dap 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.travis.yml +8 -0
- data/Gemfile +3 -3
- data/Gemfile.lock +22 -18
- data/README.md +24 -1
- data/Rakefile +22 -0
- data/data/vulndb.rb +83 -0
- data/lib/dap/filter.rb +2 -1
- data/lib/dap/filter/http.rb +4 -0
- data/lib/dap/filter/simple.rb +25 -1
- data/lib/dap/filter/udp.rb +255 -48
- data/lib/dap/filter/vulnmatch.rb +68 -0
- data/lib/dap/output.rb +111 -11
- data/lib/dap/proto/mssql.rb +114 -0
- data/lib/dap/version.rb +1 -1
- data/spec/dap/proto/ipmi_spec.rb +4 -4
- data/tools/geo-ip-summary.rb +26 -12
- data/tools/ipmi-vulns.rb +4 -2
- data/tools/json-summarize.rb +78 -33
- data/tools/netbios-counts.rb +13 -6
- data/tools/upnp-vulns.rb +9 -9
- metadata +36 -53
data/tools/ipmi-vulns.rb
CHANGED
data/tools/json-summarize.rb
CHANGED
@@ -3,6 +3,7 @@ require 'oj'
|
|
3
3
|
require 'optparse'
|
4
4
|
|
5
5
|
HELP=<<EOF
|
6
|
+
|
6
7
|
This script is used to locate the frequency of a given key in a json document. It will
|
7
8
|
inspect and increment the frequency count for each instance of the key found in the json
|
8
9
|
document, then order them in descending order and output a json document with the top n
|
@@ -14,11 +15,22 @@ HELP=<<EOF
|
|
14
15
|
unpigz -c /tmp/2014-05-05-mssql-udp-decoded.json.gz | ruby ~/src/dap/tools/json-summarize.rb --top 20 --key data.mssql.Version
|
15
16
|
EOF
|
16
17
|
|
18
|
+
|
19
|
+
def stringify(o)
|
20
|
+
if o.kind_of?( ::String )
|
21
|
+
o.to_s.encode(o.encoding, "UTF-8", :invalid => :replace, :undef => :replace, :replace => '')
|
22
|
+
else
|
23
|
+
o.to_s
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
17
27
|
def parse_command_line(args)
|
18
28
|
|
19
|
-
options={
|
20
|
-
:key
|
21
|
-
:number =>
|
29
|
+
options = {
|
30
|
+
:key => nil,
|
31
|
+
:number => 100,
|
32
|
+
:subkey => nil,
|
33
|
+
:subnumber => 100
|
22
34
|
}
|
23
35
|
|
24
36
|
OptionParser.new do | opts |
|
@@ -27,55 +39,88 @@ def parse_command_line(args)
|
|
27
39
|
|
28
40
|
opts.separator 'GeoIP name options:'
|
29
41
|
|
30
|
-
opts.on( '--key keyname', 'The name of json key to be summarized.') do | val |
|
42
|
+
opts.on( '--key keyname', 'The name of the json key to be summarized.') do | val |
|
31
43
|
options[:key] = val
|
32
44
|
end
|
45
|
+
|
46
|
+
opts.on( '--subkey keyname', 'The name of the json subkey to be summarized under each key') do | val |
|
47
|
+
options[:subkey] = val
|
48
|
+
end
|
33
49
|
|
34
50
|
opts.on( '--top num_items', 'Return top n occurrences.') do | val |
|
35
51
|
options[:number] = val.to_i
|
36
52
|
end
|
53
|
+
|
54
|
+
opts.on( '--subtop num_items', 'Return top n occurrences in each subkey.') do | val |
|
55
|
+
options[:subnumber] = val.to_i
|
56
|
+
end
|
37
57
|
|
38
58
|
opts.on_tail('-h', '--help', 'Show this message') do
|
39
|
-
puts opts
|
40
|
-
exit(
|
59
|
+
$stderr.puts puts opts
|
60
|
+
exit(1)
|
41
61
|
end
|
42
62
|
opts.parse!(args)
|
43
|
-
options
|
44
|
-
end
|
45
|
-
options
|
46
|
-
end
|
47
63
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
keys = h.keys.sort { | k1,k2 |
|
53
|
-
ret = ( h[k1] <=> h[k2] ) * -1
|
54
|
-
ret = k1 <=> k2 if ret == 0 && k1!=nil && k2!=nil
|
55
|
-
ret
|
56
|
-
}
|
57
|
-
# build up return hash
|
58
|
-
ret_hash = {}
|
59
|
-
keys.each do | key |
|
60
|
-
ret_hash[key] = h[key]
|
64
|
+
if not options[:key]
|
65
|
+
$stderr.puts opts
|
66
|
+
exit(1)
|
67
|
+
end
|
61
68
|
end
|
62
69
|
|
63
|
-
|
70
|
+
options
|
64
71
|
end
|
65
72
|
|
66
73
|
|
74
|
+
summary = {}
|
75
|
+
opts = parse_command_line(ARGV)
|
76
|
+
key = opts[:key]
|
77
|
+
skey = opts[:subkey]
|
67
78
|
|
79
|
+
$stdin.each_line do |line|
|
80
|
+
json = Oj.load(line.to_s.unpack("C*").pack("C*").strip) rescue nil
|
81
|
+
next unless ( json && json[key] )
|
68
82
|
|
69
|
-
|
70
|
-
|
71
|
-
|
83
|
+
if json[key].kind_of?(Array)
|
84
|
+
vals = json[key]
|
85
|
+
else
|
86
|
+
vals = [json[key],]
|
87
|
+
end
|
72
88
|
|
73
|
-
|
74
|
-
val =
|
75
|
-
|
76
|
-
summary[val]
|
89
|
+
vals.each do |val|
|
90
|
+
val = stringify(val)
|
91
|
+
|
92
|
+
summary[val] ||= {}
|
93
|
+
summary[val][:count] ||= 0
|
94
|
+
summary[val][:count] += 1
|
95
|
+
|
96
|
+
if skey
|
97
|
+
if json[skey].kind_of?(Array)
|
98
|
+
svals = json[skey]
|
99
|
+
else
|
100
|
+
svals = [json[skey],]
|
101
|
+
end
|
102
|
+
|
103
|
+
svals.each do |sval|
|
104
|
+
sval = stringify(sval)
|
105
|
+
summary[val][sval] ||= {}
|
106
|
+
summary[val][sval][:count] ||= 0
|
107
|
+
summary[val][sval][:count] += 1
|
108
|
+
end
|
109
|
+
end
|
77
110
|
end
|
78
111
|
|
79
|
-
|
80
|
-
|
112
|
+
end
|
113
|
+
|
114
|
+
output = {}
|
115
|
+
summary.keys.sort{|a,b| summary[b][:count] <=> summary[a][:count] }[0, opts[:number]].each do |k|
|
116
|
+
unless skey
|
117
|
+
output[k] = summary[k][:count]
|
118
|
+
else
|
119
|
+
output[k] = { "count" => summary[k][:count], skey => {} }
|
120
|
+
summary[k].keys.select{|x| x != :count}.sort{|a,b| summary[k][b][:count] <=> summary[k][a][:count] }[0, opts[:subnumber]].each do |sk|
|
121
|
+
output[k][skey][sk] = summary[k][sk][:count]
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
81
125
|
|
126
|
+
$stdout.puts Oj.dump(output)
|
data/tools/netbios-counts.rb
CHANGED
@@ -129,11 +129,12 @@ class GeoCounter
|
|
129
129
|
def count(hash)
|
130
130
|
city = hash['ip.city'].to_s
|
131
131
|
country_code = hash['ip.country_code'].to_s
|
132
|
+
country_name = hash['ip.country_name'].to_s
|
132
133
|
region = hash['ip.region'].to_s
|
133
134
|
region_name = hash['ip.region_name'].to_s
|
134
135
|
|
135
136
|
@cities[[city, country_code]] += 1 unless city.empty?
|
136
|
-
@countries[country_code] += 1 unless country_code.empty?
|
137
|
+
@countries[[country_code, country_name]] += 1 unless country_code.empty?
|
137
138
|
@regions[[region, region_name]] += 1 unless region.empty?
|
138
139
|
end
|
139
140
|
|
@@ -152,7 +153,11 @@ class GeoCounter
|
|
152
153
|
def top_countries
|
153
154
|
[].tap do |counts|
|
154
155
|
ordered_countries.to_a.take(NUM_TOP_RECORDS).each do |values|
|
155
|
-
counts << {
|
156
|
+
counts << {
|
157
|
+
'country_code' => values[0][0],
|
158
|
+
'country_name' => values[0][1],
|
159
|
+
'count' => values[1]
|
160
|
+
}
|
156
161
|
end
|
157
162
|
end
|
158
163
|
end
|
@@ -258,14 +263,16 @@ unless options.exclude_default_counts
|
|
258
263
|
counters << GeoCounter.new
|
259
264
|
counters << SambaCounter.new
|
260
265
|
end
|
266
|
+
|
261
267
|
counters << HostnameContainingCounter.new(options.hostname_containing) unless options.hostname_containing.nil?
|
262
268
|
|
263
|
-
|
264
|
-
|
265
|
-
|
269
|
+
$stdin.each_line do |line|
|
270
|
+
json = Oj.load(line.unpack("C*").pack("C*").strip) rescue nil
|
271
|
+
next unless json
|
272
|
+
counters.each { |counter| counter.count(json) }
|
266
273
|
end
|
267
274
|
|
268
275
|
summary = {}
|
269
276
|
counters.each { |counter| counter.apply_to(summary) }
|
270
277
|
|
271
|
-
puts JSON.pretty_generate(summary)
|
278
|
+
puts JSON.pretty_generate(summary)
|
data/tools/upnp-vulns.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
1
2
|
require 'oj'
|
2
3
|
|
3
|
-
|
4
4
|
# Searches contains each of the services, within each service it contains
|
5
5
|
# a hash key that will be compared against each of the items in the
|
6
6
|
# regex hash, and if a hit is returned the value from the regex is inserted
|
@@ -11,9 +11,9 @@ SEARCHES = {
|
|
11
11
|
:hash_key => 'data.upnp_server',
|
12
12
|
:output_key => 'vulnerability',
|
13
13
|
:regex => {
|
14
|
-
/MiniUPnPd\/1\.0([\.\,\-\~\s]|$)/mi => 'CVE-2013-0229',
|
15
|
-
/MiniUPnPd\/1\.[0-3]([\.\,\-\~\s]|$)/mi => 'CVE-2013-0230',
|
16
|
-
/Intel SDK for UPnP devices.*|Portable SDK for UPnP devices(\/?\s*$|\/1\.([0-5]\..*|8\.0.*|(6\.[0-9]|6\.1[0-7])([\.\,\-\~\s]|$)))/mi =>
|
14
|
+
/MiniUPnPd\/1\.0([\.\,\-\~\s]|$)/mi => ['CVE-2013-0229'],
|
15
|
+
/MiniUPnPd\/1\.[0-3]([\.\,\-\~\s]|$)/mi => ['CVE-2013-0230'],
|
16
|
+
/Intel SDK for UPnP devices.*|Portable SDK for UPnP devices(\/?\s*$|\/1\.([0-5]\..*|8\.0.*|(6\.[0-9]|6\.1[0-7])([\.\,\-\~\s]|$)))/mi => ['CVE-2012-5958', 'CVE-2012-5959']
|
17
17
|
}
|
18
18
|
}
|
19
19
|
}
|
@@ -22,14 +22,14 @@ def search(hash, service)
|
|
22
22
|
SEARCHES[service][:regex].each do | regex, value |
|
23
23
|
if regex =~ hash[SEARCHES[service][:hash_key]].force_encoding('BINARY')
|
24
24
|
# Handle cases that could be multiple hits, not for upnp but could be others.
|
25
|
-
hash[SEARCHES[service][:output_key]] = ( hash[SEARCHES[service][:output_key]] ? hash[SEARCHES[service][:output_key]] +
|
25
|
+
hash[SEARCHES[service][:output_key]] = ( hash[SEARCHES[service][:output_key]] ? hash[SEARCHES[service][:output_key]] + value : value )
|
26
26
|
end
|
27
27
|
end if hash[SEARCHES[service][:hash_key]]
|
28
28
|
hash
|
29
29
|
end
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
puts Oj.dump(
|
31
|
+
$stdin.each_line do |line|
|
32
|
+
json = Oj.load(line.unpack("C*").pack("C*").strip) rescue nil
|
33
|
+
next unless json
|
34
|
+
puts Oj.dump(search(json, :upnp))
|
35
35
|
end
|
metadata
CHANGED
@@ -1,177 +1,156 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dap
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.2
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Rapid7 Research
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2014-
|
11
|
+
date: 2014-10-29 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: rspec
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - ">="
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :development
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: cucumber
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - ">="
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :development
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - ">="
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: aruba
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - ">="
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: '0'
|
54
48
|
type: :development
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - ">="
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: '0'
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: nokogiri
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
|
-
- -
|
59
|
+
- - ">="
|
68
60
|
- !ruby/object:Gem::Version
|
69
61
|
version: '0'
|
70
62
|
type: :runtime
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
|
-
- -
|
66
|
+
- - ">="
|
76
67
|
- !ruby/object:Gem::Version
|
77
68
|
version: '0'
|
78
69
|
- !ruby/object:Gem::Dependency
|
79
70
|
name: oj
|
80
71
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
72
|
requirements:
|
83
|
-
- -
|
73
|
+
- - ">="
|
84
74
|
- !ruby/object:Gem::Version
|
85
75
|
version: '0'
|
86
76
|
type: :runtime
|
87
77
|
prerelease: false
|
88
78
|
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
79
|
requirements:
|
91
|
-
- -
|
80
|
+
- - ">="
|
92
81
|
- !ruby/object:Gem::Version
|
93
82
|
version: '0'
|
94
83
|
- !ruby/object:Gem::Dependency
|
95
84
|
name: htmlentities
|
96
85
|
requirement: !ruby/object:Gem::Requirement
|
97
|
-
none: false
|
98
86
|
requirements:
|
99
|
-
- -
|
87
|
+
- - ">="
|
100
88
|
- !ruby/object:Gem::Version
|
101
89
|
version: '0'
|
102
90
|
type: :runtime
|
103
91
|
prerelease: false
|
104
92
|
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
none: false
|
106
93
|
requirements:
|
107
|
-
- -
|
94
|
+
- - ">="
|
108
95
|
- !ruby/object:Gem::Version
|
109
96
|
version: '0'
|
110
97
|
- !ruby/object:Gem::Dependency
|
111
98
|
name: net-dns
|
112
99
|
requirement: !ruby/object:Gem::Requirement
|
113
|
-
none: false
|
114
100
|
requirements:
|
115
|
-
- -
|
101
|
+
- - ">="
|
116
102
|
- !ruby/object:Gem::Version
|
117
103
|
version: '0'
|
118
104
|
type: :runtime
|
119
105
|
prerelease: false
|
120
106
|
version_requirements: !ruby/object:Gem::Requirement
|
121
|
-
none: false
|
122
107
|
requirements:
|
123
|
-
- -
|
108
|
+
- - ">="
|
124
109
|
- !ruby/object:Gem::Version
|
125
110
|
version: '0'
|
126
111
|
- !ruby/object:Gem::Dependency
|
127
112
|
name: bit-struct
|
128
113
|
requirement: !ruby/object:Gem::Requirement
|
129
|
-
none: false
|
130
114
|
requirements:
|
131
|
-
- -
|
115
|
+
- - ">="
|
132
116
|
- !ruby/object:Gem::Version
|
133
117
|
version: '0'
|
134
118
|
type: :runtime
|
135
119
|
prerelease: false
|
136
120
|
version_requirements: !ruby/object:Gem::Requirement
|
137
|
-
none: false
|
138
121
|
requirements:
|
139
|
-
- -
|
122
|
+
- - ">="
|
140
123
|
- !ruby/object:Gem::Version
|
141
124
|
version: '0'
|
142
125
|
- !ruby/object:Gem::Dependency
|
143
126
|
name: geoip-c
|
144
127
|
requirement: !ruby/object:Gem::Requirement
|
145
|
-
none: false
|
146
128
|
requirements:
|
147
|
-
- -
|
129
|
+
- - ">="
|
148
130
|
- !ruby/object:Gem::Version
|
149
131
|
version: '0'
|
150
132
|
type: :runtime
|
151
133
|
prerelease: false
|
152
134
|
version_requirements: !ruby/object:Gem::Requirement
|
153
|
-
none: false
|
154
135
|
requirements:
|
155
|
-
- -
|
136
|
+
- - ">="
|
156
137
|
- !ruby/object:Gem::Version
|
157
138
|
version: '0'
|
158
139
|
- !ruby/object:Gem::Dependency
|
159
140
|
name: recog
|
160
141
|
requirement: !ruby/object:Gem::Requirement
|
161
|
-
none: false
|
162
142
|
requirements:
|
163
|
-
- -
|
143
|
+
- - ">="
|
164
144
|
- !ruby/object:Gem::Version
|
165
145
|
version: '0'
|
166
146
|
type: :runtime
|
167
147
|
prerelease: false
|
168
148
|
version_requirements: !ruby/object:Gem::Requirement
|
169
|
-
none: false
|
170
149
|
requirements:
|
171
|
-
- -
|
150
|
+
- - ">="
|
172
151
|
- !ruby/object:Gem::Version
|
173
152
|
version: '0'
|
174
|
-
description:
|
153
|
+
description: 'DAP reads data using an input plugin, transforms it through a series
|
175
154
|
of filters, and prints it out again using an output plugin. Every record is treated
|
176
155
|
as a document (aka: hash/dict) and filters are used to reduce, expand, and transform
|
177
156
|
these documents as they pass through. Think of DAP as a mashup between sed, awk,
|
@@ -183,15 +162,18 @@ executables:
|
|
183
162
|
extensions: []
|
184
163
|
extra_rdoc_files: []
|
185
164
|
files:
|
186
|
-
- .gitignore
|
187
|
-
- .rspec
|
165
|
+
- ".gitignore"
|
166
|
+
- ".rspec"
|
167
|
+
- ".travis.yml"
|
188
168
|
- Gemfile
|
189
169
|
- Gemfile.lock
|
190
170
|
- LICENSE
|
191
171
|
- README.md
|
172
|
+
- Rakefile
|
192
173
|
- bin/dap
|
193
174
|
- dap.gemspec
|
194
175
|
- data/.gitkeep
|
176
|
+
- data/vulndb.rb
|
195
177
|
- lib/dap.rb
|
196
178
|
- lib/dap/filter.rb
|
197
179
|
- lib/dap/filter/base.rb
|
@@ -202,6 +184,7 @@ files:
|
|
202
184
|
- lib/dap/filter/recog.rb
|
203
185
|
- lib/dap/filter/simple.rb
|
204
186
|
- lib/dap/filter/udp.rb
|
187
|
+
- lib/dap/filter/vulnmatch.rb
|
205
188
|
- lib/dap/input.rb
|
206
189
|
- lib/dap/input/csv.rb
|
207
190
|
- lib/dap/input/warc.rb
|
@@ -209,6 +192,7 @@ files:
|
|
209
192
|
- lib/dap/proto/addp.rb
|
210
193
|
- lib/dap/proto/dtls.rb
|
211
194
|
- lib/dap/proto/ipmi.rb
|
195
|
+
- lib/dap/proto/mssql.rb
|
212
196
|
- lib/dap/proto/natpmp.rb
|
213
197
|
- lib/dap/proto/wdbrpc.rb
|
214
198
|
- lib/dap/utils/oui.rb
|
@@ -238,27 +222,26 @@ files:
|
|
238
222
|
- tools/value-counts-to-md-table.rb
|
239
223
|
homepage: https://www.github.com/rapid7/dap
|
240
224
|
licenses: []
|
225
|
+
metadata: {}
|
241
226
|
post_install_message:
|
242
227
|
rdoc_options: []
|
243
228
|
require_paths:
|
244
229
|
- lib
|
245
230
|
required_ruby_version: !ruby/object:Gem::Requirement
|
246
|
-
none: false
|
247
231
|
requirements:
|
248
|
-
- -
|
232
|
+
- - ">="
|
249
233
|
- !ruby/object:Gem::Version
|
250
234
|
version: '0'
|
251
235
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
252
|
-
none: false
|
253
236
|
requirements:
|
254
|
-
- -
|
237
|
+
- - ">="
|
255
238
|
- !ruby/object:Gem::Version
|
256
239
|
version: '0'
|
257
240
|
requirements: []
|
258
241
|
rubyforge_project:
|
259
|
-
rubygems_version:
|
242
|
+
rubygems_version: 2.2.2
|
260
243
|
signing_key:
|
261
|
-
specification_version:
|
262
|
-
summary:
|
244
|
+
specification_version: 4
|
245
|
+
summary: 'DAP: The Data Analysis Pipeline'
|
263
246
|
test_files: []
|
264
247
|
has_rdoc:
|