dap 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/tools/ipmi-vulns.rb CHANGED
@@ -22,6 +22,8 @@ def search(hash)
22
22
  hash
23
23
  end
24
24
 
25
- while line=gets
26
- puts Oj.dump(search(Oj.load(line.strip)))
25
+ $stdin.each_line do |line|
26
+ json = Oj.load(line.unpack("C*").pack("C*").strip) rescue nil
27
+ next unless json
28
+ puts Oj.dump(search(json))
27
29
  end
@@ -3,6 +3,7 @@ require 'oj'
3
3
  require 'optparse'
4
4
 
5
5
  HELP=<<EOF
6
+
6
7
  This script is used to locate the frequency of a given key in a json document. It will
7
8
  inspect and increment the frequency count for each instance of the key found in the json
8
9
  document, then order them in descending order and output a json document with the top n
@@ -14,11 +15,22 @@ HELP=<<EOF
14
15
  unpigz -c /tmp/2014-05-05-mssql-udp-decoded.json.gz | ruby ~/src/dap/tools/json-summarize.rb --top 20 --key data.mssql.Version
15
16
  EOF
16
17
 
18
+
19
+ def stringify(o)
20
+ if o.kind_of?( ::String )
21
+ o.to_s.encode(o.encoding, "UTF-8", :invalid => :replace, :undef => :replace, :replace => '')
22
+ else
23
+ o.to_s
24
+ end
25
+ end
26
+
17
27
  def parse_command_line(args)
18
28
 
19
- options={
20
- :key => nil,
21
- :number => nil
29
+ options = {
30
+ :key => nil,
31
+ :number => 100,
32
+ :subkey => nil,
33
+ :subnumber => 100
22
34
  }
23
35
 
24
36
  OptionParser.new do | opts |
@@ -27,55 +39,88 @@ def parse_command_line(args)
27
39
 
28
40
  opts.separator 'GeoIP name options:'
29
41
 
30
- opts.on( '--key keyname', 'The name of json key to be summarized.') do | val |
42
+ opts.on( '--key keyname', 'The name of the json key to be summarized.') do | val |
31
43
  options[:key] = val
32
44
  end
45
+
46
+ opts.on( '--subkey keyname', 'The name of the json subkey to be summarized under each key') do | val |
47
+ options[:subkey] = val
48
+ end
33
49
 
34
50
  opts.on( '--top num_items', 'Return top n occurrences.') do | val |
35
51
  options[:number] = val.to_i
36
52
  end
53
+
54
+ opts.on( '--subtop num_items', 'Return top n occurrences in each subkey.') do | val |
55
+ options[:subnumber] = val.to_i
56
+ end
37
57
 
38
58
  opts.on_tail('-h', '--help', 'Show this message') do
39
- puts opts
40
- exit(0)
59
+ $stderr.puts puts opts
60
+ exit(1)
41
61
  end
42
62
  opts.parse!(args)
43
- options
44
- end
45
- options
46
- end
47
63
 
48
- # Sorts the hash in descending numerical value for the values
49
- # part of the hash, returning the sorted hash.
50
- #
51
- def order_hash(h)
52
- keys = h.keys.sort { | k1,k2 |
53
- ret = ( h[k1] <=> h[k2] ) * -1
54
- ret = k1 <=> k2 if ret == 0 && k1!=nil && k2!=nil
55
- ret
56
- }
57
- # build up return hash
58
- ret_hash = {}
59
- keys.each do | key |
60
- ret_hash[key] = h[key]
64
+ if not options[:key]
65
+ $stderr.puts opts
66
+ exit(1)
67
+ end
61
68
  end
62
69
 
63
- ret_hash
70
+ options
64
71
  end
65
72
 
66
73
 
74
+ summary = {}
75
+ opts = parse_command_line(ARGV)
76
+ key = opts[:key]
77
+ skey = opts[:subkey]
67
78
 
79
+ $stdin.each_line do |line|
80
+ json = Oj.load(line.to_s.unpack("C*").pack("C*").strip) rescue nil
81
+ next unless ( json && json[key] )
68
82
 
69
- summary={}
70
- opts = parse_command_line(ARGV)
71
- key = opts[:key]
83
+ if json[key].kind_of?(Array)
84
+ vals = json[key]
85
+ else
86
+ vals = [json[key],]
87
+ end
72
88
 
73
- while line = gets
74
- val = Oj.load(line.chomp.strip)[key]
75
- summary[val] ||= 0
76
- summary[val] += 1
89
+ vals.each do |val|
90
+ val = stringify(val)
91
+
92
+ summary[val] ||= {}
93
+ summary[val][:count] ||= 0
94
+ summary[val][:count] += 1
95
+
96
+ if skey
97
+ if json[skey].kind_of?(Array)
98
+ svals = json[skey]
99
+ else
100
+ svals = [json[skey],]
101
+ end
102
+
103
+ svals.each do |sval|
104
+ sval = stringify(sval)
105
+ summary[val][sval] ||= {}
106
+ summary[val][sval][:count] ||= 0
107
+ summary[val][sval][:count] += 1
108
+ end
109
+ end
77
110
  end
78
111
 
79
- summary = Hash[ *order_hash(summary).flatten.slice(0,2*opts[:number]) ]
80
- puts Oj.dump(summary)
112
+ end
113
+
114
+ output = {}
115
+ summary.keys.sort{|a,b| summary[b][:count] <=> summary[a][:count] }[0, opts[:number]].each do |k|
116
+ unless skey
117
+ output[k] = summary[k][:count]
118
+ else
119
+ output[k] = { "count" => summary[k][:count], skey => {} }
120
+ summary[k].keys.select{|x| x != :count}.sort{|a,b| summary[k][b][:count] <=> summary[k][a][:count] }[0, opts[:subnumber]].each do |sk|
121
+ output[k][skey][sk] = summary[k][sk][:count]
122
+ end
123
+ end
124
+ end
81
125
 
126
+ $stdout.puts Oj.dump(output)
@@ -129,11 +129,12 @@ class GeoCounter
129
129
  def count(hash)
130
130
  city = hash['ip.city'].to_s
131
131
  country_code = hash['ip.country_code'].to_s
132
+ country_name = hash['ip.country_name'].to_s
132
133
  region = hash['ip.region'].to_s
133
134
  region_name = hash['ip.region_name'].to_s
134
135
 
135
136
  @cities[[city, country_code]] += 1 unless city.empty?
136
- @countries[country_code] += 1 unless country_code.empty?
137
+ @countries[[country_code, country_name]] += 1 unless country_code.empty?
137
138
  @regions[[region, region_name]] += 1 unless region.empty?
138
139
  end
139
140
 
@@ -152,7 +153,11 @@ class GeoCounter
152
153
  def top_countries
153
154
  [].tap do |counts|
154
155
  ordered_countries.to_a.take(NUM_TOP_RECORDS).each do |values|
155
- counts << { 'country_code' => values[0], 'count' => values[1] }
156
+ counts << {
157
+ 'country_code' => values[0][0],
158
+ 'country_name' => values[0][1],
159
+ 'count' => values[1]
160
+ }
156
161
  end
157
162
  end
158
163
  end
@@ -258,14 +263,16 @@ unless options.exclude_default_counts
258
263
  counters << GeoCounter.new
259
264
  counters << SambaCounter.new
260
265
  end
266
+
261
267
  counters << HostnameContainingCounter.new(options.hostname_containing) unless options.hostname_containing.nil?
262
268
 
263
- while line=gets
264
- hash = Oj.load(line.strip)
265
- counters.each { |counter| counter.count(hash) }
269
+ $stdin.each_line do |line|
270
+ json = Oj.load(line.unpack("C*").pack("C*").strip) rescue nil
271
+ next unless json
272
+ counters.each { |counter| counter.count(json) }
266
273
  end
267
274
 
268
275
  summary = {}
269
276
  counters.each { |counter| counter.apply_to(summary) }
270
277
 
271
- puts JSON.pretty_generate(summary)
278
+ puts JSON.pretty_generate(summary)
data/tools/upnp-vulns.rb CHANGED
@@ -1,6 +1,6 @@
1
+ #!/usr/bin/env ruby
1
2
  require 'oj'
2
3
 
3
-
4
4
  # Searches contains each of the services, within each service it contains
5
5
  # a hash key that will be compared against each of the items in the
6
6
  # regex hash, and if a hit is returned the value from the regex is inserted
@@ -11,9 +11,9 @@ SEARCHES = {
11
11
  :hash_key => 'data.upnp_server',
12
12
  :output_key => 'vulnerability',
13
13
  :regex => {
14
- /MiniUPnPd\/1\.0([\.\,\-\~\s]|$)/mi => 'CVE-2013-0229',
15
- /MiniUPnPd\/1\.[0-3]([\.\,\-\~\s]|$)/mi => 'CVE-2013-0230',
16
- /Intel SDK for UPnP devices.*|Portable SDK for UPnP devices(\/?\s*$|\/1\.([0-5]\..*|8\.0.*|(6\.[0-9]|6\.1[0-7])([\.\,\-\~\s]|$)))/mi => 'CVE-2012-5958 , CVE-2012-5959'
14
+ /MiniUPnPd\/1\.0([\.\,\-\~\s]|$)/mi => ['CVE-2013-0229'],
15
+ /MiniUPnPd\/1\.[0-3]([\.\,\-\~\s]|$)/mi => ['CVE-2013-0230'],
16
+ /Intel SDK for UPnP devices.*|Portable SDK for UPnP devices(\/?\s*$|\/1\.([0-5]\..*|8\.0.*|(6\.[0-9]|6\.1[0-7])([\.\,\-\~\s]|$)))/mi => ['CVE-2012-5958', 'CVE-2012-5959']
17
17
  }
18
18
  }
19
19
  }
@@ -22,14 +22,14 @@ def search(hash, service)
22
22
  SEARCHES[service][:regex].each do | regex, value |
23
23
  if regex =~ hash[SEARCHES[service][:hash_key]].force_encoding('BINARY')
24
24
  # Handle cases that could be multiple hits, not for upnp but could be others.
25
- hash[SEARCHES[service][:output_key]] = ( hash[SEARCHES[service][:output_key]] ? hash[SEARCHES[service][:output_key]] + ',' + value : value )
25
+ hash[SEARCHES[service][:output_key]] = ( hash[SEARCHES[service][:output_key]] ? hash[SEARCHES[service][:output_key]] + value : value )
26
26
  end
27
27
  end if hash[SEARCHES[service][:hash_key]]
28
28
  hash
29
29
  end
30
30
 
31
- while line=gets
32
- #line.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '')
33
- #line.force_encoding('BINARY')
34
- puts Oj.dump( search( Oj.load(line.strip), :upnp ))
31
+ $stdin.each_line do |line|
32
+ json = Oj.load(line.unpack("C*").pack("C*").strip) rescue nil
33
+ next unless json
34
+ puts Oj.dump(search(json, :upnp))
35
35
  end
metadata CHANGED
@@ -1,177 +1,156 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dap
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
5
- prerelease:
4
+ version: 0.0.2
6
5
  platform: ruby
7
6
  authors:
8
7
  - Rapid7 Research
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2014-06-17 00:00:00.000000000 Z
11
+ date: 2014-10-29 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: rspec
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - ">="
20
18
  - !ruby/object:Gem::Version
21
19
  version: '0'
22
20
  type: :development
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - ">="
28
25
  - !ruby/object:Gem::Version
29
26
  version: '0'
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: cucumber
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - ">="
36
32
  - !ruby/object:Gem::Version
37
33
  version: '0'
38
34
  type: :development
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - ">="
44
39
  - !ruby/object:Gem::Version
45
40
  version: '0'
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: aruba
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - ">="
52
46
  - !ruby/object:Gem::Version
53
47
  version: '0'
54
48
  type: :development
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - ">="
60
53
  - !ruby/object:Gem::Version
61
54
  version: '0'
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: nokogiri
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
- - - ! '>='
59
+ - - ">="
68
60
  - !ruby/object:Gem::Version
69
61
  version: '0'
70
62
  type: :runtime
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
- - - ! '>='
66
+ - - ">="
76
67
  - !ruby/object:Gem::Version
77
68
  version: '0'
78
69
  - !ruby/object:Gem::Dependency
79
70
  name: oj
80
71
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
72
  requirements:
83
- - - ! '>='
73
+ - - ">="
84
74
  - !ruby/object:Gem::Version
85
75
  version: '0'
86
76
  type: :runtime
87
77
  prerelease: false
88
78
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
79
  requirements:
91
- - - ! '>='
80
+ - - ">="
92
81
  - !ruby/object:Gem::Version
93
82
  version: '0'
94
83
  - !ruby/object:Gem::Dependency
95
84
  name: htmlentities
96
85
  requirement: !ruby/object:Gem::Requirement
97
- none: false
98
86
  requirements:
99
- - - ! '>='
87
+ - - ">="
100
88
  - !ruby/object:Gem::Version
101
89
  version: '0'
102
90
  type: :runtime
103
91
  prerelease: false
104
92
  version_requirements: !ruby/object:Gem::Requirement
105
- none: false
106
93
  requirements:
107
- - - ! '>='
94
+ - - ">="
108
95
  - !ruby/object:Gem::Version
109
96
  version: '0'
110
97
  - !ruby/object:Gem::Dependency
111
98
  name: net-dns
112
99
  requirement: !ruby/object:Gem::Requirement
113
- none: false
114
100
  requirements:
115
- - - ! '>='
101
+ - - ">="
116
102
  - !ruby/object:Gem::Version
117
103
  version: '0'
118
104
  type: :runtime
119
105
  prerelease: false
120
106
  version_requirements: !ruby/object:Gem::Requirement
121
- none: false
122
107
  requirements:
123
- - - ! '>='
108
+ - - ">="
124
109
  - !ruby/object:Gem::Version
125
110
  version: '0'
126
111
  - !ruby/object:Gem::Dependency
127
112
  name: bit-struct
128
113
  requirement: !ruby/object:Gem::Requirement
129
- none: false
130
114
  requirements:
131
- - - ! '>='
115
+ - - ">="
132
116
  - !ruby/object:Gem::Version
133
117
  version: '0'
134
118
  type: :runtime
135
119
  prerelease: false
136
120
  version_requirements: !ruby/object:Gem::Requirement
137
- none: false
138
121
  requirements:
139
- - - ! '>='
122
+ - - ">="
140
123
  - !ruby/object:Gem::Version
141
124
  version: '0'
142
125
  - !ruby/object:Gem::Dependency
143
126
  name: geoip-c
144
127
  requirement: !ruby/object:Gem::Requirement
145
- none: false
146
128
  requirements:
147
- - - ! '>='
129
+ - - ">="
148
130
  - !ruby/object:Gem::Version
149
131
  version: '0'
150
132
  type: :runtime
151
133
  prerelease: false
152
134
  version_requirements: !ruby/object:Gem::Requirement
153
- none: false
154
135
  requirements:
155
- - - ! '>='
136
+ - - ">="
156
137
  - !ruby/object:Gem::Version
157
138
  version: '0'
158
139
  - !ruby/object:Gem::Dependency
159
140
  name: recog
160
141
  requirement: !ruby/object:Gem::Requirement
161
- none: false
162
142
  requirements:
163
- - - ! '>='
143
+ - - ">="
164
144
  - !ruby/object:Gem::Version
165
145
  version: '0'
166
146
  type: :runtime
167
147
  prerelease: false
168
148
  version_requirements: !ruby/object:Gem::Requirement
169
- none: false
170
149
  requirements:
171
- - - ! '>='
150
+ - - ">="
172
151
  - !ruby/object:Gem::Version
173
152
  version: '0'
174
- description: ! 'DAP reads data using an input plugin, transforms it through a series
153
+ description: 'DAP reads data using an input plugin, transforms it through a series
175
154
  of filters, and prints it out again using an output plugin. Every record is treated
176
155
  as a document (aka: hash/dict) and filters are used to reduce, expand, and transform
177
156
  these documents as they pass through. Think of DAP as a mashup between sed, awk,
@@ -183,15 +162,18 @@ executables:
183
162
  extensions: []
184
163
  extra_rdoc_files: []
185
164
  files:
186
- - .gitignore
187
- - .rspec
165
+ - ".gitignore"
166
+ - ".rspec"
167
+ - ".travis.yml"
188
168
  - Gemfile
189
169
  - Gemfile.lock
190
170
  - LICENSE
191
171
  - README.md
172
+ - Rakefile
192
173
  - bin/dap
193
174
  - dap.gemspec
194
175
  - data/.gitkeep
176
+ - data/vulndb.rb
195
177
  - lib/dap.rb
196
178
  - lib/dap/filter.rb
197
179
  - lib/dap/filter/base.rb
@@ -202,6 +184,7 @@ files:
202
184
  - lib/dap/filter/recog.rb
203
185
  - lib/dap/filter/simple.rb
204
186
  - lib/dap/filter/udp.rb
187
+ - lib/dap/filter/vulnmatch.rb
205
188
  - lib/dap/input.rb
206
189
  - lib/dap/input/csv.rb
207
190
  - lib/dap/input/warc.rb
@@ -209,6 +192,7 @@ files:
209
192
  - lib/dap/proto/addp.rb
210
193
  - lib/dap/proto/dtls.rb
211
194
  - lib/dap/proto/ipmi.rb
195
+ - lib/dap/proto/mssql.rb
212
196
  - lib/dap/proto/natpmp.rb
213
197
  - lib/dap/proto/wdbrpc.rb
214
198
  - lib/dap/utils/oui.rb
@@ -238,27 +222,26 @@ files:
238
222
  - tools/value-counts-to-md-table.rb
239
223
  homepage: https://www.github.com/rapid7/dap
240
224
  licenses: []
225
+ metadata: {}
241
226
  post_install_message:
242
227
  rdoc_options: []
243
228
  require_paths:
244
229
  - lib
245
230
  required_ruby_version: !ruby/object:Gem::Requirement
246
- none: false
247
231
  requirements:
248
- - - ! '>='
232
+ - - ">="
249
233
  - !ruby/object:Gem::Version
250
234
  version: '0'
251
235
  required_rubygems_version: !ruby/object:Gem::Requirement
252
- none: false
253
236
  requirements:
254
- - - ! '>='
237
+ - - ">="
255
238
  - !ruby/object:Gem::Version
256
239
  version: '0'
257
240
  requirements: []
258
241
  rubyforge_project:
259
- rubygems_version: 1.8.23
242
+ rubygems_version: 2.2.2
260
243
  signing_key:
261
- specification_version: 3
262
- summary: ! 'DAP: The Data Analysis Pipeline'
244
+ specification_version: 4
245
+ summary: 'DAP: The Data Analysis Pipeline'
263
246
  test_files: []
264
247
  has_rdoc: