dap 0.0.1 → 0.0.2

data/tools/ipmi-vulns.rb

@@ -22,6 +22,8 @@ def search(hash)
   hash
 end
 
-while line=gets
-  puts Oj.dump(search(Oj.load(line.strip)))
+$stdin.each_line do |line|
+  json = Oj.load(line.unpack("C*").pack("C*").strip) rescue nil
+  next unless json
+  puts Oj.dump(search(json))
 end

data/tools/json-summarize.rb

@@ -3,6 +3,7 @@ require 'oj'
 require 'optparse'
 
 HELP=<<EOF
+
  This script is used to locate the frequency of a given key in a json document. It will
  inspect and increment the frequency count for each instance of the key found in the json
  document, then order them in descending order and output a json document with the top n
@@ -14,11 +15,22 @@ HELP=<<EOF
     unpigz -c /tmp/2014-05-05-mssql-udp-decoded.json.gz | ruby ~/src/dap/tools/json-summarize.rb --top 20 --key data.mssql.Version
 EOF
 
+
+def stringify(o)
+  if o.kind_of?( ::String ) 
+    o.to_s.encode(o.encoding, "UTF-8", :invalid => :replace, :undef => :replace, :replace => '')
+  else
+    o.to_s
+  end
+end
+
 def parse_command_line(args)
 
-  options={
-      :key => nil,
-      :number => nil
+  options = {
+      :key    => nil,
+      :number => 100,
+      :subkey => nil,
+      :subnumber => 100
   }
 
   OptionParser.new do | opts |
@@ -27,55 +39,88 @@ def parse_command_line(args)
 
     opts.separator 'GeoIP name options:'
 
-    opts.on( '--key keyname', 'The name of json key to be summarized.') do | val |
+    opts.on( '--key keyname', 'The name of the json key to be summarized.') do | val |
       options[:key] = val
     end
+    
+    opts.on( '--subkey keyname', 'The name of the json subkey to be summarized under each key') do | val |
+      options[:subkey] = val
+    end
 
     opts.on( '--top num_items', 'Return top n occurrences.') do | val |
       options[:number] = val.to_i
     end
+    
+    opts.on( '--subtop num_items', 'Return top n occurrences in each subkey.') do | val |
+      options[:subnumber] = val.to_i
+    end
 
     opts.on_tail('-h', '--help', 'Show this message') do
-      puts opts
-      exit(0)
+      $stderr.puts puts opts
+      exit(1)
     end
     opts.parse!(args)
-    options
-  end
-  options
-end
 
-# Sorts the hash in descending numerical value for the values
-# part of the hash, returning the sorted hash.
-#
-def order_hash(h)
-  keys = h.keys.sort { | k1,k2 |
-    ret = ( h[k1] <=> h[k2] ) * -1
-    ret = k1 <=> k2 if ret == 0 && k1!=nil && k2!=nil
-    ret
-  }
-  # build up return hash
-  ret_hash = {}
-  keys.each do | key |
-    ret_hash[key] = h[key]
+    if not options[:key]
+      $stderr.puts opts
+      exit(1)
+    end
   end
 
-  ret_hash
+  options
 end
 
 
+summary = {}
+opts = parse_command_line(ARGV)
+key  = opts[:key]
+skey = opts[:subkey]
 
+$stdin.each_line do |line|
+  json = Oj.load(line.to_s.unpack("C*").pack("C*").strip) rescue nil
+  next unless ( json && json[key] )
 
-  summary={}
-  opts = parse_command_line(ARGV)
-  key = opts[:key]
+  if json[key].kind_of?(Array)
+    vals = json[key]
+  else
+    vals = [json[key],]
+  end
 
-  while line = gets
-    val = Oj.load(line.chomp.strip)[key]
-    summary[val] ||= 0
-    summary[val] += 1
+  vals.each do |val|
+    val = stringify(val)
+
+    summary[val] ||= {}
+    summary[val][:count] ||= 0
+    summary[val][:count]  += 1
+
+    if skey
+      if json[skey].kind_of?(Array)
+        svals = json[skey]
+      else
+        svals = [json[skey],]
+      end
+
+      svals.each do |sval|
+        sval = stringify(sval)
+        summary[val][sval] ||= {}
+        summary[val][sval][:count] ||= 0
+        summary[val][sval][:count]  += 1
+      end
+    end
   end
 
-  summary = Hash[ *order_hash(summary).flatten.slice(0,2*opts[:number]) ]
-  puts Oj.dump(summary)
+end
+
+output = {}
+summary.keys.sort{|a,b| summary[b][:count] <=> summary[a][:count] }[0, opts[:number]].each do |k|
+  unless skey
+    output[k] = summary[k][:count]
+  else
+    output[k] = { "count" => summary[k][:count], skey => {} }
+    summary[k].keys.select{|x| x != :count}.sort{|a,b| summary[k][b][:count] <=> summary[k][a][:count] }[0, opts[:subnumber]].each do |sk|
+      output[k][skey][sk] = summary[k][sk][:count]
+    end
+  end
+end
 
+$stdout.puts Oj.dump(output)

data/tools/netbios-counts.rb

@@ -129,11 +129,12 @@ class GeoCounter
   def count(hash)
     city         = hash['ip.city'].to_s
     country_code = hash['ip.country_code'].to_s
+    country_name = hash['ip.country_name'].to_s
     region       = hash['ip.region'].to_s
     region_name  = hash['ip.region_name'].to_s
 
     @cities[[city, country_code]] += 1 unless city.empty?
-    @countries[country_code] += 1 unless country_code.empty?
+    @countries[[country_code, country_name]] += 1 unless country_code.empty?
     @regions[[region, region_name]] += 1 unless region.empty?
   end
   
@@ -152,7 +153,11 @@ class GeoCounter
   def top_countries
     [].tap do |counts|
       ordered_countries.to_a.take(NUM_TOP_RECORDS).each do |values|
-        counts << { 'country_code' => values[0], 'count' => values[1] }
+        counts << { 
+          'country_code' => values[0][0],
+          'country_name' => values[0][1], 
+          'count' => values[1] 
+        }
       end
     end
   end
@@ -258,14 +263,16 @@ unless options.exclude_default_counts
   counters << GeoCounter.new
   counters << SambaCounter.new
 end
+
 counters << HostnameContainingCounter.new(options.hostname_containing) unless options.hostname_containing.nil?
 
-while line=gets
-  hash = Oj.load(line.strip)
-  counters.each { |counter| counter.count(hash) }
+$stdin.each_line do |line|
+  json = Oj.load(line.unpack("C*").pack("C*").strip) rescue nil
+  next unless json
+  counters.each { |counter| counter.count(json) }
 end
 
 summary = {}
 counters.each { |counter| counter.apply_to(summary) }
 
-puts JSON.pretty_generate(summary)
+puts JSON.pretty_generate(summary)

data/tools/upnp-vulns.rb

@@ -1,6 +1,6 @@
+#!/usr/bin/env ruby
 require 'oj'
 
-
 # Searches contains each of the services, within each service it contains
 # a hash key that will be compared against each of the items in the
 # regex hash, and if a hit is returned the value from the regex is inserted
@@ -11,9 +11,9 @@ SEARCHES = {
       :hash_key   => 'data.upnp_server',
       :output_key => 'vulnerability',
       :regex      => {
-        /MiniUPnPd\/1\.0([\.\,\-\~\s]|$)/mi     => 'CVE-2013-0229',
-        /MiniUPnPd\/1\.[0-3]([\.\,\-\~\s]|$)/mi => 'CVE-2013-0230',
-        /Intel SDK for UPnP devices.*|Portable SDK for UPnP devices(\/?\s*$|\/1\.([0-5]\..*|8\.0.*|(6\.[0-9]|6\.1[0-7])([\.\,\-\~\s]|$)))/mi =>  'CVE-2012-5958 , CVE-2012-5959'
+        /MiniUPnPd\/1\.0([\.\,\-\~\s]|$)/mi     => ['CVE-2013-0229'],
+        /MiniUPnPd\/1\.[0-3]([\.\,\-\~\s]|$)/mi => ['CVE-2013-0230'],
+        /Intel SDK for UPnP devices.*|Portable SDK for UPnP devices(\/?\s*$|\/1\.([0-5]\..*|8\.0.*|(6\.[0-9]|6\.1[0-7])([\.\,\-\~\s]|$)))/mi => ['CVE-2012-5958', 'CVE-2012-5959']
       }
   }
 }
@@ -22,14 +22,14 @@ def search(hash, service)
   SEARCHES[service][:regex].each do | regex, value |
     if regex =~ hash[SEARCHES[service][:hash_key]].force_encoding('BINARY')
       # Handle cases that could be multiple hits, not for upnp but could be others.
-      hash[SEARCHES[service][:output_key]] = ( hash[SEARCHES[service][:output_key]] ? hash[SEARCHES[service][:output_key]] + ',' + value : value )
+      hash[SEARCHES[service][:output_key]] = ( hash[SEARCHES[service][:output_key]] ? hash[SEARCHES[service][:output_key]] + value : value )
     end
   end if hash[SEARCHES[service][:hash_key]]
   hash
 end
 
-while line=gets
-  #line.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '')
-  #line.force_encoding('BINARY')
-  puts Oj.dump( search( Oj.load(line.strip), :upnp ))
+$stdin.each_line do |line|
+  json = Oj.load(line.unpack("C*").pack("C*").strip) rescue nil
+  next unless json
+  puts Oj.dump(search(json, :upnp))
 end

metadata

@@ -1,177 +1,156 @@
 --- !ruby/object:Gem::Specification
 name: dap
 version: !ruby/object:Gem::Version
-  version: 0.0.1
-  prerelease: 
+  version: 0.0.2
 platform: ruby
 authors:
 - Rapid7 Research
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-17 00:00:00.000000000 Z
+date: 2014-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: cucumber
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: aruba
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: net-dns
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bit-struct
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: geoip-c
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: recog
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ! 'DAP reads data using an input plugin, transforms it through a series
+description: 'DAP reads data using an input plugin, transforms it through a series
   of filters, and prints it out again using an output plugin. Every record is treated
   as a document (aka: hash/dict) and filters are used to reduce, expand, and transform
   these documents as they pass through. Think of DAP as a mashup between sed, awk,
@@ -183,15 +162,18 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE
 - README.md
+- Rakefile
 - bin/dap
 - dap.gemspec
 - data/.gitkeep
+- data/vulndb.rb
 - lib/dap.rb
 - lib/dap/filter.rb
 - lib/dap/filter/base.rb
@@ -202,6 +184,7 @@ files:
 - lib/dap/filter/recog.rb
 - lib/dap/filter/simple.rb
 - lib/dap/filter/udp.rb
+- lib/dap/filter/vulnmatch.rb
 - lib/dap/input.rb
 - lib/dap/input/csv.rb
 - lib/dap/input/warc.rb
@@ -209,6 +192,7 @@ files:
 - lib/dap/proto/addp.rb
 - lib/dap/proto/dtls.rb
 - lib/dap/proto/ipmi.rb
+- lib/dap/proto/mssql.rb
 - lib/dap/proto/natpmp.rb
 - lib/dap/proto/wdbrpc.rb
 - lib/dap/utils/oui.rb
@@ -238,27 +222,26 @@ files:
 - tools/value-counts-to-md-table.rb
 homepage: https://www.github.com/rapid7/dap
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
-summary: ! 'DAP: The Data Analysis Pipeline'
+specification_version: 4
+summary: 'DAP: The Data Analysis Pipeline'
 test_files: []
 has_rdoc: