dap 0.0.1

data/tools/upnp-vulns.rb

@@ -0,0 +1,35 @@
+require 'oj'
+
+
+# Searches contains each of the services, within each service it contains
+# a hash key that will be compared against each of the items in the
+# regex hash, and if a hit is returned the value from the regex is inserted
+# into the hash with the output_key as the key.
+#
+SEARCHES = {
+    :upnp => {
+      :hash_key   => 'data.upnp_server',
+      :output_key => 'vulnerability',
+      :regex      => {
+        /MiniUPnPd\/1\.0([\.\,\-\~\s]|$)/mi     => 'CVE-2013-0229',
+        /MiniUPnPd\/1\.[0-3]([\.\,\-\~\s]|$)/mi => 'CVE-2013-0230',
+        /Intel SDK for UPnP devices.*|Portable SDK for UPnP devices(\/?\s*$|\/1\.([0-5]\..*|8\.0.*|(6\.[0-9]|6\.1[0-7])([\.\,\-\~\s]|$)))/mi =>  'CVE-2012-5958 , CVE-2012-5959'
+      }
+  }
+}
+
+def search(hash, service)
+  SEARCHES[service][:regex].each do | regex, value |
+    if regex =~ hash[SEARCHES[service][:hash_key]].force_encoding('BINARY')
+      # Handle cases that could be multiple hits, not for upnp but could be others.
+      hash[SEARCHES[service][:output_key]] = ( hash[SEARCHES[service][:output_key]] ? hash[SEARCHES[service][:output_key]] + ',' + value : value )
+    end
+  end if hash[SEARCHES[service][:hash_key]]
+  hash
+end
+
+while line=gets
+  #line.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '')
+  #line.force_encoding('BINARY')
+  puts Oj.dump( search( Oj.load(line.strip), :upnp ))
+end

data/tools/value-counts-to-md-table.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+info = {}
+$stdin.each_line do |line|
+  line = line.unpack("C*").pack("C*").strip
+  info[line] ||= 0
+  info[line]  +=1
+end
+
+
+puts "
+
+#### Top Values
+| Count        | Value |
+|:------------- | ------------- |"
+
+max = 100
+cnt = 0
+info.keys.sort {|a,b| info[b] <=> info[a] }.each do |k|
+  puts "| #{info[k]} | #{k} |"
+  cnt +=1
+  break if cnt > max
+end
+puts ""

metadata

@@ -0,0 +1,264 @@
+--- !ruby/object:Gem::Specification
+name: dap
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Rapid7 Research
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-dns
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bit-struct
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: geoip-c
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: recog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! 'DAP reads data using an input plugin, transforms it through a series
+  of filters, and prints it out again using an output plugin. Every record is treated
+  as a document (aka: hash/dict) and filters are used to reduce, expand, and transform
+  these documents as they pass through. Think of DAP as a mashup between sed, awk,
+  grep, csvtool, and jq, with map/reduce capabilities.'
+email:
+- research@rapid7.com
+executables:
+- dap
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- bin/dap
+- dap.gemspec
+- data/.gitkeep
+- lib/dap.rb
+- lib/dap/filter.rb
+- lib/dap/filter/base.rb
+- lib/dap/filter/geoip.rb
+- lib/dap/filter/http.rb
+- lib/dap/filter/names.rb
+- lib/dap/filter/openssl.rb
+- lib/dap/filter/recog.rb
+- lib/dap/filter/simple.rb
+- lib/dap/filter/udp.rb
+- lib/dap/input.rb
+- lib/dap/input/csv.rb
+- lib/dap/input/warc.rb
+- lib/dap/output.rb
+- lib/dap/proto/addp.rb
+- lib/dap/proto/dtls.rb
+- lib/dap/proto/ipmi.rb
+- lib/dap/proto/natpmp.rb
+- lib/dap/proto/wdbrpc.rb
+- lib/dap/utils/oui.rb
+- lib/dap/version.rb
+- samples/http_get_reply.ic12.bz2
+- samples/http_get_reply.ic12.sh
+- samples/http_get_reply_iframes.json.bz2
+- samples/http_get_reply_iframes.json.sh
+- samples/http_get_reply_links.json.sh
+- samples/iawide.warc.bz2
+- samples/iawide_warc.sh
+- samples/ipmi_chan_auth_replies.crd.bz2
+- samples/ipmi_chan_auth_replies.sh
+- samples/ssl_certs.bz2
+- samples/ssl_certs_geo.sh
+- samples/ssl_certs_names.sh
+- samples/ssl_certs_names_expanded.sh
+- samples/ssl_certs_org.sh
+- samples/udp-netbios.csv.bz2
+- samples/udp-netbios.sh
+- spec/dap/proto/ipmi_spec.rb
+- tools/geo-ip-summary.rb
+- tools/ipmi-vulns.rb
+- tools/json-summarize.rb
+- tools/netbios-counts.rb
+- tools/upnp-vulns.rb
+- tools/value-counts-to-md-table.rb
+homepage: https://www.github.com/rapid7/dap
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: ! 'DAP: The Data Analysis Pipeline'
+test_files: []
+has_rdoc: