dap 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+# Ignore rvm files
+.ruby-version
+.ruby-gemset
+
+# Ignore geoip data file
+data/geoip.dat

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--format d

data/Gemfile

@@ -0,0 +1,15 @@
+source 'https://rubygems.org'
+
+gem 'nokogiri'
+gem 'oj'
+gem 'htmlentities'
+gem 'net-dns'
+gem 'bit-struct'
+gem 'geoip-c'
+gem 'recog'
+
+group :test do
+  gem 'rspec', '~> 2.14.1'
+  gem 'cucumber', '~> 1.3.8'
+  gem 'aruba', '~> 0.5.3'
+end

data/Gemfile.lock

@@ -0,0 +1,55 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aruba (0.5.4)
+      childprocess (>= 0.3.6)
+      cucumber (>= 1.1.1)
+      rspec-expectations (>= 2.7.0)
+    bit-struct (0.15.0)
+    builder (3.2.2)
+    childprocess (0.5.3)
+      ffi (~> 1.0, >= 1.0.11)
+    cucumber (1.3.15)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.1)
+    diff-lcs (1.2.5)
+    ffi (1.9.3)
+    geoip-c (0.9.1)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    htmlentities (4.3.1)
+    mini_portile (0.6.0)
+    multi_json (1.10.0)
+    multi_test (0.1.1)
+    net-dns (0.8.0)
+    nokogiri (1.6.2.1)
+      mini_portile (= 0.6.0)
+    oj (2.9.0)
+    recog (0.01)
+      nokogiri
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.8)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aruba (~> 0.5.3)
+  bit-struct
+  cucumber (~> 1.3.8)
+  geoip-c
+  htmlentities
+  net-dns
+  nokogiri
+  oj
+  recog
+  rspec (~> 2.14.1)

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Rapid7
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,15 @@
+# DAP: The Data Analysis Pipeline
+
+DAP was created to transform text-based data on the command-line, specializing in transforms that are annoying or difficult to do with existing tools.
+
+DAP reads data using an input plugin, transforms it through a series of filters, and prints it out again using an output plugin. Every record is treated as a document (aka: hash/dict) and filters are used to reduce, expand, and transform these documents as they pass through. Think of DAP as a mashup between sed, awk, grep, csvtool, and jq, with map/reduce capabilities.
+
+DAP was written to process terabyte-sized public scan datasets, such as those provided by https://scans.io/. Although DAP isn't particularly fast, it can be used across multiple cores (and machines) by splitting the input source and wrapping the execution with GNU Parallel.
+
+## Prerequisites
+
+DAP depends on GeoIP (http://dev.maxmind.com/geoip/legacy/downloadable/) to be able to append geographic metadata to analyzed datasets.  At least on Ubuntu, the libgeoip-dev package provides this capability.
+
+## Usage
+
+See [tree/master/samples](/tree/master/samples)

data/bin/dap

@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'rubygems'
+require 'bundler/setup'
+require 'shellwords'
+require 'dap'
+
+def version
+  $stderr.puts "dap #{Dap::VERSION}"
+  exit(0)
+end
+
+def usage
+  $stderr.puts ""
+  $stderr.puts "  Usage: #{$0}  [input] + [filter] + [output]"
+  $stderr.puts "       --inputs"
+  $stderr.puts "       --outputs"
+  $stderr.puts "       --filters"
+  $stderr.puts ""
+  $stderr.puts "Example: echo world | #{$0} lines stdin + rename line=hello + json stdout"
+  $stderr.puts ""
+  exit(1)
+end
+
+def show_inputs
+  $stderr.puts "Inputs:"
+  Dap::Factory.inputs.each_pair do |k,v|
+    $stderr.puts "  * #{k}"
+  end
+  $stderr.puts
+  exit(1)
+end
+
+def show_outputs
+  $stderr.puts "Outputs:"
+  Dap::Factory.outputs.each_pair do |k,v|
+    $stderr.puts "  * #{k}"
+  end  
+  $stderr.puts
+  exit(1)
+end
+
+def show_filters
+  $stderr.puts "Filters:"
+  Dap::Factory.filters.each_pair do |k,v|
+    $stderr.puts "  * #{k}"
+  end  
+  $stderr.puts
+  exit(1)
+end
+
+trace = false
+args  = []
+
+#
+# Tokenize on + then treat each stage as a separate name + argument list
+#
+ARGV.join(' ').split(/\s*\+\s*/).each do |bit|
+  
+  # Handle quoted arguments as needed
+  # XXX: Doesn't work as expected since ARGV parsing gobbles them up
+  aset = Shellwords.shellwords(bit)
+
+  # Check the first argument for help or usage flags
+  arg  = aset.first
+
+  if arg == "--trace"
+    trace = true
+    arg   = aset.shift
+  end
+
+  if arg == "-h" or arg == "--help"
+    usage
+  end
+
+  if arg == "--version" or arg == "-v"
+    version
+  end
+
+  if arg == "--inputs"
+    show_inputs
+  end
+
+  if arg == "--outputs"
+    show_outputs
+  end
+
+  if arg == "--filters"
+    show_filters
+  end
+
+  args << aset if aset.length > 0
+end
+
+inp_args = args.shift
+out_args = args.pop
+
+usage if (inp_args == nil or out_args == nil)
+
+filters = []
+
+inp = Dap::Factory.create_input(inp_args)
+out = Dap::Factory.create_output(out_args)
+args.each do |a|
+  filters << Dap::Factory.create_filter(a)
+end
+
+out.start
+
+while true
+  data = inp.read_record
+  break if data == Dap::Input::Error::EOF
+  next if data == Dap::Input::Error::Empty
+
+  docs = [ data ]
+  
+  fcount = 1
+  filters.each do |f|
+    $stderr.puts "T:  #{" " * (fcount * 2)}#{f.name} -> #{docs.inspect} " if trace
+    docs = docs.collect {|doc| f.process(doc) }.flatten
+    $stderr.puts "T:  #{" " * (fcount * 2)}#{" " * f.name.length} == #{docs.inspect}" if trace
+    fcount += 1
+    break if docs.length == 0
+  end
+
+  begin
+    docs.each do |doc|
+      out.write_record(doc)
+    end
+  rescue ::Errno::EPIPE
+    break
+  end
+end
+
+out.stop 

data/dap.gemspec

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'dap/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'dap'
+  s.version     = Dap::VERSION
+  s.authors     = [
+      'Rapid7 Research'
+  ]
+  s.email       = [
+      'research@rapid7.com'
+  ]
+  s.homepage    = "https://www.github.com/rapid7/dap"
+  s.summary     = %q{DAP: The Data Analysis Pipeline}
+  s.description = %q{
+    DAP reads data using an input plugin, transforms it through a series of filters, and prints it out again 
+    using an output plugin. Every record is treated as a document (aka: hash/dict) and filters are used to 
+    reduce, expand, and transform these documents as they pass through. Think of DAP as a mashup between 
+    sed, awk, grep, csvtool, and jq, with map/reduce capabilities.
+  }.gsub(/\s+/, ' ').strip
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  # ---- Dependencies ----
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'cucumber'
+  s.add_development_dependency 'aruba'
+
+  s.add_runtime_dependency 'nokogiri'
+  s.add_runtime_dependency 'oj'
+  s.add_runtime_dependency 'htmlentities'
+  s.add_runtime_dependency 'net-dns'
+  s.add_runtime_dependency 'bit-struct'
+  s.add_runtime_dependency 'geoip-c'
+  s.add_runtime_dependency 'recog'
+
+end

data/lib/dap.rb

@@ -0,0 +1,101 @@
+module Dap
+
+  require 'bundler/setup'
+  
+  require 'dap/version'
+  require 'dap/input'
+  require 'dap/output'
+  require 'dap/filter'
+
+  class Factory
+
+    @@inputs  = {}
+    @@outputs = {}
+    @@filters = {}
+
+    def self.create_input(args)
+      name = args.shift
+      raise RuntimeError, "Invalid input plugin: #{name}" unless @@inputs[name]
+      @@inputs[name].new(args)
+    end
+    
+    def self.create_output(args)
+      name = args.shift
+      raise RuntimeError, "Invalid output plugin: #{name}" unless @@outputs[name]
+      @@outputs[name].new(args)
+    end
+
+    def self.create_filter(args)
+      name = args.shift
+      raise RuntimeError, "Invalid filter plugin: #{name}" unless @@filters[name]
+      @@filters[name].new(args)
+    end
+
+    #
+    # Create nice-looking filter names from classes
+    # Ex: FilterHTTPDecode => http_decode
+    # Ex: FilterLimitLen => limit_len
+    #
+    def self.name_from_class(name)
+      name.to_s.split('::').last.
+      gsub(/([A-Z][a-z])/) { |c| "_#{c[0,1].downcase}#{c[1,1]}" }.
+      gsub(/([a-z][A-Z])/) { |c| "#{c[0,1]}_#{c[1,1].downcase}" }.
+      gsub(/_+/, '_').
+      sub(/^_(input|filter|output)_/, '').downcase
+    end
+    
+    #
+    # Load input formats
+    #
+    def self.load_inputs
+      Dap::Input.constants.each do |c|
+        next unless c.to_s =~ /^Input/
+        o = Dap::Input.const_get(c)
+        @@inputs[ name_from_class(c) ] = o
+      end
+    end
+
+    #
+    # Load output formats
+    #
+    def self.load_outputs
+      Dap::Output.constants.each do |c|
+        o = Dap::Output.const_get(c)
+        next unless c.to_s =~ /^Output/
+        @@outputs[ name_from_class(c) ] = o        
+      end      
+    end
+
+    #
+    # Load filters
+    #
+    def self.load_filters
+      Dap::Filter.constants.each do |c|
+        o = Dap::Filter.const_get(c)
+        next unless c.to_s =~ /^Filter/
+        @@filters[ name_from_class(c) ] = o        
+      end
+    end
+
+    def self.inputs
+      @@inputs
+    end
+
+    def self.outputs
+      @@outputs
+    end
+
+    def self.filters
+      @@filters
+    end    
+
+    def self.load_modules
+      self.load_inputs
+      self.load_outputs
+      self.load_filters
+    end
+  end
+
+  Factory.load_modules
+
+end

data/lib/dap/filter.rb

@@ -0,0 +1,8 @@
+require 'dap/filter/base'
+require 'dap/filter/simple'
+require 'dap/filter/http'
+require 'dap/filter/udp'
+require 'dap/filter/openssl'
+require 'dap/filter/names'
+require 'dap/filter/geoip'
+require 'dap/filter/recog'

data/lib/dap/filter/base.rb

@@ -0,0 +1,37 @@
+module Dap
+module Filter
+
+module Base
+  attr_accessor :name, :opts
+
+  def initialize(args)
+    self.opts = {}
+    args.each do |arg|
+        k,v = arg.split("=", 2)
+        self.opts[k] = v
+    end
+    self.name = Dap::Factory.name_from_class(self.class)
+  end
+
+  def process(doc)
+    raise RuntimeError, "No process() method defined for filter #{self.name}"
+  end
+
+end
+
+module BaseDecoder
+  include Base
+  def process(doc)
+    self.opts.each_pair do |k,v|
+      next unless doc.has_key?(k)
+      info = decode(doc[k]) || {}
+      info.each_pair do |x,y|
+        doc[ "#{k}.#{x}" ] = y
+      end
+    end
+   [ doc ]
+  end
+end
+
+end
+end