dap 0.0.1

data/lib/dap/version.rb

@@ -0,0 +1,3 @@
+module Dap
+  VERSION = "0.0.1"
+end

data/samples/http_get_reply.ic12.sh

@@ -0,0 +1 @@
+bzcat http_get_reply.ic12.bz2 | ../bin/dap lines + field_split_tab line + rename line.f1=ip line.f4=data + select ip data + transform data=qprintdecode + decode_http_reply data + select ip data.http_code data.http_server + json

data/samples/http_get_reply_iframes.json.sh

@@ -0,0 +1 @@
+bzcat http_get_reply_iframes.json.bz2 | ../bin/dap json + transform data=base64decode + include data='<iframe'  + html_iframes data + select ip iframe + json

data/samples/http_get_reply_links.json.sh

@@ -0,0 +1 @@
+bzcat http_get_reply_iframes.json.bz2 | ../bin/dap json + transform data=base64decode + html_links data + select ip link element + decode_uri link + json

data/samples/iawide_warc.sh

@@ -0,0 +1 @@
+bzcat iawide.warc.bz2 | ../bin/dap warc + html_links content + select ip link element + decode_uri link + json

data/samples/ipmi_chan_auth_replies.sh

@@ -0,0 +1 @@
+bzcat ipmi_chan_auth_replies.crd.bz2 | ../bin/dap lines + field_split_tab line +  rename line.f2=ip line.f6=data + select ip data + transform data=hexdecode + decode_ipmi_chan_auth_reply data + remove data +  json

data/samples/ssl_certs_geo.sh

@@ -0,0 +1 @@
+bzcat ssl_certs.bz2 | ../bin/dap json + select host_ip ssl_version port cipher + geo_ip host_ip + json

data/samples/ssl_certs_names.sh

@@ -0,0 +1 @@
+bzcat ssl_certs.bz2 | ../bin/dap json + field_split_array certs + transform certs.f1=base64decode + remove certs + decode_x509 certs.f1 + select certs.f1.names + exists certs.f1.names + split_array certs.f1.names + select certs.f1.names.item + exists certs.f1.names.item + lines

data/samples/ssl_certs_names_expanded.sh

@@ -0,0 +1 @@
+bzcat ssl_certs.bz2 | ../bin/dap json + field_split_array certs + transform certs.f1=base64decode + remove certs + decode_x509 certs.f1 + select certs.f1.names + exists certs.f1.names + split_array certs.f1.names + select certs.f1.names.item + exists certs.f1.names.item + rename certs.f1.names.item=hostname  + extract_hostname hostname + select hostname.hostname + split_domains hostname.hostname + select hostname.hostname.domain + rename hostname.hostname.domain=name + prepend_subdomains name=www,dns,mail,vpn,secure,ssl + json

data/samples/ssl_certs_org.sh

@@ -0,0 +1 @@
+bzcat ssl_certs.bz2 | ../bin/dap json + select host_ip ssl_version port cipher + geo_ip_org host_ip + json

data/samples/udp-netbios.sh

@@ -0,0 +1 @@
+bzcat udp-netbios.csv.bz2 | ../bin/dap csv - header=y + select saddr data + rename saddr=ip + transform data=hexdecode + decode_netbios_status_reply data + remove data + geo_ip ip + json 

data/spec/dap/proto/ipmi_spec.rb

@@ -0,0 +1,19 @@
+require 'bit-struct'
+require_relative '../../../lib/dap/proto/ipmi'
+
+module Dap
+module Proto
+module IPMI
+
+describe Channel_Auth_Reply do
+  it "valid with the proper rmcp version and message length" do
+    expect(subject.valid?).to be_false
+    expect(Channel_Auth_Reply.new(rmcp_version: 6).valid?).to be_false
+    expect(Channel_Auth_Reply.new(message_length: 16).valid?).to be_false
+    expect(Channel_Auth_Reply.new(rmcp_version: 6, message_length: 16).valid?).to be_true
+  end
+end
+
+end
+end
+end

data/tools/geo-ip-summary.rb

@@ -0,0 +1,149 @@
+#!/usr/bin/env ruby
+require 'oj'
+require 'optparse'
+
+class GeoIPSummary
+  attr_accessor :country_name, :region_name, :city_name, :tree
+  #
+  # Pass the hash keys for the country name, region name and
+  # city name that we'll encounter during the process_hash function.
+  #
+  def initialize(country_name, region_name, city_name)
+    @country_name = country_name
+    @region_name = region_name
+    @city_name = city_name
+    @tree = {}
+    @tree['count'] = 0
+  end
+
+  def process_hash( json_hash )
+    country = json_hash[@country_name]
+    region  = json_hash[@region_name] || 'Undefined Region'
+    city    = json_hash[@city_name]   || 'Undefined City'
+
+    # Create subhashes and values as needed on down the tree
+    @tree[country] ||= {}
+    @tree[country]['count'] ||=0
+    @tree[country][region] ||= {}
+    @tree[country][region]['count'] ||= 0
+    @tree[country][region][city] ||= 0
+
+    # Now increment counters
+    @tree['count'] += 1
+    @tree[country]['count'] += 1
+    @tree[country][region]['count'] +=1
+    @tree[country][region][city] += 1
+  end
+
+  # Performs the final sorting of the hash, with descending order of counts
+  #
+  def order_tree
+    @tree.each do | country, country_hash|
+      if country != 'count'
+        country_hash.each do | region, region_hash |
+          @tree[country][region] = order_hash(@tree[country][region]) if region != 'count'
+        end
+        @tree[country] = order_hash(@tree[country])
+      end
+    end
+    @tree = order_hash(@tree)
+  end
+
+  private
+
+  # Sorts the hash, and returns a copy of the hash in sorted order by their counts, or if
+  # counts are equal then by their names.
+  def order_hash(h)
+    keys = h.keys.sort { | k1,k2 |
+      if k1 == 'count'
+        ret = -1
+      elsif k2 == 'count'
+        ret = 1
+      else
+        # Cities level is slightly different form, if hash at this level then compare
+        # count value within hash, otherwise just compare values. mult by -1 to reverse
+        # ordering
+        if h[k1].class == Hash
+          ret = ( h[k1]['count'] <=> h[k2]['count'] ) * -1
+          ret = k1 <=> k2 if ret == 0 && k1!=nil && k2!=nil
+        else
+          ret = ( h[k1] <=> h[k2] ) * -1
+          ret = k1 <=> k2 if ret == 0 && k1!=nil && k2!=nil
+        end
+      end
+      ret
+    }
+
+    # build up return hash
+    ret_hash = {}
+    keys.each do | key |
+      ret_hash[key] = h[key]
+    end
+
+    ret_hash
+  end
+end
+HELP=<<EOF
+ This script is used to summarize geoip data from data in a json file. The name of the json element for
+ the country, region, and city must be provided. The output is a hash with the country/region/city data and
+ the count of occurrences from the input file; this output hash is sorted in count descending order so that
+ the most common country, region within a country, and city within a region is returned first.
+
+ Example with dap:
+   bzcat ../samples/ssl_certs.bz2 | ../bin/dap json + select host_ip + geo_ip host_ip + json | ./geo-ip-summary.rb --var host_ip > /tmp/ssl_geo.json
+EOF
+
+def parse_command_line(args)
+
+  options={
+      :country => nil,
+      :region => nil,
+      :city => nil,
+      :var => nil
+  }
+
+  OptionParser.new do | opts |
+    opts.banner = HELP
+    opts.separator ''
+
+    opts.separator 'GeoIP name options:'
+
+    opts.on( '--country country_key', 'The name of json key for the country.') do | val |
+      options[:country] = val
+    end
+
+    opts.on( '--region region_key', 'The name of the json key for the region.') do | val |
+      options[:region] = val
+    end
+
+    opts.on( '--city city_key', 'The name of the json key for the city.' ) do | val |
+      options[:city] = val
+    end
+
+    opts.on('--var top-level-var', 'Sets the top level json name, for defining all of country/region/city') do | val |
+      options[:var] = val
+      options[:country] = "#{val}.country_name"
+      options[:region] = "#{val}.region"
+      options[:city]   = "#{val}.city"
+    end
+
+    opts.on_tail('-h', '--help', 'Show this message') do
+      puts opts
+      exit(0)
+    end
+    opts.parse!(args)
+    options
+  end
+  options
+end
+  opts = parse_command_line(ARGV)
+  raise 'Need json key names for country,region and city.' if opts[:country].nil? || opts[:region].nil? || opts[:city].nil?
+
+  summarizer = GeoIPSummary.new(opts[:country], opts[:region], opts[:city])
+  while line=gets
+    summarizer.process_hash(Oj.load(line.strip))
+  end
+
+  Oj.default_options={:indent=>2}
+
+  puts Oj.dump(summarizer.order_tree)

data/tools/ipmi-vulns.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'oj'
+
+SEARCHES = {
+  "data.ipmi_compat_password"           => { value: "1", name: "straight-pass" },
+  "data.ipmi_compat_md2"                => { value: "1", name: "md2" },
+  "data.ipmi_compat_none"               => { value: "1", name: "noauth" },
+  "data.ipmi_user_disable_message_auth" => { value: "1", name: "permsg" },
+  "data.ipmi_user_disable_user_auth"    => { value: "1", name: "usrlvl" }
+}
+
+def search(hash)
+  SEARCHES.each do | key, vuln |
+    if hash[key] == vuln[:value]
+      hash["VULN-IPMI-#{vuln[:name].upcase}"] = "true"
+    end
+  end
+  if (hash['data.ipmi_user_non_null'] == "0") && (hash['data.ipmi_user_null'] == "0")
+    hash["VULN-IPMI-ANON"] = "true"
+  end 
+  hash
+end
+
+while line=gets
+  puts Oj.dump(search(Oj.load(line.strip)))
+end

data/tools/json-summarize.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+require 'oj'
+require 'optparse'
+
+HELP=<<EOF
+ This script is used to locate the frequency of a given key in a json document. It will
+ inspect and increment the frequency count for each instance of the key found in the json
+ document, then order them in descending order and output a json document with the top n
+ occurrences of the key value.
+
+  Note that if passed a key that has unique values, this script can consume a lot of memory.
+
+  Sample:
+    unpigz -c /tmp/2014-05-05-mssql-udp-decoded.json.gz | ruby ~/src/dap/tools/json-summarize.rb --top 20 --key data.mssql.Version
+EOF
+
+def parse_command_line(args)
+
+  options={
+      :key => nil,
+      :number => nil
+  }
+
+  OptionParser.new do | opts |
+    opts.banner = HELP
+    opts.separator ''
+
+    opts.separator 'GeoIP name options:'
+
+    opts.on( '--key keyname', 'The name of json key to be summarized.') do | val |
+      options[:key] = val
+    end
+
+    opts.on( '--top num_items', 'Return top n occurrences.') do | val |
+      options[:number] = val.to_i
+    end
+
+    opts.on_tail('-h', '--help', 'Show this message') do
+      puts opts
+      exit(0)
+    end
+    opts.parse!(args)
+    options
+  end
+  options
+end
+
+# Sorts the hash in descending numerical value for the values
+# part of the hash, returning the sorted hash.
+#
+def order_hash(h)
+  keys = h.keys.sort { | k1,k2 |
+    ret = ( h[k1] <=> h[k2] ) * -1
+    ret = k1 <=> k2 if ret == 0 && k1!=nil && k2!=nil
+    ret
+  }
+  # build up return hash
+  ret_hash = {}
+  keys.each do | key |
+    ret_hash[key] = h[key]
+  end
+
+  ret_hash
+end
+
+
+
+
+  summary={}
+  opts = parse_command_line(ARGV)
+  key = opts[:key]
+
+  while line = gets
+    val = Oj.load(line.chomp.strip)[key]
+    summary[val] ||= 0
+    summary[val] += 1
+  end
+
+  summary = Hash[ *order_hash(summary).flatten.slice(0,2*opts[:number]) ]
+  puts Oj.dump(summary)
+

data/tools/netbios-counts.rb

@@ -0,0 +1,271 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'ostruct'
+require 'oj'
+require 'json'
+
+options = OpenStruct.new
+options.top_count = 5
+options.exclude_default_counts = false
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: netbios-counts.rb [options]"
+
+  opts.on("-c", "--count [NUM]", OptionParser::DecimalInteger, 
+          "Specify the number of top count results") do |count|
+    options.top_count = count if count > 1
+  end
+
+  opts.on("--count-hostnames-containing [TEXT]", "Count hostnames that include the speified text") do |text|
+    options.hostname_containing = text
+  end 
+
+  opts.on("--exclude-default-counts", "Exclude the provided top counts") do
+    options.exclude_default_counts = true
+  end
+end.parse!
+
+NUM_TOP_RECORDS = options.top_count
+
+module Counter
+  def count(hash)
+    value = countable_value(hash)
+    @counts[value] += 1 unless (value.empty? || value == 'UNKNOWN')
+  end
+
+  def top_counts
+    [].tap do |counts|
+      ordered_by_count.to_a.take(NUM_TOP_RECORDS).each do |values|
+        counts << count_hash(values) 
+      end
+    end
+  end
+
+  def ordered_by_count
+    Hash[@counts.sort_by{|k, v| v}.reverse] 
+  end
+end
+
+class CompanyNameCounter
+  include Counter
+
+  def initialize
+    @counts = Hash.new(0)
+  end
+
+  def countable_value(hash)
+    hash['data.netbios_mac_company'].to_s
+  end
+
+  def count_hash(values)
+    { 'name' => values[0], 'count' => values[1] }
+  end
+
+  def apply_to(hash)
+    hash['top_companies'] = top_counts
+  end
+end
+
+class NetbiosNameCounter
+  include Counter
+
+  def initialize
+    @counts = Hash.new(0)
+  end
+
+  def countable_value(hash)
+    hash['data.netbios_hname'].to_s
+  end
+
+  def count_hash(values)
+    { 'hostname' => values[0], 'count' => values[1] }
+  end
+
+  def apply_to(hash)
+    hash['top_netbios_hostnames'] = top_counts
+  end
+end
+
+class MacAddressCounter
+  include Counter
+
+  def initialize
+    @counts = Hash.new(0)
+  end
+
+  def countable_value(hash)
+    address = hash['data.netbios_mac'].to_s
+    [].tap do |data|
+      unless (address.empty? || address == '00:00:00:00:00:00')
+        data << address
+        data << hash['data.netbios_hname']
+        data << hash['data.netbios_mac_company']
+      end
+    end
+  end
+
+  def count_hash(values)
+    { 
+      'mac_address' => values[0][0], 
+      'hostname'    => values[0][1],
+      'company'     => values[0][2],
+      'count'       => values[1] 
+    }
+  end
+
+  def apply_to(hash)
+    hash['top_mac_addresses'] = top_counts
+  end
+end
+
+class GeoCounter
+  def initialize
+    @cities    = Hash.new(0)
+    @countries = Hash.new(0)
+    @regions   = Hash.new(0)
+  end
+
+  def count(hash)
+    city         = hash['ip.city'].to_s
+    country_code = hash['ip.country_code'].to_s
+    region       = hash['ip.region'].to_s
+    region_name  = hash['ip.region_name'].to_s
+
+    @cities[[city, country_code]] += 1 unless city.empty?
+    @countries[country_code] += 1 unless country_code.empty?
+    @regions[[region, region_name]] += 1 unless region.empty?
+  end
+  
+  def top_cities
+    [].tap do |counts|
+      ordered_cities.to_a.take(NUM_TOP_RECORDS).each do |values|
+        counts << { 
+          'city'    => values[0][0], 
+          'country_code' => values[0][1], 
+          'count'   => values[1] 
+        }
+      end
+    end
+  end
+
+  def top_countries
+    [].tap do |counts|
+      ordered_countries.to_a.take(NUM_TOP_RECORDS).each do |values|
+        counts << { 'country_code' => values[0], 'count' => values[1] }
+      end
+    end
+  end
+
+  def top_regions
+    [].tap do |counts|
+      ordered_regions.to_a.take(NUM_TOP_RECORDS).each do |values|
+        counts << { 
+          'region'      => values[0][0], 
+          'region_name' => values[0][1], 
+          'count'       => values[1] 
+        }
+      end
+    end
+  end
+
+  def ordered_cities
+    Hash[@cities.sort_by{|k, v| v}.reverse] 
+  end
+
+  def ordered_countries
+    Hash[@countries.sort_by{|k, v| v}.reverse] 
+  end
+
+  def ordered_regions
+    Hash[@regions.sort_by{|k, v| v}.reverse] 
+  end
+
+  def apply_to(hash)
+    hash['top_cities']    = top_cities unless top_cities.empty?
+    hash['top_countries'] = top_countries unless top_countries.empty?
+    hash['top_regions']   = top_regions unless top_regions.empty?
+  end
+end
+
+class SambaCounter
+  include Counter
+
+  def initialize
+    @counts = Hash.new(0)
+  end
+
+  def countable_value(hash)
+    address = hash['data.netbios_mac'].to_s
+    if (address == '00:00:00:00:00:00')
+      hash['data.netbios_hname']
+    else
+      ''
+    end
+  end
+
+  def count_hash(values)
+    { 'name'  => values[0], 'count' => values[1] }
+  end
+
+  def apply_to(hash)
+    hash['top_samba_names'] = top_counts
+  end
+end
+
+class HostnameContainingCounter
+  include Counter
+
+  def initialize(text)
+    @text = text
+    @counts = Hash.new(0)
+  end
+
+  def countable_value(hash)
+    hostname = hash['data.netbios_hname'].to_s
+    [].tap do |data|
+      if hostname.include?(@text)
+        data << hostname
+        data << hash['data.netbios_mac_company']
+        data << hash['ip.city'].to_s
+        data << hash['ip.country_code'].to_s
+        data << hash['ip.country_name'].to_s
+      end
+    end
+  end
+
+  def count_hash(values)
+    { 
+      'hostname'     => values[0][0],
+      'company'      => values[0][1],
+      'city'         => values[0][2],
+      'country_code' => values[0][3],
+      'country_name' => values[0][4],
+      'count'        => values[1] 
+    }
+  end
+
+  def apply_to(hash)
+    hash["hostnames with '#{@text}'"] = top_counts
+  end
+end
+
+counters = []
+unless options.exclude_default_counts
+  counters << CompanyNameCounter.new 
+  counters << NetbiosNameCounter.new
+  counters << MacAddressCounter.new
+  counters << GeoCounter.new
+  counters << SambaCounter.new
+end
+counters << HostnameContainingCounter.new(options.hostname_containing) unless options.hostname_containing.nil?
+
+while line=gets
+  hash = Oj.load(line.strip)
+  counters.each { |counter| counter.count(hash) }
+end
+
+summary = {}
+counters.each { |counter| counter.apply_to(summary) }
+
+puts JSON.pretty_generate(summary)