dap 1.0.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9102bebe944c3db1ab080d8bfae9990a1bcd4830
-  data.tar.gz: a3baf5ac569e0bf6871f12a7fa11aa8c6911135c
+  metadata.gz: '0467108dd6edf2120b8de36390c3a338aadbfaf8'
+  data.tar.gz: 0a3a5c8f96087bd1c09af9534867e5b7f619d2b0
 SHA512:
-  metadata.gz: 54bd662a9ac9363f2a5bca07abf398796310cedd08a97f1f90619146699a8288ff230e78f636c6b108aeadcabf1b54e50b87309ed13b0dc1cd94070dc6007409
-  data.tar.gz: a16f006e782180fbb418f1f9032fb361bca480dac9b6b2724f6d85e0c314b048a85032a07523cb48138fc6704d9065149dce3f8c8be7b323463d160265cbc6f0
+  metadata.gz: 622ed2ba3f267ab350d7323f8fb70e44a196d4800fe85fd6e7d2e64add8abdee448d83cd6c2d1577f873c55cdb6dd32d256c09f2e40154b1e14fff66506e08e1
+  data.tar.gz: bf214ba2c71977e1f0d26d7dc35193ccf28fc9d0239fb900be3c539ddc6ee742046ddc92c556d7986a704a2c73613611aef1fe7da9cc399f95b2fe98364acdad

data/Dockerfile.testing

@@ -14,13 +14,21 @@ RUN /bin/bash -l -c "rvm use 2.4.5 && gem update --system && gem install bundler
 ADD Gemfile* $TEST_DIR/
 RUN /bin/bash -l -c "cd $TEST_DIR && rvm use 2.4.5 && bundle install"
 
-# install maxmind data
+# install maxmind legacy data
 RUN mkdir /var/lib/geoip
-WORKDIR /var/lib/geoip
-RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIP.dat
-RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPCity.dat -O GeoLiteCity.dat
-RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPASNum.dat
-RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPOrg.dat -O geoip_org.dat
+COPY test/test_data/geoip/*.dat /var/lib/geoip/
+# Note that these test files were copied from
+# https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPCity.dat
+# https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPASNum.dat
+# https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPOrg.dat
+
+# install maxmind geoip2 data
+RUN mkdir /var/lib/geoip2
+COPY test/test_data/geoip2/*.mmdb /var/lib/geoip2/
+# Note that these test files were copied from
+# https://github.com/maxmind/MaxMind-DB/raw/f6ed981c23b0eb33d7c07568e2177236252afda6/test-data/GeoLite2-ASN-Test.mmdb
+# https://github.com/maxmind/MaxMind-DB/raw/f6ed981c23b0eb33d7c07568e2177236252afda6/test-data/GeoIP2-City-Test.mmdb
+# https://github.com/maxmind/MaxMind-DB/blob/f6ed981c23b0eb33d7c07568e2177236252afda6/test-data/GeoIP2-ISP-Test.mmdb
 
 # install bats
 RUN git clone https://github.com/sstephenson/bats.git && cd bats && ./install.sh /usr

data/Gemfile

@@ -5,7 +5,8 @@ gem 'htmlentities'
 gem 'net-dns'
 gem 'bit-struct'
 gem 'geoip-c'
-gem 'recog', '>=2.1.12'
+gem 'maxmind-db', '~> 1.0.0'
+gem 'recog', '>=2.3.0'
 
 group :test do
   gem 'rspec', '~> 3.1.0'

data/Gemfile.lock

@@ -21,14 +21,15 @@ GEM
     gherkin (2.12.2)
       multi_json (~> 1.3)
     htmlentities (4.3.4)
-    mini_portile2 (2.2.0)
+    maxmind-db (1.0.0)
+    mini_portile2 (2.4.0)
     multi_json (1.11.2)
     multi_test (0.1.2)
     net-dns (0.9.0)
-    nokogiri (1.8.0)
-      mini_portile2 (~> 2.2.0)
+    nokogiri (1.10.3)
+      mini_portile2 (~> 2.4.0)
     oj (3.7.9)
-    recog (2.1.12)
+    recog (2.3.0)
       nokogiri
     rspec (3.1.0)
       rspec-core (~> 3.1.0)
@@ -52,9 +53,10 @@ DEPENDENCIES
   cucumber (~> 1.3.16)
   geoip-c
   htmlentities
+  maxmind-db (~> 1.0.0)
   net-dns
   oj
-  recog (>= 2.1.12)
+  recog (>= 2.3.0)
   rspec (~> 3.1.0)
 
 BUNDLED WITH

data/lib/dap/filter.rb

@@ -5,6 +5,7 @@ require 'dap/filter/udp'
 require 'dap/filter/openssl'
 require 'dap/filter/names'
 require 'dap/filter/geoip'
+require 'dap/filter/geoip2'
 require 'dap/filter/recog'
 require 'dap/filter/vulnmatch'
 require 'dap/filter/ssh_keyscan'

data/lib/dap/filter/geoip.rb

@@ -16,26 +16,54 @@ module GeoIPLibrary
   @@geo_orgs = nil
   @@geo_asn = nil
 
-  GEOIP_DIRS.each do |d|
-    GEOIP_CITY.each do |f|
-      path = File.join(d, f)
-      if ::File.exist?(path)
-        @@geo_city = GeoIP::City.new(path)
-        break
+  GEOIP_CITY_DATABASE_PATH = ENV["GEOIP_CITY_DATABASE_PATH"]
+  GEOIP_ASN_DATABASE_PATH = ENV["GEOIP_ASN_DATABASE_PATH"]
+  GEOIP_ORG_DATABASE_PATH = ENV["GEOIP_ORG_DATABASE_PATH"]
+
+  if GEOIP_CITY_DATABASE_PATH
+    if ::File.exist?(GEOIP_CITY_DATABASE_PATH)
+      @@geo_city = GeoIP::City.new(GEOIP_CITY_DATABASE_PATH)
+    end
+  else
+    GEOIP_DIRS.each do |d|
+      GEOIP_CITY.each do |f|
+        path = File.join(d, f)
+        if ::File.exist?(path)
+          @@geo_city = GeoIP::City.new(path)
+          break
+        end
       end
     end
-    GEOIP_ORGS.each do |f|
-      path = File.join(d, f)
-      if ::File.exist?( path )
-        @@geo_orgs = GeoIP::Organization.new(path)
-        break
+  end
+
+  if GEOIP_ORG_DATABASE_PATH
+    if ::File.exist?(GEOIP_ORG_DATABASE_PATH)
+      @@geo_orgs = GeoIP::Organization.new(GEOIP_ORG_DATABASE_PATH)
+    end
+  else
+    GEOIP_DIRS.each do |d|
+      GEOIP_ORGS.each do |f|
+        path = File.join(d, f)
+        if ::File.exist?( path )
+          @@geo_orgs = GeoIP::Organization.new(path)
+          break
+        end
       end
     end
-    GEOIP_ASN.each do |f|
-      path = File.join(d, f)
-      if ::File.exist?(path)
-        @@geo_asn = GeoIP::Organization.new(path)
-        break
+  end
+
+  if GEOIP_ASN_DATABASE_PATH
+    if ::File.exist?(GEOIP_ASN_DATABASE_PATH)
+      @@geo_asn = GeoIP::Organization.new(GEOIP_ASN_DATABASE_PATH)
+    end
+  else
+    GEOIP_DIRS.each do |d|
+      GEOIP_ASN.each do |f|
+        path = File.join(d, f)
+        if ::File.exist?(path)
+          @@geo_asn = GeoIP::Organization.new(path)
+          break
+        end
       end
     end
   end

data/lib/dap/filter/geoip2.rb

@@ -0,0 +1,286 @@
+require 'maxmind/db'
+
+module Dap
+module Filter
+
+require 'dap/utils/misc'
+
+module GeoIP2Library
+  GEOIP2_DIRS = [
+    File.expand_path( File.join( File.dirname(__FILE__), "..", "..", "..", "data")),
+    "/var/lib/geoip",
+    "/var/lib/geoip2"
+  ]
+  GEOIP2_CITY = %W{ GeoLite2-City.mmdb }
+  GEOIP2_ASN = %W{ GeoLite2-ASN.mmdb }
+  GEOIP2_ISP = %W{ GeoIP2-ISP.mmdb }
+
+  def self.find_db(db_file_names, db_dirs, env_path)
+    if env_path
+      if ::File.exist?(env_path)
+        return MaxMind::DB.new(env_path, mode: MaxMind::DB::MODE_MEMORY)
+      end
+    else
+      db_dirs.each do |d|
+        db_file_names.each do |f|
+          path = File.join(d, f)
+          if ::File.exist?(path)
+            return MaxMind::DB.new(path, mode: MaxMind::DB::MODE_MEMORY)
+          end
+        end
+      end
+    end
+    nil
+  end
+
+  @@geo_asn = find_db(GEOIP2_ASN, GEOIP2_DIRS, ENV["GEOIP2_ASN_DATABASE_PATH"])
+  @@geo_city = find_db(GEOIP2_CITY, GEOIP2_DIRS, ENV["GEOIP2_CITY_DATABASE_PATH"])
+  @@geo_isp = find_db(GEOIP2_ISP, GEOIP2_DIRS, ENV["GEOIP2_ISP_DATABASE_PATH"])
+end
+
+
+#
+# Add GeoIP2 tags using the MaxMind GeoIP2::City
+#
+class FilterGeoIP2City
+  include BaseDecoder
+  include GeoIP2Library
+
+  GEOIP2_LANGUAGE = ENV["GEOIP2_LANGUAGE"] || "en"
+  LOCALE_SPECIFIC_NAMES = %w(city.names continent.names country.names registered_country.names represented_country.names)
+  DESIRED_GEOIP2_KEYS = %w(
+    city.geoname_id
+    continent.code continent.geoname_id
+    country.geoname_id country.iso_code country.is_in_european_union
+    location.accuracy_radius location.latitude location.longitude location.metro_code location.time_zone
+    postal.code
+    registered_country.geoname_id registered_country.iso_code registered_country.is_in_european_union
+    represented_country.geoname_id represented_country.iso_code represented_country.is_in_european_union represented_country.type
+    traits.is_anonymous_proxy traits.is_satellite_provider
+  )
+
+  attr_reader :locale_specific_names
+  def initialize(args={})
+    @locale_specific_names = LOCALE_SPECIFIC_NAMES.map { |lsn| "#{lsn}.#{GEOIP2_LANGUAGE}" }
+    super
+  end
+
+  def decode(ip)
+    unless @@geo_city
+      raise "No MaxMind GeoIP2::City data found"
+    end
+    return unless (geo_hash = @@geo_city.get(ip))
+    ret = defaults
+
+    if geo_hash.include?("subdivisions")
+      # handle countries that are divided into various subdivisions.  generally 1, sometimes 2
+      subdivisions = geo_hash["subdivisions"]
+      geo_hash.delete("subdivisions")
+      ret["geoip2.city.subdivisions.length"] = subdivisions.size.to_s
+      subdivisions.each_index do |i|
+        subdivision = subdivisions[i]
+        subdivision.each_pair do |k,v|
+          if %w(geoname_id iso_code).include?(k)
+            ret["geoip2.city.subdivisions.#{i}.#{k}"] = v.to_s
+          elsif k == "names"
+            if v.include?(GEOIP2_LANGUAGE)
+              ret["geoip2.city.subdivisions.#{i}.name"] = subdivision["names"][GEOIP2_LANGUAGE]
+            end
+          end
+        end
+      end
+    end
+
+    Dap::Utils::Misc.flatten_hash(geo_hash).each_pair do |k,v|
+      if DESIRED_GEOIP2_KEYS.include?(k)
+        # these keys we can just copy directly over
+        ret["geoip2.city.#{k}"] = v
+      elsif @locale_specific_names.include?(k)
+        # these keys we need to pick the locale-specific name and set the key accordingly
+        lsn_renamed = k.gsub(/\.names.#{GEOIP2_LANGUAGE}/, ".name")
+        ret["geoip2.city.#{lsn_renamed}"] = v
+      end
+    end
+    ret
+  end
+
+  def defaults()
+    ret = {}
+    default_int_suffixes = %w(geoname_id metro_code)
+    default_bool_suffixes = %w(is_in_european_union is_anonymous_proxy is_satellite_provider)
+    DESIRED_GEOIP2_KEYS.each do |k|
+      suffix = k.split(/\./)[-1]
+      if default_int_suffixes.include?(suffix)
+        ret["geoip2.city.#{k}"] = "0"
+      elsif default_bool_suffixes.include?(suffix)
+        ret["geoip2.city.#{k}"] = "false"
+      else
+        ret["geoip2.city.#{k}"] = ""
+      end
+    end
+    ret
+  end
+end
+
+#
+# Add GeoIP2 ASN and Org tags using the MaxMind GeoIP2::ASN database
+#
+class FilterGeoIP2Asn
+  include BaseDecoder
+  include GeoIP2Library
+
+  def decode(ip)
+    unless @@geo_asn
+      raise "No MaxMind GeoIP2::ASN data found"
+    end
+    geo_hash = @@geo_asn.get(ip)
+    return unless geo_hash
+
+    ret = {}
+
+    if geo_hash.include?("autonomous_system_number")
+      ret["geoip2.asn.asn"] = "AS#{geo_hash["autonomous_system_number"]}"
+    else
+      ret["geoip2.asn.asn"] = ""
+    end
+
+    if geo_hash.include?("autonomous_system_organization")
+      ret["geoip2.asn.asn_org"] = "#{geo_hash["autonomous_system_organization"]}"
+    else
+      ret["geoip2.asn.asn_org"] = ""
+    end
+
+    ret
+  end
+end
+
+#
+# Add GeoIP2 ISP tags using the MaxMind GeoIP2::ISP database
+#
+class FilterGeoIP2Isp
+  include BaseDecoder
+  include GeoIP2Library
+  def decode(ip)
+    unless @@geo_isp
+      raise "No MaxMind GeoIP2::ISP data found"
+    end
+    geo_hash = @@geo_isp.get(ip)
+    return unless geo_hash
+
+    ret = {}
+
+    if geo_hash.include?("autonomous_system_number")
+      ret["geoip2.isp.asn"] = "AS#{geo_hash["autonomous_system_number"]}"
+    else
+      ret["geoip2.isp.asn"] = ""
+    end
+
+    if geo_hash.include?("autonomous_system_organization")
+      ret["geoip2.isp.asn_org"] = geo_hash["autonomous_system_organization"]
+    else
+      ret["geoip2.isp.asn_org"] = ""
+    end
+
+    if geo_hash.include?("isp")
+      ret["geoip2.isp.isp"] = geo_hash["isp"]
+    else
+      ret["geoip2.isp.isp"] = ""
+    end
+
+    if geo_hash.include?("organization")
+      ret["geoip2.isp.org"] = geo_hash["organization"]
+    else
+      ret["geoip2.isp.org"] = ""
+    end
+
+    ret
+  end
+end
+
+#
+# Convert GeoIP2 data as closely as possible to the legacy GeoIP data as generated by geo_ip, geo_ip_asn and geo_ip_org
+#
+class FilterGeoIP2LegacyCompat
+  include Base
+
+  attr_accessor :base_field
+
+  def initialize(args)
+    super
+    fail "Expected 1 arguments to '#{self.name}' but got #{args.size}" unless args.size == 1
+    self.base_field = args.first
+  end
+
+  def process(doc)
+    # all of these values we just take directly and rename
+    remap = {
+      # geoip2 name -> geoip name
+      "city.country.iso_code": "country_code",
+      "city.country.name": "country.name",
+      "city.postal.code": "postal_code",
+      "city.location.latitude": "latitude",
+      "city.location.longitude": "longitude",
+      "city.city.name": "city",
+      "city.subdivisions.0.iso_code": "region",
+      "city.subdivisions.0.name": "region_name",
+      "asn.asn": "asn",
+      "isp.asn": "asn",
+    }
+
+    remap.each_pair do |geoip2,geoip|
+      geoip2_key = "#{self.base_field}.geoip2.#{geoip2}"
+      if doc.include?(geoip2_key)
+        doc["#{self.base_field}.#{geoip}"] = doc[geoip2_key]
+      end
+    end
+
+    # these values all require special handling
+
+    # https://dev.maxmind.com/geoip/geoip2/whats-new-in-geoip2/#Custom_Country_Codes
+    # which basically says if traits.is_anonymous_proxy is true, previously the
+    # country_code would have had a special value of A1.  Similarly, if
+    # traits.is_satellite_provider is true, previously the country_code would
+    # have a special value of A2.
+    anon_key = "#{self.base_field}.geoip2.city.traits.is_anonymous_proxy"
+    if doc.include?(anon_key)
+      anon_value = doc[anon_key]
+      if anon_value == "true"
+        doc["#{self.base_field}.country_code"] = "A1"
+      end
+    end
+
+    satellite_key = "#{self.base_field}.geoip2.city.traits.is_satellite_provider"
+    if doc.include?(satellite_key)
+      satellite_value = doc[satellite_key]
+      if satellite_value == "true"
+        doc["#{self.base_field}.country_code"] = "A1"
+      end
+    end
+
+    # only set dma_code if location.metro_code was set and not empty or 0
+    metro_key = "#{self.base_field}.geoip2.city.location.metro_code}"
+    if doc.include?(metro_key)
+      metro_value = doc[metro_key]
+      if !metro_value.empty? && metro_value != "0"
+        doc["#{self.base_field}.dma_code"] = metro_value
+      end
+    end
+
+    # get the org key from 3 possible fields in decreasing order of preference
+    asn_org_key = "#{self.base_field}.geoip2.asn.asn_org"
+    isp_asn_org_key = "#{self.base_field}.geoip2.isp.asn_org"
+    isp_org_key = "#{self.base_field}.geoip2.isp.asn_org"
+    [ isp_org_key, isp_asn_org_key, asn_org_key ].each do |k|
+      v = doc[k]
+      if v && !v.empty?
+        doc["#{self.base_field}.org"] = v
+        break
+      end
+    end
+
+    [ doc ]
+  end
+end
+
+end
+end

data/lib/dap/utils/misc.rb

@@ -0,0 +1,22 @@
+module Dap
+module Utils
+module Misc
+
+  def self.flatten_hash(h)
+    ret = {}
+    h.each_pair do |k,v|
+      next unless k
+      if v.is_a?(Hash)
+        flatten_hash(v).each_pair do |fk,fv|
+          ret["#{k}.#{fk}"] = fv.to_s
+        end
+      else
+        ret[k.to_s] = v.to_s
+      end
+    end
+    ret
+  end
+
+end
+end
+end

data/lib/dap/version.rb

@@ -1,3 +1,3 @@
 module Dap
-  VERSION = "1.0.2"
+  VERSION = "1.2.0"
 end

data/spec/dap/utils/misc_spec.rb

@@ -0,0 +1,12 @@
+describe Dap::Utils::Misc do
+  describe '.flatten_hash' do
+    context 'with mixed nested data' do
+      let(:test_hash) { {"foo0": "bar0", "foo1": {"bar1": "stuff", "more": 1}, "foo2": {"bar2": "stuff", "more": 1, "morestuff": {"foo1": "thing1"}}} }
+      let(:expected_flat) { {'foo0'=>'bar0', 'foo1.bar1'=>'stuff', 'foo1.more'=>'1', 'foo2.bar2'=>'stuff', 'foo2.more'=>'1', 'foo2.morestuff.foo1'=>'thing1'} }
+      let(:actual_flat) { Dap::Utils::Misc.flatten_hash(test_hash) }
+      it 'flattens properly' do
+        expect(actual_flat).to eq(expected_flat)
+      end
+    end
+  end
+end

data/test/filters.bats

@@ -109,11 +109,9 @@ load ./test_common
 }
 
 @test "recog_match" {
-  # currently differs from godap, need to figure out which is correct
-  skip
   run bash -c "echo '9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2' | $DAP_EXECUTABLE lines + recog line=dns.versionbind + json | jq -Sc ."
   assert_success
-  assert_output '{"line":"9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2","line.recog.os.cpe23":"cpe:/o:redhat:enterprise_linux:6","line.recog.os.family":"Linux","line.recog.os.product":"Enterprise Linux","line.recog.os.vendor":"Red Hat","line.recog.os.version":"6","line.recog.os.version.version":"9","line.recog.service.cpe23":"cpe:/a:isc:bind:9.8.2rc1","line.recog.service.family":"BIND","line.recog.service.product":"BIND","line.recog.service.vendor":"ISC","line.recog.service.version":"9.8.2rc1"}'
+  assert_output '{"line":"9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2","line.recog.fingerprint_db":"dns.versionbind","line.recog.matched":"ISC BIND: Red Hat Enterprise Linux","line.recog.os.cpe23":"cpe:/o:redhat:enterprise_linux:6","line.recog.os.family":"Linux","line.recog.os.product":"Enterprise Linux","line.recog.os.vendor":"Red Hat","line.recog.os.version":"6","line.recog.os.version.version":"9","line.recog.service.cpe23":"cpe:/a:isc:bind:9.8.2rc1","line.recog.service.family":"BIND","line.recog.service.product":"BIND","line.recog.service.protocol":"dns","line.recog.service.vendor":"ISC","line.recog.service.version":"9.8.2rc1"}'
 }
 
 @test "recog_nomatch" {
@@ -128,3 +126,69 @@ load ./test_common
   run bash -c "echo 'test' | $DAP_EXECUTABLE lines + recog + json"
   assert_failure
 }
+
+@test "geo_ip yields valid fields" {
+  run bash -c "echo 66.92.181.240 | GEOIP_CITY_DATABASE_PATH=./test/test_data/geoip/GeoIPCity.dat $DAP_EXECUTABLE lines + geo_ip line + json | jq -Sc ."
+  assert_success
+  assert_output '{"line":"66.92.181.240","line.area_code":"510","line.city":"Fremont","line.country_code":"US","line.country_code3":"USA","line.country_name":"United States","line.dma_code":"807","line.latitude":"37.50790023803711","line.longitude":"-121.95999908447266","line.postal_code":"94538","line.region":"CA","line.region_name":"California"}'
+}
+
+@test "geo_ip_org yields valid fields" {
+  run bash -c "echo 12.87.118.0 | GEOIP_ORG_DATABASE_PATH=./test/test_data/geoip/GeoIPOrg.dat $DAP_EXECUTABLE lines + geo_ip_org line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"12.87.118.0","line.org":"AT&T Worldnet Services"}'
+}
+
+@test "geo_ip_asn" {
+  run bash -c "echo 12.87.118.0 | GEOIP_ASN_DATABASE_PATH=./test/test_data/geoip/GeoIPASNum.dat $DAP_EXECUTABLE lines + geo_ip_asn line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"12.87.118.0","line.asn":"AS7018"}'
+}
+
+@test "geo_ip2_city" {
+  # test with default language
+  run bash -c "echo 81.2.69.142 | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"81.2.69.142","line.geoip2.city.city.geoname_id":"2643743","line.geoip2.city.city.name":"London","line.geoip2.city.continent.code":"EU","line.geoip2.city.continent.geoname_id":"6255148","line.geoip2.city.continent.name":"Europe","line.geoip2.city.country.geoname_id":"2635167","line.geoip2.city.country.is_in_european_union":"true","line.geoip2.city.country.iso_code":"GB","line.geoip2.city.country.name":"United Kingdom","line.geoip2.city.location.accuracy_radius":"10","line.geoip2.city.location.latitude":"51.5142","line.geoip2.city.location.longitude":"-0.0931","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/London","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"6252001","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"US","line.geoip2.city.registered_country.name":"United States","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.subdivisions.0.geoname_id":"6269131","line.geoip2.city.subdivisions.0.iso_code":"ENG","line.geoip2.city.subdivisions.0.name":"England","line.geoip2.city.subdivisions.length":"1","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
+
+  # test with non-default language
+  run bash -c "echo 67.43.156.0 | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb GEOIP2_LANGUAGE=fr $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"67.43.156.0","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asie","line.geoip2.city.country.geoname_id":"1252634","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"BT","line.geoip2.city.country.name":"Bhutan","line.geoip2.city.location.accuracy_radius":"534","line.geoip2.city.location.latitude":"27.5","line.geoip2.city.location.longitude":"90.5","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Asia/Thimphu","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"798549","line.geoip2.city.registered_country.is_in_european_union":"true","line.geoip2.city.registered_country.iso_code":"RO","line.geoip2.city.registered_country.name":"Roumanie","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.traits.is_anonymous_proxy":"true","line.geoip2.city.traits.is_satellite_provider":"false"}'
+
+  # test IPv6
+  run bash -c "echo 2a02:d9c0:: | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"2a02:d9c0::","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asia","line.geoip2.city.country.geoname_id":"298795","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"TR","line.geoip2.city.country.name":"Turkey","line.geoip2.city.location.accuracy_radius":"100","line.geoip2.city.location.latitude":"39.05901","line.geoip2.city.location.longitude":"34.91155","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/Istanbul","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"298795","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"TR","line.geoip2.city.registered_country.name":"Turkey","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
+}
+
+@test "geo_ip2_asn" {
+  run bash -c "echo 12.81.92.0 | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_asn line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"12.81.92.0","line.geoip2.asn.asn":"AS7018","line.geoip2.asn.asn_org":"AT&T Services"}'
+
+  # test IPv6
+  run bash -c "echo 2600:7000:: | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_asn line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"2600:7000::","line.geoip2.asn.asn":"AS6939","line.geoip2.asn.asn_org":"Hurricane Electric, Inc."}'
+}
+
+@test "geo_ip2_isp" {
+  run bash -c "echo -e '12.81.92.0\n2600:7000::' | GEOIP2_ISP_DATABASE_PATH=test/test_data/geoip2/GeoIP2-ISP-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_isp line + json | jq -Sc -r ."
+  assert_line --index 0 '{"line":"12.81.92.0","line.geoip2.isp.asn":"AS7018","line.geoip2.isp.asn_org":"","line.geoip2.isp.isp":"AT&T Services","line.geoip2.isp.org":"AT&T Services"}'
+  # test IPv6
+  assert_line --index 1 '{"line":"2600:7000::","line.geoip2.isp.asn":"AS6939","line.geoip2.isp.asn_org":"Hurricane Electric, Inc.","line.geoip2.isp.isp":"","line.geoip2.isp.org":""}'
+}
+
+@test "geo_ip2_legacy_compat" {
+  run bash -c "echo -e '81.2.69.142\n12.81.92.0\n2a02:d9c0::\n2a01:1000::' | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb GEOIP2_ISP_DATABASE_PATH=test/test_data/geoip2/GeoIP2-ISP-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + geo_ip2_asn line + geo_ip2_isp line + geo_ip2_legacy_compat line + match_remove line.geoip2 + json | jq -Sc -r ."
+  assert_success
+  # this one only has city data, not ASN/org/ISP
+  assert_line --index 0 '{"line":"81.2.69.142","line.city":"London","line.country.name":"United Kingdom","line.country_code":"GB","line.latitude":"51.5142","line.longitude":"-0.0931","line.postal_code":"","line.region":"ENG","line.region_name":"England"}'
+  # this one has ASN/org data in the test databases but none in the city DB
+  assert_line --index 1 '{"line":"12.81.92.0","line.asn":"AS7018","line.org":"AT&T Services"}'
+  # exists only city
+  assert_line --index 2 '{"line":"2a02:d9c0::","line.country.name":"Turkey","line.country_code":"TR","line.latitude":"39.05901","line.longitude":"34.91155","line.postal_code":""}'
+  # exists in ISP
+  assert_line --index 3 '{"line":"2a01:1000::","line.asn":"AS5617","line.org":"Telekomunikacja Polska S.A."}'
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dap
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Rapid7 Research
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-23 00:00:00.000000000 Z
+date: 2019-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -183,6 +183,7 @@ files:
 - lib/dap/filter.rb
 - lib/dap/filter/base.rb
 - lib/dap/filter/geoip.rb
+- lib/dap/filter/geoip2.rb
 - lib/dap/filter/gquic.rb
 - lib/dap/filter/http.rb
 - lib/dap/filter/ldap.rb
@@ -205,6 +206,7 @@ files:
 - lib/dap/proto/mssql.rb
 - lib/dap/proto/natpmp.rb
 - lib/dap/proto/wdbrpc.rb
+- lib/dap/utils/misc.rb
 - lib/dap/utils/oui.rb
 - lib/dap/version.rb
 - samples/http_get_reply.ic12.bz2
@@ -231,10 +233,17 @@ files:
 - spec/dap/input/json_spec.rb
 - spec/dap/proto/ipmi_spec.rb
 - spec/dap/proto/ldap_proto_spec.rb
+- spec/dap/utils/misc_spec.rb
 - spec/spec_helper.rb
 - test/filters.bats
 - test/inputs.bats
 - test/test_common.bash
+- test/test_data/geoip/GeoIPASNum.dat
+- test/test_data/geoip/GeoIPCity.dat
+- test/test_data/geoip/GeoIPOrg.dat
+- test/test_data/geoip2/GeoIP2-City-Test.mmdb
+- test/test_data/geoip2/GeoIP2-ISP-Test.mmdb
+- test/test_data/geoip2/GeoLite2-ASN-Test.mmdb
 - tools/geo-ip-summary.rb
 - tools/ipmi-vulns.rb
 - tools/json-summarize.rb
@@ -273,7 +282,14 @@ test_files:
 - spec/dap/input/json_spec.rb
 - spec/dap/proto/ipmi_spec.rb
 - spec/dap/proto/ldap_proto_spec.rb
+- spec/dap/utils/misc_spec.rb
 - spec/spec_helper.rb
 - test/filters.bats
 - test/inputs.bats
 - test/test_common.bash
+- test/test_data/geoip/GeoIPASNum.dat
+- test/test_data/geoip/GeoIPCity.dat
+- test/test_data/geoip/GeoIPOrg.dat
+- test/test_data/geoip2/GeoIP2-City-Test.mmdb
+- test/test_data/geoip2/GeoIP2-ISP-Test.mmdb
+- test/test_data/geoip2/GeoLite2-ASN-Test.mmdb