dap 1.2.3 → 1.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: caec5115c33ffe3301d3522090ef651e8ef80cba
-  data.tar.gz: f78403df2edc26901708dae7cc326289696d34dd
+SHA256:
+  metadata.gz: 2e0ed16a920babb2efdaa5afc286a117525da9dd9c0603b1c5c3af46226cd808
+  data.tar.gz: 59207589322edf4886493847efcc449ef4eb6fccfebf7c3f0ea96fd940e958f5
 SHA512:
-  metadata.gz: 06f2a048acf014e685c7f7b869ed660253d5d0fa51f3436562e569927594c10820346134197706f027d58747417d5a111c5f0715f3244f7c67d9fada22b25bb0
-  data.tar.gz: e0ebcd198d2477bcec6335243906a2ca70437b8cc3aaa51af9703414ba3c3aff0d57f8044cf1812c6920457a8c6fae88073642bbc028f9415662d2372e28fbe6
+  metadata.gz: e46e7fcaf7c884f78ef7698a860984d623dfabf61f58986830bedcc7b06d1df138a943859269b2c8fd5a9ed8e2e492fca0ef57da829363a0db5b2d927310a9fd
+  data.tar.gz: 67c341369782f84f542b82babf35a176c8f866de0d59ea6539c1332eedcce1b6df4180bf66beede6b11c6c45070440635a64c40f80bf2f503212b5c897e0328c

data/.gitignore

@@ -4,3 +4,13 @@
 
 # Ignore geoip data file
 data/geoip.dat
+
+/pkg/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# Misc
+**/.DS_Store

data/Dockerfile

@@ -1,5 +1,6 @@
 FROM ubuntu:latest
 
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
 RUN apt-get install -y \
   ca-certificates \

data/Dockerfile.testing

@@ -1,6 +1,8 @@
 FROM ubuntu:18.04
 
 ENV TEST_DIR /opt/bats_testing
+
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
 RUN apt-get install -y build-essential ca-certificates curl git jq libffi-dev libgeoip-dev libxml2-dev wget zlib1g-dev
 

data/Gemfile

@@ -6,10 +6,10 @@ gem 'net-dns'
 gem 'bit-struct'
 gem 'geoip-c'
 gem 'maxmind-db', '~> 1.0.0'
-gem 'recog', '>=2.3.0'
+gem 'recog', '>=2.3.8'
 
 group :test do
-  gem 'rspec', '~> 3.1.0'
+  gem 'rspec', '~> 3.9.0'
   gem 'cucumber', '~> 1.3.16'
   gem 'aruba', '~> 0.6.1'
 end

data/Gemfile.lock

@@ -5,44 +5,43 @@ GEM
       childprocess (>= 0.3.6)
       cucumber (>= 1.1.1)
       rspec-expectations (>= 2.7.0)
-    bit-struct (0.16.0)
-    builder (3.2.2)
-    childprocess (0.5.8)
-      ffi (~> 1.0, >= 1.0.11)
+    bit-struct (0.16)
+    builder (3.2.4)
+    childprocess (3.0.0)
     cucumber (1.3.20)
       builder (>= 2.1.2)
       diff-lcs (>= 1.1.3)
       gherkin (~> 2.12)
       multi_json (>= 1.7.5, < 2.0)
       multi_test (>= 0.1.2)
-    diff-lcs (1.2.5)
-    ffi (1.9.10)
+    diff-lcs (1.3)
     geoip-c (0.9.1)
     gherkin (2.12.2)
       multi_json (~> 1.3)
     htmlentities (4.3.4)
     maxmind-db (1.0.0)
     mini_portile2 (2.4.0)
-    multi_json (1.11.2)
+    multi_json (1.14.1)
     multi_test (0.1.2)
     net-dns (0.9.0)
-    nokogiri (1.10.3)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
-    oj (3.7.9)
-    recog (2.3.0)
+    oj (3.10.6)
+    recog (2.3.8)
       nokogiri
-    rspec (3.1.0)
-      rspec-core (~> 3.1.0)
-      rspec-expectations (~> 3.1.0)
-      rspec-mocks (~> 3.1.0)
-    rspec-core (3.1.7)
-      rspec-support (~> 3.1.0)
-    rspec-expectations (3.1.2)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.1.0)
-    rspec-mocks (3.1.3)
-      rspec-support (~> 3.1.0)
-    rspec-support (3.1.2)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
 
 PLATFORMS
   ruby
@@ -56,8 +55,8 @@ DEPENDENCIES
   maxmind-db (~> 1.0.0)
   net-dns
   oj
-  recog (>= 2.3.0)
-  rspec (~> 3.1.0)
+  recog (>= 2.3.8)
+  rspec (~> 3.9.0)
 
 BUNDLED WITH
-   2.0.1
+   2.0.2

data/README.md

@@ -9,8 +9,6 @@ DAP reads data using an input plugin, transforms it through a series of filters,
 
 DAP was written to process terabyte-sized public scan datasets, such as those provided by https://scans.io/. Although DAP isn't particularly fast, it can be used across multiple cores (and machines) by splitting the input source and wrapping the execution with GNU Parallel.
 
-
-
 ## Installation
 
 ### Prerequisites
@@ -61,7 +59,10 @@ gem install dap
 ### OS X
 
 ```bash
+# Install the GeoIP C library required by DAP
 brew update
+brew install geoip
+
 gem install dap
 ```
 
@@ -69,7 +70,7 @@ gem install dap
 
 In its simplest form, DAP takes input, applies zero or more filters which modify the input, and then outputs the result.  The input, filters and output are separated by plus signs (`+`).  As seen from `dap -h`:
 
-```
+```shell
 Usage: dap  [input] + [filter] + [output]
        --inputs
        --outputs
@@ -80,7 +81,7 @@ To see which input/output formats are supported and what filters are available,
 
 This example reads as input a single IP address from `STDIN` in line form, applies geo-ip transformations as a filter on that line, and then returns the output as JSON:
 
-```
+```shell
 $   echo 8.8.8.8 | bin/dap + lines + geo_ip2_city line + json | jq .
 {
   "line": "8.8.8.8",

data/Rakefile

@@ -19,4 +19,4 @@ Cucumber::Rake::Task.new(:features) do |t|
 end
 
 task :default => [ :spec, :features, :yard ]
-
+task :tests => [ :spec ]

data/lib/dap/filter/geoip2.rb

@@ -33,6 +33,22 @@ module GeoIP2Library
     nil
   end
 
+  def get_maxmind_data(db, ip)
+    begin
+      db.get(ip)
+    rescue IPAddr::InvalidAddressError
+    end
+  end
+
+  def remove_empties(hash)
+    hash.each_pair do |k,v|
+      if v.empty?
+        hash.delete(k)
+      end
+    end
+    hash
+  end
+
   @@geo_asn = find_db(GEOIP2_ASN, GEOIP2_DIRS, ENV["GEOIP2_ASN_DATABASE_PATH"])
   @@geo_city = find_db(GEOIP2_CITY, GEOIP2_DIRS, ENV["GEOIP2_CITY_DATABASE_PATH"])
   @@geo_isp = find_db(GEOIP2_ISP, GEOIP2_DIRS, ENV["GEOIP2_ISP_DATABASE_PATH"])
@@ -69,8 +85,10 @@ class FilterGeoIP2City
     unless @@geo_city
       raise "No MaxMind GeoIP2::City data found"
     end
-    return unless (geo_hash = @@geo_city.get(ip))
+
     ret = defaults
+    geo_hash = get_maxmind_data(@@geo_city, ip)
+    return unless geo_hash
 
     if geo_hash.include?("subdivisions")
       # handle countries that are divided into various subdivisions.  generally 1, sometimes 2
@@ -101,7 +119,8 @@ class FilterGeoIP2City
         ret["geoip2.city.#{lsn_renamed}"] = v
       end
     end
-    ret
+
+    remove_empties(ret)
   end
 
   def defaults()
@@ -133,9 +152,9 @@ class FilterGeoIP2Asn
     unless @@geo_asn
       raise "No MaxMind GeoIP2::ASN data found"
     end
-    geo_hash = @@geo_asn.get(ip)
-    return unless geo_hash
 
+    geo_hash = get_maxmind_data(@@geo_asn, ip)
+    return unless geo_hash
     ret = {}
 
     if geo_hash.include?("autonomous_system_number")
@@ -150,7 +169,7 @@ class FilterGeoIP2Asn
       ret["geoip2.asn.asn_org"] = ""
     end
 
-    ret
+    remove_empties(ret)
   end
 end
 
@@ -164,9 +183,9 @@ class FilterGeoIP2Isp
     unless @@geo_isp
       raise "No MaxMind GeoIP2::ISP data found"
     end
-    geo_hash = @@geo_isp.get(ip)
-    return unless geo_hash
 
+    geo_hash = get_maxmind_data(@@geo_isp, ip)
+    return unless geo_hash
     ret = {}
 
     if geo_hash.include?("autonomous_system_number")
@@ -193,7 +212,7 @@ class FilterGeoIP2Isp
       ret["geoip2.isp.org"] = ""
     end
 
-    ret
+    remove_empties(ret)
   end
 end
 
@@ -202,6 +221,7 @@ end
 #
 class FilterGeoIP2LegacyCompat
   include Base
+  include GeoIP2Library
 
   attr_accessor :base_field
 
@@ -227,10 +247,11 @@ class FilterGeoIP2LegacyCompat
       "isp.asn": "asn",
     }
 
+    ret = {}
     remap.each_pair do |geoip2,geoip|
       geoip2_key = "#{self.base_field}.geoip2.#{geoip2}"
       if doc.include?(geoip2_key)
-        doc["#{self.base_field}.#{geoip}"] = doc[geoip2_key]
+        ret["#{self.base_field}.#{geoip}"] = doc[geoip2_key]
       end
     end
 
@@ -245,7 +266,7 @@ class FilterGeoIP2LegacyCompat
     if doc.include?(anon_key)
       anon_value = doc[anon_key]
       if anon_value == "true"
-        doc["#{self.base_field}.country_code"] = "A1"
+        ret["#{self.base_field}.country_code"] = "A1"
       end
     end
 
@@ -253,7 +274,7 @@ class FilterGeoIP2LegacyCompat
     if doc.include?(satellite_key)
       satellite_value = doc[satellite_key]
       if satellite_value == "true"
-        doc["#{self.base_field}.country_code"] = "A1"
+        ret["#{self.base_field}.country_code"] = "A1"
       end
     end
 
@@ -262,7 +283,7 @@ class FilterGeoIP2LegacyCompat
     if doc.include?(metro_key)
       metro_value = doc[metro_key]
       if !metro_value.empty? && metro_value != "0"
-        doc["#{self.base_field}.dma_code"] = metro_value
+        ret["#{self.base_field}.dma_code"] = metro_value
       end
     end
 
@@ -273,12 +294,12 @@ class FilterGeoIP2LegacyCompat
     [ isp_org_key, isp_asn_org_key, asn_org_key ].each do |k|
       v = doc[k]
       if v && !v.empty?
-        doc["#{self.base_field}.org"] = v
+        ret["#{self.base_field}.org"] = v
         break
       end
     end
 
-    [ doc ]
+    [ doc.merge(remove_empties(ret)) ]
   end
 end
 

data/lib/dap/filter/http.rb

@@ -15,7 +15,7 @@ module HTMLGhetto
     data.
       to_s.
       encode('UTF-8', invalid: :replace, undef: :replace, replace: '').
-      scan(/<([^>]+)>/m).each do |e|
+      scan(/<([^<>]{1,4096})>/m).each do |e|
 
       e = e.first
 

data/lib/dap/version.rb

@@ -1,3 +1,3 @@
 module Dap
-  VERSION = "1.2.3"
+  VERSION = "1.2.8"
 end

data/spec/dap/filter/http_filter_spec.rb

@@ -151,5 +151,70 @@ describe Dap::Filter::FilterHTMLLinks do
         expect(processed.map { |p| p['link'] }).to eq(%w(a b))
       end
     end
+
+    context 'repeated less than symbol' do
+      body =
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<a href="a"/>'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
+        '<a href="b"/>'
+      let(:processed) { filter.process({'data' => body}) }
+      it 'extracted the correct links' do
+        expect(processed.map { |p| p['link'] }).to eq(%w(a b))
+      end
+    end
   end
 end

data/test/filters.bats

@@ -149,17 +149,22 @@ load ./test_common
   # test with default language
   run bash -c "echo 81.2.69.142 | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
   assert_success
-  assert_output '{"line":"81.2.69.142","line.geoip2.city.city.geoname_id":"2643743","line.geoip2.city.city.name":"London","line.geoip2.city.continent.code":"EU","line.geoip2.city.continent.geoname_id":"6255148","line.geoip2.city.continent.name":"Europe","line.geoip2.city.country.geoname_id":"2635167","line.geoip2.city.country.is_in_european_union":"true","line.geoip2.city.country.iso_code":"GB","line.geoip2.city.country.name":"United Kingdom","line.geoip2.city.location.accuracy_radius":"10","line.geoip2.city.location.latitude":"51.5142","line.geoip2.city.location.longitude":"-0.0931","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/London","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"6252001","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"US","line.geoip2.city.registered_country.name":"United States","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.subdivisions.0.geoname_id":"6269131","line.geoip2.city.subdivisions.0.iso_code":"ENG","line.geoip2.city.subdivisions.0.name":"England","line.geoip2.city.subdivisions.length":"1","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
+  assert_output '{"line":"81.2.69.142","line.geoip2.city.city.geoname_id":"2643743","line.geoip2.city.city.name":"London","line.geoip2.city.continent.code":"EU","line.geoip2.city.continent.geoname_id":"6255148","line.geoip2.city.continent.name":"Europe","line.geoip2.city.country.geoname_id":"2635167","line.geoip2.city.country.is_in_european_union":"true","line.geoip2.city.country.iso_code":"GB","line.geoip2.city.country.name":"United Kingdom","line.geoip2.city.location.accuracy_radius":"10","line.geoip2.city.location.latitude":"51.5142","line.geoip2.city.location.longitude":"-0.0931","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/London","line.geoip2.city.registered_country.geoname_id":"6252001","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"US","line.geoip2.city.registered_country.name":"United States","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.subdivisions.0.geoname_id":"6269131","line.geoip2.city.subdivisions.0.iso_code":"ENG","line.geoip2.city.subdivisions.0.name":"England","line.geoip2.city.subdivisions.length":"1","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
 
   # test with non-default language
   run bash -c "echo 67.43.156.0 | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb GEOIP2_LANGUAGE=fr $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
   assert_success
-  assert_output '{"line":"67.43.156.0","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asie","line.geoip2.city.country.geoname_id":"1252634","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"BT","line.geoip2.city.country.name":"Bhutan","line.geoip2.city.location.accuracy_radius":"534","line.geoip2.city.location.latitude":"27.5","line.geoip2.city.location.longitude":"90.5","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Asia/Thimphu","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"798549","line.geoip2.city.registered_country.is_in_european_union":"true","line.geoip2.city.registered_country.iso_code":"RO","line.geoip2.city.registered_country.name":"Roumanie","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.traits.is_anonymous_proxy":"true","line.geoip2.city.traits.is_satellite_provider":"false"}'
+  assert_output '{"line":"67.43.156.0","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asie","line.geoip2.city.country.geoname_id":"1252634","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"BT","line.geoip2.city.country.name":"Bhutan","line.geoip2.city.location.accuracy_radius":"534","line.geoip2.city.location.latitude":"27.5","line.geoip2.city.location.longitude":"90.5","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Asia/Thimphu","line.geoip2.city.registered_country.geoname_id":"798549","line.geoip2.city.registered_country.is_in_european_union":"true","line.geoip2.city.registered_country.iso_code":"RO","line.geoip2.city.registered_country.name":"Roumanie","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.traits.is_anonymous_proxy":"true","line.geoip2.city.traits.is_satellite_provider":"false"}'
 
   # test IPv6
   run bash -c "echo 2a02:d9c0:: | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
   assert_success
-  assert_output '{"line":"2a02:d9c0::","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asia","line.geoip2.city.country.geoname_id":"298795","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"TR","line.geoip2.city.country.name":"Turkey","line.geoip2.city.location.accuracy_radius":"100","line.geoip2.city.location.latitude":"39.05901","line.geoip2.city.location.longitude":"34.91155","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/Istanbul","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"298795","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"TR","line.geoip2.city.registered_country.name":"Turkey","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
+  assert_output '{"line":"2a02:d9c0::","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asia","line.geoip2.city.country.geoname_id":"298795","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"TR","line.geoip2.city.country.name":"Turkey","line.geoip2.city.location.accuracy_radius":"100","line.geoip2.city.location.latitude":"39.05901","line.geoip2.city.location.longitude":"34.91155","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/Istanbul","line.geoip2.city.registered_country.geoname_id":"298795","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"TR","line.geoip2.city.registered_country.name":"Turkey","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
+
+  # test invalid IP
+  run bash -c "echo test | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"test"}'
 }
 
 @test "geo_ip2_asn" {
@@ -171,24 +176,35 @@ load ./test_common
   run bash -c "echo 2600:7000:: | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_asn line + json | jq -Sc -r ."
   assert_success
   assert_output '{"line":"2600:7000::","line.geoip2.asn.asn":"AS6939","line.geoip2.asn.asn_org":"Hurricane Electric, Inc."}'
+
+  # test invalid IP
+  run bash -c "echo test | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_asn line + json | jq -Sc -r ."
+  assert_success
+  assert_output '{"line":"test"}'
 }
 
 @test "geo_ip2_isp" {
-  run bash -c "echo -e '12.81.92.0\n2600:7000::' | GEOIP2_ISP_DATABASE_PATH=test/test_data/geoip2/GeoIP2-ISP-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_isp line + json | jq -Sc -r ."
-  assert_line --index 0 '{"line":"12.81.92.0","line.geoip2.isp.asn":"AS7018","line.geoip2.isp.asn_org":"","line.geoip2.isp.isp":"AT&T Services","line.geoip2.isp.org":"AT&T Services"}'
+  run bash -c "echo -e '12.81.92.0\n2600:7000::\ntest' | GEOIP2_ISP_DATABASE_PATH=test/test_data/geoip2/GeoIP2-ISP-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_isp line + json | jq -Sc -r ."
+  assert_line --index 0 '{"line":"12.81.92.0","line.geoip2.isp.asn":"AS7018","line.geoip2.isp.isp":"AT&T Services","line.geoip2.isp.org":"AT&T Services"}'
   # test IPv6
-  assert_line --index 1 '{"line":"2600:7000::","line.geoip2.isp.asn":"AS6939","line.geoip2.isp.asn_org":"Hurricane Electric, Inc.","line.geoip2.isp.isp":"","line.geoip2.isp.org":""}'
+  assert_line --index 1 '{"line":"2600:7000::","line.geoip2.isp.asn":"AS6939","line.geoip2.isp.asn_org":"Hurricane Electric, Inc."}'
+  # test invalid IP
+  assert_line --index 2 '{"line":"test"}'
 }
 
 @test "geo_ip2_legacy_compat" {
   run bash -c "echo -e '81.2.69.142\n12.81.92.0\n2a02:d9c0::\n2a01:1000::' | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb GEOIP2_ISP_DATABASE_PATH=test/test_data/geoip2/GeoIP2-ISP-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + geo_ip2_asn line + geo_ip2_isp line + geo_ip2_legacy_compat line + match_remove line.geoip2 + json | jq -Sc -r ."
   assert_success
   # this one only has city data, not ASN/org/ISP
-  assert_line --index 0 '{"line":"81.2.69.142","line.city":"London","line.country_code":"GB","line.country_name":"United Kingdom","line.latitude":"51.5142","line.longitude":"-0.0931","line.postal_code":"","line.region":"ENG","line.region_name":"England"}'
+  assert_line --index 0 '{"line":"81.2.69.142","line.city":"London","line.country_code":"GB","line.country_name":"United Kingdom","line.latitude":"51.5142","line.longitude":"-0.0931","line.region":"ENG","line.region_name":"England"}'
   # this one has ASN/org data in the test databases but none in the city DB
   assert_line --index 1 '{"line":"12.81.92.0","line.asn":"AS7018","line.org":"AT&T Services"}'
   # exists only city
-  assert_line --index 2 '{"line":"2a02:d9c0::","line.country_code":"TR","line.country_name":"Turkey","line.latitude":"39.05901","line.longitude":"34.91155","line.postal_code":""}'
+  assert_line --index 2 '{"line":"2a02:d9c0::","line.country_code":"TR","line.country_name":"Turkey","line.latitude":"39.05901","line.longitude":"34.91155"}'
   # exists in ISP
   assert_line --index 3 '{"line":"2a01:1000::","line.asn":"AS5617","line.org":"Telekomunikacja Polska S.A."}'
+
+  run bash -c "echo '{\"ip\": \"4.2.2.1\", \"something_empty\": \"\", \"some_int\": 80}' | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb dap json + geo_ip2_city ip + geo_ip2_legacy_compat ip + match_remove ip. + json"
+  assert_success
+  assert_output '{"ip":"4.2.2.1","something_empty":"","some_int":80}'
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dap
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.8
 platform: ruby
 authors:
 - Rapid7 Research
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-01 00:00:00.000000000 Z
+date: 2020-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -267,7 +267,7 @@ files:
 homepage: https://www.github.com/rapid7/dap
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -282,9 +282,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.3
-signing_key: 
+rubygems_version: 3.0.6
+signing_key:
 specification_version: 4
 summary: 'DAP: The Data Analysis Pipeline'
 test_files: