dap 1.2.3 → 1.2.8

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: caec5115c33ffe3301d3522090ef651e8ef80cba
4
- data.tar.gz: f78403df2edc26901708dae7cc326289696d34dd
2
+ SHA256:
3
+ metadata.gz: 2e0ed16a920babb2efdaa5afc286a117525da9dd9c0603b1c5c3af46226cd808
4
+ data.tar.gz: 59207589322edf4886493847efcc449ef4eb6fccfebf7c3f0ea96fd940e958f5
5
5
  SHA512:
6
- metadata.gz: 06f2a048acf014e685c7f7b869ed660253d5d0fa51f3436562e569927594c10820346134197706f027d58747417d5a111c5f0715f3244f7c67d9fada22b25bb0
7
- data.tar.gz: e0ebcd198d2477bcec6335243906a2ca70437b8cc3aaa51af9703414ba3c3aff0d57f8044cf1812c6920457a8c6fae88073642bbc028f9415662d2372e28fbe6
6
+ metadata.gz: e46e7fcaf7c884f78ef7698a860984d623dfabf61f58986830bedcc7b06d1df138a943859269b2c8fd5a9ed8e2e492fca0ef57da829363a0db5b2d927310a9fd
7
+ data.tar.gz: 67c341369782f84f542b82babf35a176c8f866de0d59ea6539c1332eedcce1b6df4180bf66beede6b11c6c45070440635a64c40f80bf2f503212b5c897e0328c
data/.gitignore CHANGED
@@ -4,3 +4,13 @@
4
4
 
5
5
  # Ignore geoip data file
6
6
  data/geoip.dat
7
+
8
+ /pkg/
9
+
10
+ ## Environment normalization:
11
+ /.bundle/
12
+ /vendor/bundle
13
+ /lib/bundler/man/
14
+
15
+ # Misc
16
+ **/.DS_Store
data/Dockerfile CHANGED
@@ -1,5 +1,6 @@
1
1
  FROM ubuntu:latest
2
2
 
3
+ ENV DEBIAN_FRONTEND=noninteractive
3
4
  RUN apt-get update
4
5
  RUN apt-get install -y \
5
6
  ca-certificates \
@@ -1,6 +1,8 @@
1
1
  FROM ubuntu:18.04
2
2
 
3
3
  ENV TEST_DIR /opt/bats_testing
4
+
5
+ ENV DEBIAN_FRONTEND=noninteractive
4
6
  RUN apt-get update
5
7
  RUN apt-get install -y build-essential ca-certificates curl git jq libffi-dev libgeoip-dev libxml2-dev wget zlib1g-dev
6
8
 
data/Gemfile CHANGED
@@ -6,10 +6,10 @@ gem 'net-dns'
6
6
  gem 'bit-struct'
7
7
  gem 'geoip-c'
8
8
  gem 'maxmind-db', '~> 1.0.0'
9
- gem 'recog', '>=2.3.0'
9
+ gem 'recog', '>=2.3.8'
10
10
 
11
11
  group :test do
12
- gem 'rspec', '~> 3.1.0'
12
+ gem 'rspec', '~> 3.9.0'
13
13
  gem 'cucumber', '~> 1.3.16'
14
14
  gem 'aruba', '~> 0.6.1'
15
15
  end
@@ -5,44 +5,43 @@ GEM
5
5
  childprocess (>= 0.3.6)
6
6
  cucumber (>= 1.1.1)
7
7
  rspec-expectations (>= 2.7.0)
8
- bit-struct (0.16.0)
9
- builder (3.2.2)
10
- childprocess (0.5.8)
11
- ffi (~> 1.0, >= 1.0.11)
8
+ bit-struct (0.16)
9
+ builder (3.2.4)
10
+ childprocess (3.0.0)
12
11
  cucumber (1.3.20)
13
12
  builder (>= 2.1.2)
14
13
  diff-lcs (>= 1.1.3)
15
14
  gherkin (~> 2.12)
16
15
  multi_json (>= 1.7.5, < 2.0)
17
16
  multi_test (>= 0.1.2)
18
- diff-lcs (1.2.5)
19
- ffi (1.9.10)
17
+ diff-lcs (1.3)
20
18
  geoip-c (0.9.1)
21
19
  gherkin (2.12.2)
22
20
  multi_json (~> 1.3)
23
21
  htmlentities (4.3.4)
24
22
  maxmind-db (1.0.0)
25
23
  mini_portile2 (2.4.0)
26
- multi_json (1.11.2)
24
+ multi_json (1.14.1)
27
25
  multi_test (0.1.2)
28
26
  net-dns (0.9.0)
29
- nokogiri (1.10.3)
27
+ nokogiri (1.10.9)
30
28
  mini_portile2 (~> 2.4.0)
31
- oj (3.7.9)
32
- recog (2.3.0)
29
+ oj (3.10.6)
30
+ recog (2.3.8)
33
31
  nokogiri
34
- rspec (3.1.0)
35
- rspec-core (~> 3.1.0)
36
- rspec-expectations (~> 3.1.0)
37
- rspec-mocks (~> 3.1.0)
38
- rspec-core (3.1.7)
39
- rspec-support (~> 3.1.0)
40
- rspec-expectations (3.1.2)
32
+ rspec (3.9.0)
33
+ rspec-core (~> 3.9.0)
34
+ rspec-expectations (~> 3.9.0)
35
+ rspec-mocks (~> 3.9.0)
36
+ rspec-core (3.9.2)
37
+ rspec-support (~> 3.9.3)
38
+ rspec-expectations (3.9.2)
41
39
  diff-lcs (>= 1.2.0, < 2.0)
42
- rspec-support (~> 3.1.0)
43
- rspec-mocks (3.1.3)
44
- rspec-support (~> 3.1.0)
45
- rspec-support (3.1.2)
40
+ rspec-support (~> 3.9.0)
41
+ rspec-mocks (3.9.1)
42
+ diff-lcs (>= 1.2.0, < 2.0)
43
+ rspec-support (~> 3.9.0)
44
+ rspec-support (3.9.3)
46
45
 
47
46
  PLATFORMS
48
47
  ruby
@@ -56,8 +55,8 @@ DEPENDENCIES
56
55
  maxmind-db (~> 1.0.0)
57
56
  net-dns
58
57
  oj
59
- recog (>= 2.3.0)
60
- rspec (~> 3.1.0)
58
+ recog (>= 2.3.8)
59
+ rspec (~> 3.9.0)
61
60
 
62
61
  BUNDLED WITH
63
- 2.0.1
62
+ 2.0.2
data/README.md CHANGED
@@ -9,8 +9,6 @@ DAP reads data using an input plugin, transforms it through a series of filters,
9
9
 
10
10
  DAP was written to process terabyte-sized public scan datasets, such as those provided by https://scans.io/. Although DAP isn't particularly fast, it can be used across multiple cores (and machines) by splitting the input source and wrapping the execution with GNU Parallel.
11
11
 
12
-
13
-
14
12
  ## Installation
15
13
 
16
14
  ### Prerequisites
@@ -61,7 +59,10 @@ gem install dap
61
59
  ### OS X
62
60
 
63
61
  ```bash
62
+ # Install the GeoIP C library required by DAP
64
63
  brew update
64
+ brew install geoip
65
+
65
66
  gem install dap
66
67
  ```
67
68
 
@@ -69,7 +70,7 @@ gem install dap
69
70
 
70
71
  In its simplest form, DAP takes input, applies zero or more filters which modify the input, and then outputs the result. The input, filters and output are separated by plus signs (`+`). As seen from `dap -h`:
71
72
 
72
- ```
73
+ ```shell
73
74
  Usage: dap [input] + [filter] + [output]
74
75
  --inputs
75
76
  --outputs
@@ -80,7 +81,7 @@ To see which input/output formats are supported and what filters are available,
80
81
 
81
82
  This example reads as input a single IP address from `STDIN` in line form, applies geo-ip transformations as a filter on that line, and then returns the output as JSON:
82
83
 
83
- ```
84
+ ```shell
84
85
  $ echo 8.8.8.8 | bin/dap + lines + geo_ip2_city line + json | jq .
85
86
  {
86
87
  "line": "8.8.8.8",
data/Rakefile CHANGED
@@ -19,4 +19,4 @@ Cucumber::Rake::Task.new(:features) do |t|
19
19
  end
20
20
 
21
21
  task :default => [ :spec, :features, :yard ]
22
-
22
+ task :tests => [ :spec ]
@@ -33,6 +33,22 @@ module GeoIP2Library
33
33
  nil
34
34
  end
35
35
 
36
+ def get_maxmind_data(db, ip)
37
+ begin
38
+ db.get(ip)
39
+ rescue IPAddr::InvalidAddressError
40
+ end
41
+ end
42
+
43
+ def remove_empties(hash)
44
+ hash.each_pair do |k,v|
45
+ if v.empty?
46
+ hash.delete(k)
47
+ end
48
+ end
49
+ hash
50
+ end
51
+
36
52
  @@geo_asn = find_db(GEOIP2_ASN, GEOIP2_DIRS, ENV["GEOIP2_ASN_DATABASE_PATH"])
37
53
  @@geo_city = find_db(GEOIP2_CITY, GEOIP2_DIRS, ENV["GEOIP2_CITY_DATABASE_PATH"])
38
54
  @@geo_isp = find_db(GEOIP2_ISP, GEOIP2_DIRS, ENV["GEOIP2_ISP_DATABASE_PATH"])
@@ -69,8 +85,10 @@ class FilterGeoIP2City
69
85
  unless @@geo_city
70
86
  raise "No MaxMind GeoIP2::City data found"
71
87
  end
72
- return unless (geo_hash = @@geo_city.get(ip))
88
+
73
89
  ret = defaults
90
+ geo_hash = get_maxmind_data(@@geo_city, ip)
91
+ return unless geo_hash
74
92
 
75
93
  if geo_hash.include?("subdivisions")
76
94
  # handle countries that are divided into various subdivisions. generally 1, sometimes 2
@@ -101,7 +119,8 @@ class FilterGeoIP2City
101
119
  ret["geoip2.city.#{lsn_renamed}"] = v
102
120
  end
103
121
  end
104
- ret
122
+
123
+ remove_empties(ret)
105
124
  end
106
125
 
107
126
  def defaults()
@@ -133,9 +152,9 @@ class FilterGeoIP2Asn
133
152
  unless @@geo_asn
134
153
  raise "No MaxMind GeoIP2::ASN data found"
135
154
  end
136
- geo_hash = @@geo_asn.get(ip)
137
- return unless geo_hash
138
155
 
156
+ geo_hash = get_maxmind_data(@@geo_asn, ip)
157
+ return unless geo_hash
139
158
  ret = {}
140
159
 
141
160
  if geo_hash.include?("autonomous_system_number")
@@ -150,7 +169,7 @@ class FilterGeoIP2Asn
150
169
  ret["geoip2.asn.asn_org"] = ""
151
170
  end
152
171
 
153
- ret
172
+ remove_empties(ret)
154
173
  end
155
174
  end
156
175
 
@@ -164,9 +183,9 @@ class FilterGeoIP2Isp
164
183
  unless @@geo_isp
165
184
  raise "No MaxMind GeoIP2::ISP data found"
166
185
  end
167
- geo_hash = @@geo_isp.get(ip)
168
- return unless geo_hash
169
186
 
187
+ geo_hash = get_maxmind_data(@@geo_isp, ip)
188
+ return unless geo_hash
170
189
  ret = {}
171
190
 
172
191
  if geo_hash.include?("autonomous_system_number")
@@ -193,7 +212,7 @@ class FilterGeoIP2Isp
193
212
  ret["geoip2.isp.org"] = ""
194
213
  end
195
214
 
196
- ret
215
+ remove_empties(ret)
197
216
  end
198
217
  end
199
218
 
@@ -202,6 +221,7 @@ end
202
221
  #
203
222
  class FilterGeoIP2LegacyCompat
204
223
  include Base
224
+ include GeoIP2Library
205
225
 
206
226
  attr_accessor :base_field
207
227
 
@@ -227,10 +247,11 @@ class FilterGeoIP2LegacyCompat
227
247
  "isp.asn": "asn",
228
248
  }
229
249
 
250
+ ret = {}
230
251
  remap.each_pair do |geoip2,geoip|
231
252
  geoip2_key = "#{self.base_field}.geoip2.#{geoip2}"
232
253
  if doc.include?(geoip2_key)
233
- doc["#{self.base_field}.#{geoip}"] = doc[geoip2_key]
254
+ ret["#{self.base_field}.#{geoip}"] = doc[geoip2_key]
234
255
  end
235
256
  end
236
257
 
@@ -245,7 +266,7 @@ class FilterGeoIP2LegacyCompat
245
266
  if doc.include?(anon_key)
246
267
  anon_value = doc[anon_key]
247
268
  if anon_value == "true"
248
- doc["#{self.base_field}.country_code"] = "A1"
269
+ ret["#{self.base_field}.country_code"] = "A1"
249
270
  end
250
271
  end
251
272
 
@@ -253,7 +274,7 @@ class FilterGeoIP2LegacyCompat
253
274
  if doc.include?(satellite_key)
254
275
  satellite_value = doc[satellite_key]
255
276
  if satellite_value == "true"
256
- doc["#{self.base_field}.country_code"] = "A1"
277
+ ret["#{self.base_field}.country_code"] = "A1"
257
278
  end
258
279
  end
259
280
 
@@ -262,7 +283,7 @@ class FilterGeoIP2LegacyCompat
262
283
  if doc.include?(metro_key)
263
284
  metro_value = doc[metro_key]
264
285
  if !metro_value.empty? && metro_value != "0"
265
- doc["#{self.base_field}.dma_code"] = metro_value
286
+ ret["#{self.base_field}.dma_code"] = metro_value
266
287
  end
267
288
  end
268
289
 
@@ -273,12 +294,12 @@ class FilterGeoIP2LegacyCompat
273
294
  [ isp_org_key, isp_asn_org_key, asn_org_key ].each do |k|
274
295
  v = doc[k]
275
296
  if v && !v.empty?
276
- doc["#{self.base_field}.org"] = v
297
+ ret["#{self.base_field}.org"] = v
277
298
  break
278
299
  end
279
300
  end
280
301
 
281
- [ doc ]
302
+ [ doc.merge(remove_empties(ret)) ]
282
303
  end
283
304
  end
284
305
 
@@ -15,7 +15,7 @@ module HTMLGhetto
15
15
  data.
16
16
  to_s.
17
17
  encode('UTF-8', invalid: :replace, undef: :replace, replace: '').
18
- scan(/<([^>]+)>/m).each do |e|
18
+ scan(/<([^<>]{1,4096})>/m).each do |e|
19
19
 
20
20
  e = e.first
21
21
 
@@ -1,3 +1,3 @@
1
1
  module Dap
2
- VERSION = "1.2.3"
2
+ VERSION = "1.2.8"
3
3
  end
@@ -151,5 +151,70 @@ describe Dap::Filter::FilterHTMLLinks do
151
151
  expect(processed.map { |p| p['link'] }).to eq(%w(a b))
152
152
  end
153
153
  end
154
+
155
+ context 'repeated less than symbol' do
156
+ body =
157
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
158
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
159
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
160
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
161
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
162
+ '<a href="a"/>'\
163
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
164
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
165
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
166
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
167
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
168
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
169
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
170
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
171
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
172
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
173
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
174
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
175
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
176
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
177
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
178
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
179
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
180
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
181
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
182
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
183
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
184
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
185
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
186
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
187
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
188
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
189
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
190
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
191
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
192
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
193
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
194
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
195
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
196
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
197
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
198
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
199
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
200
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
201
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
202
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
203
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
204
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
205
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
206
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
207
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
208
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
209
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
210
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
211
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
212
+ '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'\
213
+ '<a href="b"/>'
214
+ let(:processed) { filter.process({'data' => body}) }
215
+ it 'extracted the correct links' do
216
+ expect(processed.map { |p| p['link'] }).to eq(%w(a b))
217
+ end
218
+ end
154
219
  end
155
220
  end
@@ -149,17 +149,22 @@ load ./test_common
149
149
  # test with default language
150
150
  run bash -c "echo 81.2.69.142 | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
151
151
  assert_success
152
- assert_output '{"line":"81.2.69.142","line.geoip2.city.city.geoname_id":"2643743","line.geoip2.city.city.name":"London","line.geoip2.city.continent.code":"EU","line.geoip2.city.continent.geoname_id":"6255148","line.geoip2.city.continent.name":"Europe","line.geoip2.city.country.geoname_id":"2635167","line.geoip2.city.country.is_in_european_union":"true","line.geoip2.city.country.iso_code":"GB","line.geoip2.city.country.name":"United Kingdom","line.geoip2.city.location.accuracy_radius":"10","line.geoip2.city.location.latitude":"51.5142","line.geoip2.city.location.longitude":"-0.0931","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/London","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"6252001","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"US","line.geoip2.city.registered_country.name":"United States","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.subdivisions.0.geoname_id":"6269131","line.geoip2.city.subdivisions.0.iso_code":"ENG","line.geoip2.city.subdivisions.0.name":"England","line.geoip2.city.subdivisions.length":"1","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
152
+ assert_output '{"line":"81.2.69.142","line.geoip2.city.city.geoname_id":"2643743","line.geoip2.city.city.name":"London","line.geoip2.city.continent.code":"EU","line.geoip2.city.continent.geoname_id":"6255148","line.geoip2.city.continent.name":"Europe","line.geoip2.city.country.geoname_id":"2635167","line.geoip2.city.country.is_in_european_union":"true","line.geoip2.city.country.iso_code":"GB","line.geoip2.city.country.name":"United Kingdom","line.geoip2.city.location.accuracy_radius":"10","line.geoip2.city.location.latitude":"51.5142","line.geoip2.city.location.longitude":"-0.0931","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/London","line.geoip2.city.registered_country.geoname_id":"6252001","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"US","line.geoip2.city.registered_country.name":"United States","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.subdivisions.0.geoname_id":"6269131","line.geoip2.city.subdivisions.0.iso_code":"ENG","line.geoip2.city.subdivisions.0.name":"England","line.geoip2.city.subdivisions.length":"1","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
153
153
 
154
154
  # test with non-default language
155
155
  run bash -c "echo 67.43.156.0 | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb GEOIP2_LANGUAGE=fr $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
156
156
  assert_success
157
- assert_output '{"line":"67.43.156.0","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asie","line.geoip2.city.country.geoname_id":"1252634","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"BT","line.geoip2.city.country.name":"Bhutan","line.geoip2.city.location.accuracy_radius":"534","line.geoip2.city.location.latitude":"27.5","line.geoip2.city.location.longitude":"90.5","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Asia/Thimphu","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"798549","line.geoip2.city.registered_country.is_in_european_union":"true","line.geoip2.city.registered_country.iso_code":"RO","line.geoip2.city.registered_country.name":"Roumanie","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.traits.is_anonymous_proxy":"true","line.geoip2.city.traits.is_satellite_provider":"false"}'
157
+ assert_output '{"line":"67.43.156.0","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asie","line.geoip2.city.country.geoname_id":"1252634","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"BT","line.geoip2.city.country.name":"Bhutan","line.geoip2.city.location.accuracy_radius":"534","line.geoip2.city.location.latitude":"27.5","line.geoip2.city.location.longitude":"90.5","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Asia/Thimphu","line.geoip2.city.registered_country.geoname_id":"798549","line.geoip2.city.registered_country.is_in_european_union":"true","line.geoip2.city.registered_country.iso_code":"RO","line.geoip2.city.registered_country.name":"Roumanie","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.traits.is_anonymous_proxy":"true","line.geoip2.city.traits.is_satellite_provider":"false"}'
158
158
 
159
159
  # test IPv6
160
160
  run bash -c "echo 2a02:d9c0:: | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
161
161
  assert_success
162
- assert_output '{"line":"2a02:d9c0::","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asia","line.geoip2.city.country.geoname_id":"298795","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"TR","line.geoip2.city.country.name":"Turkey","line.geoip2.city.location.accuracy_radius":"100","line.geoip2.city.location.latitude":"39.05901","line.geoip2.city.location.longitude":"34.91155","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/Istanbul","line.geoip2.city.postal.code":"","line.geoip2.city.registered_country.geoname_id":"298795","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"TR","line.geoip2.city.registered_country.name":"Turkey","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.represented_country.iso_code":"","line.geoip2.city.represented_country.type":"","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
162
+ assert_output '{"line":"2a02:d9c0::","line.geoip2.city.city.geoname_id":"0","line.geoip2.city.continent.code":"AS","line.geoip2.city.continent.geoname_id":"6255147","line.geoip2.city.continent.name":"Asia","line.geoip2.city.country.geoname_id":"298795","line.geoip2.city.country.is_in_european_union":"false","line.geoip2.city.country.iso_code":"TR","line.geoip2.city.country.name":"Turkey","line.geoip2.city.location.accuracy_radius":"100","line.geoip2.city.location.latitude":"39.05901","line.geoip2.city.location.longitude":"34.91155","line.geoip2.city.location.metro_code":"0","line.geoip2.city.location.time_zone":"Europe/Istanbul","line.geoip2.city.registered_country.geoname_id":"298795","line.geoip2.city.registered_country.is_in_european_union":"false","line.geoip2.city.registered_country.iso_code":"TR","line.geoip2.city.registered_country.name":"Turkey","line.geoip2.city.represented_country.geoname_id":"0","line.geoip2.city.represented_country.is_in_european_union":"false","line.geoip2.city.traits.is_anonymous_proxy":"false","line.geoip2.city.traits.is_satellite_provider":"false"}'
163
+
164
+ # test invalid IP
165
+ run bash -c "echo test | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + json | jq -Sc -r ."
166
+ assert_success
167
+ assert_output '{"line":"test"}'
163
168
  }
164
169
 
165
170
  @test "geo_ip2_asn" {
@@ -171,24 +176,35 @@ load ./test_common
171
176
  run bash -c "echo 2600:7000:: | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_asn line + json | jq -Sc -r ."
172
177
  assert_success
173
178
  assert_output '{"line":"2600:7000::","line.geoip2.asn.asn":"AS6939","line.geoip2.asn.asn_org":"Hurricane Electric, Inc."}'
179
+
180
+ # test invalid IP
181
+ run bash -c "echo test | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_asn line + json | jq -Sc -r ."
182
+ assert_success
183
+ assert_output '{"line":"test"}'
174
184
  }
175
185
 
176
186
  @test "geo_ip2_isp" {
177
- run bash -c "echo -e '12.81.92.0\n2600:7000::' | GEOIP2_ISP_DATABASE_PATH=test/test_data/geoip2/GeoIP2-ISP-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_isp line + json | jq -Sc -r ."
178
- assert_line --index 0 '{"line":"12.81.92.0","line.geoip2.isp.asn":"AS7018","line.geoip2.isp.asn_org":"","line.geoip2.isp.isp":"AT&T Services","line.geoip2.isp.org":"AT&T Services"}'
187
+ run bash -c "echo -e '12.81.92.0\n2600:7000::\ntest' | GEOIP2_ISP_DATABASE_PATH=test/test_data/geoip2/GeoIP2-ISP-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_isp line + json | jq -Sc -r ."
188
+ assert_line --index 0 '{"line":"12.81.92.0","line.geoip2.isp.asn":"AS7018","line.geoip2.isp.isp":"AT&T Services","line.geoip2.isp.org":"AT&T Services"}'
179
189
  # test IPv6
180
- assert_line --index 1 '{"line":"2600:7000::","line.geoip2.isp.asn":"AS6939","line.geoip2.isp.asn_org":"Hurricane Electric, Inc.","line.geoip2.isp.isp":"","line.geoip2.isp.org":""}'
190
+ assert_line --index 1 '{"line":"2600:7000::","line.geoip2.isp.asn":"AS6939","line.geoip2.isp.asn_org":"Hurricane Electric, Inc."}'
191
+ # test invalid IP
192
+ assert_line --index 2 '{"line":"test"}'
181
193
  }
182
194
 
183
195
  @test "geo_ip2_legacy_compat" {
184
196
  run bash -c "echo -e '81.2.69.142\n12.81.92.0\n2a02:d9c0::\n2a01:1000::' | GEOIP2_ASN_DATABASE_PATH=test/test_data/geoip2/GeoLite2-ASN-Test.mmdb GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb GEOIP2_ISP_DATABASE_PATH=test/test_data/geoip2/GeoIP2-ISP-Test.mmdb $DAP_EXECUTABLE lines + geo_ip2_city line + geo_ip2_asn line + geo_ip2_isp line + geo_ip2_legacy_compat line + match_remove line.geoip2 + json | jq -Sc -r ."
185
197
  assert_success
186
198
  # this one only has city data, not ASN/org/ISP
187
- assert_line --index 0 '{"line":"81.2.69.142","line.city":"London","line.country_code":"GB","line.country_name":"United Kingdom","line.latitude":"51.5142","line.longitude":"-0.0931","line.postal_code":"","line.region":"ENG","line.region_name":"England"}'
199
+ assert_line --index 0 '{"line":"81.2.69.142","line.city":"London","line.country_code":"GB","line.country_name":"United Kingdom","line.latitude":"51.5142","line.longitude":"-0.0931","line.region":"ENG","line.region_name":"England"}'
188
200
  # this one has ASN/org data in the test databases but none in the city DB
189
201
  assert_line --index 1 '{"line":"12.81.92.0","line.asn":"AS7018","line.org":"AT&T Services"}'
190
202
  # exists only city
191
- assert_line --index 2 '{"line":"2a02:d9c0::","line.country_code":"TR","line.country_name":"Turkey","line.latitude":"39.05901","line.longitude":"34.91155","line.postal_code":""}'
203
+ assert_line --index 2 '{"line":"2a02:d9c0::","line.country_code":"TR","line.country_name":"Turkey","line.latitude":"39.05901","line.longitude":"34.91155"}'
192
204
  # exists in ISP
193
205
  assert_line --index 3 '{"line":"2a01:1000::","line.asn":"AS5617","line.org":"Telekomunikacja Polska S.A."}'
206
+
207
+ run bash -c "echo '{\"ip\": \"4.2.2.1\", \"something_empty\": \"\", \"some_int\": 80}' | GEOIP2_CITY_DATABASE_PATH=test/test_data/geoip2/GeoIP2-City-Test.mmdb dap json + geo_ip2_city ip + geo_ip2_legacy_compat ip + match_remove ip. + json"
208
+ assert_success
209
+ assert_output '{"ip":"4.2.2.1","something_empty":"","some_int":80}'
194
210
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dap
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.3
4
+ version: 1.2.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rapid7 Research
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-05-01 00:00:00.000000000 Z
11
+ date: 2020-06-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec
@@ -267,7 +267,7 @@ files:
267
267
  homepage: https://www.github.com/rapid7/dap
268
268
  licenses: []
269
269
  metadata: {}
270
- post_install_message:
270
+ post_install_message:
271
271
  rdoc_options: []
272
272
  require_paths:
273
273
  - lib
@@ -282,9 +282,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
282
282
  - !ruby/object:Gem::Version
283
283
  version: '0'
284
284
  requirements: []
285
- rubyforge_project:
286
- rubygems_version: 2.6.14.3
287
- signing_key:
285
+ rubygems_version: 3.0.6
286
+ signing_key:
288
287
  specification_version: 4
289
288
  summary: 'DAP: The Data Analysis Pipeline'
290
289
  test_files: