fluent-plugin-geoip 0.6.1 → 0.7.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,9 +1,11 @@
1
1
  require 'fluent/mixin/rewrite_tag_name'
2
+ require 'fluent/plugin/geoip'
2
3
 
3
4
  class Fluent::GeoipOutput < Fluent::BufferedOutput
4
5
  Fluent::Plugin.register_output('geoip', self)
5
6
 
6
7
  config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
8
+ config_param :geoip2_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
7
9
  config_param :geoip_lookup_key, :string, :default => 'host'
8
10
  config_param :tag, :string, :default => nil
9
11
  config_param :skip_adding_null_record, :bool, :default => false
@@ -18,21 +20,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
18
20
  config_param :flush_interval, :time, :default => 0
19
21
  config_param :log_level, :string, :default => 'warn'
20
22
 
21
- # Define `log` method for v0.10.42 or earlier
22
- unless method_defined?(:log)
23
- define_method("log") { $log }
24
- end
25
-
26
- # To support Fluentd v0.10.57 or earlier
27
- unless method_defined?(:router)
28
- define_method("router") { Fluent::Engine }
29
- end
30
-
31
- def initialize
32
- require 'fluent/plugin/geoip'
33
-
34
- super
35
- end
23
+ config_param :backend_library, :enum, :list => Fluent::GeoIP::BACKEND_LIBRARIES, :default => :geoip
36
24
 
37
25
  def configure(conf)
38
26
  super
@@ -7,14 +7,7 @@ class GeoipFilterTest < Test::Unit::TestCase
7
7
  @time = Fluent::Engine.now
8
8
  end
9
9
 
10
- CONFIG = %[
11
- geoip_lookup_key host
12
- enable_key_city geoip_city
13
- remove_tag_prefix input.
14
- tag geoip.${tag}
15
- ]
16
-
17
- def create_driver(conf=CONFIG, tag='test', use_v1=false)
10
+ def create_driver(conf='', tag='test', use_v1=false)
18
11
  Fluent::Test::FilterTestDriver.new(Fluent::GeoipFilter, tag).configure(conf, use_v1)
19
12
  end
20
13
 
@@ -29,408 +22,992 @@ class GeoipFilterTest < Test::Unit::TestCase
29
22
  filtered.map {|m| m[2] }
30
23
  end
31
24
 
32
- def test_configure
33
- assert_nothing_raised {
34
- create_driver('')
35
- }
36
- assert_raise(Fluent::ConfigError) {
37
- create_driver('enable_key_cities')
38
- }
39
- d = create_driver %[
40
- enable_key_city geoip_city
41
- remove_tag_prefix input.
42
- tag geoip.${tag}
43
- ]
44
- assert_equal 'geoip_city', d.instance.config['enable_key_city']
45
-
46
- # multiple key config
47
- d = create_driver %[
48
- geoip_lookup_key from.ip, to.ip
49
- enable_key_city from_city, to_city
50
- remove_tag_prefix input.
51
- tag geoip.${tag}
52
- ]
53
- assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
54
-
55
- # multiple key config (bad configure)
56
- assert_raise(Fluent::ConfigError) {
25
+ sub_test_case "configure" do
26
+ test "empty" do
27
+ assert_nothing_raised {
28
+ create_driver('')
29
+ }
30
+ end
31
+
32
+ test "missing required parameters" do
33
+ assert_raise(Fluent::ConfigError) {
34
+ create_driver('enable_key_cities')
35
+ }
36
+ end
37
+
38
+ test "minimum" do
57
39
  d = create_driver %[
58
- geoip_lookup_key from.ip, to.ip
59
- enable_key_city from_city
60
- enable_key_region from_region
61
- remove_tag_prefix input.
62
- tag geoip.${tag}
40
+ enable_key_city geoip_city
63
41
  ]
64
- }
42
+ assert_equal 'geoip_city', d.instance.config['enable_key_city']
43
+ end
65
44
 
66
- # invalid json structure
67
- assert_raise(Fluent::ConfigError) {
45
+ test "multiple key config" do
68
46
  d = create_driver %[
47
+ geoip_lookup_key from.ip, to.ip
48
+ enable_key_city from_city, to_city
49
+ ]
50
+ assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
51
+ end
52
+
53
+ test "multiple key config (bad configure)" do
54
+ assert_raise(Fluent::ConfigError) {
55
+ create_driver %[
56
+ geoip_lookup_key from.ip, to.ip
57
+ enable_key_city from_city
58
+ enable_key_region from_region
59
+ ]
60
+ }
61
+ end
62
+
63
+ test "invalid json structure w/ Ruby hash like" do
64
+ assert_raise(Fluent::ConfigError) {
65
+ create_driver %[
66
+ geoip_lookup_key host
67
+ <record>
68
+ invalid_json {"foo" => 123}
69
+ </record>
70
+ ]
71
+ }
72
+ end
73
+
74
+ test "invalid json structure w/ unquoted string literal" do
75
+ assert_raise(Fluent::ConfigError) {
76
+ create_driver %[
77
+ geoip_lookup_key host
78
+ <record>
79
+ invalid_json {"foo" : string, "bar" : 123}
80
+ </record>
81
+ ]
82
+ }
83
+ end
84
+
85
+ data(geoip: "geoip",
86
+ geoip2_compat: "geoip2_compat")
87
+ test "unsupported key" do |backend|
88
+ assert_raise(Fluent::ConfigError.new("#{backend}: unsupported key unknown")) do
89
+ create_driver %[
90
+ backend_library #{backend}
91
+ <record>
92
+ city ${unknown["host"]}
93
+ </record>
94
+ ]
95
+ end
96
+ end
97
+
98
+ data(geoip: ["geoip", '${city["host"]}'],
99
+ geoip2_compat: ["geoip2_compat", '${city["host"]}'],
100
+ geoip2_c: ["geoip2_c", '${city.names.en["host"]}'])
101
+ test "supported backend" do |(backend, placeholder)|
102
+ create_driver %[
103
+ backend_library #{backend}
104
+ <record>
105
+ city #{placeholder}
106
+ </record>
107
+ ]
108
+ end
109
+
110
+ test "unsupported backend" do
111
+ assert_raise(Fluent::ConfigError) do
112
+ create_driver %[
113
+ backend_library hive_geoip2
114
+ <record>
115
+ city ${city["host"]}
116
+ </record>
117
+ ]
118
+ end
119
+ end
120
+ end
121
+
122
+ sub_test_case "geoip2_c" do
123
+ def test_filter_with_dot_key
124
+ config = %[
125
+ backend_library geoip2_c
126
+ geoip_lookup_key ip.origin, ip.dest
127
+ <record>
128
+ origin_country ${country.iso_code['ip.origin']}
129
+ dest_country ${country.iso_code['ip.dest']}
130
+ </record>
131
+ ]
132
+ messages = [
133
+ {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'}
134
+ ]
135
+ expected = [
136
+ {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8',
137
+ 'origin_country' => 'US', 'dest_country' => 'US'}
138
+ ]
139
+ filtered = filter(config, messages)
140
+ assert_equal(expected, filtered)
141
+ end
142
+
143
+ def test_filter_with_unknown_address
144
+ config = %[
145
+ backend_library geoip2_c
69
146
  geoip_lookup_key host
70
147
  <record>
71
- invalid_json {"foo" => 123}
148
+ geoip_city ${city.names.en['host']}
149
+ geopoint [${location.longitude['host']}, ${location.latitude['host']}]
72
150
  </record>
73
- remove_tag_prefix input.
74
- tag geoip.${tag}
151
+ skip_adding_null_record false
75
152
  ]
76
- }
77
- assert_raise(Fluent::ConfigError) {
78
- d = create_driver %[
153
+ # 203.0.113.1 is a test address described in RFC5737
154
+ messages = [
155
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
156
+ {'host' => '0', 'message' => 'invalid ip'}
157
+ ]
158
+ expected = [
159
+ {'host' => '203.0.113.1', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
160
+ {'host' => '0', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]}
161
+ ]
162
+ filtered = filter(config, messages)
163
+ assert_equal(expected, filtered)
164
+ end
165
+
166
+ def test_filter_with_skip_unknown_address
167
+ config = %[
168
+ backend_library geoip2_c
79
169
  geoip_lookup_key host
80
170
  <record>
81
- invalid_json {"foo" : string, "bar" : 123}
171
+ geoip_city ${city.names.en['host']}
172
+ geopoint [${location.longitude['host']}, ${location.latitude['host']}]
82
173
  </record>
83
- remove_tag_prefix input.
84
- tag geoip.${tag}
174
+ skip_adding_null_record true
85
175
  ]
86
- }
87
- end
176
+ # 203.0.113.1 is a test address described in RFC5737
177
+ messages = [
178
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
179
+ {'host' => '0', 'message' => 'invalid ip'},
180
+ {'host' => '8.8.8.8', 'message' => 'google public dns'}
181
+ ]
182
+ expected = [
183
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
184
+ {'host' => '0', 'message' => 'invalid ip'},
185
+ {'host' => '8.8.8.8', 'message' => 'google public dns',
186
+ 'geoip_city' => 'Mountain View', 'geopoint' => [-122.0838, 37.386]}
187
+ ]
188
+ filtered = filter(config, messages)
189
+ assert_equal(expected, filtered)
190
+ end
88
191
 
89
- def test_filter
90
- messages = [
91
- {'host' => '66.102.3.80', 'message' => 'valid ip'},
92
- {'message' => 'missing field'},
93
- ]
94
- expected = [
95
- {'host' => '66.102.3.80', 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
96
- {'message' => 'missing field', 'geoip_city' => nil},
97
- ]
98
- filtered = filter(CONFIG, messages)
99
- assert_equal(expected, filtered)
100
- end
192
+ def test_filter_record_directive
193
+ config = %[
194
+ backend_library geoip2_c
195
+ geoip_lookup_key from.ip
196
+ <record>
197
+ from_city ${city.names.en['from.ip']}
198
+ from_country ${country.names.en['from.ip']}
199
+ latitude ${location.latitude['from.ip']}
200
+ longitude ${location.longitude['from.ip']}
201
+ float_concat ${location.latitude['from.ip']},${location.longitude['from.ip']}
202
+ float_array [${location.longitude['from.ip']}, ${location.latitude['from.ip']}]
203
+ float_nest { "lat" : ${location.latitude['from.ip']}, "lon" : ${location.longitude['from.ip']}}
204
+ string_concat ${city.names.en['from.ip']},${country.names.en['from.ip']}
205
+ string_array [${city.names.en['from.ip']}, ${country.names.en['from.ip']}]
206
+ string_nest { "city" : ${city.names.en['from.ip']}, "country_name" : ${country.names.en['from.ip']}}
207
+ unknown_city ${city.names.en['unknown_key']}
208
+ undefined ${city.names.en['undefined']}
209
+ broken_array1 [${location.longitude['from.ip']}, ${location.latitude['undefined']}]
210
+ broken_array2 [${location.longitude['undefined']}, ${location.latitude['undefined']}]
211
+ </record>
212
+ ]
213
+ messages = [
214
+ { 'from' => {'ip' => '66.102.3.80'} },
215
+ { 'message' => 'missing field' },
216
+ ]
217
+ expected = [
218
+ {
219
+ 'from' => {'ip' => '66.102.3.80'},
220
+ 'from_city' => 'Mountain View',
221
+ 'from_country' => 'United States',
222
+ 'latitude' => 37.419200000000004,
223
+ 'longitude' => -122.0574,
224
+ 'float_concat' => '37.419200000000004,-122.0574',
225
+ 'float_array' => [-122.0574, 37.419200000000004],
226
+ 'float_nest' => { 'lat' => 37.4192000000000004, 'lon' => -122.0574 },
227
+ 'string_concat' => 'Mountain View,United States',
228
+ 'string_array' => ["Mountain View", "United States"],
229
+ 'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
230
+ 'unknown_city' => nil,
231
+ 'undefined' => nil,
232
+ 'broken_array1' => [-122.0574, nil],
233
+ 'broken_array2' => [nil, nil]
234
+ },
235
+ {
236
+ 'message' => 'missing field',
237
+ 'from_city' => nil,
238
+ 'from_country' => nil,
239
+ 'latitude' => nil,
240
+ 'longitude' => nil,
241
+ 'float_concat' => ',',
242
+ 'float_array' => [nil, nil],
243
+ 'float_nest' => { 'lat' => nil, 'lon' => nil },
244
+ 'string_concat' => ',',
245
+ 'string_array' => [nil, nil],
246
+ 'string_nest' => { "city" => nil, "country_name" => nil },
247
+ 'unknown_city' => nil,
248
+ 'undefined' => nil,
249
+ 'broken_array1' => [nil, nil],
250
+ 'broken_array2' => [nil, nil]
251
+ },
252
+ ]
253
+ filtered = filter(config, messages)
254
+ # test-unit cannot calculate diff between large Array
255
+ assert_equal(expected[0], filtered[0])
256
+ assert_equal(expected[1], filtered[1])
257
+ end
101
258
 
102
- def test_filter_with_dot_key
103
- config = %[
104
- geoip_lookup_key ip.origin, ip.dest
105
- <record>
106
- origin_country ${country_code['ip.origin']}
107
- dest_country ${country_code['ip.dest']}
108
- </record>
109
- ]
110
- messages = [
111
- {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'}
112
- ]
113
- expected = [
114
- {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8',
115
- 'origin_country' => 'US', 'dest_country' => 'US'}
116
- ]
117
- filtered = filter(config, messages)
118
- assert_equal(expected, filtered)
119
- end
259
+ def test_filter_record_directive_multiple_record
260
+ config = %[
261
+ backend_library geoip2_c
262
+ geoip_lookup_key from.ip, to.ip
263
+ <record>
264
+ from_city ${city.names.en['from.ip']}
265
+ to_city ${city.names.en['to.ip']}
266
+ from_country ${country.names.en['from.ip']}
267
+ to_country ${country.names.en['to.ip']}
268
+ string_array [${country.names.en['from.ip']}, ${country.names.en['to.ip']}]
269
+ </record>
270
+ ]
271
+ messages = [
272
+ {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
273
+ {'message' => 'missing field'}
274
+ ]
275
+ expected = [
276
+ {
277
+ 'from' => { 'ip' => '66.102.3.80' },
278
+ 'to' => { 'ip' => '125.54.15.42' },
279
+ 'from_city' => 'Mountain View',
280
+ 'from_country' => 'United States',
281
+ 'to_city' => 'Tokorozawa',
282
+ 'to_country' => 'Japan',
283
+ 'string_array' => ['United States', 'Japan']
284
+ },
285
+ {
286
+ 'message' => 'missing field',
287
+ 'from_city' => nil,
288
+ 'from_country' => nil,
289
+ 'to_city' => nil,
290
+ 'to_country' => nil,
291
+ 'string_array' => [nil, nil]
292
+ }
293
+ ]
294
+ filtered = filter(config, messages)
295
+ assert_equal(expected, filtered)
296
+ end
120
297
 
121
- def test_filter_nested_attr
122
- config = %[
123
- geoip_lookup_key host.ip
124
- enable_key_city geoip_city
125
- ]
126
- messages = [
127
- {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'},
128
- {'message' => 'missing field'}
129
- ]
130
- expected = [
131
- {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
132
- {'message' => 'missing field', 'geoip_city' => nil}
133
- ]
134
- filtered = filter(config, messages)
135
- assert_equal(expected, filtered)
136
- end
298
+ def config_quoted_record
299
+ %[
300
+ backend_library geoip2_c
301
+ geoip_lookup_key host
302
+ <record>
303
+ location_properties '{ "country_code" : "${country.iso_code["host"]}", "lat": ${location.latitude["host"]}, "lon": ${location.longitude["host"]} }'
304
+ location_string ${location.latitude['host']},${location.longitude['host']}
305
+ location_string2 ${country.iso_code["host"]}
306
+ location_array "[${location.longitude['host']},${location.latitude['host']}]"
307
+ location_array2 '[${location.longitude["host"]},${location.latitude["host"]}]'
308
+ peculiar_pattern '[GEOIP] message => {"lat":${location.latitude["host"]}, "lon":${location.longitude["host"]}}'
309
+ </record>
310
+ ]
311
+ end
137
312
 
138
- def test_filter_with_unknown_address
139
- config = %[
140
- geoip_lookup_key host
141
- <record>
142
- geoip_city ${city['host']}
143
- geopoint [${longitude['host']}, ${latitude['host']}]
144
- </record>
145
- skip_adding_null_record false
146
- ]
147
- # 203.0.113.1 is a test address described in RFC5737
148
- messages = [
149
- {'host' => '203.0.113.1', 'message' => 'invalid ip'},
150
- {'host' => '0', 'message' => 'invalid ip'}
151
- ]
152
- expected = [
153
- {'host' => '203.0.113.1', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
154
- {'host' => '0', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]}
155
- ]
156
- filtered = filter(config, messages)
157
- assert_equal(expected, filtered)
158
- end
313
+ def test_filter_quoted_record
314
+ messages = [
315
+ {'host' => '66.102.3.80', 'message' => 'valid ip'}
316
+ ]
317
+ expected = [
318
+ {
319
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
320
+ 'location_properties' => {
321
+ 'country_code' => 'US',
322
+ 'lat' => 37.419200000000004,
323
+ 'lon' => -122.0574
324
+ },
325
+ 'location_string' => '37.419200000000004,-122.0574',
326
+ 'location_string2' => 'US',
327
+ 'location_array' => [-122.0574, 37.419200000000004],
328
+ 'location_array2' => [-122.0574, 37.419200000000004],
329
+ 'peculiar_pattern' => '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}'
330
+ }
331
+ ]
332
+ filtered = filter(config_quoted_record, messages)
333
+ assert_equal(expected, filtered)
334
+ end
159
335
 
160
- def test_filter_with_skip_unknown_address
161
- config = %[
162
- geoip_lookup_key host
163
- <record>
164
- geoip_city ${city['host']}
165
- geopoint [${longitude['host']}, ${latitude['host']}]
166
- </record>
167
- skip_adding_null_record true
168
- ]
169
- # 203.0.113.1 is a test address described in RFC5737
170
- messages = [
171
- {'host' => '203.0.113.1', 'message' => 'invalid ip'},
172
- {'host' => '0', 'message' => 'invalid ip'},
173
- {'host' => '8.8.8.8', 'message' => 'google public dns'}
174
- ]
175
- expected = [
176
- {'host' => '203.0.113.1', 'message' => 'invalid ip'},
177
- {'host' => '0', 'message' => 'invalid ip'},
178
- {'host' => '8.8.8.8', 'message' => 'google public dns',
179
- 'geoip_city' => 'Mountain View', 'geopoint' => [-122.08380126953125, 37.38600158691406]}
180
- ]
181
- filtered = filter(config, messages)
182
- assert_equal(expected, filtered)
183
- end
336
+ def test_filter_v1_config_compatibility
337
+ messages = [
338
+ {'host' => '66.102.3.80', 'message' => 'valid ip'}
339
+ ]
340
+ expected = [
341
+ {
342
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
343
+ 'location_properties' => {
344
+ 'country_code' => 'US',
345
+ 'lat' => 37.419200000000004,
346
+ 'lon' => -122.0574
347
+ },
348
+ 'location_string' => '37.419200000000004,-122.0574',
349
+ 'location_string2' => 'US',
350
+ 'location_array' => [-122.0574, 37.419200000000004],
351
+ 'location_array2' => [-122.0574, 37.419200000000004],
352
+ 'peculiar_pattern' => '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}'
353
+ }
354
+ ]
355
+ filtered = filter(config_quoted_record, messages, true)
356
+ assert_equal(expected, filtered)
357
+ end
184
358
 
185
- def test_filter_multiple_key
186
- config = %[
187
- geoip_lookup_key from.ip, to.ip
188
- enable_key_city from_city, to_city
189
- ]
190
- messages = [
191
- {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
192
- {'message' => 'missing field'}
193
- ]
194
- expected = [
195
- {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'},
196
- 'from_city' => 'Mountain View', 'to_city' => 'Tokorozawa'},
197
- {'message' => 'missing field', 'from_city' => nil, 'to_city' => nil}
198
- ]
199
- filtered = filter(config, messages)
200
- assert_equal(expected, filtered)
359
+ def test_filter_multiline_v1_config
360
+ config = %[
361
+ backend_library geoip2_c
362
+ geoip_lookup_key host
363
+ <record>
364
+ location_properties {
365
+ "city": "${city.names.en['host']}",
366
+ "country_code": "${country.iso_code['host']}",
367
+ "latitude": "${location.latitude['host']}",
368
+ "longitude": "${location.longitude['host']}"
369
+ }
370
+ </record>
371
+ ]
372
+ messages = [
373
+ { 'host' => '66.102.3.80', 'message' => 'valid ip' }
374
+ ]
375
+ expected = [
376
+ {
377
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
378
+ "location_properties" => {
379
+ "city" => "Mountain View",
380
+ "country_code" => "US",
381
+ "latitude" => 37.419200000000004,
382
+ "longitude" => -122.0574
383
+ }
384
+ }
385
+ ]
386
+ filtered = filter(config, messages, true)
387
+ assert_equal(expected, filtered)
388
+ end
201
389
  end
202
390
 
203
- def test_filter_multiple_key_multiple_record
204
- config = %[
205
- geoip_lookup_key from.ip, to.ip
206
- enable_key_city from_city, to_city
207
- enable_key_country_name from_country, to_country
208
- ]
209
- messages = [
210
- {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
211
- {'from' => {'ip' => '66.102.3.80'}},
212
- {'message' => 'missing field'}
213
- ]
214
- expected = [
215
- {
216
- 'from' => {'ip' => '66.102.3.80'},
217
- 'to' => {'ip' => '125.54.15.42'},
218
- 'from_city' => 'Mountain View',
219
- 'from_country' => 'United States',
220
- 'to_city' => 'Tokorozawa',
221
- 'to_country' => 'Japan'
222
- },
223
- {
224
- 'from' => {'ip' => '66.102.3.80'},
225
- 'from_city' => 'Mountain View',
226
- 'from_country' => 'United States',
227
- 'to_city' => nil,
228
- 'to_country' => nil
229
- },
230
- {
231
- 'message' => 'missing field',
232
- 'from_city' => nil,
233
- 'from_country' => nil,
234
- 'to_city' => nil,
235
- 'to_country' => nil
236
- }
237
- ]
238
- filtered = filter(config, messages)
239
- assert_equal(expected, filtered)
240
- end
391
+ sub_test_case "geoip2_compat" do
392
+ def test_filter_with_dot_key
393
+ config = %[
394
+ backend_library geoip2_compat
395
+ geoip_lookup_key ip.origin, ip.dest
396
+ <record>
397
+ origin_country ${country_code['ip.origin']}
398
+ dest_country ${country_code['ip.dest']}
399
+ </record>
400
+ ]
401
+ messages = [
402
+ {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'}
403
+ ]
404
+ expected = [
405
+ {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8',
406
+ 'origin_country' => 'US', 'dest_country' => 'US'}
407
+ ]
408
+ filtered = filter(config, messages)
409
+ assert_equal(expected, filtered)
410
+ end
241
411
 
242
- def test_filter_record_directive
243
- config = %[
244
- geoip_lookup_key from.ip
245
- <record>
246
- from_city ${city['from.ip']}
247
- from_country ${country_name['from.ip']}
248
- latitude ${latitude['from.ip']}
249
- longitude ${longitude['from.ip']}
250
- float_concat ${latitude['from.ip']},${longitude['from.ip']}
251
- float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
252
- float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
253
- string_concat ${city['from.ip']},${country_name['from.ip']}
254
- string_array [${city['from.ip']}, ${country_name['from.ip']}]
255
- string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
256
- unknown_city ${city['unknown_key']}
257
- undefined ${city['undefined']}
258
- broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
259
- broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
260
- </record>
261
- ]
262
- messages = [
263
- { 'from' => {'ip' => '66.102.3.80'} },
264
- { 'message' => 'missing field' },
265
- ]
266
- expected = [
267
- {
268
- 'from' => {'ip' => '66.102.3.80'},
269
- 'from_city' => 'Mountain View',
270
- 'from_country' => 'United States',
271
- 'latitude' => 37.4192008972168,
272
- 'longitude' => -122.05740356445312,
273
- 'float_concat' => '37.4192008972168,-122.05740356445312',
274
- 'float_array' => [-122.05740356445312, 37.4192008972168],
275
- 'float_nest' => { 'lat' => 37.4192008972168, 'lon' => -122.05740356445312 },
276
- 'string_concat' => 'Mountain View,United States',
277
- 'string_array' => ["Mountain View", "United States"],
278
- 'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
279
- 'unknown_city' => nil,
280
- 'undefined' => nil,
281
- 'broken_array1' => [-122.05740356445312, nil],
282
- 'broken_array2' => [nil, nil]
283
- },
284
- {
285
- 'message' => 'missing field',
286
- 'from_city' => nil,
287
- 'from_country' => nil,
288
- 'latitude' => nil,
289
- 'longitude' => nil,
290
- 'float_concat' => ',',
291
- 'float_array' => [nil, nil],
292
- 'float_nest' => { 'lat' => nil, 'lon' => nil },
293
- 'string_concat' => ',',
294
- 'string_array' => [nil, nil],
295
- 'string_nest' => { "city" => nil, "country_name" => nil },
296
- 'unknown_city' => nil,
297
- 'undefined' => nil,
298
- 'broken_array1' => [nil, nil],
299
- 'broken_array2' => [nil, nil]
300
- },
301
- ]
302
- filtered = filter(config, messages)
303
- # test-unit cannot calculate diff between large Array
304
- assert_equal(expected[0], filtered[0])
305
- assert_equal(expected[1], filtered[1])
306
- end
412
+ def test_filter_with_unknown_address
413
+ config = %[
414
+ backend_library geoip2_compat
415
+ geoip_lookup_key host
416
+ <record>
417
+ geoip_city ${city['host']}
418
+ geopoint [${longitude['host']}, ${latitude['host']}]
419
+ </record>
420
+ skip_adding_null_record false
421
+ ]
422
+ # 203.0.113.1 is a test address described in RFC5737
423
+ messages = [
424
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
425
+ {'host' => '0', 'message' => 'invalid ip'}
426
+ ]
427
+ expected = [
428
+ {'host' => '203.0.113.1', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
429
+ {'host' => '0', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]}
430
+ ]
431
+ filtered = filter(config, messages)
432
+ assert_equal(expected, filtered)
433
+ end
307
434
 
308
- def test_filter_record_directive_multiple_record
309
- config = %[
310
- geoip_lookup_key from.ip, to.ip
311
- <record>
312
- from_city ${city['from.ip']}
313
- to_city ${city['to.ip']}
314
- from_country ${country_name['from.ip']}
315
- to_country ${country_name['to.ip']}
316
- string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
317
- </record>
318
- ]
319
- messages = [
320
- {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
321
- {'message' => 'missing field'}
322
- ]
323
- expected = [
324
- {
325
- 'from' => { 'ip' => '66.102.3.80' },
326
- 'to' => { 'ip' => '125.54.15.42' },
327
- 'from_city' => 'Mountain View',
328
- 'from_country' => 'United States',
329
- 'to_city' => 'Tokorozawa',
330
- 'to_country' => 'Japan',
331
- 'string_array' => ['United States', 'Japan']
332
- },
333
- {
334
- 'message' => 'missing field',
335
- 'from_city' => nil,
336
- 'from_country' => nil,
337
- 'to_city' => nil,
338
- 'to_country' => nil,
339
- 'string_array' => [nil, nil]
340
- }
341
- ]
342
- filtered = filter(config, messages)
343
- assert_equal(expected, filtered)
344
- end
435
+ def test_filter_with_skip_unknown_address
436
+ config = %[
437
+ backend_library geoip2_compat
438
+ geoip_lookup_key host
439
+ <record>
440
+ geoip_city ${city['host']}
441
+ geopoint [${longitude['host']}, ${latitude['host']}]
442
+ </record>
443
+ skip_adding_null_record true
444
+ ]
445
+ # 203.0.113.1 is a test address described in RFC5737
446
+ messages = [
447
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
448
+ {'host' => '0', 'message' => 'invalid ip'},
449
+ {'host' => '8.8.8.8', 'message' => 'google public dns'}
450
+ ]
451
+ expected = [
452
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
453
+ {'host' => '0', 'message' => 'invalid ip'},
454
+ {'host' => '8.8.8.8', 'message' => 'google public dns',
455
+ 'geoip_city' => 'Mountain View', 'geopoint' => [-122.0838, 37.386]}
456
+ ]
457
+ filtered = filter(config, messages)
458
+ assert_equal(expected, filtered)
459
+ end
345
460
 
346
- CONFIG_QUOTED_RECORD = %[
347
- geoip_lookup_key host
348
- <record>
349
- location_properties '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
350
- location_string ${latitude['host']},${longitude['host']}
351
- location_string2 ${country_code["host"]}
352
- location_array "[${longitude['host']},${latitude['host']}]"
353
- location_array2 '[${longitude["host"]},${latitude["host"]}]'
354
- peculiar_pattern '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
355
- </record>
356
- remove_tag_prefix input.
357
- tag geoip.${tag}
358
- ]
359
-
360
- def test_filter_quoted_record
361
- messages = [
362
- {'host' => '66.102.3.80', 'message' => 'valid ip'}
363
- ]
364
- expected = [
365
- {
366
- 'host' => '66.102.3.80', 'message' => 'valid ip',
367
- 'location_properties' => {
368
- 'country_code' => 'US',
369
- 'lat' => 37.4192008972168,
370
- 'lon' => -122.05740356445312
461
+ def test_filter_record_directive
462
+ config = %[
463
+ backend_library geoip2_compat
464
+ geoip_lookup_key from.ip
465
+ <record>
466
+ from_city ${city['from.ip']}
467
+ from_country ${country_name['from.ip']}
468
+ latitude ${latitude['from.ip']}
469
+ longitude ${longitude['from.ip']}
470
+ float_concat ${latitude['from.ip']},${longitude['from.ip']}
471
+ float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
472
+ float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
473
+ string_concat ${city['from.ip']},${country_name['from.ip']}
474
+ string_array [${city['from.ip']}, ${country_name['from.ip']}]
475
+ string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
476
+ unknown_city ${city['unknown_key']}
477
+ undefined ${city['undefined']}
478
+ broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
479
+ broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
480
+ </record>
481
+ ]
482
+ messages = [
483
+ { 'from' => {'ip' => '66.102.3.80'} },
484
+ { 'message' => 'missing field' },
485
+ ]
486
+ expected = [
487
+ {
488
+ 'from' => {'ip' => '66.102.3.80'},
489
+ 'from_city' => 'Mountain View',
490
+ 'from_country' => 'United States',
491
+ 'latitude' => 37.419200000000004,
492
+ 'longitude' => -122.0574,
493
+ 'float_concat' => '37.419200000000004,-122.0574',
494
+ 'float_array' => [-122.0574, 37.419200000000004],
495
+ 'float_nest' => { 'lat' => 37.4192000000000004, 'lon' => -122.0574 },
496
+ 'string_concat' => 'Mountain View,United States',
497
+ 'string_array' => ["Mountain View", "United States"],
498
+ 'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
499
+ 'unknown_city' => nil,
500
+ 'undefined' => nil,
501
+ 'broken_array1' => [-122.0574, nil],
502
+ 'broken_array2' => [nil, nil]
371
503
  },
372
- 'location_string' => '37.4192008972168,-122.05740356445312',
373
- 'location_string2' => 'US',
374
- 'location_array' => [-122.05740356445312, 37.4192008972168],
375
- 'location_array2' => [-122.05740356445312, 37.4192008972168],
376
- 'peculiar_pattern' => '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}'
377
- }
378
- ]
379
- filtered = filter(CONFIG_QUOTED_RECORD, messages)
380
- assert_equal(expected, filtered)
381
- end
504
+ {
505
+ 'message' => 'missing field',
506
+ 'from_city' => nil,
507
+ 'from_country' => nil,
508
+ 'latitude' => nil,
509
+ 'longitude' => nil,
510
+ 'float_concat' => ',',
511
+ 'float_array' => [nil, nil],
512
+ 'float_nest' => { 'lat' => nil, 'lon' => nil },
513
+ 'string_concat' => ',',
514
+ 'string_array' => [nil, nil],
515
+ 'string_nest' => { "city" => nil, "country_name" => nil },
516
+ 'unknown_city' => nil,
517
+ 'undefined' => nil,
518
+ 'broken_array1' => [nil, nil],
519
+ 'broken_array2' => [nil, nil]
520
+ },
521
+ ]
522
+ filtered = filter(config, messages)
523
+ # test-unit cannot calculate diff between large Array
524
+ assert_equal(expected[0], filtered[0])
525
+ assert_equal(expected[1], filtered[1])
526
+ end
382
527
 
383
- def test_filter_v1_config_compatibility
384
- messages = [
385
- {'host' => '66.102.3.80', 'message' => 'valid ip'}
386
- ]
387
- expected = [
388
- {
389
- 'host' => '66.102.3.80', 'message' => 'valid ip',
390
- 'location_properties' => {
391
- 'country_code' => 'US',
392
- 'lat' => 37.4192008972168,
393
- 'lon' => -122.05740356445312
528
+ def test_filter_record_directive_multiple_record
529
+ config = %[
530
+ backend_library geoip2_compat
531
+ geoip_lookup_key from.ip, to.ip
532
+ <record>
533
+ from_city ${city['from.ip']}
534
+ to_city ${city['to.ip']}
535
+ from_country ${country_name['from.ip']}
536
+ to_country ${country_name['to.ip']}
537
+ string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
538
+ </record>
539
+ ]
540
+ messages = [
541
+ {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
542
+ {'message' => 'missing field'}
543
+ ]
544
+ expected = [
545
+ {
546
+ 'from' => { 'ip' => '66.102.3.80' },
547
+ 'to' => { 'ip' => '125.54.15.42' },
548
+ 'from_city' => 'Mountain View',
549
+ 'from_country' => 'United States',
550
+ 'to_city' => 'Tokorozawa',
551
+ 'to_country' => 'Japan',
552
+ 'string_array' => ['United States', 'Japan']
394
553
  },
395
- 'location_string' => '37.4192008972168,-122.05740356445312',
396
- 'location_string2' => 'US',
397
- 'location_array' => [-122.05740356445312, 37.4192008972168],
398
- 'location_array2' => [-122.05740356445312, 37.4192008972168],
399
- 'peculiar_pattern' => '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}'
400
- }
401
- ]
402
- filtered = filter(CONFIG_QUOTED_RECORD, messages, true)
403
- assert_equal(expected, filtered)
554
+ {
555
+ 'message' => 'missing field',
556
+ 'from_city' => nil,
557
+ 'from_country' => nil,
558
+ 'to_city' => nil,
559
+ 'to_country' => nil,
560
+ 'string_array' => [nil, nil]
561
+ }
562
+ ]
563
+ filtered = filter(config, messages)
564
+ assert_equal(expected, filtered)
565
+ end
566
+
567
+ def config_quoted_record
568
+ %[
569
+ backend_library geoip2_compat
570
+ geoip_lookup_key host
571
+ <record>
572
+ location_properties '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
573
+ location_string ${latitude['host']},${longitude['host']}
574
+ location_string2 ${country_code["host"]}
575
+ location_array "[${longitude['host']},${latitude['host']}]"
576
+ location_array2 '[${longitude["host"]},${latitude["host"]}]'
577
+ peculiar_pattern '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
578
+ </record>
579
+ ]
580
+ end
581
+
582
+ def test_filter_quoted_record
583
+ messages = [
584
+ {'host' => '66.102.3.80', 'message' => 'valid ip'}
585
+ ]
586
+ expected = [
587
+ {
588
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
589
+ 'location_properties' => {
590
+ 'country_code' => 'US',
591
+ 'lat' => 37.419200000000004,
592
+ 'lon' => -122.0574
593
+ },
594
+ 'location_string' => '37.419200000000004,-122.0574',
595
+ 'location_string2' => 'US',
596
+ 'location_array' => [-122.0574, 37.419200000000004],
597
+ 'location_array2' => [-122.0574, 37.419200000000004],
598
+ 'peculiar_pattern' => '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}'
599
+ }
600
+ ]
601
+ filtered = filter(config_quoted_record, messages)
602
+ assert_equal(expected, filtered)
603
+ end
604
+
605
+ def test_filter_v1_config_compatibility
606
+ messages = [
607
+ {'host' => '66.102.3.80', 'message' => 'valid ip'}
608
+ ]
609
+ expected = [
610
+ {
611
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
612
+ 'location_properties' => {
613
+ 'country_code' => 'US',
614
+ 'lat' => 37.419200000000004,
615
+ 'lon' => -122.0574
616
+ },
617
+ 'location_string' => '37.419200000000004,-122.0574',
618
+ 'location_string2' => 'US',
619
+ 'location_array' => [-122.0574, 37.419200000000004],
620
+ 'location_array2' => [-122.0574, 37.419200000000004],
621
+ 'peculiar_pattern' => '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}'
622
+ }
623
+ ]
624
+ filtered = filter(config_quoted_record, messages, true)
625
+ assert_equal(expected, filtered)
626
+ end
627
+
628
+ def test_filter_multiline_v1_config
629
+ config = %[
630
+ backend_library geoip2_compat
631
+ geoip_lookup_key host
632
+ <record>
633
+ location_properties {
634
+ "city": "${city['host']}",
635
+ "country_code": "${country_code['host']}",
636
+ "latitude": "${latitude['host']}",
637
+ "longitude": "${longitude['host']}"
638
+ }
639
+ </record>
640
+ ]
641
+ messages = [
642
+ { 'host' => '66.102.3.80', 'message' => 'valid ip' }
643
+ ]
644
+ expected = [
645
+ {
646
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
647
+ "location_properties" => {
648
+ "city" => "Mountain View",
649
+ "country_code" => "US",
650
+ "latitude" => 37.419200000000004,
651
+ "longitude" => -122.0574
652
+ }
653
+ }
654
+ ]
655
+ filtered = filter(config, messages, true)
656
+ assert_equal(expected, filtered)
657
+ end
404
658
  end
405
659
 
406
- def test_filter_multiline_v1_config
407
- config = %[
408
- geoip_lookup_key host
409
- <record>
410
- location_properties {
411
- "city": "${city['host']}",
412
- "country_code": "${country_code['host']}",
413
- "latitude": "${latitude['host']}",
414
- "longitude": "${longitude['host']}"
660
+ sub_test_case "geoip legacy" do
661
+ def test_filter
662
+ config = %[
663
+ geoip_lookup_key host
664
+ enable_key_city geoip_city
665
+ ]
666
+ messages = [
667
+ {'host' => '66.102.3.80', 'message' => 'valid ip'},
668
+ {'message' => 'missing field'},
669
+ ]
670
+ expected = [
671
+ {'host' => '66.102.3.80', 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
672
+ {'message' => 'missing field', 'geoip_city' => nil},
673
+ ]
674
+ filtered = filter(config, messages)
675
+ assert_equal(expected, filtered)
676
+ end
677
+
678
+ def test_filter_with_dot_key
679
+ config = %[
680
+ geoip_lookup_key ip.origin, ip.dest
681
+ <record>
682
+ origin_country ${country_code['ip.origin']}
683
+ dest_country ${country_code['ip.dest']}
684
+ </record>
685
+ ]
686
+ messages = [
687
+ {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'}
688
+ ]
689
+ expected = [
690
+ {'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8',
691
+ 'origin_country' => 'US', 'dest_country' => 'US'}
692
+ ]
693
+ filtered = filter(config, messages)
694
+ assert_equal(expected, filtered)
695
+ end
696
+
697
+ def test_filter_nested_attr
698
+ config = %[
699
+ geoip_lookup_key host.ip
700
+ enable_key_city geoip_city
701
+ ]
702
+ messages = [
703
+ {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'},
704
+ {'message' => 'missing field'}
705
+ ]
706
+ expected = [
707
+ {'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
708
+ {'message' => 'missing field', 'geoip_city' => nil}
709
+ ]
710
+ filtered = filter(config, messages)
711
+ assert_equal(expected, filtered)
712
+ end
713
+
714
+ def test_filter_with_unknown_address
715
+ config = %[
716
+ geoip_lookup_key host
717
+ <record>
718
+ geoip_city ${city['host']}
719
+ geopoint [${longitude['host']}, ${latitude['host']}]
720
+ </record>
721
+ skip_adding_null_record false
722
+ ]
723
+ # 203.0.113.1 is a test address described in RFC5737
724
+ messages = [
725
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
726
+ {'host' => '0', 'message' => 'invalid ip'}
727
+ ]
728
+ expected = [
729
+ {'host' => '203.0.113.1', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
730
+ {'host' => '0', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]}
731
+ ]
732
+ filtered = filter(config, messages)
733
+ assert_equal(expected, filtered)
734
+ end
735
+
736
+ def test_filter_with_skip_unknown_address
737
+ config = %[
738
+ geoip_lookup_key host
739
+ <record>
740
+ geoip_city ${city['host']}
741
+ geopoint [${longitude['host']}, ${latitude['host']}]
742
+ </record>
743
+ skip_adding_null_record true
744
+ ]
745
+ # 203.0.113.1 is a test address described in RFC5737
746
+ messages = [
747
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
748
+ {'host' => '0', 'message' => 'invalid ip'},
749
+ {'host' => '8.8.8.8', 'message' => 'google public dns'}
750
+ ]
751
+ expected = [
752
+ {'host' => '203.0.113.1', 'message' => 'invalid ip'},
753
+ {'host' => '0', 'message' => 'invalid ip'},
754
+ {'host' => '8.8.8.8', 'message' => 'google public dns',
755
+ 'geoip_city' => 'Mountain View', 'geopoint' => [-122.08380126953125, 37.38600158691406]}
756
+ ]
757
+ filtered = filter(config, messages)
758
+ assert_equal(expected, filtered)
759
+ end
760
+
761
+ def test_filter_multiple_key
762
+ config = %[
763
+ geoip_lookup_key from.ip, to.ip
764
+ enable_key_city from_city, to_city
765
+ ]
766
+ messages = [
767
+ {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
768
+ {'message' => 'missing field'}
769
+ ]
770
+ expected = [
771
+ {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'},
772
+ 'from_city' => 'Mountain View', 'to_city' => 'Tokorozawa'},
773
+ {'message' => 'missing field', 'from_city' => nil, 'to_city' => nil}
774
+ ]
775
+ filtered = filter(config, messages)
776
+ assert_equal(expected, filtered)
777
+ end
778
+
779
+ def test_filter_multiple_key_multiple_record
780
+ config = %[
781
+ geoip_lookup_key from.ip, to.ip
782
+ enable_key_city from_city, to_city
783
+ enable_key_country_name from_country, to_country
784
+ ]
785
+ messages = [
786
+ {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
787
+ {'from' => {'ip' => '66.102.3.80'}},
788
+ {'message' => 'missing field'}
789
+ ]
790
+ expected = [
791
+ {
792
+ 'from' => {'ip' => '66.102.3.80'},
793
+ 'to' => {'ip' => '125.54.15.42'},
794
+ 'from_city' => 'Mountain View',
795
+ 'from_country' => 'United States',
796
+ 'to_city' => 'Tokorozawa',
797
+ 'to_country' => 'Japan'
798
+ },
799
+ {
800
+ 'from' => {'ip' => '66.102.3.80'},
801
+ 'from_city' => 'Mountain View',
802
+ 'from_country' => 'United States',
803
+ 'to_city' => nil,
804
+ 'to_country' => nil
805
+ },
806
+ {
807
+ 'message' => 'missing field',
808
+ 'from_city' => nil,
809
+ 'from_country' => nil,
810
+ 'to_city' => nil,
811
+ 'to_country' => nil
812
+ }
813
+ ]
814
+ filtered = filter(config, messages)
815
+ assert_equal(expected, filtered)
816
+ end
817
+
818
+ def test_filter_record_directive
819
+ config = %[
820
+ geoip_lookup_key from.ip
821
+ <record>
822
+ from_city ${city['from.ip']}
823
+ from_country ${country_name['from.ip']}
824
+ latitude ${latitude['from.ip']}
825
+ longitude ${longitude['from.ip']}
826
+ float_concat ${latitude['from.ip']},${longitude['from.ip']}
827
+ float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
828
+ float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
829
+ string_concat ${city['from.ip']},${country_name['from.ip']}
830
+ string_array [${city['from.ip']}, ${country_name['from.ip']}]
831
+ string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
832
+ unknown_city ${city['unknown_key']}
833
+ undefined ${city['undefined']}
834
+ broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
835
+ broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
836
+ </record>
837
+ ]
838
+ messages = [
839
+ { 'from' => {'ip' => '66.102.3.80'} },
840
+ { 'message' => 'missing field' },
841
+ ]
842
+ expected = [
843
+ {
844
+ 'from' => {'ip' => '66.102.3.80'},
845
+ 'from_city' => 'Mountain View',
846
+ 'from_country' => 'United States',
847
+ 'latitude' => 37.4192008972168,
848
+ 'longitude' => -122.05740356445312,
849
+ 'float_concat' => '37.4192008972168,-122.05740356445312',
850
+ 'float_array' => [-122.05740356445312, 37.4192008972168],
851
+ 'float_nest' => { 'lat' => 37.4192008972168, 'lon' => -122.05740356445312 },
852
+ 'string_concat' => 'Mountain View,United States',
853
+ 'string_array' => ["Mountain View", "United States"],
854
+ 'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
855
+ 'unknown_city' => nil,
856
+ 'undefined' => nil,
857
+ 'broken_array1' => [-122.05740356445312, nil],
858
+ 'broken_array2' => [nil, nil]
859
+ },
860
+ {
861
+ 'message' => 'missing field',
862
+ 'from_city' => nil,
863
+ 'from_country' => nil,
864
+ 'latitude' => nil,
865
+ 'longitude' => nil,
866
+ 'float_concat' => ',',
867
+ 'float_array' => [nil, nil],
868
+ 'float_nest' => { 'lat' => nil, 'lon' => nil },
869
+ 'string_concat' => ',',
870
+ 'string_array' => [nil, nil],
871
+ 'string_nest' => { "city" => nil, "country_name" => nil },
872
+ 'unknown_city' => nil,
873
+ 'undefined' => nil,
874
+ 'broken_array1' => [nil, nil],
875
+ 'broken_array2' => [nil, nil]
876
+ },
877
+ ]
878
+ filtered = filter(config, messages)
879
+ # test-unit cannot calculate diff between large Array
880
+ assert_equal(expected[0], filtered[0])
881
+ assert_equal(expected[1], filtered[1])
882
+ end
883
+
884
+ def test_filter_record_directive_multiple_record
885
+ config = %[
886
+ geoip_lookup_key from.ip, to.ip
887
+ <record>
888
+ from_city ${city['from.ip']}
889
+ to_city ${city['to.ip']}
890
+ from_country ${country_name['from.ip']}
891
+ to_country ${country_name['to.ip']}
892
+ string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
893
+ </record>
894
+ ]
895
+ messages = [
896
+ {'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
897
+ {'message' => 'missing field'}
898
+ ]
899
+ expected = [
900
+ {
901
+ 'from' => { 'ip' => '66.102.3.80' },
902
+ 'to' => { 'ip' => '125.54.15.42' },
903
+ 'from_city' => 'Mountain View',
904
+ 'from_country' => 'United States',
905
+ 'to_city' => 'Tokorozawa',
906
+ 'to_country' => 'Japan',
907
+ 'string_array' => ['United States', 'Japan']
908
+ },
909
+ {
910
+ 'message' => 'missing field',
911
+ 'from_city' => nil,
912
+ 'from_country' => nil,
913
+ 'to_city' => nil,
914
+ 'to_country' => nil,
915
+ 'string_array' => [nil, nil]
916
+ }
917
+ ]
918
+ filtered = filter(config, messages)
919
+ assert_equal(expected, filtered)
920
+ end
921
+
922
+ def config_quoted_record
923
+ %[
924
+ geoip_lookup_key host
925
+ <record>
926
+ location_properties '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
927
+ location_string ${latitude['host']},${longitude['host']}
928
+ location_string2 ${country_code["host"]}
929
+ location_array "[${longitude['host']},${latitude['host']}]"
930
+ location_array2 '[${longitude["host"]},${latitude["host"]}]'
931
+ peculiar_pattern '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
932
+ </record>
933
+ ]
934
+ end
935
+
936
+ def test_filter_quoted_record
937
+ messages = [
938
+ {'host' => '66.102.3.80', 'message' => 'valid ip'}
939
+ ]
940
+ expected = [
941
+ {
942
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
943
+ 'location_properties' => {
944
+ 'country_code' => 'US',
945
+ 'lat' => 37.4192008972168,
946
+ 'lon' => -122.05740356445312
947
+ },
948
+ 'location_string' => '37.4192008972168,-122.05740356445312',
949
+ 'location_string2' => 'US',
950
+ 'location_array' => [-122.05740356445312, 37.4192008972168],
951
+ 'location_array2' => [-122.05740356445312, 37.4192008972168],
952
+ 'peculiar_pattern' => '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}'
415
953
  }
416
- </record>
417
- ]
418
- messages = [
419
- { 'host' => '66.102.3.80', 'message' => 'valid ip' }
420
- ]
421
- expected = [
422
- {
423
- 'host' => '66.102.3.80', 'message' => 'valid ip',
424
- "location_properties" => {
425
- "city" => "Mountain View",
426
- "country_code" => "US",
427
- "latitude" => 37.4192008972168,
428
- "longitude" => -122.05740356445312
954
+ ]
955
+ filtered = filter(config_quoted_record, messages)
956
+ assert_equal(expected, filtered)
957
+ end
958
+
959
+ def test_filter_v1_config_compatibility
960
+ messages = [
961
+ {'host' => '66.102.3.80', 'message' => 'valid ip'}
962
+ ]
963
+ expected = [
964
+ {
965
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
966
+ 'location_properties' => {
967
+ 'country_code' => 'US',
968
+ 'lat' => 37.4192008972168,
969
+ 'lon' => -122.05740356445312
970
+ },
971
+ 'location_string' => '37.4192008972168,-122.05740356445312',
972
+ 'location_string2' => 'US',
973
+ 'location_array' => [-122.05740356445312, 37.4192008972168],
974
+ 'location_array2' => [-122.05740356445312, 37.4192008972168],
975
+ 'peculiar_pattern' => '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}'
429
976
  }
430
- }
431
- ]
432
- filtered = filter(config, messages, true)
433
- assert_equal(expected, filtered)
977
+ ]
978
+ filtered = filter(config_quoted_record, messages, true)
979
+ assert_equal(expected, filtered)
980
+ end
981
+
982
+ def test_filter_multiline_v1_config
983
+ config = %[
984
+ geoip_lookup_key host
985
+ <record>
986
+ location_properties {
987
+ "city": "${city['host']}",
988
+ "country_code": "${country_code['host']}",
989
+ "latitude": "${latitude['host']}",
990
+ "longitude": "${longitude['host']}"
991
+ }
992
+ </record>
993
+ ]
994
+ messages = [
995
+ { 'host' => '66.102.3.80', 'message' => 'valid ip' }
996
+ ]
997
+ expected = [
998
+ {
999
+ 'host' => '66.102.3.80', 'message' => 'valid ip',
1000
+ "location_properties" => {
1001
+ "city" => "Mountain View",
1002
+ "country_code" => "US",
1003
+ "latitude" => 37.4192008972168,
1004
+ "longitude" => -122.05740356445312
1005
+ }
1006
+ }
1007
+ ]
1008
+ filtered = filter(config, messages, true)
1009
+ assert_equal(expected, filtered)
1010
+ end
434
1011
  end
435
1012
  end
436
1013