fluent-plugin-geoip 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +31 -20
- data/Appraisals +0 -4
- data/README.md +50 -11
- data/docker-compose.yml +31 -0
- data/dockerfiles/Dockerfile-ruby2.1 +8 -0
- data/dockerfiles/Dockerfile-ruby2.2 +8 -0
- data/dockerfiles/Dockerfile-ruby2.3 +8 -0
- data/dockerfiles/Dockerfile-ruby2.4 +8 -0
- data/fluent-plugin-geoip.gemspec +4 -2
- data/lib/fluent/plugin/filter_geoip.rb +4 -6
- data/lib/fluent/plugin/geoip.rb +47 -18
- data/lib/fluent/plugin/out_geoip.rb +3 -15
- data/test/plugin/test_filter_geoip.rb +957 -380
- data/test/plugin/test_out_geoip.rb +1035 -385
- metadata +49 -3
- data/gemfiles/fluentd_v0.10.gemfile +0 -8
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
require 'fluent/mixin/rewrite_tag_name'
|
|
2
|
+
require 'fluent/plugin/geoip'
|
|
2
3
|
|
|
3
4
|
class Fluent::GeoipOutput < Fluent::BufferedOutput
|
|
4
5
|
Fluent::Plugin.register_output('geoip', self)
|
|
5
6
|
|
|
6
7
|
config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
|
|
8
|
+
config_param :geoip2_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
|
|
7
9
|
config_param :geoip_lookup_key, :string, :default => 'host'
|
|
8
10
|
config_param :tag, :string, :default => nil
|
|
9
11
|
config_param :skip_adding_null_record, :bool, :default => false
|
|
@@ -18,21 +20,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
|
|
|
18
20
|
config_param :flush_interval, :time, :default => 0
|
|
19
21
|
config_param :log_level, :string, :default => 'warn'
|
|
20
22
|
|
|
21
|
-
|
|
22
|
-
unless method_defined?(:log)
|
|
23
|
-
define_method("log") { $log }
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
# To support Fluentd v0.10.57 or earlier
|
|
27
|
-
unless method_defined?(:router)
|
|
28
|
-
define_method("router") { Fluent::Engine }
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
def initialize
|
|
32
|
-
require 'fluent/plugin/geoip'
|
|
33
|
-
|
|
34
|
-
super
|
|
35
|
-
end
|
|
23
|
+
config_param :backend_library, :enum, :list => Fluent::GeoIP::BACKEND_LIBRARIES, :default => :geoip
|
|
36
24
|
|
|
37
25
|
def configure(conf)
|
|
38
26
|
super
|
|
@@ -7,14 +7,7 @@ class GeoipFilterTest < Test::Unit::TestCase
|
|
|
7
7
|
@time = Fluent::Engine.now
|
|
8
8
|
end
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
geoip_lookup_key host
|
|
12
|
-
enable_key_city geoip_city
|
|
13
|
-
remove_tag_prefix input.
|
|
14
|
-
tag geoip.${tag}
|
|
15
|
-
]
|
|
16
|
-
|
|
17
|
-
def create_driver(conf=CONFIG, tag='test', use_v1=false)
|
|
10
|
+
def create_driver(conf='', tag='test', use_v1=false)
|
|
18
11
|
Fluent::Test::FilterTestDriver.new(Fluent::GeoipFilter, tag).configure(conf, use_v1)
|
|
19
12
|
end
|
|
20
13
|
|
|
@@ -29,408 +22,992 @@ class GeoipFilterTest < Test::Unit::TestCase
|
|
|
29
22
|
filtered.map {|m| m[2] }
|
|
30
23
|
end
|
|
31
24
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
# multiple key config
|
|
47
|
-
d = create_driver %[
|
|
48
|
-
geoip_lookup_key from.ip, to.ip
|
|
49
|
-
enable_key_city from_city, to_city
|
|
50
|
-
remove_tag_prefix input.
|
|
51
|
-
tag geoip.${tag}
|
|
52
|
-
]
|
|
53
|
-
assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
|
|
54
|
-
|
|
55
|
-
# multiple key config (bad configure)
|
|
56
|
-
assert_raise(Fluent::ConfigError) {
|
|
25
|
+
sub_test_case "configure" do
|
|
26
|
+
test "empty" do
|
|
27
|
+
assert_nothing_raised {
|
|
28
|
+
create_driver('')
|
|
29
|
+
}
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
test "missing required parameters" do
|
|
33
|
+
assert_raise(Fluent::ConfigError) {
|
|
34
|
+
create_driver('enable_key_cities')
|
|
35
|
+
}
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
test "minimum" do
|
|
57
39
|
d = create_driver %[
|
|
58
|
-
|
|
59
|
-
enable_key_city from_city
|
|
60
|
-
enable_key_region from_region
|
|
61
|
-
remove_tag_prefix input.
|
|
62
|
-
tag geoip.${tag}
|
|
40
|
+
enable_key_city geoip_city
|
|
63
41
|
]
|
|
64
|
-
|
|
42
|
+
assert_equal 'geoip_city', d.instance.config['enable_key_city']
|
|
43
|
+
end
|
|
65
44
|
|
|
66
|
-
|
|
67
|
-
assert_raise(Fluent::ConfigError) {
|
|
45
|
+
test "multiple key config" do
|
|
68
46
|
d = create_driver %[
|
|
47
|
+
geoip_lookup_key from.ip, to.ip
|
|
48
|
+
enable_key_city from_city, to_city
|
|
49
|
+
]
|
|
50
|
+
assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
test "multiple key config (bad configure)" do
|
|
54
|
+
assert_raise(Fluent::ConfigError) {
|
|
55
|
+
create_driver %[
|
|
56
|
+
geoip_lookup_key from.ip, to.ip
|
|
57
|
+
enable_key_city from_city
|
|
58
|
+
enable_key_region from_region
|
|
59
|
+
]
|
|
60
|
+
}
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
test "invalid json structure w/ Ruby hash like" do
|
|
64
|
+
assert_raise(Fluent::ConfigError) {
|
|
65
|
+
create_driver %[
|
|
66
|
+
geoip_lookup_key host
|
|
67
|
+
<record>
|
|
68
|
+
invalid_json {"foo" => 123}
|
|
69
|
+
</record>
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
test "invalid json structure w/ unquoted string literal" do
|
|
75
|
+
assert_raise(Fluent::ConfigError) {
|
|
76
|
+
create_driver %[
|
|
77
|
+
geoip_lookup_key host
|
|
78
|
+
<record>
|
|
79
|
+
invalid_json {"foo" : string, "bar" : 123}
|
|
80
|
+
</record>
|
|
81
|
+
]
|
|
82
|
+
}
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
data(geoip: "geoip",
|
|
86
|
+
geoip2_compat: "geoip2_compat")
|
|
87
|
+
test "unsupported key" do |backend|
|
|
88
|
+
assert_raise(Fluent::ConfigError.new("#{backend}: unsupported key unknown")) do
|
|
89
|
+
create_driver %[
|
|
90
|
+
backend_library #{backend}
|
|
91
|
+
<record>
|
|
92
|
+
city ${unknown["host"]}
|
|
93
|
+
</record>
|
|
94
|
+
]
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
data(geoip: ["geoip", '${city["host"]}'],
|
|
99
|
+
geoip2_compat: ["geoip2_compat", '${city["host"]}'],
|
|
100
|
+
geoip2_c: ["geoip2_c", '${city.names.en["host"]}'])
|
|
101
|
+
test "supported backend" do |(backend, placeholder)|
|
|
102
|
+
create_driver %[
|
|
103
|
+
backend_library #{backend}
|
|
104
|
+
<record>
|
|
105
|
+
city #{placeholder}
|
|
106
|
+
</record>
|
|
107
|
+
]
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
test "unsupported backend" do
|
|
111
|
+
assert_raise(Fluent::ConfigError) do
|
|
112
|
+
create_driver %[
|
|
113
|
+
backend_library hive_geoip2
|
|
114
|
+
<record>
|
|
115
|
+
city ${city["host"]}
|
|
116
|
+
</record>
|
|
117
|
+
]
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
sub_test_case "geoip2_c" do
|
|
123
|
+
def test_filter_with_dot_key
|
|
124
|
+
config = %[
|
|
125
|
+
backend_library geoip2_c
|
|
126
|
+
geoip_lookup_key ip.origin, ip.dest
|
|
127
|
+
<record>
|
|
128
|
+
origin_country ${country.iso_code['ip.origin']}
|
|
129
|
+
dest_country ${country.iso_code['ip.dest']}
|
|
130
|
+
</record>
|
|
131
|
+
]
|
|
132
|
+
messages = [
|
|
133
|
+
{'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'}
|
|
134
|
+
]
|
|
135
|
+
expected = [
|
|
136
|
+
{'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8',
|
|
137
|
+
'origin_country' => 'US', 'dest_country' => 'US'}
|
|
138
|
+
]
|
|
139
|
+
filtered = filter(config, messages)
|
|
140
|
+
assert_equal(expected, filtered)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def test_filter_with_unknown_address
|
|
144
|
+
config = %[
|
|
145
|
+
backend_library geoip2_c
|
|
69
146
|
geoip_lookup_key host
|
|
70
147
|
<record>
|
|
71
|
-
|
|
148
|
+
geoip_city ${city.names.en['host']}
|
|
149
|
+
geopoint [${location.longitude['host']}, ${location.latitude['host']}]
|
|
72
150
|
</record>
|
|
73
|
-
|
|
74
|
-
tag geoip.${tag}
|
|
151
|
+
skip_adding_null_record false
|
|
75
152
|
]
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
153
|
+
# 203.0.113.1 is a test address described in RFC5737
|
|
154
|
+
messages = [
|
|
155
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
156
|
+
{'host' => '0', 'message' => 'invalid ip'}
|
|
157
|
+
]
|
|
158
|
+
expected = [
|
|
159
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
|
|
160
|
+
{'host' => '0', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]}
|
|
161
|
+
]
|
|
162
|
+
filtered = filter(config, messages)
|
|
163
|
+
assert_equal(expected, filtered)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def test_filter_with_skip_unknown_address
|
|
167
|
+
config = %[
|
|
168
|
+
backend_library geoip2_c
|
|
79
169
|
geoip_lookup_key host
|
|
80
170
|
<record>
|
|
81
|
-
|
|
171
|
+
geoip_city ${city.names.en['host']}
|
|
172
|
+
geopoint [${location.longitude['host']}, ${location.latitude['host']}]
|
|
82
173
|
</record>
|
|
83
|
-
|
|
84
|
-
tag geoip.${tag}
|
|
174
|
+
skip_adding_null_record true
|
|
85
175
|
]
|
|
86
|
-
|
|
87
|
-
|
|
176
|
+
# 203.0.113.1 is a test address described in RFC5737
|
|
177
|
+
messages = [
|
|
178
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
179
|
+
{'host' => '0', 'message' => 'invalid ip'},
|
|
180
|
+
{'host' => '8.8.8.8', 'message' => 'google public dns'}
|
|
181
|
+
]
|
|
182
|
+
expected = [
|
|
183
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
184
|
+
{'host' => '0', 'message' => 'invalid ip'},
|
|
185
|
+
{'host' => '8.8.8.8', 'message' => 'google public dns',
|
|
186
|
+
'geoip_city' => 'Mountain View', 'geopoint' => [-122.0838, 37.386]}
|
|
187
|
+
]
|
|
188
|
+
filtered = filter(config, messages)
|
|
189
|
+
assert_equal(expected, filtered)
|
|
190
|
+
end
|
|
88
191
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
192
|
+
def test_filter_record_directive
|
|
193
|
+
config = %[
|
|
194
|
+
backend_library geoip2_c
|
|
195
|
+
geoip_lookup_key from.ip
|
|
196
|
+
<record>
|
|
197
|
+
from_city ${city.names.en['from.ip']}
|
|
198
|
+
from_country ${country.names.en['from.ip']}
|
|
199
|
+
latitude ${location.latitude['from.ip']}
|
|
200
|
+
longitude ${location.longitude['from.ip']}
|
|
201
|
+
float_concat ${location.latitude['from.ip']},${location.longitude['from.ip']}
|
|
202
|
+
float_array [${location.longitude['from.ip']}, ${location.latitude['from.ip']}]
|
|
203
|
+
float_nest { "lat" : ${location.latitude['from.ip']}, "lon" : ${location.longitude['from.ip']}}
|
|
204
|
+
string_concat ${city.names.en['from.ip']},${country.names.en['from.ip']}
|
|
205
|
+
string_array [${city.names.en['from.ip']}, ${country.names.en['from.ip']}]
|
|
206
|
+
string_nest { "city" : ${city.names.en['from.ip']}, "country_name" : ${country.names.en['from.ip']}}
|
|
207
|
+
unknown_city ${city.names.en['unknown_key']}
|
|
208
|
+
undefined ${city.names.en['undefined']}
|
|
209
|
+
broken_array1 [${location.longitude['from.ip']}, ${location.latitude['undefined']}]
|
|
210
|
+
broken_array2 [${location.longitude['undefined']}, ${location.latitude['undefined']}]
|
|
211
|
+
</record>
|
|
212
|
+
]
|
|
213
|
+
messages = [
|
|
214
|
+
{ 'from' => {'ip' => '66.102.3.80'} },
|
|
215
|
+
{ 'message' => 'missing field' },
|
|
216
|
+
]
|
|
217
|
+
expected = [
|
|
218
|
+
{
|
|
219
|
+
'from' => {'ip' => '66.102.3.80'},
|
|
220
|
+
'from_city' => 'Mountain View',
|
|
221
|
+
'from_country' => 'United States',
|
|
222
|
+
'latitude' => 37.419200000000004,
|
|
223
|
+
'longitude' => -122.0574,
|
|
224
|
+
'float_concat' => '37.419200000000004,-122.0574',
|
|
225
|
+
'float_array' => [-122.0574, 37.419200000000004],
|
|
226
|
+
'float_nest' => { 'lat' => 37.4192000000000004, 'lon' => -122.0574 },
|
|
227
|
+
'string_concat' => 'Mountain View,United States',
|
|
228
|
+
'string_array' => ["Mountain View", "United States"],
|
|
229
|
+
'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
|
|
230
|
+
'unknown_city' => nil,
|
|
231
|
+
'undefined' => nil,
|
|
232
|
+
'broken_array1' => [-122.0574, nil],
|
|
233
|
+
'broken_array2' => [nil, nil]
|
|
234
|
+
},
|
|
235
|
+
{
|
|
236
|
+
'message' => 'missing field',
|
|
237
|
+
'from_city' => nil,
|
|
238
|
+
'from_country' => nil,
|
|
239
|
+
'latitude' => nil,
|
|
240
|
+
'longitude' => nil,
|
|
241
|
+
'float_concat' => ',',
|
|
242
|
+
'float_array' => [nil, nil],
|
|
243
|
+
'float_nest' => { 'lat' => nil, 'lon' => nil },
|
|
244
|
+
'string_concat' => ',',
|
|
245
|
+
'string_array' => [nil, nil],
|
|
246
|
+
'string_nest' => { "city" => nil, "country_name" => nil },
|
|
247
|
+
'unknown_city' => nil,
|
|
248
|
+
'undefined' => nil,
|
|
249
|
+
'broken_array1' => [nil, nil],
|
|
250
|
+
'broken_array2' => [nil, nil]
|
|
251
|
+
},
|
|
252
|
+
]
|
|
253
|
+
filtered = filter(config, messages)
|
|
254
|
+
# test-unit cannot calculate diff between large Array
|
|
255
|
+
assert_equal(expected[0], filtered[0])
|
|
256
|
+
assert_equal(expected[1], filtered[1])
|
|
257
|
+
end
|
|
101
258
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
]
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
259
|
+
def test_filter_record_directive_multiple_record
|
|
260
|
+
config = %[
|
|
261
|
+
backend_library geoip2_c
|
|
262
|
+
geoip_lookup_key from.ip, to.ip
|
|
263
|
+
<record>
|
|
264
|
+
from_city ${city.names.en['from.ip']}
|
|
265
|
+
to_city ${city.names.en['to.ip']}
|
|
266
|
+
from_country ${country.names.en['from.ip']}
|
|
267
|
+
to_country ${country.names.en['to.ip']}
|
|
268
|
+
string_array [${country.names.en['from.ip']}, ${country.names.en['to.ip']}]
|
|
269
|
+
</record>
|
|
270
|
+
]
|
|
271
|
+
messages = [
|
|
272
|
+
{'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
|
|
273
|
+
{'message' => 'missing field'}
|
|
274
|
+
]
|
|
275
|
+
expected = [
|
|
276
|
+
{
|
|
277
|
+
'from' => { 'ip' => '66.102.3.80' },
|
|
278
|
+
'to' => { 'ip' => '125.54.15.42' },
|
|
279
|
+
'from_city' => 'Mountain View',
|
|
280
|
+
'from_country' => 'United States',
|
|
281
|
+
'to_city' => 'Tokorozawa',
|
|
282
|
+
'to_country' => 'Japan',
|
|
283
|
+
'string_array' => ['United States', 'Japan']
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
'message' => 'missing field',
|
|
287
|
+
'from_city' => nil,
|
|
288
|
+
'from_country' => nil,
|
|
289
|
+
'to_city' => nil,
|
|
290
|
+
'to_country' => nil,
|
|
291
|
+
'string_array' => [nil, nil]
|
|
292
|
+
}
|
|
293
|
+
]
|
|
294
|
+
filtered = filter(config, messages)
|
|
295
|
+
assert_equal(expected, filtered)
|
|
296
|
+
end
|
|
120
297
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
{'host'
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
assert_equal(expected, filtered)
|
|
136
|
-
end
|
|
298
|
+
def config_quoted_record
|
|
299
|
+
%[
|
|
300
|
+
backend_library geoip2_c
|
|
301
|
+
geoip_lookup_key host
|
|
302
|
+
<record>
|
|
303
|
+
location_properties '{ "country_code" : "${country.iso_code["host"]}", "lat": ${location.latitude["host"]}, "lon": ${location.longitude["host"]} }'
|
|
304
|
+
location_string ${location.latitude['host']},${location.longitude['host']}
|
|
305
|
+
location_string2 ${country.iso_code["host"]}
|
|
306
|
+
location_array "[${location.longitude['host']},${location.latitude['host']}]"
|
|
307
|
+
location_array2 '[${location.longitude["host"]},${location.latitude["host"]}]'
|
|
308
|
+
peculiar_pattern '[GEOIP] message => {"lat":${location.latitude["host"]}, "lon":${location.longitude["host"]}}'
|
|
309
|
+
</record>
|
|
310
|
+
]
|
|
311
|
+
end
|
|
137
312
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
313
|
+
def test_filter_quoted_record
|
|
314
|
+
messages = [
|
|
315
|
+
{'host' => '66.102.3.80', 'message' => 'valid ip'}
|
|
316
|
+
]
|
|
317
|
+
expected = [
|
|
318
|
+
{
|
|
319
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
320
|
+
'location_properties' => {
|
|
321
|
+
'country_code' => 'US',
|
|
322
|
+
'lat' => 37.419200000000004,
|
|
323
|
+
'lon' => -122.0574
|
|
324
|
+
},
|
|
325
|
+
'location_string' => '37.419200000000004,-122.0574',
|
|
326
|
+
'location_string2' => 'US',
|
|
327
|
+
'location_array' => [-122.0574, 37.419200000000004],
|
|
328
|
+
'location_array2' => [-122.0574, 37.419200000000004],
|
|
329
|
+
'peculiar_pattern' => '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}'
|
|
330
|
+
}
|
|
331
|
+
]
|
|
332
|
+
filtered = filter(config_quoted_record, messages)
|
|
333
|
+
assert_equal(expected, filtered)
|
|
334
|
+
end
|
|
159
335
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
assert_equal(expected, filtered)
|
|
183
|
-
end
|
|
336
|
+
def test_filter_v1_config_compatibility
|
|
337
|
+
messages = [
|
|
338
|
+
{'host' => '66.102.3.80', 'message' => 'valid ip'}
|
|
339
|
+
]
|
|
340
|
+
expected = [
|
|
341
|
+
{
|
|
342
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
343
|
+
'location_properties' => {
|
|
344
|
+
'country_code' => 'US',
|
|
345
|
+
'lat' => 37.419200000000004,
|
|
346
|
+
'lon' => -122.0574
|
|
347
|
+
},
|
|
348
|
+
'location_string' => '37.419200000000004,-122.0574',
|
|
349
|
+
'location_string2' => 'US',
|
|
350
|
+
'location_array' => [-122.0574, 37.419200000000004],
|
|
351
|
+
'location_array2' => [-122.0574, 37.419200000000004],
|
|
352
|
+
'peculiar_pattern' => '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}'
|
|
353
|
+
}
|
|
354
|
+
]
|
|
355
|
+
filtered = filter(config_quoted_record, messages, true)
|
|
356
|
+
assert_equal(expected, filtered)
|
|
357
|
+
end
|
|
184
358
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
359
|
+
def test_filter_multiline_v1_config
|
|
360
|
+
config = %[
|
|
361
|
+
backend_library geoip2_c
|
|
362
|
+
geoip_lookup_key host
|
|
363
|
+
<record>
|
|
364
|
+
location_properties {
|
|
365
|
+
"city": "${city.names.en['host']}",
|
|
366
|
+
"country_code": "${country.iso_code['host']}",
|
|
367
|
+
"latitude": "${location.latitude['host']}",
|
|
368
|
+
"longitude": "${location.longitude['host']}"
|
|
369
|
+
}
|
|
370
|
+
</record>
|
|
371
|
+
]
|
|
372
|
+
messages = [
|
|
373
|
+
{ 'host' => '66.102.3.80', 'message' => 'valid ip' }
|
|
374
|
+
]
|
|
375
|
+
expected = [
|
|
376
|
+
{
|
|
377
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
378
|
+
"location_properties" => {
|
|
379
|
+
"city" => "Mountain View",
|
|
380
|
+
"country_code" => "US",
|
|
381
|
+
"latitude" => 37.419200000000004,
|
|
382
|
+
"longitude" => -122.0574
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
]
|
|
386
|
+
filtered = filter(config, messages, true)
|
|
387
|
+
assert_equal(expected, filtered)
|
|
388
|
+
end
|
|
201
389
|
end
|
|
202
390
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
'
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
{
|
|
224
|
-
'from' => {'ip' => '66.102.3.80'},
|
|
225
|
-
'from_city' => 'Mountain View',
|
|
226
|
-
'from_country' => 'United States',
|
|
227
|
-
'to_city' => nil,
|
|
228
|
-
'to_country' => nil
|
|
229
|
-
},
|
|
230
|
-
{
|
|
231
|
-
'message' => 'missing field',
|
|
232
|
-
'from_city' => nil,
|
|
233
|
-
'from_country' => nil,
|
|
234
|
-
'to_city' => nil,
|
|
235
|
-
'to_country' => nil
|
|
236
|
-
}
|
|
237
|
-
]
|
|
238
|
-
filtered = filter(config, messages)
|
|
239
|
-
assert_equal(expected, filtered)
|
|
240
|
-
end
|
|
391
|
+
sub_test_case "geoip2_compat" do
|
|
392
|
+
def test_filter_with_dot_key
|
|
393
|
+
config = %[
|
|
394
|
+
backend_library geoip2_compat
|
|
395
|
+
geoip_lookup_key ip.origin, ip.dest
|
|
396
|
+
<record>
|
|
397
|
+
origin_country ${country_code['ip.origin']}
|
|
398
|
+
dest_country ${country_code['ip.dest']}
|
|
399
|
+
</record>
|
|
400
|
+
]
|
|
401
|
+
messages = [
|
|
402
|
+
{'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'}
|
|
403
|
+
]
|
|
404
|
+
expected = [
|
|
405
|
+
{'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8',
|
|
406
|
+
'origin_country' => 'US', 'dest_country' => 'US'}
|
|
407
|
+
]
|
|
408
|
+
filtered = filter(config, messages)
|
|
409
|
+
assert_equal(expected, filtered)
|
|
410
|
+
end
|
|
241
411
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
{ 'message' => 'missing field' },
|
|
265
|
-
]
|
|
266
|
-
expected = [
|
|
267
|
-
{
|
|
268
|
-
'from' => {'ip' => '66.102.3.80'},
|
|
269
|
-
'from_city' => 'Mountain View',
|
|
270
|
-
'from_country' => 'United States',
|
|
271
|
-
'latitude' => 37.4192008972168,
|
|
272
|
-
'longitude' => -122.05740356445312,
|
|
273
|
-
'float_concat' => '37.4192008972168,-122.05740356445312',
|
|
274
|
-
'float_array' => [-122.05740356445312, 37.4192008972168],
|
|
275
|
-
'float_nest' => { 'lat' => 37.4192008972168, 'lon' => -122.05740356445312 },
|
|
276
|
-
'string_concat' => 'Mountain View,United States',
|
|
277
|
-
'string_array' => ["Mountain View", "United States"],
|
|
278
|
-
'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
|
|
279
|
-
'unknown_city' => nil,
|
|
280
|
-
'undefined' => nil,
|
|
281
|
-
'broken_array1' => [-122.05740356445312, nil],
|
|
282
|
-
'broken_array2' => [nil, nil]
|
|
283
|
-
},
|
|
284
|
-
{
|
|
285
|
-
'message' => 'missing field',
|
|
286
|
-
'from_city' => nil,
|
|
287
|
-
'from_country' => nil,
|
|
288
|
-
'latitude' => nil,
|
|
289
|
-
'longitude' => nil,
|
|
290
|
-
'float_concat' => ',',
|
|
291
|
-
'float_array' => [nil, nil],
|
|
292
|
-
'float_nest' => { 'lat' => nil, 'lon' => nil },
|
|
293
|
-
'string_concat' => ',',
|
|
294
|
-
'string_array' => [nil, nil],
|
|
295
|
-
'string_nest' => { "city" => nil, "country_name" => nil },
|
|
296
|
-
'unknown_city' => nil,
|
|
297
|
-
'undefined' => nil,
|
|
298
|
-
'broken_array1' => [nil, nil],
|
|
299
|
-
'broken_array2' => [nil, nil]
|
|
300
|
-
},
|
|
301
|
-
]
|
|
302
|
-
filtered = filter(config, messages)
|
|
303
|
-
# test-unit cannot calculate diff between large Array
|
|
304
|
-
assert_equal(expected[0], filtered[0])
|
|
305
|
-
assert_equal(expected[1], filtered[1])
|
|
306
|
-
end
|
|
412
|
+
def test_filter_with_unknown_address
|
|
413
|
+
config = %[
|
|
414
|
+
backend_library geoip2_compat
|
|
415
|
+
geoip_lookup_key host
|
|
416
|
+
<record>
|
|
417
|
+
geoip_city ${city['host']}
|
|
418
|
+
geopoint [${longitude['host']}, ${latitude['host']}]
|
|
419
|
+
</record>
|
|
420
|
+
skip_adding_null_record false
|
|
421
|
+
]
|
|
422
|
+
# 203.0.113.1 is a test address described in RFC5737
|
|
423
|
+
messages = [
|
|
424
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
425
|
+
{'host' => '0', 'message' => 'invalid ip'}
|
|
426
|
+
]
|
|
427
|
+
expected = [
|
|
428
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
|
|
429
|
+
{'host' => '0', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]}
|
|
430
|
+
]
|
|
431
|
+
filtered = filter(config, messages)
|
|
432
|
+
assert_equal(expected, filtered)
|
|
433
|
+
end
|
|
307
434
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
'
|
|
326
|
-
'
|
|
327
|
-
'
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
{
|
|
334
|
-
'message' => 'missing field',
|
|
335
|
-
'from_city' => nil,
|
|
336
|
-
'from_country' => nil,
|
|
337
|
-
'to_city' => nil,
|
|
338
|
-
'to_country' => nil,
|
|
339
|
-
'string_array' => [nil, nil]
|
|
340
|
-
}
|
|
341
|
-
]
|
|
342
|
-
filtered = filter(config, messages)
|
|
343
|
-
assert_equal(expected, filtered)
|
|
344
|
-
end
|
|
435
|
+
def test_filter_with_skip_unknown_address
|
|
436
|
+
config = %[
|
|
437
|
+
backend_library geoip2_compat
|
|
438
|
+
geoip_lookup_key host
|
|
439
|
+
<record>
|
|
440
|
+
geoip_city ${city['host']}
|
|
441
|
+
geopoint [${longitude['host']}, ${latitude['host']}]
|
|
442
|
+
</record>
|
|
443
|
+
skip_adding_null_record true
|
|
444
|
+
]
|
|
445
|
+
# 203.0.113.1 is a test address described in RFC5737
|
|
446
|
+
messages = [
|
|
447
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
448
|
+
{'host' => '0', 'message' => 'invalid ip'},
|
|
449
|
+
{'host' => '8.8.8.8', 'message' => 'google public dns'}
|
|
450
|
+
]
|
|
451
|
+
expected = [
|
|
452
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
453
|
+
{'host' => '0', 'message' => 'invalid ip'},
|
|
454
|
+
{'host' => '8.8.8.8', 'message' => 'google public dns',
|
|
455
|
+
'geoip_city' => 'Mountain View', 'geopoint' => [-122.0838, 37.386]}
|
|
456
|
+
]
|
|
457
|
+
filtered = filter(config, messages)
|
|
458
|
+
assert_equal(expected, filtered)
|
|
459
|
+
end
|
|
345
460
|
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
461
|
+
def test_filter_record_directive
|
|
462
|
+
config = %[
|
|
463
|
+
backend_library geoip2_compat
|
|
464
|
+
geoip_lookup_key from.ip
|
|
465
|
+
<record>
|
|
466
|
+
from_city ${city['from.ip']}
|
|
467
|
+
from_country ${country_name['from.ip']}
|
|
468
|
+
latitude ${latitude['from.ip']}
|
|
469
|
+
longitude ${longitude['from.ip']}
|
|
470
|
+
float_concat ${latitude['from.ip']},${longitude['from.ip']}
|
|
471
|
+
float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
|
|
472
|
+
float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
|
|
473
|
+
string_concat ${city['from.ip']},${country_name['from.ip']}
|
|
474
|
+
string_array [${city['from.ip']}, ${country_name['from.ip']}]
|
|
475
|
+
string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
|
|
476
|
+
unknown_city ${city['unknown_key']}
|
|
477
|
+
undefined ${city['undefined']}
|
|
478
|
+
broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
|
|
479
|
+
broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
|
|
480
|
+
</record>
|
|
481
|
+
]
|
|
482
|
+
messages = [
|
|
483
|
+
{ 'from' => {'ip' => '66.102.3.80'} },
|
|
484
|
+
{ 'message' => 'missing field' },
|
|
485
|
+
]
|
|
486
|
+
expected = [
|
|
487
|
+
{
|
|
488
|
+
'from' => {'ip' => '66.102.3.80'},
|
|
489
|
+
'from_city' => 'Mountain View',
|
|
490
|
+
'from_country' => 'United States',
|
|
491
|
+
'latitude' => 37.419200000000004,
|
|
492
|
+
'longitude' => -122.0574,
|
|
493
|
+
'float_concat' => '37.419200000000004,-122.0574',
|
|
494
|
+
'float_array' => [-122.0574, 37.419200000000004],
|
|
495
|
+
'float_nest' => { 'lat' => 37.4192000000000004, 'lon' => -122.0574 },
|
|
496
|
+
'string_concat' => 'Mountain View,United States',
|
|
497
|
+
'string_array' => ["Mountain View", "United States"],
|
|
498
|
+
'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
|
|
499
|
+
'unknown_city' => nil,
|
|
500
|
+
'undefined' => nil,
|
|
501
|
+
'broken_array1' => [-122.0574, nil],
|
|
502
|
+
'broken_array2' => [nil, nil]
|
|
371
503
|
},
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
504
|
+
{
|
|
505
|
+
'message' => 'missing field',
|
|
506
|
+
'from_city' => nil,
|
|
507
|
+
'from_country' => nil,
|
|
508
|
+
'latitude' => nil,
|
|
509
|
+
'longitude' => nil,
|
|
510
|
+
'float_concat' => ',',
|
|
511
|
+
'float_array' => [nil, nil],
|
|
512
|
+
'float_nest' => { 'lat' => nil, 'lon' => nil },
|
|
513
|
+
'string_concat' => ',',
|
|
514
|
+
'string_array' => [nil, nil],
|
|
515
|
+
'string_nest' => { "city" => nil, "country_name" => nil },
|
|
516
|
+
'unknown_city' => nil,
|
|
517
|
+
'undefined' => nil,
|
|
518
|
+
'broken_array1' => [nil, nil],
|
|
519
|
+
'broken_array2' => [nil, nil]
|
|
520
|
+
},
|
|
521
|
+
]
|
|
522
|
+
filtered = filter(config, messages)
|
|
523
|
+
# test-unit cannot calculate diff between large Array
|
|
524
|
+
assert_equal(expected[0], filtered[0])
|
|
525
|
+
assert_equal(expected[1], filtered[1])
|
|
526
|
+
end
|
|
382
527
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
'
|
|
392
|
-
'
|
|
393
|
-
|
|
528
|
+
def test_filter_record_directive_multiple_record
|
|
529
|
+
config = %[
|
|
530
|
+
backend_library geoip2_compat
|
|
531
|
+
geoip_lookup_key from.ip, to.ip
|
|
532
|
+
<record>
|
|
533
|
+
from_city ${city['from.ip']}
|
|
534
|
+
to_city ${city['to.ip']}
|
|
535
|
+
from_country ${country_name['from.ip']}
|
|
536
|
+
to_country ${country_name['to.ip']}
|
|
537
|
+
string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
|
|
538
|
+
</record>
|
|
539
|
+
]
|
|
540
|
+
messages = [
|
|
541
|
+
{'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
|
|
542
|
+
{'message' => 'missing field'}
|
|
543
|
+
]
|
|
544
|
+
expected = [
|
|
545
|
+
{
|
|
546
|
+
'from' => { 'ip' => '66.102.3.80' },
|
|
547
|
+
'to' => { 'ip' => '125.54.15.42' },
|
|
548
|
+
'from_city' => 'Mountain View',
|
|
549
|
+
'from_country' => 'United States',
|
|
550
|
+
'to_city' => 'Tokorozawa',
|
|
551
|
+
'to_country' => 'Japan',
|
|
552
|
+
'string_array' => ['United States', 'Japan']
|
|
394
553
|
},
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
554
|
+
{
|
|
555
|
+
'message' => 'missing field',
|
|
556
|
+
'from_city' => nil,
|
|
557
|
+
'from_country' => nil,
|
|
558
|
+
'to_city' => nil,
|
|
559
|
+
'to_country' => nil,
|
|
560
|
+
'string_array' => [nil, nil]
|
|
561
|
+
}
|
|
562
|
+
]
|
|
563
|
+
filtered = filter(config, messages)
|
|
564
|
+
assert_equal(expected, filtered)
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
def config_quoted_record
|
|
568
|
+
%[
|
|
569
|
+
backend_library geoip2_compat
|
|
570
|
+
geoip_lookup_key host
|
|
571
|
+
<record>
|
|
572
|
+
location_properties '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
|
|
573
|
+
location_string ${latitude['host']},${longitude['host']}
|
|
574
|
+
location_string2 ${country_code["host"]}
|
|
575
|
+
location_array "[${longitude['host']},${latitude['host']}]"
|
|
576
|
+
location_array2 '[${longitude["host"]},${latitude["host"]}]'
|
|
577
|
+
peculiar_pattern '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
|
|
578
|
+
</record>
|
|
579
|
+
]
|
|
580
|
+
end
|
|
581
|
+
|
|
582
|
+
def test_filter_quoted_record
|
|
583
|
+
messages = [
|
|
584
|
+
{'host' => '66.102.3.80', 'message' => 'valid ip'}
|
|
585
|
+
]
|
|
586
|
+
expected = [
|
|
587
|
+
{
|
|
588
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
589
|
+
'location_properties' => {
|
|
590
|
+
'country_code' => 'US',
|
|
591
|
+
'lat' => 37.419200000000004,
|
|
592
|
+
'lon' => -122.0574
|
|
593
|
+
},
|
|
594
|
+
'location_string' => '37.419200000000004,-122.0574',
|
|
595
|
+
'location_string2' => 'US',
|
|
596
|
+
'location_array' => [-122.0574, 37.419200000000004],
|
|
597
|
+
'location_array2' => [-122.0574, 37.419200000000004],
|
|
598
|
+
'peculiar_pattern' => '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}'
|
|
599
|
+
}
|
|
600
|
+
]
|
|
601
|
+
filtered = filter(config_quoted_record, messages)
|
|
602
|
+
assert_equal(expected, filtered)
|
|
603
|
+
end
|
|
604
|
+
|
|
605
|
+
def test_filter_v1_config_compatibility
|
|
606
|
+
messages = [
|
|
607
|
+
{'host' => '66.102.3.80', 'message' => 'valid ip'}
|
|
608
|
+
]
|
|
609
|
+
expected = [
|
|
610
|
+
{
|
|
611
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
612
|
+
'location_properties' => {
|
|
613
|
+
'country_code' => 'US',
|
|
614
|
+
'lat' => 37.419200000000004,
|
|
615
|
+
'lon' => -122.0574
|
|
616
|
+
},
|
|
617
|
+
'location_string' => '37.419200000000004,-122.0574',
|
|
618
|
+
'location_string2' => 'US',
|
|
619
|
+
'location_array' => [-122.0574, 37.419200000000004],
|
|
620
|
+
'location_array2' => [-122.0574, 37.419200000000004],
|
|
621
|
+
'peculiar_pattern' => '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}'
|
|
622
|
+
}
|
|
623
|
+
]
|
|
624
|
+
filtered = filter(config_quoted_record, messages, true)
|
|
625
|
+
assert_equal(expected, filtered)
|
|
626
|
+
end
|
|
627
|
+
|
|
628
|
+
def test_filter_multiline_v1_config
|
|
629
|
+
config = %[
|
|
630
|
+
backend_library geoip2_compat
|
|
631
|
+
geoip_lookup_key host
|
|
632
|
+
<record>
|
|
633
|
+
location_properties {
|
|
634
|
+
"city": "${city['host']}",
|
|
635
|
+
"country_code": "${country_code['host']}",
|
|
636
|
+
"latitude": "${latitude['host']}",
|
|
637
|
+
"longitude": "${longitude['host']}"
|
|
638
|
+
}
|
|
639
|
+
</record>
|
|
640
|
+
]
|
|
641
|
+
messages = [
|
|
642
|
+
{ 'host' => '66.102.3.80', 'message' => 'valid ip' }
|
|
643
|
+
]
|
|
644
|
+
expected = [
|
|
645
|
+
{
|
|
646
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
647
|
+
"location_properties" => {
|
|
648
|
+
"city" => "Mountain View",
|
|
649
|
+
"country_code" => "US",
|
|
650
|
+
"latitude" => 37.419200000000004,
|
|
651
|
+
"longitude" => -122.0574
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
]
|
|
655
|
+
filtered = filter(config, messages, true)
|
|
656
|
+
assert_equal(expected, filtered)
|
|
657
|
+
end
|
|
404
658
|
end
|
|
405
659
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
660
|
+
sub_test_case "geoip legacy" do
|
|
661
|
+
def test_filter
|
|
662
|
+
config = %[
|
|
663
|
+
geoip_lookup_key host
|
|
664
|
+
enable_key_city geoip_city
|
|
665
|
+
]
|
|
666
|
+
messages = [
|
|
667
|
+
{'host' => '66.102.3.80', 'message' => 'valid ip'},
|
|
668
|
+
{'message' => 'missing field'},
|
|
669
|
+
]
|
|
670
|
+
expected = [
|
|
671
|
+
{'host' => '66.102.3.80', 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
|
|
672
|
+
{'message' => 'missing field', 'geoip_city' => nil},
|
|
673
|
+
]
|
|
674
|
+
filtered = filter(config, messages)
|
|
675
|
+
assert_equal(expected, filtered)
|
|
676
|
+
end
|
|
677
|
+
|
|
678
|
+
def test_filter_with_dot_key
|
|
679
|
+
config = %[
|
|
680
|
+
geoip_lookup_key ip.origin, ip.dest
|
|
681
|
+
<record>
|
|
682
|
+
origin_country ${country_code['ip.origin']}
|
|
683
|
+
dest_country ${country_code['ip.dest']}
|
|
684
|
+
</record>
|
|
685
|
+
]
|
|
686
|
+
messages = [
|
|
687
|
+
{'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'}
|
|
688
|
+
]
|
|
689
|
+
expected = [
|
|
690
|
+
{'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8',
|
|
691
|
+
'origin_country' => 'US', 'dest_country' => 'US'}
|
|
692
|
+
]
|
|
693
|
+
filtered = filter(config, messages)
|
|
694
|
+
assert_equal(expected, filtered)
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
def test_filter_nested_attr
|
|
698
|
+
config = %[
|
|
699
|
+
geoip_lookup_key host.ip
|
|
700
|
+
enable_key_city geoip_city
|
|
701
|
+
]
|
|
702
|
+
messages = [
|
|
703
|
+
{'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'},
|
|
704
|
+
{'message' => 'missing field'}
|
|
705
|
+
]
|
|
706
|
+
expected = [
|
|
707
|
+
{'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip', 'geoip_city' => 'Mountain View'},
|
|
708
|
+
{'message' => 'missing field', 'geoip_city' => nil}
|
|
709
|
+
]
|
|
710
|
+
filtered = filter(config, messages)
|
|
711
|
+
assert_equal(expected, filtered)
|
|
712
|
+
end
|
|
713
|
+
|
|
714
|
+
def test_filter_with_unknown_address
|
|
715
|
+
config = %[
|
|
716
|
+
geoip_lookup_key host
|
|
717
|
+
<record>
|
|
718
|
+
geoip_city ${city['host']}
|
|
719
|
+
geopoint [${longitude['host']}, ${latitude['host']}]
|
|
720
|
+
</record>
|
|
721
|
+
skip_adding_null_record false
|
|
722
|
+
]
|
|
723
|
+
# 203.0.113.1 is a test address described in RFC5737
|
|
724
|
+
messages = [
|
|
725
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
726
|
+
{'host' => '0', 'message' => 'invalid ip'}
|
|
727
|
+
]
|
|
728
|
+
expected = [
|
|
729
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]},
|
|
730
|
+
{'host' => '0', 'message' => 'invalid ip', 'geoip_city' => nil, 'geopoint' => [nil, nil]}
|
|
731
|
+
]
|
|
732
|
+
filtered = filter(config, messages)
|
|
733
|
+
assert_equal(expected, filtered)
|
|
734
|
+
end
|
|
735
|
+
|
|
736
|
+
def test_filter_with_skip_unknown_address
|
|
737
|
+
config = %[
|
|
738
|
+
geoip_lookup_key host
|
|
739
|
+
<record>
|
|
740
|
+
geoip_city ${city['host']}
|
|
741
|
+
geopoint [${longitude['host']}, ${latitude['host']}]
|
|
742
|
+
</record>
|
|
743
|
+
skip_adding_null_record true
|
|
744
|
+
]
|
|
745
|
+
# 203.0.113.1 is a test address described in RFC5737
|
|
746
|
+
messages = [
|
|
747
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
748
|
+
{'host' => '0', 'message' => 'invalid ip'},
|
|
749
|
+
{'host' => '8.8.8.8', 'message' => 'google public dns'}
|
|
750
|
+
]
|
|
751
|
+
expected = [
|
|
752
|
+
{'host' => '203.0.113.1', 'message' => 'invalid ip'},
|
|
753
|
+
{'host' => '0', 'message' => 'invalid ip'},
|
|
754
|
+
{'host' => '8.8.8.8', 'message' => 'google public dns',
|
|
755
|
+
'geoip_city' => 'Mountain View', 'geopoint' => [-122.08380126953125, 37.38600158691406]}
|
|
756
|
+
]
|
|
757
|
+
filtered = filter(config, messages)
|
|
758
|
+
assert_equal(expected, filtered)
|
|
759
|
+
end
|
|
760
|
+
|
|
761
|
+
def test_filter_multiple_key
|
|
762
|
+
config = %[
|
|
763
|
+
geoip_lookup_key from.ip, to.ip
|
|
764
|
+
enable_key_city from_city, to_city
|
|
765
|
+
]
|
|
766
|
+
messages = [
|
|
767
|
+
{'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
|
|
768
|
+
{'message' => 'missing field'}
|
|
769
|
+
]
|
|
770
|
+
expected = [
|
|
771
|
+
{'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'},
|
|
772
|
+
'from_city' => 'Mountain View', 'to_city' => 'Tokorozawa'},
|
|
773
|
+
{'message' => 'missing field', 'from_city' => nil, 'to_city' => nil}
|
|
774
|
+
]
|
|
775
|
+
filtered = filter(config, messages)
|
|
776
|
+
assert_equal(expected, filtered)
|
|
777
|
+
end
|
|
778
|
+
|
|
779
|
+
def test_filter_multiple_key_multiple_record
|
|
780
|
+
config = %[
|
|
781
|
+
geoip_lookup_key from.ip, to.ip
|
|
782
|
+
enable_key_city from_city, to_city
|
|
783
|
+
enable_key_country_name from_country, to_country
|
|
784
|
+
]
|
|
785
|
+
messages = [
|
|
786
|
+
{'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
|
|
787
|
+
{'from' => {'ip' => '66.102.3.80'}},
|
|
788
|
+
{'message' => 'missing field'}
|
|
789
|
+
]
|
|
790
|
+
expected = [
|
|
791
|
+
{
|
|
792
|
+
'from' => {'ip' => '66.102.3.80'},
|
|
793
|
+
'to' => {'ip' => '125.54.15.42'},
|
|
794
|
+
'from_city' => 'Mountain View',
|
|
795
|
+
'from_country' => 'United States',
|
|
796
|
+
'to_city' => 'Tokorozawa',
|
|
797
|
+
'to_country' => 'Japan'
|
|
798
|
+
},
|
|
799
|
+
{
|
|
800
|
+
'from' => {'ip' => '66.102.3.80'},
|
|
801
|
+
'from_city' => 'Mountain View',
|
|
802
|
+
'from_country' => 'United States',
|
|
803
|
+
'to_city' => nil,
|
|
804
|
+
'to_country' => nil
|
|
805
|
+
},
|
|
806
|
+
{
|
|
807
|
+
'message' => 'missing field',
|
|
808
|
+
'from_city' => nil,
|
|
809
|
+
'from_country' => nil,
|
|
810
|
+
'to_city' => nil,
|
|
811
|
+
'to_country' => nil
|
|
812
|
+
}
|
|
813
|
+
]
|
|
814
|
+
filtered = filter(config, messages)
|
|
815
|
+
assert_equal(expected, filtered)
|
|
816
|
+
end
|
|
817
|
+
|
|
818
|
+
def test_filter_record_directive
|
|
819
|
+
config = %[
|
|
820
|
+
geoip_lookup_key from.ip
|
|
821
|
+
<record>
|
|
822
|
+
from_city ${city['from.ip']}
|
|
823
|
+
from_country ${country_name['from.ip']}
|
|
824
|
+
latitude ${latitude['from.ip']}
|
|
825
|
+
longitude ${longitude['from.ip']}
|
|
826
|
+
float_concat ${latitude['from.ip']},${longitude['from.ip']}
|
|
827
|
+
float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
|
|
828
|
+
float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
|
|
829
|
+
string_concat ${city['from.ip']},${country_name['from.ip']}
|
|
830
|
+
string_array [${city['from.ip']}, ${country_name['from.ip']}]
|
|
831
|
+
string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
|
|
832
|
+
unknown_city ${city['unknown_key']}
|
|
833
|
+
undefined ${city['undefined']}
|
|
834
|
+
broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
|
|
835
|
+
broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
|
|
836
|
+
</record>
|
|
837
|
+
]
|
|
838
|
+
messages = [
|
|
839
|
+
{ 'from' => {'ip' => '66.102.3.80'} },
|
|
840
|
+
{ 'message' => 'missing field' },
|
|
841
|
+
]
|
|
842
|
+
expected = [
|
|
843
|
+
{
|
|
844
|
+
'from' => {'ip' => '66.102.3.80'},
|
|
845
|
+
'from_city' => 'Mountain View',
|
|
846
|
+
'from_country' => 'United States',
|
|
847
|
+
'latitude' => 37.4192008972168,
|
|
848
|
+
'longitude' => -122.05740356445312,
|
|
849
|
+
'float_concat' => '37.4192008972168,-122.05740356445312',
|
|
850
|
+
'float_array' => [-122.05740356445312, 37.4192008972168],
|
|
851
|
+
'float_nest' => { 'lat' => 37.4192008972168, 'lon' => -122.05740356445312 },
|
|
852
|
+
'string_concat' => 'Mountain View,United States',
|
|
853
|
+
'string_array' => ["Mountain View", "United States"],
|
|
854
|
+
'string_nest' => {"city" => "Mountain View", "country_name" => "United States"},
|
|
855
|
+
'unknown_city' => nil,
|
|
856
|
+
'undefined' => nil,
|
|
857
|
+
'broken_array1' => [-122.05740356445312, nil],
|
|
858
|
+
'broken_array2' => [nil, nil]
|
|
859
|
+
},
|
|
860
|
+
{
|
|
861
|
+
'message' => 'missing field',
|
|
862
|
+
'from_city' => nil,
|
|
863
|
+
'from_country' => nil,
|
|
864
|
+
'latitude' => nil,
|
|
865
|
+
'longitude' => nil,
|
|
866
|
+
'float_concat' => ',',
|
|
867
|
+
'float_array' => [nil, nil],
|
|
868
|
+
'float_nest' => { 'lat' => nil, 'lon' => nil },
|
|
869
|
+
'string_concat' => ',',
|
|
870
|
+
'string_array' => [nil, nil],
|
|
871
|
+
'string_nest' => { "city" => nil, "country_name" => nil },
|
|
872
|
+
'unknown_city' => nil,
|
|
873
|
+
'undefined' => nil,
|
|
874
|
+
'broken_array1' => [nil, nil],
|
|
875
|
+
'broken_array2' => [nil, nil]
|
|
876
|
+
},
|
|
877
|
+
]
|
|
878
|
+
filtered = filter(config, messages)
|
|
879
|
+
# test-unit cannot calculate diff between large Array
|
|
880
|
+
assert_equal(expected[0], filtered[0])
|
|
881
|
+
assert_equal(expected[1], filtered[1])
|
|
882
|
+
end
|
|
883
|
+
|
|
884
|
+
def test_filter_record_directive_multiple_record
|
|
885
|
+
config = %[
|
|
886
|
+
geoip_lookup_key from.ip, to.ip
|
|
887
|
+
<record>
|
|
888
|
+
from_city ${city['from.ip']}
|
|
889
|
+
to_city ${city['to.ip']}
|
|
890
|
+
from_country ${country_name['from.ip']}
|
|
891
|
+
to_country ${country_name['to.ip']}
|
|
892
|
+
string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
|
|
893
|
+
</record>
|
|
894
|
+
]
|
|
895
|
+
messages = [
|
|
896
|
+
{'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}},
|
|
897
|
+
{'message' => 'missing field'}
|
|
898
|
+
]
|
|
899
|
+
expected = [
|
|
900
|
+
{
|
|
901
|
+
'from' => { 'ip' => '66.102.3.80' },
|
|
902
|
+
'to' => { 'ip' => '125.54.15.42' },
|
|
903
|
+
'from_city' => 'Mountain View',
|
|
904
|
+
'from_country' => 'United States',
|
|
905
|
+
'to_city' => 'Tokorozawa',
|
|
906
|
+
'to_country' => 'Japan',
|
|
907
|
+
'string_array' => ['United States', 'Japan']
|
|
908
|
+
},
|
|
909
|
+
{
|
|
910
|
+
'message' => 'missing field',
|
|
911
|
+
'from_city' => nil,
|
|
912
|
+
'from_country' => nil,
|
|
913
|
+
'to_city' => nil,
|
|
914
|
+
'to_country' => nil,
|
|
915
|
+
'string_array' => [nil, nil]
|
|
916
|
+
}
|
|
917
|
+
]
|
|
918
|
+
filtered = filter(config, messages)
|
|
919
|
+
assert_equal(expected, filtered)
|
|
920
|
+
end
|
|
921
|
+
|
|
922
|
+
def config_quoted_record
|
|
923
|
+
%[
|
|
924
|
+
geoip_lookup_key host
|
|
925
|
+
<record>
|
|
926
|
+
location_properties '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
|
|
927
|
+
location_string ${latitude['host']},${longitude['host']}
|
|
928
|
+
location_string2 ${country_code["host"]}
|
|
929
|
+
location_array "[${longitude['host']},${latitude['host']}]"
|
|
930
|
+
location_array2 '[${longitude["host"]},${latitude["host"]}]'
|
|
931
|
+
peculiar_pattern '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
|
|
932
|
+
</record>
|
|
933
|
+
]
|
|
934
|
+
end
|
|
935
|
+
|
|
936
|
+
def test_filter_quoted_record
|
|
937
|
+
messages = [
|
|
938
|
+
{'host' => '66.102.3.80', 'message' => 'valid ip'}
|
|
939
|
+
]
|
|
940
|
+
expected = [
|
|
941
|
+
{
|
|
942
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
943
|
+
'location_properties' => {
|
|
944
|
+
'country_code' => 'US',
|
|
945
|
+
'lat' => 37.4192008972168,
|
|
946
|
+
'lon' => -122.05740356445312
|
|
947
|
+
},
|
|
948
|
+
'location_string' => '37.4192008972168,-122.05740356445312',
|
|
949
|
+
'location_string2' => 'US',
|
|
950
|
+
'location_array' => [-122.05740356445312, 37.4192008972168],
|
|
951
|
+
'location_array2' => [-122.05740356445312, 37.4192008972168],
|
|
952
|
+
'peculiar_pattern' => '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}'
|
|
415
953
|
}
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
'host' => '66.102.3.80', 'message' => 'valid ip'
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
954
|
+
]
|
|
955
|
+
filtered = filter(config_quoted_record, messages)
|
|
956
|
+
assert_equal(expected, filtered)
|
|
957
|
+
end
|
|
958
|
+
|
|
959
|
+
def test_filter_v1_config_compatibility
|
|
960
|
+
messages = [
|
|
961
|
+
{'host' => '66.102.3.80', 'message' => 'valid ip'}
|
|
962
|
+
]
|
|
963
|
+
expected = [
|
|
964
|
+
{
|
|
965
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
966
|
+
'location_properties' => {
|
|
967
|
+
'country_code' => 'US',
|
|
968
|
+
'lat' => 37.4192008972168,
|
|
969
|
+
'lon' => -122.05740356445312
|
|
970
|
+
},
|
|
971
|
+
'location_string' => '37.4192008972168,-122.05740356445312',
|
|
972
|
+
'location_string2' => 'US',
|
|
973
|
+
'location_array' => [-122.05740356445312, 37.4192008972168],
|
|
974
|
+
'location_array2' => [-122.05740356445312, 37.4192008972168],
|
|
975
|
+
'peculiar_pattern' => '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}'
|
|
429
976
|
}
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
977
|
+
]
|
|
978
|
+
filtered = filter(config_quoted_record, messages, true)
|
|
979
|
+
assert_equal(expected, filtered)
|
|
980
|
+
end
|
|
981
|
+
|
|
982
|
+
def test_filter_multiline_v1_config
|
|
983
|
+
config = %[
|
|
984
|
+
geoip_lookup_key host
|
|
985
|
+
<record>
|
|
986
|
+
location_properties {
|
|
987
|
+
"city": "${city['host']}",
|
|
988
|
+
"country_code": "${country_code['host']}",
|
|
989
|
+
"latitude": "${latitude['host']}",
|
|
990
|
+
"longitude": "${longitude['host']}"
|
|
991
|
+
}
|
|
992
|
+
</record>
|
|
993
|
+
]
|
|
994
|
+
messages = [
|
|
995
|
+
{ 'host' => '66.102.3.80', 'message' => 'valid ip' }
|
|
996
|
+
]
|
|
997
|
+
expected = [
|
|
998
|
+
{
|
|
999
|
+
'host' => '66.102.3.80', 'message' => 'valid ip',
|
|
1000
|
+
"location_properties" => {
|
|
1001
|
+
"city" => "Mountain View",
|
|
1002
|
+
"country_code" => "US",
|
|
1003
|
+
"latitude" => 37.4192008972168,
|
|
1004
|
+
"longitude" => -122.05740356445312
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
]
|
|
1008
|
+
filtered = filter(config, messages, true)
|
|
1009
|
+
assert_equal(expected, filtered)
|
|
1010
|
+
end
|
|
434
1011
|
end
|
|
435
1012
|
end
|
|
436
1013
|
|