fluent-plugin-geoip 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +90 -211
- data/fluent-plugin-geoip.gemspec +2 -1
- data/lib/fluent/plugin/filter_geoip.rb +180 -5
- data/test/helper.rb +1 -0
- data/test/plugin/test_filter_geoip.rb +137 -69
- metadata +17 -7
- data/lib/fluent/plugin/geoip.rb +0 -171
- data/lib/fluent/plugin/out_geoip.rb +0 -56
- data/test/plugin/test_out_geoip.rb +0 -1048
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
require 'fluent/plugin/output'
|
|
2
|
-
require 'fluent/plugin/geoip'
|
|
3
|
-
|
|
4
|
-
class Fluent::Plugin::GeoipOutput < Fluent::Plugin::Output
|
|
5
|
-
Fluent::Plugin.register_output('geoip', self)
|
|
6
|
-
|
|
7
|
-
helpers :event_emitter, :inject, :compat_parameters
|
|
8
|
-
|
|
9
|
-
config_param :geoip_database, :string, default: File.expand_path('../../../data/GeoLiteCity.dat', __dir__)
|
|
10
|
-
config_param :geoip2_database, :string, default: File.expand_path('../../../data/GeoLite2-City.mmdb', __dir__)
|
|
11
|
-
config_param :geoip_lookup_keys, :array, value_type: :string, default: ["host"]
|
|
12
|
-
config_param :geoip_lookup_key, :string, default: nil, deprecated: "Use geoip_lookup_keys instead"
|
|
13
|
-
config_param :tag, :string, default: nil
|
|
14
|
-
config_param :skip_adding_null_record, :bool, default: false
|
|
15
|
-
|
|
16
|
-
config_set_default :@log_level, "warn"
|
|
17
|
-
|
|
18
|
-
config_param :backend_library, :enum, list: Fluent::GeoIP::BACKEND_LIBRARIES, default: :geoip2_c
|
|
19
|
-
config_section :buffer do
|
|
20
|
-
config_set_default :@type, :memory
|
|
21
|
-
config_set_default :chunk_keys, ['tag']
|
|
22
|
-
config_set_default :flush_interval, 0
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def configure(conf)
|
|
26
|
-
compat_parameters_convert(conf, :buffer, :inject, default_chunk_key: 'tag')
|
|
27
|
-
super
|
|
28
|
-
raise Fluetn::ConfigError, "chunk key must include 'tag'" unless @chunk_key_tag
|
|
29
|
-
placeholder_validate!(:tag, @tag) if @tag
|
|
30
|
-
@geoip = Fluent::GeoIP.new(self, conf)
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def format(tag, time, record)
|
|
34
|
-
record = inject_values_to_record(tag, time, record)
|
|
35
|
-
[tag, time, record].to_msgpack
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def formatted_to_msgpack_binary
|
|
39
|
-
true
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def write(chunk)
|
|
43
|
-
es = Fluent::MultiEventStream.new
|
|
44
|
-
tag = ""
|
|
45
|
-
chunk.each do |_tag, time, record|
|
|
46
|
-
tag = _tag
|
|
47
|
-
es.add(time, @geoip.add_geoip_field(record))
|
|
48
|
-
end
|
|
49
|
-
tag = extract_placeholders(@tag, chunk.metadata) if @tag
|
|
50
|
-
router.emit_stream(tag, es)
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def multi_workers_ready?
|
|
54
|
-
true
|
|
55
|
-
end
|
|
56
|
-
end
|
|
@@ -1,1048 +0,0 @@
|
|
|
1
|
-
require 'helper'
|
|
2
|
-
require 'fluent/plugin/out_geoip'
|
|
3
|
-
require 'fluent/test/driver/output'
|
|
4
|
-
|
|
5
|
-
class GeoipOutputTest < Test::Unit::TestCase
|
|
6
|
-
def setup
|
|
7
|
-
Fluent::Test.setup
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
CONFIG = %[
|
|
11
|
-
geoip_lookup_keys host
|
|
12
|
-
<record>
|
|
13
|
-
geoip_city ${city['host']}
|
|
14
|
-
</record>
|
|
15
|
-
tag geoip.${tag[1]}
|
|
16
|
-
]
|
|
17
|
-
|
|
18
|
-
def create_driver(conf = CONFIG, syntax: :v1)
|
|
19
|
-
Fluent::Test::Driver::Output.new(Fluent::Plugin::GeoipOutput).configure(conf, syntax: syntax)
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
sub_test_case "configure" do
|
|
23
|
-
test "empty" do
|
|
24
|
-
assert_nothing_raised do
|
|
25
|
-
create_driver('')
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
test "obsoleted configuration" do
|
|
30
|
-
assert_raise(Fluent::ConfigError) {
|
|
31
|
-
create_driver('enable_key_city geoip_city')
|
|
32
|
-
}
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
test "deprecated configuration geoip_lookup_key" do
|
|
36
|
-
conf = %[
|
|
37
|
-
geoip_lookup_key host,ip
|
|
38
|
-
<record>
|
|
39
|
-
geoip_city ${city['host']}
|
|
40
|
-
</record>
|
|
41
|
-
tag geoip.${tag[1]}
|
|
42
|
-
]
|
|
43
|
-
d = create_driver(conf)
|
|
44
|
-
assert_equal(["host", "ip"],
|
|
45
|
-
d.instance.instance_variable_get(:@geoip).instance_variable_get(:@geoip_lookup_keys))
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
test "invalid json structure w/ Ruby hash like" do
|
|
49
|
-
assert_raise(Fluent::ConfigParseError) do
|
|
50
|
-
create_driver %[
|
|
51
|
-
geoip_lookup_keys host
|
|
52
|
-
<record>
|
|
53
|
-
invalid_json {"foo" => 123}
|
|
54
|
-
</record>
|
|
55
|
-
tag geoip.${tag[1]}
|
|
56
|
-
]
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
test "invalid json structure w/ unquoted string literal" do
|
|
61
|
-
assert_raise(Fluent::ConfigParseError) do
|
|
62
|
-
create_driver %[
|
|
63
|
-
geoip_lookup_keys host
|
|
64
|
-
<record>
|
|
65
|
-
invalid_json {"foo" : string, "bar" : 123}
|
|
66
|
-
</record>
|
|
67
|
-
tag geoip.${tag[1]}
|
|
68
|
-
]
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
data(geoip: "geoip",
|
|
73
|
-
geoip2_compat: "geoip2_compat")
|
|
74
|
-
test "unsupported key" do |backend|
|
|
75
|
-
assert_raise(Fluent::ConfigError.new("#{backend}: unsupported key unknown")) do
|
|
76
|
-
create_driver %[
|
|
77
|
-
backend_library #{backend}
|
|
78
|
-
<record>
|
|
79
|
-
city ${unknown["host"]}
|
|
80
|
-
</record>
|
|
81
|
-
tag geoip.${tag[1]}
|
|
82
|
-
]
|
|
83
|
-
end
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
data(geoip: ["geoip", '${city["host"]}'],
|
|
87
|
-
geoip2_compat: ["geoip2_compat", '${city["host"]}'],
|
|
88
|
-
geoip2_c: ["geoip2_c", '${city.names.en["host"]}'])
|
|
89
|
-
test "supported backend" do |(backend, placeholder)|
|
|
90
|
-
create_driver %[
|
|
91
|
-
backend_library #{backend}
|
|
92
|
-
<record>
|
|
93
|
-
city #{placeholder}
|
|
94
|
-
</record>
|
|
95
|
-
tag geoip.${tag[1]}
|
|
96
|
-
]
|
|
97
|
-
end
|
|
98
|
-
|
|
99
|
-
test "unsupported backend" do
|
|
100
|
-
assert_raise(Fluent::ConfigError) do
|
|
101
|
-
create_driver %[
|
|
102
|
-
backend_library hive_geoip2
|
|
103
|
-
<record>
|
|
104
|
-
city ${city["host"]}
|
|
105
|
-
</record>
|
|
106
|
-
tag geoip.${tag[1]}
|
|
107
|
-
]
|
|
108
|
-
end
|
|
109
|
-
end
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
sub_test_case "geoip2_c" do
|
|
113
|
-
def test_emit_tag_option
|
|
114
|
-
d1 = create_driver(%[
|
|
115
|
-
backend_library geoip2_c
|
|
116
|
-
geoip_lookup_keys host
|
|
117
|
-
<record>
|
|
118
|
-
geoip_city ${city.names.en['host']}
|
|
119
|
-
</record>
|
|
120
|
-
tag geoip.${tag[1]}
|
|
121
|
-
])
|
|
122
|
-
d1.run(default_tag: 'input.access') do
|
|
123
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
124
|
-
d1.feed({'message' => 'missing field'})
|
|
125
|
-
end
|
|
126
|
-
events = d1.events
|
|
127
|
-
assert_equal 2, events.length
|
|
128
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
129
|
-
assert_equal 'Mountain View', events[0][2]['geoip_city']
|
|
130
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
131
|
-
end
|
|
132
|
-
|
|
133
|
-
def test_emit_tag_parts
|
|
134
|
-
d1 = create_driver(%[
|
|
135
|
-
backend_library geoip2_c
|
|
136
|
-
geoip_lookup_keys host
|
|
137
|
-
<record>
|
|
138
|
-
geoip_city ${city.names.en['host']}
|
|
139
|
-
</record>
|
|
140
|
-
tag geoip.${tag[1]}.${tag[2]}.${tag[3]}
|
|
141
|
-
])
|
|
142
|
-
d1.run(default_tag: '0.1.2.3') do
|
|
143
|
-
d1.feed({'host' => '66.102.3.80'})
|
|
144
|
-
end
|
|
145
|
-
events = d1.events
|
|
146
|
-
assert_equal 1, events.length
|
|
147
|
-
assert_equal 'geoip.1.2.3', events[0][0] # tag
|
|
148
|
-
assert_equal 'Mountain View', events[0][2]['geoip_city']
|
|
149
|
-
end
|
|
150
|
-
|
|
151
|
-
def test_emit_with_dot_key
|
|
152
|
-
d1 = create_driver(%[
|
|
153
|
-
backend_library geoip2_c
|
|
154
|
-
geoip_lookup_keys ip.origin, ip.dest
|
|
155
|
-
<record>
|
|
156
|
-
origin_country ${country.iso_code['ip.origin']}
|
|
157
|
-
dest_country ${country.iso_code['ip.dest']}
|
|
158
|
-
</record>
|
|
159
|
-
tag geoip.${tag[1]}
|
|
160
|
-
])
|
|
161
|
-
d1.run(default_tag: 'input.access') do
|
|
162
|
-
d1.feed({'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'})
|
|
163
|
-
end
|
|
164
|
-
events = d1.events
|
|
165
|
-
assert_equal 1, events.length
|
|
166
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
167
|
-
assert_equal 'US', events[0][2]['origin_country']
|
|
168
|
-
assert_equal 'US', events[0][2]['dest_country']
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
def test_emit_with_unknown_address
|
|
172
|
-
d1 = create_driver(%[
|
|
173
|
-
backend_library geoip2_c
|
|
174
|
-
geoip_lookup_keys host
|
|
175
|
-
<record>
|
|
176
|
-
geoip_city ${city.names.en['host']}
|
|
177
|
-
geopoint [${location.longitude['host']}, ${location.latitude['host']}]
|
|
178
|
-
</record>
|
|
179
|
-
skip_adding_null_record false
|
|
180
|
-
tag geoip.${tag[1]}
|
|
181
|
-
], syntax: :v0)
|
|
182
|
-
d1.run(default_tag: 'input.access') do
|
|
183
|
-
# 203.0.113.1 is a test address described in RFC5737
|
|
184
|
-
d1.feed({'host' => '203.0.113.1', 'message' => 'invalid ip'})
|
|
185
|
-
d1.feed({'host' => '0', 'message' => 'invalid ip'})
|
|
186
|
-
end
|
|
187
|
-
events = d1.events
|
|
188
|
-
assert_equal 2, events.length
|
|
189
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
190
|
-
assert_equal nil, events[0][2]['geoip_city']
|
|
191
|
-
assert_equal 'geoip.access', events[1][0] # tag
|
|
192
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
193
|
-
end
|
|
194
|
-
|
|
195
|
-
def test_emit_with_skip_unknown_address
|
|
196
|
-
d1 = create_driver(%[
|
|
197
|
-
backend_library geoip2_c
|
|
198
|
-
geoip_lookup_keys host
|
|
199
|
-
<record>
|
|
200
|
-
geoip_city ${city.names.en['host']}
|
|
201
|
-
geopoint [${location.longitude['host']}, ${location.latitude['host']}]
|
|
202
|
-
</record>
|
|
203
|
-
skip_adding_null_record true
|
|
204
|
-
tag geoip.${tag[1]}
|
|
205
|
-
], syntax: :v0)
|
|
206
|
-
d1.run(default_tag: 'input.access') do
|
|
207
|
-
# 203.0.113.1 is a test address described in RFC5737
|
|
208
|
-
d1.feed({'host' => '203.0.113.1', 'message' => 'invalid ip'})
|
|
209
|
-
d1.feed({'host' => '0', 'message' => 'invalid ip'})
|
|
210
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'google bot'})
|
|
211
|
-
end
|
|
212
|
-
events = d1.events
|
|
213
|
-
assert_equal 3, events.length
|
|
214
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
215
|
-
assert_equal nil, events[0][2]['geoip_city']
|
|
216
|
-
assert_equal nil, events[0][2]['geopoint']
|
|
217
|
-
assert_equal 'geoip.access', events[1][0] # tag
|
|
218
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
219
|
-
assert_equal nil, events[1][2]['geopoint']
|
|
220
|
-
assert_equal 'Mountain View', events[2][2]['geoip_city']
|
|
221
|
-
assert_equal [-122.0574, 37.419200000000004], events[2][2]['geopoint']
|
|
222
|
-
end
|
|
223
|
-
|
|
224
|
-
def test_emit_record_directive
|
|
225
|
-
d1 = create_driver(%[
|
|
226
|
-
backend_library geoip2_c
|
|
227
|
-
geoip_lookup_keys from.ip
|
|
228
|
-
<record>
|
|
229
|
-
from_city ${city.names.en['from.ip']}
|
|
230
|
-
from_country ${country.names.en['from.ip']}
|
|
231
|
-
latitude ${location.latitude['from.ip']}
|
|
232
|
-
longitude ${location.longitude['from.ip']}
|
|
233
|
-
float_concat ${location.latitude['from.ip']},${location.longitude['from.ip']}
|
|
234
|
-
float_array [${location.longitude['from.ip']}, ${location.latitude['from.ip']}]
|
|
235
|
-
float_nest { "lat" : ${location.latitude['from.ip']}, "lon" : ${location.longitude['from.ip']}}
|
|
236
|
-
string_concat ${location.latitude['from.ip']},${location.longitude['from.ip']}
|
|
237
|
-
string_array [${city.names.en['from.ip']}, ${country.names.en['from.ip']}]
|
|
238
|
-
string_nest { "city" : ${city.names.en['from.ip']}, "country_name" : ${country.names.en['from.ip']}}
|
|
239
|
-
unknown_city ${city.names.en['unknown_key']}
|
|
240
|
-
undefined ${city.names.en['undefined']}
|
|
241
|
-
broken_array1 [${location.longitude['from.ip']}, ${location.latitude['undefined']}]
|
|
242
|
-
broken_array2 [${location.longitude['undefined']}, ${location.latitude['undefined']}]
|
|
243
|
-
</record>
|
|
244
|
-
tag geoip.${tag[1]}
|
|
245
|
-
], syntax: :v0)
|
|
246
|
-
d1.run(default_tag: 'input.access') do
|
|
247
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}})
|
|
248
|
-
d1.feed({'message' => 'missing field'})
|
|
249
|
-
end
|
|
250
|
-
events = d1.events
|
|
251
|
-
assert_equal 2, events.length
|
|
252
|
-
|
|
253
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
254
|
-
assert_equal 'Mountain View', events[0][2]['from_city']
|
|
255
|
-
assert_equal 'United States', events[0][2]['from_country']
|
|
256
|
-
assert_equal 37.419200000000004, events[0][2]['latitude']
|
|
257
|
-
assert_equal(-122.0574, events[0][2]['longitude'])
|
|
258
|
-
assert_equal '37.419200000000004,-122.0574', events[0][2]['float_concat']
|
|
259
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['float_array']
|
|
260
|
-
float_nest = {"lat" => 37.419200000000004, "lon" => -122.0574 }
|
|
261
|
-
assert_equal float_nest, events[0][2]['float_nest']
|
|
262
|
-
assert_equal '37.419200000000004,-122.0574', events[0][2]['string_concat']
|
|
263
|
-
assert_equal ["Mountain View", "United States"], events[0][2]['string_array']
|
|
264
|
-
string_nest = {"city" => "Mountain View", "country_name" => "United States"}
|
|
265
|
-
assert_equal string_nest, events[0][2]['string_nest']
|
|
266
|
-
assert_equal nil, events[0][2]['unknown_city']
|
|
267
|
-
assert_equal nil, events[0][2]['undefined']
|
|
268
|
-
assert_equal [-122.0574, nil], events[0][2]['broken_array1']
|
|
269
|
-
assert_equal [nil, nil], events[0][2]['broken_array2']
|
|
270
|
-
|
|
271
|
-
assert_equal nil, events[1][2]['from_city']
|
|
272
|
-
assert_equal nil, events[1][2]['from_country']
|
|
273
|
-
assert_equal nil, events[1][2]['latitude']
|
|
274
|
-
assert_equal nil, events[1][2]['longitude']
|
|
275
|
-
assert_equal ',', events[1][2]['float_concat']
|
|
276
|
-
assert_equal [nil, nil], events[1][2]['float_array']
|
|
277
|
-
float_nest = {"lat" => nil, "lon" => nil}
|
|
278
|
-
assert_equal float_nest, events[1][2]['float_nest']
|
|
279
|
-
assert_equal ',', events[1][2]['string_concat']
|
|
280
|
-
assert_equal [nil, nil], events[1][2]['string_array']
|
|
281
|
-
string_nest = {"city" => nil, "country_name" => nil}
|
|
282
|
-
assert_equal string_nest, events[1][2]['string_nest']
|
|
283
|
-
assert_equal nil, events[1][2]['unknown_city']
|
|
284
|
-
assert_equal nil, events[1][2]['undefined']
|
|
285
|
-
assert_equal [nil, nil], events[1][2]['broken_array1']
|
|
286
|
-
assert_equal [nil, nil], events[1][2]['broken_array2']
|
|
287
|
-
end
|
|
288
|
-
|
|
289
|
-
def test_emit_record_directive_multiple_record
|
|
290
|
-
d1 = create_driver(%[
|
|
291
|
-
backend_library geoip2_c
|
|
292
|
-
geoip_lookup_keys from.ip, to.ip
|
|
293
|
-
<record>
|
|
294
|
-
from_city ${city.names.en['from.ip']}
|
|
295
|
-
to_city ${city.names.en['to.ip']}
|
|
296
|
-
from_country ${country.names.en['from.ip']}
|
|
297
|
-
to_country ${country.names.en['to.ip']}
|
|
298
|
-
string_array [${country.names.en['from.ip']}, ${country.names.en['to.ip']}]
|
|
299
|
-
</record>
|
|
300
|
-
tag geoip.${tag[1]}
|
|
301
|
-
], syntax: :v0)
|
|
302
|
-
d1.run(default_tag: 'input.access') do
|
|
303
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}})
|
|
304
|
-
d1.feed({'message' => 'missing field'})
|
|
305
|
-
end
|
|
306
|
-
events = d1.events
|
|
307
|
-
assert_equal 2, events.length
|
|
308
|
-
|
|
309
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
310
|
-
assert_equal 'Mountain View', events[0][2]['from_city']
|
|
311
|
-
assert_equal 'United States', events[0][2]['from_country']
|
|
312
|
-
assert_equal 'Tokorozawa', events[0][2]['to_city']
|
|
313
|
-
assert_equal 'Japan', events[0][2]['to_country']
|
|
314
|
-
assert_equal ['United States','Japan'], events[0][2]['string_array']
|
|
315
|
-
|
|
316
|
-
assert_equal nil, events[1][2]['from_city']
|
|
317
|
-
assert_equal nil, events[1][2]['to_city']
|
|
318
|
-
assert_equal nil, events[1][2]['from_country']
|
|
319
|
-
assert_equal nil, events[1][2]['to_country']
|
|
320
|
-
assert_equal [nil, nil], events[1][2]['string_array']
|
|
321
|
-
end
|
|
322
|
-
|
|
323
|
-
def config_quoted_record
|
|
324
|
-
%[
|
|
325
|
-
backend_library geoip2_c
|
|
326
|
-
geoip_lookup_keys host
|
|
327
|
-
<record>
|
|
328
|
-
location_properties '{ "country_code" : "${country.iso_code["host"]}", "lat": ${location.latitude["host"]}, "lon": ${location.longitude["host"]} }'
|
|
329
|
-
location_string ${location.latitude['host']},${location.longitude['host']}
|
|
330
|
-
location_string2 ${country.iso_code["host"]}
|
|
331
|
-
location_array "[${location.longitude['host']},${location.latitude['host']}]"
|
|
332
|
-
location_array2 '[${location.longitude["host"]},${location.latitude["host"]}]'
|
|
333
|
-
peculiar_pattern '[GEOIP] message => {"lat":${location.latitude["host"]}, "lon":${location.longitude["host"]}}'
|
|
334
|
-
</record>
|
|
335
|
-
tag geoip.${tag[1]}
|
|
336
|
-
]
|
|
337
|
-
end
|
|
338
|
-
|
|
339
|
-
def test_emit_quoted_record
|
|
340
|
-
d1 = create_driver(config_quoted_record)
|
|
341
|
-
d1.run(default_tag: 'input.access') do
|
|
342
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
343
|
-
end
|
|
344
|
-
events = d1.events
|
|
345
|
-
assert_equal 1, events.length
|
|
346
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
347
|
-
location_properties = { "country_code" => "US", "lat" => 37.419200000000004, "lon"=> -122.0574 }
|
|
348
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
349
|
-
assert_equal '37.419200000000004,-122.0574', events[0][2]['location_string']
|
|
350
|
-
assert_equal 'US', events[0][2]['location_string2']
|
|
351
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['location_array']
|
|
352
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['location_array2']
|
|
353
|
-
assert_equal '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}', events[0][2]['peculiar_pattern']
|
|
354
|
-
end
|
|
355
|
-
|
|
356
|
-
def test_emit_v1_config_compatibility
|
|
357
|
-
d1 = create_driver(config_quoted_record)
|
|
358
|
-
d1.run(default_tag: 'input.access') do
|
|
359
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
360
|
-
end
|
|
361
|
-
events = d1.events
|
|
362
|
-
assert_equal 1, events.length
|
|
363
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
364
|
-
location_properties = { "country_code" => "US", "lat" => 37.419200000000004, "lon"=> -122.0574 }
|
|
365
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
366
|
-
assert_equal '37.419200000000004,-122.0574', events[0][2]['location_string']
|
|
367
|
-
assert_equal 'US', events[0][2]['location_string2']
|
|
368
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['location_array']
|
|
369
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['location_array2']
|
|
370
|
-
assert_equal '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}', events[0][2]['peculiar_pattern']
|
|
371
|
-
end
|
|
372
|
-
|
|
373
|
-
def test_emit_multiline_v1_config
|
|
374
|
-
d1 = create_driver(%[
|
|
375
|
-
backend_library geoip2_c
|
|
376
|
-
geoip_lookup_keys host
|
|
377
|
-
<record>
|
|
378
|
-
location_properties {
|
|
379
|
-
"city": "${city.names.en['host']}",
|
|
380
|
-
"country_code": "${country.iso_code['host']}",
|
|
381
|
-
"latitude": "${location.latitude['host']}",
|
|
382
|
-
"longitude": "${location.longitude['host']}"
|
|
383
|
-
}
|
|
384
|
-
</record>
|
|
385
|
-
tag geoip.${tag[1]}
|
|
386
|
-
])
|
|
387
|
-
d1.run(default_tag: 'input.access') do
|
|
388
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
389
|
-
end
|
|
390
|
-
events = d1.events
|
|
391
|
-
assert_equal 1, events.length
|
|
392
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
393
|
-
location_properties = { "city"=>"Mountain View", "country_code"=>"US", "latitude"=>37.419200000000004, "longitude"=>-122.0574 }
|
|
394
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
395
|
-
end
|
|
396
|
-
end
|
|
397
|
-
|
|
398
|
-
sub_test_case "geoip2_compat" do
|
|
399
|
-
def test_emit_tag_option
|
|
400
|
-
d1 = create_driver(%[
|
|
401
|
-
backend_library geoip2_compat
|
|
402
|
-
geoip_lookup_keys host
|
|
403
|
-
<record>
|
|
404
|
-
geoip_city ${city['host']}
|
|
405
|
-
</record>
|
|
406
|
-
tag geoip.${tag[1]}
|
|
407
|
-
])
|
|
408
|
-
d1.run(default_tag: 'input.access') do
|
|
409
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
410
|
-
d1.feed({'message' => 'missing field'})
|
|
411
|
-
end
|
|
412
|
-
events = d1.events
|
|
413
|
-
assert_equal 2, events.length
|
|
414
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
415
|
-
assert_equal 'Mountain View', events[0][2]['geoip_city']
|
|
416
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
417
|
-
end
|
|
418
|
-
|
|
419
|
-
def test_emit_tag_parts
|
|
420
|
-
d1 = create_driver(%[
|
|
421
|
-
backend_library geoip2_compat
|
|
422
|
-
geoip_lookup_keys host
|
|
423
|
-
<record>
|
|
424
|
-
geoip_city ${city['host']}
|
|
425
|
-
</record>
|
|
426
|
-
tag geoip.${tag[1]}.${tag[2]}.${tag[3]}
|
|
427
|
-
])
|
|
428
|
-
d1.run(default_tag: '0.1.2.3') do
|
|
429
|
-
d1.feed({'host' => '66.102.3.80'})
|
|
430
|
-
end
|
|
431
|
-
events = d1.events
|
|
432
|
-
assert_equal 1, events.length
|
|
433
|
-
assert_equal 'geoip.1.2.3', events[0][0] # tag
|
|
434
|
-
assert_equal 'Mountain View', events[0][2]['geoip_city']
|
|
435
|
-
end
|
|
436
|
-
|
|
437
|
-
def test_emit_with_dot_key
|
|
438
|
-
d1 = create_driver(%[
|
|
439
|
-
backend_library geoip2_compat
|
|
440
|
-
geoip_lookup_keys ip.origin, ip.dest
|
|
441
|
-
<record>
|
|
442
|
-
origin_country ${country_code['ip.origin']}
|
|
443
|
-
dest_country ${country_code['ip.dest']}
|
|
444
|
-
</record>
|
|
445
|
-
tag geoip.${tag[1]}
|
|
446
|
-
])
|
|
447
|
-
d1.run(default_tag: 'input.access') do
|
|
448
|
-
d1.feed({'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'})
|
|
449
|
-
end
|
|
450
|
-
events = d1.events
|
|
451
|
-
assert_equal 1, events.length
|
|
452
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
453
|
-
assert_equal 'US', events[0][2]['origin_country']
|
|
454
|
-
assert_equal 'US', events[0][2]['dest_country']
|
|
455
|
-
end
|
|
456
|
-
|
|
457
|
-
def test_emit_with_unknown_address
|
|
458
|
-
d1 = create_driver(%[
|
|
459
|
-
backend_library geoip2_compat
|
|
460
|
-
geoip_lookup_keys host
|
|
461
|
-
<record>
|
|
462
|
-
geoip_city ${city['host']}
|
|
463
|
-
geopoint [${longitude['host']}, ${latitude['host']}]
|
|
464
|
-
</record>
|
|
465
|
-
skip_adding_null_record false
|
|
466
|
-
tag geoip.${tag[1]}
|
|
467
|
-
], syntax: :v0)
|
|
468
|
-
d1.run(default_tag: 'input.access') do
|
|
469
|
-
# 203.0.113.1 is a test address described in RFC5737
|
|
470
|
-
d1.feed({'host' => '203.0.113.1', 'message' => 'invalid ip'})
|
|
471
|
-
d1.feed({'host' => '0', 'message' => 'invalid ip'})
|
|
472
|
-
end
|
|
473
|
-
events = d1.events
|
|
474
|
-
assert_equal 2, events.length
|
|
475
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
476
|
-
assert_equal nil, events[0][2]['geoip_city']
|
|
477
|
-
assert_equal 'geoip.access', events[1][0] # tag
|
|
478
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
479
|
-
end
|
|
480
|
-
|
|
481
|
-
def test_emit_with_skip_unknown_address
|
|
482
|
-
d1 = create_driver(%[
|
|
483
|
-
backend_library geoip2_compat
|
|
484
|
-
geoip_lookup_keys host
|
|
485
|
-
<record>
|
|
486
|
-
geoip_city ${city['host']}
|
|
487
|
-
geopoint [${longitude['host']}, ${latitude['host']}]
|
|
488
|
-
</record>
|
|
489
|
-
skip_adding_null_record true
|
|
490
|
-
tag geoip.${tag[1]}
|
|
491
|
-
], syntax: :v0)
|
|
492
|
-
d1.run(default_tag: 'input.access') do
|
|
493
|
-
# 203.0.113.1 is a test address described in RFC5737
|
|
494
|
-
d1.feed({'host' => '203.0.113.1', 'message' => 'invalid ip'})
|
|
495
|
-
d1.feed({'host' => '0', 'message' => 'invalid ip'})
|
|
496
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'google bot'})
|
|
497
|
-
end
|
|
498
|
-
events = d1.events
|
|
499
|
-
assert_equal 3, events.length
|
|
500
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
501
|
-
assert_equal nil, events[0][2]['geoip_city']
|
|
502
|
-
assert_equal nil, events[0][2]['geopoint']
|
|
503
|
-
assert_equal 'geoip.access', events[1][0] # tag
|
|
504
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
505
|
-
assert_equal nil, events[1][2]['geopoint']
|
|
506
|
-
assert_equal 'Mountain View', events[2][2]['geoip_city']
|
|
507
|
-
assert_equal [-122.0574, 37.419200000000004], events[2][2]['geopoint']
|
|
508
|
-
end
|
|
509
|
-
|
|
510
|
-
def test_emit_record_directive
|
|
511
|
-
d1 = create_driver(%[
|
|
512
|
-
backend_library geoip2_compat
|
|
513
|
-
geoip_lookup_keys from.ip
|
|
514
|
-
<record>
|
|
515
|
-
from_city ${city['from.ip']}
|
|
516
|
-
from_country ${country_name['from.ip']}
|
|
517
|
-
latitude ${latitude['from.ip']}
|
|
518
|
-
longitude ${longitude['from.ip']}
|
|
519
|
-
float_concat ${latitude['from.ip']},${longitude['from.ip']}
|
|
520
|
-
float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
|
|
521
|
-
float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
|
|
522
|
-
string_concat ${latitude['from.ip']},${longitude['from.ip']}
|
|
523
|
-
string_array [${city['from.ip']}, ${country_name['from.ip']}]
|
|
524
|
-
string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
|
|
525
|
-
unknown_city ${city['unknown_key']}
|
|
526
|
-
undefined ${city['undefined']}
|
|
527
|
-
broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
|
|
528
|
-
broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
|
|
529
|
-
</record>
|
|
530
|
-
tag geoip.${tag[1]}
|
|
531
|
-
], syntax: :v0)
|
|
532
|
-
d1.run(default_tag: 'input.access') do
|
|
533
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}})
|
|
534
|
-
d1.feed({'message' => 'missing field'})
|
|
535
|
-
end
|
|
536
|
-
events = d1.events
|
|
537
|
-
assert_equal 2, events.length
|
|
538
|
-
|
|
539
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
540
|
-
assert_equal 'Mountain View', events[0][2]['from_city']
|
|
541
|
-
assert_equal 'United States', events[0][2]['from_country']
|
|
542
|
-
assert_equal 37.419200000000004, events[0][2]['latitude']
|
|
543
|
-
assert_equal(-122.0574, events[0][2]['longitude'])
|
|
544
|
-
assert_equal '37.419200000000004,-122.0574', events[0][2]['float_concat']
|
|
545
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['float_array']
|
|
546
|
-
float_nest = {"lat" => 37.419200000000004, "lon" => -122.0574 }
|
|
547
|
-
assert_equal float_nest, events[0][2]['float_nest']
|
|
548
|
-
assert_equal '37.419200000000004,-122.0574', events[0][2]['string_concat']
|
|
549
|
-
assert_equal ["Mountain View", "United States"], events[0][2]['string_array']
|
|
550
|
-
string_nest = {"city" => "Mountain View", "country_name" => "United States"}
|
|
551
|
-
assert_equal string_nest, events[0][2]['string_nest']
|
|
552
|
-
assert_equal nil, events[0][2]['unknown_city']
|
|
553
|
-
assert_equal nil, events[0][2]['undefined']
|
|
554
|
-
assert_equal [-122.0574, nil], events[0][2]['broken_array1']
|
|
555
|
-
assert_equal [nil, nil], events[0][2]['broken_array2']
|
|
556
|
-
|
|
557
|
-
assert_equal nil, events[1][2]['from_city']
|
|
558
|
-
assert_equal nil, events[1][2]['from_country']
|
|
559
|
-
assert_equal nil, events[1][2]['latitude']
|
|
560
|
-
assert_equal nil, events[1][2]['longitude']
|
|
561
|
-
assert_equal ',', events[1][2]['float_concat']
|
|
562
|
-
assert_equal [nil, nil], events[1][2]['float_array']
|
|
563
|
-
float_nest = {"lat" => nil, "lon" => nil}
|
|
564
|
-
assert_equal float_nest, events[1][2]['float_nest']
|
|
565
|
-
assert_equal ',', events[1][2]['string_concat']
|
|
566
|
-
assert_equal [nil, nil], events[1][2]['string_array']
|
|
567
|
-
string_nest = {"city" => nil, "country_name" => nil}
|
|
568
|
-
assert_equal string_nest, events[1][2]['string_nest']
|
|
569
|
-
assert_equal nil, events[1][2]['unknown_city']
|
|
570
|
-
assert_equal nil, events[1][2]['undefined']
|
|
571
|
-
assert_equal [nil, nil], events[1][2]['broken_array1']
|
|
572
|
-
assert_equal [nil, nil], events[1][2]['broken_array2']
|
|
573
|
-
end
|
|
574
|
-
|
|
575
|
-
def test_emit_record_directive_multiple_record
|
|
576
|
-
d1 = create_driver(%[
|
|
577
|
-
backend_library geoip2_compat
|
|
578
|
-
geoip_lookup_keys from.ip, to.ip
|
|
579
|
-
<record>
|
|
580
|
-
from_city ${city['from.ip']}
|
|
581
|
-
to_city ${city['to.ip']}
|
|
582
|
-
from_country ${country_name['from.ip']}
|
|
583
|
-
to_country ${country_name['to.ip']}
|
|
584
|
-
string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
|
|
585
|
-
</record>
|
|
586
|
-
tag geoip.${tag[1]}
|
|
587
|
-
], syntax: :v0)
|
|
588
|
-
d1.run(default_tag: 'input.access') do
|
|
589
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}})
|
|
590
|
-
d1.feed({'message' => 'missing field'})
|
|
591
|
-
end
|
|
592
|
-
events = d1.events
|
|
593
|
-
assert_equal 2, events.length
|
|
594
|
-
|
|
595
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
596
|
-
assert_equal 'Mountain View', events[0][2]['from_city']
|
|
597
|
-
assert_equal 'United States', events[0][2]['from_country']
|
|
598
|
-
assert_equal 'Tokorozawa', events[0][2]['to_city']
|
|
599
|
-
assert_equal 'Japan', events[0][2]['to_country']
|
|
600
|
-
assert_equal ['United States','Japan'], events[0][2]['string_array']
|
|
601
|
-
|
|
602
|
-
assert_equal nil, events[1][2]['from_city']
|
|
603
|
-
assert_equal nil, events[1][2]['to_city']
|
|
604
|
-
assert_equal nil, events[1][2]['from_country']
|
|
605
|
-
assert_equal nil, events[1][2]['to_country']
|
|
606
|
-
assert_equal [nil, nil], events[1][2]['string_array']
|
|
607
|
-
end
|
|
608
|
-
|
|
609
|
-
def config_quoted_record
|
|
610
|
-
%[
|
|
611
|
-
backend_library geoip2_compat
|
|
612
|
-
geoip_lookup_keys host
|
|
613
|
-
<record>
|
|
614
|
-
location_properties '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
|
|
615
|
-
location_string ${latitude['host']},${longitude['host']}
|
|
616
|
-
location_string2 ${country_code["host"]}
|
|
617
|
-
location_array "[${longitude['host']},${latitude['host']}]"
|
|
618
|
-
location_array2 '[${longitude["host"]},${latitude["host"]}]'
|
|
619
|
-
peculiar_pattern '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
|
|
620
|
-
</record>
|
|
621
|
-
tag geoip.${tag[1]}
|
|
622
|
-
]
|
|
623
|
-
end
|
|
624
|
-
|
|
625
|
-
def test_emit_quoted_record
|
|
626
|
-
d1 = create_driver(config_quoted_record)
|
|
627
|
-
d1.run(default_tag: 'input.access') do
|
|
628
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
629
|
-
end
|
|
630
|
-
events = d1.events
|
|
631
|
-
assert_equal 1, events.length
|
|
632
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
633
|
-
location_properties = { "country_code" => "US", "lat" => 37.419200000000004, "lon"=> -122.0574 }
|
|
634
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
635
|
-
assert_equal '37.419200000000004,-122.0574', events[0][2]['location_string']
|
|
636
|
-
assert_equal 'US', events[0][2]['location_string2']
|
|
637
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['location_array']
|
|
638
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['location_array2']
|
|
639
|
-
assert_equal '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}', events[0][2]['peculiar_pattern']
|
|
640
|
-
end
|
|
641
|
-
|
|
642
|
-
def test_emit_v1_config_compatibility
|
|
643
|
-
d1 = create_driver(config_quoted_record)
|
|
644
|
-
d1.run(default_tag: 'input.access') do
|
|
645
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
646
|
-
end
|
|
647
|
-
events = d1.events
|
|
648
|
-
assert_equal 1, events.length
|
|
649
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
650
|
-
location_properties = { "country_code" => "US", "lat" => 37.419200000000004, "lon"=> -122.0574 }
|
|
651
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
652
|
-
assert_equal '37.419200000000004,-122.0574', events[0][2]['location_string']
|
|
653
|
-
assert_equal 'US', events[0][2]['location_string2']
|
|
654
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['location_array']
|
|
655
|
-
assert_equal [-122.0574, 37.419200000000004], events[0][2]['location_array2']
|
|
656
|
-
assert_equal '[GEOIP] message => {"lat":37.419200000000004, "lon":-122.0574}', events[0][2]['peculiar_pattern']
|
|
657
|
-
end
|
|
658
|
-
|
|
659
|
-
def test_emit_multiline_v1_config
|
|
660
|
-
d1 = create_driver(%[
|
|
661
|
-
backend_library geoip2_compat
|
|
662
|
-
geoip_lookup_keys host
|
|
663
|
-
<record>
|
|
664
|
-
location_properties {
|
|
665
|
-
"city": "${city['host']}",
|
|
666
|
-
"country_code": "${country_code['host']}",
|
|
667
|
-
"latitude": "${latitude['host']}",
|
|
668
|
-
"longitude": "${longitude['host']}"
|
|
669
|
-
}
|
|
670
|
-
</record>
|
|
671
|
-
tag geoip.${tag[1]}
|
|
672
|
-
])
|
|
673
|
-
d1.run(default_tag: 'input.access') do
|
|
674
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
675
|
-
end
|
|
676
|
-
events = d1.events
|
|
677
|
-
assert_equal 1, events.length
|
|
678
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
679
|
-
location_properties = { "city"=>"Mountain View", "country_code"=>"US", "latitude"=>37.419200000000004, "longitude"=>-122.0574 }
|
|
680
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
681
|
-
end
|
|
682
|
-
end
|
|
683
|
-
|
|
684
|
-
sub_test_case "geoip legacy" do
|
|
685
|
-
def test_emit_tag_option
|
|
686
|
-
d1 = create_driver(%[
|
|
687
|
-
backend_library geoip
|
|
688
|
-
geoip_lookup_keys host
|
|
689
|
-
<record>
|
|
690
|
-
geoip_city ${city['host']}
|
|
691
|
-
</record>
|
|
692
|
-
tag geoip.${tag[1]}
|
|
693
|
-
])
|
|
694
|
-
d1.run(default_tag: 'input.access') do
|
|
695
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
696
|
-
d1.feed({'message' => 'missing field'})
|
|
697
|
-
end
|
|
698
|
-
events = d1.events
|
|
699
|
-
assert_equal 2, events.length
|
|
700
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
701
|
-
assert_equal 'Mountain View', events[0][2]['geoip_city']
|
|
702
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
703
|
-
end
|
|
704
|
-
|
|
705
|
-
def test_emit_tag_parts
|
|
706
|
-
d1 = create_driver(%[
|
|
707
|
-
backend_library geoip
|
|
708
|
-
geoip_lookup_keys host
|
|
709
|
-
<record>
|
|
710
|
-
geoip_city ${city['host']}
|
|
711
|
-
</record>
|
|
712
|
-
tag geoip.${tag[1]}.${tag[2]}.${tag[3]}
|
|
713
|
-
])
|
|
714
|
-
d1.run(default_tag: '0.1.2.3') do
|
|
715
|
-
d1.feed({'host' => '66.102.3.80'})
|
|
716
|
-
end
|
|
717
|
-
events = d1.events
|
|
718
|
-
assert_equal 1, events.length
|
|
719
|
-
assert_equal 'geoip.1.2.3', events[0][0] # tag
|
|
720
|
-
assert_equal 'Mountain View', events[0][2]['geoip_city']
|
|
721
|
-
end
|
|
722
|
-
|
|
723
|
-
def test_emit_with_dot_key
|
|
724
|
-
d1 = create_driver(%[
|
|
725
|
-
backend_library geoip
|
|
726
|
-
geoip_lookup_keys ip.origin, ip.dest
|
|
727
|
-
<record>
|
|
728
|
-
origin_country ${country_code['ip.origin']}
|
|
729
|
-
dest_country ${country_code['ip.dest']}
|
|
730
|
-
</record>
|
|
731
|
-
tag geoip.${tag[1]}
|
|
732
|
-
])
|
|
733
|
-
d1.run(default_tag: 'input.access') do
|
|
734
|
-
d1.feed({'ip.origin' => '66.102.3.80', 'ip.dest' => '8.8.8.8'})
|
|
735
|
-
end
|
|
736
|
-
events = d1.events
|
|
737
|
-
assert_equal 1, events.length
|
|
738
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
739
|
-
assert_equal 'US', events[0][2]['origin_country']
|
|
740
|
-
assert_equal 'US', events[0][2]['dest_country']
|
|
741
|
-
end
|
|
742
|
-
|
|
743
|
-
def test_emit_nested_attr
|
|
744
|
-
d1 = create_driver(%[
|
|
745
|
-
backend_library geoip
|
|
746
|
-
geoip_lookup_keys host.ip
|
|
747
|
-
<record>
|
|
748
|
-
geoip_city ${city['host.ip']}
|
|
749
|
-
</record>
|
|
750
|
-
tag geoip.${tag[1]}
|
|
751
|
-
])
|
|
752
|
-
d1.run(default_tag: 'input.access') do
|
|
753
|
-
d1.feed({'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'})
|
|
754
|
-
d1.feed({'message' => 'missing field'})
|
|
755
|
-
end
|
|
756
|
-
events = d1.events
|
|
757
|
-
assert_equal 2, events.length
|
|
758
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
759
|
-
assert_equal 'Mountain View', events[0][2]['geoip_city']
|
|
760
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
761
|
-
end
|
|
762
|
-
|
|
763
|
-
def test_emit_with_unknown_address
|
|
764
|
-
d1 = create_driver(%[
|
|
765
|
-
backend_library geoip
|
|
766
|
-
geoip_lookup_keys host
|
|
767
|
-
<record>
|
|
768
|
-
geoip_city ${city['host']}
|
|
769
|
-
geopoint [${longitude['host']}, ${latitude['host']}]
|
|
770
|
-
</record>
|
|
771
|
-
skip_adding_null_record false
|
|
772
|
-
tag geoip.${tag[1]}
|
|
773
|
-
], syntax: :v0)
|
|
774
|
-
d1.run(default_tag: 'input.access') do
|
|
775
|
-
# 203.0.113.1 is a test address described in RFC5737
|
|
776
|
-
d1.feed({'host' => '203.0.113.1', 'message' => 'invalid ip'})
|
|
777
|
-
d1.feed({'host' => '0', 'message' => 'invalid ip'})
|
|
778
|
-
end
|
|
779
|
-
events = d1.events
|
|
780
|
-
assert_equal 2, events.length
|
|
781
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
782
|
-
assert_equal nil, events[0][2]['geoip_city']
|
|
783
|
-
assert_equal 'geoip.access', events[1][0] # tag
|
|
784
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
785
|
-
end
|
|
786
|
-
|
|
787
|
-
def test_emit_with_skip_unknown_address
|
|
788
|
-
d1 = create_driver(%[
|
|
789
|
-
backend_library geoip
|
|
790
|
-
geoip_lookup_keys host
|
|
791
|
-
<record>
|
|
792
|
-
geoip_city ${city['host']}
|
|
793
|
-
geopoint [${longitude['host']}, ${latitude['host']}]
|
|
794
|
-
</record>
|
|
795
|
-
skip_adding_null_record true
|
|
796
|
-
tag geoip.${tag[1]}
|
|
797
|
-
], syntax: :v0)
|
|
798
|
-
d1.run(default_tag: 'input.access') do
|
|
799
|
-
# 203.0.113.1 is a test address described in RFC5737
|
|
800
|
-
d1.feed({'host' => '203.0.113.1', 'message' => 'invalid ip'})
|
|
801
|
-
d1.feed({'host' => '0', 'message' => 'invalid ip'})
|
|
802
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'google bot'})
|
|
803
|
-
end
|
|
804
|
-
events = d1.events
|
|
805
|
-
assert_equal 3, events.length
|
|
806
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
807
|
-
assert_equal nil, events[0][2]['geoip_city']
|
|
808
|
-
assert_equal nil, events[0][2]['geopoint']
|
|
809
|
-
assert_equal 'geoip.access', events[1][0] # tag
|
|
810
|
-
assert_equal nil, events[1][2]['geoip_city']
|
|
811
|
-
assert_equal nil, events[1][2]['geopoint']
|
|
812
|
-
assert_equal 'Mountain View', events[2][2]['geoip_city']
|
|
813
|
-
assert_equal [-122.05740356445312, 37.4192008972168], events[2][2]['geopoint']
|
|
814
|
-
end
|
|
815
|
-
|
|
816
|
-
def test_emit_multiple_key
|
|
817
|
-
d1 = create_driver(%[
|
|
818
|
-
backend_library geoip
|
|
819
|
-
geoip_lookup_keys from.ip, to.ip
|
|
820
|
-
<record>
|
|
821
|
-
from_city ${city['from.ip']}
|
|
822
|
-
to_city ${city['to.ip']}
|
|
823
|
-
</record>
|
|
824
|
-
tag geoip.${tag[1]}
|
|
825
|
-
])
|
|
826
|
-
d1.run(default_tag: 'input.access') do
|
|
827
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}})
|
|
828
|
-
d1.feed({'message' => 'missing field'})
|
|
829
|
-
end
|
|
830
|
-
events = d1.events
|
|
831
|
-
assert_equal 2, events.length
|
|
832
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
833
|
-
assert_equal 'Mountain View', events[0][2]['from_city']
|
|
834
|
-
assert_equal 'Tokorozawa', events[0][2]['to_city']
|
|
835
|
-
assert_equal nil, events[1][2]['from_city']
|
|
836
|
-
assert_equal nil, events[1][2]['to_city']
|
|
837
|
-
end
|
|
838
|
-
|
|
839
|
-
def test_emit_multiple_key_multiple_record
|
|
840
|
-
d1 = create_driver(%[
|
|
841
|
-
backend_library geoip
|
|
842
|
-
geoip_lookup_keys from.ip, to.ip
|
|
843
|
-
<record>
|
|
844
|
-
from_city ${city['from.ip']}
|
|
845
|
-
from_country ${country_name['from.ip']}
|
|
846
|
-
to_city ${city['to.ip']}
|
|
847
|
-
to_country ${country_name['to.ip']}
|
|
848
|
-
</record>
|
|
849
|
-
tag geoip.${tag[1]}
|
|
850
|
-
])
|
|
851
|
-
d1.run(default_tag: 'input.access') do
|
|
852
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}})
|
|
853
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}})
|
|
854
|
-
d1.feed({'message' => 'missing field'})
|
|
855
|
-
end
|
|
856
|
-
events = d1.events
|
|
857
|
-
assert_equal 3, events.length
|
|
858
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
859
|
-
assert_equal 'Mountain View', events[0][2]['from_city']
|
|
860
|
-
assert_equal 'United States', events[0][2]['from_country']
|
|
861
|
-
assert_equal 'Tokorozawa', events[0][2]['to_city']
|
|
862
|
-
assert_equal 'Japan', events[0][2]['to_country']
|
|
863
|
-
|
|
864
|
-
assert_equal 'Mountain View', events[1][2]['from_city']
|
|
865
|
-
assert_equal 'United States', events[1][2]['from_country']
|
|
866
|
-
assert_equal nil, events[1][2]['to_city']
|
|
867
|
-
assert_equal nil, events[1][2]['to_country']
|
|
868
|
-
|
|
869
|
-
assert_equal nil, events[2][2]['from_city']
|
|
870
|
-
assert_equal nil, events[2][2]['from_country']
|
|
871
|
-
assert_equal nil, events[2][2]['to_city']
|
|
872
|
-
assert_equal nil, events[2][2]['to_country']
|
|
873
|
-
end
|
|
874
|
-
|
|
875
|
-
def test_emit_record_directive
|
|
876
|
-
d1 = create_driver(%[
|
|
877
|
-
backend_library geoip
|
|
878
|
-
geoip_lookup_keys from.ip
|
|
879
|
-
<record>
|
|
880
|
-
from_city ${city['from.ip']}
|
|
881
|
-
from_country ${country_name['from.ip']}
|
|
882
|
-
latitude ${latitude['from.ip']}
|
|
883
|
-
longitude ${longitude['from.ip']}
|
|
884
|
-
float_concat ${latitude['from.ip']},${longitude['from.ip']}
|
|
885
|
-
float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
|
|
886
|
-
float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
|
|
887
|
-
string_concat ${latitude['from.ip']},${longitude['from.ip']}
|
|
888
|
-
string_array [${city['from.ip']}, ${country_name['from.ip']}]
|
|
889
|
-
string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
|
|
890
|
-
unknown_city ${city['unknown_key']}
|
|
891
|
-
undefined ${city['undefined']}
|
|
892
|
-
broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
|
|
893
|
-
broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
|
|
894
|
-
</record>
|
|
895
|
-
tag geoip.${tag[1]}
|
|
896
|
-
], syntax: :v0)
|
|
897
|
-
d1.run(default_tag: 'input.access') do
|
|
898
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}})
|
|
899
|
-
d1.feed({'message' => 'missing field'})
|
|
900
|
-
end
|
|
901
|
-
events = d1.events
|
|
902
|
-
assert_equal 2, events.length
|
|
903
|
-
|
|
904
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
905
|
-
assert_equal 'Mountain View', events[0][2]['from_city']
|
|
906
|
-
assert_equal 'United States', events[0][2]['from_country']
|
|
907
|
-
assert_equal 37.4192008972168, events[0][2]['latitude']
|
|
908
|
-
assert_equal(-122.05740356445312, events[0][2]['longitude'])
|
|
909
|
-
assert_equal '37.4192008972168,-122.05740356445312', events[0][2]['float_concat']
|
|
910
|
-
assert_equal [-122.05740356445312, 37.4192008972168], events[0][2]['float_array']
|
|
911
|
-
float_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
|
|
912
|
-
assert_equal float_nest, events[0][2]['float_nest']
|
|
913
|
-
assert_equal '37.4192008972168,-122.05740356445312', events[0][2]['string_concat']
|
|
914
|
-
assert_equal ["Mountain View", "United States"], events[0][2]['string_array']
|
|
915
|
-
string_nest = {"city" => "Mountain View", "country_name" => "United States"}
|
|
916
|
-
assert_equal string_nest, events[0][2]['string_nest']
|
|
917
|
-
assert_equal nil, events[0][2]['unknown_city']
|
|
918
|
-
assert_equal nil, events[0][2]['undefined']
|
|
919
|
-
assert_equal [-122.05740356445312, nil], events[0][2]['broken_array1']
|
|
920
|
-
assert_equal [nil, nil], events[0][2]['broken_array2']
|
|
921
|
-
|
|
922
|
-
assert_equal nil, events[1][2]['from_city']
|
|
923
|
-
assert_equal nil, events[1][2]['from_country']
|
|
924
|
-
assert_equal nil, events[1][2]['latitude']
|
|
925
|
-
assert_equal nil, events[1][2]['longitude']
|
|
926
|
-
assert_equal ',', events[1][2]['float_concat']
|
|
927
|
-
assert_equal [nil, nil], events[1][2]['float_array']
|
|
928
|
-
float_nest = {"lat" => nil, "lon" => nil}
|
|
929
|
-
assert_equal float_nest, events[1][2]['float_nest']
|
|
930
|
-
assert_equal ',', events[1][2]['string_concat']
|
|
931
|
-
assert_equal [nil, nil], events[1][2]['string_array']
|
|
932
|
-
string_nest = {"city" => nil, "country_name" => nil}
|
|
933
|
-
assert_equal string_nest, events[1][2]['string_nest']
|
|
934
|
-
assert_equal nil, events[1][2]['unknown_city']
|
|
935
|
-
assert_equal nil, events[1][2]['undefined']
|
|
936
|
-
assert_equal [nil, nil], events[1][2]['broken_array1']
|
|
937
|
-
assert_equal [nil, nil], events[1][2]['broken_array2']
|
|
938
|
-
end
|
|
939
|
-
|
|
940
|
-
def test_emit_record_directive_multiple_record
|
|
941
|
-
d1 = create_driver(%[
|
|
942
|
-
backend_library geoip
|
|
943
|
-
geoip_lookup_keys from.ip, to.ip
|
|
944
|
-
<record>
|
|
945
|
-
from_city ${city['from.ip']}
|
|
946
|
-
to_city ${city['to.ip']}
|
|
947
|
-
from_country ${country_name['from.ip']}
|
|
948
|
-
to_country ${country_name['to.ip']}
|
|
949
|
-
string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
|
|
950
|
-
</record>
|
|
951
|
-
tag geoip.${tag[1]}
|
|
952
|
-
], syntax: :v0)
|
|
953
|
-
d1.run(default_tag: 'input.access') do
|
|
954
|
-
d1.feed({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.15.42'}})
|
|
955
|
-
d1.feed({'message' => 'missing field'})
|
|
956
|
-
end
|
|
957
|
-
events = d1.events
|
|
958
|
-
assert_equal 2, events.length
|
|
959
|
-
|
|
960
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
961
|
-
assert_equal 'Mountain View', events[0][2]['from_city']
|
|
962
|
-
assert_equal 'United States', events[0][2]['from_country']
|
|
963
|
-
assert_equal 'Tokorozawa', events[0][2]['to_city']
|
|
964
|
-
assert_equal 'Japan', events[0][2]['to_country']
|
|
965
|
-
assert_equal ['United States','Japan'], events[0][2]['string_array']
|
|
966
|
-
|
|
967
|
-
assert_equal nil, events[1][2]['from_city']
|
|
968
|
-
assert_equal nil, events[1][2]['to_city']
|
|
969
|
-
assert_equal nil, events[1][2]['from_country']
|
|
970
|
-
assert_equal nil, events[1][2]['to_country']
|
|
971
|
-
assert_equal [nil, nil], events[1][2]['string_array']
|
|
972
|
-
end
|
|
973
|
-
|
|
974
|
-
def config_quoted_record
|
|
975
|
-
%[
|
|
976
|
-
backend_library geoip
|
|
977
|
-
geoip_lookup_keys host
|
|
978
|
-
<record>
|
|
979
|
-
location_properties '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
|
|
980
|
-
location_string ${latitude['host']},${longitude['host']}
|
|
981
|
-
location_string2 ${country_code["host"]}
|
|
982
|
-
location_array "[${longitude['host']},${latitude['host']}]"
|
|
983
|
-
location_array2 '[${longitude["host"]},${latitude["host"]}]'
|
|
984
|
-
peculiar_pattern '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
|
|
985
|
-
</record>
|
|
986
|
-
tag geoip.${tag[1]}
|
|
987
|
-
]
|
|
988
|
-
end
|
|
989
|
-
|
|
990
|
-
def test_emit_quoted_record
|
|
991
|
-
d1 = create_driver(config_quoted_record)
|
|
992
|
-
d1.run(default_tag: 'input.access') do
|
|
993
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
994
|
-
end
|
|
995
|
-
events = d1.events
|
|
996
|
-
assert_equal 1, events.length
|
|
997
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
998
|
-
location_properties = { "country_code" => "US", "lat" => 37.4192008972168, "lon"=> -122.05740356445312 }
|
|
999
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
1000
|
-
assert_equal '37.4192008972168,-122.05740356445312', events[0][2]['location_string']
|
|
1001
|
-
assert_equal 'US', events[0][2]['location_string2']
|
|
1002
|
-
assert_equal [-122.05740356445312, 37.4192008972168], events[0][2]['location_array']
|
|
1003
|
-
assert_equal [-122.05740356445312, 37.4192008972168], events[0][2]['location_array2']
|
|
1004
|
-
assert_equal '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}', events[0][2]['peculiar_pattern']
|
|
1005
|
-
end
|
|
1006
|
-
|
|
1007
|
-
def test_emit_v1_config_compatibility
|
|
1008
|
-
d1 = create_driver(config_quoted_record)
|
|
1009
|
-
d1.run(default_tag: 'input.access') do
|
|
1010
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
1011
|
-
end
|
|
1012
|
-
events = d1.events
|
|
1013
|
-
assert_equal 1, events.length
|
|
1014
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
1015
|
-
location_properties = { "country_code" => "US", "lat" => 37.4192008972168, "lon"=> -122.05740356445312 }
|
|
1016
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
1017
|
-
assert_equal '37.4192008972168,-122.05740356445312', events[0][2]['location_string']
|
|
1018
|
-
assert_equal 'US', events[0][2]['location_string2']
|
|
1019
|
-
assert_equal [-122.05740356445312, 37.4192008972168], events[0][2]['location_array']
|
|
1020
|
-
assert_equal [-122.05740356445312, 37.4192008972168], events[0][2]['location_array2']
|
|
1021
|
-
assert_equal '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}', events[0][2]['peculiar_pattern']
|
|
1022
|
-
end
|
|
1023
|
-
|
|
1024
|
-
def test_emit_multiline_v1_config
|
|
1025
|
-
d1 = create_driver(%[
|
|
1026
|
-
backend_library geoip
|
|
1027
|
-
geoip_lookup_keys host
|
|
1028
|
-
<record>
|
|
1029
|
-
location_properties {
|
|
1030
|
-
"city": "${city['host']}",
|
|
1031
|
-
"country_code": "${country_code['host']}",
|
|
1032
|
-
"latitude": "${latitude['host']}",
|
|
1033
|
-
"longitude": "${longitude['host']}"
|
|
1034
|
-
}
|
|
1035
|
-
</record>
|
|
1036
|
-
tag geoip.${tag[1]}
|
|
1037
|
-
])
|
|
1038
|
-
d1.run(default_tag: 'input.access') do
|
|
1039
|
-
d1.feed({'host' => '66.102.3.80', 'message' => 'valid ip'})
|
|
1040
|
-
end
|
|
1041
|
-
events = d1.events
|
|
1042
|
-
assert_equal 1, events.length
|
|
1043
|
-
assert_equal 'geoip.access', events[0][0] # tag
|
|
1044
|
-
location_properties = { "city"=>"Mountain View", "country_code"=>"US", "latitude"=>37.4192008972168, "longitude"=>-122.05740356445312 }
|
|
1045
|
-
assert_equal location_properties, events[0][2]['location_properties']
|
|
1046
|
-
end
|
|
1047
|
-
end
|
|
1048
|
-
end
|