fluent-plugin-ipinfo 1.0.2 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +45 -1
- data/fluent-plugin-ipinfo.gemspec +1 -1
- data/lib/fluent/plugin/filter_ipinfo.rb +59 -24
- data/test/plugin/test_filter_ipinfo.rb +164 -28
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2e715ffb9f1f67768b47a75a2909da48be93f43c882d78e135d69c70a3dc9678
|
|
4
|
+
data.tar.gz: 3c002e087f57d526771d43408f2692a0c459b4b266a14623f655d19e83fc79d8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f9269e76e2018277b28e2572b540612fa968d7ebb71ea2e60580f762f3bdfbed5c5aa83d799a4b9ac085c6ffca07a451be4cf01988ce5c5e6409ee421e9031e2
|
|
7
|
+
data.tar.gz: 07210a8a3082bfe1ce5dfb47d0433deb9157d7aaeff4e03b7e3bdb90233e4535547ade6c9d4ad229f11ce0506c1cd926ffcf7f1540aea98b4cfa1ab08d73ae6b
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -39,7 +39,7 @@ In this example, the following event:
|
|
|
39
39
|
}
|
|
40
40
|
```
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
would be enriched and returned as following:
|
|
43
43
|
|
|
44
44
|
```json
|
|
45
45
|
{
|
|
@@ -68,6 +68,8 @@ Would be enriched and returned as following:
|
|
|
68
68
|
The token to be used with the IPInfo API for paid plans.
|
|
69
69
|
To use the free plan (limited to 50k requests per month), do not use the `access_token` parameter.
|
|
70
70
|
|
|
71
|
+
If the value provided for `access_token` is an empty string (`""` or `" "`), the default value (`nil`) is used instead.
|
|
72
|
+
|
|
71
73
|
### `key_name`
|
|
72
74
|
|
|
73
75
|
| type | required | default |
|
|
@@ -76,6 +78,8 @@ To use the free plan (limited to 50k requests per month), do not use the `access
|
|
|
76
78
|
|
|
77
79
|
The name of the key containing the IP address.
|
|
78
80
|
|
|
81
|
+
If the value provided for `key_name` is an empty string (`""` or `" "`) or `nil`, the default value (`ip_address`) is used instead.
|
|
82
|
+
|
|
79
83
|
### `out_key`
|
|
80
84
|
|
|
81
85
|
| type | required | default |
|
|
@@ -84,6 +88,46 @@ The name of the key containing the IP address.
|
|
|
84
88
|
|
|
85
89
|
The name of the key to store the geographical location data in.
|
|
86
90
|
|
|
91
|
+
If the value provided for `out_key` is an empty string (`""` or `" "`) or `nil`, the default value (`ipinfo`) is used instead.
|
|
92
|
+
|
|
93
|
+
If the record has already a key with the same name as the value of `out_key`, its value will be overwritten with the geographical location data as shown in the example below:
|
|
94
|
+
|
|
95
|
+
```xml
|
|
96
|
+
<filter foo.bar>
|
|
97
|
+
@type ipinfo
|
|
98
|
+
access_token 1a2b3c4d5e
|
|
99
|
+
key_name ip_address
|
|
100
|
+
out_key data
|
|
101
|
+
fields ["country_name", "region", "city", "latitude", "longitude"]
|
|
102
|
+
</filter>
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The following event:
|
|
106
|
+
|
|
107
|
+
```json
|
|
108
|
+
{
|
|
109
|
+
"message":"Can you get me the geographical location for this IP addresse ?",
|
|
110
|
+
"ip_address":"8.8.8.8",
|
|
111
|
+
"data": "This value is going to be overwritten."
|
|
112
|
+
}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
would be enriched and returned as following:
|
|
116
|
+
|
|
117
|
+
```json
|
|
118
|
+
{
|
|
119
|
+
"message": "Can you get me the geographical location for this IP addresse ?",
|
|
120
|
+
"ip_address": "8.8.8.8",
|
|
121
|
+
"data": {
|
|
122
|
+
"country_name": "United States",
|
|
123
|
+
"region": "California",
|
|
124
|
+
"city": "Mountain View",
|
|
125
|
+
"latitude": "37.4056",
|
|
126
|
+
"longitude": "-122.0775"
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
```
|
|
130
|
+
|
|
87
131
|
### `fields`
|
|
88
132
|
|
|
89
133
|
| type | required | default |
|
|
@@ -35,41 +35,76 @@ module Fluent::Plugin
|
|
|
35
35
|
|
|
36
36
|
def initialize
|
|
37
37
|
super
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
@
|
|
38
|
+
# Keep the data in cache for one week
|
|
39
|
+
# Number of entries to keep in cache
|
|
40
|
+
@ipinfo_cache_settings = {:ttl => 60 * 60 * 24 * 7, :maxsize => 4096}
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
def configure(conf)
|
|
44
44
|
super
|
|
45
|
+
|
|
46
|
+
if !@access_token.nil? and @access_token.strip.empty?
|
|
47
|
+
log.warn "access_token value is an empty string. Falling back to default value."
|
|
48
|
+
@access_token = nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
if @key_name.nil? or @key_name.strip.empty?
|
|
52
|
+
log.warn "key_name value is '#{@key_name}'. Falling back to default value."
|
|
53
|
+
@key_name = 'ip_address'
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
if @out_key.nil? or @out_key.strip.empty?
|
|
57
|
+
log.warn "out_key value is '#{@out_key}'. Falling back to default value."
|
|
58
|
+
@out_key = 'ipinfo'
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Delete duplicates and empty fields (nil values included)
|
|
62
|
+
@fields = @fields.uniq.delete_if {|f| f.strip.empty? or f.nil?}
|
|
63
|
+
|
|
64
|
+
unless @fields.length() > 0
|
|
65
|
+
raise Fluent::ConfigError, "Fields array is empty. You need to specify at least one field."
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Create the IPInfo client handler
|
|
45
69
|
unless @access_token.nil?
|
|
46
|
-
@ipinfo_handler = IPinfo::create(@access_token, @
|
|
70
|
+
@ipinfo_handler = IPinfo::create(@access_token, @ipinfo_cache_settings)
|
|
47
71
|
else
|
|
48
|
-
@ipinfo_handler = IPinfo::create(nil, @
|
|
72
|
+
@ipinfo_handler = IPinfo::create(nil, @ipinfo_cache_settings)
|
|
49
73
|
end
|
|
50
74
|
end
|
|
51
75
|
|
|
52
76
|
def filter(tag, time, record)
|
|
53
|
-
|
|
54
|
-
unless
|
|
55
|
-
#
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
#
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
77
|
+
# Check that the record has a key whose name is the value of @key_name
|
|
78
|
+
unless record.key?(@key_name)
|
|
79
|
+
log.error "key '#{@key_name}' is not present in the record. Ignoring the record."
|
|
80
|
+
else
|
|
81
|
+
ip_address = record[@key_name]
|
|
82
|
+
# Check that the ip_address is not 'nil'
|
|
83
|
+
unless ip_address.nil?
|
|
84
|
+
begin
|
|
85
|
+
# Fetch geolocation details using IPInfo API
|
|
86
|
+
ipinfo_details = @ipinfo_handler.details(ip_address)
|
|
87
|
+
# IPInfo ruby wrapper returns a dict based on symbols, we need to stringify the symbols
|
|
88
|
+
# to be able to use them easily
|
|
89
|
+
all_details = ipinfo_details.all
|
|
90
|
+
geodata = all_details.stringify_keys
|
|
91
|
+
# Get the final list of fields by running a join operation on the fields provided by the user and the ones
|
|
92
|
+
# returned by IPInfo API
|
|
93
|
+
ipinfo_returned_fields = geodata.keys
|
|
94
|
+
final_fields = ipinfo_returned_fields & @fields
|
|
95
|
+
if final_fields.length() != @fields.length()
|
|
96
|
+
ignored_fields = @fields - final_fields
|
|
97
|
+
ignored_fields.each{|field|
|
|
98
|
+
log.warn "Field '#{@field}' not present in IPInfo payload. Ignoring it."
|
|
99
|
+
}
|
|
100
|
+
end
|
|
101
|
+
# Extract a subhash from the geolocation data returned by IPInfo API using the final_fields list as keys.
|
|
102
|
+
record[@out_key] = extract_subhash(geodata, final_fields)
|
|
103
|
+
rescue Exception => e
|
|
104
|
+
log.error 'An error occured while fetching geographical location data.', error: e
|
|
105
|
+
log.warn_backtrace e.backtrace
|
|
106
|
+
end
|
|
70
107
|
end
|
|
71
|
-
# Extract a subhash from the geolocation data returned by IPInfo API using the final_fields list as keys.
|
|
72
|
-
record[@out_key] = extract_subhash(geodata, final_fields)
|
|
73
108
|
end
|
|
74
109
|
record
|
|
75
110
|
end
|
|
@@ -51,14 +51,105 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
51
51
|
d.filtered_records
|
|
52
52
|
end
|
|
53
53
|
|
|
54
|
+
sub_test_case 'plugin will use default values' do
|
|
55
|
+
test 'access_token nil' do
|
|
56
|
+
config = %[
|
|
57
|
+
@type ipinfo
|
|
58
|
+
access_token
|
|
59
|
+
]
|
|
60
|
+
d = create_driver(config)
|
|
61
|
+
assert_equal d.instance.access_token, nil
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
test 'access_token empty string' do
|
|
65
|
+
config = %[
|
|
66
|
+
@type ipinfo
|
|
67
|
+
access_token ""
|
|
68
|
+
]
|
|
69
|
+
d = create_driver(config)
|
|
70
|
+
assert_equal d.instance.access_token, nil
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
test 'key_name nil' do
|
|
74
|
+
config = %[
|
|
75
|
+
@type ipinfo
|
|
76
|
+
key_name
|
|
77
|
+
]
|
|
78
|
+
d = create_driver(config)
|
|
79
|
+
assert_equal d.instance.key_name, 'ip_address'
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
test 'key_name empty string' do
|
|
83
|
+
config = %[
|
|
84
|
+
@type ipinfo
|
|
85
|
+
key_name
|
|
86
|
+
]
|
|
87
|
+
d = create_driver(config)
|
|
88
|
+
assert_equal d.instance.key_name, 'ip_address'
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
test 'out_key nil' do
|
|
92
|
+
config = %[
|
|
93
|
+
@type ipinfo
|
|
94
|
+
out_key
|
|
95
|
+
]
|
|
96
|
+
d = create_driver(config)
|
|
97
|
+
assert_equal d.instance.out_key, 'ipinfo'
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
test 'out_key empty string' do
|
|
101
|
+
config = %[
|
|
102
|
+
@type ipinfo
|
|
103
|
+
out_key
|
|
104
|
+
]
|
|
105
|
+
d = create_driver(config)
|
|
106
|
+
assert_equal d.instance.out_key, 'ipinfo'
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
sub_test_case 'plugin will raise Fluent::ConfigError' do
|
|
111
|
+
test 'empty fields' do
|
|
112
|
+
config = %[
|
|
113
|
+
@type ipinfo
|
|
114
|
+
fields []
|
|
115
|
+
]
|
|
116
|
+
assert_raise Fluent::ConfigError do
|
|
117
|
+
create_driver(config)
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
test 'fields with one empty string' do
|
|
122
|
+
config = %[
|
|
123
|
+
@type ipinfo
|
|
124
|
+
fields [""]
|
|
125
|
+
]
|
|
126
|
+
assert_raise Fluent::ConfigError do
|
|
127
|
+
create_driver(config)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
test 'fields with multiple empty string' do
|
|
132
|
+
config = %[
|
|
133
|
+
@type ipinfo
|
|
134
|
+
fields [""]
|
|
135
|
+
]
|
|
136
|
+
assert_raise Fluent::ConfigError do
|
|
137
|
+
create_driver(config)
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
54
142
|
sub_test_case 'plugin will fetch geolocation data' do
|
|
55
143
|
test 'add ipinfo to record with default fields' do
|
|
56
|
-
|
|
144
|
+
config = CONFIG
|
|
57
145
|
messages = [
|
|
58
|
-
{
|
|
146
|
+
{
|
|
147
|
+
'ip_address' => '8.8.8.8'
|
|
148
|
+
}
|
|
59
149
|
]
|
|
60
150
|
expected = [
|
|
61
|
-
{
|
|
151
|
+
{
|
|
152
|
+
'ip_address' => '8.8.8.8',
|
|
62
153
|
'ipinfo' => {
|
|
63
154
|
'country_name' => 'United States',
|
|
64
155
|
'region' => 'California',
|
|
@@ -68,20 +159,23 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
68
159
|
}
|
|
69
160
|
}
|
|
70
161
|
]
|
|
71
|
-
filtered_records = filter(
|
|
162
|
+
filtered_records = filter(config, messages)
|
|
72
163
|
assert_equal(expected, filtered_records)
|
|
73
164
|
end
|
|
74
165
|
|
|
75
166
|
test 'add ipinfo to record with custom key_name' do
|
|
76
|
-
|
|
167
|
+
config = %[
|
|
77
168
|
@type ipinfo
|
|
78
169
|
key_name ip
|
|
79
170
|
]
|
|
80
171
|
messages = [
|
|
81
|
-
{
|
|
172
|
+
{
|
|
173
|
+
'ip' => '8.8.8.8'
|
|
174
|
+
}
|
|
82
175
|
]
|
|
83
176
|
expected = [
|
|
84
|
-
{
|
|
177
|
+
{
|
|
178
|
+
'ip' => '8.8.8.8',
|
|
85
179
|
'ipinfo' => {
|
|
86
180
|
'country_name' => 'United States',
|
|
87
181
|
'region' => 'California',
|
|
@@ -91,20 +185,23 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
91
185
|
}
|
|
92
186
|
}
|
|
93
187
|
]
|
|
94
|
-
filtered_records = filter(
|
|
188
|
+
filtered_records = filter(config, messages)
|
|
95
189
|
assert_equal(expected, filtered_records)
|
|
96
190
|
end
|
|
97
191
|
|
|
98
192
|
test 'add ipinfo to record with custom out_key' do
|
|
99
|
-
|
|
193
|
+
config = %[
|
|
100
194
|
@type ipinfo
|
|
101
195
|
out_key geodata
|
|
102
196
|
]
|
|
103
197
|
messages = [
|
|
104
|
-
{
|
|
198
|
+
{
|
|
199
|
+
'ip_address' => '8.8.8.8'
|
|
200
|
+
}
|
|
105
201
|
]
|
|
106
202
|
expected = [
|
|
107
|
-
{
|
|
203
|
+
{
|
|
204
|
+
'ip_address' => '8.8.8.8',
|
|
108
205
|
'geodata' => {
|
|
109
206
|
'country_name' => 'United States',
|
|
110
207
|
'region' => 'California',
|
|
@@ -114,62 +211,98 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
114
211
|
}
|
|
115
212
|
}
|
|
116
213
|
]
|
|
117
|
-
filtered_records = filter(
|
|
214
|
+
filtered_records = filter(config, messages)
|
|
215
|
+
assert_equal(expected, filtered_records)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
test 'add ipinfo to record with custom out_key that is already defined' do
|
|
219
|
+
config = %[
|
|
220
|
+
@type ipinfo
|
|
221
|
+
out_key geodata
|
|
222
|
+
]
|
|
223
|
+
messages = [
|
|
224
|
+
{
|
|
225
|
+
'ip_address' => '8.8.8.8',
|
|
226
|
+
'geodata' => 'bye!'
|
|
227
|
+
}
|
|
228
|
+
]
|
|
229
|
+
expected = [
|
|
230
|
+
{
|
|
231
|
+
'ip_address' => '8.8.8.8',
|
|
232
|
+
'geodata' => {
|
|
233
|
+
'country_name' => 'United States',
|
|
234
|
+
'region' => 'California',
|
|
235
|
+
'city' => 'Mountain View',
|
|
236
|
+
'latitude' => '37.4056',
|
|
237
|
+
'longitude' => '-122.0775'
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
]
|
|
241
|
+
filtered_records = filter(config, messages)
|
|
118
242
|
assert_equal(expected, filtered_records)
|
|
119
243
|
end
|
|
120
244
|
|
|
121
245
|
test 'add ipinfo to record with custom fields' do
|
|
122
|
-
|
|
246
|
+
config = %[
|
|
123
247
|
@type ipinfo
|
|
124
248
|
fields ["country_name", "city"]
|
|
125
249
|
]
|
|
126
250
|
messages = [
|
|
127
|
-
{
|
|
251
|
+
{
|
|
252
|
+
'ip_address' => '8.8.8.8'
|
|
253
|
+
}
|
|
128
254
|
]
|
|
129
255
|
expected = [
|
|
130
|
-
{
|
|
256
|
+
{
|
|
257
|
+
'ip_address' => '8.8.8.8',
|
|
131
258
|
'ipinfo' => {
|
|
132
259
|
'country_name' => 'United States',
|
|
133
260
|
'city' => 'Mountain View'
|
|
134
261
|
}
|
|
135
262
|
}
|
|
136
263
|
]
|
|
137
|
-
filtered_records = filter(
|
|
264
|
+
filtered_records = filter(config, messages)
|
|
138
265
|
assert_equal(expected, filtered_records)
|
|
139
266
|
end
|
|
140
267
|
|
|
141
268
|
test 'add ipinfo to record with custom key_name and custom fields' do
|
|
142
|
-
|
|
269
|
+
config = %[
|
|
143
270
|
@type ipinfo
|
|
144
271
|
key_name ip
|
|
145
272
|
fields ["country_name", "city"]
|
|
146
273
|
]
|
|
147
274
|
messages = [
|
|
148
|
-
{
|
|
275
|
+
{
|
|
276
|
+
'ip' => '8.8.8.8'
|
|
277
|
+
}
|
|
149
278
|
]
|
|
150
279
|
expected = [
|
|
151
|
-
{
|
|
280
|
+
{
|
|
281
|
+
'ip' => '8.8.8.8',
|
|
152
282
|
'ipinfo' => {
|
|
153
283
|
'country_name' => 'United States',
|
|
154
284
|
'city' => 'Mountain View'
|
|
155
285
|
}
|
|
156
286
|
}
|
|
157
287
|
]
|
|
158
|
-
filtered_records = filter(
|
|
288
|
+
filtered_records = filter(config, messages)
|
|
159
289
|
assert_equal(expected, filtered_records)
|
|
160
290
|
end
|
|
161
291
|
|
|
162
292
|
test 'add ipinfo to record with custom key_name and custom out_key' do
|
|
163
|
-
|
|
293
|
+
config = %[
|
|
164
294
|
@type ipinfo
|
|
165
295
|
key_name ip
|
|
166
296
|
out_key geodata
|
|
167
297
|
]
|
|
168
298
|
messages = [
|
|
169
|
-
{
|
|
299
|
+
{
|
|
300
|
+
'ip' => '8.8.8.8'
|
|
301
|
+
}
|
|
170
302
|
]
|
|
171
303
|
expected = [
|
|
172
|
-
{
|
|
304
|
+
{
|
|
305
|
+
'ip' => '8.8.8.8',
|
|
173
306
|
'geodata' => {
|
|
174
307
|
'country_name' => 'United States',
|
|
175
308
|
'region' => 'California',
|
|
@@ -179,29 +312,32 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
179
312
|
}
|
|
180
313
|
}
|
|
181
314
|
]
|
|
182
|
-
filtered_records = filter(
|
|
315
|
+
filtered_records = filter(config, messages)
|
|
183
316
|
assert_equal(expected, filtered_records)
|
|
184
317
|
end
|
|
185
318
|
|
|
186
319
|
test 'add ipinfo to record with custom key_name, custom out_name and custom fields' do
|
|
187
|
-
|
|
320
|
+
config = %[
|
|
188
321
|
@type ipinfo
|
|
189
322
|
key_name ip
|
|
190
323
|
out_key geodata
|
|
191
324
|
fields ["country_name", "city"]
|
|
192
325
|
]
|
|
193
326
|
messages = [
|
|
194
|
-
{
|
|
327
|
+
{
|
|
328
|
+
'ip' => '8.8.8.8'
|
|
329
|
+
}
|
|
195
330
|
]
|
|
196
331
|
expected = [
|
|
197
|
-
{
|
|
332
|
+
{
|
|
333
|
+
'ip' => '8.8.8.8',
|
|
198
334
|
'geodata' => {
|
|
199
335
|
'country_name' => 'United States',
|
|
200
336
|
'city' => 'Mountain View'
|
|
201
337
|
}
|
|
202
338
|
}
|
|
203
339
|
]
|
|
204
|
-
filtered_records = filter(
|
|
340
|
+
filtered_records = filter(config, messages)
|
|
205
341
|
assert_equal(expected, filtered_records)
|
|
206
342
|
end
|
|
207
343
|
end
|