fluent-plugin-ipinfo 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +45 -1
- data/fluent-plugin-ipinfo.gemspec +1 -1
- data/lib/fluent/plugin/filter_ipinfo.rb +59 -24
- data/test/plugin/test_filter_ipinfo.rb +164 -28
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2e715ffb9f1f67768b47a75a2909da48be93f43c882d78e135d69c70a3dc9678
|
|
4
|
+
data.tar.gz: 3c002e087f57d526771d43408f2692a0c459b4b266a14623f655d19e83fc79d8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f9269e76e2018277b28e2572b540612fa968d7ebb71ea2e60580f762f3bdfbed5c5aa83d799a4b9ac085c6ffca07a451be4cf01988ce5c5e6409ee421e9031e2
|
|
7
|
+
data.tar.gz: 07210a8a3082bfe1ce5dfb47d0433deb9157d7aaeff4e03b7e3bdb90233e4535547ade6c9d4ad229f11ce0506c1cd926ffcf7f1540aea98b4cfa1ab08d73ae6b
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -39,7 +39,7 @@ In this example, the following event:
|
|
|
39
39
|
}
|
|
40
40
|
```
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
would be enriched and returned as following:
|
|
43
43
|
|
|
44
44
|
```json
|
|
45
45
|
{
|
|
@@ -68,6 +68,8 @@ Would be enriched and returned as following:
|
|
|
68
68
|
The token to be used with the IPInfo API for paid plans.
|
|
69
69
|
To use the free plan (limited to 50k requests per month), do not use the `access_token` parameter.
|
|
70
70
|
|
|
71
|
+
If the value provided for `access_token` is an empty string (`""` or `" "`), the default value (`nil`) is used instead.
|
|
72
|
+
|
|
71
73
|
### `key_name`
|
|
72
74
|
|
|
73
75
|
| type | required | default |
|
|
@@ -76,6 +78,8 @@ To use the free plan (limited to 50k requests per month), do not use the `access
|
|
|
76
78
|
|
|
77
79
|
The name of the key containing the IP address.
|
|
78
80
|
|
|
81
|
+
If the value provided for `key_name` is an empty string (`""` or `" "`) or `nil`, the default value (`ip_address`) is used instead.
|
|
82
|
+
|
|
79
83
|
### `out_key`
|
|
80
84
|
|
|
81
85
|
| type | required | default |
|
|
@@ -84,6 +88,46 @@ The name of the key containing the IP address.
|
|
|
84
88
|
|
|
85
89
|
The name of the key to store the geographical location data in.
|
|
86
90
|
|
|
91
|
+
If the value provided for `out_key` is an empty string (`""` or `" "`) or `nil`, the default value (`ipinfo`) is used instead.
|
|
92
|
+
|
|
93
|
+
If the record has already a key with the same name as the value of `out_key`, its value will be overwritten with the geographical location data as shown in the example below:
|
|
94
|
+
|
|
95
|
+
```xml
|
|
96
|
+
<filter foo.bar>
|
|
97
|
+
@type ipinfo
|
|
98
|
+
access_token 1a2b3c4d5e
|
|
99
|
+
key_name ip_address
|
|
100
|
+
out_key data
|
|
101
|
+
fields ["country_name", "region", "city", "latitude", "longitude"]
|
|
102
|
+
</filter>
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The following event:
|
|
106
|
+
|
|
107
|
+
```json
|
|
108
|
+
{
|
|
109
|
+
"message":"Can you get me the geographical location for this IP addresse ?",
|
|
110
|
+
"ip_address":"8.8.8.8",
|
|
111
|
+
"data": "This value is going to be overwritten."
|
|
112
|
+
}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
would be enriched and returned as following:
|
|
116
|
+
|
|
117
|
+
```json
|
|
118
|
+
{
|
|
119
|
+
"message": "Can you get me the geographical location for this IP addresse ?",
|
|
120
|
+
"ip_address": "8.8.8.8",
|
|
121
|
+
"data": {
|
|
122
|
+
"country_name": "United States",
|
|
123
|
+
"region": "California",
|
|
124
|
+
"city": "Mountain View",
|
|
125
|
+
"latitude": "37.4056",
|
|
126
|
+
"longitude": "-122.0775"
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
```
|
|
130
|
+
|
|
87
131
|
### `fields`
|
|
88
132
|
|
|
89
133
|
| type | required | default |
|
|
@@ -35,41 +35,76 @@ module Fluent::Plugin
|
|
|
35
35
|
|
|
36
36
|
def initialize
|
|
37
37
|
super
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
@
|
|
38
|
+
# Keep the data in cache for one week
|
|
39
|
+
# Number of entries to keep in cache
|
|
40
|
+
@ipinfo_cache_settings = {:ttl => 60 * 60 * 24 * 7, :maxsize => 4096}
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
def configure(conf)
|
|
44
44
|
super
|
|
45
|
+
|
|
46
|
+
if !@access_token.nil? and @access_token.strip.empty?
|
|
47
|
+
log.warn "access_token value is an empty string. Falling back to default value."
|
|
48
|
+
@access_token = nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
if @key_name.nil? or @key_name.strip.empty?
|
|
52
|
+
log.warn "key_name value is '#{@key_name}'. Falling back to default value."
|
|
53
|
+
@key_name = 'ip_address'
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
if @out_key.nil? or @out_key.strip.empty?
|
|
57
|
+
log.warn "out_key value is '#{@out_key}'. Falling back to default value."
|
|
58
|
+
@out_key = 'ipinfo'
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Delete duplicates and empty fields (nil values included)
|
|
62
|
+
@fields = @fields.uniq.delete_if {|f| f.strip.empty? or f.nil?}
|
|
63
|
+
|
|
64
|
+
unless @fields.length() > 0
|
|
65
|
+
raise Fluent::ConfigError, "Fields array is empty. You need to specify at least one field."
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Create the IPInfo client handler
|
|
45
69
|
unless @access_token.nil?
|
|
46
|
-
@ipinfo_handler = IPinfo::create(@access_token, @
|
|
70
|
+
@ipinfo_handler = IPinfo::create(@access_token, @ipinfo_cache_settings)
|
|
47
71
|
else
|
|
48
|
-
@ipinfo_handler = IPinfo::create(nil, @
|
|
72
|
+
@ipinfo_handler = IPinfo::create(nil, @ipinfo_cache_settings)
|
|
49
73
|
end
|
|
50
74
|
end
|
|
51
75
|
|
|
52
76
|
def filter(tag, time, record)
|
|
53
|
-
|
|
54
|
-
unless
|
|
55
|
-
#
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
#
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
77
|
+
# Check that the record has a key whose name is the value of @key_name
|
|
78
|
+
unless record.key?(@key_name)
|
|
79
|
+
log.error "key '#{@key_name}' is not present in the record. Ignoring the record."
|
|
80
|
+
else
|
|
81
|
+
ip_address = record[@key_name]
|
|
82
|
+
# Check that the ip_address is not 'nil'
|
|
83
|
+
unless ip_address.nil?
|
|
84
|
+
begin
|
|
85
|
+
# Fetch geolocation details using IPInfo API
|
|
86
|
+
ipinfo_details = @ipinfo_handler.details(ip_address)
|
|
87
|
+
# IPInfo ruby wrapper returns a dict based on symbols, we need to stringify the symbols
|
|
88
|
+
# to be able to use them easily
|
|
89
|
+
all_details = ipinfo_details.all
|
|
90
|
+
geodata = all_details.stringify_keys
|
|
91
|
+
# Get the final list of fields by running a join operation on the fields provided by the user and the ones
|
|
92
|
+
# returned by IPInfo API
|
|
93
|
+
ipinfo_returned_fields = geodata.keys
|
|
94
|
+
final_fields = ipinfo_returned_fields & @fields
|
|
95
|
+
if final_fields.length() != @fields.length()
|
|
96
|
+
ignored_fields = @fields - final_fields
|
|
97
|
+
ignored_fields.each{|field|
|
|
98
|
+
log.warn "Field '#{@field}' not present in IPInfo payload. Ignoring it."
|
|
99
|
+
}
|
|
100
|
+
end
|
|
101
|
+
# Extract a subhash from the geolocation data returned by IPInfo API using the final_fields list as keys.
|
|
102
|
+
record[@out_key] = extract_subhash(geodata, final_fields)
|
|
103
|
+
rescue Exception => e
|
|
104
|
+
log.error 'An error occured while fetching geographical location data.', error: e
|
|
105
|
+
log.warn_backtrace e.backtrace
|
|
106
|
+
end
|
|
70
107
|
end
|
|
71
|
-
# Extract a subhash from the geolocation data returned by IPInfo API using the final_fields list as keys.
|
|
72
|
-
record[@out_key] = extract_subhash(geodata, final_fields)
|
|
73
108
|
end
|
|
74
109
|
record
|
|
75
110
|
end
|
|
@@ -51,14 +51,105 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
51
51
|
d.filtered_records
|
|
52
52
|
end
|
|
53
53
|
|
|
54
|
+
sub_test_case 'plugin will use default values' do
|
|
55
|
+
test 'access_token nil' do
|
|
56
|
+
config = %[
|
|
57
|
+
@type ipinfo
|
|
58
|
+
access_token
|
|
59
|
+
]
|
|
60
|
+
d = create_driver(config)
|
|
61
|
+
assert_equal d.instance.access_token, nil
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
test 'access_token empty string' do
|
|
65
|
+
config = %[
|
|
66
|
+
@type ipinfo
|
|
67
|
+
access_token ""
|
|
68
|
+
]
|
|
69
|
+
d = create_driver(config)
|
|
70
|
+
assert_equal d.instance.access_token, nil
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
test 'key_name nil' do
|
|
74
|
+
config = %[
|
|
75
|
+
@type ipinfo
|
|
76
|
+
key_name
|
|
77
|
+
]
|
|
78
|
+
d = create_driver(config)
|
|
79
|
+
assert_equal d.instance.key_name, 'ip_address'
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
test 'key_name empty string' do
|
|
83
|
+
config = %[
|
|
84
|
+
@type ipinfo
|
|
85
|
+
key_name
|
|
86
|
+
]
|
|
87
|
+
d = create_driver(config)
|
|
88
|
+
assert_equal d.instance.key_name, 'ip_address'
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
test 'out_key nil' do
|
|
92
|
+
config = %[
|
|
93
|
+
@type ipinfo
|
|
94
|
+
out_key
|
|
95
|
+
]
|
|
96
|
+
d = create_driver(config)
|
|
97
|
+
assert_equal d.instance.out_key, 'ipinfo'
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
test 'out_key empty string' do
|
|
101
|
+
config = %[
|
|
102
|
+
@type ipinfo
|
|
103
|
+
out_key
|
|
104
|
+
]
|
|
105
|
+
d = create_driver(config)
|
|
106
|
+
assert_equal d.instance.out_key, 'ipinfo'
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
sub_test_case 'plugin will raise Fluent::ConfigError' do
|
|
111
|
+
test 'empty fields' do
|
|
112
|
+
config = %[
|
|
113
|
+
@type ipinfo
|
|
114
|
+
fields []
|
|
115
|
+
]
|
|
116
|
+
assert_raise Fluent::ConfigError do
|
|
117
|
+
create_driver(config)
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
test 'fields with one empty string' do
|
|
122
|
+
config = %[
|
|
123
|
+
@type ipinfo
|
|
124
|
+
fields [""]
|
|
125
|
+
]
|
|
126
|
+
assert_raise Fluent::ConfigError do
|
|
127
|
+
create_driver(config)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
test 'fields with multiple empty string' do
|
|
132
|
+
config = %[
|
|
133
|
+
@type ipinfo
|
|
134
|
+
fields [""]
|
|
135
|
+
]
|
|
136
|
+
assert_raise Fluent::ConfigError do
|
|
137
|
+
create_driver(config)
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
54
142
|
sub_test_case 'plugin will fetch geolocation data' do
|
|
55
143
|
test 'add ipinfo to record with default fields' do
|
|
56
|
-
|
|
144
|
+
config = CONFIG
|
|
57
145
|
messages = [
|
|
58
|
-
{
|
|
146
|
+
{
|
|
147
|
+
'ip_address' => '8.8.8.8'
|
|
148
|
+
}
|
|
59
149
|
]
|
|
60
150
|
expected = [
|
|
61
|
-
{
|
|
151
|
+
{
|
|
152
|
+
'ip_address' => '8.8.8.8',
|
|
62
153
|
'ipinfo' => {
|
|
63
154
|
'country_name' => 'United States',
|
|
64
155
|
'region' => 'California',
|
|
@@ -68,20 +159,23 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
68
159
|
}
|
|
69
160
|
}
|
|
70
161
|
]
|
|
71
|
-
filtered_records = filter(
|
|
162
|
+
filtered_records = filter(config, messages)
|
|
72
163
|
assert_equal(expected, filtered_records)
|
|
73
164
|
end
|
|
74
165
|
|
|
75
166
|
test 'add ipinfo to record with custom key_name' do
|
|
76
|
-
|
|
167
|
+
config = %[
|
|
77
168
|
@type ipinfo
|
|
78
169
|
key_name ip
|
|
79
170
|
]
|
|
80
171
|
messages = [
|
|
81
|
-
{
|
|
172
|
+
{
|
|
173
|
+
'ip' => '8.8.8.8'
|
|
174
|
+
}
|
|
82
175
|
]
|
|
83
176
|
expected = [
|
|
84
|
-
{
|
|
177
|
+
{
|
|
178
|
+
'ip' => '8.8.8.8',
|
|
85
179
|
'ipinfo' => {
|
|
86
180
|
'country_name' => 'United States',
|
|
87
181
|
'region' => 'California',
|
|
@@ -91,20 +185,23 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
91
185
|
}
|
|
92
186
|
}
|
|
93
187
|
]
|
|
94
|
-
filtered_records = filter(
|
|
188
|
+
filtered_records = filter(config, messages)
|
|
95
189
|
assert_equal(expected, filtered_records)
|
|
96
190
|
end
|
|
97
191
|
|
|
98
192
|
test 'add ipinfo to record with custom out_key' do
|
|
99
|
-
|
|
193
|
+
config = %[
|
|
100
194
|
@type ipinfo
|
|
101
195
|
out_key geodata
|
|
102
196
|
]
|
|
103
197
|
messages = [
|
|
104
|
-
{
|
|
198
|
+
{
|
|
199
|
+
'ip_address' => '8.8.8.8'
|
|
200
|
+
}
|
|
105
201
|
]
|
|
106
202
|
expected = [
|
|
107
|
-
{
|
|
203
|
+
{
|
|
204
|
+
'ip_address' => '8.8.8.8',
|
|
108
205
|
'geodata' => {
|
|
109
206
|
'country_name' => 'United States',
|
|
110
207
|
'region' => 'California',
|
|
@@ -114,62 +211,98 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
114
211
|
}
|
|
115
212
|
}
|
|
116
213
|
]
|
|
117
|
-
filtered_records = filter(
|
|
214
|
+
filtered_records = filter(config, messages)
|
|
215
|
+
assert_equal(expected, filtered_records)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
test 'add ipinfo to record with custom out_key that is already defined' do
|
|
219
|
+
config = %[
|
|
220
|
+
@type ipinfo
|
|
221
|
+
out_key geodata
|
|
222
|
+
]
|
|
223
|
+
messages = [
|
|
224
|
+
{
|
|
225
|
+
'ip_address' => '8.8.8.8',
|
|
226
|
+
'geodata' => 'bye!'
|
|
227
|
+
}
|
|
228
|
+
]
|
|
229
|
+
expected = [
|
|
230
|
+
{
|
|
231
|
+
'ip_address' => '8.8.8.8',
|
|
232
|
+
'geodata' => {
|
|
233
|
+
'country_name' => 'United States',
|
|
234
|
+
'region' => 'California',
|
|
235
|
+
'city' => 'Mountain View',
|
|
236
|
+
'latitude' => '37.4056',
|
|
237
|
+
'longitude' => '-122.0775'
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
]
|
|
241
|
+
filtered_records = filter(config, messages)
|
|
118
242
|
assert_equal(expected, filtered_records)
|
|
119
243
|
end
|
|
120
244
|
|
|
121
245
|
test 'add ipinfo to record with custom fields' do
|
|
122
|
-
|
|
246
|
+
config = %[
|
|
123
247
|
@type ipinfo
|
|
124
248
|
fields ["country_name", "city"]
|
|
125
249
|
]
|
|
126
250
|
messages = [
|
|
127
|
-
{
|
|
251
|
+
{
|
|
252
|
+
'ip_address' => '8.8.8.8'
|
|
253
|
+
}
|
|
128
254
|
]
|
|
129
255
|
expected = [
|
|
130
|
-
{
|
|
256
|
+
{
|
|
257
|
+
'ip_address' => '8.8.8.8',
|
|
131
258
|
'ipinfo' => {
|
|
132
259
|
'country_name' => 'United States',
|
|
133
260
|
'city' => 'Mountain View'
|
|
134
261
|
}
|
|
135
262
|
}
|
|
136
263
|
]
|
|
137
|
-
filtered_records = filter(
|
|
264
|
+
filtered_records = filter(config, messages)
|
|
138
265
|
assert_equal(expected, filtered_records)
|
|
139
266
|
end
|
|
140
267
|
|
|
141
268
|
test 'add ipinfo to record with custom key_name and custom fields' do
|
|
142
|
-
|
|
269
|
+
config = %[
|
|
143
270
|
@type ipinfo
|
|
144
271
|
key_name ip
|
|
145
272
|
fields ["country_name", "city"]
|
|
146
273
|
]
|
|
147
274
|
messages = [
|
|
148
|
-
{
|
|
275
|
+
{
|
|
276
|
+
'ip' => '8.8.8.8'
|
|
277
|
+
}
|
|
149
278
|
]
|
|
150
279
|
expected = [
|
|
151
|
-
{
|
|
280
|
+
{
|
|
281
|
+
'ip' => '8.8.8.8',
|
|
152
282
|
'ipinfo' => {
|
|
153
283
|
'country_name' => 'United States',
|
|
154
284
|
'city' => 'Mountain View'
|
|
155
285
|
}
|
|
156
286
|
}
|
|
157
287
|
]
|
|
158
|
-
filtered_records = filter(
|
|
288
|
+
filtered_records = filter(config, messages)
|
|
159
289
|
assert_equal(expected, filtered_records)
|
|
160
290
|
end
|
|
161
291
|
|
|
162
292
|
test 'add ipinfo to record with custom key_name and custom out_key' do
|
|
163
|
-
|
|
293
|
+
config = %[
|
|
164
294
|
@type ipinfo
|
|
165
295
|
key_name ip
|
|
166
296
|
out_key geodata
|
|
167
297
|
]
|
|
168
298
|
messages = [
|
|
169
|
-
{
|
|
299
|
+
{
|
|
300
|
+
'ip' => '8.8.8.8'
|
|
301
|
+
}
|
|
170
302
|
]
|
|
171
303
|
expected = [
|
|
172
|
-
{
|
|
304
|
+
{
|
|
305
|
+
'ip' => '8.8.8.8',
|
|
173
306
|
'geodata' => {
|
|
174
307
|
'country_name' => 'United States',
|
|
175
308
|
'region' => 'California',
|
|
@@ -179,29 +312,32 @@ class IPinfoFilterTest < Test::Unit::TestCase
|
|
|
179
312
|
}
|
|
180
313
|
}
|
|
181
314
|
]
|
|
182
|
-
filtered_records = filter(
|
|
315
|
+
filtered_records = filter(config, messages)
|
|
183
316
|
assert_equal(expected, filtered_records)
|
|
184
317
|
end
|
|
185
318
|
|
|
186
319
|
test 'add ipinfo to record with custom key_name, custom out_name and custom fields' do
|
|
187
|
-
|
|
320
|
+
config = %[
|
|
188
321
|
@type ipinfo
|
|
189
322
|
key_name ip
|
|
190
323
|
out_key geodata
|
|
191
324
|
fields ["country_name", "city"]
|
|
192
325
|
]
|
|
193
326
|
messages = [
|
|
194
|
-
{
|
|
327
|
+
{
|
|
328
|
+
'ip' => '8.8.8.8'
|
|
329
|
+
}
|
|
195
330
|
]
|
|
196
331
|
expected = [
|
|
197
|
-
{
|
|
332
|
+
{
|
|
333
|
+
'ip' => '8.8.8.8',
|
|
198
334
|
'geodata' => {
|
|
199
335
|
'country_name' => 'United States',
|
|
200
336
|
'city' => 'Mountain View'
|
|
201
337
|
}
|
|
202
338
|
}
|
|
203
339
|
]
|
|
204
|
-
filtered_records = filter(
|
|
340
|
+
filtered_records = filter(config, messages)
|
|
205
341
|
assert_equal(expected, filtered_records)
|
|
206
342
|
end
|
|
207
343
|
end
|