fluent-plugin-geoip 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +31 -20
- data/Appraisals +0 -4
- data/README.md +50 -11
- data/docker-compose.yml +31 -0
- data/dockerfiles/Dockerfile-ruby2.1 +8 -0
- data/dockerfiles/Dockerfile-ruby2.2 +8 -0
- data/dockerfiles/Dockerfile-ruby2.3 +8 -0
- data/dockerfiles/Dockerfile-ruby2.4 +8 -0
- data/fluent-plugin-geoip.gemspec +4 -2
- data/lib/fluent/plugin/filter_geoip.rb +4 -6
- data/lib/fluent/plugin/geoip.rb +47 -18
- data/lib/fluent/plugin/out_geoip.rb +3 -15
- data/test/plugin/test_filter_geoip.rb +957 -380
- data/test/plugin/test_out_geoip.rb +1035 -385
- metadata +49 -3
- data/gemfiles/fluentd_v0.10.gemfile +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6064156b8f481d751a7f1ee92c5e80b189c52f2c
|
4
|
+
data.tar.gz: 95e5509ad5ec16ff48e9d71687caa205bd4dbf85
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c3fcd73dcac19ab974974099f05147e1ba082cf339b0c45ae00971049cadf94cf6d8c1a438351c0c08cd7a51f5fcba03c01013760ff21422275757cc7adcd270
|
7
|
+
data.tar.gz: f6d8bb596b0c423e041d69f520cbb0fd6268c7a36bf7910e32b1a189fe2c5da560a6c41f2f95fa5a93b83c70cce1edf999994bac65f28a0f88b7f0fdc01e3bd6
|
data/.travis.yml
CHANGED
@@ -1,26 +1,37 @@
|
|
1
1
|
sudo: required
|
2
|
-
|
2
|
+
services:
|
3
|
+
- docker
|
3
4
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
5
|
+
env:
|
6
|
+
global:
|
7
|
+
DOCKER_COMPOSE_VERSION: 1.8.1
|
8
|
+
matrix:
|
9
|
+
- TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/Gemfile
|
10
|
+
- TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/Gemfile
|
11
|
+
- TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/Gemfile
|
12
|
+
- TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/Gemfile
|
13
|
+
- TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
|
14
|
+
- TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
|
15
|
+
- TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
|
16
|
+
- TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
|
10
17
|
|
11
18
|
before_install:
|
12
|
-
|
13
|
-
|
14
|
-
-
|
19
|
+
#- sudo apt-get update
|
20
|
+
#- sudo apt-get install -y docker-engine
|
21
|
+
- sudo rm /usr/local/bin/docker-compose
|
22
|
+
- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) > docker-compose
|
23
|
+
- chmod +x docker-compose
|
24
|
+
- sudo mv docker-compose /usr/local/bin/
|
25
|
+
- test -f data/GeoLite2-City.mmdb || curl http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip > data/GeoLite2-City.mmdb
|
15
26
|
|
16
|
-
|
17
|
-
- Gemfile
|
18
|
-
- gemfiles/fluentd_v0.10.gemfile
|
19
|
-
- gemfiles/fluentd_v0.12.gemfile
|
27
|
+
install: true
|
20
28
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
29
|
+
before_script:
|
30
|
+
- echo BUNDLE_GEMFILE=${BUNDLE_GEMFILE} > env
|
31
|
+
- docker-compose build test-ruby${TARGET_RUBY_VERSION}
|
32
|
+
- docker-compose ps
|
33
|
+
- docker-compose up -d test-ruby${TARGET_RUBY_VERSION}
|
34
|
+
- docker-compose exec test-ruby${TARGET_RUBY_VERSION} bundle install
|
35
|
+
|
36
|
+
script:
|
37
|
+
- docker-compose exec test-ruby${TARGET_RUBY_VERSION} bundle exec rake test
|
data/Appraisals
CHANGED
data/README.md
CHANGED
@@ -18,13 +18,16 @@ before use, install dependent library as:
|
|
18
18
|
# for RHEL/CentOS
|
19
19
|
$ sudo yum group install "Development Tools"
|
20
20
|
$ sudo yum install geoip-devel --enablerepo=epel
|
21
|
+
$ sudo yum install libmaxminddb-devel --enablerepo=epel
|
21
22
|
|
22
23
|
# for Ubuntu/Debian
|
23
24
|
$ sudo apt-get install build-essential
|
24
25
|
$ sudo apt-get install libgeoip-dev
|
26
|
+
$ sudo apt-get install libmaxminddb-dev
|
25
27
|
|
26
28
|
# for OS X
|
27
29
|
$ brew install geoip
|
30
|
+
$ brew install libmaxminddb
|
28
31
|
```
|
29
32
|
|
30
33
|
## Installation
|
@@ -48,7 +51,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
|
|
48
51
|
|
49
52
|
```xml
|
50
53
|
<match access.apache>
|
51
|
-
type geoip
|
54
|
+
@type geoip
|
52
55
|
|
53
56
|
# Specify one or more geoip lookup field which has ip address (default: host)
|
54
57
|
# in the case of accessing nested value, delimit keys by dot like 'host.ip'.
|
@@ -89,7 +92,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
|
|
89
92
|
|
90
93
|
```xml
|
91
94
|
<match access.apache>
|
92
|
-
type geoip
|
95
|
+
@type geoip
|
93
96
|
geoip_lookup_key user1_host, user2_host
|
94
97
|
<record>
|
95
98
|
user1_city ${city["user1_host"]}
|
@@ -107,12 +110,12 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
|
|
107
110
|
|
108
111
|
```
|
109
112
|
<match access.apache>
|
110
|
-
type
|
113
|
+
@type geoip
|
111
114
|
geoip_lookup_key host
|
112
115
|
<record>
|
113
116
|
# lat lon as properties
|
114
117
|
# ex. {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
|
115
|
-
location_properties { "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }
|
118
|
+
location_properties '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
|
116
119
|
|
117
120
|
# lat lon as string
|
118
121
|
# ex. "37.4192008972168,-122.05740356445312"
|
@@ -120,7 +123,7 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
|
|
120
123
|
|
121
124
|
# GeoJSON (lat lon as array) is useful for Kibana's bettermap.
|
122
125
|
# ex. [-122.05740356445312, 37.4192008972168]
|
123
|
-
location_array [${longitude["host"]},${latitude["host"]}]
|
126
|
+
location_array '[${longitude["host"]},${latitude["host"]}]'
|
124
127
|
</record>
|
125
128
|
remove_tag_prefix access.
|
126
129
|
tag geoip.${tag}
|
@@ -134,7 +137,7 @@ On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ...
|
|
134
137
|
|
135
138
|
```
|
136
139
|
<match access.apache>
|
137
|
-
type
|
140
|
+
@type geoip
|
138
141
|
geoip_lookup_key host
|
139
142
|
<record>
|
140
143
|
location_properties '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
|
@@ -194,16 +197,16 @@ Note that filter version of geoip plugin does not have handling tag feature.
|
|
194
197
|
|
195
198
|
```xml
|
196
199
|
<source>
|
197
|
-
type forward
|
200
|
+
@type forward
|
198
201
|
</source>
|
199
202
|
|
200
203
|
<match test.geoip>
|
201
|
-
type copy
|
204
|
+
@type copy
|
202
205
|
<store>
|
203
|
-
type stdout
|
206
|
+
@type stdout
|
204
207
|
</store>
|
205
208
|
<store>
|
206
|
-
type geoip
|
209
|
+
@type geoip
|
207
210
|
geoip_lookup_key host
|
208
211
|
<record>
|
209
212
|
lat ${latitude["host"]}
|
@@ -216,7 +219,7 @@ Note that filter version of geoip plugin does not have handling tag feature.
|
|
216
219
|
</match>
|
217
220
|
|
218
221
|
<match debug.**>
|
219
|
-
type stdout
|
222
|
+
@type stdout
|
220
223
|
</match>
|
221
224
|
```
|
222
225
|
|
@@ -275,6 +278,8 @@ http://dev.maxmind.com/geoip/legacy/csv/
|
|
275
278
|
|
276
279
|
## Placeholders
|
277
280
|
|
281
|
+
### GeoIP legacy
|
282
|
+
|
278
283
|
Provides these placeholders for adding field of geolocate results.<br />
|
279
284
|
For more example of geolocating, you can try these websites like [Geo IP Address View](http://www.geoipview.com/) or [View my IP information](http://www.geoiptool.com/en/).
|
280
285
|
|
@@ -292,6 +297,40 @@ For more example of geolocating, you can try these websites like [Geo IP Address
|
|
292
297
|
|
293
298
|
Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/#GeoIP_City_Edition_CSV_Database_Fields
|
294
299
|
|
300
|
+
### GeoIP2
|
301
|
+
|
302
|
+
You can get any fields in the
|
303
|
+
[GeoLite2](http://dev.maxmind.com/geoip/geoip2/geolite2/) database and
|
304
|
+
[GeoIP2 Downloadable Databases](http://dev.maxmind.com/geoip/geoip2/downloadable/).
|
305
|
+
|
306
|
+
For example(geoip2_c backend):
|
307
|
+
|
308
|
+
| placeholder attributes | output example | note |
|
309
|
+
|------------------------------------------|--------------------|------|
|
310
|
+
| ${city.names.en[lookup_field]} | "Mountain View" | - |
|
311
|
+
| ${location.latitude[lookup_field]} | 37.419200000000004 | - |
|
312
|
+
| ${location.longitude[lookup_field]} | -122.0574 | - |
|
313
|
+
| ${country.iso_code[lookup_field]} | "US" | - |
|
314
|
+
| ${country.names.en[lookup_field]} | "United States" | - |
|
315
|
+
| ${postal.code[lookup_field]} | "94043" | - |
|
316
|
+
| ${subdivisions.0.iso_code[lookup_field]} | "CA" | - |
|
317
|
+
| ${subdivisions.0.names.en[lookup_field]} | "California" | - |
|
318
|
+
|
319
|
+
For example(geoip2_compat backend):
|
320
|
+
|
321
|
+
| placeholder attributes | output example | note |
|
322
|
+
|-------------------------------|--------------------|------|
|
323
|
+
| ${city[lookup_field]} | "Mountain View" | - |
|
324
|
+
| ${latitude[lookup_field]} | 37.419200000000004 | - |
|
325
|
+
| ${longitude[lookup_field]} | -122.0574 | - |
|
326
|
+
| ${country_code[lookup_field]} | "US" | - |
|
327
|
+
| ${country_name[lookup_field]} | "United States" | - |
|
328
|
+
| ${postal_code[lookup_field]} | "94043" | |
|
329
|
+
| ${region[lookup_field]} | "CA" | - |
|
330
|
+
| ${region_name[lookup_field]} | "California" | - |
|
331
|
+
|
332
|
+
**NOTE**: geoip2_compat backend supports only above fields.
|
333
|
+
|
295
334
|
## Parameters
|
296
335
|
|
297
336
|
### GeoipOutput
|
data/docker-compose.yml
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
version: "2"
|
2
|
+
services:
|
3
|
+
test-ruby2.4:
|
4
|
+
build:
|
5
|
+
context: .
|
6
|
+
dockerfile: dockerfiles/Dockerfile-ruby2.4
|
7
|
+
env_file:
|
8
|
+
- env
|
9
|
+
command: ruby -run -e httpd .
|
10
|
+
test-ruby2.3:
|
11
|
+
build:
|
12
|
+
context: .
|
13
|
+
dockerfile: dockerfiles/Dockerfile-ruby2.3
|
14
|
+
env_file:
|
15
|
+
- env
|
16
|
+
command: ruby -run -e httpd .
|
17
|
+
test-ruby2.2:
|
18
|
+
build:
|
19
|
+
context: .
|
20
|
+
dockerfile: dockerfiles/Dockerfile-ruby2.2
|
21
|
+
env_file:
|
22
|
+
- env
|
23
|
+
command: ruby -run -e httpd .
|
24
|
+
test-ruby2.1:
|
25
|
+
build:
|
26
|
+
context: .
|
27
|
+
dockerfile: dockerfiles/Dockerfile-ruby2.1
|
28
|
+
env_file:
|
29
|
+
- env
|
30
|
+
command: ruby -run -e httpd .
|
31
|
+
|
data/fluent-plugin-geoip.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-geoip"
|
7
|
-
spec.version = "0.
|
7
|
+
spec.version = "0.7.0"
|
8
8
|
spec.authors = ["Kentaro Yoshida"]
|
9
9
|
spec.email = ["y.ken.studio@gmail.com"]
|
10
10
|
spec.summary = %q{Fluentd Filter plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
|
@@ -20,9 +20,11 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_development_dependency "rake"
|
21
21
|
spec.add_development_dependency "appraisal"
|
22
22
|
spec.add_development_dependency "test-unit", ">= 3.1.0"
|
23
|
+
spec.add_development_dependency "geoip2_compat"
|
24
|
+
spec.add_development_dependency "geoip2_c"
|
23
25
|
|
24
26
|
spec.add_runtime_dependency "fluentd"
|
25
27
|
spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
|
26
28
|
spec.add_runtime_dependency "geoip-c"
|
29
|
+
spec.add_runtime_dependency "dig_rb"
|
27
30
|
end
|
28
|
-
|
@@ -1,8 +1,11 @@
|
|
1
|
+
require 'fluent/plugin/geoip'
|
2
|
+
|
1
3
|
module Fluent
|
2
4
|
class GeoipFilter < Filter
|
3
5
|
Plugin.register_filter('geoip', self)
|
4
6
|
|
5
7
|
config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
|
8
|
+
config_param :geoip2_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
|
6
9
|
config_param :geoip_lookup_key, :string, :default => 'host'
|
7
10
|
config_param :skip_adding_null_record, :bool, :default => false
|
8
11
|
|
@@ -13,12 +16,7 @@ module Fluent
|
|
13
16
|
config_param :flush_interval, :time, :default => 0
|
14
17
|
config_param :log_level, :string, :default => 'warn'
|
15
18
|
|
16
|
-
|
17
|
-
def initialize
|
18
|
-
require 'fluent/plugin/geoip'
|
19
|
-
|
20
|
-
super
|
21
|
-
end
|
19
|
+
config_param :backend_library, :enum, :list => Fluent::GeoIP::BACKEND_LIBRARIES, :default => :geoip
|
22
20
|
|
23
21
|
def configure(conf)
|
24
22
|
super
|
data/lib/fluent/plugin/geoip.rb
CHANGED
@@ -1,12 +1,16 @@
|
|
1
1
|
require 'geoip'
|
2
2
|
require 'yajl'
|
3
|
+
require 'dig_rb'
|
3
4
|
|
4
5
|
module Fluent
|
5
6
|
class GeoIP
|
7
|
+
BACKEND_LIBRARIES = [:geoip, :geoip2_compat, :geoip2_c]
|
8
|
+
|
6
9
|
REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\[['"](?<record_key>-?[^'"]+)['"]\]\}$/
|
7
10
|
REGEXP_PLACEHOLDER_SCAN = /['"]?(\$\{[^\}]+?\})['"]?/
|
8
11
|
|
9
12
|
GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
|
13
|
+
GEOIP2_COMPAT_KEYS = %w(city country_code country_name latitude longitude postal_code region region_name)
|
10
14
|
|
11
15
|
attr_reader :log
|
12
16
|
|
@@ -25,7 +29,7 @@ module Fluent
|
|
25
29
|
if record_key.nil?
|
26
30
|
raise Fluent::ConfigError, "geoip: missing value found at '#{key} #{lookup_field}'"
|
27
31
|
end
|
28
|
-
@map
|
32
|
+
@map[record_key] = "${#{geoip_key}['#{lookup_field}']}"
|
29
33
|
end
|
30
34
|
end
|
31
35
|
if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
|
@@ -53,7 +57,16 @@ module Fluent
|
|
53
57
|
@placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
|
54
58
|
@placeholder_keys.each do |key|
|
55
59
|
geoip_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:geoip_key]
|
56
|
-
|
60
|
+
case plugin.backend_library
|
61
|
+
when :geoip
|
62
|
+
raise Fluent::ConfigError, "#{plugin.backend_library}: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
|
63
|
+
when :geoip2_compat
|
64
|
+
raise Fluent::ConfigError, "#{plugin.backend_library}: unsupported key #{geoip_key}" unless GEOIP2_COMPAT_KEYS.include?(geoip_key)
|
65
|
+
when :geoip2_c
|
66
|
+
# Nothing to do.
|
67
|
+
# We cannot define supported key(s) before we fetch values from GeoIP2 database
|
68
|
+
# because geoip2_c can fetch any fields in GeoIP2 database.
|
69
|
+
end
|
57
70
|
end
|
58
71
|
|
59
72
|
if plugin.is_a?(Fluent::BufferedOutput)
|
@@ -63,7 +76,7 @@ module Fluent
|
|
63
76
|
end
|
64
77
|
end
|
65
78
|
|
66
|
-
@geoip =
|
79
|
+
@geoip = load_database(plugin)
|
67
80
|
end
|
68
81
|
|
69
82
|
def add_geoip_field(record)
|
@@ -81,7 +94,7 @@ module Fluent
|
|
81
94
|
else
|
82
95
|
rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
|
83
96
|
end
|
84
|
-
record
|
97
|
+
record[record_key] = rewrited
|
85
98
|
end
|
86
99
|
return record
|
87
100
|
end
|
@@ -116,25 +129,25 @@ module Fluent
|
|
116
129
|
def get_address(record)
|
117
130
|
address = {}
|
118
131
|
@geoip_lookup_key.each do |field|
|
119
|
-
address
|
120
|
-
key = field.split('.')
|
121
|
-
obj = record
|
122
|
-
key.each {|k|
|
123
|
-
break obj = nil if not obj.has_key?(k)
|
124
|
-
obj = obj[k]
|
125
|
-
}
|
126
|
-
address.store(field, obj)
|
132
|
+
address[field] = record[field] || record.dig(*field.split('.'))
|
127
133
|
end
|
128
|
-
|
134
|
+
address
|
129
135
|
end
|
130
136
|
|
131
137
|
def geolocate(addresses)
|
132
138
|
geodata = {}
|
133
139
|
addresses.each do |field, ip|
|
134
|
-
geo =
|
135
|
-
|
140
|
+
geo = nil
|
141
|
+
if ip
|
142
|
+
geo = if @geoip.respond_to?(:look_up)
|
143
|
+
@geoip.look_up(ip)
|
144
|
+
else
|
145
|
+
@geoip.lookup(ip)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
geodata[field] = geo
|
136
149
|
end
|
137
|
-
|
150
|
+
geodata
|
138
151
|
end
|
139
152
|
|
140
153
|
def create_placeholder(geodata)
|
@@ -142,9 +155,25 @@ module Fluent
|
|
142
155
|
@placeholder_keys.each do |placeholder_key|
|
143
156
|
position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
|
144
157
|
next if position.nil? or geodata[position[:record_key]].nil?
|
145
|
-
|
158
|
+
keys = [position[:record_key]] + position[:geoip_key].split('.').map(&:to_sym)
|
159
|
+
placeholder[placeholder_key] = geodata.dig(*keys)
|
160
|
+
end
|
161
|
+
placeholder
|
162
|
+
end
|
163
|
+
|
164
|
+
def load_database(plugin)
|
165
|
+
case plugin.backend_library
|
166
|
+
when :geoip
|
167
|
+
::GeoIP::City.new(plugin.geoip_database, :memory, false)
|
168
|
+
when :geoip2_compat
|
169
|
+
require 'geoip2_compat'
|
170
|
+
GeoIP2Compat.new(plugin.geoip2_database)
|
171
|
+
when :geoip2_c
|
172
|
+
require 'geoip2'
|
173
|
+
GeoIP2::Database.new(plugin.geoip2_database)
|
146
174
|
end
|
147
|
-
|
175
|
+
rescue LoadError
|
176
|
+
raise Fluent::ConfigError, "You must install #{plugin.backend_library} gem."
|
148
177
|
end
|
149
178
|
end
|
150
179
|
end
|