fluent-plugin-geoip 0.6.1 → 0.7.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8fdb172feeffd90a77f0a83f3f44b58b88ddb138
4
- data.tar.gz: 3abc009ded4a7ced18e046c27d9f50094f11b865
3
+ metadata.gz: 6064156b8f481d751a7f1ee92c5e80b189c52f2c
4
+ data.tar.gz: 95e5509ad5ec16ff48e9d71687caa205bd4dbf85
5
5
  SHA512:
6
- metadata.gz: c5ef5143cdd8f2ea9fe372d5a2e0ee5bbe5981a4a1c2cd84b34364fe7f6679a897b5a40732d8b97ad9dfdf5f36b3dc257ec529357c2411e817cfdb34822e05f0
7
- data.tar.gz: 70d27191a5cc8f7f4ee5153845149f7d0881c5bf08ed7554ab5424a65f7806b78f36909fa8319d1850d49c13d2247fa41f466cf954777e85d80d45065948eedb
6
+ metadata.gz: c3fcd73dcac19ab974974099f05147e1ba082cf339b0c45ae00971049cadf94cf6d8c1a438351c0c08cd7a51f5fcba03c01013760ff21422275757cc7adcd270
7
+ data.tar.gz: f6d8bb596b0c423e041d69f520cbb0fd6268c7a36bf7910e32b1a189fe2c5da560a6c41f2f95fa5a93b83c70cce1edf999994bac65f28a0f88b7f0fdc01e3bd6
@@ -1,26 +1,37 @@
1
1
  sudo: required
2
- language: ruby
2
+ services:
3
+ - docker
3
4
 
4
- rvm:
5
- - 2.3.0
6
- - 2.2
7
- - 2.1
8
- - 2.0.0
9
- - 1.9.3
5
+ env:
6
+ global:
7
+ DOCKER_COMPOSE_VERSION: 1.8.1
8
+ matrix:
9
+ - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/Gemfile
10
+ - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/Gemfile
11
+ - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/Gemfile
12
+ - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/Gemfile
13
+ - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
14
+ - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
15
+ - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
16
+ - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
10
17
 
11
18
  before_install:
12
- - sudo apt-get update
13
- - sudo apt-get install libgeoip-dev
14
- - gem update bundler
19
+ #- sudo apt-get update
20
+ #- sudo apt-get install -y docker-engine
21
+ - sudo rm /usr/local/bin/docker-compose
22
+ - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) > docker-compose
23
+ - chmod +x docker-compose
24
+ - sudo mv docker-compose /usr/local/bin/
25
+ - test -f data/GeoLite2-City.mmdb || curl http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip > data/GeoLite2-City.mmdb
15
26
 
16
- gemfile:
17
- - Gemfile
18
- - gemfiles/fluentd_v0.10.gemfile
19
- - gemfiles/fluentd_v0.12.gemfile
27
+ install: true
20
28
 
21
- matrix:
22
- fast_finish: true
23
- allow_failures:
24
- - gemfile: gemfiles/fluentd_v0.10.gemfile
25
- - rvm: 1.9.3
26
- - rvm: 2.0.0
29
+ before_script:
30
+ - echo BUNDLE_GEMFILE=${BUNDLE_GEMFILE} > env
31
+ - docker-compose build test-ruby${TARGET_RUBY_VERSION}
32
+ - docker-compose ps
33
+ - docker-compose up -d test-ruby${TARGET_RUBY_VERSION}
34
+ - docker-compose exec test-ruby${TARGET_RUBY_VERSION} bundle install
35
+
36
+ script:
37
+ - docker-compose exec test-ruby${TARGET_RUBY_VERSION} bundle exec rake test
data/Appraisals CHANGED
@@ -1,7 +1,3 @@
1
- appraise "fluentd v0.10" do
2
- gem "fluentd", "~> 0.10.46"
3
- end
4
-
5
1
  appraise "fluentd v0.12" do
6
2
  gem "fluentd", "~> 0.12.0"
7
3
  end
data/README.md CHANGED
@@ -18,13 +18,16 @@ before use, install dependent library as:
18
18
  # for RHEL/CentOS
19
19
  $ sudo yum group install "Development Tools"
20
20
  $ sudo yum install geoip-devel --enablerepo=epel
21
+ $ sudo yum install libmaxminddb-devel --enablerepo=epel
21
22
 
22
23
  # for Ubuntu/Debian
23
24
  $ sudo apt-get install build-essential
24
25
  $ sudo apt-get install libgeoip-dev
26
+ $ sudo apt-get install libmaxminddb-dev
25
27
 
26
28
  # for OS X
27
29
  $ brew install geoip
30
+ $ brew install libmaxminddb
28
31
  ```
29
32
 
30
33
  ## Installation
@@ -48,7 +51,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
48
51
 
49
52
  ```xml
50
53
  <match access.apache>
51
- type geoip
54
+ @type geoip
52
55
 
53
56
  # Specify one or more geoip lookup field which has ip address (default: host)
54
57
  # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
@@ -89,7 +92,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
89
92
 
90
93
  ```xml
91
94
  <match access.apache>
92
- type geoip
95
+ @type geoip
93
96
  geoip_lookup_key user1_host, user2_host
94
97
  <record>
95
98
  user1_city ${city["user1_host"]}
@@ -107,12 +110,12 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
107
110
 
108
111
  ```
109
112
  <match access.apache>
110
- type geoip
113
+ @type geoip
111
114
  geoip_lookup_key host
112
115
  <record>
113
116
  # lat lon as properties
114
117
  # ex. {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
115
- location_properties { "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }
118
+ location_properties '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
116
119
 
117
120
  # lat lon as string
118
121
  # ex. "37.4192008972168,-122.05740356445312"
@@ -120,7 +123,7 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
120
123
 
121
124
  # GeoJSON (lat lon as array) is useful for Kibana's bettermap.
122
125
  # ex. [-122.05740356445312, 37.4192008972168]
123
- location_array [${longitude["host"]},${latitude["host"]}]
126
+ location_array '[${longitude["host"]},${latitude["host"]}]'
124
127
  </record>
125
128
  remove_tag_prefix access.
126
129
  tag geoip.${tag}
@@ -134,7 +137,7 @@ On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ...
134
137
 
135
138
  ```
136
139
  <match access.apache>
137
- type geoip
140
+ @type geoip
138
141
  geoip_lookup_key host
139
142
  <record>
140
143
  location_properties '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
@@ -194,16 +197,16 @@ Note that filter version of geoip plugin does not have handling tag feature.
194
197
 
195
198
  ```xml
196
199
  <source>
197
- type forward
200
+ @type forward
198
201
  </source>
199
202
 
200
203
  <match test.geoip>
201
- type copy
204
+ @type copy
202
205
  <store>
203
- type stdout
206
+ @type stdout
204
207
  </store>
205
208
  <store>
206
- type geoip
209
+ @type geoip
207
210
  geoip_lookup_key host
208
211
  <record>
209
212
  lat ${latitude["host"]}
@@ -216,7 +219,7 @@ Note that filter version of geoip plugin does not have handling tag feature.
216
219
  </match>
217
220
 
218
221
  <match debug.**>
219
- type stdout
222
+ @type stdout
220
223
  </match>
221
224
  ```
222
225
 
@@ -275,6 +278,8 @@ http://dev.maxmind.com/geoip/legacy/csv/
275
278
 
276
279
  ## Placeholders
277
280
 
281
+ ### GeoIP legacy
282
+
278
283
  Provides these placeholders for adding field of geolocate results.<br />
279
284
  For more example of geolocating, you can try these websites like [Geo IP Address View](http://www.geoipview.com/) or [View my IP information](http://www.geoiptool.com/en/).
280
285
 
@@ -292,6 +297,40 @@ For more example of geolocating, you can try these websites like [Geo IP Address
292
297
 
293
298
  Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/#GeoIP_City_Edition_CSV_Database_Fields
294
299
 
300
+ ### GeoIP2
301
+
302
+ You can get any fields in the
303
+ [GeoLite2](http://dev.maxmind.com/geoip/geoip2/geolite2/) database and
304
+ [GeoIP2 Downloadable Databases](http://dev.maxmind.com/geoip/geoip2/downloadable/).
305
+
306
+ For example(geoip2_c backend):
307
+
308
+ | placeholder attributes | output example | note |
309
+ |------------------------------------------|--------------------|------|
310
+ | ${city.names.en[lookup_field]} | "Mountain View" | - |
311
+ | ${location.latitude[lookup_field]} | 37.419200000000004 | - |
312
+ | ${location.longitude[lookup_field]} | -122.0574 | - |
313
+ | ${country.iso_code[lookup_field]} | "US" | - |
314
+ | ${country.names.en[lookup_field]} | "United States" | - |
315
+ | ${postal.code[lookup_field]} | "94043" | - |
316
+ | ${subdivisions.0.iso_code[lookup_field]} | "CA" | - |
317
+ | ${subdivisions.0.names.en[lookup_field]} | "California" | - |
318
+
319
+ For example(geoip2_compat backend):
320
+
321
+ | placeholder attributes | output example | note |
322
+ |-------------------------------|--------------------|------|
323
+ | ${city[lookup_field]} | "Mountain View" | - |
324
+ | ${latitude[lookup_field]} | 37.419200000000004 | - |
325
+ | ${longitude[lookup_field]} | -122.0574 | - |
326
+ | ${country_code[lookup_field]} | "US" | - |
327
+ | ${country_name[lookup_field]} | "United States" | - |
328
+ | ${postal_code[lookup_field]} | "94043" | |
329
+ | ${region[lookup_field]} | "CA" | - |
330
+ | ${region_name[lookup_field]} | "California" | - |
331
+
332
+ **NOTE**: geoip2_compat backend supports only above fields.
333
+
295
334
  ## Parameters
296
335
 
297
336
  ### GeoipOutput
@@ -0,0 +1,31 @@
1
+ version: "2"
2
+ services:
3
+ test-ruby2.4:
4
+ build:
5
+ context: .
6
+ dockerfile: dockerfiles/Dockerfile-ruby2.4
7
+ env_file:
8
+ - env
9
+ command: ruby -run -e httpd .
10
+ test-ruby2.3:
11
+ build:
12
+ context: .
13
+ dockerfile: dockerfiles/Dockerfile-ruby2.3
14
+ env_file:
15
+ - env
16
+ command: ruby -run -e httpd .
17
+ test-ruby2.2:
18
+ build:
19
+ context: .
20
+ dockerfile: dockerfiles/Dockerfile-ruby2.2
21
+ env_file:
22
+ - env
23
+ command: ruby -run -e httpd .
24
+ test-ruby2.1:
25
+ build:
26
+ context: .
27
+ dockerfile: dockerfiles/Dockerfile-ruby2.1
28
+ env_file:
29
+ - env
30
+ command: ruby -run -e httpd .
31
+
@@ -0,0 +1,8 @@
1
+ FROM ruby:2.1-alpine
2
+
3
+ ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
+
5
+ RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
6
+
7
+ WORKDIR /app
8
+ COPY . .
@@ -0,0 +1,8 @@
1
+ FROM ruby:2.2-alpine
2
+
3
+ ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
+
5
+ RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
6
+
7
+ WORKDIR /app
8
+ COPY . .
@@ -0,0 +1,8 @@
1
+ FROM ruby:2.3-alpine
2
+
3
+ ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
+
5
+ RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
6
+
7
+ WORKDIR /app
8
+ COPY . .
@@ -0,0 +1,8 @@
1
+ FROM ruby:2.4-alpine
2
+
3
+ ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
+
5
+ RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
6
+
7
+ WORKDIR /app
8
+ COPY . .
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-geoip"
7
- spec.version = "0.6.1"
7
+ spec.version = "0.7.0"
8
8
  spec.authors = ["Kentaro Yoshida"]
9
9
  spec.email = ["y.ken.studio@gmail.com"]
10
10
  spec.summary = %q{Fluentd Filter plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -20,9 +20,11 @@ Gem::Specification.new do |spec|
20
20
  spec.add_development_dependency "rake"
21
21
  spec.add_development_dependency "appraisal"
22
22
  spec.add_development_dependency "test-unit", ">= 3.1.0"
23
+ spec.add_development_dependency "geoip2_compat"
24
+ spec.add_development_dependency "geoip2_c"
23
25
 
24
26
  spec.add_runtime_dependency "fluentd"
25
27
  spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
26
28
  spec.add_runtime_dependency "geoip-c"
29
+ spec.add_runtime_dependency "dig_rb"
27
30
  end
28
-
@@ -1,8 +1,11 @@
1
+ require 'fluent/plugin/geoip'
2
+
1
3
  module Fluent
2
4
  class GeoipFilter < Filter
3
5
  Plugin.register_filter('geoip', self)
4
6
 
5
7
  config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
8
+ config_param :geoip2_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
6
9
  config_param :geoip_lookup_key, :string, :default => 'host'
7
10
  config_param :skip_adding_null_record, :bool, :default => false
8
11
 
@@ -13,12 +16,7 @@ module Fluent
13
16
  config_param :flush_interval, :time, :default => 0
14
17
  config_param :log_level, :string, :default => 'warn'
15
18
 
16
-
17
- def initialize
18
- require 'fluent/plugin/geoip'
19
-
20
- super
21
- end
19
+ config_param :backend_library, :enum, :list => Fluent::GeoIP::BACKEND_LIBRARIES, :default => :geoip
22
20
 
23
21
  def configure(conf)
24
22
  super
@@ -1,12 +1,16 @@
1
1
  require 'geoip'
2
2
  require 'yajl'
3
+ require 'dig_rb'
3
4
 
4
5
  module Fluent
5
6
  class GeoIP
7
+ BACKEND_LIBRARIES = [:geoip, :geoip2_compat, :geoip2_c]
8
+
6
9
  REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\[['"](?<record_key>-?[^'"]+)['"]\]\}$/
7
10
  REGEXP_PLACEHOLDER_SCAN = /['"]?(\$\{[^\}]+?\})['"]?/
8
11
 
9
12
  GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
13
+ GEOIP2_COMPAT_KEYS = %w(city country_code country_name latitude longitude postal_code region region_name)
10
14
 
11
15
  attr_reader :log
12
16
 
@@ -25,7 +29,7 @@ module Fluent
25
29
  if record_key.nil?
26
30
  raise Fluent::ConfigError, "geoip: missing value found at '#{key} #{lookup_field}'"
27
31
  end
28
- @map.store(record_key, "${#{geoip_key}['#{lookup_field}']}")
32
+ @map[record_key] = "${#{geoip_key}['#{lookup_field}']}"
29
33
  end
30
34
  end
31
35
  if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
@@ -53,7 +57,16 @@ module Fluent
53
57
  @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
54
58
  @placeholder_keys.each do |key|
55
59
  geoip_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:geoip_key]
56
- raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
60
+ case plugin.backend_library
61
+ when :geoip
62
+ raise Fluent::ConfigError, "#{plugin.backend_library}: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
63
+ when :geoip2_compat
64
+ raise Fluent::ConfigError, "#{plugin.backend_library}: unsupported key #{geoip_key}" unless GEOIP2_COMPAT_KEYS.include?(geoip_key)
65
+ when :geoip2_c
66
+ # Nothing to do.
67
+ # We cannot define supported key(s) before we fetch values from GeoIP2 database
68
+ # because geoip2_c can fetch any fields in GeoIP2 database.
69
+ end
57
70
  end
58
71
 
59
72
  if plugin.is_a?(Fluent::BufferedOutput)
@@ -63,7 +76,7 @@ module Fluent
63
76
  end
64
77
  end
65
78
 
66
- @geoip = ::GeoIP::City.new(plugin.geoip_database, :memory, false)
79
+ @geoip = load_database(plugin)
67
80
  end
68
81
 
69
82
  def add_geoip_field(record)
@@ -81,7 +94,7 @@ module Fluent
81
94
  else
82
95
  rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
83
96
  end
84
- record.store(record_key, rewrited)
97
+ record[record_key] = rewrited
85
98
  end
86
99
  return record
87
100
  end
@@ -116,25 +129,25 @@ module Fluent
116
129
  def get_address(record)
117
130
  address = {}
118
131
  @geoip_lookup_key.each do |field|
119
- address.store(field, record[field]); next if not record[field].nil?
120
- key = field.split('.')
121
- obj = record
122
- key.each {|k|
123
- break obj = nil if not obj.has_key?(k)
124
- obj = obj[k]
125
- }
126
- address.store(field, obj)
132
+ address[field] = record[field] || record.dig(*field.split('.'))
127
133
  end
128
- return address
134
+ address
129
135
  end
130
136
 
131
137
  def geolocate(addresses)
132
138
  geodata = {}
133
139
  addresses.each do |field, ip|
134
- geo = ip.nil? ? nil : @geoip.look_up(ip)
135
- geodata.store(field, geo)
140
+ geo = nil
141
+ if ip
142
+ geo = if @geoip.respond_to?(:look_up)
143
+ @geoip.look_up(ip)
144
+ else
145
+ @geoip.lookup(ip)
146
+ end
147
+ end
148
+ geodata[field] = geo
136
149
  end
137
- return geodata
150
+ geodata
138
151
  end
139
152
 
140
153
  def create_placeholder(geodata)
@@ -142,9 +155,25 @@ module Fluent
142
155
  @placeholder_keys.each do |placeholder_key|
143
156
  position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
144
157
  next if position.nil? or geodata[position[:record_key]].nil?
145
- placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
158
+ keys = [position[:record_key]] + position[:geoip_key].split('.').map(&:to_sym)
159
+ placeholder[placeholder_key] = geodata.dig(*keys)
160
+ end
161
+ placeholder
162
+ end
163
+
164
+ def load_database(plugin)
165
+ case plugin.backend_library
166
+ when :geoip
167
+ ::GeoIP::City.new(plugin.geoip_database, :memory, false)
168
+ when :geoip2_compat
169
+ require 'geoip2_compat'
170
+ GeoIP2Compat.new(plugin.geoip2_database)
171
+ when :geoip2_c
172
+ require 'geoip2'
173
+ GeoIP2::Database.new(plugin.geoip2_database)
146
174
  end
147
- return placeholder
175
+ rescue LoadError
176
+ raise Fluent::ConfigError, "You must install #{plugin.backend_library} gem."
148
177
  end
149
178
  end
150
179
  end