fluent-plugin-geoip 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8fdb172feeffd90a77f0a83f3f44b58b88ddb138
4
- data.tar.gz: 3abc009ded4a7ced18e046c27d9f50094f11b865
3
+ metadata.gz: 6064156b8f481d751a7f1ee92c5e80b189c52f2c
4
+ data.tar.gz: 95e5509ad5ec16ff48e9d71687caa205bd4dbf85
5
5
  SHA512:
6
- metadata.gz: c5ef5143cdd8f2ea9fe372d5a2e0ee5bbe5981a4a1c2cd84b34364fe7f6679a897b5a40732d8b97ad9dfdf5f36b3dc257ec529357c2411e817cfdb34822e05f0
7
- data.tar.gz: 70d27191a5cc8f7f4ee5153845149f7d0881c5bf08ed7554ab5424a65f7806b78f36909fa8319d1850d49c13d2247fa41f466cf954777e85d80d45065948eedb
6
+ metadata.gz: c3fcd73dcac19ab974974099f05147e1ba082cf339b0c45ae00971049cadf94cf6d8c1a438351c0c08cd7a51f5fcba03c01013760ff21422275757cc7adcd270
7
+ data.tar.gz: f6d8bb596b0c423e041d69f520cbb0fd6268c7a36bf7910e32b1a189fe2c5da560a6c41f2f95fa5a93b83c70cce1edf999994bac65f28a0f88b7f0fdc01e3bd6
@@ -1,26 +1,37 @@
1
1
  sudo: required
2
- language: ruby
2
+ services:
3
+ - docker
3
4
 
4
- rvm:
5
- - 2.3.0
6
- - 2.2
7
- - 2.1
8
- - 2.0.0
9
- - 1.9.3
5
+ env:
6
+ global:
7
+ DOCKER_COMPOSE_VERSION: 1.8.1
8
+ matrix:
9
+ - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/Gemfile
10
+ - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/Gemfile
11
+ - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/Gemfile
12
+ - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/Gemfile
13
+ - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
14
+ - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
15
+ - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
16
+ - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
10
17
 
11
18
  before_install:
12
- - sudo apt-get update
13
- - sudo apt-get install libgeoip-dev
14
- - gem update bundler
19
+ #- sudo apt-get update
20
+ #- sudo apt-get install -y docker-engine
21
+ - sudo rm /usr/local/bin/docker-compose
22
+ - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) > docker-compose
23
+ - chmod +x docker-compose
24
+ - sudo mv docker-compose /usr/local/bin/
25
+ - test -f data/GeoLite2-City.mmdb || curl http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip > data/GeoLite2-City.mmdb
15
26
 
16
- gemfile:
17
- - Gemfile
18
- - gemfiles/fluentd_v0.10.gemfile
19
- - gemfiles/fluentd_v0.12.gemfile
27
+ install: true
20
28
 
21
- matrix:
22
- fast_finish: true
23
- allow_failures:
24
- - gemfile: gemfiles/fluentd_v0.10.gemfile
25
- - rvm: 1.9.3
26
- - rvm: 2.0.0
29
+ before_script:
30
+ - echo BUNDLE_GEMFILE=${BUNDLE_GEMFILE} > env
31
+ - docker-compose build test-ruby${TARGET_RUBY_VERSION}
32
+ - docker-compose ps
33
+ - docker-compose up -d test-ruby${TARGET_RUBY_VERSION}
34
+ - docker-compose exec test-ruby${TARGET_RUBY_VERSION} bundle install
35
+
36
+ script:
37
+ - docker-compose exec test-ruby${TARGET_RUBY_VERSION} bundle exec rake test
data/Appraisals CHANGED
@@ -1,7 +1,3 @@
1
- appraise "fluentd v0.10" do
2
- gem "fluentd", "~> 0.10.46"
3
- end
4
-
5
1
  appraise "fluentd v0.12" do
6
2
  gem "fluentd", "~> 0.12.0"
7
3
  end
data/README.md CHANGED
@@ -18,13 +18,16 @@ before use, install dependent library as:
18
18
  # for RHEL/CentOS
19
19
  $ sudo yum group install "Development Tools"
20
20
  $ sudo yum install geoip-devel --enablerepo=epel
21
+ $ sudo yum install libmaxminddb-devel --enablerepo=epel
21
22
 
22
23
  # for Ubuntu/Debian
23
24
  $ sudo apt-get install build-essential
24
25
  $ sudo apt-get install libgeoip-dev
26
+ $ sudo apt-get install libmaxminddb-dev
25
27
 
26
28
  # for OS X
27
29
  $ brew install geoip
30
+ $ brew install libmaxminddb
28
31
  ```
29
32
 
30
33
  ## Installation
@@ -48,7 +51,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
48
51
 
49
52
  ```xml
50
53
  <match access.apache>
51
- type geoip
54
+ @type geoip
52
55
 
53
56
  # Specify one or more geoip lookup field which has ip address (default: host)
54
57
  # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
@@ -89,7 +92,7 @@ $ sudo td-agent-gem install fluent-plugin-geoip
89
92
 
90
93
  ```xml
91
94
  <match access.apache>
92
- type geoip
95
+ @type geoip
93
96
  geoip_lookup_key user1_host, user2_host
94
97
  <record>
95
98
  user1_city ${city["user1_host"]}
@@ -107,12 +110,12 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
107
110
 
108
111
  ```
109
112
  <match access.apache>
110
- type geoip
113
+ @type geoip
111
114
  geoip_lookup_key host
112
115
  <record>
113
116
  # lat lon as properties
114
117
  # ex. {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
115
- location_properties { "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }
118
+ location_properties '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
116
119
 
117
120
  # lat lon as string
118
121
  # ex. "37.4192008972168,-122.05740356445312"
@@ -120,7 +123,7 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
120
123
 
121
124
  # GeoJSON (lat lon as array) is useful for Kibana's bettermap.
122
125
  # ex. [-122.05740356445312, 37.4192008972168]
123
- location_array [${longitude["host"]},${latitude["host"]}]
126
+ location_array '[${longitude["host"]},${latitude["host"]}]'
124
127
  </record>
125
128
  remove_tag_prefix access.
126
129
  tag geoip.${tag}
@@ -134,7 +137,7 @@ On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ...
134
137
 
135
138
  ```
136
139
  <match access.apache>
137
- type geoip
140
+ @type geoip
138
141
  geoip_lookup_key host
139
142
  <record>
140
143
  location_properties '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
@@ -194,16 +197,16 @@ Note that filter version of geoip plugin does not have handling tag feature.
194
197
 
195
198
  ```xml
196
199
  <source>
197
- type forward
200
+ @type forward
198
201
  </source>
199
202
 
200
203
  <match test.geoip>
201
- type copy
204
+ @type copy
202
205
  <store>
203
- type stdout
206
+ @type stdout
204
207
  </store>
205
208
  <store>
206
- type geoip
209
+ @type geoip
207
210
  geoip_lookup_key host
208
211
  <record>
209
212
  lat ${latitude["host"]}
@@ -216,7 +219,7 @@ Note that filter version of geoip plugin does not have handling tag feature.
216
219
  </match>
217
220
 
218
221
  <match debug.**>
219
- type stdout
222
+ @type stdout
220
223
  </match>
221
224
  ```
222
225
 
@@ -275,6 +278,8 @@ http://dev.maxmind.com/geoip/legacy/csv/
275
278
 
276
279
  ## Placeholders
277
280
 
281
+ ### GeoIP legacy
282
+
278
283
  Provides these placeholders for adding field of geolocate results.<br />
279
284
  For more example of geolocating, you can try these websites like [Geo IP Address View](http://www.geoipview.com/) or [View my IP information](http://www.geoiptool.com/en/).
280
285
 
@@ -292,6 +297,40 @@ For more example of geolocating, you can try these websites like [Geo IP Address
292
297
 
293
298
  Further more specification available at http://dev.maxmind.com/geoip/legacy/csv/#GeoIP_City_Edition_CSV_Database_Fields
294
299
 
300
+ ### GeoIP2
301
+
302
+ You can get any fields in the
303
+ [GeoLite2](http://dev.maxmind.com/geoip/geoip2/geolite2/) database and
304
+ [GeoIP2 Downloadable Databases](http://dev.maxmind.com/geoip/geoip2/downloadable/).
305
+
306
+ For example(geoip2_c backend):
307
+
308
+ | placeholder attributes | output example | note |
309
+ |------------------------------------------|--------------------|------|
310
+ | ${city.names.en[lookup_field]} | "Mountain View" | - |
311
+ | ${location.latitude[lookup_field]} | 37.419200000000004 | - |
312
+ | ${location.longitude[lookup_field]} | -122.0574 | - |
313
+ | ${country.iso_code[lookup_field]} | "US" | - |
314
+ | ${country.names.en[lookup_field]} | "United States" | - |
315
+ | ${postal.code[lookup_field]} | "94043" | - |
316
+ | ${subdivisions.0.iso_code[lookup_field]} | "CA" | - |
317
+ | ${subdivisions.0.names.en[lookup_field]} | "California" | - |
318
+
319
+ For example(geoip2_compat backend):
320
+
321
+ | placeholder attributes | output example | note |
322
+ |-------------------------------|--------------------|------|
323
+ | ${city[lookup_field]} | "Mountain View" | - |
324
+ | ${latitude[lookup_field]} | 37.419200000000004 | - |
325
+ | ${longitude[lookup_field]} | -122.0574 | - |
326
+ | ${country_code[lookup_field]} | "US" | - |
327
+ | ${country_name[lookup_field]} | "United States" | - |
328
+ | ${postal_code[lookup_field]} | "94043" | |
329
+ | ${region[lookup_field]} | "CA" | - |
330
+ | ${region_name[lookup_field]} | "California" | - |
331
+
332
+ **NOTE**: geoip2_compat backend supports only above fields.
333
+
295
334
  ## Parameters
296
335
 
297
336
  ### GeoipOutput
@@ -0,0 +1,31 @@
1
+ version: "2"
2
+ services:
3
+ test-ruby2.4:
4
+ build:
5
+ context: .
6
+ dockerfile: dockerfiles/Dockerfile-ruby2.4
7
+ env_file:
8
+ - env
9
+ command: ruby -run -e httpd .
10
+ test-ruby2.3:
11
+ build:
12
+ context: .
13
+ dockerfile: dockerfiles/Dockerfile-ruby2.3
14
+ env_file:
15
+ - env
16
+ command: ruby -run -e httpd .
17
+ test-ruby2.2:
18
+ build:
19
+ context: .
20
+ dockerfile: dockerfiles/Dockerfile-ruby2.2
21
+ env_file:
22
+ - env
23
+ command: ruby -run -e httpd .
24
+ test-ruby2.1:
25
+ build:
26
+ context: .
27
+ dockerfile: dockerfiles/Dockerfile-ruby2.1
28
+ env_file:
29
+ - env
30
+ command: ruby -run -e httpd .
31
+
@@ -0,0 +1,8 @@
1
+ FROM ruby:2.1-alpine
2
+
3
+ ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
+
5
+ RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
6
+
7
+ WORKDIR /app
8
+ COPY . .
@@ -0,0 +1,8 @@
1
+ FROM ruby:2.2-alpine
2
+
3
+ ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
+
5
+ RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
6
+
7
+ WORKDIR /app
8
+ COPY . .
@@ -0,0 +1,8 @@
1
+ FROM ruby:2.3-alpine
2
+
3
+ ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
+
5
+ RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
6
+
7
+ WORKDIR /app
8
+ COPY . .
@@ -0,0 +1,8 @@
1
+ FROM ruby:2.4-alpine
2
+
3
+ ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
+
5
+ RUN apk --no-cache --update add build-base ruby-dev libc6-compat libmaxminddb-dev geoip-dev git
6
+
7
+ WORKDIR /app
8
+ COPY . .
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-geoip"
7
- spec.version = "0.6.1"
7
+ spec.version = "0.7.0"
8
8
  spec.authors = ["Kentaro Yoshida"]
9
9
  spec.email = ["y.ken.studio@gmail.com"]
10
10
  spec.summary = %q{Fluentd Filter plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -20,9 +20,11 @@ Gem::Specification.new do |spec|
20
20
  spec.add_development_dependency "rake"
21
21
  spec.add_development_dependency "appraisal"
22
22
  spec.add_development_dependency "test-unit", ">= 3.1.0"
23
+ spec.add_development_dependency "geoip2_compat"
24
+ spec.add_development_dependency "geoip2_c"
23
25
 
24
26
  spec.add_runtime_dependency "fluentd"
25
27
  spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
26
28
  spec.add_runtime_dependency "geoip-c"
29
+ spec.add_runtime_dependency "dig_rb"
27
30
  end
28
-
@@ -1,8 +1,11 @@
1
+ require 'fluent/plugin/geoip'
2
+
1
3
  module Fluent
2
4
  class GeoipFilter < Filter
3
5
  Plugin.register_filter('geoip', self)
4
6
 
5
7
  config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
8
+ config_param :geoip2_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLite2-City.mmdb'
6
9
  config_param :geoip_lookup_key, :string, :default => 'host'
7
10
  config_param :skip_adding_null_record, :bool, :default => false
8
11
 
@@ -13,12 +16,7 @@ module Fluent
13
16
  config_param :flush_interval, :time, :default => 0
14
17
  config_param :log_level, :string, :default => 'warn'
15
18
 
16
-
17
- def initialize
18
- require 'fluent/plugin/geoip'
19
-
20
- super
21
- end
19
+ config_param :backend_library, :enum, :list => Fluent::GeoIP::BACKEND_LIBRARIES, :default => :geoip
22
20
 
23
21
  def configure(conf)
24
22
  super
@@ -1,12 +1,16 @@
1
1
  require 'geoip'
2
2
  require 'yajl'
3
+ require 'dig_rb'
3
4
 
4
5
  module Fluent
5
6
  class GeoIP
7
+ BACKEND_LIBRARIES = [:geoip, :geoip2_compat, :geoip2_c]
8
+
6
9
  REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\[['"](?<record_key>-?[^'"]+)['"]\]\}$/
7
10
  REGEXP_PLACEHOLDER_SCAN = /['"]?(\$\{[^\}]+?\})['"]?/
8
11
 
9
12
  GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
13
+ GEOIP2_COMPAT_KEYS = %w(city country_code country_name latitude longitude postal_code region region_name)
10
14
 
11
15
  attr_reader :log
12
16
 
@@ -25,7 +29,7 @@ module Fluent
25
29
  if record_key.nil?
26
30
  raise Fluent::ConfigError, "geoip: missing value found at '#{key} #{lookup_field}'"
27
31
  end
28
- @map.store(record_key, "${#{geoip_key}['#{lookup_field}']}")
32
+ @map[record_key] = "${#{geoip_key}['#{lookup_field}']}"
29
33
  end
30
34
  end
31
35
  if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
@@ -53,7 +57,16 @@ module Fluent
53
57
  @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
54
58
  @placeholder_keys.each do |key|
55
59
  geoip_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:geoip_key]
56
- raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
60
+ case plugin.backend_library
61
+ when :geoip
62
+ raise Fluent::ConfigError, "#{plugin.backend_library}: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
63
+ when :geoip2_compat
64
+ raise Fluent::ConfigError, "#{plugin.backend_library}: unsupported key #{geoip_key}" unless GEOIP2_COMPAT_KEYS.include?(geoip_key)
65
+ when :geoip2_c
66
+ # Nothing to do.
67
+ # We cannot define supported key(s) before we fetch values from GeoIP2 database
68
+ # because geoip2_c can fetch any fields in GeoIP2 database.
69
+ end
57
70
  end
58
71
 
59
72
  if plugin.is_a?(Fluent::BufferedOutput)
@@ -63,7 +76,7 @@ module Fluent
63
76
  end
64
77
  end
65
78
 
66
- @geoip = ::GeoIP::City.new(plugin.geoip_database, :memory, false)
79
+ @geoip = load_database(plugin)
67
80
  end
68
81
 
69
82
  def add_geoip_field(record)
@@ -81,7 +94,7 @@ module Fluent
81
94
  else
82
95
  rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
83
96
  end
84
- record.store(record_key, rewrited)
97
+ record[record_key] = rewrited
85
98
  end
86
99
  return record
87
100
  end
@@ -116,25 +129,25 @@ module Fluent
116
129
  def get_address(record)
117
130
  address = {}
118
131
  @geoip_lookup_key.each do |field|
119
- address.store(field, record[field]); next if not record[field].nil?
120
- key = field.split('.')
121
- obj = record
122
- key.each {|k|
123
- break obj = nil if not obj.has_key?(k)
124
- obj = obj[k]
125
- }
126
- address.store(field, obj)
132
+ address[field] = record[field] || record.dig(*field.split('.'))
127
133
  end
128
- return address
134
+ address
129
135
  end
130
136
 
131
137
  def geolocate(addresses)
132
138
  geodata = {}
133
139
  addresses.each do |field, ip|
134
- geo = ip.nil? ? nil : @geoip.look_up(ip)
135
- geodata.store(field, geo)
140
+ geo = nil
141
+ if ip
142
+ geo = if @geoip.respond_to?(:look_up)
143
+ @geoip.look_up(ip)
144
+ else
145
+ @geoip.lookup(ip)
146
+ end
147
+ end
148
+ geodata[field] = geo
136
149
  end
137
- return geodata
150
+ geodata
138
151
  end
139
152
 
140
153
  def create_placeholder(geodata)
@@ -142,9 +155,25 @@ module Fluent
142
155
  @placeholder_keys.each do |placeholder_key|
143
156
  position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
144
157
  next if position.nil? or geodata[position[:record_key]].nil?
145
- placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
158
+ keys = [position[:record_key]] + position[:geoip_key].split('.').map(&:to_sym)
159
+ placeholder[placeholder_key] = geodata.dig(*keys)
160
+ end
161
+ placeholder
162
+ end
163
+
164
+ def load_database(plugin)
165
+ case plugin.backend_library
166
+ when :geoip
167
+ ::GeoIP::City.new(plugin.geoip_database, :memory, false)
168
+ when :geoip2_compat
169
+ require 'geoip2_compat'
170
+ GeoIP2Compat.new(plugin.geoip2_database)
171
+ when :geoip2_c
172
+ require 'geoip2'
173
+ GeoIP2::Database.new(plugin.geoip2_database)
146
174
  end
147
- return placeholder
175
+ rescue LoadError
176
+ raise Fluent::ConfigError, "You must install #{plugin.backend_library} gem."
148
177
  end
149
178
  end
150
179
  end