fluent-plugin-geoip 0.0.6 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -2,8 +2,8 @@
2
2
 
3
3
  Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.
4
4
 
5
- fluent-plugin-geoip has bundled cost-free [GeoLite City database](http://dev.maxmind.com/geoip/legacy/geolite/) by default.
6
- Also you can use purchased [GeoIP City database](http://www.maxmind.com/en/city) ([lang:ja](http://www.maxmind.com/ja/city)) which costs starting from $50.
5
+ fluent-plugin-geoip has bundled cost-free [GeoLite City database](http://dev.maxmind.com/geoip/legacy/geolite/) by default.<br />
6
+ Also you can use purchased [GeoIP City database](http://www.maxmind.com/en/city) ([lang:ja](http://www.maxmind.com/ja/city)) which costs starting from $50.
7
7
 
8
8
  The accuracy details for GeoLite City (free) and GeoIP City (purchased) has described at the page below.
9
9
 
@@ -40,31 +40,35 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
40
40
  <match access.apache>
41
41
  type geoip
42
42
 
43
- # Specify geoip lookup field (default: host)
43
+ # Specify one or more geoip lookup field which has ip address (default: host)
44
44
  # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
45
- geoip_lookup_key host
45
+ geoip_lookup_key host
46
46
 
47
47
  # Specify geoip database (using bundled GeoLiteCity databse by default)
48
- geoip_database 'data/GeoLiteCity.dat'
49
-
50
- # Set adding field of geolocate results (more than one settings are required.)
51
- enable_key_city geoip_city
52
- enable_key_latitude geoip_lat
53
- enable_key_longitude geoip_lon
54
- enable_key_country_code3 geoip_country3
55
- enable_key_country_code geoip_country
56
- enable_key_country_name geoip_country_name
57
- enable_key_dma_code geoip_dma
58
- enable_key_area_code geoip_area
59
- enable_key_region geoip_region
60
-
61
- # Setting for tag
62
- remove_tag_prefix access.
63
- add_tag_prefix geoip.
64
- include_tag_key false
65
-
66
- # Buffering time (default: 60s)
67
- flush_interval 1s
48
+ geoip_database 'data/GeoLiteCity.dat'
49
+
50
+ # Set adding field with placeholder (more than one settings are required.)
51
+ <record>
52
+ city ${city['host']}
53
+ latitude ${latitude['host']}
54
+ longitude ${longitude['host']}
55
+ country_code3 ${country_code3['host']}
56
+ country ${country['host']}
57
+ country_name ${country_name['host']}
58
+ dma ${dma['host']}
59
+ area ${area['host']}
60
+ region ${region['host']}
61
+ </record>
62
+
63
+ # Settings for tag
64
+ remove_tag_prefix access.
65
+ tag geoip.${tag}
66
+
67
+ # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
68
+ log_level info
69
+
70
+ # Set buffering time (default: 0s)
71
+ flush_interval 1s
68
72
  </match>
69
73
  ```
70
74
 
@@ -73,20 +77,39 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
73
77
  ```xml
74
78
  <match access.apache>
75
79
  type geoip
80
+ geoip_lookup_key user1_host, user2_host
81
+ <record>
82
+ user1_city ${city['user1_host']}
83
+ user2_city ${city['user2_host']}
84
+ </record>
85
+ remove_tag_prefix access.
86
+ tag geoip.${tag}
87
+ </match>
88
+ ```
76
89
 
77
- # Set ip address key to geolocate
78
- geoip_lookup_key user1_host, user2_host
79
-
80
- # Set adding field of geolocate results
81
- enable_key_city user1_city, user2_city
82
- enable_key_country_name user1_country, user2_country
90
+ #### Advanced config samples
83
91
 
84
- # Setting for tag
85
- remove_tag_prefix access.
86
- add_tag_prefix geoip.
92
+ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (JSON) parser.
87
93
 
88
- # Buffering time
89
- flush_interval 1s
94
+ ```
95
+ <match input.access>
96
+ type geoip
97
+ geoip_lookup_key host
98
+ <record>
99
+ # lat lon as properties
100
+ # ex. {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
101
+ location_properties { "lat":${latitude['host']}, "lon":${longitude['host']}}
102
+
103
+ # lat lon as string
104
+ # ex. "37.4192008972168,-122.05740356445312"
105
+ location_string ${latitude['host']},${longitude['host']}
106
+
107
+ # lat lon as array (it is useful for Kibana's bettermap.)
108
+ # ex. [-122.05740356445312, 37.4192008972168]
109
+ location_array [${longitude['host']},${latitude['host']}]
110
+ </record>
111
+ remove_tag_prefix access.
112
+ tag geoip.${tag}
90
113
  </match>
91
114
  ```
92
115
 
@@ -105,14 +128,15 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
105
128
  type stdout
106
129
  </store>
107
130
  <store>
108
- type geoip
109
- geoip_lookup_key host
110
- enable_key_city city
111
- enable_key_latitude lat
112
- enable_key_longitude lon
113
- remove_tag_prefix test.
114
- add_tag_prefix debug.
115
- flush_interval 5s
131
+ type geoip
132
+ geoip_lookup_key host
133
+ <record>
134
+ city ${city['host']}
135
+ lat ${latitude['host']}
136
+ lon ${longitude['host']}
137
+ </record>
138
+ remove_tag_prefix test.
139
+ tag debug.${tag}
116
140
  </store>
117
141
  </match>
118
142
 
@@ -133,30 +157,69 @@ $ tail /var/log/td-agent/td-agent.log
133
157
  2013-08-04 16:21:32 +0900 debug.geoip: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
134
158
  ```
135
159
 
136
- For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.
160
+ For more details of geoip data format is described at the page below in section `GeoIP City Edition CSV Database Fields`.<br />
137
161
  http://dev.maxmind.com/geoip/legacy/csv/
138
162
 
163
+ ## Placeholders
164
+
165
+ Provides these placeholders for adding field of geolocate results.
166
+
167
+ * ${city}
168
+ * ${latitude}
169
+ * ${longitude}
170
+ * ${country_code3}
171
+ * ${country_code}
172
+ * ${country_name}
173
+ * ${dma_code}
174
+ * ${area_code}
175
+ * ${region}
176
+
177
+ ## Parameters
178
+
179
+ * `include_tag_key` (default: false)
180
+ * `tag_key`
181
+
182
+ Add original tag name into filtered record using SetTagKeyMixin.<br />
183
+ Further details are written at http://docs.fluentd.org/articles/in_exec
184
+
185
+ * `remove_tag_prefix`
186
+ * `remove_tag_suffix`
187
+ * `add_tag_prefix`
188
+ * `add_tag_suffix`
189
+
190
+ Set one or more option are required unless using `tag` option for editing tag name. (HandleTagNameMixin feature)
191
+
192
+ * `tag`
193
+
194
+ On using this option with tag placeholder like `tag geoip.${tag}` (test code is available at [test_out_geoip.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
195
+
196
+ * `flush_interval` (default: 0 sec)
197
+
198
+ Set buffering time to execute bulk lookup geoip.
199
+
139
200
  ## Articles
140
201
 
141
- * [IPアドレスを元に位置情報をリアルタイムに付与する fluent-plugin-geoip v0.0.1をリリースしました #fluentd - Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released)
202
+ * [IPアドレスを元に位置情報をリアルタイムに付与する fluent-plugin-geoip v0.0.1をリリースしました #fluentd - Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released)<br />
142
203
  http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-has-released
143
204
 
144
- * [初の安定版 fluent-plugin-geoip v0.0.3 をリリースしました #fluentd- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.3)
205
+ * [初の安定版 fluent-plugin-geoip v0.0.3 をリリースしました #fluentd- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.3)<br />
145
206
  http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.3
146
207
 
147
- * [fluent-plugin-geoip v0.0.4 をリリースしました。ElasticSearch+Kibanaの世界地図に位置情報をプロットするために必要なFluentdの設定サンプルも紹介します- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.4)
208
+ * [fluent-plugin-geoip v0.0.4 をリリースしました。ElasticSearch+Kibanaの世界地図に位置情報をプロットするために必要なFluentdの設定サンプルも紹介します- Y-Ken Studio](http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.4)<br />
148
209
  http://y-ken.hatenablog.com/entry/fluent-plugin-geoip-v0.0.4
149
210
 
150
- * [Released GeoIP plugin to work together with ElasticSearch + Kibana v3](https://groups.google.com/d/topic/fluentd/OVIcH_SKBwM/discussion)
211
+ * [Released GeoIP plugin to work together with ElasticSearch + Kibana v3](https://groups.google.com/d/topic/fluentd/OVIcH_SKBwM/discussion)<br />
151
212
  https://groups.google.com/d/topic/fluentd/OVIcH_SKBwM/discussion
152
213
 
153
- * [Fluentd、Amazon RedshiftとTableauを用いたカジュアルなデータ可視化 | SmartNews開発者ブログ](http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-redshift-and-tableau/)
214
+ * [Fluentd、Amazon RedshiftとTableauを用いたカジュアルなデータ可視化 | SmartNews開発者ブログ](http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-redshift-and-tableau/)<br />
154
215
  http://developer.smartnews.be/blog/2013/10/03/easy-data-analysis-using-fluentd-redshift-and-tableau/
155
216
 
156
217
  ## TODO
157
218
 
158
219
  Pull requests are very welcome!!
159
220
 
221
+ * support [GeoIP2](http://dev.maxmind.com/geoip/geoip2/whats-new-in-geoip2/)
222
+
160
223
  ## Contributing
161
224
 
162
225
  1. Fork it
@@ -167,7 +230,7 @@ Pull requests are very welcome!!
167
230
 
168
231
  ## Copyright
169
232
 
170
- Copyright (c) 2013- Kentaro Yoshida (@yoshi_ken)
233
+ Copyright (c) 2013- Kentaro Yoshida ([@yoshi_ken](https://twitter.com/yoshi_ken))
171
234
 
172
235
  ## License
173
236
 
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-geoip"
7
- spec.version = "0.0.6"
7
+ spec.version = "0.1.0"
8
8
  spec.authors = ["Kentaro Yoshida"]
9
9
  spec.email = ["y.ken.studio@gmail.com"]
10
10
  spec.summary = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -19,5 +19,6 @@ Gem::Specification.new do |spec|
19
19
  spec.add_development_dependency "bundler"
20
20
  spec.add_development_dependency "rake"
21
21
  spec.add_runtime_dependency "fluentd"
22
+ spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
22
23
  spec.add_runtime_dependency "geoip-c"
23
24
  end
@@ -1,43 +1,75 @@
1
+ require 'fluent/mixin/rewrite_tag_name'
2
+
1
3
  class Fluent::GeoipOutput < Fluent::BufferedOutput
2
4
  Fluent::Plugin.register_output('geoip', self)
3
5
 
6
+ REGEXP_PLACEHOLDER = /^\$\{(?<geoip_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
4
7
  GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
8
+
5
9
  config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
6
10
  config_param :geoip_lookup_key, :string, :default => 'host'
11
+ config_param :tag, :string, :default => nil
7
12
 
8
13
  include Fluent::HandleTagNameMixin
9
14
  include Fluent::SetTagKeyMixin
10
15
  config_set_default :include_tag_key, false
11
- attr_reader :geoip_keys_map
16
+
17
+ include Fluent::Mixin::RewriteTagName
18
+ config_param :hostname_command, :string, :default => 'hostname'
19
+
20
+ config_param :flush_interval, :time, :default => 0
21
+ config_param :log_level, :string, :default => 'warn'
22
+
23
+ # Define `log` method for v0.10.42 or earlier
24
+ unless method_defined?(:log)
25
+ define_method("log") { $log }
26
+ end
12
27
 
13
28
  def initialize
14
29
  require 'geoip'
30
+ require 'yajl'
31
+
15
32
  super
16
33
  end
17
34
 
18
35
  def configure(conf)
19
36
  super
20
37
 
21
- @geoip_keys_map = Hash.new
38
+ @map = {}
39
+ @geoip_lookup_key = @geoip_lookup_key.split(/\s*,\s*/)
40
+
41
+ # enable_key_* format (legacy format)
22
42
  conf.keys.select{|k| k =~ /^enable_key_/}.each do |key|
23
- geoip_key_name = key.sub('enable_key_','')
24
- raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key_name}" unless GEOIP_KEYS.include?(geoip_key_name)
25
- @geoip_keys_map.store(geoip_key_name, conf[key].split(/\s*,\s*/))
43
+ geoip_key = key.sub('enable_key_','')
44
+ raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
45
+ @geoip_lookup_key.zip(conf[key].split(/\s*,\s*/)).each do |lookup_field,record_key|
46
+ if record_key.nil?
47
+ raise Fluent::ConfigError, "geoip: missing value found at '#{key} #{lookup_field}'"
48
+ end
49
+ @map.store(record_key, "${#{geoip_key}['#{lookup_field}']}")
50
+ end
51
+ end
52
+ if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
53
+ log.warn "geoip: 'enable_key_*' config format is obsoleted. use <record></record> directive for now."
54
+ log.warn "geoip: for further details referable to https://github.com/y-ken/fluent-plugin-geoip"
26
55
  end
27
56
 
28
- @geoip_lookup_key = @geoip_lookup_key.split(/\s*,\s*/).map {|lookupkey|
29
- lookupkey.split(".")
30
- }
31
- if @geoip_lookup_key.size > 1
32
- @geoip_keys_map.each{|name, key|
33
- if key.size != @geoip_lookup_key.size
34
- raise Fluent::ConfigError, "geoip: lookup key length is not match #{name}"
35
- end
57
+ # <record></record> directive
58
+ conf.elements.select { |element| element.name == 'record' }.each { |element|
59
+ element.each_pair { |k, v|
60
+ element.has_key?(k) # to suppress unread configuration warning
61
+ @map[k] = v
36
62
  }
63
+ }
64
+ @placeholder_keys = @map.values.join.scan(/(\$\{[^}]+\})/).map{ |placeholder| placeholder[0] }.uniq
65
+ @placeholder_keys.each do |key|
66
+ geoip_key = key.match(REGEXP_PLACEHOLDER)[:geoip_key]
67
+ raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
37
68
  end
69
+ @placeholder_expander = PlaceholderExpander.new
38
70
 
39
- if ( !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
40
- raise Fluent::ConfigError, "geoip: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
71
+ if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
72
+ raise Fluent::ConfigError, "geoip: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."
41
73
  end
42
74
 
43
75
  @geoip = GeoIP::City.new(@geoip_database, :memory, false)
@@ -61,27 +93,63 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
61
93
  end
62
94
  end
63
95
 
96
+
97
+ private
98
+ def add_geoip_field(record)
99
+ placeholder = create_placeholder(geolocate(get_address(record)))
100
+ @map.each do |record_key, value|
101
+ if value.match(REGEXP_PLACEHOLDER)
102
+ rewrited = placeholder[value]
103
+ else
104
+ rewrited = value.gsub(/\$\{[^\}]+?\}/, placeholder)
105
+ if rewrited.match(/(^[\[\{]|^[\d\.\-]+$)/)
106
+ rewrited = parse_json(rewrited)
107
+ end
108
+ end
109
+ record.store(record_key, rewrited)
110
+ end
111
+ return record
112
+ end
113
+
114
+ def parse_json(message)
115
+ begin
116
+ return Yajl::Parser.parse(message)
117
+ rescue Yajl::ParseError => e
118
+ log.info "geoip: failed to parse '#{message}' as json.", :error_class => e.class, :error => e.message
119
+ return nil
120
+ end
121
+ end
122
+
64
123
  def get_address(record)
65
- @geoip_lookup_key.map {|key|
124
+ address = {}
125
+ @geoip_lookup_key.each do |field|
126
+ key = field.split('.')
66
127
  obj = record
67
128
  key.each {|k|
68
129
  break obj = nil if not obj.has_key?(k)
69
130
  obj = obj[k]
70
131
  }
71
- obj
72
- }
132
+ address.store(field, obj)
133
+ end
134
+ return address
73
135
  end
74
136
 
75
- def add_geoip_field(record)
76
- addresses = get_address(record)
77
- return record if addresses.all? {|address| address == nil }
78
- results = addresses.map {|address| @geoip.look_up(address) }
79
- return record if results.all? {|result| result == nil }
80
- @geoip_keys_map.each do |geoip_key,record_keys|
81
- record_keys.each_with_index {|record_key, idx|
82
- record.store(record_key, results[idx][geoip_key.to_sym])
83
- }
137
+ def geolocate(addresses)
138
+ geodata = {}
139
+ addresses.each do |field, ip|
140
+ geo = ip.nil? ? nil : @geoip.look_up(ip)
141
+ geodata.store(field, geo)
84
142
  end
85
- return record
143
+ return geodata
144
+ end
145
+
146
+ def create_placeholder(geodata)
147
+ placeholder = {}
148
+ @placeholder_keys.each do |placeholder_key|
149
+ position = placeholder_key.match(REGEXP_PLACEHOLDER)
150
+ next if position.nil? or geodata[position[:record_key]].nil?
151
+ placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
152
+ end
153
+ return placeholder
86
154
  end
87
155
  end
@@ -29,7 +29,6 @@ class GeoipOutputTest < Test::Unit::TestCase
29
29
  add_tag_prefix geoip.
30
30
  ]
31
31
  assert_equal 'geoip_city', d.instance.config['enable_key_city']
32
- assert_equal ['geoip_city'], d.instance.geoip_keys_map['city']
33
32
 
34
33
  # multiple key config
35
34
  d = create_driver %[
@@ -39,7 +38,6 @@ class GeoipOutputTest < Test::Unit::TestCase
39
38
  add_tag_prefix geoip.
40
39
  ]
41
40
  assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
42
- assert_equal ['from_city', 'to_city'], d.instance.geoip_keys_map['city']
43
41
 
44
42
  # multiple key config (bad configure)
45
43
  assert_raise(Fluent::ConfigError) {
@@ -66,6 +64,24 @@ class GeoipOutputTest < Test::Unit::TestCase
66
64
  assert_equal nil, emits[1][2]['geoip_city']
67
65
  end
68
66
 
67
+ def test_emit_tag_option
68
+ d1 = create_driver(%[
69
+ geoip_lookup_key host
70
+ enable_key_city geoip_city
71
+ remove_tag_prefix input.
72
+ tag geoip.${tag}
73
+ ], 'input.access')
74
+ d1.run do
75
+ d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
76
+ d1.emit({'message' => 'missing field'})
77
+ end
78
+ emits = d1.emits
79
+ assert_equal 2, emits.length
80
+ assert_equal 'geoip.access', emits[0][0] # tag
81
+ assert_equal 'Mountain View', emits[0][2]['geoip_city']
82
+ assert_equal nil, emits[1][2]['geoip_city']
83
+ end
84
+
69
85
  def test_emit_nested_attr
70
86
  d1 = create_driver(%[
71
87
  geoip_lookup_key host.ip
@@ -127,6 +143,82 @@ class GeoipOutputTest < Test::Unit::TestCase
127
143
  remove_tag_prefix input.
128
144
  add_tag_prefix geoip.
129
145
  ], 'input.access')
146
+ d1.run do
147
+ d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
148
+ d1.emit({'from' => {'ip' => '66.102.3.80'}})
149
+ d1.emit({'message' => 'missing field'})
150
+ end
151
+ emits = d1.emits
152
+ assert_equal 3, emits.length
153
+ assert_equal 'geoip.access', emits[0][0] # tag
154
+ assert_equal 'Mountain View', emits[0][2]['from_city']
155
+ assert_equal 'United States', emits[0][2]['from_country']
156
+ assert_equal 'Musashino', emits[0][2]['to_city']
157
+ assert_equal 'Japan', emits[0][2]['to_country']
158
+
159
+ assert_equal 'Mountain View', emits[1][2]['from_city']
160
+ assert_equal 'United States', emits[1][2]['from_country']
161
+ assert_equal nil, emits[1][2]['to_city']
162
+ assert_equal nil, emits[1][2]['to_country']
163
+
164
+ assert_equal nil, emits[2][2]['from_city']
165
+ assert_equal nil, emits[2][2]['from_country']
166
+ assert_equal nil, emits[2][2]['to_city']
167
+ assert_equal nil, emits[2][2]['to_country']
168
+ end
169
+
170
+ def test_emit_record_directive
171
+ d1 = create_driver(%[
172
+ geoip_lookup_key from.ip
173
+ <record>
174
+ from_city ${city['from.ip']}
175
+ from_country ${country_name['from.ip']}
176
+ latitude ${latitude['from.ip']}
177
+ longitude ${longitude['from.ip']}
178
+ location_string ${latitude['from.ip']},${longitude['from.ip']}
179
+ location_array [${longitude['from.ip']},${latitude['from.ip']}]
180
+ location_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
181
+ </record>
182
+ remove_tag_prefix input.
183
+ tag geoip.${tag}
184
+ ], 'input.access')
185
+ d1.run do
186
+ d1.emit({'from' => {'ip' => '66.102.3.80'}})
187
+ d1.emit({'message' => 'missing field'})
188
+ end
189
+ emits = d1.emits
190
+ assert_equal 2, emits.length
191
+ assert_equal 'geoip.access', emits[0][0] # tag
192
+ assert_equal 'Mountain View', emits[0][2]['from_city']
193
+ assert_equal 'United States', emits[0][2]['from_country']
194
+ assert_equal 37.4192008972168, emits[0][2]['latitude']
195
+ assert_equal -122.05740356445312, emits[0][2]['longitude']
196
+ assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string']
197
+ assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array']
198
+ location_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
199
+ assert_equal location_nest, emits[0][2]['location_nest']
200
+ assert_equal nil, emits[0][2]['undefined']
201
+ assert_equal nil, emits[1][2]['from_city']
202
+ assert_equal nil, emits[1][2]['from_country']
203
+ assert_equal nil, emits[1][2]['latitude']
204
+ assert_equal nil, emits[1][2]['longitude']
205
+ assert_equal ',', emits[1][2]['location_string']
206
+ assert_equal nil, emits[1][2]['location_array']
207
+ assert_equal nil, emits[1][2]['location_nest']
208
+ end
209
+
210
+ def test_emit_record_directive_multiple_record
211
+ d1 = create_driver(%[
212
+ geoip_lookup_key from.ip, to.ip
213
+ <record>
214
+ from_city ${city['from.ip']}
215
+ to_city ${city['to.ip']}
216
+ from_country ${country_name['from.ip']}
217
+ to_country ${country_name['to.ip']}
218
+ </record>
219
+ remove_tag_prefix input.
220
+ tag geoip.${tag}
221
+ ], 'input.access')
130
222
  d1.run do
131
223
  d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
132
224
  d1.emit({'message' => 'missing field'})
@@ -140,6 +232,44 @@ class GeoipOutputTest < Test::Unit::TestCase
140
232
  assert_equal 'Japan', emits[0][2]['to_country']
141
233
  assert_equal nil, emits[1][2]['from_city']
142
234
  assert_equal nil, emits[1][2]['to_city']
235
+ assert_equal nil, emits[1][2]['from_country']
236
+ assert_equal nil, emits[1][2]['to_country']
143
237
  end
144
238
 
239
+ def test_emit_record_directive_aggressive
240
+ d1 = create_driver(%[
241
+ geoip_lookup_key from.ip
242
+ <record>
243
+ city ${city['from.ip']}
244
+ latitude ${latitude['from.ip']}
245
+ longitude ${longitude['from.ip']}
246
+ unknown_city ${city['unknown_key']}
247
+ undefined ${city['undefined']}
248
+ broken_array1 [${longitude['from.ip']},${latitude['undefined']}]
249
+ broken_array2 [${longitude['undefined']},${latitude['undefined']}]
250
+ </record>
251
+ remove_tag_prefix input.
252
+ tag geoip.${tag}
253
+ ], 'input.access')
254
+ d1.run do
255
+ d1.emit({'from' => {'ip' => '66.102.3.80'}})
256
+ d1.emit({'message' => 'missing field'})
257
+ end
258
+ emits = d1.emits
259
+ assert_equal 2, emits.length
260
+ assert_equal 'geoip.access', emits[0][0] # tag
261
+ assert_equal 'Mountain View', emits[0][2]['city']
262
+ assert_equal 37.4192008972168, emits[0][2]['latitude']
263
+ assert_equal -122.05740356445312, emits[0][2]['longitude']
264
+ assert_equal nil, emits[0][2]['unknown_city']
265
+ assert_equal nil, emits[0][2]['undefined']
266
+ assert_equal nil, emits[0][2]['broken_array1']
267
+ assert_equal nil, emits[0][2]['broken_array2']
268
+
269
+ assert_equal nil, emits[1][2]['city']
270
+ assert_equal nil, emits[1][2]['unknown_city']
271
+ assert_equal nil, emits[1][2]['undefined']
272
+ assert_equal nil, emits[1][2]['broken_array1']
273
+ assert_equal nil, emits[1][2]['broken_array2']
274
+ end
145
275
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-geoip
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.6
4
+ version: 0.1.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-02-05 00:00:00.000000000 Z
12
+ date: 2014-03-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -59,6 +59,22 @@ dependencies:
59
59
  - - ! '>='
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: fluent-mixin-rewrite-tag-name
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :runtime
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
62
78
  - !ruby/object:Gem::Dependency
63
79
  name: geoip-c
64
80
  requirement: !ruby/object:Gem::Requirement