fluent-plugin-geoip 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -164,15 +164,15 @@ http://dev.maxmind.com/geoip/legacy/csv/
164
164
 
165
165
  Provides these placeholders for adding field of geolocate results.
166
166
 
167
- * ${city}
168
- * ${latitude}
169
- * ${longitude}
170
- * ${country_code3}
171
- * ${country_code}
172
- * ${country_name}
173
- * ${dma_code}
174
- * ${area_code}
175
- * ${region}
167
+ * ${city[lookup_field]}
168
+ * ${latitude[lookup_field]}
169
+ * ${longitude[lookup_field]}
170
+ * ${country_code3[lookup_field]}
171
+ * ${country_code[lookup_field]}
172
+ * ${country_name[lookup_field]}
173
+ * ${dma_code[lookup_field]}
174
+ * ${area_code[lookup_field]}
175
+ * ${region[lookup_field]}
176
176
 
177
177
  ## Parameters
178
178
 
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-geoip"
7
- spec.version = "0.1.0"
7
+ spec.version = "0.1.1"
8
8
  spec.authors = ["Kentaro Yoshida"]
9
9
  spec.email = ["y.ken.studio@gmail.com"]
10
10
  spec.summary = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}
@@ -3,7 +3,9 @@ require 'fluent/mixin/rewrite_tag_name'
3
3
  class Fluent::GeoipOutput < Fluent::BufferedOutput
4
4
  Fluent::Plugin.register_output('geoip', self)
5
5
 
6
- REGEXP_PLACEHOLDER = /^\$\{(?<geoip_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
6
+ REGEXP_JSON = /(^[\[\{].+[\]\}]$|^[\d\.\-]+$)/
7
+ REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
8
+ REGEXP_PLACEHOLDER_SCAN = /(\$\{[^\}]+?\})/
7
9
  GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
8
10
 
9
11
  config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
@@ -59,11 +61,20 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
59
61
  element.each_pair { |k, v|
60
62
  element.has_key?(k) # to suppress unread configuration warning
61
63
  @map[k] = v
64
+ validate_json = Proc.new {
65
+ begin
66
+ dummy_text = '12345'
67
+ Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text)) if v.match(REGEXP_JSON)
68
+ rescue Yajl::ParseError => e
69
+ raise Fluent::ConfigError, "geoip: failed to parse '#{v}' as json."
70
+ end
71
+ }
72
+ validate_json.call
62
73
  }
63
74
  }
64
- @placeholder_keys = @map.values.join.scan(/(\$\{[^}]+\})/).map{ |placeholder| placeholder[0] }.uniq
75
+ @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
65
76
  @placeholder_keys.each do |key|
66
- geoip_key = key.match(REGEXP_PLACEHOLDER)[:geoip_key]
77
+ geoip_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:geoip_key]
67
78
  raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
68
79
  end
69
80
  @placeholder_expander = PlaceholderExpander.new
@@ -93,18 +104,19 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
93
104
  end
94
105
  end
95
106
 
96
-
97
107
  private
98
108
  def add_geoip_field(record)
99
109
  placeholder = create_placeholder(geolocate(get_address(record)))
100
110
  @map.each do |record_key, value|
101
- if value.match(REGEXP_PLACEHOLDER)
111
+ if value.match(REGEXP_PLACEHOLDER_SINGLE)
102
112
  rewrited = placeholder[value]
113
+ elsif value.match(REGEXP_JSON)
114
+ rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
115
+ Yajl::Encoder.encode(placeholder[match])
116
+ }
117
+ rewrited = parse_json(rewrited)
103
118
  else
104
- rewrited = value.gsub(/\$\{[^\}]+?\}/, placeholder)
105
- if rewrited.match(/(^[\[\{]|^[\d\.\-]+$)/)
106
- rewrited = parse_json(rewrited)
107
- end
119
+ rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
108
120
  end
109
121
  record.store(record_key, rewrited)
110
122
  end
@@ -146,7 +158,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
146
158
  def create_placeholder(geodata)
147
159
  placeholder = {}
148
160
  @placeholder_keys.each do |placeholder_key|
149
- position = placeholder_key.match(REGEXP_PLACEHOLDER)
161
+ position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
150
162
  next if position.nil? or geodata[position[:record_key]].nil?
151
163
  placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
152
164
  end
@@ -9,7 +9,7 @@ class GeoipOutputTest < Test::Unit::TestCase
9
9
  geoip_lookup_key host
10
10
  enable_key_city geoip_city
11
11
  remove_tag_prefix input.
12
- add_tag_prefix geoip.
12
+ tag geoip.${tag}
13
13
  ]
14
14
 
15
15
  def create_driver(conf=CONFIG,tag='test')
@@ -24,9 +24,9 @@ class GeoipOutputTest < Test::Unit::TestCase
24
24
  d = create_driver('enable_key_cities')
25
25
  }
26
26
  d = create_driver %[
27
- enable_key_city geoip_city
27
+ enable_key_city geoip_city
28
28
  remove_tag_prefix input.
29
- add_tag_prefix geoip.
29
+ tag geoip.${tag}
30
30
  ]
31
31
  assert_equal 'geoip_city', d.instance.config['enable_key_city']
32
32
 
@@ -35,7 +35,7 @@ class GeoipOutputTest < Test::Unit::TestCase
35
35
  geoip_lookup_key from.ip, to.ip
36
36
  enable_key_city from_city, to_city
37
37
  remove_tag_prefix input.
38
- add_tag_prefix geoip.
38
+ tag geoip.${tag}
39
39
  ]
40
40
  assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
41
41
 
@@ -46,7 +46,29 @@ class GeoipOutputTest < Test::Unit::TestCase
46
46
  enable_key_city from_city
47
47
  enable_key_region from_region
48
48
  remove_tag_prefix input.
49
- add_tag_prefix geoip.
49
+ tag geoip.${tag}
50
+ ]
51
+ }
52
+
53
+ # invalid json structure
54
+ assert_raise(Fluent::ConfigError) {
55
+ d = create_driver %[
56
+ geoip_lookup_key host
57
+ <record>
58
+ invalid_json {"foo" => 123}
59
+ </record>
60
+ remove_tag_prefix input.
61
+ tag geoip.${tag}
62
+ ]
63
+ }
64
+ assert_raise(Fluent::ConfigError) {
65
+ d = create_driver %[
66
+ geoip_lookup_key host
67
+ <record>
68
+ invalid_json {"foo" : string, "bar" : 123}
69
+ </record>
70
+ remove_tag_prefix input.
71
+ tag geoip.${tag}
50
72
  ]
51
73
  }
52
74
  end
@@ -175,9 +197,16 @@ class GeoipOutputTest < Test::Unit::TestCase
175
197
  from_country ${country_name['from.ip']}
176
198
  latitude ${latitude['from.ip']}
177
199
  longitude ${longitude['from.ip']}
178
- location_string ${latitude['from.ip']},${longitude['from.ip']}
179
- location_array [${longitude['from.ip']},${latitude['from.ip']}]
180
- location_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
200
+ float_concat ${latitude['from.ip']},${longitude['from.ip']}
201
+ float_array [${longitude['from.ip']}, ${latitude['from.ip']}]
202
+ float_nest { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
203
+ string_concat ${latitude['from.ip']},${longitude['from.ip']}
204
+ string_array [${city['from.ip']}, ${country_name['from.ip']}]
205
+ string_nest { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
206
+ unknown_city ${city['unknown_key']}
207
+ undefined ${city['undefined']}
208
+ broken_array1 [${longitude['from.ip']}, ${latitude['undefined']}]
209
+ broken_array2 [${longitude['undefined']}, ${latitude['undefined']}]
181
210
  </record>
182
211
  remove_tag_prefix input.
183
212
  tag geoip.${tag}
@@ -188,23 +217,41 @@ class GeoipOutputTest < Test::Unit::TestCase
188
217
  end
189
218
  emits = d1.emits
190
219
  assert_equal 2, emits.length
220
+
191
221
  assert_equal 'geoip.access', emits[0][0] # tag
192
222
  assert_equal 'Mountain View', emits[0][2]['from_city']
193
223
  assert_equal 'United States', emits[0][2]['from_country']
194
224
  assert_equal 37.4192008972168, emits[0][2]['latitude']
195
225
  assert_equal -122.05740356445312, emits[0][2]['longitude']
196
- assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string']
197
- assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array']
198
- location_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
199
- assert_equal location_nest, emits[0][2]['location_nest']
226
+ assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['float_concat']
227
+ assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['float_array']
228
+ float_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
229
+ assert_equal float_nest, emits[0][2]['float_nest']
230
+ assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['string_concat']
231
+ assert_equal ["Mountain View", "United States"], emits[0][2]['string_array']
232
+ string_nest = {"city" => "Mountain View", "country_name" => "United States"}
233
+ assert_equal string_nest, emits[0][2]['string_nest']
234
+ assert_equal nil, emits[0][2]['unknown_city']
200
235
  assert_equal nil, emits[0][2]['undefined']
236
+ assert_equal [-122.05740356445312, nil], emits[0][2]['broken_array1']
237
+ assert_equal [nil, nil], emits[0][2]['broken_array2']
238
+
201
239
  assert_equal nil, emits[1][2]['from_city']
202
240
  assert_equal nil, emits[1][2]['from_country']
203
241
  assert_equal nil, emits[1][2]['latitude']
204
242
  assert_equal nil, emits[1][2]['longitude']
205
- assert_equal ',', emits[1][2]['location_string']
206
- assert_equal nil, emits[1][2]['location_array']
207
- assert_equal nil, emits[1][2]['location_nest']
243
+ assert_equal ',', emits[1][2]['float_concat']
244
+ assert_equal [nil, nil], emits[1][2]['float_array']
245
+ float_nest = {"lat" => nil, "lon" => nil}
246
+ assert_equal float_nest, emits[1][2]['float_nest']
247
+ assert_equal ',', emits[1][2]['string_concat']
248
+ assert_equal [nil, nil], emits[1][2]['string_array']
249
+ string_nest = {"city" => nil, "country_name" => nil}
250
+ assert_equal string_nest, emits[1][2]['string_nest']
251
+ assert_equal nil, emits[1][2]['unknown_city']
252
+ assert_equal nil, emits[1][2]['undefined']
253
+ assert_equal [nil, nil], emits[1][2]['broken_array1']
254
+ assert_equal [nil, nil], emits[1][2]['broken_array2']
208
255
  end
209
256
 
210
257
  def test_emit_record_directive_multiple_record
@@ -215,6 +262,7 @@ class GeoipOutputTest < Test::Unit::TestCase
215
262
  to_city ${city['to.ip']}
216
263
  from_country ${country_name['from.ip']}
217
264
  to_country ${country_name['to.ip']}
265
+ string_array [${country_name['from.ip']}, ${country_name['to.ip']}]
218
266
  </record>
219
267
  remove_tag_prefix input.
220
268
  tag geoip.${tag}
@@ -225,51 +273,18 @@ class GeoipOutputTest < Test::Unit::TestCase
225
273
  end
226
274
  emits = d1.emits
227
275
  assert_equal 2, emits.length
276
+
228
277
  assert_equal 'geoip.access', emits[0][0] # tag
229
278
  assert_equal 'Mountain View', emits[0][2]['from_city']
230
279
  assert_equal 'United States', emits[0][2]['from_country']
231
280
  assert_equal 'Musashino', emits[0][2]['to_city']
232
281
  assert_equal 'Japan', emits[0][2]['to_country']
282
+ assert_equal ['United States','Japan'], emits[0][2]['string_array']
283
+
233
284
  assert_equal nil, emits[1][2]['from_city']
234
285
  assert_equal nil, emits[1][2]['to_city']
235
286
  assert_equal nil, emits[1][2]['from_country']
236
287
  assert_equal nil, emits[1][2]['to_country']
237
- end
238
-
239
- def test_emit_record_directive_aggressive
240
- d1 = create_driver(%[
241
- geoip_lookup_key from.ip
242
- <record>
243
- city ${city['from.ip']}
244
- latitude ${latitude['from.ip']}
245
- longitude ${longitude['from.ip']}
246
- unknown_city ${city['unknown_key']}
247
- undefined ${city['undefined']}
248
- broken_array1 [${longitude['from.ip']},${latitude['undefined']}]
249
- broken_array2 [${longitude['undefined']},${latitude['undefined']}]
250
- </record>
251
- remove_tag_prefix input.
252
- tag geoip.${tag}
253
- ], 'input.access')
254
- d1.run do
255
- d1.emit({'from' => {'ip' => '66.102.3.80'}})
256
- d1.emit({'message' => 'missing field'})
257
- end
258
- emits = d1.emits
259
- assert_equal 2, emits.length
260
- assert_equal 'geoip.access', emits[0][0] # tag
261
- assert_equal 'Mountain View', emits[0][2]['city']
262
- assert_equal 37.4192008972168, emits[0][2]['latitude']
263
- assert_equal -122.05740356445312, emits[0][2]['longitude']
264
- assert_equal nil, emits[0][2]['unknown_city']
265
- assert_equal nil, emits[0][2]['undefined']
266
- assert_equal nil, emits[0][2]['broken_array1']
267
- assert_equal nil, emits[0][2]['broken_array2']
268
-
269
- assert_equal nil, emits[1][2]['city']
270
- assert_equal nil, emits[1][2]['unknown_city']
271
- assert_equal nil, emits[1][2]['undefined']
272
- assert_equal nil, emits[1][2]['broken_array1']
273
- assert_equal nil, emits[1][2]['broken_array2']
288
+ assert_equal [nil, nil], emits[1][2]['string_array']
274
289
  end
275
290
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-geoip
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-03-28 00:00:00.000000000 Z
12
+ date: 2014-03-31 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler