fluent-plugin-geoip 0.3.0 → 0.3.1

data/README.md

@@ -24,7 +24,7 @@ $ sudo apt-get install libgeoip-dev
 
 ## Installation
 
-install with `gem` or `fluent-gem` command as:
+install with `gem` or td-agent provided command as:
 
 ```bash
 # for fluentd
@@ -32,6 +32,9 @@ $ gem install fluent-plugin-geoip
 
 # for td-agent
 $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
+
+# for td-agent2
+$ sudo td-agent-gem install fluent-plugin-geoip
 ```
 
 ## Usage
@@ -45,20 +48,20 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
   geoip_lookup_key  host
 
   # Specify optional geoip database (using bundled GeoLiteCity databse by default)
-  geoip_database    '/path/to/your/GeoIPCity.dat'
+  geoip_database    "/path/to/your/GeoIPCity.dat"
 
   # Set adding field with placeholder (more than one settings are required.)
   <record>
-    city            ${city['host']}
-    latitude        ${latitude['host']}
-    longitude       ${longitude['host']}
-    country_code3   ${country_code3['host']}
-    country_code2   ${country_code2['host']}
-    country         ${country_code['host']}
-    country_name    ${country_name['host']}
-    dma             ${dma_code['host']}
-    area            ${area_code['host']}
-    region          ${region['host']}
+    city            ${city["host"]}
+    latitude        ${latitude["host"]}
+    longitude       ${longitude["host"]}
+    country_code3   ${country_code3["host"]}
+    country_code2   ${country_code2["host"]}
+    country         ${country_code["host"]}
+    country_name    ${country_name["host"]}
+    dma             ${dma_code["host"]}
+    area            ${area_code["host"]}
+    region          ${region["host"]}
   </record>
 
   # Settings for tag
@@ -80,8 +83,8 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
   type geoip
   geoip_lookup_key  user1_host, user2_host
   <record>
-    user1_city      ${city['user1_host']}
-    user2_city      ${city['user2_host']}
+    user1_city      ${city["user1_host"]}
+    user2_city      ${city["user2_host"]}
   </record>
   remove_tag_prefix access.
   tag               geoip.${tag}
@@ -90,24 +93,41 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-geoip
 
 #### Advanced config samples
 
-It is a sample to get friendly geo point recdords for elasticsearch with Yajl (JSON) parser.
+It is a sample to get friendly geo point recdords for elasticsearch with Yajl (JSON) parser.<br />
+
 
 ```
-<match input.access>
+<match access.apache>
   type                   geoip
   geoip_lookup_key       host
   <record>
     # lat lon as properties
     # ex. {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
-    location_properties  { "lat":${latitude['host']}, "lon":${longitude['host']}}
+    location_properties  { "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }
   
     # lat lon as string
     # ex. "37.4192008972168,-122.05740356445312"
-    location_string      ${latitude['host']},${longitude['host']}
+    location_string      ${latitude["host"]},${longitude["host"]}
     
     # lat lon as array (it is useful for Kibana's bettermap.)
     # ex. [-122.05740356445312, 37.4192008972168]
-    location_array       [${longitude['host']},${latitude['host']}]
+    location_array       [${longitude["host"]},${latitude["host"]}]
+  </record>
+  remove_tag_prefix      access.
+  tag                    geoip.${tag}
+</match>
+```
+
+On the case of using td-agent2 (v1-config), it have to quote `{ ... }` or `[ ... ]` block with quotation like below.
+
+```
+<match access.apache>
+  type                   geoip
+  geoip_lookup_key       host
+  <record>
+    location_properties  '{ "lat" : ${latitude["host"]}, "lon" : ${longitude["host"]} }'
+    location_string      ${latitude["host"]},${longitude["host"]}
+    location_array       '[${longitude["host"]},${latitude["host"]}]'
   </record>
   remove_tag_prefix      access.
   tag                    geoip.${tag}
@@ -132,9 +152,9 @@ It is a sample to get friendly geo point recdords for elasticsearch with Yajl (J
     type    geoip
     geoip_lookup_key  host
     <record>
-      city  ${city['host']}
-      lat   ${latitude['host']}
-      lon   ${longitude['host']}
+      city  ${city["host"]}
+      lat   ${latitude["host"]}
+      lon   ${longitude["host"]}
     </record>
     remove_tag_prefix test.
     tag     debug.${tag}

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.3.0"
+  spec.version       = "0.3.1"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}

data/lib/fluent/plugin/out_geoip.rb

@@ -3,8 +3,7 @@ require 'fluent/mixin/rewrite_tag_name'
 class Fluent::GeoipOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('geoip', self)
 
-  REGEXP_JSON = /(^[\[\{].+[\]\}]$|^[\d\.\-]+$)/
-  REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
+  REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\[['"](?<record_key>-?[^'"]+)['"]\]\}$/
   REGEXP_PLACEHOLDER_SCAN = /(\$\{[^\}]+?\})/
   GEOIP_KEYS = %w(city latitude longitude country_code3 country_code2 country_code country_name dma_code area_code region)
 
@@ -60,6 +59,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
     conf.elements.select { |element| element.name == 'record' }.each { |element|
       element.each_pair { |k, v|
         element.has_key?(k) # to suppress unread configuration warning
+        v = v[1..v.size-2] if quoted_json?(v)
         @map[k] = v
         validate_json = Proc.new { 
           begin
@@ -69,7 +69,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
             raise Fluent::ConfigError, "geoip: failed to parse '#{v}' as json."
           end
         }
-        validate_json.call if v.match(REGEXP_JSON)
+        validate_json.call if json?(v.tr('\'"\\', ''))
       }
     }
     @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
@@ -105,12 +105,31 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   end
 
   private
+
+  def json?(text)
+    text.match(/^\[.+\]$/) || text.match(/^\{.+\}$/) || text.match(/^[\d\.\-]+$/)
+  end
+
+  def quoted_json?(text)
+    # to improbe compatibility with fluentd v1-config
+    trim_quote = text[1..text.size-2]
+    text.match(/(^'.+'$|^".+"$)/) && (json_array_block?(trim_quote) || json_hash_block?(trim_quote))
+  end
+
+  def json_array_block?(text)
+    text.match(/^\[[^\{]*?\{.+\}[^\}]*?\]$/)
+  end
+
+  def json_hash_block?(text)
+    text.match(/^\{[^\{]*?\{.+\}[^\}]*?\}$/)
+  end
+
   def add_geoip_field(record)
     placeholder = create_placeholder(geolocate(get_address(record)))
     @map.each do |record_key, value|
       if value.match(REGEXP_PLACEHOLDER_SINGLE)
         rewrited = placeholder[value]
-      elsif value.match(REGEXP_JSON)
+      elsif json?(value)
         rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
           Yajl::Encoder.encode(placeholder[match])
         }

data/test/plugin/test_out_geoip.rb

@@ -326,4 +326,36 @@ class GeoipOutputTest < Test::Unit::TestCase
     assert_equal nil, emits[1][2]['to_country']
     assert_equal [nil, nil], emits[1][2]['string_array']
   end
+
+
+
+  def test_emit_v1_parser
+    d1 = create_driver(%[
+      geoip_lookup_key  host
+      <record>
+        location_properties  '{ "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
+        location_string      ${latitude['host']},${longitude['host']}
+        location_string2     ${latitude["host"]},${longitude["host"]}
+        location_array       "[${longitude['host']},${latitude['host']}]"
+        location_array2      '[${longitude["host"]},${latitude["host"]}]'
+        peculiar_pattern     [GEOIP] message => {"lat":${latitude['host']}, "lon":${longitude['host']}}
+      </record>
+      remove_tag_prefix input.
+      tag               geoip.${tag}
+    ], 'input.access')
+    d1.run do
+      d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    location_properties = { "lat" => 37.4192008972168, "lon"=> -122.05740356445312 }
+    assert_equal location_properties, emits[0][2]['location_properties']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string2']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array2']
+    assert_equal '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}', emits[0][2]['peculiar_pattern']
+  end
 end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-20 00:00:00.000000000 Z
+date: 2014-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler