fluent-plugin-geoip 0.3.1 → 0.4.0

data/README.md

@@ -56,7 +56,6 @@ $ sudo td-agent-gem install fluent-plugin-geoip
     latitude        ${latitude["host"]}
     longitude       ${longitude["host"]}
     country_code3   ${country_code3["host"]}
-    country_code2   ${country_code2["host"]}
     country         ${country_code["host"]}
     country_name    ${country_name["host"]}
     dma             ${dma_code["host"]}
@@ -189,7 +188,6 @@ Provides these placeholders for adding field of geolocate results.
 * ${latitude[lookup_field]}
 * ${longitude[lookup_field]}
 * ${country_code3[lookup_field]}
-* ${country_code2[lookup_field]}
 * ${country_code[lookup_field]}
 * ${country_name[lookup_field]}
 * ${dma_code[lookup_field]}

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.3.1"
+  spec.version       = "0.4.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}

data/lib/fluent/plugin/out_geoip.rb

@@ -4,8 +4,9 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('geoip', self)
 
   REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\[['"](?<record_key>-?[^'"]+)['"]\]\}$/
-  REGEXP_PLACEHOLDER_SCAN = /(\$\{[^\}]+?\})/
-  GEOIP_KEYS = %w(city latitude longitude country_code3 country_code2 country_code country_name dma_code area_code region)
+  REGEXP_PLACEHOLDER_SCAN = /['"]?(\$\{[^\}]+?\})['"]?/
+
+  GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
 
   config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
   config_param :geoip_lookup_key, :string, :default => 'host'
@@ -59,7 +60,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
     conf.elements.select { |element| element.name == 'record' }.each { |element|
       element.each_pair { |k, v|
         element.has_key?(k) # to suppress unread configuration warning
-        v = v[1..v.size-2] if quoted_json?(v)
+        v = v[1..v.size-2] if quoted_value?(v)
         @map[k] = v
         validate_json = Proc.new { 
           begin
@@ -107,21 +108,13 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   private
 
   def json?(text)
-    text.match(/^\[.+\]$/) || text.match(/^\{.+\}$/) || text.match(/^[\d\.\-]+$/)
+    text.match(/^\[.+\]$/) || text.match(/^\{.+\}$/)
   end
 
-  def quoted_json?(text)
+  def quoted_value?(text)
     # to improbe compatibility with fluentd v1-config
     trim_quote = text[1..text.size-2]
-    text.match(/(^'.+'$|^".+"$)/) && (json_array_block?(trim_quote) || json_hash_block?(trim_quote))
-  end
-
-  def json_array_block?(text)
-    text.match(/^\[[^\{]*?\{.+\}[^\}]*?\]$/)
-  end
-
-  def json_hash_block?(text)
-    text.match(/^\{[^\{]*?\{.+\}[^\}]*?\}$/)
+    text.match(/(^'.+'$|^".+"$)/)
   end
 
   def add_geoip_field(record)
@@ -131,6 +124,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
         rewrited = placeholder[value]
       elsif json?(value)
         rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
+          match = match[1..match.size-2] if quoted_value?(match)
           Yajl::Encoder.encode(placeholder[match])
         }
         rewrited = parse_json(rewrited)

data/test/plugin/test_out_geoip.rb

@@ -12,8 +12,8 @@ class GeoipOutputTest < Test::Unit::TestCase
     tag               geoip.${tag}
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::GeoipOutput, tag).configure(conf)
+  def create_driver(conf=CONFIG,tag='test',use_v1=false)
+    Fluent::Test::OutputTestDriver.new(Fluent::GeoipOutput, tag).configure(conf, use_v1)
   end
 
   def test_configure
@@ -327,35 +327,76 @@ class GeoipOutputTest < Test::Unit::TestCase
     assert_equal [nil, nil], emits[1][2]['string_array']
   end
 
+  CONFIG_QUOTED_RECORD = %[
+    geoip_lookup_key  host
+    <record>
+      location_properties  '{ "country_code" : "${country_code["host"]}", "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
+      location_string      ${latitude['host']},${longitude['host']}
+      location_string2     ${country_code["host"]}
+      location_array       "[${longitude['host']},${latitude['host']}]"
+      location_array2      '[${longitude["host"]},${latitude["host"]}]'
+      peculiar_pattern     '[GEOIP] message => {"lat":${latitude["host"]}, "lon":${longitude["host"]}}'
+    </record>
+    remove_tag_prefix input.
+    tag               geoip.${tag}
+  ]
+
+  def test_emit_quoted_record
+    d1 = create_driver(CONFIG_QUOTED_RECORD, 'input.access')
+    d1.run do
+      d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    location_properties = { "country_code" => "US", "lat" => 37.4192008972168, "lon"=> -122.05740356445312 }
+    assert_equal location_properties, emits[0][2]['location_properties']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string']
+    assert_equal 'US', emits[0][2]['location_string2']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array2']
+    assert_equal '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}', emits[0][2]['peculiar_pattern']
+  end
 
+  def test_emit_v1_config_compatibility
+    d1 = create_driver(CONFIG_QUOTED_RECORD, 'input.access', true)
+    d1.run do
+      d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'geoip.access', emits[0][0] # tag
+    location_properties = { "country_code" => "US", "lat" => 37.4192008972168, "lon"=> -122.05740356445312 }
+    assert_equal location_properties, emits[0][2]['location_properties']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string']
+    assert_equal 'US', emits[0][2]['location_string2']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array2']
+    assert_equal '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}', emits[0][2]['peculiar_pattern']
+  end
 
-  def test_emit_v1_parser
+  def test_emit_multiline_v1_config
     d1 = create_driver(%[
       geoip_lookup_key  host
       <record>
-        location_properties  '{ "lat": ${latitude["host"]}, "lon": ${longitude["host"]} }'
-        location_string      ${latitude['host']},${longitude['host']}
-        location_string2     ${latitude["host"]},${longitude["host"]}
-        location_array       "[${longitude['host']},${latitude['host']}]"
-        location_array2      '[${longitude["host"]},${latitude["host"]}]'
-        peculiar_pattern     [GEOIP] message => {"lat":${latitude['host']}, "lon":${longitude['host']}}
+        location_properties  {
+          "city": "${city['host']}",
+          "country_code": "${country_code['host']}",
+          "latitude": "${latitude['host']}",
+          "longitude": "${longitude['host']}"
+        }
       </record>
       remove_tag_prefix input.
       tag               geoip.${tag}
-    ], 'input.access')
+    ], 'input.access', true)
     d1.run do
       d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
     end
     emits = d1.emits
     assert_equal 1, emits.length
     assert_equal 'geoip.access', emits[0][0] # tag
-    location_properties = { "lat" => 37.4192008972168, "lon"=> -122.05740356445312 }
+    location_properties = { "city"=>"Mountain View", "country_code"=>"US", "latitude"=>37.4192008972168, "longitude"=>-122.05740356445312 }
     assert_equal location_properties, emits[0][2]['location_properties']
-    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string']
-    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string2']
-    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array']
-    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array2']
-    assert_equal '[GEOIP] message => {"lat":37.4192008972168, "lon":-122.05740356445312}', emits[0][2]['peculiar_pattern']
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-25 00:00:00.000000000 Z
+date: 2014-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler