fluent-plugin-geoip 0.1.0 → 0.1.1

data/README.md

@@ -164,15 +164,15 @@ http://dev.maxmind.com/geoip/legacy/csv/
 
 Provides these placeholders for adding field of geolocate results.
 
-* ${city}
-* ${latitude}
-* ${longitude}
-* ${country_code3}
-* ${country_code}
-* ${country_name}
-* ${dma_code}
-* ${area_code}
-* ${region}
+* ${city[lookup_field]}
+* ${latitude[lookup_field]}
+* ${longitude[lookup_field]}
+* ${country_code3[lookup_field]}
+* ${country_code[lookup_field]}
+* ${country_name[lookup_field]}
+* ${dma_code[lookup_field]}
+* ${area_code[lookup_field]}
+* ${region[lookup_field]}
 
 ## Parameters
 

data/fluent-plugin-geoip.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-geoip"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with Maxmind GeoIP databases.}

data/lib/fluent/plugin/out_geoip.rb

@@ -3,7 +3,9 @@ require 'fluent/mixin/rewrite_tag_name'
 class Fluent::GeoipOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('geoip', self)
 
-  REGEXP_PLACEHOLDER = /^\$\{(?<geoip_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
+  REGEXP_JSON = /(^[\[\{].+[\]\}]$|^[\d\.\-]+$)/
+  REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<geoip_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
+  REGEXP_PLACEHOLDER_SCAN = /(\$\{[^\}]+?\})/
   GEOIP_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
 
   config_param :geoip_database, :string, :default => File.dirname(__FILE__) + '/../../../data/GeoLiteCity.dat'
@@ -59,11 +61,20 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
       element.each_pair { |k, v|
         element.has_key?(k) # to suppress unread configuration warning
         @map[k] = v
+        validate_json = Proc.new { 
+          begin
+            dummy_text = '12345'
+            Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text)) if v.match(REGEXP_JSON)
+          rescue Yajl::ParseError => e
+            raise Fluent::ConfigError, "geoip: failed to parse '#{v}' as json."
+          end
+        }
+        validate_json.call
       }
     }
-    @placeholder_keys = @map.values.join.scan(/(\$\{[^}]+\})/).map{ |placeholder| placeholder[0] }.uniq
+    @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
     @placeholder_keys.each do |key|
-      geoip_key = key.match(REGEXP_PLACEHOLDER)[:geoip_key]
+      geoip_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:geoip_key]
       raise Fluent::ConfigError, "geoip: unsupported key #{geoip_key}" unless GEOIP_KEYS.include?(geoip_key)
     end
     @placeholder_expander = PlaceholderExpander.new
@@ -93,18 +104,19 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
     end
   end
 
-
   private
   def add_geoip_field(record)
     placeholder = create_placeholder(geolocate(get_address(record)))
     @map.each do |record_key, value|
-      if value.match(REGEXP_PLACEHOLDER)
+      if value.match(REGEXP_PLACEHOLDER_SINGLE)
         rewrited = placeholder[value]
+      elsif value.match(REGEXP_JSON)
+        rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
+          Yajl::Encoder.encode(placeholder[match])
+        }
+        rewrited = parse_json(rewrited)
       else
-        rewrited = value.gsub(/\$\{[^\}]+?\}/, placeholder)
-        if rewrited.match(/(^[\[\{]|^[\d\.\-]+$)/) 
-          rewrited = parse_json(rewrited)
-        end
+        rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
       end
       record.store(record_key, rewrited)
     end
@@ -146,7 +158,7 @@ class Fluent::GeoipOutput < Fluent::BufferedOutput
   def create_placeholder(geodata)
     placeholder = {}
     @placeholder_keys.each do |placeholder_key|
-      position = placeholder_key.match(REGEXP_PLACEHOLDER)
+      position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
       next if position.nil? or geodata[position[:record_key]].nil?
       placeholder.store(placeholder_key, geodata[position[:record_key]][position[:geoip_key].to_sym])
     end

data/test/plugin/test_out_geoip.rb

@@ -9,7 +9,7 @@ class GeoipOutputTest < Test::Unit::TestCase
     geoip_lookup_key  host
     enable_key_city   geoip_city
     remove_tag_prefix input.
-    add_tag_prefix    geoip.
+    tag               geoip.${tag}
   ]
 
   def create_driver(conf=CONFIG,tag='test')
@@ -24,9 +24,9 @@ class GeoipOutputTest < Test::Unit::TestCase
       d = create_driver('enable_key_cities')
     }
     d = create_driver %[
-      enable_key_city geoip_city
+      enable_key_city   geoip_city
       remove_tag_prefix input.
-      add_tag_prefix    geoip.
+      tag               geoip.${tag}
     ]
     assert_equal 'geoip_city', d.instance.config['enable_key_city']
 
@@ -35,7 +35,7 @@ class GeoipOutputTest < Test::Unit::TestCase
       geoip_lookup_key  from.ip, to.ip
       enable_key_city   from_city, to_city
       remove_tag_prefix input.
-      add_tag_prefix    geoip.
+      tag               geoip.${tag}
     ]
     assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
 
@@ -46,7 +46,29 @@ class GeoipOutputTest < Test::Unit::TestCase
         enable_key_city   from_city
         enable_key_region from_region
         remove_tag_prefix input.
-        add_tag_prefix    geoip.
+        tag               geoip.${tag}
+      ]
+    }
+
+    # invalid json structure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        geoip_lookup_key  host
+        <record>
+          invalid_json    {"foo" => 123}
+        </record>
+        remove_tag_prefix input.
+        tag               geoip.${tag}
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        geoip_lookup_key  host
+        <record>
+          invalid_json    {"foo" : string, "bar" : 123}
+        </record>
+        remove_tag_prefix input.
+        tag               geoip.${tag}
       ]
     }
   end
@@ -175,9 +197,16 @@ class GeoipOutputTest < Test::Unit::TestCase
         from_country    ${country_name['from.ip']}
         latitude        ${latitude['from.ip']}
         longitude       ${longitude['from.ip']}
-        location_string ${latitude['from.ip']},${longitude['from.ip']}
-        location_array  [${longitude['from.ip']},${latitude['from.ip']}]
-        location_nest   { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
+        float_concat    ${latitude['from.ip']},${longitude['from.ip']}
+        float_array     [${longitude['from.ip']}, ${latitude['from.ip']}]
+        float_nest      { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
+        string_concat   ${latitude['from.ip']},${longitude['from.ip']}
+        string_array    [${city['from.ip']}, ${country_name['from.ip']}]
+        string_nest     { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
+        unknown_city    ${city['unknown_key']}
+        undefined       ${city['undefined']}
+        broken_array1   [${longitude['from.ip']}, ${latitude['undefined']}]
+        broken_array2   [${longitude['undefined']}, ${latitude['undefined']}]
       </record>
       remove_tag_prefix input.
       tag               geoip.${tag}
@@ -188,23 +217,41 @@ class GeoipOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 2, emits.length
+
     assert_equal 'geoip.access', emits[0][0] # tag
     assert_equal 'Mountain View', emits[0][2]['from_city']
     assert_equal 'United States', emits[0][2]['from_country']
     assert_equal 37.4192008972168, emits[0][2]['latitude']
     assert_equal -122.05740356445312, emits[0][2]['longitude']
-    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['location_string']
-    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['location_array']
-    location_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
-    assert_equal location_nest, emits[0][2]['location_nest']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['float_concat']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['float_array']
+    float_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
+    assert_equal float_nest, emits[0][2]['float_nest']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['string_concat']
+    assert_equal ["Mountain View", "United States"], emits[0][2]['string_array']
+    string_nest = {"city" => "Mountain View", "country_name" => "United States"}
+    assert_equal string_nest, emits[0][2]['string_nest']
+    assert_equal nil, emits[0][2]['unknown_city']
     assert_equal nil, emits[0][2]['undefined']
+    assert_equal [-122.05740356445312, nil], emits[0][2]['broken_array1']
+    assert_equal [nil, nil], emits[0][2]['broken_array2']
+
     assert_equal nil, emits[1][2]['from_city']
     assert_equal nil, emits[1][2]['from_country']
     assert_equal nil, emits[1][2]['latitude']
     assert_equal nil, emits[1][2]['longitude']
-    assert_equal ',', emits[1][2]['location_string']
-    assert_equal nil, emits[1][2]['location_array']
-    assert_equal nil, emits[1][2]['location_nest']
+    assert_equal ',', emits[1][2]['float_concat']
+    assert_equal [nil, nil], emits[1][2]['float_array']
+    float_nest = {"lat" => nil, "lon" => nil}
+    assert_equal float_nest, emits[1][2]['float_nest']
+    assert_equal ',', emits[1][2]['string_concat']
+    assert_equal [nil, nil], emits[1][2]['string_array']
+    string_nest = {"city" => nil, "country_name" => nil}
+    assert_equal string_nest, emits[1][2]['string_nest']
+    assert_equal nil, emits[1][2]['unknown_city']
+    assert_equal nil, emits[1][2]['undefined']
+    assert_equal [nil, nil], emits[1][2]['broken_array1']
+    assert_equal [nil, nil], emits[1][2]['broken_array2']
   end
 
   def test_emit_record_directive_multiple_record
@@ -215,6 +262,7 @@ class GeoipOutputTest < Test::Unit::TestCase
         to_city         ${city['to.ip']}
         from_country    ${country_name['from.ip']}
         to_country      ${country_name['to.ip']}
+        string_array    [${country_name['from.ip']}, ${country_name['to.ip']}]
       </record>
       remove_tag_prefix input.
       tag               geoip.${tag}
@@ -225,51 +273,18 @@ class GeoipOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 2, emits.length
+
     assert_equal 'geoip.access', emits[0][0] # tag
     assert_equal 'Mountain View', emits[0][2]['from_city']
     assert_equal 'United States', emits[0][2]['from_country']
     assert_equal 'Musashino', emits[0][2]['to_city']
     assert_equal 'Japan', emits[0][2]['to_country']
+    assert_equal ['United States','Japan'], emits[0][2]['string_array']
+
     assert_equal nil, emits[1][2]['from_city']
     assert_equal nil, emits[1][2]['to_city']
     assert_equal nil, emits[1][2]['from_country']
     assert_equal nil, emits[1][2]['to_country']
-  end
-
-  def test_emit_record_directive_aggressive
-    d1 = create_driver(%[
-      geoip_lookup_key  from.ip
-      <record>
-        city            ${city['from.ip']}
-        latitude        ${latitude['from.ip']}
-        longitude       ${longitude['from.ip']}
-        unknown_city    ${city['unknown_key']}
-        undefined       ${city['undefined']}
-        broken_array1   [${longitude['from.ip']},${latitude['undefined']}]
-        broken_array2   [${longitude['undefined']},${latitude['undefined']}]
-      </record>
-      remove_tag_prefix input.
-      tag               geoip.${tag}
-    ], 'input.access')
-    d1.run do
-      d1.emit({'from' => {'ip' => '66.102.3.80'}})
-      d1.emit({'message' => 'missing field'})
-    end
-    emits = d1.emits
-    assert_equal 2, emits.length
-    assert_equal 'geoip.access', emits[0][0] # tag
-    assert_equal 'Mountain View', emits[0][2]['city']
-    assert_equal 37.4192008972168, emits[0][2]['latitude']
-    assert_equal -122.05740356445312, emits[0][2]['longitude']
-    assert_equal nil, emits[0][2]['unknown_city']
-    assert_equal nil, emits[0][2]['undefined']
-    assert_equal nil, emits[0][2]['broken_array1']
-    assert_equal nil, emits[0][2]['broken_array2']
-
-    assert_equal nil, emits[1][2]['city']
-    assert_equal nil, emits[1][2]['unknown_city']
-    assert_equal nil, emits[1][2]['undefined']
-    assert_equal nil, emits[1][2]['broken_array1']
-    assert_equal nil, emits[1][2]['broken_array2']
+    assert_equal [nil, nil], emits[1][2]['string_array']
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-geoip
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-28 00:00:00.000000000 Z
+date: 2014-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler