fluent-plugin-qqwry 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 13b7b4e4a05362dd611e2ca9bd8d9cf1fb8fa5b5
+  data.tar.gz: 811fcc85eaa7dae22545904f30f4687a2c11a152
+SHA512:
+  metadata.gz: 054d4542cb901f72524851855e6229cd52311580ab4cd5a9b75e88e1448ab4b11989f086239d6582061ef787d096179b0bce034e9b88f796d69dc1dea6de46e7
+  data.tar.gz: 7ca2d977db399a85c87c1531a74d62a4fe23e42e6f49c67cc046c63f6bacf5dbcca58ece36d834db44954c0186967034cf2df13a70ed3ccf6c7bdd7e2c5ba61c

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.swp
+vendor/
+Gemfile.lock
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-geoip.gemspec
+gemspec
+
+gem "qqwry"

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2013- Kentaro Yoshida
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/README.md

@@ -0,0 +1,150 @@
+# fluent-plugin-qqwry
+
+Fluentd Output plugin to add information about geographical location of IP addresses with QQWry databases.
+
+fluent-plugin-geoip has bundled qqwry.dat
+
+## Dependency
+
+before use, install dependent library as:
+
+## Installation
+
+install with `gem` or `fluent-gem` command as:
+
+```bash
+# for fluentd
+$ gem install fluent-plugin-qqwry
+
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-qqwry
+```
+
+## Usage
+
+```xml
+<match access.apache>
+  type qqwry
+
+  # Specify one or more qqwry lookup field which has ip address (default: host)
+  # in the case of accessing nested value, delimit keys by dot like 'host.ip'.
+  qqwry_lookup_key  host
+
+  # Specify optional qqwry database (using bundled GeoLiteCity databse by default)
+  qqwry_database    '/path/to/your/qqwry.dat'
+
+  # Set adding field with placeholder (more than one settings are required.)
+  <record>
+    city            ${city['host']}
+    area            ${area_code['host']}
+    country         ${country['host']}
+    province        ${province['host']}
+  </record>
+
+  # Settings for tag
+  remove_tag_prefix access.
+  tag               qqwry.${tag}
+
+  # Set log_level for fluentd-v0.10.43 or earlier (default: warn)
+  log_level         info
+
+  # Set buffering time (default: 0s)
+  flush_interval    1s
+</match>
+```
+
+#### Tips: how to geolocate multiple key
+
+```xml
+<match access.apache>
+  type qqwry
+  qqwry_lookup_key  user1_host, user2_host
+  <record>
+    user1_city      ${city['user1_host']}
+    user2_city      ${city['user2_host']}
+  </record>
+  remove_tag_prefix access.
+  tag               qqwry.${tag}
+</match>
+```
+
+## Tutorial
+
+#### configuration
+
+```xml
+<source>
+  type forward
+</source>
+
+<match test.qqwry>
+  type copy
+  <store>
+    type stdout
+  </store>
+  <store>
+    type    qqwry
+    qqwry_lookup_key  host
+    <record>
+      city  ${city['host']}
+      lat   ${latitude['host']}
+      lon   ${longitude['host']}
+    </record>
+    remove_tag_prefix test.
+    tag     debug.${tag}
+  </store>
+</match>
+
+<match debug.**>
+  type stdout
+</match>
+```
+
+#### result
+
+```bash
+# forward record with Google's ip address.
+$ echo '{"host":"66.102.9.80","message":"test"}' | fluent-cat test.qqwry
+
+# check the result at stdout
+$ tail /var/log/td-agent/td-agent.log
+2013-08-04 16:21:32 +0900 test.qqwry: {"host":"66.102.9.80","message":"test"}
+2013-08-04 16:21:32 +0900 debug.qqwry: {"host":"66.102.9.80","message":"test","city":"Mountain View","lat":37.4192008972168,"lon":-122.05740356445312}
+```
+
+## Parameters
+
+* `include_tag_key` (default: false)
+* `tag_key`
+
+Add original tag name into filtered record using SetTagKeyMixin.<br />
+Further details are written at http://docs.fluentd.org/articles/in_exec
+
+* `remove_tag_prefix`
+* `remove_tag_suffix`
+* `add_tag_prefix`
+* `add_tag_suffix`
+
+Set one or more option are required unless using `tag` option for editing tag name. (HandleTagNameMixin feature)
+
+* `tag`
+
+On using this option with tag placeholder like `tag qqwry.${tag}` (test code is available at [test_out_qqwry.rb](https://github.com/y-ken/fluent-plugin-geoip/blob/master/test/plugin/test_out_geoip.rb)), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
+
+* `flush_interval` (default: 0 sec)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Copyright
+
+Copyright (c) 2014- Chris Song ([@fakechris](http://weibo.com/songchris))
+
+## License
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-qqwry.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-qqwry"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Chris Song"]
+  spec.email         = ["fakechris@gmail.com"]
+  spec.summary       = %q{Fluentd Output plugin to add information about geographical location of IP addresses with QQWry databases.}
+  spec.homepage      = "https://github.com/fakechris/fluent-plugin-qqwry"
+  spec.license       = "Apache License, Version 2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
+  spec.add_runtime_dependency "qqwry"
+end

data/lib/fluent/plugin/out_qqwry.rb

@@ -0,0 +1,166 @@
+require 'fluent/mixin/rewrite_tag_name'
+
+class Fluent::QQWryOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('qqwry', self)
+
+  REGEXP_JSON = /(^[\[\{].+[\]\}]$|^[\d\.\-]+$)/
+  REGEXP_PLACEHOLDER_SINGLE = /^\$\{(?<qqwry_key>-?[^\[]+)\['(?<record_key>-?[^']+)'\]\}$/
+  REGEXP_PLACEHOLDER_SCAN = /(\$\{[^\}]+?\})/
+  QQWRY_KEYS = %w(city latitude longitude country_code3 country_code country_name dma_code area_code region)
+
+  config_param :qqwry_database, :string, :default => File.dirname(__FILE__) + '/../../../data/qqwry.dat'
+  config_param :qqwry_lookup_key, :string, :default => 'host'
+  config_param :tag, :string, :default => nil
+
+  include Fluent::HandleTagNameMixin
+  include Fluent::SetTagKeyMixin
+  config_set_default :include_tag_key, false
+
+  include Fluent::Mixin::RewriteTagName
+  config_param :hostname_command, :string, :default => 'hostname'
+
+  config_param :flush_interval, :time, :default => 0
+  config_param :log_level, :string, :default => 'warn'
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def initialize
+    require 'qqwry'
+
+    super
+  end
+
+  def configure(conf)
+    super
+
+    @map = {}
+    @qqwry_lookup_key = @qqwry_lookup_key.split(/\s*,\s*/)
+
+    # enable_key_* format (legacy format)
+    conf.keys.select{|k| k =~ /^enable_key_/}.each do |key|
+      qqwry_key = key.sub('enable_key_','')
+      raise Fluent::ConfigError, "qqwry: unsupported key #{qqwry_key}" unless QQWRY_KEYS.include?(qqwry_key)
+      @qqwry_lookup_key.zip(conf[key].split(/\s*,\s*/)).each do |lookup_field,record_key|
+        if record_key.nil?
+          raise Fluent::ConfigError, "qqwry: missing value found at '#{key} #{lookup_field}'"
+        end
+        @map.store(record_key, "${#{qqwry_key}['#{lookup_field}']}")
+      end
+    end
+    if conf.keys.select{|k| k =~ /^enable_key_/}.size > 0
+      log.warn "qqwry: 'enable_key_*' config format is obsoleted. use <record></record> directive for now."
+      log.warn "qqwry: for further details referable to https://github.com/fakechris/fluent-plugin-qqwry"
+    end
+
+    # <record></record> directive
+    conf.elements.select { |element| element.name == 'record' }.each { |element|
+      element.each_pair { |k, v|
+        element.has_key?(k) # to suppress unread configuration warning
+        @map[k] = v
+        validate_json = Proc.new {
+          begin
+            dummy_text = Yajl::Encoder.encode('dummy_text')
+            Yajl::Parser.parse(v.gsub(REGEXP_PLACEHOLDER_SCAN, dummy_text))
+          rescue Yajl::ParseError => e
+            raise Fluent::ConfigError, "qqwry: failed to parse '#{v}' as json."
+          end
+        }
+        validate_json.call if v.match(REGEXP_JSON)
+      }
+    }
+    @placeholder_keys = @map.values.join.scan(REGEXP_PLACEHOLDER_SCAN).map{ |placeholder| placeholder[0] }.uniq
+    @placeholder_keys.each do |key|
+      qqwry_key = key.match(REGEXP_PLACEHOLDER_SINGLE)[:qqwry_key]
+      raise Fluent::ConfigError, "qqwry: unsupported key #{qqwry_key}" unless QQWRY_KEYS.include?(qqwry_key)
+    end
+    @placeholder_expander = PlaceholderExpander.new
+
+    if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
+      raise Fluent::ConfigError, "qqwry: required at least one option of 'tag', 'remove_tag_prefix', 'remove_tag_suffix', 'add_tag_prefix', 'add_tag_suffix'."
+    end
+
+    @qqwry = QQWry::Database.new(@qqwry_database)
+  end
+
+  def start
+    super
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def shutdown
+    super
+  end
+
+  def write(chunk)
+    chunk.msgpack_each do |tag, time, record|
+      Fluent::Engine.emit(tag, time, add_qqwry_field(record))
+    end
+  end
+
+  private
+  def add_qqwry_field(record)
+    placeholder = create_placeholder(geolocate(get_address(record)))
+    @map.each do |record_key, value|
+      if value.match(REGEXP_PLACEHOLDER_SINGLE)
+        rewrited = placeholder[value]
+      elsif value.match(REGEXP_JSON)
+        rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN) {|match|
+          Yajl::Encoder.encode(placeholder[match])
+        }
+        rewrited = parse_json(rewrited)
+      else
+        rewrited = value.gsub(REGEXP_PLACEHOLDER_SCAN, placeholder)
+      end
+      record.store(record_key, rewrited)
+    end
+    return record
+  end
+
+  def parse_json(message)
+    begin
+      return Yajl::Parser.parse(message)
+    rescue Yajl::ParseError => e
+      log.info "qqwry: failed to parse '#{message}' as json.", :error_class => e.class, :error => e.message
+      return nil
+    end
+  end
+
+  def get_address(record)
+    address = {}
+    @qqwry_lookup_key.each do |field|
+      key = field.split('.')
+      obj = record
+      key.each {|k|
+        break obj = nil if not obj.has_key?(k)
+        obj = obj[k]
+      }
+      address.store(field, obj)
+    end
+    return address
+  end
+
+  def geolocate(addresses)
+    geodata = {}
+    addresses.each do |field, ip|
+      geo = ip.nil? ? nil : @qqwry.query(ip)
+      geodata.store(field, geo)
+    end
+    return geodata
+  end
+
+  def create_placeholder(geodata)
+    placeholder = {}
+    @placeholder_keys.each do |placeholder_key|
+      position = placeholder_key.match(REGEXP_PLACEHOLDER_SINGLE)
+      next if position.nil? or geodata[position[:record_key]].nil?
+      placeholder.store(placeholder_key, geodata[position[:record_key]][position[:qqwry_key].to_sym])
+    end
+    return placeholder
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_geoip'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_qqwry.rb

@@ -0,0 +1,309 @@
+require 'helper'
+
+class QQWryOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    qqwry_lookup_key  host
+    enable_key_city   qqwry_city
+    remove_tag_prefix input.
+    tag               qqwry.${tag}
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::QQWryOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('enable_key_cities')
+    }
+    d = create_driver %[
+      enable_key_city   qqwry_city
+      remove_tag_prefix input.
+      tag               qqwry.${tag}
+    ]
+    assert_equal 'qqwry_city', d.instance.config['enable_key_city']
+
+    # multiple key config
+    d = create_driver %[
+      qqwry_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+      remove_tag_prefix input.
+      tag               qqwry.${tag}
+    ]
+    assert_equal 'from_city, to_city', d.instance.config['enable_key_city']
+
+    # multiple key config (bad configure)
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        qqwry_lookup_key  from.ip, to.ip
+        enable_key_city   from_city
+        enable_key_region from_region
+        remove_tag_prefix input.
+        tag               qqwry.${tag}
+      ]
+    }
+
+    # invalid json structure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        qqwry_lookup_key  host
+        <record>
+          invalid_json    {"foo" => 123}
+        </record>
+        remove_tag_prefix input.
+        tag               qqwry.${tag}
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        qqwry_lookup_key  host
+        <record>
+          invalid_json    {"foo" : string, "bar" : 123}
+        </record>
+        remove_tag_prefix input.
+        tag               qqwry.${tag}
+      ]
+    }
+  end
+
+  def test_emit
+    d1 = create_driver(CONFIG, 'input.access')
+    d1.run do
+      d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'qqwry.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['qqwry_city']
+    assert_equal nil, emits[1][2]['qqwry_city']
+  end
+
+  def test_emit_tag_option
+    d1 = create_driver(%[
+      qqwry_lookup_key  host
+      <record>
+        qqwry_city      ${city['host']}
+      </record>
+      remove_tag_prefix input.
+      tag               qqwry.${tag}
+    ], 'input.access')
+    d1.run do
+      d1.emit({'host' => '66.102.3.80', 'message' => 'valid ip'})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'qqwry.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['qqwry_city']
+    assert_equal nil, emits[1][2]['qqwry_city']
+  end
+
+  def test_emit_tag_parts
+    d1 = create_driver(%[
+      qqwry_lookup_key  host
+      <record>
+        qqwry_city      ${city['host']}
+      </record>
+      tag               qqwry.${tag_parts[1]}.${tag_parts[2..3]}.${tag_parts[-1]}
+    ], '0.1.2.3')
+    d1.run do
+      d1.emit({'host' => '66.102.3.80'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'qqwry.1.2.3.3', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['qqwry_city']
+  end
+
+  def test_emit_nested_attr
+    d1 = create_driver(%[
+      qqwry_lookup_key  host.ip
+      enable_key_city   qqwry_city
+      remove_tag_prefix input.
+      add_tag_prefix    qqwry.
+    ], 'input.access')
+    d1.run do
+      d1.emit({'host' => {'ip' => '66.102.3.80'}, 'message' => 'valid ip'})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'qqwry.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['qqwry_city']
+    assert_equal nil, emits[1][2]['qqwry_city']
+  end
+
+  def test_emit_with_unknown_address
+    d1 = create_driver(CONFIG, 'input.access')
+    d1.run do
+      # 203.0.113.1 is a test address described in RFC5737
+      d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'})
+      d1.emit({'host' => '0', 'message' => 'invalid ip'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'qqwry.access', emits[0][0] # tag
+    assert_equal nil, emits[0][2]['qqwry_city']
+    assert_equal 'qqwry.access', emits[1][0] # tag
+    assert_equal nil, emits[1][2]['qqwry_city']
+  end
+
+  def test_emit_multiple_key
+    d1 = create_driver(%[
+      qqwry_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+      remove_tag_prefix input.
+      add_tag_prefix    qqwry.
+    ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    assert_equal 'qqwry.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['from_city']
+    assert_equal 'Musashino', emits[0][2]['to_city']
+    assert_equal nil, emits[1][2]['from_city']
+    assert_equal nil, emits[1][2]['to_city']
+  end
+
+  def test_emit_multiple_key_multiple_record
+    d1 = create_driver(%[
+      qqwry_lookup_key  from.ip, to.ip
+      enable_key_city   from_city, to_city
+      enable_key_country_name from_country, to_country
+      remove_tag_prefix input.
+      add_tag_prefix    qqwry.
+    ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
+      d1.emit({'from' => {'ip' => '66.102.3.80'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 3, emits.length
+    assert_equal 'qqwry.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['from_city']
+    assert_equal 'United States', emits[0][2]['from_country']
+    assert_equal 'Musashino', emits[0][2]['to_city']
+    assert_equal 'Japan', emits[0][2]['to_country']
+
+    assert_equal 'Mountain View', emits[1][2]['from_city']
+    assert_equal 'United States', emits[1][2]['from_country']
+    assert_equal nil, emits[1][2]['to_city']
+    assert_equal nil, emits[1][2]['to_country']
+
+    assert_equal nil, emits[2][2]['from_city']
+    assert_equal nil, emits[2][2]['from_country']
+    assert_equal nil, emits[2][2]['to_city']
+    assert_equal nil, emits[2][2]['to_country']
+  end
+
+  def test_emit_record_directive
+    d1 = create_driver(%[
+      qqwry_lookup_key  from.ip
+      <record>
+        from_city       ${city['from.ip']}
+        from_country    ${country_name['from.ip']}
+        latitude        ${latitude['from.ip']}
+        longitude       ${longitude['from.ip']}
+        float_concat    ${latitude['from.ip']},${longitude['from.ip']}
+        float_array     [${longitude['from.ip']}, ${latitude['from.ip']}]
+        float_nest      { "lat" : ${latitude['from.ip']}, "lon" : ${longitude['from.ip']}}
+        string_concat   ${latitude['from.ip']},${longitude['from.ip']}
+        string_array    [${city['from.ip']}, ${country_name['from.ip']}]
+        string_nest     { "city" : ${city['from.ip']}, "country_name" : ${country_name['from.ip']}}
+        unknown_city    ${city['unknown_key']}
+        undefined       ${city['undefined']}
+        broken_array1   [${longitude['from.ip']}, ${latitude['undefined']}]
+        broken_array2   [${longitude['undefined']}, ${latitude['undefined']}]
+      </record>
+      remove_tag_prefix input.
+      tag               qqwry.${tag}
+    ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+
+    assert_equal 'qqwry.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['from_city']
+    assert_equal 'United States', emits[0][2]['from_country']
+    assert_equal 37.4192008972168, emits[0][2]['latitude']
+    assert_equal -122.05740356445312, emits[0][2]['longitude']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['float_concat']
+    assert_equal [-122.05740356445312, 37.4192008972168], emits[0][2]['float_array']
+    float_nest = {"lat" => 37.4192008972168, "lon" => -122.05740356445312 }
+    assert_equal float_nest, emits[0][2]['float_nest']
+    assert_equal '37.4192008972168,-122.05740356445312', emits[0][2]['string_concat']
+    assert_equal ["Mountain View", "United States"], emits[0][2]['string_array']
+    string_nest = {"city" => "Mountain View", "country_name" => "United States"}
+    assert_equal string_nest, emits[0][2]['string_nest']
+    assert_equal nil, emits[0][2]['unknown_city']
+    assert_equal nil, emits[0][2]['undefined']
+    assert_equal [-122.05740356445312, nil], emits[0][2]['broken_array1']
+    assert_equal [nil, nil], emits[0][2]['broken_array2']
+
+    assert_equal nil, emits[1][2]['from_city']
+    assert_equal nil, emits[1][2]['from_country']
+    assert_equal nil, emits[1][2]['latitude']
+    assert_equal nil, emits[1][2]['longitude']
+    assert_equal ',', emits[1][2]['float_concat']
+    assert_equal [nil, nil], emits[1][2]['float_array']
+    float_nest = {"lat" => nil, "lon" => nil}
+    assert_equal float_nest, emits[1][2]['float_nest']
+    assert_equal ',', emits[1][2]['string_concat']
+    assert_equal [nil, nil], emits[1][2]['string_array']
+    string_nest = {"city" => nil, "country_name" => nil}
+    assert_equal string_nest, emits[1][2]['string_nest']
+    assert_equal nil, emits[1][2]['unknown_city']
+    assert_equal nil, emits[1][2]['undefined']
+    assert_equal [nil, nil], emits[1][2]['broken_array1']
+    assert_equal [nil, nil], emits[1][2]['broken_array2']
+  end
+
+  def test_emit_record_directive_multiple_record
+    d1 = create_driver(%[
+      qqwry_lookup_key  from.ip, to.ip
+      <record>
+        from_city       ${city['from.ip']}
+        to_city         ${city['to.ip']}
+        from_country    ${country_name['from.ip']}
+        to_country      ${country_name['to.ip']}
+        string_array    [${country_name['from.ip']}, ${country_name['to.ip']}]
+      </record>
+      remove_tag_prefix input.
+      tag               qqwry.${tag}
+    ], 'input.access')
+    d1.run do
+      d1.emit({'from' => {'ip' => '66.102.3.80'}, 'to' => {'ip' => '125.54.95.42'}})
+      d1.emit({'message' => 'missing field'})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+
+    assert_equal 'qqwry.access', emits[0][0] # tag
+    assert_equal 'Mountain View', emits[0][2]['from_city']
+    assert_equal 'United States', emits[0][2]['from_country']
+    assert_equal 'Musashino', emits[0][2]['to_city']
+    assert_equal 'Japan', emits[0][2]['to_country']
+    assert_equal ['United States','Japan'], emits[0][2]['string_array']
+
+    assert_equal nil, emits[1][2]['from_city']
+    assert_equal nil, emits[1][2]['to_city']
+    assert_equal nil, emits[1][2]['from_country']
+    assert_equal nil, emits[1][2]['to_country']
+    assert_equal [nil, nil], emits[1][2]['string_array']
+  end
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-qqwry
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Chris Song
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-rewrite-tag-name
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: qqwry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- fakechris@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- data/qqwry.dat
+- fluent-plugin-qqwry.gemspec
+- lib/fluent/plugin/out_qqwry.rb
+- test/helper.rb
+- test/plugin/test_out_qqwry.rb
+homepage: https://github.com/fakechris/fluent-plugin-qqwry
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd Output plugin to add information about geographical location of IP
+  addresses with QQWry databases.
+test_files:
+- test/helper.rb
+- test/plugin/test_out_qqwry.rb