fluent-plugin-lookup 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bc552128863c56880e1ed789e3dc9956f8063bfe
+  data.tar.gz: a870d43673d7027df0f90fdf6343dae2f4f397f1
+SHA512:
+  metadata.gz: a4752b33dcba97bd4aaa359e63678efc02437f61315f3ea5c858715bb406ba22ca134aebc7299bdff21f0be591341c84851dfe88bbe986893b1533ddb01cddcc
+  data.tar.gz: 45bcabbafc264a61d6686ed6c305997d88b7726760da4102428c26db0537caca08697e50e6ae1a93a070abd176d75b53602c975a057fecacc90ec90e4c169f3d

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-filter_custom_cart.gemspec
+gemspec

data/README.md

@@ -0,0 +1,91 @@
+fluent-plugin-lookup
+====================
+
+(Yet another Fluentd plugin)
+
+What
+----
+
+Allows to replace record values for specific keys, using a lookup table from a *CSV* file.
+
+How
+---
+
+You basically want to define :
+- Input field : **field**.
+- Output field : **output_field** (omitting this parameter will replace *input* field value).
+- Lookup table CSV file : **table_file** (two columns per row, separated by a comma).
+- Sanity check, raises error if empty, malformed file or duplicates entries inside the file : **strict** (omitting this parameter will default to *false*).
+
+Use this filter multiple times if you need to repalce multiple fields.
+
+Examples
+========
+
+This is our *lookup.csv* file :
+
+```
+value,other_value
+nicolas,cage
+input,output
+1,one
+two,2
+```
+
+Example 1
+---------
+
+```
+<match *.test>
+    type lookup
+    add_tag_prefix lookup.
+    table_file /usr/share/my/lookup.csv
+    field key1
+    output_field key2
+</match>
+```
+
+Example of records :
+```
+{
+    'key1' => "nicolas",
+    'foo' => "bar"
+}
+```
+... will output :
+```
+{
+    'key1' => "nicolas",
+    'key2' => "cage",
+    'foo' => "bar"
+}
+```
+
+Example 2
+---------
+
+```
+<match *.test>
+    type lookup
+    add_tag_prefix lookup.
+    table_file /usr/share/my/lookup.csv
+    field key1
+</match>
+```
+
+Example of records :
+```
+{
+    'key1' => "nicolas",
+    'foo' => "bar"
+}
+```
+... will output :
+```
+{
+    'key1' => "cage",
+    'foo' => "bar"
+}
+```
+
+Since *output_field* is not defined, the input *field* value is replaced.

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+desc 'Default: run test.'
+task :default => :test
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/fluent-plugin-lookup.gemspec

@@ -0,0 +1,19 @@
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-lookup"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Neozaru"]
+  spec.email         = ["neozaru@mailoo.org"]
+  spec.description   = %q{Fluentd custom plugin to replace fields values using lookup table file}
+  spec.summary       = %q{Fluentd custom plugin to replace fields values using lookup table file}
+  spec.homepage      = "https://github.com/Neozaru/fluent-plugin-lookup.git"
+  spec.license       = "WTFPL"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "fluentd"
+end

data/lib/fluent/plugin/out_lookup.rb

@@ -0,0 +1,101 @@
+# coding: utf-8
+require "csv"
+
+module Fluent
+  class LookupOutput < Output
+    include Fluent::HandleTagNameMixin
+
+    Fluent::Plugin.register_output('lookup', self)
+
+    config_param :table_file, :string, :default => nil
+    config_param :field, :string, :default => nil
+    config_param :output_field, :string, :default => nil
+    config_param :strict, :bool, :default => false
+
+    def handle_row(lookup_table, row)
+      if (row.length < 2)
+        return handle_row_error(row, "Too few columns : #{row.length} instead of 2")
+      end
+
+      # If too much columns
+      if (strict && row.length > 2)
+        return handle_row_error(row, "Too much columns : #{row.length} instead of 2")
+      end
+
+      # If duplicates
+      if (strict && lookup_table.has_key?(row[0]))
+        return handle_row_error(row, "Duplicate entry")
+      end
+
+      lookup_table[row[0]] = row[1]
+
+    end
+
+
+    def create_lookup_table(file)
+      lookup_table = {}
+      CSV.foreach(file) do |row|
+        handle_row(lookup_table, row)
+      end
+
+      if (strict && lookup_table.length == 0)
+        raise ConfigError, "Lookup file is empty"
+      end
+
+      return lookup_table
+    rescue Errno::ENOENT => e
+      handle_file_err(file, e)
+    rescue Errno::EACCES => e
+      handle_file_err(file, e)
+    end
+
+
+    def configure(conf)
+      super
+
+      if (field.nil? || table_file.nil?) 
+        raise ConfigError, "lookup: Both 'field', and 'table_file' are required to be set."
+      end
+
+      @lookup_table = create_lookup_table(table_file)
+      @field = field
+      @output_field = output_field || field
+
+    end
+
+    def emit(tag, es, chain)
+      es.each { |time, record|
+        t = tag.dup
+        filter_record(t, time, record)
+        Engine.emit(t, time, record)
+      }
+
+      chain.next
+    end
+
+    private
+
+    def filter_record(tag, time, record)
+      super(tag, time, record)
+      if (not record.has_key?(@field))
+        return
+      end
+      record[@output_field] = process(record[@field]) || record[@field]
+    end
+
+    def process(value)
+      return @lookup_table[value]
+    end
+
+
+    def handle_row_error(row, e)
+      raise ConfigError, "Error at row #{row} : #{e}"
+    end
+
+
+    def handle_file_err(file, e)
+      raise ConfigError, "Unable to open file '#{file}' : #{e.message}"
+    end
+
+  end
+end

data/test/plugin/correct.csv

@@ -0,0 +1,5 @@
+foo,bar
+nicolas,cage
+input,output
+1,one
+two,2

data/test/plugin/duplicates.csv

@@ -0,0 +1,6 @@
+foo,bar
+nicolas,cage
+foo,bar
+input,output
+1,one
+two,2

data/test/plugin/test_out_lookup.rb

@@ -0,0 +1,375 @@
+# coding: utf-8
+
+require 'test_helper'
+require 'fluent/plugin/out_lookup'
+
+class LookupOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    dir = File.dirname(__FILE__)
+    @nonexisting_file = "#{dir}/tobeornottobe.csv"
+    @correct_file = "#{dir}/correct.csv"
+    @duplicates_file = "#{dir}/duplicates.csv"
+    @empty_file = "#{dir}/empty.csv"
+  end
+
+  def create_driver(conf, tag = 'test')
+    Fluent::Test::OutputTestDriver.new(
+      Fluent::LookupOutput, tag
+    ).configure(conf)
+  end
+
+
+  def test_configure_on_success
+
+
+    # All set
+    d = create_driver(%[
+      strict true
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+      output_field key2
+    ])
+
+    assert_equal 'lookup.', d.instance.add_tag_prefix
+    assert_equal 'key1',    d.instance.field
+    assert_equal 'key2', d.instance.output_field
+    assert_equal true, d.instance.strict
+    assert_equal @correct_file, d.instance.table_file
+
+
+    # "Strict" omitted
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+      output_field key2
+    ])
+
+    assert_equal 'lookup.', d.instance.add_tag_prefix
+    assert_equal 'key1',    d.instance.field
+    assert_equal 'key2', d.instance.output_field
+    assert_equal false, d.instance.strict
+    assert_equal @correct_file, d.instance.table_file
+
+
+    # "output_field" omitted
+    d = create_driver(%[
+      strict true
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    assert_equal 'lookup.', d.instance.add_tag_prefix
+    assert_equal 'key1',    d.instance.field
+    assert_equal 'key1', d.instance.output_field
+    assert_equal true, d.instance.strict
+    assert_equal @correct_file, d.instance.table_file
+
+
+    # File with duplicates in non-strict mode
+    d = create_driver(%[
+      strict false
+      add_tag_prefix lookup.
+      table_file #{@duplicates_file}
+      field key1
+      output_field key2
+    ])
+
+    assert_equal 'lookup.', d.instance.add_tag_prefix
+    assert_equal 'key1',    d.instance.field
+    assert_equal 'key2', d.instance.output_field
+    assert_equal false, d.instance.strict
+    assert_equal @duplicates_file, d.instance.table_file
+
+
+    # Empty file in non-strict mode
+    d = create_driver(%[
+      strict false
+      add_tag_prefix lookup.
+      table_file #{@empty_file}
+      field key1
+      output_field key2
+    ])
+
+    assert_equal 'lookup.', d.instance.add_tag_prefix
+    assert_equal 'key1',    d.instance.field
+    assert_equal 'key2', d.instance.output_field
+    assert_equal false, d.instance.strict
+    assert_equal @empty_file, d.instance.table_file
+
+  end
+
+  def test_configure_on_failure
+    # when mandatory keys not set
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        blah blah
+      ])
+    end
+
+    # 'field' is missing
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        strict true
+        add_tag_prefix lookup.
+        table_file #{@correct_file}
+        output_field key2
+      ])
+    end
+
+    # 'table_file' is missing
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        strict true
+        add_tag_prefix lookup.
+        field key1
+        output_field key2
+      ])
+    end
+
+    # 'table_file' is not readable in strict mode
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        strict true
+        add_tag_prefix lookup.
+        table_file #{@nonexisting_file}
+        field key1
+        output_field key2
+      ])
+    end
+
+    # 'table_file' is not readable in non-strict mode
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        strict false
+        add_tag_prefix lookup.
+        table_file #{@nonexisting_file}
+        field key1
+        output_field key2
+      ])
+    end
+
+    # 'table_file' contains duplicates in strict mode
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        strict true
+        add_tag_prefix lookup.
+        table_file #{@duplicates_file}
+        field key1
+        output_field key2
+      ])
+    end
+
+    # 'table_file' is empty in strict mode
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        strict true
+        add_tag_prefix lookup.
+        table_file #{@empty_file}
+        field key1
+        output_field key2
+      ])
+    end
+
+  end
+
+
+  def test_unit_create_lookup_table
+
+    # Correct file
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+      output_field key2
+    ])
+
+    table = d.instance.create_lookup_table(@correct_file)
+    assert_equal({"foo"=>"bar", "nicolas"=>"cage", "input"=>"output", "1"=>"one", "two"=>"2"}, table)
+
+    # With duplicates
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@duplicates_file}
+      field key1
+      output_field key2
+    ])
+
+    table = d.instance.create_lookup_table(@duplicates_file)
+    assert_equal({"foo"=>"bar", "nicolas"=>"cage", "input"=>"output", "1"=>"one", "two"=>"2"}, table)
+
+    # Empty file
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@empty_file}
+      field key1
+      output_field key2
+    ])
+
+    table = d.instance.create_lookup_table(@empty_file)
+    assert_equal({}, table)
+
+  end
+
+
+  def test_unit_handle_row
+    # Correct row
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    table = {}
+    d.instance.handle_row(table, ["foo", "bar"])
+    assert_equal({"foo"=>"bar"}, table)
+
+    # Too small row
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    table = {}
+    assert_raise do d.instance.handle_row(table, ["foo"]) end
+    assert_equal({}, table)
+
+    # Too large row non strict
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    table = {}
+    d.instance.handle_row(table, ["foo", "bar", "baz"])
+    assert_equal({"foo" => "bar"}, table)
+
+    # Too large row strict
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      strict true
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    table = {}
+    assert_raise do d.instance.handle_row(table, ["foo", "bar", "baz"]) end
+    assert_equal({}, table)
+
+
+    # Too duplicate row non strict
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    table = {}
+    d.instance.handle_row(table, ["foo", "bar"])
+    d.instance.handle_row(table, ["foo", "baz"])
+    assert_equal({"foo" => "baz"}, table)
+
+
+    # Too duplicate row strict
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      strict true
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    table = {}
+    d.instance.handle_row(table, ["foo", "bar"])
+    assert_raise do d.instance.handle_row(table, ["foo", "baz"]) end
+    assert_equal({"foo" => "bar"}, table)
+  end
+
+
+  def test_emit_with_output_field
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+      output_field key2
+    ])
+
+    record = {
+      'key1' => "nicolas",
+      'foo' => "bar"
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'cage', emits[0][2]['key2']
+  end
+
+  def test_emit_with_output_field_no_correspondance
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+      output_field key2
+    ])
+
+    record = {
+      'key1' => "myvalue",
+      'foo' => "bar"
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'myvalue', emits[0][2]['key2']
+  end
+
+  def test_emit_without_output_field
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    record = {
+      'key1' => "nicolas",
+      'foo' => "bar"
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'cage', emits[0][2]['key1']
+  end
+
+  def test_emit_without_output_field_no_correspondance
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field key1
+    ])
+
+    record = {
+      'key1' => "myvalue",
+      'foo' => "bar"
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'myvalue', emits[0][2]['key1']
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,19 @@
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-lookup
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Neozaru
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd custom plugin to replace fields values using lookup table file
+email:
+- neozaru@mailoo.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-lookup.gemspec
+- lib/fluent/plugin/out_lookup.rb
+- test/plugin/correct.csv
+- test/plugin/duplicates.csv
+- test/plugin/empty.csv
+- test/plugin/test_out_lookup.rb
+- test/test_helper.rb
+homepage: https://github.com/Neozaru/fluent-plugin-lookup.git
+licenses:
+- WTFPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Fluentd custom plugin to replace fields values using lookup table file
+test_files:
+- test/plugin/correct.csv
+- test/plugin/duplicates.csv
+- test/plugin/empty.csv
+- test/plugin/test_out_lookup.rb
+- test/test_helper.rb