fluent-plugin-lookup 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ac6a0b2172ab5435936e89fa771c3612039d93c
-  data.tar.gz: f7646ff1776bc57cb6519adc9b29e09f07d042fe
+  metadata.gz: 752a9b8b44e789257e24b27a8d08a2d538397aa0
+  data.tar.gz: 9b02e0d6a7f5f67543561c3e933c6c3fd57573d3
 SHA512:
-  metadata.gz: c7aed8a806f45ba877107089297bcf8770dcc3e4e49f878e5ec2372c40294915d2953516bf3138211e64f74a92d4a8b9caa0c1f68f3feaf3ba0cce7fe204ea3e
-  data.tar.gz: fcebd49bb86c18a85503bd68dd0743b9cc5f3a2c8ddc36d4ff6a79c1868de07af4cdf444b99bbee3a69c2b7801d35c91f8fcccc13f9fff68a712044ba059bfcc
+  metadata.gz: 80769b096d7e29a452bd59c70d4384025350a0d533473a859013f48ef84319a19a06bf5afe0502f75542c60790887fb9ab8057ed5d89302fe05359477583e695
+  data.tar.gz: 8e69986b886057d210bf23f8fe6f6903f6df163dd1fe05f3bf4a9d2efe9b3c49ccfa0004604648f057294f559bffb6aac39b7b385c0688af5fd855ca84905adf

data/README.md

@@ -87,5 +87,63 @@ Example of records :
     'foo' => "bar"
 }
 ```
+Example 3
+---------
+
+```
+<match *.test>
+    type lookup
+    add_tag_prefix lookup.
+    table_file /usr/share/my/lookup.csv
+    field nested.key1
+</match>
+```
+
+Example of records :
+```
+{
+    'nested' => {
+        'key1' => "nicolas",
+    },
+    'foo' => "bar"
+}
+```
+... will output :
+```
+{
+    'nested' => {
+        'key1' => "cage",
+    },
+    'foo' => "bar"
+}
+```
+
+Example 4
+---------
+
+Renaming key
+
+```
+<match *.test>
+    type lookup
+    add_tag_prefix lookup.
+    table_file /usr/share/my/lookup.csv
+    field key1
+    rename_key true
+</match>
+```
+
+Example of records :
+```
+{
+    'input' => "nicolas",
+    'foo' => "bar"
+}
+```
+... will output :
+```
+{
+    'output' => "nicolas",
+    'foo' => "bar"
+}
 
-Since *output_field* is not defined, the input *field* value is replaced.

data/fluent-plugin-lookup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-lookup"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["Neozaru"]
   spec.email         = ["neozaru@mailoo.org"]
   spec.description   = %q{Fluentd custom plugin to replace fields values using lookup table file}

data/lib/fluent/plugin/out_lookup.rb

@@ -11,6 +11,7 @@ module Fluent
     config_param :field, :string, :default => nil
     config_param :output_field, :string, :default => nil
     config_param :strict, :bool, :default => false
+    config_param :rename_key, :bool, :default => false
 
     def handle_row(lookup_table, row)
       if (row.length < 2)
@@ -53,6 +54,11 @@ module Fluent
     def configure(conf)
       super
 
+      @assign_method = method(:assign)
+      @assign_self_method = method(:assign_self)
+      @return_method = method(:return)
+      @rename_method = method(:rename)
+
       if (field.nil? || table_file.nil?) 
         raise ConfigError, "lookup: Both 'field', and 'table_file' are required to be set."
       end
@@ -60,6 +66,11 @@ module Fluent
       @lookup_table = create_lookup_table(table_file)
       @field = field.split(".")
 
+      if (rename_key)
+        @filter_method = method(:filter_rename_key)
+        return
+      end
+
       if (output_field.nil?)
         @filter_method = method(:filter_no_output)
       else
@@ -67,9 +78,6 @@ module Fluent
         @filter_method = method(:filter_with_output)
       end
 
-      @assign_method = method(:assign)
-      @assign_self_method = method(:assign_self)
-      @return_method = method(:return)
 
     end
 
@@ -93,10 +101,17 @@ module Fluent
       record[key] = process(value) || value
     end
 
-    def return(record, key, value) 
+    def return(record, key, value_nouse) 
       return record[key]
     end
 
+    def rename(record, key, value_nouse)
+      new_key = process(key) || return
+      field_value = record[key]
+      record.delete(key)
+      record[new_key] = field_value
+    end
+    
 
     def filter_record(tag, time, record)
       super(tag, time, record)
@@ -116,6 +131,11 @@ module Fluent
       end
     end
 
+    # Rename key (will NOT copy or move the field, just rename the key)
+    def filter_rename_key(record)
+      value = dig_cb(record, @field, nil, false, @rename_method)
+    end
+
     # Generic function to dig into map. 
     def dig_cb(record, path, value, alter, cb)
       digged_record = record

data/test/plugin/test_out_lookup.rb

@@ -415,6 +415,39 @@ class LookupOutputTest < Test::Unit::TestCase
     assert_equal 'lookup.test', emits[0][0]
     assert_equal nil, emits[0][2]['nested']
     assert_equal nil, emits[0][2]['new']
+    assert_equal 'bar', emits[0][2]['foo']
+  end
+
+  # Checks that the record is not modified if the input_field is not found
+  def test_emit_nested_with_partial_output_structure
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field nested.key1
+      output_field new.foo
+    ])
+
+    record = {
+      'foo' => "bar",
+      'nested' => {
+        'key1' => "nicolas",
+      },
+      'new' => {
+        'field1' => "value1",
+        'field2' => "value2"
+      }
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'nicolas', emits[0][2]['nested']['key1']
+    assert_equal 'cage', emits[0][2]['new']['foo']
+    assert_equal 'value1', emits[0][2]['new']['field1']
+    assert_equal 'value2', emits[0][2]['new']['field2']
+
   end
 
   # Checks that the record is not modified if the input_field is not found
@@ -457,4 +490,90 @@ class LookupOutputTest < Test::Unit::TestCase
     assert_equal 'myvalue', emits[0][2]['key1']
   end
 
+  def test_emit_rename_key
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field nicolas
+      rename_key true
+    ])
+
+    record = {
+      'nicolas' => "bar"
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'bar', emits[0][2]['cage']
+    assert_equal nil, emits[0][2]['nicolas']
+  end
+
+  def test_emit_rename_key_erase
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field nicolas
+      rename_key true
+    ])
+
+    record = {
+      'nicolas' => "bar",
+      'cage' => "foo"
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'bar', emits[0][2]['cage']
+    assert_equal nil, emits[0][2]['nicolas']
+  end
+
+  def test_emit_rename_key_nested
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field nested.nicolas
+      rename_key true
+    ])
+
+    record = {
+      'nested' => {
+        'nicolas' => "bar",
+      }
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'bar', emits[0][2]['nested']['cage']
+    assert_equal nil, emits[0][2]['nested']['nicolas']
+  end
+
+  def test_emit_rename_key_not_existing
+    d = create_driver(%[
+      add_tag_prefix lookup.
+      table_file #{@correct_file}
+      field dummy
+      rename_key true
+    ])
+
+    record = {
+      'nicolas' => "bar"
+    }
+
+    d.run { d.emit(record) }
+    emits = d.emits
+
+    assert_equal 1,           emits.count
+    assert_equal 'lookup.test', emits[0][0]
+    assert_equal 'bar', emits[0][2]['nicolas']
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-lookup
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Neozaru
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-13 00:00:00.000000000 Z
+date: 2015-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler