fluent-plugin-record-modifier 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 344506e1d7deb2d3211df17f02386283fa8a30ab
-  data.tar.gz: 44e6d422823ddf0c686abe066b7da36fc96e4e19
+  metadata.gz: 3ff1d6de463980c890752f4eb32719f3f160c026
+  data.tar.gz: 4ed772c23ab2316e1bf064a9fd3c1ed498c1aae8
 SHA512:
-  metadata.gz: 45f15173d92fa1d91e20c62337ffe757852bf42bd006c7847459ca61521cd21cc63b604715489cfec05555f8ce95ba91d182816e470c64771bb3e81cecacad17
-  data.tar.gz: 7c63568717b13b17bdec90bf43c6edf30884639dd9d586efe85bd90f02787d99a7cbbd26484f6ab2676bdddb2facae032148307b0a35aec6c36428f0e602f3ca
+  metadata.gz: 2477196fe7d51f5d26546107d4b9a01907e97e3dc38a0d5a2a1a4cc8182f298d24abb996b197e3609cae7e0a534781b93f0d5bd802d7afc31c0297f2f5e31924
+  data.tar.gz: e0826bc7e9203ea5a6bf24f715b5b63d2b13a95c407a4d298cfd8b9f2f7c781afed10c69248774e8930b7a0f86cc1f569c75a9c2e3c00e689e3dffc4fe2f6e5d

data/.travis.yml

@@ -1,9 +1,10 @@
 language: ruby
 
 rvm:
-  - 1.9.3
   - 2.0.0
-  - 2.1.1
+  - 2.1
+  - 2.2.3
+  - 2.3.0
   - ruby-head
   - rbx
 
@@ -11,9 +12,13 @@ branches:
   only:
     - master
 
+gemfile:
+  - Gemfile
+
 matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx
 
+before_install: gem update bundler
 script: bundle exec rake test

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.4.0 - 2016/01/08
+
+  * Introduce <record> directive
+  * Support ${xxx} placeholders
+    https://github.com/repeatedly/fluent-plugin-record-modifier/pull/6
+
+
 Release 0.3.0 - 2015/06/09
 
   * Add record_modifier filter

data/README.md

@@ -1,4 +1,4 @@
-# Output filter plugin for modifying each event record for [Fluentd](http://fluentd.org)
+# Filter plugin for modifying each event record for [Fluentd](http://fluentd.org)
 
 Adding arbitary field to event record without custmizing existence plugin.
 
@@ -13,13 +13,16 @@ Use RubyGems:
 
 ## Configuration
 
-    <match pattern>
-      type record_modifier
-      tag foo.filtered
+Use `record_modifier` filter.
 
-      gen_host ${hostname}
-      foo bar
-    </match>
+    <filter pattern>
+      @type record_modifier
+
+      <record>
+        gen_host ${hostname}
+        foo bar
+      </record>
+    </filter>
 
 If following record is passed:
 
@@ -33,6 +36,37 @@ then you got new record like below:
 {"message":"hello world!", "gen_host":"oreore-mac.local", "foo":"bar"}
 ```
 
+You can also use `record_transformer` like `${xxx}` placeholders and access `tag`, `time`, `record` and `tag_parts` values by Ruby code.
+
+    <filter pattern>
+      @type record_modifier
+
+      <record>
+        tag ${tag}
+        tag_extract ${tag_parts[0]}-${tag_pars[1]}-foo
+        formatted_time ${Time.at(time).to_s}
+        new_field foo:${record['key1'] + record['dict']['key']} 
+      </record>
+    </filter>
+
+`record_modifier` is faster than `record_transformer`. See [this comment](https://github.com/repeatedly/fluent-plugin-record-modifier/pull/7#issuecomment-169843012).
+But unlike `record_transformer`, `record_modifier` doesn't support following features for now.
+
+- tag_suffix and tag_prefix
+- dynamic key placeholder
+
+### record_modifier output
+
+In v0.10, you can use `record_modifier` output to emulate filter. `record_modifier` output doesn't support `<record>` way.
+
+    <match pattern>
+      type record_modifier
+      tag foo.filtered
+
+      gen_host ${hostname}
+      foo bar
+    </match>
+
 ### char_encoding
 
 Fluentd including some plugins treats the logs as a BINARY by default to forward.
@@ -41,7 +75,7 @@ But an user sometimes processes the logs depends on their requirements, e.g. han
 `char_encoding` parameter is useful for this case.
 
 ```conf
-<match pattern>
+<filter pattern>
   type record_modifier
 
   # set UTF-8 encoding information to string.
@@ -49,7 +83,7 @@ But an user sometimes processes the logs depends on their requirements, e.g. han
 
   # change char encoding from 'UTF-8' to 'EUC-JP'
   char_encoding utf-8:euc-jp
-</match>
+</filter>
 ```
 
 ### remove_keys
@@ -58,12 +92,12 @@ The logs include needless record keys in some cases.
 You can remove it by using `remove_keys` parameter.
 
 ```conf
-<match pattern>
+<filter pattern>
   type record_modifier
 
   # remove key1 and key2 keys from record
   remove_keys key1,key2
-</match>
+</filter>
 ```
 
 If following record is passed:
@@ -80,7 +114,6 @@ then you got new record like below:
 
 ### Mixins
 
-* [SetTagKeyMixin](https://github.com/fluent/fluentd/blob/master/lib/fluent/mixin.rb#L181)
 * [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders)
 
 ## TODO

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/fluent-plugin-record-modifier.gemspec

@@ -20,4 +20,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
 end

data/lib/fluent/plugin/filter_record_modifier.rb

@@ -7,19 +7,23 @@ module Fluent
     config_param :char_encoding, :string, :default => nil
     config_param :remove_keys, :string, :default => nil
 
-    include SetTagKeyMixin
     include Fluent::Mixin::ConfigPlaceholders
 
-    BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id include_tag_key tag_key char_encoding remove_keys)
+    BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys)
 
     def configure(conf)
       super
 
+      if conf.has_key?('include_tag_key')
+        raise ConfigError, "include_tag_key and tag_key parameters are removed. Use 'tag ${tag}' in <record> section"
+      end
+
       @map = {}
       conf.each_pair { |k, v|
         unless BUILTIN_CONFIGURATIONS.include?(k)
           conf.has_key?(k)
-          @map[k] = v
+          $log.warn "top level definition is deprecated. Please put parameters inside <record>: '#{k} #{v}'"
+          @map[k] = DynamicExpander.new(k, v)
         end
       }
 
@@ -39,33 +43,46 @@ module Fluent
         end
       end
 
+      @has_tag_parts = false
+      conf.elements.select { |element| element.name == 'record' }.each do |element|
+        element.each_pair do |k, v|
+          element.has_key?(k) # to suppress unread configuration warning
+          @has_tag_parts = true if v.include?('tag_parts')
+          @map[k] = DynamicExpander.new(k, v)
+        end
+      end
+
       if @remove_keys
-        @remove_keys = @remove_keys.split(',').map {|e| e.strip }
+        @remove_keys = @remove_keys.split(',').map { |e| e.strip }
       end
-    end
 
-    def filter(tag, time, record)
-      filter_record(tag, time, record)
-      modify_record(record)
+      # Collect DynamicExpander related garbage instructions
+      GC.start
     end
 
-    private
-
-    def modify_record(record)
-      @map.each_pair { |k, v|
-        record[k] = v
-      }
+    def filter_stream(tag, es)
+      new_es = MultiEventStream.new
+      tag_parts = @has_tag_parts ? tag.split('.') : nil
 
-      if @remove_keys
-        @remove_keys.each { |v|
-          record.delete(v)
+      es.each { |time, record|
+        @map.each_pair { |k, v|
+          record[k] = v.expand(tag, time, record, tag_parts)
         }
-      end
 
-      record = change_encoding(record) if @char_encoding
-      record
+        if @remove_keys
+          @remove_keys.each { |v|
+            record.delete(v)
+          }
+        end
+
+        record = change_encoding(record) if @char_encoding
+        new_es.add(time, record)
+      }
+      new_es
     end
 
+    private
+
     def set_encoding(record)
       record.each_pair { |k, v|
         if v.is_a?(String)
@@ -82,5 +99,53 @@ module Fluent
         end
       }
     end
+
+    class DynamicExpander
+      def initialize(param_key, param_value)
+        if param_value.include?('${')
+          __str_eval_code__ = parse_parameter(param_value)
+
+          # Use class_eval with string instead of define_method for performance.
+          # It can't share instructions but this is 2x+ faster than define_method in filter case.
+          # Refer: http://tenderlovemaking.com/2013/03/03/dynamic_method_definitions.html
+          (class << self; self; end).class_eval <<-EORUBY,  __FILE__, __LINE__ + 1
+            def expand(tag, time, record, tag_parts)
+              #{__str_eval_code__}
+            end
+          EORUBY
+        else
+          @param_value = param_value
+        end
+
+        begin
+          # check eval genarates wrong code or not
+          expand(nil, nil, nil, nil)
+        rescue SyntaxError
+          raise ConfigError, "Pass invalid syntax parameter : key = #{param_key}, value = #{param_value}"
+        rescue
+          # Ignore other runtime errors
+        end
+      end
+
+      # Default implementation for fixed value. This is overwritten when parameter contains '${xxx}' placeholder
+      def expand(tag, time, record, tag_parts)
+        @param_value
+      end
+
+      private
+
+      def parse_parameter(value)
+        num_placeholders = value.scan('${').size
+        if num_placeholders == 1
+          if value.start_with?('${') && value.end_with?('}')
+            return value[2..-2]
+          else
+            "\"#{value.gsub('${', '#{')}\""
+          end
+        else
+          "\"#{value.gsub('${', '#{')}\""
+        end
+      end
+    end
   end if defined?(Filter)
 end

data/test/test_filter_record_modifier.rb

@@ -12,12 +12,14 @@ class RecordModifierFilterTest < Test::Unit::TestCase
 
   CONFIG = %[
     type record_modifier
-
-    gen_host ${hostname}
-    foo bar
-    include_tag_key
-    tag_key included_tag
     remove_keys hoge
+
+    <record>
+      gen_host ${hostname}
+      foo bar
+      included_tag ${tag}
+      tag_wrap -${tag_parts[0]}-${tag_parts[1]}-
+    </record>
   ]
 
   def create_driver(conf = CONFIG)
@@ -33,8 +35,9 @@ class RecordModifierFilterTest < Test::Unit::TestCase
     d = create_driver
     map = d.instance.instance_variable_get(:@map)
 
-    assert_equal get_hostname, map['gen_host']
-    assert_equal 'bar', map['foo']
+    map.each_pair { |k, v|
+      assert v.is_a?(Fluent::RecordModifierFilter::DynamicExpander)
+    }
   end
 
   def test_format
@@ -45,7 +48,7 @@ class RecordModifierFilterTest < Test::Unit::TestCase
       d.emit("a" => 2)
     end
 
-    mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => @tag}
+    mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => @tag, 'tag_wrap' => "-#{@tag.split('.')[0]}-#{@tag.split('.')[1]}-"}
     assert_equal [
       {"a" => 1}.merge(mapped),
       {"a" => 2}.merge(mapped),

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-modifier
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-09 00:00:00.000000000 Z
+date: 2016-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -58,6 +58,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.4
 description: Output filter plugin for modifying each event record
 email: repeatedly@gmail.com
 executables: []