fluent-plugin-record-reformer 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6cb811fb3f7274893d0f871c070fa298b0f9c138
-  data.tar.gz: 504bb732f1e60a2bc98f80b44a78294de6770627
+  metadata.gz: 2166c7d160e63370e3400203a591cbb9c2de22e7
+  data.tar.gz: e01e518088e90bdb32e4040ed5a2fbbca5ee6bf8
 SHA512:
-  metadata.gz: 29c43c2e9b76b18de8c5744219814a1a9ca4babfa1f332c88d84c7b1db2e89fe38a57e4f78fc6c8a320cacb4ab88542174a05728f997a913857bc0603511d0b5
-  data.tar.gz: b6a984d357f3a1f74221284a3982265001a01338c6c3667b60d1ad1d7455270553e46345351172510c7d4d6775409daf59dda35fbd686b4bbcee80b22c903455
+  metadata.gz: b2e3901a0af27f800d9a81e34477409f8696b13bed9c2e47425911cd45ae0295455a5686e42969daa31a4a327c8cf779b821f0fe4a8b4625eedfe72fce58b0f9
+  data.tar.gz: 8812a93c884c231525b95bf29b14e5f9d4381ea0f9cece5a3109b3d7439b3cfff677c99831741d7aa7f509798bc4cea68348cbe4747a15676c45c471ff26cb62

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.2.0 (2014/01/15)
+
+Enhancement:
+
+* Support a record directive
+* Add `remove_keys` option
+* Add `extends` option
+* Add `enable_ruby` option
+
 ## 0.1.1 (2013/11/21)
 
 Changes:

data/README.md

@@ -1,6 +1,6 @@
 # fluent-plugin-record-reformer
 
-[![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-record-reformer.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-record-reformer) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-record-reformer.png)](https://gemnasium.com/sonots/fluent-plugin-record-reformer)
+[![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-record-reformer.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-record-reformer)
 
 Fluentd pluging to add or replace fields of a event record
 
@@ -16,63 +16,108 @@ Example:
 
     <match foo.**>
       type record_reformer
-      output_tag reformed.${tag}
+      remove_keys remove_me
+      renew_record false
+      enable_ruby false
       
-      hostname ${hostname}
-      tag ${tag}
-      time ${time.strftime('%Y-%m-%dT%H:%M:%S%z')}
-      message ${hostname} ${tag_parts.last} ${message}
+      output_tag reformed.${tag}
+      <record>
+        hostname ${hostname}
+        input_tag ${tag}
+        message ${message}, ${tag_parts[-1]}
+      </record>
     </match>
 
-Assume following input is coming:
+Assume following input is coming (indented):
 
 ```js
-foo.bar {"message":"hello world!", "foo":"bar"}
+foo.bar {
+  "remove_me":"bar",
+  "foo":"bar",
+  "message":"Hello world!"
+}
 ```
 
 then output becomes as below (indented):
 
 ```js
 reformed.foo.bar {
-  "hostname":"your_hostname", 
-  "tag":"foo.bar",
-  "time":"2013-05-01T01:13:14Z",
-  "message":"your_hostname bar hello world!",
-  "foo":"bar"
+  "foo":"bar",
+  "hostname":"YOUR_HOSTNAME",
+  "input_tag":"foo.bar",
+  "message":"Hello world!, bar",
 }
 ```
 
-Note that the keyword `output_tag` is reserved for the output tag. It can not be used as a record key. 
+## Configuration (Classic Style)
+
+Example:
+
+    <match foo.**>
+      type record_reformer
+      remove_keys remove_me
+      renew_record false
+      enable_ruby false
+      output_tag reformed.${tag}
+      
+      hostname ${hostname}
+      input_tag ${tag}
+      message ${message}, ${tag_parts[-1]}
+    </match>
+
+This results in same, but please note that following option parameters are reserved, so can not be used as a record key.
+
+## Parameters
+
+- output_tag
+
+    The output tag name
+
+- remove_keys
+
+    Specify record keys to be removed by a string separated by , (comma) like
+
+        remove_keys message,foo
 
-### Placeholders
+- renew_record *bool*
+
+    Set to `true` if you do not want to extend (or merge) the input record fields. Default is `false`.
+
+- enable_ruby *bool*
+
+    Enable to use ruby codes in placeholders. See `Placeholders` section.
+    Default is `true` (just for lower version compatibility). 
+
+## Placeholders
 
 The keys of input json are available as placeholders. In the above example, 
 
 * ${foo}
 * ${message}
+* ${remove_me}
 
 shall be available. In addition, following placeholders are reserved: 
 
 * ${hostname} hostname
 * ${tag} input tag
-* ${tags} input tag splitted by '.' (obsolete. use tag_parts)
-* ${tag_parts} input tag splitted by '.'
 * ${time} time of the event
+* ${tags[N]} input tag splitted by '.' (obsolete. use tag\_parts)
+* ${tag\_parts[N]} input tag splitted by '.' indexed with N such as `${tag_parts[0]}`, `${tag_parts[-1]}`. 
 
-It is also possible to write a ruby code in placeholders, so you may write some codes as
+It is also possible to write a ruby code in placeholders if you set `enable_ruby true` option, so you may write some codes as
 
 * ${time.strftime('%Y-%m-%dT%H:%M:%S%z')}
-* ${tag_parts[0]}
-* ${tag_parts.last}
-
-## Notice
+* ${tag\_parts.last}
 
-Please note that this plugin enables to execute any ruby codes. Do not allow anyone to write fluentd configuration from outside of your system by security reasons.
+but, please note that enabling ruby codes is not encouraged by security reasons and also in terms of the performance.
 
 ## Relatives
 
-I created this plugin inspired by [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier). 
-I chose not to send pull requests because the implementation of this plugin became completely different with it.
+Following plugins look similar:
+
+* [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier)
+* [fluent-plugin-format](https://github.com/mach/fluent-plugin-format)
+* [fluent-plugin-add](https://github.com/yu-yamada/fluent-plugin-add)
 
 ## ChangeLog
 

data/fluent-plugin-record-reformer.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-record-reformer"
-  gem.version     = "0.1.1"
+  gem.version     = "0.2.0"
   gem.authors     = ["Naotoshi Seo"]
   gem.email       = "sonots@gmail.com"
   gem.homepage    = "https://github.com/sonots/fluent-plugin-record-reformer"
@@ -21,4 +21,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec"
   gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry-nav"
 end

data/lib/fluent/plugin/out_record_reformer.rb

@@ -13,8 +13,11 @@ module Fluent
     end
 
     config_param :output_tag, :string
+    config_param :remove_keys, :string, :default => nil
+    config_param :renew_record, :bool, :default => false
+    config_param :enable_ruby, :bool, :default => true # true for lower version compatibility
 
-    BUILTIN_CONFIGURATIONS = %W(type output_tag)
+    BUILTIN_CONFIGURATIONS = %W(type output_tag remove_keys renew_record enable_ruby)
 
     def configure(conf)
       super
@@ -22,9 +25,34 @@ module Fluent
       @map = {}
       conf.each_pair { |k, v|
         next if BUILTIN_CONFIGURATIONS.include?(k)
-        conf.has_key?(k)
+        conf.has_key?(k) # to suppress unread configuration warning
         @map[k] = v
       }
+      # <record></record> directive
+      conf.elements.select { |element| element.name == 'record' }.each { |element|
+        element.each_pair { |k, v|
+          element.has_key?(k) # to suppress unread configuration warning
+          @map[k] = v
+        }
+      }
+
+      if @remove_keys
+        @remove_keys = @remove_keys.split(',')
+      end
+
+      @expand_placeholder_proc =
+        if @enable_ruby
+          Proc.new {|str, record, tag, tag_parts, time| expand_ruby_placeholder(str, record, tag, tag_parts, time) }
+        else
+          Proc.new {|str, record, tag, tag_parts, time| expand_placeholder(str, record, tag, tag_parts, time) }
+        end
+
+      @time_proc = # hmm, want to remove ${time} placeholder ...
+        if @enable_ruby
+          Proc.new {|time| Time.at(time) }
+        else
+          Proc.new {|time| time }
+        end
 
       @hostname = Socket.gethostname
     end
@@ -32,23 +60,58 @@ module Fluent
     def emit(tag, es, chain)
       tag_parts = tag.split('.')
       es.each { |time, record|
-        t_time = Time.at(time)
-        output_tag = expand_placeholder(@output_tag, record, tag, tag_parts, t_time)
+        t_time = @time_proc.call(time)
+        output_tag = @expand_placeholder_proc.call(@output_tag, record, tag, tag_parts, t_time)
         Engine.emit(output_tag, time, replace_record(record, tag, tag_parts, t_time))
       }
       chain.next
     rescue => e
-      $log.warn e.message
-      $log.warn e.backtrace.join(', ')
+      $log.warn "record_reformer: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
     end
 
     private
 
     def replace_record(record, tag, tag_parts, time)
+      new_record = @renew_record ? {} : record.dup
       @map.each_pair { |k, v|
-        record[k] = expand_placeholder(v, record, tag, tag_parts, time)
+        new_record[k] = @expand_placeholder_proc.call(v, record, tag, tag_parts, time)
       }
-      record
+      @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
+      new_record
+    end
+
+    def expand_placeholder(str, record, tag, tag_parts, time)
+      # referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter, thanks!
+      placeholders = get_placeholders(record, tag, tag_parts, time)
+      str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) do
+        $log.warn "record_reformer: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless placeholders.include?($1)
+        placeholders[$1]
+      end
+    end
+
+    def get_placeholders(record, tag, tag_parts, time)
+      placeholders = {
+        '${time}' => time,
+        '${tag}' => tag,
+        '${hostname}' => @hostname,
+      }
+
+      size = tag_parts.size
+      tag_parts.each_with_index do |t, idx|
+        placeholders.store("${tag_parts[#{idx}]}", t)
+        placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
+      end
+      # tags is just for old version compatibility
+      tag_parts.each_with_index do |t, idx|
+        placeholders.store("${tags[#{idx}]}", t)
+        placeholders.store("${tags[#{idx-size}]}", t) # support tags[-1]
+      end
+
+      record.each {|k, v|
+        placeholders.store("${#{k}}", v)
+      }
+
+      return placeholders
     end
 
     # Replace placeholders in a string
@@ -58,7 +121,7 @@ module Fluent
     # @param [String] tag         the tag
     # @param [Array]  tag_parts   the tag parts (tag splitted by .)
     # @param [Time]   time        the time
-    def expand_placeholder(str, record, tag, tag_parts, time)
+    def expand_ruby_placeholder(str, record, tag, tag_parts, time)
       struct = UndefOpenStruct.new(record)
       struct.tag  = tag
       struct.tags = struct.tag_parts = tag_parts # tags is for old version compatibility

data/spec/out_record_reformer_spec.rb

@@ -75,5 +75,106 @@ describe Fluent::RecordReformerOutput do
       end
       it { emit }
     end
+
+    context 'record directive' do
+      let(:config) {%[
+        type reformed
+        output_tag reformed.${tag}
+
+        <record>
+          hostname ${hostname}
+          output_tag ${tag}
+          time ${time.strftime('%S')}
+          message ${hostname} ${tag_parts.last} ${message}
+        </record>
+      ]}
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'foo' => 'bar',
+          'hostname' => hostname,
+          'output_tag' => tag,
+          'time' => time.strftime('%S'),
+          'message' => "#{hostname} #{tag_parts.last} 1",
+        })
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'foo' => 'bar',
+          'hostname' => hostname,
+          'output_tag' => tag,
+          'time' => time.strftime('%S'),
+          'message' => "#{hostname} #{tag_parts.last} 2",
+        })
+      end
+      it { emit }
+    end
+
+    context 'remove_keys' do
+      let(:config) { CONFIG + %[remove_keys foo,message] }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.strftime('%S'),
+        })
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.strftime('%S'),
+        })
+      end
+      it { emit }
+    end
+
+    context 'renew_record true' do
+      let(:config) { CONFIG + %[renew_record true] }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.strftime('%S'),
+          'message' => "#{hostname} #{tag_parts.last} 1",
+        })
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.strftime('%S'),
+          'message' => "#{hostname} #{tag_parts.last} 2",
+        })
+      end
+      it { emit }
+    end
+
+    context 'enable_ruby no' do
+      let(:config) {%[
+        type reformed
+        output_tag reformed.${tag}
+        enable_ruby no
+
+        hostname ${hostname}
+        tag ${tag}
+        time ${time}
+        message ${hostname} ${tag_parts[-1]} ${message}
+      ]}
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'foo' => 'bar',
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.to_i.to_s, # hmm, want to remove ${time} placeholder
+          'message' => "#{hostname} #{tag_parts.last} 1",
+        })
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'foo' => 'bar',
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.to_i.to_s, # hmm, want to remove ${time} placeholder
+          'message' => "#{hostname} #{tag_parts.last} 2",
+        })
+      end
+      it { emit }
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-reformer
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-17 00:00:00.000000000 Z
+date: 2014-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -66,6 +66,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Output filter plugin for reforming each event record
 email: sonots@gmail.com
 executables: []