fluent-plugin-record-reformer 0.2.8 → 0.2.9

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.9 (2014/05/14)
+
+Enhancements:
+
+* Add `keep_keys` option
+
 ## 0.2.8 (2014/04/12)
 
 Changes:

data/README.md

@@ -20,11 +20,12 @@ Example:
       renew_record false
       enable_ruby false
       
-      tag reformed.${tag}
+      tag reformed.${tag_prefix[-2]}
       <record>
         hostname ${hostname}
         input_tag ${tag}
-        message ${message}, ${tag_parts[-1]}
+        last_tag ${tag_parts[-1]}
+        message ${message}, yay!
       </record>
     </match>
 
@@ -33,7 +34,7 @@ Assume following input is coming (indented):
 ```js
 foo.bar {
   "remove_me":"bar",
-  "foo":"bar",
+  "not_remove_me":"bar",
   "message":"Hello world!"
 }
 ```
@@ -41,11 +42,12 @@ foo.bar {
 then output becomes as below (indented):
 
 ```js
-reformed.foo.bar {
-  "foo":"bar",
+reformed.foo {
+  "not_remove_me":"bar",
   "hostname":"YOUR_HOSTNAME",
   "input_tag":"foo.bar",
-  "message":"Hello world!, bar",
+  "last_tag":"bar",
+  "message":"Hello world!, yay!",
 }
 ```
 
@@ -58,11 +60,12 @@ Example:
       remove_keys remove_me
       renew_record false
       enable_ruby false
-      tag reformed.${tag}
+      tag reformed.${tag_prefix[-2]}
       
       hostname ${hostname}
       input_tag ${tag}
-      message ${message}, ${tag_parts[-1]}
+      last_tag ${tag_parts[-1]}
+      message ${message}, yay!
     </match>
 
 This results in same, but please note that following option parameters are reserved, so can not be used as a record key.
@@ -85,7 +88,13 @@ This results in same, but please note that following option parameters are reser
 
 - renew_record *bool*
 
-    Set to `true` if you do not want to extend (or merge) the input record fields. Default is `false`.
+    `renew_record true` creates an output record newly without extending (merging) the input record fields. Default is `false`.
+
+- keep_keys
+
+    You may use with `renew_record true`. Specify record keys to be kept by a string separated by , (comma) like
+
+        keep_keys message,foo
 
 - enable_ruby *bool*
 

data/fluent-plugin-record-reformer.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-record-reformer"
-  gem.version     = "0.2.8"
+  gem.version     = "0.2.9"
   gem.authors     = ["Naotoshi Seo"]
   gem.email       = "sonots@gmail.com"
   gem.homepage    = "https://github.com/sonots/fluent-plugin-record-reformer"

data/lib/fluent/plugin/out_record_reformer.rb

@@ -12,10 +12,11 @@ module Fluent
     config_param :output_tag, :string, :default => nil # obsolete
     config_param :tag, :string, :default => nil
     config_param :remove_keys, :string, :default => nil
+    config_param :keep_keys, :string, :default => nil
     config_param :renew_record, :bool, :default => false
     config_param :enable_ruby, :bool, :default => true # true for lower version compatibility
 
-    BUILTIN_CONFIGURATIONS = %W(type tag output_tag remove_keys renew_record enable_ruby)
+    BUILTIN_CONFIGURATIONS = %W(type tag output_tag remove_keys renew_record keep_keys enable_ruby)
 
     # To support log_level option implemented by Fluentd v0.10.43
     unless method_defined?(:log)
@@ -43,6 +44,11 @@ module Fluent
         @remove_keys = @remove_keys.split(',')
       end
 
+      if @keep_keys
+        raise Fluent::ConfigError, "out_record_reformer: `renew_record` must be true to use `keep_keys`" unless @renew_record
+        @keep_keys = @keep_keys.split(',')
+      end
+
       if @output_tag and @tag.nil? # for lower version compatibility
         log.warn "out_record_reformer: `output_tag` is deprecated. Use `tag` option instead."
         @tag = @output_tag
@@ -96,8 +102,9 @@ module Fluent
       new_tag = @placeholder_expander.expand(tag)
 
       new_record = @renew_record ? {} : record.dup
-      @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
-      @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
+      @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
+      @map.each_pair {|k, v| new_record[k] = @placeholder_expander.expand(v) }
+      @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       [new_tag, new_record]
     end

data/spec/out_record_reformer_spec.rb

@@ -29,6 +29,11 @@ describe Fluent::RecordReformerOutput do
         let(:config) { %[] }
         it { expect { subject }.to raise_error(Fluent::ConfigError) }
       end
+
+      context "keep_keys must be specified togerther with renew_record true" do
+        let(:config) { %[keep_keys a] }
+        it { expect { subject }.to raise_error(Fluent::ConfigError) }
+      end
     end
   end
 
@@ -131,6 +136,21 @@ describe Fluent::RecordReformerOutput do
       it { emit }
     end
 
+    context 'keep_keys' do
+      let(:emit) do
+        driver.run { driver.emit({'foo'=>'bar', 'message' => 1}, time.to_i) }
+      end
+      let(:config) { %[tag reformed.${tag}\nrenew_record true\nkeep_keys foo,message] }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'foo' => 'bar',
+          'message' => 1, # this keep type
+        })
+      end
+      it { emit }
+    end
+
     context 'unknown placeholder (enable_ruby no)' do
       let(:emit) do
         driver.run { driver.emit({}, time.to_i) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-reformer
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-12 00:00:00.000000000 Z
+date: 2014-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -125,7 +125,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1062048915
+      hash: 689323407
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -134,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1062048915
+      hash: 689323407
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23