fluent-plugin-record-reformer 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55ba6719a96af4f1c33c52d970579a9006ef9a7c
-  data.tar.gz: 15cd6f42ed2f5b0b995f20560cf9974d12e23bda
+  metadata.gz: f79ddbaecca21d91a17132b991c9f031b5a19b01
+  data.tar.gz: e6f3777a3a2fdc0fe14887c3d745dfd38e39c60a
 SHA512:
-  metadata.gz: 8758e09cd82e2f2fd50076811bf7a25961a2782f5ac6081185567ff8e874b1bb312aeaae546acfa17411a26f6c0af2c85ed47af99416cb7448ac4a65176e5815
-  data.tar.gz: 4ae98eedf0aafea25057b67a9d2f2f189f54c1063d00ecd70f90075c317171089e614bc3f00115f027037ca5a23eee51c0c7b3c75f6f8b150174792f1c0cf972
+  metadata.gz: 5193f5ee853ff6af02e119759bc8ec6a6c018f1a69bad34511757f62bd95918c2c3dc766eb3bc485ad47e65829a88e44da16e63a5238150314b9dca4f137e678
+  data.tar.gz: dc3313b3754851512e110247fd0c184c9130e9c2137f5c6adc39c62c173e9c6f2a3379b55b29dd2bf453c88e036722817a19455e72389e05e25f00e02da77585

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.2 (2014/01/20)
+
+Enhancement:
+
+* Add `tag_prefix` and `tag_suffix` placeholders. Thanks to [xthexder](https://github.com/xthexder). 
+
 ## 0.2.1 (2014/01/15)
 
 Enhancement:

data/README.md

@@ -67,7 +67,7 @@ Example:
 
 This results in same, but please note that following option parameters are reserved, so can not be used as a record key.
 
-## Parameters
+## Option Parameters
 
 - output_tag
 
@@ -98,11 +98,28 @@ The keys of input json are available as placeholders. In the above example,
 
 shall be available. In addition, following placeholders are reserved: 
 
-* ${hostname} hostname
-* ${tag} input tag
-* ${time} time of the event
-* ${tags[N]} input tag splitted by '.' (obsolete. use tag\_parts)
-* ${tag\_parts[N]} input tag splitted by '.' indexed with N such as `${tag_parts[0]}`, `${tag_parts[-1]}`. 
+* ${hostname} Hostname of the running machine
+* ${tag} Input tag
+* ${time} Time of the event
+* ${tags[N]} (Obsolete. Use tag\_parts) Input tag splitted by '.'
+* ${tag\_parts[N]} Input tag splitted by '.' indexed with N such as `${tag_parts[0]}`, `${tag_parts[-1]}`. 
+* ${tag\_prefix[N]} Tag parts before and on the index N. For example,
+
+        Input tag: prefix.test.tag.suffix
+        
+        ${tag_prefix[0]}  => prefix
+        ${tag_prefix[1]}  => prefix.test
+        ${tag_prefix[-2]} => prefix.test.tag
+        ${tag_prefix[-1]} => prefix.test.tag.suffix
+
+* ${tag\_suffix[N]} Tag parts after and on the index N. For example,
+
+        Input tag: prefix.test.tag.suffix
+    
+        ${tag_suffix[0]}  => prefix.test.tag.suffix
+        ${tag_suffix[1]}  => test.tag.suffix
+        ${tag_suffix[-2]} => tag.suffix
+        ${tag_suffix[-1]} => suffix
 
 It is also possible to write a ruby code in placeholders if you set `enable_ruby true` option, so you may write some codes as
 

data/fluent-plugin-record-reformer.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-record-reformer"
-  gem.version     = "0.2.1"
+  gem.version     = "0.2.2"
   gem.authors     = ["Naotoshi Seo"]
   gem.email       = "sonots@gmail.com"
   gem.homepage    = "https://github.com/sonots/fluent-plugin-record-reformer"

data/lib/fluent/plugin/out_record_reformer.rb

@@ -6,10 +6,6 @@ module Fluent
 
     def initialize
       super
-      # require utilities for placeholder
-      require 'pathname'
-      require 'uri'
-      require 'cgi'
     end
 
     config_param :output_tag, :string
@@ -42,6 +38,10 @@ module Fluent
 
       @placeholder_expander =
         if @enable_ruby
+          # require utilities which would be used in ruby placeholders
+          require 'pathname'
+          require 'uri'
+          require 'cgi'
           RubyPlaceholderExpander.new
         else
           PlaceholderExpander.new
@@ -59,19 +59,21 @@ module Fluent
 
     def emit(tag, es, chain)
       tag_parts = tag.split('.')
+      tag_prefix = tag_prefix(tag_parts)
+      tag_suffix = tag_suffix(tag_parts)
       es.each { |time, record|
-        new_tag, new_record = reform(@output_tag, record, tag, tag_parts, @time_proc.call(time))
+        new_tag, new_record = reform(@output_tag, record, tag, tag_parts, tag_prefix, tag_suffix, @time_proc.call(time))
         Engine.emit(new_tag, time, new_record)
       }
       chain.next
     rescue => e
-      $log.warn "record_reformer: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
+      $log.warn "record_reformer: #{e.class} #{e.message} #{e.backtrace.first}"
     end
 
     private
 
-    def reform(output_tag, record, tag, tag_parts, time)
-      @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @hostname, time)
+    def reform(output_tag, record, tag, tag_parts, tag_prefix, tag_suffix, time)
+      @placeholder_expander.prepare_placeholders(record, tag, tag_parts, tag_prefix, tag_suffix, @hostname, time)
       new_tag = @placeholder_expander.expand(output_tag)
 
       new_record = @renew_record ? {} : record.dup
@@ -81,11 +83,30 @@ module Fluent
       [new_tag, new_record]
     end
 
+    def tag_prefix(tag_parts)
+      return [] if tag_parts.empty?
+      tag_prefix = [tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
+      end
+      tag_prefix
+    end
+
+    def tag_suffix(tag_parts)
+      return [] if tag_parts.empty?
+      rev_tag_parts = tag_parts.reverse
+      rev_tag_suffix = [rev_tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
+      end
+      rev_tag_suffix.reverse
+    end
+
     class PlaceholderExpander
       # referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter, thanks!
       attr_reader :placeholders
 
-      def prepare_placeholders(record, tag, tag_parts, hostname, time)
+      def prepare_placeholders(record, tag, tag_parts, tag_prefix, tag_suffix, hostname, time)
         placeholders = {
           '${time}' => time,
           '${tag}' => tag,
@@ -93,17 +114,27 @@ module Fluent
         }
 
         size = tag_parts.size
-        tag_parts.each_with_index do |t, idx|
+
+        tag_parts.each_with_index { |t, idx|
           placeholders.store("${tag_parts[#{idx}]}", t)
           placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
-        end
-        # tags is just for old version compatibility
-        tag_parts.each_with_index do |t, idx|
+
+          # tags is just for old version compatibility
           placeholders.store("${tags[#{idx}]}", t)
           placeholders.store("${tags[#{idx-size}]}", t) # support tags[-1]
-        end
+        }
+
+        tag_prefix.each_with_index { |t, idx|
+          placeholders.store("${tag_prefix[#{idx}]}", t)
+          placeholders.store("${tag_prefix[#{idx-size}]}", t) # support tag_prefix[-1]
+        }
 
-        record.each {|k, v|
+        tag_suffix.each_with_index { |t, idx|
+          placeholders.store("${tag_suffix[#{idx}]}", t)
+          placeholders.store("${tag_suffix[#{idx-size}]}", t) # support tag_suffix[-1]
+        }
+
+        record.each { |k, v|
           placeholders.store("${#{k}}", v)
         }
 
@@ -111,10 +142,10 @@ module Fluent
       end
 
       def expand(str)
-        str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) do
-          $log.warn "record_reformer: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless @placeholders.include?($1)
+        str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
+          $log.warn "record_reformer: unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
           @placeholders[$1]
-        end
+        }
       end
     end
 
@@ -126,12 +157,16 @@ module Fluent
       # @param [Hash]   record      the record, one of information
       # @param [String] tag         the tag
       # @param [Array]  tag_parts   the tag parts (tag splitted by .)
+      # @param [Array]  tag_prefix  the tag prefix parts
+      # @param [Array]  tag_suffix  the tag suffix parts
       # @param [String] hostname    the hostname
       # @param [Time]   time        the time
-      def prepare_placeholders(record, tag, tag_parts, hostname, time)
+      def prepare_placeholders(record, tag, tag_parts, tag_prefix, tag_suffix, hostname, time)
         struct = UndefOpenStruct.new(record)
         struct.tag  = tag
         struct.tags = struct.tag_parts = tag_parts # tags is for old version compatibility
+        struct.tag_prefix = tag_prefix
+        struct.tag_suffix = tag_suffix
         struct.time = time
         struct.hostname = hostname
         @placeholders = struct

data/spec/out_record_reformer_spec.rb

@@ -10,7 +10,7 @@ describe Fluent::RecordReformerOutput do
     hostname ${hostname}
     tag ${tag}
     time ${time.strftime('%S')}
-    message ${hostname} ${tag_parts.last} ${message}
+    message ${hostname} ${tag_parts.last} ${URI.escape(message)}
   ]
   let(:tag) { 'test.tag' }
   let(:tag_parts) { tag.split('.') }
@@ -31,8 +31,8 @@ describe Fluent::RecordReformerOutput do
     let(:time) { Time.now }
     let(:emit) do
       driver.run do
-        driver.emit({'foo'=>'bar', 'message' => 1}, time.to_i)
-        driver.emit({'foo'=>'bar', 'message' => 2}, time.to_i)
+        driver.emit({'foo'=>'bar', 'message' => '1'}, time.to_i)
+        driver.emit({'foo'=>'bar', 'message' => '2'}, time.to_i)
       end
     end
 
@@ -176,5 +176,66 @@ describe Fluent::RecordReformerOutput do
       end
       it { emit }
     end
+
+    context '${tag_prefix[N]} and ${tag_suffix[N]}' do
+      let(:config) {%[
+        type reformed
+        output_tag ${tag_suffix[-2]}
+        enable_ruby no
+
+        hostname ${hostname}
+        tag ${tag}
+        time ${time}
+        message ${tag_prefix[1]} ${tag_prefix[-2]} ${tag_suffix[2]} ${tag_suffix[-3]} ${message}
+      ]}
+      let(:tag) { 'prefix.test.tag.suffix' }
+      let(:tag_parts) { tag.split('.') }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("tag.suffix", time.to_i, {
+          'foo' => 'bar',
+          'message' => "prefix.test prefix.test.tag tag.suffix test.tag.suffix 1",
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.to_i.to_s, # hmm, want to remove ${time} placeholder
+        })
+        Fluent::Engine.should_receive(:emit).with("tag.suffix", time.to_i, {
+          'foo' => 'bar',
+          'message' => "prefix.test prefix.test.tag tag.suffix test.tag.suffix 2",
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.to_i.to_s, # hmm, want to remove ${time} placeholder
+        })
+      end
+      it { emit }
+    end
+
+    context '${tag_prefix[N]} and ${tag_suffix[N]} with ruby enabled' do
+      let(:config) { CONFIG + %[
+        test_tag ${tag_prefix[1]} ${tag_prefix[-2]} ${tag_suffix[2]} ${tag_suffix[-3]}
+      ]}
+      let(:tag) { 'prefix.test.tag.suffix' }
+      let(:tag_parts) { tag.split('.') }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'foo' => 'bar',
+          'message' => "#{hostname} #{tag_parts.last} 1",
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.strftime('%S'),
+          'test_tag' => "prefix.test prefix.test.tag tag.suffix test.tag.suffix",
+        })
+        Fluent::Engine.should_receive(:emit).with("reformed.#{tag}", time.to_i, {
+          'foo' => 'bar',
+          'message' => "#{hostname} #{tag_parts.last} 2",
+          'hostname' => hostname,
+          'tag' => tag,
+          'time' => time.strftime('%S'),
+          'test_tag' => "prefix.test prefix.test.tag tag.suffix test.tag.suffix",
+        })
+      end
+      it { emit }
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-reformer
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-15 00:00:00.000000000 Z
+date: 2014-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd