fluent-plugin-yohoushi 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e999a6aedf4bd3f07f2a40298b9826da39a5dd68
-  data.tar.gz: abd025986465c4ee61f437f656a2439c6f12d3d0
+  metadata.gz: 4e938d9760ec8f7885633d08d9909f17274b9b42
+  data.tar.gz: ea2a1c8be04c55d11c0b8a15bee8712fa8166b7e
 SHA512:
-  metadata.gz: 7ba26bb0f27b83ac24b059e24989e12097b5cf8e5f50b039da0059e85952856820c5082f5d474fd1767180fa27363be9e553c7af308987ab2a5779a544d8954c
-  data.tar.gz: 48f200e029d13f71655c13bf18301616aa80eb66ce69c289a0af0f199e7542a902e641eebb013c9458f406180279fd3c3c9b2dfc495d2f78bcb705114f080f76
+  metadata.gz: 57efe427de701f149cc1932cf0daab759c36ae18021a15f5f1a7a52fc4e192bc08225d942eae37f0a6920d6ce888ed09be649c267b958f7a775be193415cae84
+  data.tar.gz: 2410dd562c892fee6ab12ec50c9f9c46542fd4f900ff0d3dde090ff433451e21372258a131516b208807b9cc1ace7d85e769040080daf369bd60d521c9a95966

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.0.4 (2014/01/25)
+
+Enhancement:
+
+* Add `enable_ruby false` option
+* Add `tag_prefix` and `tag_suffix` placeholders
+
 ## 0.0.3 (2013/12/12)
 
 Changes:

data/README.md

@@ -66,6 +66,10 @@ then fluent-plugin-yohoushi posts data to yohoshi similarly like
 
     The graph mode (either of gauge, count, modified, or derive). Just same as mode of GrowthForecast POST parameter. Default is gauge.
 
+- enable\_ruby *bool*
+
+    Enable to use ruby codes in placeholders. See Placeholders section. Default is true (just for lower version compatibility).
+
 ### Placeholders
 
 The keys of input json are available as placeholders. In the above example, 
@@ -76,17 +80,36 @@ The keys of input json are available as placeholders. In the above example,
 shall be available. In addition, following placeholders are reserved: 
 
 * ${hostname} hostname
-* ${tag} input tag
-* ${tag\_parts} input tag splitted by '.'
-* ${tags} input tag splitted by '.' (obsolete)
 * ${time} time of the event
 * ${key} the matched key value with `key_pattern` or `key1`, `key2`, ...
+* ${tag} input tag
+* ${tags[N]} (Obsolete. Use tag\_parts) Input tag splitted by '.'
+* ${tag\_parts[N]} Input tag splitted by '.' indexed with N such as `${tag_parts[0]}`, `${tag_parts[-1]}`. 
+* ${tag\_prefix[N]} Tag parts before and on the index N. For example,
 
-It is also possible to write a ruby code in placeholders, so you may write some codes as
+        Input tag: prefix.test.tag.suffix
+        
+        ${tag_prefix[0]}  => prefix
+        ${tag_prefix[1]}  => prefix.test
+        ${tag_prefix[-2]} => prefix.test.tag
+        ${tag_prefix[-1]} => prefix.test.tag.suffix
+
+* ${tag\_suffix[N]} Tag parts after and on the index N. For example,
+
+        Input tag: prefix.test.tag.suffix
+    
+        ${tag_suffix[0]}  => prefix.test.tag.suffix
+        ${tag_suffix[1]}  => test.tag.suffix
+        ${tag_suffix[-2]} => tag.suffix
+        ${tag_suffix[-1]} => suffix
+
+It is also possible to write a ruby code in placeholders if you set `enable_ruby true` option, so you may write some codes as
 
 * ${time.strftime('%Y-%m-%dT%H:%M:%S%z')}
 * ${tag\_parts.last}
 
+but, please note that enabling ruby codes is not encouraged by security reasons and also in terms of the performance.
+
 ## ChangeLog
 
 See [CHANGELOG.md](CHANGELOG.md) for details.
@@ -101,5 +124,5 @@ See [CHANGELOG.md](CHANGELOG.md) for details.
 
 ## Copyright
 
-Copyright (c) 2013 Naotoshi SEO. See [LICENSE](LICENSE) for details.
+Copyright (c) 2013 Naotoshi Seo. See [LICENSE](LICENSE) for details.
 

data/fluent-plugin-yohoushi.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-yohoushi"
-  s.version     = "0.0.3"
+  s.version     = "0.0.4"
   s.authors     = ["Naotoshi Seo"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-yohoushi"
-  s.summary     = "fluentd plugin to post data to yohoushi"
+  s.summary     = "Fluentd plugin to post data to yohoushi"
   s.description = s.summary
   s.licenses    = ["MIT"]
 

data/lib/fluent/plugin/out_yohoushi.rb

@@ -30,6 +30,7 @@ class Fluent::YohoushiOutput < Fluent::Output
       raise Fluent::ConfigError, "stdout output output_type should be `gauge`, `count`, `modified`, or `derive`"
     end
   end
+  config_param :enable_ruby, :bool, :default => true # true for lower version compatibility
 
   # for test
   attr_reader :client
@@ -70,6 +71,17 @@ class Fluent::YohoushiOutput < Fluent::Output
     end
     raise Fluent::ConfigError, "Either of `key_pattern` or `key1` must be specified" if (@key_pattern.nil? and @keys.empty?)
 
+    @placeholder_expander =
+      if @enable_ruby
+        # require utilities which would be used in ruby placeholders
+        require 'pathname'
+        require 'uri'
+        require 'cgi'
+        RubyPlaceholderExpander.new
+      else
+        PlaceholderExpander.new
+      end
+
     @hostname = Socket.gethostname
   rescue => e
     raise Fluent::ConfigError, "#{e.class} #{e.message} #{e.backtrace.first}"
@@ -95,11 +107,22 @@ class Fluent::YohoushiOutput < Fluent::Output
 
   def emit(tag, es, chain)
     tag_parts = tag.split('.')
+    tag_prefix = tag_prefix(tag_parts)
+    tag_suffix = tag_suffix(tag_parts)
+    placeholders = {
+      'tag' => tag,
+      'tags' => tag_parts, # for lower compatibility
+      'tag_parts' => tag_parts,
+      'tag_prefix' => tag_prefix,
+      'tag_suffix' => tag_suffix,
+      'hostname' => @hostname,
+    }
     if @key_pattern
       es.each do |time, record|
         record.each do |key, value|
           next unless key =~ @key_pattern
-          path = expand_placeholder(@key_pattern_path, record, tag, tag_parts, time, key)
+          placeholders['key'] = key
+          path = expand_placeholder(@key_pattern_path, time, record, placeholders)
           post(path, value)
         end
       end
@@ -107,7 +130,8 @@ class Fluent::YohoushiOutput < Fluent::Output
       es.each do |time, record|
         @keys.each do |key, path|
           next unless value = record[key]
-          path = expand_placeholder(path, record, tag, tag_parts, time, key)
+          placeholders['key'] = key
+          path = expand_placeholder(path, time, record, placeholders)
           post(path, value)
         end
       end
@@ -118,21 +142,87 @@ class Fluent::YohoushiOutput < Fluent::Output
     $log.warn "out_yohoushi: #{e.class} #{e.message} #{e.backtrace.first}"
   end
 
-  def expand_placeholder(str, record, tag, tag_parts, time, key)
-    struct = UndefOpenStruct.new(record)
-    struct.tag  = tag
-    struct.tags = tag_parts # obsolete
-    struct.tag_parts = tag_parts
-    struct.time = time
-    struct.key  = key
-    struct.hostname = @hostname
-    str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-    eval "\"#{str}\"", struct.instance_eval { binding }
+  def expand_placeholder(value, time, record, opts)
+    @placeholder_expander.prepare_placeholders(time, record, opts)
+    @placeholder_expander.expand(value)
+  end
+
+  def tag_prefix(tag_parts)
+    return [] if tag_parts.empty?
+    tag_prefix = [tag_parts.first]
+    1.upto(tag_parts.size-1).each do |i|
+      tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
+    end
+    tag_prefix
+  end
+
+  def tag_suffix(tag_parts)
+    return [] if tag_parts.empty?
+    rev_tag_parts = tag_parts.reverse
+    rev_tag_suffix = [rev_tag_parts.first]
+    1.upto(tag_parts.size-1).each do |i|
+      rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
+    end
+    rev_tag_suffix.reverse
+  end
+
+  class PlaceholderExpander
+    attr_reader :placeholders
+
+    def prepare_placeholders(time, record, opts)
+      placeholders = { '${time}' => time.to_s }
+      record.each {|key, val| placeholders.store("${#{key}}", val) }
+
+      opts.each do |key, val|
+        if val.kind_of?(Array)
+          size = val.size
+          val.each_with_index { |t, idx|
+            placeholders.store("${#{key}[#{idx}]}", t)
+            placeholders.store("${#{key}[#{idx-size}]}", t) # support [-1]
+          }
+        else # string, interger, float, and others?
+          placeholders["${#{key}}"] = val
+        end
+      end
+
+      @placeholders = placeholders
+    end
+
+    def expand(str)
+      str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
+        $log.warn "record_reformer: unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
+        @placeholders[$1]
+      }
+    end
   end
 
-  class UndefOpenStruct < OpenStruct
-    (Object.instance_methods).each do |m|
-      undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+  class RubyPlaceholderExpander
+    attr_reader :placeholders
+
+    # Get placeholders as a struct
+    #
+    # @param [Time]   time        the time
+    # @param [Hash]   record      the record
+    # @param [Hash]   opts        others
+    def prepare_placeholders(time, record, opts)
+      struct = UndefOpenStruct.new(record)
+      struct.time = time
+      opts.each {|key, val| struct.__send__("#{key}=", val) }
+      @placeholders = struct
+    end
+
+    # Replace placeholders in a string
+    #
+    # @param [String] str         the string to be replaced
+    def expand(str)
+      str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
+      eval "\"#{str}\"", @placeholders.instance_eval { binding }
+    end
+
+    class UndefOpenStruct < OpenStruct
+      (Object.instance_methods).each do |m|
+        undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+      end
     end
   end
 end

data/spec/out_yohoushi_spec.rb

@@ -139,21 +139,61 @@ describe Fluent::YohoushiOutput do
     end
   end
 
-  describe 'expand_placeholder' do
-    let(:config) { %[mapping1 / http://foo\nkey1 foo bar] }
+  describe 'expand_placeholder with enable_ruby true' do
+    let(:config) { %[mapping1 / http://foo] }
     let(:tag) { 'fluent.error' }
-    let(:record) { { 'foo_count' => "1", 'bar_count' => "1" } }
-    let(:tag_parts) { tag.split('.') }
     let(:time) { Time.now.to_i }
+    let(:record) { { 'foo_count' => "1" } }
+    let(:emit) { driver.run { driver.emit(record, time) } }
+    let(:expected_path)  { '/fluent/error/fluent.error/1/foo_count' }
+    let(:expected_value) { '1' }
+    before { driver.instance.should_receive(:post).with(expected_path, expected_value) }
 
-    context 'tags (obsolete)' do
-      let(:path) { '/path/to/${tags[-1]}' }
-      it { instance.expand_placeholder(path, record, tag, tag_parts, time, 'foo_count').should == '/path/to/error' }
+    context 'keyN' do
+      let(:config) {%[
+        mapping1 / http://foobar
+        key1 foo_count /${tags.first}/${tag_parts.last}/${tag_prefix[1]}/${foo_count}/${CGI.unescape(key)}
+        enable_ruby true
+      ]}
+      it { emit }
+    end
+
+    context 'key_pattern' do
+      let(:config) {%[
+        mapping1 / http://foobar
+        key_pattern _count$ /${tags.first}/${tag_parts.last}/${tag_prefix[1]}/${foo_count}/${URI.unescape(key)}
+        enable_ruby true
+      ]}
+      it { emit }
+    end
+  end
+
+  describe 'expand_placeholder with enable_ruby false' do
+    let(:config) { %[mapping1 / http://foo] }
+    let(:tag) { 'fluent.error' }
+    let(:time) { Time.now.to_i }
+    let(:record) { { 'foo_count' => "1" } }
+    let(:emit) { driver.run { driver.emit(record, time) } }
+    let(:expected_path)  { '/fluent/error/fluent.error/1/foo_count' }
+    let(:expected_value) { '1' }
+    before { driver.instance.should_receive(:post).with(expected_path, expected_value) }
+
+    context 'keyN' do
+      let(:config) {%[
+        mapping1 / http://foobar
+        key1 foo_count /${tags[0]}/${tag_parts[-1]}/${tag_prefix[1]}/${foo_count}/${key}
+        enable_ruby false
+      ]}
+      it { emit }
     end
 
-    context 'tag_parts' do
-      let(:path) { '/path/to/${tag_parts[-1]}' }
-      it { instance.expand_placeholder(path, record, tag, tag_parts, time, 'foo_count').should == '/path/to/error' }
+    context 'key_pattern' do
+      let(:config) {%[
+        mapping1 / http://foobar
+        key_pattern _count$ /${tags[0]}/${tag_parts[-1]}/${tag_prefix[1]}/${foo_count}/${key}
+        enable_ruby false
+      ]}
+      it { emit }
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-yohoushi
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-17 00:00:00.000000000 Z
+date: 2014-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -108,7 +108,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: fluentd plugin to post data to yohoushi
+description: Fluentd plugin to post data to yohoushi
 email:
 - sonots@gmail.com
 executables: []
@@ -151,7 +151,7 @@ rubyforge_project: fluent-plugin-yohoushi
 rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
-summary: fluentd plugin to post data to yohoushi
+summary: Fluentd plugin to post data to yohoushi
 test_files:
 - spec/out_yohoushi_spec.rb
 - spec/spec_helper.rb