fluent-plugin-hash-forward 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 34a4f0ca25be43b69b64a1338daabc8372a2008b
-  data.tar.gz: be4337ab0434d3fe45be14de13dc381580d8d49d
+  metadata.gz: 38bfa4c2b8ecb078ef39859c02f535d445f6abc5
+  data.tar.gz: 14a48e69abed66f8ed07ae185286d82d91c3aa21
 SHA512:
-  metadata.gz: e037f72f0fa5fbc9980e11a7a9ec94a336488b05dc45eeb69589182fa404a5c02921ae210277b0288e20eaf60002102a7e18d1051834830851a7b2af729ac0a4
-  data.tar.gz: 7a20b3102b6a98e193608b2f21aa45b0ebd5b5dfef0222a39ef8ad50f6a01ad4336a5ad4d63539ae96d6b601f47398969ea540dc471a69026ee6029131ac13e1
+  metadata.gz: 993925c2ddb9e8909c7f1bd508f14b6d0ab22b63d4c267b01808d8bd9c3dd4789b871cc7388e20db5d132469fe19c8d00749a37b116e6d2841572c024c0ef441
+  data.tar.gz: e738d73bdc48f65f64e23287ba1fa8c941a6a5d777686aff5f6c7c82e144919494e9ec925f130cbd4f050420ac744c846fe9b86ebff5bd913347f3bdfb05a2c5

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# 0.0.2 (2013/10/31)
+
+Changes
+
+* Change `hash_key` option to `hash_key_slice` option. Stopped to use a placeholder. 
+
+# 0.0.1 (2013/10/31)
+
+* Initial release
+

data/README.md

@@ -35,21 +35,32 @@ Basically same with out\_forward plugin. See [http://docs.fluentd.org/articles/o
 
 Following parameters are additionally available:
 
-* hash\_key
+* hash\_key\_slice *min*..*max*
 
-    Specify a placeholder string to be used as a key for hashing. See Placeholders section for more details. Default uses `${tag}`as a hash key.
+    Use sliced `tag` as a hash key to determine a forwarding node. Default: use entire `tag`. 
 
-### Placeholders
+    For example, assume tags of input messages are like
 
-You can use following placeholders:
+        foo.bar.host1
+        foo.bar.host2
 
-* ${tag} input tag
-* ${tags} input tag splitted by '.'
+    but, you want to forward these messages to the same node, configure like
 
-It is also possible to write a ruby code in placeholders, so you may write some codes as
+        hash_key_slice 0..-2
 
-* ${tags[0]}
-* ${tags.last}
+    then, hash\_key becomes as `foo.bar` which results in forwarding these messages to the same node.
+
+    FYI: This option behaves like `tag.split('.').slice(min..max)`.
+
+## ToDo
+
+* Consistent hashing
+
+   * Consistent hashing is useful on adding or removing nodes dynamically, but currently `out_hash_forward` does not support such a dynamical feature, so consistent hashing is just useless now. To effectively support consistent hashing, this plugin must support ways to add or remove nodes dynamically by preparing http api or reading nodes information from redis or memcached, etc. 
+
+## ChangeLog
+
+See [CHANGELOG.md](CHANGELOG.md) for details. 
 
 ## Copyright
 

data/fluent-plugin-hash-forward.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-hash-forward"
-  s.version     = "0.0.1"
+  s.version     = "0.0.2"
   s.authors     = ["Ryosuke IWANAGA", "Naotoshi SEO"]
   s.email       = ["riywo.jp@gmail.com", "sonots@gmail.com"]
   s.homepage    = "https://github.com/riywo/fluent-plugin-hash-forward"

data/lib/fluent/plugin/out_hash_forward.rb

@@ -3,15 +3,28 @@ require 'fluent/plugin/out_forward'
 class Fluent::HashForwardOutput < Fluent::ForwardOutput
   Fluent::Plugin.register_output('hash_forward', self)
 
-  config_param :hash_key, :string, :default => nil
+  config_param :hash_key_slice, :string, :default => nil
 
   def configure(conf)
     super
     @standby_nodes, @regular_nodes = @nodes.partition {|n| n.standby? }
+
+    if @hash_key_slice
+      lindex, rindex = @hash_key_slice.split('..', 2)
+      if lindex.nil? or rindex.nil? or lindex !~ /^-?\d+$/ or rindex !~ /^-?\d+$/
+        raise Fluent::ConfigError, "out_hash_forard: hash_key_slice must be formatted like [num]..[num]"
+      else
+        @hash_key_slice_lindex = lindex.to_i
+        @hash_key_slice_rindex = rindex.to_i
+      end
+    end
   end
 
+  # for test
   attr_reader :regular_nodes
   attr_reader :standby_nodes
+  attr_accessor :hash_key_slice_lindex
+  attr_accessor :hash_key_slice_rindex
 
   # Override
   def write_objects(tag, chunk)
@@ -47,7 +60,7 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
 
   # Get nodes (a regular_node and a standby_node if available) using hash algorithm
   def nodes(tag)
-    hash_key = @hash_key ? expand_placeholder(@hash_key, tag) : tag
+    hash_key = @hash_key_slice ? perform_hash_key_slice(tag) : tag
     regular_index = get_index(hash_key, regular_nodes.size)
     standby_index = standby_nodes.size > 0 ? get_index(hash_key, standby_nodes.size) : 0
     [regular_nodes[regular_index], standby_nodes[standby_index]].compact
@@ -59,21 +72,9 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
     MurmurHash3::V32.str_hash(key) % size
   end
 
-  # Replace ${tag} and ${tags} placeholders in a string
-  #
-  # @param [String] str    the string to be expanded
-  # @param [String] tag    tag of a message
-  def expand_placeholder(str, tag)
-    struct = UndefOpenStruct.new
-    struct.tag  = tag
-    struct.tags = tag.split('.')
-    str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-    eval "\"#{str}\"", struct.instance_eval { binding }
-  end
-
-  class UndefOpenStruct < OpenStruct
-    (Object.instance_methods).each do |m|
-      undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
-    end
+  def perform_hash_key_slice(tag)
+    tags = tag.split('.')
+    sliced = tags[@hash_key_slice_lindex..@hash_key_slice_rindex]
+    return sliced.nil? ? "" : sliced.join('.')
   end
 end

data/spec/out_hash_forward_spec.rb

@@ -29,6 +29,73 @@ describe Fluent::HashForwardOutput do
     context 'default behavior' do
       it { lambda{ driver }.should_not raise_error }
     end
+
+    describe 'bad hash_key_slice' do
+     context 'string' do
+       let(:config) { CONFIG + %[hash_key_slice a..b] }
+       it { lambda{ driver }.should raise_error(Fluent::ConfigError) }
+     end
+
+     context 'no rindex' do
+       let(:config) { CONFIG + %[hash_key_slice 0..] }
+       it { lambda{ driver }.should raise_error(Fluent::ConfigError) }
+     end
+
+     context 'no lindex' do
+       let(:config) { CONFIG + %[hash_key_slice ..1] }
+       it { lambda{ driver }.should raise_error(Fluent::ConfigError) }
+     end
+
+     context 'bad format' do
+       let(:config) { CONFIG + %[hash_key_slice 0,1] }
+       it { lambda{ driver }.should raise_error(Fluent::ConfigError) }
+     end
+    end
+  end
+
+  describe 'test perform_hash_key_slice' do
+    # actually the same behavior with ruby Array, so just conforming ruby Array#slice
+    let(:tag) { 'tag0.tag1' }
+    context 'larger than tags size' do
+      let(:config) { CONFIG + %[hash_key_slice 1..10] }
+      it { expect(driver.instance.perform_hash_key_slice(tag)).to eq('tag1') }
+    end
+
+    context 'rindex is smaller than lindex' do
+      let(:config) { CONFIG + %[hash_key_slice 1..0] }
+      it { expect(driver.instance.perform_hash_key_slice(tag)).to eq('') }
+    end
+
+    context 'rindex is -1' do
+      let(:config) { CONFIG + %[hash_key_slice 0..-1] }
+      it { expect(driver.instance.perform_hash_key_slice(tag)).to eq(tag) }
+    end
+
+    context 'rindex is -2' do
+      let(:config) { CONFIG + %[hash_key_slice 0..-2] }
+      it { expect(driver.instance.perform_hash_key_slice(tag)).to eq('tag0') }
+    end
+
+    context 'rindex is large negative integer' do
+      let(:config) { CONFIG + %[hash_key_slice 0..-10] }
+      it { expect(driver.instance.perform_hash_key_slice(tag)).to eq('') }
+    end
+
+    context 'lindex is -1' do
+      let(:config) { CONFIG + %[hash_key_slice -1..10] }
+      it { expect(driver.instance.perform_hash_key_slice(tag)).to eq('tag1') }
+    end
+
+    context 'lindex is -2' do
+      let(:config) { CONFIG + %[hash_key_slice -2..10] }
+      it { expect(driver.instance.perform_hash_key_slice(tag)).to eq(tag) }
+    end
+
+    context 'lindex is large negatize integer' do
+      # this behavior looks wierd for me
+      let(:config) { CONFIG + %[hash_key_slice -3..10] }
+      it { expect(driver.instance.perform_hash_key_slice(tag)).to eq('') }
+    end
   end
 
   describe 'test hashing' do
@@ -57,10 +124,10 @@ describe Fluent::HashForwardOutput do
       end
     end
 
-    context 'test hash_key' do
+    context 'test hash_key_slice' do
       let(:tag1) { 'test.tag1' }
       let(:tag2) { 'test.tag2' }
-      let(:config) { CONFIG + %[hash_key ${tags[0..-2]}] }
+      let(:config) { CONFIG + %[hash_key_slice 0..-2] }
       before do
         @node1 = driver.instance.nodes(tag1).first
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-hash-forward
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Ryosuke IWANAGA
@@ -105,6 +105,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - Gemfile
 - README.md
 - Rakefile