fluent-plugin-hash-forward 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 34a4f0ca25be43b69b64a1338daabc8372a2008b
+  data.tar.gz: be4337ab0434d3fe45be14de13dc381580d8d49d
+SHA512:
+  metadata.gz: e037f72f0fa5fbc9980e11a7a9ec94a336488b05dc45eeb69589182fa404a5c02921ae210277b0288e20eaf60002102a7e18d1051834830851a7b2af729ac0a4
+  data.tar.gz: 7a20b3102b6a98e193608b2f21aa45b0ebd5b5dfef0222a39ef8ad50f6a01ad4336a5ad4d63539ae96d6b601f47398969ea540dc471a69026ee6029131ac13e1

data/.gitignore

@@ -0,0 +1,14 @@
+/*.gem
+~*
+#*
+*~
+.bundle
+Gemfile.lock
+.rbenv-version
+vendor
+doc/*
+tmp/*
+.yardoc
+pck/*
+.ruby-version
+pkg/*

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,60 @@
+# fluent-plugin-hash-forward
+
+Fluentd plugin to keep forwarding messages of a specific tag pattern to a specific node
+
+- Forward some servers
+- Same tag messages forward to the same server
+    - Using Murmurhash
+
+## Configuration
+
+Example:
+
+    <match pattern>
+      type hash_forward
+      flush_interval 1s
+    
+      <server>
+        host 192.168.1.3
+        port 24224
+      </server>
+      <server>
+        host 192.168.1.4
+        port 24224
+      </server>
+    
+      <secondary>
+        type file
+        path /var/log/fluent/forward-failed
+      </secondary>
+    </match>
+
+## Parameters
+
+Basically same with out\_forward plugin. See [http://docs.fluentd.org/articles/out_forward](http://docs.fluentd.org/ja/articles/out_forward). 
+
+Following parameters are additionally available:
+
+* hash\_key
+
+    Specify a placeholder string to be used as a key for hashing. See Placeholders section for more details. Default uses `${tag}`as a hash key.
+
+### Placeholders
+
+You can use following placeholders:
+
+* ${tag} input tag
+* ${tags} input tag splitted by '.'
+
+It is also possible to write a ruby code in placeholders, so you may write some codes as
+
+* ${tags[0]}
+* ${tags.last}
+
+## Copyright
+
+* Copyright
+  * Copyright (c) 2012- Ryosuke IWANAGA (riywo)
+  * Copyright (c) 2013- Naotoshi SEO (sonots)
+* License
+  * Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+task :default => :spec
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+    sh 'irb -rubygems -I lib'
+end
+task :c => :console

data/fluent-plugin-hash-forward.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-hash-forward"
+  s.version     = "0.0.1"
+  s.authors     = ["Ryosuke IWANAGA", "Naotoshi SEO"]
+  s.email       = ["riywo.jp@gmail.com", "sonots@gmail.com"]
+  s.homepage    = "https://github.com/riywo/fluent-plugin-hash-forward"
+  s.summary     = %q{Fluentd plugin to keep forwarding messsages of a specific tag pattern to a specific node}
+  s.description = s.summary
+  s.licenses    = ["MIT"]
+
+  s.rubyforge_project = "fluent-plugin-hash-forward"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "murmurhash3"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "pry"
+  s.add_development_dependency "pry-nav"
+end

data/lib/fluent/plugin/out_hash_forward.rb

@@ -0,0 +1,79 @@
+require 'fluent/plugin/out_forward'
+
+class Fluent::HashForwardOutput < Fluent::ForwardOutput
+  Fluent::Plugin.register_output('hash_forward', self)
+
+  config_param :hash_key, :string, :default => nil
+
+  def configure(conf)
+    super
+    @standby_nodes, @regular_nodes = @nodes.partition {|n| n.standby? }
+  end
+
+  attr_reader :regular_nodes
+  attr_reader :standby_nodes
+
+  # Override
+  def write_objects(tag, chunk)
+    return if chunk.empty?
+
+    error = nil
+
+    nodes = nodes(tag)
+
+    # below is just copy from out_forward
+    nodes.each do |node|
+      if node.available?
+        begin
+          send_data(node, tag, chunk)
+          return
+        rescue
+          # for load balancing during detecting crashed servers
+          error = $!  # use the latest error
+        end
+      end
+    end
+
+    if error
+      raise error
+    else
+      raise "no nodes are available"  # TODO message
+    end
+  end
+
+  # Override: I don't use weight
+  def rebuild_weight_array
+  end
+
+  # Get nodes (a regular_node and a standby_node if available) using hash algorithm
+  def nodes(tag)
+    hash_key = @hash_key ? expand_placeholder(@hash_key, tag) : tag
+    regular_index = get_index(hash_key, regular_nodes.size)
+    standby_index = standby_nodes.size > 0 ? get_index(hash_key, standby_nodes.size) : 0
+    [regular_nodes[regular_index], standby_nodes[standby_index]].compact
+  end
+
+  # hashing(key) mod N
+  def get_index(key, size)
+    require 'murmurhash3'
+    MurmurHash3::V32.str_hash(key) % size
+  end
+
+  # Replace ${tag} and ${tags} placeholders in a string
+  #
+  # @param [String] str    the string to be expanded
+  # @param [String] tag    tag of a message
+  def expand_placeholder(str, tag)
+    struct = UndefOpenStruct.new
+    struct.tag  = tag
+    struct.tags = tag.split('.')
+    str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
+    eval "\"#{str}\"", struct.instance_eval { binding }
+  end
+
+  class UndefOpenStruct < OpenStruct
+    (Object.instance_methods).each do |m|
+      undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+    end
+  end
+end

data/spec/out_hash_forward_spec.rb

@@ -0,0 +1,114 @@
+# encoding: UTF-8
+require_relative 'spec_helper'
+
+describe Fluent::HashForwardOutput do
+  before { Fluent::Test.setup }
+  CONFIG = %[
+    type hash_forward
+    flush_interval 1s
+
+    <server>
+      host 192.168.1.3
+      port 24224
+    </server>
+    <server>
+      host 192.168.1.4
+      port 24224
+    </server>
+
+    <secondary>
+      type file
+      path /var/log/fluent/forward-failed
+    </secondary>
+  ]
+  let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::HashForwardOutput, tag).configure(config) }
+
+  describe 'test configure' do
+    let(:tag) { 'test.tag' }
+    let(:config) { CONFIG }
+    context 'default behavior' do
+      it { lambda{ driver }.should_not raise_error }
+    end
+  end
+
+  describe 'test hashing' do
+    let(:tag) { 'test.tag' }
+    let(:config) { CONFIG }
+
+    context 'test consistency' do
+      before do
+        @node = driver.instance.nodes(tag).first
+      end
+      it 'should forward to the same node' do
+        expect(driver.instance.nodes(tag).first).to eq(@node)
+      end
+    end
+
+    context 'test distribution' do
+      let(:tag1) { 'test.tag1' }
+      let(:tag2) { 'test.tag2' }
+      before do
+        MurmurHash3::V32.stub(:str_hash).with(tag1).and_return(0)
+        MurmurHash3::V32.stub(:str_hash).with(tag2).and_return(1)
+        @node1 = driver.instance.nodes(tag1).first
+      end
+      it 'should forward to the different node' do
+        expect(driver.instance.nodes(tag2).first).not_to eq(@node1)
+      end
+    end
+
+    context 'test hash_key' do
+      let(:tag1) { 'test.tag1' }
+      let(:tag2) { 'test.tag2' }
+      let(:config) { CONFIG + %[hash_key ${tags[0..-2]}] }
+      before do
+        @node1 = driver.instance.nodes(tag1).first
+      end
+      it 'should forward to the different node' do
+        expect(driver.instance.nodes(tag2).first).to eq(@node1)
+      end
+    end
+  end
+
+  describe 'test emit' do
+    let(:tag) { 'test.tag' }
+    let(:time) { Time.now.to_i }
+    let(:es) { Array.new(1) }
+    let(:chain) { Fluent::NullOutputChain.instance }
+    let(:config) { CONFIG }
+
+    context 'default behavior' do
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        node = driver.instance.nodes(tag).first
+        driver.instance.should_receive(:send_data).with(node, tag, es)
+      end
+      it 'should forward' do
+        driver.instance.write_objects(tag, es)
+      end
+    end
+
+    context 'test standby' do
+      let(:config) {
+        CONFIG + %[
+        <server>
+          host 192.168.1.5
+          port 24224
+          standby true
+        </server>
+        ]
+      }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        regular_node = driver.instance.nodes(tag)[0]
+        standby_node = driver.instance.nodes(tag)[1]
+        regular_node.stub(:available?).and_return(false) # stub as regular node is not available
+        driver.instance.should_receive(:send_data).with(standby_node, tag, es)
+      end
+      it 'should forward to the standby node if regular node is not available' do
+        driver.instance.write_objects(tag, es)
+      end
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+# encoding: UTF-8
+require 'rubygems'
+require 'bundler'
+Bundler.setup(:default, :test)
+Bundler.require(:default, :test)
+
+require 'fluent/test'
+require 'rspec'
+require 'pry'
+
+$TESTING=true
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'fluent/plugin/out_hash_forward'

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-hash-forward
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ryosuke IWANAGA
+- Naotoshi SEO
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: murmurhash3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to keep forwarding messsages of a specific tag pattern
+  to a specific node
+email:
+- riywo.jp@gmail.com
+- sonots@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-hash-forward.gemspec
+- lib/fluent/plugin/out_hash_forward.rb
+- spec/out_hash_forward_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/riywo/fluent-plugin-hash-forward
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: fluent-plugin-hash-forward
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to keep forwarding messsages of a specific tag pattern to
+  a specific node
+test_files:
+- spec/out_hash_forward_spec.rb
+- spec/spec_helper.rb