fluent-plugin-multiprocess 0.1.0

data/README.md

@@ -0,0 +1,35 @@
+# Multiprocess agent plugin for Fluentd
+
+## multiprocess
+
+**multiprocess** agent plugin runs some child fluentd processes.
+
+
+### Configuration
+
+    <source>
+      type multiprocess
+
+      # optional:
+      #graceful_kill_interval 2s
+      #graceful_kill_interval_increment 3s
+      #graceful_kill_timeout 60s
+
+      <process>
+        cmdline -c /etc/fluent/fluentd_child1.conf
+      </process>
+      <process>
+        cmdline -c /etc/fluent/fluentd_child2.conf
+        sleep_before_start 5s
+      </process>
+      <process>
+        cmdline -c /etc/fluent/fluentd_child3.conf
+        sleep_before_shutdown 5s
+      </process>
+    </source>
+
+- **process** section sets command line arguments of a child process. This plugin creates one child process for each \<process\> section
+- **cmdline** option is required in a \<process\> section
+- **sleep\_before\_start** sets wait time before starting the process. Note that child processes **start from last to first** (`fluentd_child3` -\> `sleep 5` -\> { `fluentd_child2`, `fluentd_child1` } in this case)
+- **sleep\_before\_shutdown** sets wait time before shutting down the process. Note that child processes **shutdown from first to last** ({ `fluentd_child1`, `fluentd_child2` } -\> `sleep 5` -> `fluentd_child3` in this case)
+

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/example.conf

@@ -0,0 +1,10 @@
+<source>
+  type multiprocess
+  <process>
+    cmdline -c example_child1.conf
+  </process>
+  <process>
+    cmdline -c example_child2.conf
+    sleep_before_shutdown 10s
+  </process>
+</source>

data/example_child1.conf

@@ -0,0 +1,14 @@
+<source>
+  type gc_stat
+  emit_interval 3s
+  tag gc_stat
+</source>
+
+<match **>
+  type forward
+  flush_interval 1s
+  <server>
+    host 127.0.0.1
+    port 24001
+  </server>
+</match>

data/example_child2.conf

@@ -0,0 +1,10 @@
+<source>
+  type forward
+  port 24001
+  bind 127.0.0.1
+</source>
+
+<match **>
+  type stdout
+</match>
+

data/fluent-plugin-multiprocess.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-multiprocess"
+  gem.description = "Multiprocess agent plugin for Fluentd event collector"
+  gem.homepage    = "https://github.com/frsyuki/fluent-plugin-fluentd"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Sadayuki Furuhashi"]
+  gem.email       = "frsyuki@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+  gem.license = "Apache 2.0"
+
+  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency "serverengine", "~> 1.5.5"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/in_multiprocess.rb

@@ -0,0 +1,84 @@
+#
+# Fluent
+#
+# Copyright (C) 2013 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+  class MultiprocessInput < Input
+    Plugin.register_input('multiprocess', self)
+
+    require 'shellwords'
+    require 'serverengine'
+
+    config_param :graceful_kill_interval, :time, :default => 2
+    config_param :graceful_kill_interval_increment, :time, :default => 3
+    config_param :graceful_kill_timeout, :time, :default => 60
+
+    class ProcessElement
+      include Configurable
+
+      config_param :cmdline, :string
+      config_param :sleep_before_start, :time, :default => 0
+      config_param :sleep_before_shutdown, :time, :default => 0
+
+      attr_accessor :process_monitor
+    end
+
+    def configure(conf)
+      @processes = conf.elements.select {|e|
+        e.name == 'process'
+      }.map {|e|
+        pe = ProcessElement.new
+        pe.configure(e)
+        pe
+      }
+    end
+
+    def start
+      @pm = ServerEngine::ProcessManager.new(
+        :auto_tick => true,
+        :auto_tick_interval => 1,
+        :graceful_kill_interval => @graceful_kill_interval,
+        :graceful_kill_interval_increment => @graceful_kill_interval_increment,
+        :graceful_kill_timeout => @graceful_kill_timeout,
+        :graceful_kill_signal => 'TERM',
+        :immediate_kill_timeout => 0,  # disabled
+      )
+
+      plugin_rb = $LOADED_FEATURES.find {|x| x =~ /fluent\/plugin\.rb\z/ }
+      fluentd_rb = File.join(File.dirname(plugin_rb), 'command', 'fluentd.rb')
+
+      @processes.reverse_each do |pe|
+        cmd = "#{Shellwords.shellescape(RbConfig.ruby)} #{Shellwords.shellescape(fluentd_rb)} #{pe.cmdline}"
+        sleep pe.sleep_before_start if pe.sleep_before_start > 0
+        $log.info "launching child fluentd #{pe.cmdline}"
+        pe.process_monitor = @pm.spawn(cmd)
+      end
+    end
+
+    def shutdown
+      @processes.each {|pe|
+        sleep pe.sleep_before_shutdown if pe.sleep_before_shutdown > 0
+        $log.info "shutting down child fluentd #{pe.cmdline}"
+        pe.process_monitor.start_graceful_stop!
+      }
+      @processes.each {|pe|
+        pe.process_monitor.join
+      }
+    end
+  end
+
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-multiprocess
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: serverengine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Multiprocess agent plugin for Fluentd event collector
+email: frsyuki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- VERSION
+- example.conf
+- example_child1.conf
+- example_child2.conf
+- fluent-plugin-multiprocess.gemspec
+- lib/fluent/plugin/in_multiprocess.rb
+homepage: https://github.com/frsyuki/fluent-plugin-fluentd
+licenses:
+- Apache 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Multiprocess agent plugin for Fluentd event collector
+test_files: []
+has_rdoc: false