fluent-output-router 0.9.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    N2I3MzMzMDk2OWQ0MjIwZGQxZTFhOTcwNDE0YmJlZTA4ZjZkNjczZQ==
+  data.tar.gz: !binary |-
+    MDUzODc1NjM2MTEzMTA5MTM3OTQ0NzRlY2UxZjY3ZjVlN2ZhMDE1Zg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NTQ5NGM1YTU0ODYxNDJiMmQ2YjU1MDM4NWM5ODQxNTBiYWUzYTNhNTU1MDZm
+    ODZlYjdkOWU2MTVmZWMxZjBkMDhhZTUwYmQ0ZGJkNzEzMTQxNmExOTMwMDBl
+    YTQwYTYxODhkNDgyYTg5OWFlZWFhNWM2NzNmZWMzNWNlMDA5YjA=
+  data.tar.gz: !binary |-
+    MDg1OTVhZDA1YTg0Zjk1OWExZWNhNGQyODEwOGFhYjlkZTYxYmZkMjhlZTdh
+    YjFlZjFjNzJlNjYwZjE2MDgyNmVlMWMxMmFmY2NkOGY4ZWZhYzc1MjYzYTBl
+    MWViNDJiMGUzZGZiOGYwNzVhYzkyZGE4YTU1M2NmZDNjOTYyMWY=

data/lib/fluent/plugin/out_router.rb

@@ -0,0 +1,152 @@
+require 'erb'
+require 'rubygems'
+require 'json'
+
+module Fluent
+  class RouterOutput < Fluent::Output
+
+    Fluent::Plugin.register_output('router', self)
+    MIN_REAPER_INTERVAL = 5 #sec
+    ROUTER_STATE_FILE = "router_state.json"
+
+    def initialize
+      super
+      @outputs = {}
+      @last_used_time = {}
+      @last_reaper_run = Time.now
+    end
+
+    def configure(conf)
+      super
+      log "Starting the out_router plugin"
+      @inactivity_timeout = conf['inactivity_timeout'].to_i
+      @output_config = conf.elements.select { |e| e.name == 'config' }.first
+      mark_used @output_config
+      filename = File.join(File.dirname(Fluent::DEFAULT_CONFIG_PATH),
+                           ROUTER_STATE_FILE)
+      @state_serializer = ConsistentArrayStorage.new(filename)
+      output_keys = @state_serializer.load
+
+      # Restarts all output plugins for each stored key after a crash
+      # or a stop.
+      # This way we ensure every needed buffered output plugin, like
+      # the s3 plugin, will start after a crash and flush its buffer
+      # files to s3.
+      log "Reading file " + filename +
+        " and found output keys: " + output_keys.to_json
+      output_keys.each do |key|
+        start_output(key)
+      end
+    end
+
+    def emit(tag, es, chain)
+      # This is the router logic. Splitting tag on character '.'
+      # uses the value after '.' as the router key.
+      key = tag.split(/\./)[1]
+      out = @outputs[key]
+      unless out
+        out = start_output(key)
+      end
+      out.emit tag, es, chain
+      now = Time.now
+      @last_used_time[key] = now
+      run_reaper(now)
+    end
+
+    private
+
+    def start_output(key)
+      log "Starting a new output of type " + @output_config['type'] +
+        " for key #{key}"
+      @outputs[key] = Fluent::Plugin.new_output @output_config['type']
+      @last_used_time[key] = Time.now
+      config = @output_config.clone
+      template! config, :key => key
+      out = @outputs[key]
+      out.configure config
+      out.start
+      @state_serializer.store(@outputs.keys)
+      out
+    end
+
+    def stop_output(key)
+      log "Stopping output of type " + @output_config['type'] +
+        " for key #{key} due to inactivity (no data comming in)"
+      out = @outputs.delete(key)
+      @last_used_time.delete(key)
+      out.shutdown
+      @state_serializer.store(@outputs.keys)
+    end
+
+    def mark_used(conf)
+      conf.used = conf.keys
+      conf.elements.each { |e| mark_used e }
+    end
+
+    # MUTABLE. WHICH ALSO MEANS IT IS TERRIBLE.
+    def template!(conf, keys)
+      conf.each do |k,v|
+        case v
+        when Hash
+          conf[k] = template v, keys
+        when String
+          conf[k] = ERB.new(v).result(binding)
+        else
+          $log.error "Template error #{v.inspect}"
+          $log.error_backtrace
+        end
+      end
+    end
+
+    def log(string)
+      $log.info "out_router: " + string
+    end
+
+    # This is the reaper that is used to avoids resource leaks
+    # It will stop any active output plugin that does not get any input
+    def run_reaper(now)
+      if now - @last_reaper_run > MIN_REAPER_INTERVAL then
+        @last_used_time.each {|key, stored_time|
+          if now - stored_time > @inactivity_timeout then
+            stop_output(key)
+          end }
+         @last_reaper_run = now
+      end
+    end
+  end
+
+    # ConsistentArrayStorage serializes/deserializes an array to disc.
+    # It uses two files.
+    # First it writes the serialized array into a tmp file and then renames
+    # the tmp file to the correct filename.
+    # The rename action is fast and Linux gives a guarantee that
+    # the rename operation is atomic (on local discs). This removes the risk
+    # of any kind of crash leaving the @store_file file in a bad state.
+    class ConsistentArrayStorage
+      def initialize(filename)
+        @store_file = filename
+        @temp_file = filename + ".tmp"
+      end
+
+      def store(array)
+        file = File.open(@temp_file, 'w')
+        file << array.to_json
+        file.close
+        File.rename(@temp_file, @store_file)
+      end
+
+      #Returns the array stored on disc.
+      #Returns an empty array if the file does not exist
+      def load
+        if File.exists?(@store_file) then
+          file = File.open(@store_file, 'r')
+          json_string = file.read
+          file.close
+          JSON.parse(json_string)
+        else
+          []
+        end
+      end
+    end
+
+end

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification
+name: fluent-output-router
+version: !ruby/object:Gem::Version
+  version: 0.9.1
+platform: ruby
+authors:
+- David Almroth
+- Torbjörn Norinder
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+description: Depending on the message tag, this fluent plugin can start new instances
+  of any other available output plugins for fluent (for example the fluent-plugin-s3).
+  This way it is possible to have one fluent input get split into separate (s3) output
+  instances. The output instances will be shut down individually when there is no
+  traffic to an instance.
+email: david@campanja.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fluent/plugin/out_router.rb
+homepage: https://github.com/campanja/fluent-output-router
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.7
+signing_key: 
+specification_version: 4
+summary: Can dynamically route fluentd messages based on the tag content
+test_files: []