fluent-plugin-querycombiner 0.0.0.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 432eccbf19e7b303cf6457b3560a43779b7051b3
+  data.tar.gz: 8a3c7c461b1db05ff3968d49f46c1dcd694b7e7a
+SHA512:
+  metadata.gz: 00f8c3f21610a21e5ad6c6887377fef64c70186cd73c1e84813240581ef9f7b2d68c140fa15543052eea7134b7086ac8f6903bb7472fc077aaa2d18fac8de5c3
+  data.tar.gz: a8bdcf68fe8adfb36cda540c8f691829e73476a38713f5105ddd3bf18084e5a980b162f863c696ee055d0a37cc2fc362301946e94d29f606e2717eadf660c769

data/.gitignore

@@ -0,0 +1,26 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*~
+\#*
+.\#*
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-querycombiner.gemspec
+gemspec

data/README.md

@@ -0,0 +1,70 @@
+fluent-plugin-querycombiner
+===========================
+This fluentd output plugin helps you to combine multiple queries.
+
+This plugin is based on [fluent-plugin-onlineuser](https://github.com/y-lan/fluent-plugin-onlineuser) written by [Yuyang Lan](https://github.com/y-lan).
+
+
+## Requirement
+  * a running Redis
+
+
+## Get started
+
+```
+<match combiner.**>
+  type query_combiner
+  tag combined.test
+
+  flush_interval  0.5
+
+  host            localhost
+  port            6379
+  db_index        0
+  redis_retry     3
+
+  query_identify  session-id, task-id
+  query_ttl       3   # sec
+  buffer_size     10   # queries
+
+  <catch>
+    condition     status == 'recog-init'
+    replace       time => time_init, status => status_init
+  </catch>
+
+  <prolong>
+    condition     status == 'recog-break'
+  </prolong>
+
+  <dump>
+    condition     status == 'recog-finish'
+    replace       time => time_finish, result => result_finish, status => status_finish
+  </dump>
+
+  <release>
+    condition     status == 'recog-error'
+  </release>
+
+</match>
+```
+
+## Configuration
+#### host, port, db_index
+The basic information for connecting to Redis. By default it's **redis://127.0.0.1:6379/0**
+
+#### redis_retry
+How many times should the plugin retry when performing a redis operation before raising a error.
+By default it's 3.
+
+### session_timeout
+The inactive expire time in seconds. By default it's 1800 (30 minutes).
+
+
+### tag
+The tag prefix for emitted event messages. By default it's `query_combiner`.
+
+## Copyright
+
+Copyright:: Copyright (c) 2014- Takahiro Kamatani
+
+License:: Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-querycombiner.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-querycombiner"
+  spec.version       = "0.0.0.pre"
+  spec.authors       = ["Takahiro Kamatani"]
+  spec.email         = ["buhii314@gmail.com"]
+  spec.description   = %q{Fluent plugin to combine multiple queries.}
+  spec.summary       = spec.description
+  spec.homepage      = "https://github.com/buhii/fluent-plugin-querycombiner"
+  spec.license       = "Apache License, Version 2.0"
+  spec.has_rdoc      = false
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "fluentd", "~> 0.10.0"
+  spec.add_runtime_dependency "redis"
+end

data/lib/fluent/plugin/out_query_combiner.rb

@@ -0,0 +1,259 @@
+# -*- coding: utf-8 -*-
+
+module Fluent
+  class QueryCombinerOutput < BufferedOutput
+    Fluent::Plugin.register_output('query_combiner', self)
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :integer, :default => 6379
+    config_param :db_index, :integer, :default => 0
+    config_param :redis_retry, :integer, :default => 3
+
+    config_param :redis_key_prefix, :string, :default => 'query_combiner:'
+    config_param :query_identify, :string, :default => 'session-id'
+    config_param :query_ttl, :integer, :default => 1800
+    config_param :buffer_size, :integer, :default => 100
+
+    config_param :flush_interval, :integer, :default => 60
+    config_param :remove_interval, :integer, :default => 10
+    config_param :tag, :string, :default => "query_combiner"
+
+    def initialize
+      super
+      require 'redis'
+      require 'msgpack'
+      require 'json'
+      require 'rubygems'
+    end
+
+    def configure(conf)
+      super
+      @host = conf.has_key?('host') ? conf['host'] : 'localhost'
+      @port = conf.has_key?('port') ? conf['port'].to_i : 6379
+      @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
+
+      @query_identify = @query_identify.split(',').map { |qid| qid.strip }
+
+      # Create functions for each conditions
+      @_cond_funcs = {}
+      @_replace_keys = {}
+
+      def get_arguments(eval_str)
+        eval_str.scan(/[\"\']?[a-zA-Z][\w\d\.\-\_]*[\"\']?/).uniq.select{|x|
+          not (x.start_with?('\'') or x.start_with?('\"')) and \
+          not %w{and or xor not}.include? x
+        }
+      end
+
+      def parse_replace_expr(element_name, condition_name, str)
+        result = {}
+        str.split(',').each{|cond|
+          before, after = cond.split('=>').map{|var| var.strip}
+          result[before] = after
+          if not (before.length > 0 and after.length > 0)
+            raise Fluent::ConfigError, "SyntaxError at replace condition `#{element_name}`: #{condition_name}"
+          end
+        }
+        if result.none?
+          raise Fluent::ConfigError, "SyntaxError at replace condition `#{element_name}`: #{condition_name}"
+        end
+        result
+      end
+
+      def create_func(var, expr)
+        begin
+          f_argv = get_arguments(expr)
+          f = eval('lambda {|' + f_argv.join(',') + '| ' + expr + '}')
+          return [f, f_argv]
+        rescue SyntaxError
+          raise Fluent::ConfigError, "SyntaxError at condition `#{var}`: #{expr}"
+        end
+      end
+      conf.elements.select { |element|
+        %w{catch prolong dump release}.include? element.name
+      }.each { |element|
+        element.each_pair { |var, expr|
+          element.has_key?(var)   # to suppress unread configuration warning
+
+          if var == 'condition'
+            formula, f_argv = create_func(var, expr)
+            @_cond_funcs[element.name] = [f_argv, formula]
+
+          elsif var == 'replace'
+            if %w{catch dump}.include? element.name
+              @_replace_keys[element.name] = parse_replace_expr(element.name, var, expr)
+            else
+              raise Fluent::ConfigError, "`replace` configuration in #{element.name}: only allowed in `catch` and `dump`"
+            end
+          end
+        }
+      }
+    end
+
+    def has_all_keys?(record, argv)
+      argv.each {|var|
+        if not record.has_key?(var)
+          return false
+        end
+      }
+      true
+    end
+
+    def exec_func(record, f_argv, formula)
+      argv = []
+      f_argv.each {|v|
+        argv.push(record[v])
+      }
+      return formula.call(*argv)
+    end
+
+    def start
+      super
+
+      begin
+        gem "hiredis"
+        @redis = Redis.new(
+            :host => @host, :port => @port, :driver => :hiredis,
+            :thread_safe => true, :db => @db_index
+        )
+      rescue LoadError
+        @redis = Redis.new(
+            :host => @host, :port => @port,
+            :thread_safe => true, :db => @db_index
+        )
+      end
+
+      start_watch
+    end
+
+    def shutdown
+      @redis.quit
+    end
+
+    def tryOnRedis(method, *args)
+      tries = 0
+      begin
+        @redis.send(method, *args) if @redis.respond_to? method
+      rescue Redis::CommandError => e
+        tries += 1
+        # retry 3 times
+        retry if tries <= @redis_retry
+        $log.warn %Q[redis command retry failed : #{method}(#{args.join(', ')})]
+        raise e.message
+      end
+    end
+
+    def start_watch
+      @watcher = Thread.new(&method(:watch))
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def do_catch(qid, record, time)
+      # replace record keys
+      @_replace_keys['catch'].each_pair { |before, after|
+        record[after] = record[before]
+        record.delete(before)
+      }
+      # save record
+      tryOnRedis 'set',    @redis_key_prefix + qid, JSON.dump(record)
+      # update qid's timestamp
+      tryOnRedis 'zadd', @redis_key_prefix, time, qid
+      tryOnRedis 'expire', @redis_key_prefix + qid, @query_ttl
+    end
+
+    def do_prolong(qid, time)
+      if (tryOnRedis 'exists', @redis_key_prefix + qid)
+        # update qid's timestamp
+        tryOnRedis 'zadd', @redis_key_prefix, time, qid
+        tryOnRedis 'expire', @redis_key_prefix + qid, @query_ttl
+      end
+    end
+
+    def do_dump(qid, record)
+      if (tryOnRedis 'exists', @redis_key_prefix + qid)
+        # replace record keys
+        @_replace_keys['dump'].each_pair { |before, after|
+          record[after] = record[before]
+          record.delete(before)
+        }
+
+        # emit
+        catched_record = JSON.load(tryOnRedis('get', @redis_key_prefix + qid))
+        combined_record = catched_record.merge(record)
+        Fluent::Engine.emit @tag, Fluent::Engine.now, combined_record
+
+        # remove qid
+        do_release(qid)
+      end
+    end
+
+    def do_release(qid)
+      tryOnRedis 'del', @redis_key_prefix + qid
+      tryOnRedis 'zrem', @redis_key_prefix, qid
+    end
+
+    def extract_qid(record)
+      qid = []
+      @query_identify.each { |attr|
+        if record.has_key?(attr)
+          qid.push(record[attr])
+        else
+          return nil
+        end
+      }
+      qid.join(':')
+    end
+
+    def write(chunk)
+
+      begin
+        chunk.msgpack_each do |(tag, time, record)|
+          if (qid = extract_qid record)
+
+            @_cond_funcs.each_pair { |cond, argv_and_func|
+              argv, func = argv_and_func
+              if exec_func(record, argv, func)
+                case cond
+                when "catch"
+                  do_catch(qid, record, time)
+                when "prolong"
+                  do_prolong(qid, time)
+                when "dump"
+                  do_dump(qid, record)
+                when "release"
+                  do_release(qid)
+                end
+                break   # very important!
+              end
+            }
+          end
+        end
+
+      end
+    end
+
+    def watch
+      @last_checked = Fluent::Engine.now
+      tick = @remove_interval
+      while true
+        sleep 0.5
+        if Fluent::Engine.now - @last_checked >= tick
+          now = Fluent::Engine.now
+          to_expire = now - @query_ttl
+
+          # Delete expired qids
+          tryOnRedis 'zremrangebyscore', @redis_key_prefix, '-inf', to_expire
+
+          # Delete buffer_size over qids
+          tryOnRedis 'zremrangebyrank', @redis_key_prefix, 0, -@buffer_size
+
+          @last_checked = now
+        end
+      end
+    end
+
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_query_combiner'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_query_combiner.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+require 'helper'
+
+class QueryCombinerOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+  ]
+
+  def create_driver(conf = CONFIG, tag='test.input')
+    Fluent::Test::OutputTestDriver.new(Fluent::QueryCombinerOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+  end
+
+  def test_write
+    d = create_driver
+  end
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-querycombiner
+version: !ruby/object:Gem::Version
+  version: 0.0.0.pre
+platform: ruby
+authors:
+- Takahiro Kamatani
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent plugin to combine multiple queries.
+email:
+- buhii314@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-querycombiner.gemspec
+- lib/fluent/plugin/out_query_combiner.rb
+- test/helper.rb
+- test/plugin/test_out_query_combiner.rb
+homepage: https://github.com/buhii/fluent-plugin-querycombiner
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluent plugin to combine multiple queries.
+test_files:
+- test/helper.rb
+- test/plugin/test_out_query_combiner.rb