fluent-plugin-redisstore 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Mzk2MzIxZmZkY2QwZjcxZjdiY2E4MmE4MDJmZjVjM2EzYmRjNDJjZA==
+  data.tar.gz: !binary |-
+    ODAzYTRmZWVmY2U4NGM0OGM5MzE5MDQ5OGM0MjRkMTAwZjVjYWFiZg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NzU0YWQzOGQwYzU1NzhhNWVjY2VhMGFiNzJhYzFiOTA4YjU4NWEyZDRmNzdi
+    MDhhMjBhOTZmMzFlNzM3MTczNzY2M2MwNGQzMDc1ZDUzMTlkZWFkY2UxYmZi
+    MWEzYTNjNDAwOGEyNmMzZjExZWU4N2IwN2ZmMzRmYzViNDNlMDE=
+  data.tar.gz: !binary |-
+    NTU1NmZjOTIxNzdmODhhZTY5MmJhZDJiYTY0ODg1ZGY2ZWI4N2UyMTNjNGI3
+    YjE1YWNhZjQ1MDgzOGQxYWU1Nzk2MzYyMDZjOWIwMDc1ODk4ZGE1OTgyMTQ0
+    MThjNzYyY2E4MTkzN2I5YjJiMDdhMzEyNTUwYTJmNzQyMGY1NTE=

data/AUTHORS

@@ -0,0 +1 @@
+moaikids

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.rdoc

@@ -0,0 +1,35 @@
+= Redis output plugin for Fluent event collector
+
+== Overview
+
+*redisstore* output plugin buffers event logs in local file and upload it to redis storage.
+
+this sources folked 'fluent-plugin-redis'. (https://github.com/yuki24/fluent-plugin-redis)
+
+== Installation
+
+== Configuration
+
+    <match pattern>
+      type redisstore
+      host REDIS_HOST_ADDRESS(string, all , localhost)
+      port REDIS_PORT(int, all, 6379)
+      db_number REDIS_DB_NUMBER(int, all , nil)
+      timeout TIMEOUT(float, all , 5.0)
+      key_prefix prefix_(string, all, '')
+      key_suffix _suffix(string, all , '')
+      store_type zset(zset|set|list|string, all, zset)
+      key_name key(string, all, nil)
+      score_name score(string, all , nil)
+      value_name value(string, all, nill)
+      key_expire 604800(int, all, nil)
+      value_expire 86400(int , zset, nil)
+      value_length 100(int, zset|list, nil)
+      order asc(asc|desc, zset/list, nil)
+    </match>
+
+== Copyright
+
+Copyright:: Copyright (c) 2013 moaikids
+License::   Apache License, Version 2.0
+

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+    test.libs << 'lib' << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-redisstore.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+    gem.name        = "fluent-plugin-redisstore"
+    gem.version     = "0.0.1"
+    gem.authors     = ["moaikids"]
+    gem.licenses    = ["Apache License Version 2.0"]
+    gem.summary     = %q{Redis(zset/set/list/string) output plugin for Fluentd}
+    gem.description = %q{Redis(zset/set/list/string) output plugin for Fluentd...}
+    gem.homepage    = "https://github.com/moaikids/fluent-plugin-redisstore"
+
+    gem.files         = `git ls-files`.split($\)
+    gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+    gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+    gem.require_paths = ["lib"]
+
+    gem.add_development_dependency "rake"
+    gem.add_development_dependency "shoulda"
+    gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/out_redisstore.rb

@@ -0,0 +1,202 @@
+module Fluent
+  class RedisOutput < BufferedOutput
+    Fluent::Plugin.register_output('redisstore', self)
+    attr_reader :host, :port, :db_number, :redis, :timeout, :key_prefix, :key_suffix, :store_type, :key_name, :fixed_key_value, :score_name, :value_name, :key_expire, :value_expire, :value_length, :order
+
+    def initialize
+      super
+      require 'redis'
+      require 'msgpack'
+    end
+
+    def configure(conf)
+      super
+
+      @host = conf.has_key?('host') ? conf['host'] : 'localhost'
+      @port = conf.has_key?('port') ? conf['port'].to_i : 6379
+      @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
+      @timeout = conf.has_key?('timeout') ? conf['timeout'].to_f : 5.0
+
+      @key_prefix = conf.has_key?('key_prefix') ? conf['key_prefix'] : ''
+      @key_suffix = conf.has_key?('key_suffix') ? conf['key_suffix'] : ''
+      @store_type = conf.has_key?('store_type') ? conf['store_type'] : 'zset'
+      @key_name = conf['key_name']
+      @fixed_key_value = conf.has_key?('fixed_key_value') ? conf['fixed_key_value'] : nil
+      @score_name = conf['score_name']
+      @value_name = conf['value_name']
+      @key_expire = conf.has_key?('key_expire') ? conf['key_expire'].to_i : -1
+      @value_expire = conf.has_key?('value_expire') ? conf['value_expire'].to_i : -1
+      @value_length = conf.has_key?('value_length') ? conf['value_length'].to_i : -1
+      @order = conf.has_key?('order') ? conf['order'] : 'asc'
+    end
+
+    def start
+      super
+      @redis = Redis.new(:host => @host, :port => @port, :timeout => @timeout,
+                         :thread_safe => true, :db => @db_number)
+    end
+
+    def shutdown
+      @redis.quit
+    end
+
+    def format(tag, time, record)
+      identifier = [tag, time].join(".")
+      [identifier, record].to_msgpack
+    end
+
+    def write(chunk)
+      @redis.pipelined {
+        chunk.open { |io|
+          begin
+            MessagePack::Unpacker.new(io).each { |message|
+              begin
+                (tag, record) = message
+                if @store_type == 'zset'
+                  operation_for_zset(record)
+                elsif @store_type == 'set'
+                  operation_for_set(record)
+                elsif @store_type == 'list'
+                  operation_for_list(record)
+                elsif @store_type == 'string'
+                  operation_for_string(record)
+                end
+              rescue NoMethodError => e
+                puts e
+              end
+            }
+          rescue EOFError
+            # EOFError always occured when reached end of chunk.
+          end
+        }
+      }
+    end
+
+    def operation_for_zset(record)
+      now = Time.now.to_i
+      if @fixed_key_value
+        k = @fixed_key_value
+      else
+        k = traverse(record, @key_name).to_s
+      end
+      if @score_name
+        s = traverse(record, @score_name)
+      else
+        s = now
+      end
+      v = traverse(record, @value_name)
+      sk = @key_prefix + k + @key_suffix
+      
+      @redis.zadd sk , s, v
+      if @key_expire > 0
+        @redis.expire sk , @key_expire
+      end
+      if @value_expire > 0
+        @redis.zremrangebyscore sk , '-inf' , (now - @value_expire)
+      end
+      if @value_length > 0
+        script = generate_zremrangebyrank_script(sk, @value_length, @order)
+        @redis.eval script
+      end
+    end
+
+    def operation_for_set(record)
+      if @fixed_key_value
+        k = @fixed_key_value
+      else
+        k = traverse(record, @key_name).to_s
+      end
+      v = traverse(record, @value_name)
+      sk = @key_prefix + k + @key_suffix
+              
+      @redis.sadd sk, v
+      if @key_expire > 0
+        @redis.expire sk, @key_expire
+      end
+    end
+
+    def operation_for_list(record)
+      if @fixed_key_value
+        k = @fixed_key_value
+      else
+        k = traverse(record, @key_name).to_s
+      end
+      v = traverse(record, @value_name)
+      sk = @key_prefix + k + @key_suffix
+
+      if @order == 'asc'
+        @redis.rpush sk, v
+      else
+        @redis.lpush sk, v
+      end             
+      if @key_expire > 0
+        @redis.expire sk, @key_expire
+      end
+      if @value_length > 0
+        script = generate_ltrim_script(sk, @value_length, @order)
+        @redis.eval script
+      end 
+    end
+
+    def operation_for_string(record)
+      if @fixed_key_value
+        k = @fixed_key_value
+      else
+        k = traverse(record, @key_name).to_s
+      end
+      v = traverse(record, @value_name)
+      sk = @key_prefix + k + @key_suffix
+         
+      @redis.set sk, v
+      if @key_expire > 0
+        @redis.expire sk, @key_expire
+      end
+    end
+
+    def generate_zremrangebyrank_script(key, maxlen, order)
+      script  = "local key = '" + key.to_s + "'\n"
+      script += "local maxlen = " + maxlen.to_s + "\n"
+      script += "local order ='" + order.to_s + "'\n"
+      script += "local len = tonumber(redis.call('ZCOUNT', key, '-inf', '+inf'))\n"
+      script += "if len > maxlen then\n"
+      script += "    if order == 'asc' then\n"
+      script += "       local l = len - maxlen\n"
+      script += "       if l >= 0 then\n"
+      script += "           return redis.call('ZREMRANGEBYRANK', key, 0, l)\n"
+      script += "       end\n"
+      script += "    else\n"
+      script += "       return redis.call('ZREMRANGEBYRANK', key, maxlen, -1)\n"
+      script += "    end\n"
+      script += "end\n"
+      return script
+    end
+
+    def generate_ltrim_script(key, maxlen, order)
+      script  = "local key = '" + key.to_s + "'\n"
+      script += "local maxlen = " + maxlen.to_s + "\n"
+      script += "local order ='" + order.to_s + "'\n"
+      script += "local len = tonumber(redis.call('LLEN', key))\n"
+      script += "if len > maxlen then\n"
+      script += "    if order == 'asc' then\n"
+      script += "        local l = len - maxlen\n"
+      script += "        return redis.call('LTRIM', key, l, -1)\n"
+      script += "    else\n"
+      script += "        return redis.call('LTRIM', key, 0, maxlen - 1)\n"
+      script += "    end\n"
+      script += "end\n"
+      return script
+    end
+
+    def traverse(data, key)
+      val = data
+      key.split('.').each{ |k|
+        if val.has_key?(k)
+          val = val[k]
+        else
+          return nil
+        end
+      }
+      return val
+    end
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-redisstore
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- moaikids
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Redis(zset/set/list/string) output plugin for Fluentd...
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- AUTHORS
+- Gemfile
+- README.rdoc
+- Rakefile
+- fluent-plugin-redisstore.gemspec
+- lib/fluent/plugin/out_redisstore.rb
+homepage: https://github.com/moaikids/fluent-plugin-redisstore
+licenses:
+- Apache License Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.7
+signing_key: 
+specification_version: 4
+summary: Redis(zset/set/list/string) output plugin for Fluentd
+test_files: []