fluent-plugin-redisstore-data 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9e1ecf5297235faeb1ac35f5e26b16eb8b1bb2f6a117d13ffaa588e4c9195f3a
+  data.tar.gz: e853015c29e622eb0283bb6a9a25e0ab204f67ba0fdf318cab054065a13e4470
+SHA512:
+  metadata.gz: 628aeefe1687b77217e18d496d4cc34aafce2b349681df75144fd780669bc94452701684052c263153f430bec2ec08fec140372fb25d5705b544981d1500e820
+  data.tar.gz: 1efa75869870ce64bda99196937472c7cd6e9434f9360ccebb24a43e8e30cb7d7e3ccab3ea9ade31b5e52fa47308ef642889d00ac3f54fee122fce1ce0f0297a

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/bundle

data/AUTHORS

@@ -0,0 +1 @@
+moaikids

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.rdoc

@@ -0,0 +1,43 @@
+= Redis output plugin for Fluent event collector
+
+== Overview
+
+*redisstore* output plugin buffers event logs in local file and upload it to redis storage.
+
+this sources folked 'fluent-plugin-redis'. (https://github.com/yuki24/fluent-plugin-redis)
+
+== Installation
+
+== Configuration
+
+    <match pattern>
+      type redisstore
+      host REDIS_HOST_ADDRESS(string, all , localhost)
+      port REDIS_PORT(int, all, 6379)
+      db_number REDIS_DB_NUMBER(int, all , nil)
+      timeout TIMEOUT(float, all , 5.0)
+      key_prefix prefix_(string, all, '')
+      key_suffix _suffix(string, all , '')
+      store_type zset(zset|set|list|string, all, zset)
+      key_name key(string, all, nil)
+      score_name score(string, all , nil)
+      value_name value(string, all, nill)
+      key_expire 604800(int, all, nil)
+      value_expire 86400(int , zset, nil)
+      value_length 100(int, zset|list, nil)
+      order asc(asc|desc, zset/list, nil)
+    </match>
+
+    # use hiredis for better performance
+    # this configuration requires "gem install hiredis"
+    <match pattern>
+      type redisstore
+      driver hiredis
+      ...
+    </match>
+
+== Copyright
+
+Copyright:: Copyright (c) 2013 moaikids
+License::   Apache License, Version 2.0
+

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+    test.libs << 'lib' << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+end
+
+task :default => :test

data/benchmarking/.gitignore

@@ -0,0 +1 @@
+dummy.log

data/benchmarking/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'fluentd'
+gem 'fluent-plugin-redisstore', :path => '../'
+gem 'dummer'

data/benchmarking/README.rdoc

@@ -0,0 +1,15 @@
+= Benchmark
+
+== worker
+
+    bundle exec fluentd -c default.conf
+    # or
+    bundle exec fluentd -c hiredis.conf
+
+== dummer
+
+    bundle exec dummer -c dummer.conf
+
+== monitor
+
+    bundle exec ruby monitor.rb

data/benchmarking/default.conf

@@ -0,0 +1,21 @@
+<source>
+  type tail
+  path dummy.log
+  pos_file /var/tmp/_var_log_dummy.pos
+  format none
+  tag dummy
+</source>
+<match dummy>
+  type redisstore
+  host 127.0.0.1
+  timeout 15.0
+  store_type zset
+  key_name method
+  key_suffix _suffix
+  score_name uri
+  value_name reqtime
+  key_expire 604800
+  value_expire 259200
+  value_length 10
+  flush_interval 0.1
+</match>

data/benchmarking/dummer.conf

@@ -0,0 +1,13 @@
+configure 'sample' do
+  output "dummy.log"
+  rate 100000
+  delimiter "\t"
+  labeled true
+  field :id, type: :integer, countup: true, format: "%04d"
+  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
+  field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
+  field :method, type: :string, any: %w[GET POST PUT]
+  field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
+  field :reqtime, type: :float, range: 0.1..5.0
+  field :foobar, type: :string, length: 8
+end 

data/benchmarking/hiredis.conf

@@ -0,0 +1,22 @@
+<source>
+  type tail
+  path dummy.log
+  pos_file /var/tmp/_var_log_dummy.pos
+  format none
+  tag dummy
+</source>
+<match dummy>
+  type redisstore
+  driver hiredis
+  host 127.0.0.1
+  timeout 15.0
+  store_type zset
+  key_name method
+  key_suffix _suffix
+  score_name uri
+  value_name reqtime
+  key_expire 604800
+  value_expire 259200
+  value_length 10
+  flush_interval 0.1
+</match>

data/benchmarking/monitor.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'terminal-table'
+require 'pp'
+require 'redis'
+require 'hiredis'
+
+class RedisInfo
+  def initialize(host,port)
+    @redis = Redis.new(:host => host, :port => port, driver: :hiredis)
+  end
+  def get_total_commands_processed
+    @redis.info["total_commands_processed"].to_i
+  end
+end
+
+chk = RedisInfo.new("127.0.0.1","6379")
+previous = 0
+
+res = []
+(1..30).each do |i|
+  current = chk.get_total_commands_processed
+  res << current - previous
+  previous = current
+  sleep 1
+end
+
+res.shift
+puts "#{1.0 * res.inject(:+) / res.size} req/s"

data/fluent-plugin-redisstore-data.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+    gem.name        = "fluent-plugin-redisstore-data"
+    gem.version     = "0.0.4"
+    gem.authors     = ["moaikids"]
+    gem.licenses    = ["Apache License Version 2.0"]
+    gem.summary     = %q{Redis(zset/set/list/string) output plugin for Fluentd}
+    gem.description = %q{Redis(zset/set/list/string) output plugin for Fluentd...}
+    gem.homepage    = "https://github.com/moaikids/fluent-plugin-redisstore"
+
+    gem.files         = `git ls-files`.split($\)
+    gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+    gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+    gem.require_paths = ["lib"]
+
+    gem.add_development_dependency "rake"
+    gem.add_development_dependency "shoulda"
+    gem.add_runtime_dependency "fluentd"
+    gem.add_runtime_dependency "redis"
+end

data/lib/fluent/plugin/out_redisstore.rb

@@ -0,0 +1,220 @@
+module Fluent
+  class RedisOutput < BufferedOutput
+    Fluent::Plugin.register_output('redisstore', self)
+    attr_reader :host, :port, :db_number, :redis, :timeout, :key_prefix, :key_suffix, :store_type, :key_name, :fixed_key_value, :score_name, :value_name, :key_expire, :value_expire, :value_length, :order
+
+    def initialize
+      super
+      require 'redis'
+      require 'msgpack'
+    end
+
+    def configure(conf)
+      super
+
+      @driver = conf.has_key?('driver') ? conf['driver'] : nil
+
+      @host = conf.has_key?('host') ? conf['host'] : 'localhost'
+      @port = conf.has_key?('port') ? conf['port'].to_i : 6379
+      @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
+      @timeout = conf.has_key?('timeout') ? conf['timeout'].to_f : 5.0
+
+      @key_prefix = conf.has_key?('key_prefix') ? conf['key_prefix'] : ''
+      @key_suffix = conf.has_key?('key_suffix') ? conf['key_suffix'] : ''
+      @store_type = conf.has_key?('store_type') ? conf['store_type'] : 'zset'
+      @key_name = conf['key_name']
+      #@fixed_key_value = conf.has_key?('fixed_key_value') ? conf['fixed_key_value'] : nil
+      @fixed_key_value = conf['fixed_key_value']
+      @score_name = conf['score_name']
+      @value_name = conf['value_name']
+      @key_expire = conf.has_key?('key_expire') ? conf['key_expire'].to_i : -1
+      @value_expire = conf.has_key?('value_expire') ? conf['value_expire'].to_i : -1
+      @value_length = conf.has_key?('value_length') ? conf['value_length'].to_i : -1
+      @order = conf.has_key?('order') ? conf['order'] : 'asc'
+
+      puts conf
+      puts 'Key_name: '+@key_name.to_s
+      puts 'Fixed_key_name: '+@fixed_key_name.to_s
+    end
+
+    def start
+      super
+
+      opt = {
+        :host => @host,
+        :port => @port,
+        :db => @db_number,
+        :timeout => @timeout,
+        :thread_safe => true,
+      }
+
+      if @driver
+        opt[:driver] = @driver.to_sym
+      end
+
+      @redis = Redis.new(opt)
+    end
+
+    def shutdown
+      super
+
+      @redis.quit
+    end
+
+    def format(tag, time, record)
+      identifier = [tag, time].join(".")
+      [identifier, record].to_msgpack
+    end
+
+    def write(chunk)
+      @redis.pipelined {
+        chunk.open { |io|
+          begin
+            MessagePack::Unpacker.new(io).each { |message|
+              begin
+                (tag, record) = message
+                if @store_type == 'zset'
+                  operation_for_zset(record)
+                elsif @store_type == 'set'
+                  operation_for_set(record)
+                elsif @store_type == 'list'
+                  operation_for_list(record)
+                elsif @store_type == 'string'
+                  operation_for_string(record)
+                end
+              rescue NoMethodError => e
+                puts e
+              end
+            }
+          rescue EOFError
+            # EOFError always occured when reached end of chunk.
+          end
+        }
+      }
+    end
+
+    def operation_for_zset(record)
+      now = Time.now.to_i
+      if @fixed_key_value
+        k = @fixed_key_value
+      else
+        k = traverse(record, @key_name).to_s
+      end
+      if @score_name
+        s = traverse(record, @score_name)
+      else
+        s = now
+      end
+      v = traverse(record, @value_name)
+      sk = @key_prefix + k + @key_suffix
+
+      @redis.zadd sk , s, v
+      if @key_expire > 0
+        @redis.expire sk , @key_expire
+      end
+      if @value_expire > 0
+        @redis.zremrangebyscore sk , '-inf' , (now - @value_expire)
+      end
+      if @value_length > 0
+        script = generate_zremrangebyrank_script(sk, @value_length, @order)
+        @redis.eval script
+      end
+    end
+
+    def operation_for_set(record)
+      if @fixed_key_value
+        k = @fixed_key_value
+      else
+        k = @key_name
+      end
+      #v = traverse(record, @value_name)
+      sk = @key_prefix + k + @key_suffix
+
+      @redis.sadd sk, v
+      if @key_expire > 0
+        @redis.expire sk, @key_expire
+      end
+    end
+
+    def operation_for_list(record)
+      if @fixed_key_value
+        k = @fixed_key_value
+      else
+        k = traverse(record, @key_name).to_s
+      end
+      v = traverse(record, @value_name)
+      sk = @key_prefix + k + @key_suffix
+
+      if @order == 'asc'
+        @redis.rpush sk, v
+      else
+        @redis.lpush sk, v
+      end
+      if @key_expire > 0
+        @redis.expire sk, @key_expire
+      end
+      if @value_length > 0
+        script = generate_ltrim_script(sk, @value_length, @order)
+        @redis.eval script
+      end
+    end
+
+    def operation_for_string(record)
+      if @fixed_key_value
+        k = @fixed_key_value
+      else
+        k = @key_name
+
+      end
+      sk = @key_prefix + k + @key_suffix
+      data = record.to_s
+      @redis.set record[sk],data
+    end
+
+    def generate_zremrangebyrank_script(key, maxlen, order)
+      script  = "local key = '" + key.to_s + "'\n"
+      script += "local maxlen = " + maxlen.to_s + "\n"
+      script += "local order ='" + order.to_s + "'\n"
+      script += "local len = tonumber(redis.call('ZCOUNT', key, '-inf', '+inf'))\n"
+      script += "if len > maxlen then\n"
+      script += "    if order == 'asc' then\n"
+      script += "       local l = len - maxlen\n"
+      script += "       if l >= 0 then\n"
+      script += "           return redis.call('ZREMRANGEBYRANK', key, 0, l)\n"
+      script += "       end\n"
+      script += "    else\n"
+      script += "       return redis.call('ZREMRANGEBYRANK', key, maxlen, -1)\n"
+      script += "    end\n"
+      script += "end\n"
+      return script
+    end
+
+    def generate_ltrim_script(key, maxlen, order)
+      script  = "local key = '" + key.to_s + "'\n"
+      script += "local maxlen = " + maxlen.to_s + "\n"
+      script += "local order ='" + order.to_s + "'\n"
+      script += "local len = tonumber(redis.call('LLEN', key))\n"
+      script += "if len > maxlen then\n"
+      script += "    if order == 'asc' then\n"
+      script += "        local l = len - maxlen\n"
+      script += "        return redis.call('LTRIM', key, l, -1)\n"
+      script += "    else\n"
+      script += "        return redis.call('LTRIM', key, 0, maxlen - 1)\n"
+      script += "    end\n"
+      script += "end\n"
+      return script
+    end
+
+    def traverse(data, key)
+      val = data
+      key.split('.').each{ |k|
+        if val.has_key?(k)
+          val = val[k]
+        else
+          return nil
+        end
+      }
+      return val
+    end
+  end
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-redisstore-data
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- moaikids
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Redis(zset/set/list/string) output plugin for Fluentd...
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- AUTHORS
+- Gemfile
+- README.rdoc
+- Rakefile
+- benchmarking/.gitignore
+- benchmarking/Gemfile
+- benchmarking/README.rdoc
+- benchmarking/default.conf
+- benchmarking/dummer.conf
+- benchmarking/hiredis.conf
+- benchmarking/monitor.rb
+- fluent-plugin-redisstore-data.gemspec
+- lib/fluent/plugin/out_redisstore.rb
+homepage: https://github.com/moaikids/fluent-plugin-redisstore
+licenses:
+- Apache License Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Redis(zset/set/list/string) output plugin for Fluentd
+test_files: []