fluent-plugin-redis-list-source 1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b6204a0bcfde45f928e8b4366c8a87af214ca5ea8529d6b9d95eb8cdb354868f
+  data.tar.gz: e11b9a3529f1b85e561489e83d8c237837183e70c5b6f5a85d2a0db7a6566c66
+SHA512:
+  metadata.gz: 075ea5acb0a40510bf218702d4b5f4801baf40a0a1dcbbe04482a26d749e2a8db80c52c8429200734a162c8e6357291aa1437d087e40a4b6f686629f779a0f6f
+  data.tar.gz: aa9a27bb0ae35c099f72902b02fc65f33222dbd8ac6ccbc74645f3bbf483da998432906302733af52a12f4531676ed5ba8d77f5a8b3305437292c25c72d4bfa8

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/README.md

@@ -0,0 +1,41 @@
+# Fluent::Plugin::RedisListPoller
+
+This gem will help you to connect redis and fluentd. With it you'll be able to get your data from redis with fluentd.
+
+## Requirements
+
+* Fluentd v0.14+
+
+## Installation
+
+```bash
+gem install redis-list-source-fluentd
+```
+
+## Configuration Options
+
+```
+<source>
+  @type       redis_listener
+  host        127.0.0.1
+  port        6379
+  password    nil
+  db          0
+  timeout     5.0
+  driver      ruby
+
+  key         redis_list_item
+  command     lpop
+  batch_size  100
+
+  tag         redis.data
+
+  poll_interval      0.01
+  sleep_interval     5
+  retry_interval    5
+
+  <parse>
+    @type json
+  </parse>
+</source>
+```

data/fluent-plugin-redis-list-source.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = 'fluent-plugin-redis-list-source'
+  spec.version       = '1.0'
+  spec.authors       = ['Nikita Kazeichev']
+  spec.email         = ['kazeichev@yandex.ru']
+
+  spec.summary       = 'Plugin wich help you to transfer data from redis to fluentd'
+  spec.description   = 'This gem will help you to connect redis and fluentd. With it you\'ll be able to get your data from redis with fluentd.'
+  spec.homepage      = 'https://github.com/kazeichev/fluent-plugin-redis_list_poller'
+  spec.license       = '0BSD'
+
+  spec.files         = `git ls-files -z`
+                       .split("\x0")
+                       .reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = []
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+
+  spec.add_runtime_dependency 'redis', ['>= 3.3.0', '< 3.4.0']
+  spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
+end

data/lib/fluent/plugin/in_redis_list_monitor.rb

@@ -0,0 +1,133 @@
+require "fluent/plugin/input"
+require "fluent/plugin_mixin/redis"
+
+module Fluent
+  module Plugin
+    # Input plugin which will monitor the size of a redis list and periodically
+    # output metrics to the login pipeline.
+    # @since 0.1.0
+    class RedisListMonitorInput < Input
+      Plugin.register_input('redis_list_monitor', self)
+      
+      include Fluent::PluginMixin::Redis
+      helpers :timer
+
+      # input plugin parameters
+      config_param :tag,      :string,  :default => nil
+
+      # Initialize new input plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def initialize
+        super
+      end
+
+      # Initialize attributes and parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure(config)
+        super
+
+        configure_params(config)
+        configure_locking(config)
+
+        @queue_length = 0
+        @retry_at     = nil
+      end
+
+      # Configure plugin parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_params(config)
+        %w(host port key tag).each do |key|
+          next if instance_variable_get("@#{key}")
+          raise Fluent::ConfigError, "configuration key missing: #{key}"
+        end
+      end
+
+      # Configure locking
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_locking(config)
+        @storage  = storage_create(type: 'local')
+        @lock_key = "redis:#{@key}:lock"
+      end
+
+      # Prepare the plugin event loop
+      #
+      # This method will initialize the Redis connection object, create any required Redis structures as well
+      # as define and begin the event pollers.
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def start
+        super
+
+        start_redis
+        start_poller
+      end
+
+      def start_poller
+        timer_execute(:poll, @poll_interval) do
+          action_poll
+        end
+      end
+
+      # Tear down the plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def shutdown
+        super
+        shutdown_redis
+      end
+
+      # Wether the poller has been temporarily disabled or should fetch messages
+      # been temporarily disabled
+      # @since 0.1.0
+      # @return [TrueClass, FalseClass]
+      def sleeping?
+        @retry_at and @retry_at >= Engine.now
+      end
+
+      # Set a sleep delay, ensuring that we will not attempt to fetch messages
+      # @since 0.1.0
+      # @param [Integer] delay, the amount of seconds to wait
+      # @return [Integer] timestamp when this expires
+      def sleep!(delay = @sleep_interval)
+        @retry_at = Engine.now + delay
+      end
+
+      # Action to execute when the monitor event watcher executes
+      #
+      # The monitor is simply responsible for outputting the queue length to 
+      # the logs as well as detecting zero length lists. 
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def action_poll
+        now = Engine.now
+
+        if sleeping?
+          log.trace "redis worker is sleeping"
+          return
+        end
+
+        list_size = @redis.llen(@key)
+
+        event = {
+          "timestamp" => now,
+          "message" => "redis queue monitor",
+          "hostname" => @host,
+          "key" => @key,
+          "size" => list_size
+        }
+
+        router.emit @tag, now, event
+      rescue => e
+        log.error "error monitoring queue", :error => e
+        log.error_backtrace
+        sleep!(@retry_interval)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_redis_list_poller.rb

@@ -0,0 +1,237 @@
+require "fluent/plugin/input"
+require "fluent/plugin/parser"
+require 'fluent/process'
+require "fluent/plugin_mixin/redis"
+
+module Fluent
+  module Plugin
+    class RedisListPollerInput < Input
+      include Fluent::PluginMixin::Redis
+
+      Plugin.register_input('redis-list-source', self)
+      helpers :storage
+      helpers :timer
+
+      # redis list details
+      # - command: redis command to execute when fetching messages
+      # - batch_size: if greater than 0, fetch messages in batches
+      config_param :command,    :string,  :default => "lpop"
+      config_param :batch_size, :integer, :default => 0
+
+      # input plugin parameters
+      config_param :tag,      :string,  :default => nil
+
+      # parser plugin parameters
+      config_section :parse, :init => true, :multi => false do
+        config_set_default :@type, "json"
+      end
+
+      # Initialize new input plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def initialize
+        super
+        require 'msgpack'
+      end
+
+      # Initialize attributes and parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure(config)
+        super
+
+        configure_params(config)
+        configure_parser(config)
+        configure_locking(config)
+
+        @retry_at     = nil
+      end
+
+      # Configure plugin parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_params(config)
+        %w(host port key command tag).each do |key|
+          next if instance_variable_get("@#{key}")
+          raise Fluent::ConfigError, "configuration key missing: #{key}"
+        end
+
+        unless %w(lpop rpop).include?(@command)
+          raise Fluent::ConfigError, "command must be either lpop or rpop"
+        end
+      end
+
+      # Configure record parser
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_parser(config)
+        parser_config = @parse.corresponding_config_element
+        parser_type = parser_config['@type']
+        @parser = Fluent::Plugin.new_parser(parser_type, :parent => self)
+        @parser.configure(parser_config)
+      end
+
+      # Configure locking
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_locking(config)
+        @storage  = storage_create(type: 'local')
+        @lock_key = "fluentd:#{@key}:lock"
+      end
+
+      # Prepare the plugin event loop
+      #
+      # This method will initialize the Redis connection object, create any required Redis structures as well
+      # as define and begin the event pollers.
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def start
+        super
+        start_redis
+        start_poller
+        start_monitor
+      end
+
+      # Prepare the Redis queue poller
+      #
+      # This timed event will routinely poll items from the Redis list and
+      # emit those through the pipeline.
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def start_poller
+        timer_execute(:poller, @poll_interval) do
+          action_poll
+        end
+      end
+
+      # Prepare the Redis queue monitor
+      #
+      # This timed event will routinely poll for a lock key and disable the
+      # queue poller if required
+      #
+      # @since 0.1.1
+      # @return [NilClass]
+      def start_monitor
+        timer_execute(:monitor, 1) do
+          action_locking_monitor
+        end
+      end
+
+      # Tear down the plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def shutdown
+        super
+        shutdown_redis
+      end
+
+      # Whether to fetch a single item or a multiple items in batch
+      # @since 0.1.0
+      # @return [TrueClass, FalseClass]
+      def batched?
+        @batch_size and @batch_size > 1
+      end
+
+      # Wether the poller has been temporarily disabled or should fetch messages
+      # been temporarily disabled
+      # @since 0.1.0
+      # @return [TrueClass, FalseClass]
+      def sleeping?
+        @retry_at and @retry_at >= Engine.now
+      end
+
+      # Whether the poller has been locked
+      # @since 0.1.0
+      # @return [TrueClass, FalseClass]
+      def locked?
+        @storage.get(@lock_key)
+      end
+
+      # Set a sleep delay, ensuring that we will not attempt to fetch messages
+      # @since 0.1.0
+      # @param [Integer] delay, the amount of seconds to wait
+      # @return [Integer] timestamp when this expires
+      def sleep!(delay = @sleep_interval)
+        @retry_at = Engine.now + delay
+      end
+
+      # Poll messages from the redis server in either single message or 
+      # batch mode.
+      # @since 0.1.0
+      # @param [&block] the block to yield single messages to
+      # @return [NilClass]
+      def poll_messages
+        commands = []
+
+        if batched?
+          @redis.pipelined do
+            @batch_size.times do
+              commands << @redis.call(@command, @key)
+            end
+          end
+        else
+          commands << @redis.call(@command, @key)
+        end
+
+        commands.each do |command|
+          yield command.is_a?(Redis::Future) ? command.value : command
+        end
+      end
+
+      # Action to execute when polling for the lock key
+      # @since 0.1.0
+      # @return [NilClass]
+      def action_locking_monitor
+        lock_value = @redis.get(@lock_key)
+        @storage.put(@lock_key, lock_value)
+      end
+
+      # Action to execute when the poller event watcher executes
+      #
+      # Given that the watcher is pretty lightweight, we simply return if the
+      # worker has been set to sleep instead of actually sleeping. Doing
+      # otherwise seemed to cause locking.
+      #
+      # Otherwise we iterate through messages, parse and emit them. 
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def action_poll
+        now = Engine.now
+        messages = []
+
+        if sleeping?
+          log.trace "redis worker is sleeping"
+          return
+        end
+
+        if locked?
+          log.trace "redis queue is locked"
+          return
+        end
+
+        poll_messages do |message|
+          if message.nil?
+            log.debug "redis queue is empty"
+            sleep!(@sleep_interval)
+            break
+          end
+
+          @parser.parse(message) do |time, record|
+            if time && record
+              router.emit @tag || @key, time || Engine.now, record
+            else
+              log.warn "failed to parse message: #{message}"
+            end
+          end
+        end
+      rescue => e
+        log.error "error fetching record", :error => e
+        log.error_backtrace
+        sleep!(@retry_interval)
+      end
+    end
+  end
+end

data/lib/fluent/plugin_mixin/redis.rb

@@ -0,0 +1,66 @@
+
+module Fluent
+  module PluginMixin
+    module Redis
+      def self.included(base)
+        base.class_eval do
+          # redis connection details
+          config_param :host,     :string,  :default => '127.0.0.1'
+          config_param :port,     :integer, :default => 6379
+          config_param :path,     :string,  :default => nil
+          config_param :password, :string,  :default => nil
+          config_param :db,       :integer, :default => 0
+          config_param :timeout,  :float,   :default => 5.0
+          config_param :driver,   :string,  :default => "ruby"
+
+          # redis list details
+          # - key: redis key of type `list` to fetch messages from
+          # - command: redis command to execute when fetching messages
+          # - batch_size: if greater than 0, fetch messages in batches
+          config_param :key, :string, :default => nil
+
+          # worker parameters
+          # - poll_inteval: interval between message polling actions
+          #                 *NOTE*: Apparently this must be greather than 0
+          # - sleep_interval: interval to wait after receiving 0 messages
+          # - retry_interval: interval to wait before retrying after an error
+          config_param :poll_interval,    :float, :default => 1
+          config_param :sleep_interval,   :float, :default => 5
+          config_param :retry_interval,   :float, :default => 5
+
+          attr_reader  :redis
+        end
+      end
+
+      # Initialize new input plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def initialize
+        require 'redis'
+        super
+      end
+
+      # Prepare the Redis conncection object
+      # @since 0.1.0
+      # @return [Redis]
+      def start_redis
+        @redis  = ::Redis.new(
+          :host => @host,
+          :port => @port,
+	  :db   => @db,
+          :driver => @driver,
+	  :timeout => @timeout,
+          :password => @password,
+          :thread_safe => true
+        )
+      end
+
+      # Destroy the Redis connection object
+      # @since 0.1.0
+      # @return [NilClass]
+      def shutdown_redis
+        @redis.quit
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-redis-list-source
+version: !ruby/object:Gem::Version
+  version: '1.0'
+platform: ruby
+authors:
+- Nikita Kazeichev
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: This gem will help you to connect redis and fluentd. With it you'll be
+  able to get your data from redis with fluentd.
+email:
+- kazeichev@yandex.ru
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- README.md
+- fluent-plugin-redis-list-source.gemspec
+- lib/fluent/plugin/in_redis_list_monitor.rb
+- lib/fluent/plugin/in_redis_list_poller.rb
+- lib/fluent/plugin_mixin/redis.rb
+homepage: https://github.com/kazeichev/fluent-plugin-redis_list_poller
+licenses:
+- 0BSD
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Plugin wich help you to transfer data from redis to fluentd
+test_files: []