fluent-plugin-redis_list_poller 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5664df7a09d8c92d2eff417ca21e3e77c1469d96
+  data.tar.gz: f23f3179502d0104f0634e7acb5ad1bdc8abd430
+SHA512:
+  metadata.gz: 60ecb470169580c8b2a27ae5d7892f55d547680433ce16e66eb58d0141c0fbabf6d8e4c71de6c100fd8bfead32edae64e94928ad75d63e71691cb09c29ed7f38
+  data.tar.gz: 5a15699078c53d68f47cdbe5543ddabff3be92676e9d670ccc67a3739e8a8b0e6725a21620509dd5c89a833681dc451b3589a84a780039aa25ab30afb32fd08e

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/CHANGELOG.txt

@@ -0,0 +1,4 @@
+# Changelog
+
+## 0.1.0
+* Initial release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-redis_poll_input.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,71 @@
+# Fluent::Plugin::RedisListPoller
+
+This gem provides a few input plugins for Fluentd which are designed to more efficiently poll large volumes of messages than other existing redis input plugins. 
+
+* *redis_list_poller*: Input plugin designed to fetch large volumes of messages
+* *redis_list_monitor*: Input plugin designed to fetch queue size metrics
+
+In additional to the standard stuff, the `redis_list_poller` input plugin also looks for a lock key in Redis to see whether it has been administratively disabled. This lock key follows the naming convention of `redis:KEY_NAME:lock`.
+
+## Requirements
+
+* Fluentd v0.14+
+
+## Installation
+
+```bash
+gem install fluent-plugin-redis_list_poller
+```
+
+## Configuration Options
+
+```
+<source>
+  @type       redis_list_poller
+  host        127.0.0.1
+  port        6379
+  path        nil
+  password    nil
+  db          0
+  timeout     5.0
+  driver      ruby
+
+  key         redis_list_item
+  command     lpop
+  batch_size  100
+
+  tag         redis.data
+  format      json
+
+  poll_inteval      0.01
+  sleep_inteval     5
+  retry_interval    5
+</source>
+```
+
+## Benchmarks
+
+These very simple benchmarks which give a rough ideas of the sorts of message rates one can expect from this input plugin.
+
+They were run within a Vagrant box, with a local Redis, running on my Macbook Pro 15'. Fluentd is running on a single CPU core and is using the examples/standalone.conf.
+
+poll_interval|batch_size|messages/second|
+-------------|----------|---------------|
+1     | 1   | 1
+0.5   | 1   | 2
+0.2   | 1   | 5
+0.01  | 1   | 100
+0.001 | 1   | 457
+poll_interval|batch_size|messages/second|
+1     | 10  | 10
+0.5   | 10  | 20
+0.2   | 10  | 50
+0.01  | 10  | 1000 
+0.001 | 10  | 1793
+poll_interval|batch_size|messages/second|
+1     | 100 | 100
+0.5   | 100 | 200
+0.2   | 100 | 500
+0.01  | 100 | 7823
+0.001 | 100 | 7600
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "aargh"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/data.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+for x in `seq 1 10000`; do
+  echo "RPUSH test \"{ \\\"message\\\": \\\"${x}\\\" }\""
+done | redis-cli --pipe
+

data/examples/standalone.conf

@@ -0,0 +1,25 @@
+#<source>
+#	@type redis_list_monitor
+#	tag   redis.metrics
+#	host  127.0.0.1
+#	key   test
+#	poll_interval 1
+#</source>
+
+<source>
+	@type redis_list_poller
+	tag 	redis.content
+	host 	127.0.0.1
+	key 	test
+	batch_size 1
+	poll_interval 1
+</source>
+
+<match redis.content>
+	@type flowcounter_simple
+	unit 	second
+</match>
+
+#<match redis.metrics>
+#  @type stdout
+#</match>

data/fluent-plugin-redis_list_poller.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-redis_list_poller"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Jonathan Serafini"]
+  spec.email         = ["jonathan@serafini.ca"]
+
+  spec.summary       = %q{A fluentd redis input plugin supporting batch operations}
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/JonathanSerafini/fluent-plugin-redis_list_poller"
+  spec.license       = "apache2"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = []
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+
+  spec.add_runtime_dependency "fluentd", [">= 0.12.0", "< 2"]
+end

data/lib/fluent/plugin/in_redis_list_monitor.rb

@@ -0,0 +1,174 @@
+require "fluent/plugin/input"
+require "fluent/plugin_mixin/redis"
+
+module Fluent
+  module Plugin
+    # Input plugin which will monitor the size of a redis list and periodically
+    # output metrics to the login pipeline.
+    # @since 0.1.0
+    class RedisListMonitorInput < Input
+      include Fluent::PluginMixin::Redis
+
+      Plugin.register_input('redis_list_monitor', self)
+
+      # input plugin parameters
+      config_param :tag,      :string,  :default => nil
+
+      # Initialize new input plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def initialize
+        super
+        require 'cool.io'
+      end
+
+      # Initialize attributes and parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure(config)
+        super
+
+        configure_params(config)
+        configure_locking(config)
+
+        @queue_length = 0
+        @retry_at     = nil
+      end
+
+      # Configure plugin parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_params(config)
+        %w(host port key tag).each do |key|
+          next if instance_variable_get("@#{key}")
+          raise Fluent::ConfigError, "configuration key missing: #{key}"
+        end
+      end
+
+      # Configure locking
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_locking(config)
+        @storage  = storage_create(type: 'local')
+        @lock_key = "redis:#{@key}:lock"
+      end
+
+      # Prepare the plugin event loop
+      #
+      # This method will initialize the Redis connection object, create any required Redis structures as well
+      # as define and begin the event pollers.
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def start
+        super
+
+        @loop = Coolio::Loop.new
+
+        start_redis
+        start_poller
+
+        @thread = Thread.new(&method(:run))
+      end
+
+      def start_poller
+        @poller = TimerWatcher.new(
+          @poll_interval,
+          log,
+          &method(:action_poll)
+        )
+
+        @loop.attach(@poller)
+      end
+
+      # Begin the logging pipeline
+      # @since 0.1.0
+      # @return [NilClass]
+      def run
+        @loop.run
+      rescue => e
+        log.error "unexpected error", :error => e
+        log.error_backtrace
+      end
+
+      # Tear down the plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def shutdown
+        @loop.watchers.each { |w| w.detach }
+        @loop.stop
+        Thread.kill(@thread)
+        @thread.join
+        shutdown_redis
+        super
+      end
+
+      # Wether the poller has been temporarily disabled or should fetch messages
+      # been temporarily disabled
+      # @since 0.1.0
+      # @return [TrueClass, FalseClass]
+      def sleeping?
+        @retry_at and @retry_at >= Engine.now
+      end
+
+      # Set a sleep delay, ensuring that we will not attempt to fetch messages
+      # @since 0.1.0
+      # @param [Integer] delay, the amount of seconds to wait
+      # @return [Integer] timestamp when this expires
+      def sleep!(delay = @sleep_interval)
+        @retry_at = Engine.now + delay
+      end
+
+      # Action to execute when the monitor event watcher executes
+      #
+      # The monitor is simply responsible for outputting the queue length to 
+      # the logs as well as detecting zero length lists. 
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def action_poll
+        now = Engine.now
+
+        if sleeping?
+          log.trace "redis worker is sleeping"
+          return
+        end
+
+        list_size = @redis.llen(@key)
+
+        event = {
+          "timestamp" => now,
+          "message" => "redis queue monitor",
+          "hostname" => @host,
+          "key" => @key,
+          "size" => list_size
+        }
+
+        router.emit @tag, now, event
+      rescue => e
+        log.error "error monitoring queue", :error => e
+        log.error_backtrace
+        sleep!(@retry_interval)
+      end
+
+      # Generic Cool.io timer which will execute a given callback on schedule.
+      # @since 0.1.0
+      class TimerWatcher < Coolio::TimerWatcher
+        attr_reader :log
+
+        def initialize(interval, log, &callback)
+          @callback = callback
+          @log = log
+          super(interval, true)
+        end
+
+        def on_timer
+          @callback.call
+        rescue => e
+          log.error "unexpected error", :error=>e
+          log.error_backtrace
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_redis_list_poller.rb

@@ -0,0 +1,263 @@
+require "fluent/plugin/input"
+require "fluent/plugin_mixin/redis"
+
+module Fluent
+  module Plugin
+    class RedisListPollerInput < Input
+      include Fluent::PluginMixin::Redis
+
+      Plugin.register_input('redis_list_poller', self)
+
+      helpers :storage
+
+      # redis list details
+      # - command: redis command to execute when fetching messages
+      # - batch_size: if greater than 0, fetch messages in batches
+      config_param :command,    :string,  :default => "lpop"
+      config_param :batch_size, :integer, :default => 0
+
+      # input plugin parameters
+      config_param :tag,      :string,  :default => nil
+      config_param :format,   :string,  :default => "json"
+
+      # Initialize new input plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def initialize
+        super
+        require 'cool.io'
+        require 'msgpack'
+      end
+
+      # Initialize attributes and parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure(config)
+        super
+
+        configure_params(config)
+        configure_parser(config)
+        configure_locking(config)
+
+        @retry_at     = nil
+      end
+
+      # Configure plugin parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_params(config)
+        %w(host port key command format tag).each do |key|
+          next if instance_variable_get("@#{key}")
+          raise Fluent::ConfigError, "configuration key missing: #{key}"
+        end
+
+        unless %w(lpop rpop).include?(@command)
+          raise Fluent::ConfigError, "command must be either lpop or rpop"
+        end
+      end
+
+      # Configure record parser
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_parser(config)
+        @parser = Plugin.new_parser(@format)
+        @parser.configure(config)
+      end
+
+      # Configure locking
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_locking(config)
+        @storage  = storage_create(type: 'local')
+        @lock_key = "fluentd:#{@key}:lock"
+      end
+
+      # Prepare the plugin event loop
+      #
+      # This method will initialize the Redis connection object, create any required Redis structures as well
+      # as define and begin the event pollers.
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def start
+        super
+
+        @loop = Coolio::Loop.new
+        start_redis
+        start_poller
+        @thread = Thread.new(&method(:run))
+      end
+
+      # Prepare the Redis queue poller
+      #
+      # This timed event will routinely poll items from the Redis list and
+      # emit those through the pipeline.
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def start_poller
+        @poller = TimerWatcher.new(
+          @poll_interval,
+          log,
+          &method(:action_poll)
+        )
+
+        @lock_monitor = TimerWatcher.new(
+          1,
+          log,
+          &method(:action_locking_monitor)
+        )
+
+        @loop.attach(@poller)
+        @loop.attach(@lock_monitor)
+      end
+
+      # Begin the logging pipeline
+      # @since 0.1.0
+      # @return [NilClass]
+      def run
+        @loop.run
+      rescue => e
+        log.error "unexpected error", :error => e
+        log.error_backtrace
+      end
+
+      # Tear down the plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def shutdown
+        @loop.watchers.each { |w| w.detach }
+        @loop.stop
+        Thread.kill(@thread)
+        @thread.join
+        shutdown_redis
+        super
+      end
+
+      # Whether to fetch a single item or a multiple items in batch
+      # @since 0.1.0
+      # @return [TrueClass, FalseClass]
+      def batched?
+        @batch_size and @batch_size > 1
+      end
+
+      # Wether the poller has been temporarily disabled or should fetch messages
+      # been temporarily disabled
+      # @since 0.1.0
+      # @return [TrueClass, FalseClass]
+      def sleeping?
+        @retry_at and @retry_at >= Engine.now
+      end
+
+      # Whether the poller has been locked
+      # @since 0.1.0
+      # @return [TrueClass, FalseClass]
+      def locked?
+        @storage.get(@lock_key)
+      end
+
+      # Set a sleep delay, ensuring that we will not attempt to fetch messages
+      # @since 0.1.0
+      # @param [Integer] delay, the amount of seconds to wait
+      # @return [Integer] timestamp when this expires
+      def sleep!(delay = @sleep_interval)
+        @retry_at = Engine.now + delay
+      end
+
+      # Poll messages from the redis server in either single message or 
+      # batch mode.
+      # @since 0.1.0
+      # @param [&block] the block to yield single messages to
+      # @return [NilClass]
+      def poll_messages
+        commands = []
+
+        if batched?
+          @redis.pipelined do
+            @batch_size.times do
+              commands << @redis.call(@command, @key)
+            end
+          end
+        else
+          commands << @redis.call(@command, @key)
+        end
+
+        commands.each do |command|
+          yield command.is_a?(Redis::Future) ? command.value : command
+        end
+      end
+
+      # Action to execute when polling for the lock key
+      # @since 0.1.0
+      # @return [NilClass]
+      def action_locking_monitor
+        lock_value = @redis.get(@lock_key)
+        @storage.put(@lock_key, lock_value)
+      end
+
+      # Action to execute when the poller event watcher executes
+      #
+      # Given that the watcher is pretty lightweight, we simply return if the
+      # worker has been set to sleep instead of actually sleeping. Doing
+      # otherwise seemed to cause locking.
+      #
+      # Otherwise we iterate through messages, parse and emit them. 
+      #
+      # @since 0.1.0
+      # @return [NilClass]
+      def action_poll
+        now = Engine.now
+        messages = []
+
+        if sleeping?
+          log.trace "redis worker is sleeping"
+          return
+        end
+
+        if locked?
+          log.trace "redis queue is locked"
+          return
+        end
+
+        poll_messages do |message|
+          if message.nil?
+            log.debug "redis queue is empty"
+            sleep!(@sleep_interval)
+            break
+          end
+
+          @parser.parse(message) do |time, record|
+            if time && record
+              router.emit @tag || @key, time || Engine.now, record
+            else
+              log.warn "failed to parse message: #{message}"
+            end
+          end
+        end
+      rescue => e
+        log.error "error fetching record", :error => e
+        log.error_backtrace
+        sleep!(@retry_interval)
+      end
+
+      # Generic Cool.io timer which will execute a given callback on schedule.
+      # @since 0.1.0
+      class TimerWatcher < Coolio::TimerWatcher
+        attr_reader :log
+
+        def initialize(interval, log, &callback)
+          @callback = callback
+          @log = log
+          super(interval, true)
+        end
+
+        def on_timer
+          @callback.call
+        rescue => e
+          log.error "unexpected error", :error => e
+          log.error_backtrace
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_mixin/redis.rb

@@ -0,0 +1,63 @@
+
+module Fluent
+  module PluginMixin
+    module Redis
+      def self.included(base)
+        base.class_eval do
+          # redis connection details
+          config_param :host,     :string,  :default => '127.0.0.1'
+          config_param :port,     :integer, :default => 6379
+          config_param :path,     :string,  :default => nil
+          config_param :password, :string,  :default => nil
+          config_param :db,       :integer, :default => 0
+          config_param :timeout,  :float,   :default => 5.0
+          config_param :driver,   :string,  :default => "ruby"
+
+          # redis list details
+          # - key: redis key of type `list` to fetch messages from
+          # - command: redis command to execute when fetching messages
+          # - batch_size: if greater than 0, fetch messages in batches
+          config_param :key,        :string,  :default => nil
+
+          # worker parameters
+          # - poll_inteval: interval between message polling actions
+          #                 *NOTE*: Apparently this must be greather than 0
+          # - sleep_interval: interval to wait after receiving 0 messages
+          # - retry_interval: interval to wait before retrying after an error
+          config_param :poll_interval,    :float, :default => 1
+          config_param :sleep_interval,   :float, :default => 5
+          config_param :retry_interval,   :float, :default => 5
+
+          attr_reader  :redis
+        end
+      end
+
+      # Initialize new input plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def initialize
+        require 'redis'
+        super
+      end
+
+      # Prepare the Redis conncection object
+      # @since 0.1.0
+      # @return [Redis]
+      def start_redis
+        @redis  = ::Redis.new(
+          :host => @host,
+          :port => @port,
+          :driver => @driver,
+          :thread_safe => true
+        )
+      end
+
+      # Destroy the Redis connection object
+      # @since 0.1.0
+      # @return [NilClass]
+      def shutdown_redis
+        @redis.quit
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-redis_list_poller
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jonathan Serafini
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-06-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: A fluentd redis input plugin supporting batch operations
+email:
+- jonathan@serafini.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGELOG.txt
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/data.sh
+- examples/standalone.conf
+- fluent-plugin-redis_list_poller.gemspec
+- lib/fluent/plugin/in_redis_list_monitor.rb
+- lib/fluent/plugin/in_redis_list_poller.rb
+- lib/fluent/plugin_mixin/redis.rb
+homepage: https://github.com/JonathanSerafini/fluent-plugin-redis_list_poller
+licenses:
+- apache2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: A fluentd redis input plugin supporting batch operations
+test_files: []
+has_rdoc: