fluent-plugin-sidekiq_metric 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7fb6a02f21d2b9f1a5438cba38f932792cd979ef
+  data.tar.gz: 678d2ba649553c195a160a107cf2ee5e9cfd8439
+SHA512:
+  metadata.gz: 03f9aa88caee11f47c26ae3f247d781fc86127bfc454b674de586d02907d44912d14996cde4aa1945d8dfa290ac49a7361f04f8a14f25247a2b87144cbd28a66
+  data.tar.gz: 8624fabee2556224acd2450859026c7a4eb9426b89eb11313833459f610a68f331240cc7e535696a325e502235d6b61eceed98e0c757c46e8a56e1e7705e9d95

data/.dockerignore

@@ -0,0 +1,2 @@
+.git
+.bundle

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/gemfiles/.bundle
+*.gemfile.lock

data/Dockerfile

@@ -0,0 +1,3 @@
+FROM ruby:2.4-alpine3.6
+
+RUN apk add --no-cache ca-certificates build-base

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 joker1007
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,120 @@
+# fluent-plugin-sidekiq_metric
+
+[Fluentd](http://fluentd.org/) input plugin to collect sidekiq metrics.
+
+## Output example
+
+```json
+{
+  "processed": 12,
+  "failed": 1,
+  "scheduled_size": 3,
+  "retry_size": 1,
+  "dead_size": 0,
+  "processes_size":1,
+  "default_queue_latency": 0,
+  "workers_size": 1,
+  "enqueued": 0
+}
+```
+
+If queue_names is set, output becomes following.
+ex: `queue_names queue1, queue2`
+
+```
+{
+  "processed": 12,
+  "failed": 1,
+  "scheduled_size": 3,
+  "retry_size": 1,
+  "dead_size": 0,
+  "processes_size":1,
+  "default_queue_latency": 0,
+  "workers_size": 1,
+  "enqueued": 0,
+  "queue1_length": 1,
+  "queue2_length": 10
+}
+```
+
+## Installation
+
+### RubyGems
+
+```
+$ gem install fluent-plugin-sidekiq_metric
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "fluent-plugin-sidekiq_metric"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+## Configuration
+
+### tag (string) (required)
+
+Tag of the output events.
+
+### redis_url (string) (required)
+
+redis URL that sidekiq uses
+
+### namespace (string) (optional)
+
+config for redis-namespace
+
+### password (string) (optional)
+
+Password for redis authentication
+
+### connect_opts (hash) (optional)
+
+Other options for redis connection
+
+Default value: `{}`.
+
+### fetch_interval (time) (optional)
+
+Interval for fetching to redis
+
+Default value: `60`.
+
+### queue_names (array) (optional)
+
+Queue names for length aggregation per queue
+
+Default value: `[]`.
+
+You can copy and paste generated documents here.
+
+## Config Example
+
+```
+<source>
+  @type sidekiq_metric
+  tag sidekiq
+  redis_url redis://redis:6379
+  fetch_interval 10s
+  queue_names default
+</source>
+
+<match sidekiq>
+  @type stdout
+</match>
+```
+
+## Copyright
+
+* Copyright(c) 2017- joker1007
+* License
+  * MIT License

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/docker-compose.yml

@@ -0,0 +1,48 @@
+version: "2"
+services:
+  redis:
+    image: redis:alpine
+
+  fluentd:
+    image: ruby:2.4-alpine3.6
+    volumes:
+      - .:/app
+      - bundle:/usr/local/bundle
+    working_dir: /app
+    environment:
+      REDIS_URL: "redis://redis:6379"
+    depends_on:
+      - redis
+      - sidekiq
+    command: ["sh", "./run_fluentd.sh"]
+
+  test:
+    image: ruby:2.4-alpine3.6
+    volumes:
+      - .:/app
+      - bundle:/usr/local/bundle
+    working_dir: /app
+    environment:
+      REDIS_URL: "redis://redis:6379"
+    depends_on:
+      - redis
+      - sidekiq
+    command: ["sh", "./run.sh"]
+
+  sidekiq:
+    build:
+      context: .
+    volumes:
+      - .:/app
+      - bundle:/usr/local/bundle
+    working_dir: /app
+    depends_on:
+      - redis
+    environment:
+      REDIS_URL: "redis://redis:6379"
+    command: ["sh", "./run_sidekiq.sh"]
+
+
+volumes:
+  bundle:
+    driver: local

data/fluent-plugin-sidekiq_metric.gemspec

@@ -0,0 +1,38 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "fluent-plugin-sidekiq_metric"
+  spec.version = "0.1.1"
+  spec.authors = ["joker1007"]
+  spec.email   = ["kakyoin.hierophant@gmail.com"]
+
+  spec.summary       = %q{sidekiq metric collector plugin for fluentd.}
+  spec.description   = %q{sidekiq metric collector plugin for fluentd.}
+  spec.homepage      = "https://github.com/joker1007/fluent-plugin-sidekiq_metric"
+  spec.license       = "MIT"
+
+  begin
+    test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
+  rescue
+    test_files, files  = Dir.glob("**/*").partition do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
+  end
+
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "sidekiq"
+
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+  spec.add_runtime_dependency "redis"
+  spec.add_runtime_dependency "oj"
+end

data/lib/fluent/plugin/in_sidekiq_metric.rb

@@ -0,0 +1,139 @@
+require "fluent/plugin/input"
+require "oj"
+require "redis"
+
+module Fluent
+  module Plugin
+    class SidekiqMetricInput < ::Fluent::Plugin::Input
+      Fluent::Plugin.register_input("sidekiq_metric", self)
+
+      helpers :timer
+
+      desc 'Tag of the output events.'
+      config_param :tag, :string
+
+      desc "redis URL that sidekiq uses"
+      config_param :redis_url, :string
+
+      desc "config for redis-namespace"
+      config_param :namespace, default: nil
+
+      desc "Password for redis authentication"
+      config_param :password, :string, secret: true, default: nil
+
+      desc "Other options for redis connection"
+      config_param :connect_opts, :hash, default: {}
+
+      desc "Interval for fetching to redis"
+      config_param :fetch_interval, :time, default: 60
+
+      desc "Queue names for length aggregation per queue"
+      config_param :queue_names, :array, default: []
+
+      def configure(conf)
+        super
+        @connect_opts = @connect_opts.map { |k, v| [k.to_sym, v] }.to_h
+      end
+
+      def start
+        super
+        @timer = timer_execute(:sidekiq_metric_timer, @fetch_interval, &method(:run))
+      end
+
+      def run
+        stats = fetch_stats
+        queue_lengths = fetch_queue_lengths
+        record = stats.merge(queue_lengths)
+        router.emit(@tag, Fluent::EventTime.now, record)
+      end
+
+      ## From sidekiq gem (lib/sidekiq/api.rb)
+      def fetch_stats
+        pipe1_res = redis.pipelined do
+          redis.get('stat:processed'.freeze)
+          redis.get('stat:failed'.freeze)
+          redis.zcard('schedule'.freeze)
+          redis.zcard('retry'.freeze)
+          redis.zcard('dead'.freeze)
+          redis.scard('processes'.freeze)
+          redis.lrange('queue:default'.freeze,  -1,  -1)
+          redis.smembers('processes'.freeze)
+          redis.smembers('queues'.freeze)
+        end
+
+        pipe2_res = redis.pipelined do
+          pipe1_res[7].each { |key| redis.hget(key, 'busy'.freeze) }
+          pipe1_res[8].each { |queue| redis.llen("queue:#{queue}") }
+        end
+
+        s = pipe1_res[7].size
+        workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
+        enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
+
+        default_queue_latency =
+          if (entry = pipe1_res[6].first)
+            job = Oj.load(entry) rescue {}
+            now = Time.now.to_f
+            thence = job['enqueued_at'.freeze] || now
+            now - thence
+          else
+            0
+          end
+
+        {
+          processed:             pipe1_res[0].to_i,
+          failed:                pipe1_res[1].to_i,
+          scheduled_size:        pipe1_res[2],
+          retry_size:            pipe1_res[3],
+          dead_size:             pipe1_res[4],
+          processes_size:        pipe1_res[5],
+
+          default_queue_latency: default_queue_latency,
+          workers_size:          workers_size,
+          enqueued:              enqueued
+        }
+      end
+
+      ## From sidekiq gem (lib/sidekiq/api.rb)
+      def fetch_queue_lengths
+        return {} if @queue_names.empty?
+
+        queues = redis.smembers('queues'.freeze) & @queue_names
+        return {} if queues.empty?
+
+        lengths = redis.pipelined do
+          queues.each do |queue|
+            redis.llen("queue:#{queue}")
+          end
+        end
+
+        i = 0
+        array_of_arrays = queues.inject({}) do |memo, queue|
+          memo["#{queue}_length"] = lengths[i]
+          i += 1
+          memo
+        end
+
+        Hash[array_of_arrays]
+      end
+
+      def redis
+        @redis ||=
+          if @namespace
+            client = Redis.new(url: @redis_url, **@connect_opts).tap do |cl|
+              cl.auth(@password) if @password
+            end
+            Redis::Namespace.new(@namespace, redis: client)
+          else
+            Redis.new(url: @redis_url, **@connect_opts).tap do |cl|
+              cl.auth(@password) if @password
+            end
+          end
+      end
+
+      def clear_redis
+        @redis = nil
+      end
+    end
+  end
+end

data/run.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+bundle install
+exec bundle exec rake 

data/run_fluentd.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+bundle install
+exec bundle exec fluentd -c ./test/fluentd.conf 

data/run_sidekiq.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+bundle install
+exec bundle exec sidekiq -r ./test/sidekiq_config.rb

data/test/fluentd.conf

@@ -0,0 +1,11 @@
+<source>
+  @type sidekiq_metric
+  tag sidekiq
+  redis_url redis://redis:6379
+  fetch_interval 10s
+  queue_names default
+</source>
+
+<match sidekiq>
+  @type stdout
+</match>

data/test/helper.rb

@@ -0,0 +1,16 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/input"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)
+
+require "sidekiq"
+
+Sidekiq.configure_client do |config|
+  config.redis = {url: ENV["REDIS_URL"]}
+end
+
+require_relative "./job"

data/test/job.rb

@@ -0,0 +1,7 @@
+class Job
+  include Sidekiq::Worker
+
+  def perform
+    :done
+  end
+end

data/test/plugin/test_in_sidekiq_metric.rb

@@ -0,0 +1,70 @@
+require "helper"
+require "fluent/plugin/in_sidekiq_metric.rb"
+
+class Sidekiq_metricInputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  DEFAULT_CONF = %q{
+    tag test
+    redis_url redis://redis:6379
+    fetch_interval 3s
+    queue_names default
+  }
+
+  test "configure" do
+    driver = create_driver(<<~CONF)
+      tag test
+      redis_url redis://redis:6379
+      connect_opts {"foo": "bar", "hoge": "fuga"}
+      queue_names queue1, queue2
+    CONF
+    assert { driver.instance.tag == "test" }
+    assert { driver.instance.redis_url == "redis://redis:6379" }
+    assert { driver.instance.connect_opts == {foo: "bar", hoge: "fuga"} }
+    assert { driver.instance.queue_names == ["queue1", "queue2"] }
+  end
+
+  test "redis" do
+    driver = create_driver
+    assert { driver.instance.redis.ping == "PONG" }
+  end
+
+  test "fetch_stats" do
+    driver = create_driver
+    Job.perform_async
+    sleep 5
+    stats = driver.instance.fetch_stats
+    assert_operator(stats[:processed], ">", 0)
+    keys = [:processed, :failed, :scheduled_size, :retry_size, :dead_size, :processes_size, :default_queue_latency, :workers_size, :enqueued]
+    keys.each do |k|
+      assert_kind_of(Integer, stats[k])
+    end
+  end
+
+  test "fetch_queue_lengths" do
+    driver = create_driver
+    queue_lengths = driver.instance.fetch_queue_lengths
+    assert_kind_of(Integer, queue_lengths["default_length"])
+  end
+
+  test "emit" do
+    driver = create_driver
+    driver.run(expect_emits: 2, timeout: 10)
+    assert { driver.events.length > 0 }
+    driver.events.each do |ev|
+      assert_equal("test", ev[0])
+      keys = [:processed, :failed, :scheduled_size, :retry_size, :dead_size, :processes_size, :default_queue_latency, :workers_size, :enqueued]
+      keys.each do |k|
+        assert_kind_of(Integer, ev[2][k])
+      end
+    end
+  end
+
+  private
+
+  def create_driver(conf = DEFAULT_CONF)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SidekiqMetricInput).configure(conf)
+  end
+end

data/test/sidekiq_config.rb

@@ -0,0 +1,9 @@
+Sidekiq.configure_server do |config|
+  config.redis = {url: ENV["REDIS_URL"]}
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = {url: ENV["REDIS_URL"]}
+end
+
+require_relative "./job"

metadata

@@ -0,0 +1,171 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-sidekiq_metric
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- joker1007
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: sidekiq metric collector plugin for fluentd.
+email:
+- kakyoin.hierophant@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".dockerignore"
+- ".gitignore"
+- Dockerfile
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- docker-compose.yml
+- fluent-plugin-sidekiq_metric.gemspec
+- lib/fluent/plugin/in_sidekiq_metric.rb
+- run.sh
+- run_fluentd.sh
+- run_sidekiq.sh
+- test/fluentd.conf
+- test/helper.rb
+- test/job.rb
+- test/plugin/test_in_sidekiq_metric.rb
+- test/sidekiq_config.rb
+homepage: https://github.com/joker1007/fluent-plugin-sidekiq_metric
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: sidekiq metric collector plugin for fluentd.
+test_files:
+- test/fluentd.conf
+- test/helper.rb
+- test/job.rb
+- test/plugin/test_in_sidekiq_metric.rb
+- test/sidekiq_config.rb