fluent-plugin-sidekiq 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 059e756a9e73cf254ee157e555267df9261a9a13
-  data.tar.gz: 7e3d0983c645de754f06f07256ba1c45410b0ea8
+SHA256:
+  metadata.gz: 8f504b306f8ff6c58c7d4301481958095054d689b44a0eb47814baa496bbbc15
+  data.tar.gz: 1d3622dbb05fca9ca6cc58dc55e3005acd90f8be778a698c00feda974bb0c54a
 SHA512:
-  metadata.gz: 1ce4437c823b089f05d6b3272dcad3d45ed5ea3a615f896ed1c8ec850a13ac51adc65f0b363873791380c756842b219a26a1f3e12333f681ebd1decdac7461f4
-  data.tar.gz: 5df444a5655b90174b24e52a84543a34d7d96cf23cb124d098bd46c2128a268ee3a637b99d00d38b759cced62f91d65dd38b636386593d808102cf9cb97ea8fd
+  metadata.gz: d266e1e43afa813a59f827439b58fc2b32a6a40e125060b76db612d32d735defce90cc33e41f410a826e2dc66af67f7bb27ab5e4523300a4164977bf46408fb4
+  data.tar.gz: df123481dddee992b042d27ecd75fc40c3581ed695d1a22375259741a9b9f76d9bad7c179213af1e9a094cf93a44b7389fc57302fe9833b0043f747940d3cab0

data/README.md

@@ -1,4 +1,4 @@
-# Fluent::Plugin::Sidekiq
+# Fluent::Plugin::Sidekiq, a plugin for [Fluentd](http://fluentd.org)
 
 TODO: Write a gem description
 

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.1.0

data/fluent-plugin-sidekiq.gemspec

@@ -17,9 +17,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "fluentd"
-  spec.add_dependency "redis"
+  spec.add_dependency "redis", "~> 3.0"
   spec.add_dependency "hiredis"
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "fluentd"
 end

data/lib/fluent/plugin/out_sidekiq.rb

@@ -1,43 +1,100 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/output'
 require "redis"
 
-class Fluent::SidekiqOutput < Fluent::BufferedOutput
-  VERSION = "0.0.2"
-  Fluent::Plugin.register_output("sidekiq", self)
+module Fluent
+  module Plugin
+    class SidekiqOutput < Output
+      class Batch
+        attr_accessor :queue, :klass, :payload
 
-  config_param :redis_url, :string, :default => 'redis://localhost:6379'
-  config_param :redis_namespace, :string, :default => nil
+        def initialize(q, k)
+          self.queue = q
+          self.klass = k
+          self.payload = {:class => k, :retry => true, :jid => "", :enqueued_at => 0, :args => [[]] }
+        end
 
-  def start
-    super
-  end
+        def acceptable_batch(q, c, p, max_size)
+          q == queue && c == klass && (payload[:args][0].length + p.length) <= max_size
+        end
 
-  def shutdown
-    super
-  end
+        def add_to_batch(p)
+          payload[:retry] = p['retry']
+          payload[:jid] = p['jid']
+          payload[:enqueued_at] = p['enqueued_at']
+          payload[:args][0] += p['args'][0]
+        end
 
-  def format(tag, time, record)
-    [tag, time, record].to_msgpack
-  end
+        def enqueue(client)
+          client.sadd("queues", queue)
+          client.lpush("queue:#{queue}", JSON.generate(payload))
+        end
+      end
 
-  def redis_client
-    opts = {url: @redis_url, driver: :hiredis}
-    client = Redis.new(opts)
-    if @redis_namespace
-      require "redis/namespace"
-      Redis::Namespace.new(@redis_namespace, client)
-    else
-      client
-    end
-  end
+      VERSION = "0.1.0"
+      Fluent::Plugin.register_output("sidekiq", self)
+
+      config_param :redis_url, :string, :default => 'redis://localhost:6379'
+      config_param :redis_namespace, :string, :default => nil
+      config_param :max_batch_size, :integer, :default => 100
+
+      def format(tag, time, record)
+        [tag, time, record].to_msgpack
+      end
+
+      def formatted_to_msgpack_binary
+        true
+      end
 
-  def write(chunk)
-    client = redis_client
-    client.multi do
-      chunk.msgpack_each do |tag, time, data|
-        at = data.delete('at')
-        if at
-          client.zadd('schedule', [at, data['payload']])
+      def redis_client
+        opts = {url: @redis_url, driver: :hiredis}
+        client = Redis.new(opts)
+        if @redis_namespace
+          require "redis/namespace"
+          Redis::Namespace.new(@redis_namespace, client)
         else
+          client
+        end
+      end
+
+      def write(chunk)
+        client = redis_client
+        scheduled_jobs = []
+        batches = []
+        manually_queue = []
+
+        chunk.msgpack_each do |tag, time, data|
+          at = data.delete('at')
+          if at
+            scheduled_jobs << [at, data['payload']]
+          else
+            payload = JSON.parse(data['payload'])
+            if payload['args'][0].kind_of?(Array) && payload['args'][0].all? { |a| a.kind_of?(Hash) }
+              queue = data.delete('queue')
+              klass = payload['class']
+
+              batch = batches.find { |b| b.acceptable_batch(queue, klass, payload['args'][0], max_batch_size) }
+              if !batch
+                batch = Batch.new(queue, klass)
+                batches << batch
+              end
+
+              batch.add_to_batch(payload)
+            else
+              # Manual queuing
+              manually_queue << data
+            end
+          end
+        end
+
+        if scheduled_jobs.length > 0
+          client.zadd('schedule', scheduled_jobs)
+        end
+        batches.each do |batch|
+          batch.enqueue(client)
+        end
+        manually_queue.each do |data|
           queue = data.delete('queue')
           client.sadd('queues', queue)
           client.lpush("queue:#{queue}", data['payload'])

metadata

@@ -1,83 +1,83 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors:
 - Alex Scarborough
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-12-21 00:00:00.000000000 Z
+date: 2022-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: redis
+  name: hiredis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: hiredis
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '1.3'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Sidekiq plugin for Fluentd
@@ -87,9 +87,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .ruby-gemset
-- .ruby-version
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -101,24 +101,23 @@ homepage: https://github.com/GoCarrot/fluent-plugin-sidekiq
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.1.11
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Sidekiq plugin for Fluentd
 test_files: []