fluent-plugin-sidekiq 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 059e756a9e73cf254ee157e555267df9261a9a13
-  data.tar.gz: 7e3d0983c645de754f06f07256ba1c45410b0ea8
+  metadata.gz: 94201545914e3c1a8a9c1f46f2900844bf3fe666
+  data.tar.gz: 905b90eba9fa9a355018e0ab69e3092c4fe32ef0
 SHA512:
-  metadata.gz: 1ce4437c823b089f05d6b3272dcad3d45ed5ea3a615f896ed1c8ec850a13ac51adc65f0b363873791380c756842b219a26a1f3e12333f681ebd1decdac7461f4
-  data.tar.gz: 5df444a5655b90174b24e52a84543a34d7d96cf23cb124d098bd46c2128a268ee3a637b99d00d38b759cced62f91d65dd38b636386593d808102cf9cb97ea8fd
+  metadata.gz: abaef90d540ef6dc3efc2a80c3073dc09911a7ee5e7fd38298d2e22c807e351d5fb0ae22e7e1196a8c62d5ee8c8efceac1d4ccb94d5a712186d3d555b70bb329
+  data.tar.gz: 054e8fcda8959847b1b5953f5d9e7dc5f82f5424ec22e4650cbf8f186eef667462266339372f2c3e361ff73327ed90222a9329211c99e6865c6a8a2f69734e1e

data/README.md

@@ -1,4 +1,4 @@
-# Fluent::Plugin::Sidekiq
+# Fluent::Plugin::Sidekiq, a plugin for [Fluentd](http://fluentd.org)
 
 TODO: Write a gem description
 

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.0.4

data/lib/fluent/plugin/out_sidekiq.rb

@@ -1,11 +1,38 @@
 require "redis"
 
 class Fluent::SidekiqOutput < Fluent::BufferedOutput
-  VERSION = "0.0.2"
+  class Batch
+    attr_accessor :queue, :klass, :payload
+
+    def initialize(q, k)
+      self.queue = q
+      self.klass = k
+      self.payload = {:class => k, :retry => true, :jid => "", :enqueued_at => 0, :args => [[]] }
+    end
+
+    def acceptable_batch(q, c, p, max_size)
+      q == queue && c == klass && (payload[:args][0].length + p.length) <= max_size
+    end
+
+    def add_to_batch(p)
+      payload[:retry] = p['retry']
+      payload[:jid] = p['jid']
+      payload[:enqueued_at] = p['enqueued_at']
+      payload[:args][0] += p['args'][0]
+    end
+
+    def enqueue(client)
+      client.sadd("queues", queue)
+      client.lpush("queue:#{queue}", JSON.generate(payload))
+    end
+  end
+
+  VERSION = "0.0.4"
   Fluent::Plugin.register_output("sidekiq", self)
 
   config_param :redis_url, :string, :default => 'redis://localhost:6379'
   config_param :redis_namespace, :string, :default => nil
+  config_param :max_batch_size, :integer, :default => 100
 
   def start
     super
@@ -32,17 +59,44 @@ class Fluent::SidekiqOutput < Fluent::BufferedOutput
 
   def write(chunk)
     client = redis_client
-    client.multi do
-      chunk.msgpack_each do |tag, time, data|
-        at = data.delete('at')
-        if at
-          client.zadd('schedule', [at, data['payload']])
-        else
+    scheduled_jobs = []
+    batches = []
+    manually_queue = []
+
+    chunk.msgpack_each do |tag, time, data|
+      at = data.delete('at')
+      if at
+        scheduled_jobs << [at, data['payload']]
+      else
+        payload = JSON.parse(data['payload'])
+        if payload['args'][0].kind_of?(Array) && payload['args'][0].all? { |a| a.kind_of?(Hash) }
           queue = data.delete('queue')
-          client.sadd('queues', queue)
-          client.lpush("queue:#{queue}", data['payload'])
+          klass = payload['class']
+
+          batch = batches.find { |b| b.acceptable_batch(queue, klass, payload['args'][0], max_batch_size) }
+          if !batch
+            batch = Batch.new(queue, klass)
+            batches << batch
+          end
+
+          batch.add_to_batch(payload)
+        else
+          # Manual queuing
+          manually_queue << data
         end
       end
     end
+
+    if scheduled_jobs.length > 0
+      client.zadd('schedule', scheduled_jobs)
+    end
+    batches.each do |batch|
+      batch.enqueue(client)
+    end
+    manually_queue.each do |data|
+      queue = data.delete('queue')
+      client.sadd('queues', queue)
+      client.lpush("queue:#{queue}", data['payload'])
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Alex Scarborough
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-21 00:00:00.000000000 Z
+date: 2014-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd