fluent-plugin-dd 0.1.4 → 0.1.5

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODQ1NTNjMTEyZjZhOTAyNTAzOGFlN2VlMDQ2MGFjMzYxZmM4ZmI5ZQ==
-  data.tar.gz: !binary |-
-    OWQxOTE3ZmQzM2Y2NmZkODZhYjlhNjkxMDhkNWJiN2ZjYTYxYTI2Nw==
+SHA1:
+  metadata.gz: 8abcfb0fc04ae21036b74510e140eb471c583ec1
+  data.tar.gz: bb1cd7f2340cb5d14a5fcafed9ab58d13923afd0
 SHA512:
-  metadata.gz: !binary |-
-    YTIyMmE0YmMwNWU4NDNlZGM0YTRlNjM1NWY4OGQ5YzZlMGI5MGExNjRjODdj
-    M2Q2OWFmMGYwMGJhNjAxZTBlMDg2NzkwZmNiOTBjMzc1NWVmNTgyOWYyMThl
-    YWQ2YjA1ODUwNDYxZDdlYTMyMWY0Y2I3ZTczYTljNDE1MDM2NDY=
-  data.tar.gz: !binary |-
-    ZmEzNjMwMTQ5NDFkODI3ZTM5ODgyMzNmZjI1M2Y5YTc2NTJkYmIwN2JlM2Ix
-    M2I3NGU3ZGFiZWJlNmE2YjRhZDY0Y2Y2MjM0ZTA0NTgzNmEyYzc5YjY3ZjM0
-    NzU5YzdkYTYwNDdkZGVmZWE3MGEwZTU0MjVjMzJlNjg5NTJlOGU=
+  metadata.gz: 0be3eeb5ba3ee78de16a52c597031d4ee943771bed50743f6ccf60a8058f1e0077a40742ea9efa09e4fd693326a71afbae55a7d250920eca807ada8fb6db49a3
+  data.tar.gz: 0235abf277ff35648cfdcbff3a05219a78cfa1c147a0bc1eb144c57aae27ad5a5938cd9711f7fbd9bd14599d9c8a5ecd205ea7e809ff07a8673ee985fd0944a1

data/fluent-plugin-dd.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-dd'
-  spec.version       = '0.1.4'
+  spec.version       = '0.1.5'
   spec.authors       = ['Genki Sugawara']
   spec.email         = ['sugawara@cookpad.com']
   spec.description   = %q{Output plugin for Datadog}

data/lib/fluent/plugin/out_dd.rb

@@ -9,6 +9,7 @@ class Fluent::DdOutput < Fluent::BufferedOutput
   config_param :host, :string, :default => nil
   config_param :use_fluentd_tag_for_datadog_tag, :bool, :default => false
   config_param :emit_in_background, :bool, :default => false
+  config_param :concurrency, :integer, :default => nil
 
   def initialize
     super
@@ -23,10 +24,12 @@ class Fluent::DdOutput < Fluent::BufferedOutput
     if @emit_in_background
       @queue = Queue.new
 
-      @thread = Thread.start do
-        while(chunk = @queue.pop)
-          emit_points(chunk)
-          Thread.pass
+      @threads = @concurrency.times.map do
+        Thread.start do
+          while (job = @queue.pop)
+            emit_points(*job)
+            Thread.pass
+          end
         end
       end
     end
@@ -36,8 +39,12 @@ class Fluent::DdOutput < Fluent::BufferedOutput
     super
 
     if @emit_in_background
-      @queue.push(false)
-      @thread.join
+      @threads.size.times do
+        @queue.push(false)
+      end
+      @threads.each do |thread|
+        thread.join
+      end
     end
   end
 
@@ -48,6 +55,11 @@ class Fluent::DdOutput < Fluent::BufferedOutput
       raise Fluent::ConfigError, '`dd_api_key` is required'
     end
 
+    if !@emit_in_background && @concurrency
+      raise Fluent::ConfigError, '`concurrency` should be used with `emit_in_background`'
+    end
+    @concurrency ||= 1
+
     unless @host
       @host = %x[hostname -f 2> /dev/null].strip
       @host = Socket.gethostname if @host.empty?
@@ -61,16 +73,24 @@ class Fluent::DdOutput < Fluent::BufferedOutput
   end
 
   def write(chunk)
-    if @emit_in_background
-      @queue.push(chunk)
-    else
-      emit_points(chunk)
+    jobs = chunk_to_jobs(chunk)
+
+    jobs.each do |job|
+      if @emit_in_background
+        @queue.push(job)
+      else
+        emit_points(*job)
+      end
     end
   end
 
   private
 
-  def emit_points(chunk)
+  def emit_points(metric, points, options)
+    @dog.emit_points(metric, points, options)
+  end
+
+  def chunk_to_jobs(chunk)
     enum = chunk.to_enum(:msgpack_each)
 
     enum.select {|tag, time, record|
@@ -87,7 +107,7 @@ class Fluent::DdOutput < Fluent::BufferedOutput
       end
 
       [dd_tag] + record.values_at('metric', 'host', 'type')
-    }.each {|i, records|
+    }.map {|i, records|
       tag, metric, host, type = i
 
       points = records.map do |tag, time, record|
@@ -101,7 +121,7 @@ class Fluent::DdOutput < Fluent::BufferedOutput
       options[:host] = host if host
       options[:type] = type if type
 
-      @dog.emit_points(metric, points, options)
+      [metric, points, options]
     }
   end
 end

data/spec/out_dd_spec.rb

@@ -22,17 +22,51 @@ describe Fluent::DdOutput do
     end
   end
 
-  it 'should be called emit_points in the background' do
-    run_driver(:emit_in_background => true) do |d, dog|
-      dog.should_receive(:emit_points).with(
-        "some.metric.name",
-        [[Time.parse("2014-02-08 04:14:15 UTC"), 50.0],
-         [Time.parse("2014-02-08 04:14:15 UTC"), 100.0]],
-        {}
-      )
+  context 'with emit_in_background' do
+    before do
+      $threads_array_for_test = []
+    end
 
-      d.emit({"metric" => "some.metric.name", "value" => 50.0}, time)
-      d.emit({"metric" => "some.metric.name", "value" => 100.0}, time)
+    after do
+      $threads_array_for_test = nil
+    end
+
+    it 'should be called emit_points in the background' do
+      run_driver(:emit_in_background => true) do |d, dog|
+        dog.should_receive(:emit_points).with(
+          "some.metric.name",
+          [[Time.parse("2014-02-08 04:14:15 UTC"), 50.0],
+           [Time.parse("2014-02-08 04:14:15 UTC"), 100.0]],
+          {}
+        )
+
+        d.emit({"metric" => "some.metric.name", "value" => 50.0}, time)
+        d.emit({"metric" => "some.metric.name", "value" => 100.0}, time)
+      end
+    end
+
+    it 'should be called emit_points in the background and in the different thread' do
+      run_driver(
+        :emit_in_background => true,
+        :concurrency => 2,
+      ) do |d, dog|
+        dog.should_receive(:emit_points).with(
+          "some.metric.name.1",
+          [[Time.parse("2014-02-08 04:14:15 UTC"), 50.0]],
+          {}
+        )
+        dog.should_receive(:emit_points).with(
+          "some.metric.name.2",
+          [[Time.parse("2014-02-08 04:14:15 UTC"), 100.0]],
+          {}
+        )
+
+        d.emit({"metric" => "some.metric.name.1", "value" => 50.0}, time)
+        d.emit({"metric" => "some.metric.name.2", "value" => 100.0}, time)
+      end
+
+      expect($threads_array_for_test.size).to eq(2)
+      expect($threads_array_for_test[0]).not_to eq($threads_array_for_test[1])
     end
   end
 

data/spec/spec_helper.rb

@@ -6,6 +6,18 @@ require 'time'
 # Disable Test::Unit
 module Test::Unit::RunCount; def run(*); end; end
 
+class Fluent::DdOutput < Fluent::BufferedOutput
+  private
+  alias_method :orig_emit_points, :emit_points
+
+  def emit_points(*args)
+    if $threads_array_for_test
+      $threads_array_for_test << Thread.current
+    end
+    orig_emit_points(*args)
+  end
+end
+
 RSpec.configure do |config|
   config.before(:all) do
     Fluent::Test.setup

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dd
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-25 00:00:00.000000000 Z
+date: 2014-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: dogapi
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -56,28 +56,28 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.11.0
 description: Output plugin for Datadog
@@ -107,20 +107,21 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Output plugin for Datadog
 test_files:
 - spec/out_dd_spec.rb
 - spec/spec_helper.rb
+has_rdoc: