logstash-output-honeycomb_json_batch 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c90c885174ae440377b402eac734980422d3192d
-  data.tar.gz: c6e29760a336290dece1dd18bbc33f10b2b2fd2d
+  metadata.gz: d0021f21a46eb6dfb4685f88f6a3350e8c32e7e9
+  data.tar.gz: b1838276379b83acb24f4b6c13836922ebb8028c
 SHA512:
-  metadata.gz: b9c755b2655e7218d4141c822a0a784cc1fc6006f943c891c1d7d2a993ce2f1c7436717dca1c860a9e016ce8cde6355f4a4c8f98647240d4951b855fca7986d1
-  data.tar.gz: 6584d584c082375794988dd6cbe94c4410acc89d478c760ff62c03de8f5817421b4e358f6a6d4956285ada7c8da390441f0d2f09c880aab910fcf168cea49866
+  metadata.gz: f01e1628b5af907e299762afbf2fb2911beabbda20dd1a37db185af9b80e641a5f211d521a8a211a80394c3b682fc7748d3bf5e725cd08690030b89cb4256637
+  data.tar.gz: 786eeb856cafd6e6e5e30ab31faeb00748650d58081835f4fb7337ed2bfa4b14f0914bf8522d44a392a5e637bd72e20e77bd10a1823bb50d35339e8646a3ba20

data/README.md

@@ -1,4 +1,4 @@
-# logstash-output-honeycomb_json_batch
+# logstash-output-honeycomb_json_batch [![Gem Version](https://badge.fury.io/rb/logstash-output-honeycomb_json_batch.svg)](https://badge.fury.io/rb/logstash-output-honeycomb_json_batch)
 
 A logstash plugin for interacting with [Honeycomb](https://honeycomb.io) at high volumes. (See here for more information about [using Honeycomb](https://honeycomb.io/intro/) and [its libraries](https://honeycomb.io/docs/send-data/sdks).)
 
@@ -14,11 +14,13 @@ The easiest way to use this plugin is by installing it through rubygems like any
 bin/logstash-plugin install logstash-output-honeycomb_json_batch
 ```
 
-## Usage
+## Compatibility
+
+This plugin requires Logstash 2.4 or greater. Please open an issue if you require support for older versions.
 
-The default batch size is 50, the default flush interval is 5 seconds, and each of those can be overridden via the plugin config.
+## Usage
 
-A simple config to test this might be:
+A simple config is:
 
 ```
 input {
@@ -35,12 +37,7 @@ output {
 ```
 
 Additional arguments to `honeycomb_json_batch`:
-
-Consider these when tuning performance:
-
-- `flush_size`: Default batch size, defaults to 50
-- `idle_flush_time`: Default flush interval in seconds, defaults to 5
-- `pool_max`: Maximum number of requests to be run in parallel, defaults to 10
+- `flush_size`: Maximum batch size, defaults to 75
 - `retry_individual`: On failed requests, whether to retry event sends individually, defaults to true
 - `api_host`: Allows you to override the Honeycomb host, defaults to https://api.honeycomb.io
 

data/lib/logstash/outputs/honeycomb_json_batch.rb

@@ -1,14 +1,15 @@
 # encoding: utf-8
+require "enumerator"
 require "logstash/outputs/base"
 require "logstash/namespace"
 require "logstash/json"
 require "uri"
-require "stud/buffer"
 require "logstash/plugin_mixins/http_client"
 
 class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
   include LogStash::PluginMixins::HttpClient
-  include Stud::Buffer
+
+  concurrency :shared
 
   config_name "honeycomb_json_batch"
 
@@ -18,24 +19,18 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
 
   config :dataset, :validate => :string, :required => true
 
-  config :flush_size, :validate => :number, :default => 50
+  config :retry_individual, :validate => :boolean, :default => true
 
-  config :idle_flush_time, :validate => :number, :default => 5
+  config :flush_size, :validate => :number, :default => 75
 
-  config :retry_individual, :validate => :boolean, :default => true
+  # The following configuration options are deprecated and do nothing.
+  config :idle_flush_time, :validate => :number, :default => 5
 
   config :pool_max, :validate => :number, :default => 10
 
   def register
-    # We count outstanding requests with this queue
-    # This queue tracks the requests to create backpressure
-    # When this queue is empty no new requests may be sent,
-    # tokens must be added back by the client on success
-    @request_tokens = SizedQueue.new(@pool_max)
-    @pool_max.times {|t| @request_tokens << true }
     @total = 0
     @total_failed = 0
-    @requests = Array.new
     if @api_host.nil?
       @api_host = "https://api.honeycomb.io"
     elsif !@api_host.start_with? "http"
@@ -43,94 +38,65 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
     end
     @api_host = @api_host.chomp
 
-    buffer_initialize(
-      :max_items => @flush_size,
-      :max_interval => @idle_flush_time,
-      :logger => @logger
-    )
     logger.info("Initialized honeycomb_json_batch with settings",
-      :flush_size => @flush_size,
-      :idle_flush_time => @idle_flush_time,
-      :request_tokens => @pool_max,
       :api_host => @api_host,
       :headers => request_headers,
       :retry_individual => @retry_individual)
-
-  end
-
-  # This module currently does not support parallel requests as that would circumvent the batching
-  def receive(event, async_type=:background)
-    buffer_receive(event)
   end
 
   def close
-    buffer_flush(:final => true)
     client.close
   end
 
-  public
-  def flush(events, close=false)
-    documents = []  #this is the array of hashes that we push to Fusion as documents
-
-    events.each do |event|
-      data = event.to_hash()
-      timestamp = data.delete("@timestamp")
-      doc = { "time" => timestamp, "data" => data }
-      if samplerate = data.delete("@samplerate")
-        doc["samplerate"] = samplerate.to_i
+  def multi_receive(events)
+    events.each_slice(@flush_size) do |chunk|
+      documents = []
+      chunk.each do |event|
+        data = event.to_hash()
+        timestamp = data.delete("@timestamp")
+        doc = { "time" => timestamp, "data" => data }
+        if samplerate = data.delete("@samplerate")
+          doc["samplerate"] = samplerate.to_i
+        end
+        documents.push(doc)
       end
-      documents.push(doc)
+      make_request(documents)
     end
-
-    make_request(documents)
-  end
-
-  def multi_receive(events)
-    events.each {|event| buffer_receive(event)}
   end
 
   private
 
   def make_request(documents)
     body = LogStash::Json.dump({ @dataset => documents })
-    # Block waiting for a token
-    token = @request_tokens.pop
-    @logger.debug("Got token", :tokens => @request_tokens.length)
 
-
-    # Create an async request
-    begin
-      request = client.post("#{@api_host}/1/batch", {
-        :body => body,
-        :headers => request_headers,
-        :async => true
-      })
-    rescue Exception => e
-      @logger.warn("An error occurred while indexing: #{e.message}")
-    end
-
-    # attach handlers before performing request
-    request.on_complete do
-      # Make sure we return the token to the pool
-      @request_tokens << token
-    end
+    url = "#{@api_host}/1/batch"
+    request = client.post(url, {
+      :body => body,
+      :headers => request_headers
+    })
 
     request.on_success do |response|
       if response.code >= 200 && response.code < 300
         @total = @total + documents.length
-        @logger.debug("Successfully submitted", 
+        @logger.debug("Successfully submitted",
           :docs => documents.length,
           :response_code => response.code,
           :total => @total)
       else
         if documents.length > 1 && @retry_individual
           if statuses = JSON.parse(response.body).values.first
-            status.each_with_index do |status, i|
-              next if status >= 200 && status < 300
+            statuses.each_with_index do |status, i|
+              code = status["status"]
+              if code == nil
+                @logger.warn("Status code missing in response: #{status}")
+                next
+              elsif code >= 200 && code < 300
+                next
+              end
               make_request([documents[i]])
             end
           end
-        else 
+        else
           @total_failed += documents.length
           log_failure(
               "Encountered non-200 HTTP code #{response.code}",
@@ -150,7 +116,7 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
         :url => url,
         :method => @http_method,
         :body => body,
-        :headers => headers,
+        :headers => request_headers,
         :message => exception.message,
         :class => exception.class.name,
         :backtrace => exception.backtrace,
@@ -158,7 +124,8 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
       )
     end
 
-    client.execute!
+    request.call
+
   rescue Exception => e
     log_failure("Got totally unexpected exception #{e.message}", :docs => documents.length)
   end

data/logstash-output-honeycomb_json_batch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-honeycomb_json_batch'
-  s.version         = '0.1.2'
+  s.version         = '0.2.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "This output lets you `POST` batches of events to the Honeycomb.io API endpoint"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/honeycomb_json_batch_spec.rb

@@ -33,22 +33,11 @@ describe LogStash::Outputs::HoneycombJSONBatch do
 
   it "should receive a single post request" do
     expect(client).to receive(:post).
-                        with("#{ api_host }/1/batch", hash_including(:body, :headers, :async)).
+                        with("#{ api_host }/1/batch", hash_including(:body, :headers)).
                         once.
                         and_call_original
 
-    5.times {|t| @honeycomb.receive(event)}
-    @honeycomb.buffer_flush(:force => true)
-  end
-
-  it "should send batches based on the specified flush_size" do
-    expect(client).to receive(:post).
-                        with("#{ api_host }/1/batch", hash_including(:body, :headers, :async)).
-                        twice.
-                        and_call_original
-
-    (flush_size + 1).times {|t| @honeycomb.receive(event)}
-    @honeycomb.buffer_flush(:force => true)
+    @honeycomb.multi_receive([event])
   end
 
   it "should attach the right headers for Honeycomb ingestion" do
@@ -59,11 +48,10 @@ describe LogStash::Outputs::HoneycombJSONBatch do
                         })).once.
                         and_call_original
 
-    @honeycomb.receive(event)
-    @honeycomb.buffer_flush(:force => true)
+    @honeycomb.multi_receive([event])
   end
 
-  it "should wrap events in the right structure Honeycomb ingestion" do
+  it "should wrap events in the right structure for Honeycomb ingestion" do
     data = event.to_hash()
     data.delete("@timestamp")
     expect(client).to receive(:post).
@@ -71,25 +59,23 @@ describe LogStash::Outputs::HoneycombJSONBatch do
                           DATASET => [ { "time" => event.timestamp.to_s, "data" => data } ]
                         }))).once.
                         and_call_original
-
-    @honeycomb.receive(event)
-    @honeycomb.buffer_flush(:force => true)
+    @honeycomb.multi_receive([event])
   end
 
   it "should extract timestamp and samplerate from the data" do
-    with_samplerate = LogStash::Event.new("alpha" => 1.0, "@samplerate" => "17.5")
+    with_samplerate = LogStash::Event.new("alpha" => 1.0, "@samplerate" => "17.5",
+                                          "@timestamp" => "2014-11-17T20:37:17.223Z")
     data = with_samplerate.to_hash()
     data.delete("@timestamp")
     data.delete("@samplerate")
 
     expect(client).to receive(:post).
                         with("#{ api_host }/1/batch", hash_including(:body => LogStash::Json.dump({
-                          DATASET => [ { "time" => event.timestamp.to_s, "data" => data, "samplerate" => 17 } ]
+                          DATASET => [ { "time" => with_samplerate.timestamp.to_s, "data" => data, "samplerate" => 17 } ]
                         }))).once.
                         and_call_original
 
-    @honeycomb.receive(with_samplerate)
-    @honeycomb.buffer_flush(:force => true)
+    @honeycomb.multi_receive([with_samplerate])
   end
 
   it "should wrap multiple events up in the right structure" do
@@ -107,9 +93,16 @@ describe LogStash::Outputs::HoneycombJSONBatch do
                         }))).once.
                         and_call_original
 
-    @honeycomb.receive(event1)
-    @honeycomb.receive(event2)
-    @honeycomb.receive(event3)
-    @honeycomb.buffer_flush(:force => true)
+    @honeycomb.multi_receive([event1, event2, event3])
+  end
+
+  it "should chunk large batches" do
+    events = []
+    (1..3*@honeycomb.flush_size).each do |i|
+      events.push(LogStash::Event.new("index" => i))
+    end
+    expect(client).to receive(:post).exactly(3).times.
+        and_call_original
+    @honeycomb.multi_receive(events)
   end
 end

metadata

@@ -1,16 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-honeycomb_json_batch
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Honeycomb
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-04 00:00:00.000000000 Z
+date: 2017-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,9 +20,8 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -31,6 +31,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-http_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,9 +40,8 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
-  name: logstash-mixin-http_client
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -51,20 +51,22 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.0.0
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: support@honeycomb.io
 executables: []
 extensions: []
@@ -82,7 +84,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -97,9 +99,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
 specification_version: 4
 summary: This output lets you `POST` batches of events to the Honeycomb.io API endpoint
 test_files: