logstash-output-honeycomb_json_batch 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2b2a7b660d8b854fb0607a4fa80dabe4329e4e41
-  data.tar.gz: c0bf451645c3755d6c7ac39cdb031f7ab08b6516
+  metadata.gz: c90c885174ae440377b402eac734980422d3192d
+  data.tar.gz: c6e29760a336290dece1dd18bbc33f10b2b2fd2d
 SHA512:
-  metadata.gz: f5055fc55d3cbd3be58411d4565894df2eb5f9f68f6fb711c5590e6e4613cc57565ed1ba5e556eb7f3c0d3da2961cd6797ee8fb1a86084fd6900368ea7116425
-  data.tar.gz: 379e0423f7822931c219a15e3f0a379d30f9f869fa48a71c0d901754840dfe502f921229abb8db1a68e52c8c61bb8aea4929be2804d3cf288087002ee5b30e83
+  metadata.gz: b9c755b2655e7218d4141c822a0a784cc1fc6006f943c891c1d7d2a993ce2f1c7436717dca1c860a9e016ce8cde6355f4a4c8f98647240d4951b855fca7986d1
+  data.tar.gz: 6584d584c082375794988dd6cbe94c4410acc89d478c760ff62c03de8f5817421b4e358f6a6d4956285ada7c8da390441f0d2f09c880aab910fcf168cea49866

data/README.md

@@ -34,6 +34,39 @@ output {
 }
 ```
 
+Additional arguments to `honeycomb_json_batch`:
+
+Consider these when tuning performance:
+
+- `flush_size`: Default batch size, defaults to 50
+- `idle_flush_time`: Default flush interval in seconds, defaults to 5
+- `pool_max`: Maximum number of requests to be run in parallel, defaults to 10
+- `retry_individual`: On failed requests, whether to retry event sends individually, defaults to true
+- `api_host`: Allows you to override the Honeycomb host, defaults to https://api.honeycomb.io
+
+Special logstash fields that will be extracted:
+
+- `@timestamp`: Logstash events contain timestamps by default, and this output will extract it for use as the Honeycomb timestamp
+- `@samplerate`: If this special field is populated (e.g. via the `filter` section, this particular event will be weighted as `@samplerate` events in Honeycomb). See the **Sampling** section below.
+
+### Sampling
+
+High volume sites may want to send only a fraction of all traffic to Honeycomb. The drop filter can drop a portion of your traffic, and a mutate filter will ensure that Honeycomb understands that transmitted events are coming through as the result of sampling.
+
+```
+filter {
+  drop {
+    # keep 1/4 of the event stream
+    percentage => 75
+  }
+  mutate {
+    add_field => {
+      # the events that do make it through represent 4 events
+      "@samplerate" => "4"
+    }
+  }
+}
+```
 
 ## Development
 

data/lib/logstash/outputs/honeycomb_json_batch.rb

@@ -12,7 +12,6 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
 
   config_name "honeycomb_json_batch"
 
-  # URL host to use, defaults to https://api.honeycomb.io
   config :api_host, :validate => :string
 
   config :write_key, :validate => :string, :required => true
@@ -25,7 +24,7 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
 
   config :retry_individual, :validate => :boolean, :default => true
 
-  config :pool_max, :validate => :number, :default => 50
+  config :pool_max, :validate => :number, :default => 10
 
   def register
     # We count outstanding requests with this queue
@@ -64,6 +63,11 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
     buffer_receive(event)
   end
 
+  def close
+    buffer_flush(:final => true)
+    client.close
+  end
+
   public
   def flush(events, close=false)
     documents = []  #this is the array of hashes that we push to Fusion as documents
@@ -71,7 +75,11 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
     events.each do |event|
       data = event.to_hash()
       timestamp = data.delete("@timestamp")
-      documents.push({ "time" => timestamp, "data" => data })
+      doc = { "time" => timestamp, "data" => data }
+      if samplerate = data.delete("@samplerate")
+        doc["samplerate"] = samplerate.to_i
+      end
+      documents.push(doc)
     end
 
     make_request(documents)
@@ -116,8 +124,11 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
           :total => @total)
       else
         if documents.length > 1 && @retry_individual
-          documents.each do |doc| 
-            make_request([doc])
+          if statuses = JSON.parse(response.body).values.first
+            status.each_with_index do |status, i|
+              next if status >= 200 && status < 300
+              make_request([documents[i]])
+            end
           end
         else 
           @total_failed += documents.length
@@ -147,14 +158,14 @@ class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
       )
     end
 
-    Thread.new do 
-      client.execute!
-    end
+    client.execute!
+  rescue Exception => e
+    log_failure("Got totally unexpected exception #{e.message}", :docs => documents.length)
   end
 
   # This is split into a separate method mostly to help testing
   def log_failure(message, opts)
-    @logger.error("[HTTP Output Failure] #{message}", opts)
+    @logger.error("[Honeycomb Batch Output Failure] #{message}", opts)
   end
 
   def request_headers()

data/logstash-output-honeycomb_json_batch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-honeycomb_json_batch'
-  s.version         = '0.1.1'
+  s.version         = '0.1.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "This output lets you `POST` batches of events to the Honeycomb.io API endpoint"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/honeycomb_json_batch_spec.rb

@@ -76,6 +76,22 @@ describe LogStash::Outputs::HoneycombJSONBatch do
     @honeycomb.buffer_flush(:force => true)
   end
 
+  it "should extract timestamp and samplerate from the data" do
+    with_samplerate = LogStash::Event.new("alpha" => 1.0, "@samplerate" => "17.5")
+    data = with_samplerate.to_hash()
+    data.delete("@timestamp")
+    data.delete("@samplerate")
+
+    expect(client).to receive(:post).
+                        with("#{ api_host }/1/batch", hash_including(:body => LogStash::Json.dump({
+                          DATASET => [ { "time" => event.timestamp.to_s, "data" => data, "samplerate" => 17 } ]
+                        }))).once.
+                        and_call_original
+
+    @honeycomb.receive(with_samplerate)
+    @honeycomb.buffer_flush(:force => true)
+  end
+
   it "should wrap multiple events up in the right structure" do
     event1 = LogStash::Event.new("alpha" => 1.0)
     event2 = LogStash::Event.new("beta" => 2.0)

metadata

@@ -1,107 +1,105 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-honeycomb_json_batch
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Honeycomb
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-31 00:00:00.000000000 Z
+date: 2017-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - <=
+    - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  type: :runtime
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - <=
+    - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: logstash-mixin-http_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.1
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
-  type: :runtime
+  name: logstash-mixin-http_client
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.1
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 5.0.0
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the
-  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
-  gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: support@honeycomb.io
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/logstash/outputs/honeycomb_json_batch.rb
-- spec/outputs/honeycomb_json_batch_spec.rb
-- logstash-output-honeycomb_json_batch.gemspec
-- README.md
 - Gemfile
 - LICENSE
+- README.md
+- lib/logstash/outputs/honeycomb_json_batch.rb
+- logstash-output-honeycomb_json_batch.gemspec
+- spec/outputs/honeycomb_json_batch_spec.rb
 homepage: https://honeycomb.io
 licenses:
 - Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.0.14.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
 specification_version: 4
 summary: This output lets you `POST` batches of events to the Honeycomb.io API endpoint
 test_files: