fluent-plugin-honeycomb 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3bbbf988b86bf83fcd88f058397137d50d10f2db
-  data.tar.gz: cb223aaa156f2860648b464f98dc539ffdb873ea
+  metadata.gz: 90ab38aaf106e01ace1f186cf8622601bf94e0ca
+  data.tar.gz: 0f976da371561c9aac100a83c53c304435f8318d
 SHA512:
-  metadata.gz: cfca8e30d8350af3587f054190d46c971885a55307af165476b6cb36ba0138fe80e654419d3ec273debd564db4d1662ac9619ccc9341f4337b55e79717a4a8ec
-  data.tar.gz: 14610fbbc0f9b77f5b026a099d07b75ec8048cf26fa59017f7d5f21c53a2d32871098841e9ab30fafe0b5c6d2f8deedff5b27f55643071913f8394132c84510e
+  metadata.gz: 52b63188d32f34689f2a7de629b134c498d432bdda2a112c0eaba972d37889c3e165e4f25abcbc5ba0b074a1f39a9d70f185a5712b20c47695c7f5d08350cbc0
+  data.tar.gz: 7f69ca94ab060668c786697bf5da4a989061e13b6e3dec0bedcf031af0041bfc7845069a8b2864c2a8d6a1b5ef683f3f1fc984d45679fb2260c3f0eac146fcf9

data/README.md

@@ -1,6 +1,6 @@
 [![Build Status](https://travis-ci.org/honeycombio/fluent-plugin-honeycomb.svg?branch=master)](https://travis-ci.org/honeycombio/fluent-plugin-honeycomb)
 
-Send your logs to [Honeycomb](https://honeycomb.io).
+Send your logs to [Honeycomb](https://honeycomb.io). (For more information about using Honeycomb, see [our docs](https://honeycomb.io/docs/).)
 
 ## Getting started
 
@@ -58,5 +58,5 @@ Travis will automatically upload tagged releases to Rubygems. To release a new
 version, run
 ```
 bump patch --tag   # Or bump minor --tag, etc.
-git push --tags
+git push --follow-tags
 ```

data/fluent-plugin-honeycomb.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name        = 'fluent-plugin-honeycomb'
-  spec.version     = '0.4.0'
-  spec.date        = '2017-02-07'
+  spec.version     = '0.4.1'
 
   spec.summary     = "Fluentd output plugin for Honeycomb.io"
   spec.description = "Fluentd output plugin for Honeycomb.io"

data/lib/fluent/plugin/out_honeycomb.rb

@@ -22,10 +22,13 @@ module Fluent
     # 'conf' is a Hash that includes configuration parameters.
     # If the configuration is invalid, raise Fluent::ConfigError.
     def configure(conf)
+      # Apply sane defaults. These override the poor fluentd defaults, but not
+      # anything explicitly specified in the configuration.
+      conf["buffer_chunk_limit"] ||= "500k"
+      conf["flush_interval"] ||= "1s"
+      conf["max_retry_wait"] ||= "30s"
+      conf["retry_limit"] ||= 17
       super
-
-      # You can also refer raw parameter via conf[name].
-      @path = conf['path']
     end
 
     # This method is called when starting.
@@ -76,59 +79,69 @@ module Fluent
         })
       end
 
+      publish_batch(batch, 0)
+    end
+
+    def publish_batch(batch, retry_count)
       if batch.length == 0
         return
       end
       log.info "publishing #{batch.length} records"
-      body = JSON.dump({ @dataset => batch })
+      body = JSON.dump(batch)
       resp = HTTP.headers(
           "User-Agent" => "fluent-plugin-honeycomb",
           "Content-Type" => "application/json",
           "X-Honeycomb-Team" => @writekey)
-          .post(URI.join(@api_host, "/1/batch"), {
+          .post(URI.join(@api_host, "/1/batch/#{@dataset}"), {
               :body => body,
           })
-      parse_response(resp)
+      failures = parse_response(batch, resp)
+      if failures.size > 0 && retry_count < @retry_limit
+        # sleep and retry with the set of failed events
+        sleep 1
+        publish_batch(failures, retry_count + 1)
+      end
     end
 
-    def parse_response(resp)
+    def parse_response(batch, resp)
       if resp.status != 200
         # Force retry
         log.error "Error sending batch: #{resp.status}, #{resp.body}"
         raise Exception.new("Error sending batch: #{resp.status}, #{resp.body}")
-      else
-        begin
-          results = JSON.parse(resp.body)
-        rescue JSON::ParserError => e
-          log.warn "Error parsing response as JSON: #{e}"
-          return
-        end
-        successes = 0
-        failures = []
-        results.each do |key, statuses|
-          if !statuses.is_a? Array
-            next
-          end
+      end
 
-          statuses.each do |s|
-            if !s.is_a? Hash
-              next
-            end
+      begin
+        results = JSON.parse(resp.body)
+      rescue JSON::ParserError => e
+        log.warn "Error parsing response as JSON: #{e}"
+        raise e
+      end
+      successes = 0
+      failures = []
+      if !results.is_a? Array
+        log.warning "Unexpected response format: #{results}"
+        raise Exception.new("Unexpected response format: #{resp.status}")
+      end
 
-            if s["status"] == 202
-              successes += 1
-            else
-              failures[r["status"]] += 1
-            end
-          end
+      results.each_with_index do |result, idx|
+        if !result.is_a? Hash
+          log.warning "Unexpected status format in response: #{result}"
+          next
         end
 
-        if failures.size > 0
-          log.warn "Errors publishing records: #{failures}"
+        if result["status"] == 202
+          successes += 1
         else
-          log.debug "Successfully published #{successes} records"
+          failures.push(batch[idx])
         end
       end
+
+      if failures.size > 0
+        log.warn "Errors publishing records: #{failures.size} failures out of #{successes + failures.size}"
+      else
+        log.debug "Successfully published #{successes} records"
+      end
+      return failures
     end
 
     def flatten(record, prefix)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-honeycomb
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - The Honeycomb.io Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-07 00:00:00.000000000 Z
+date: 2017-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd