fluent-plugin-honeycomb 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4327708ee0469056f9c618d2d35ebff5981436bd
-  data.tar.gz: adad738e9a96926e5bb39e8975520d3b55732f80
+  metadata.gz: 7083d11330f158765d8eb1e0079b7e50b1c0f979
+  data.tar.gz: a25a7b1023946e9c3a28dbdfb700bf3046f48717
 SHA512:
-  metadata.gz: 87c309c3d1c33c837d4b431b50787093d544491222cc125a7cf5d3fa6fee3353cc9ceca727f25b515df5471b7c6a27160419fbcbfb2bcb6ff0f7dd74ba5cc0ed
-  data.tar.gz: 084e70f13259e597560dd85089a15f791485e605cc12282a29518cce979636277db5f7aaed1a0a409ab89087a9d3d58fc9f4d007fe7fbbbd2ce89d1556d9edf9
+  metadata.gz: 7be1985ff4cbe433a9fd29b3e97dfb9ffe92eab9be59089f27177081fcea34be4341aa9a8dbe310bf1c11fc5f6caa5494d78358d6a01d3d88216dc6de017ce73
+  data.tar.gz: c8515220ffb762a2b3942795afb55460db2c2a41f64ae7a9c9b9f98d1ba22a52af110b29fcf1847004e7afce3a6fbe4203a1a0c79a271651372e7476652d43ce

data/README.md

@@ -27,9 +27,8 @@ Parameter | Type | Required? | Description
 | `sample_rate` | integer | no | Sample your event stream by sending 1 out of every N events. |
 | `include_tag_key` | bool | no | Whether to include the Fluentd tag in the submitted event. |
 | `tag_key` | string | no | If `include_tag_key` is `true`, the tag key name in the event (default: `fluentd_tag`).
-| `flatten_keys` | array | no | Flatten nested JSON data under these keys into
-the top-level event.
-
+| `flatten_keys` | array | no | Flatten nested JSON data under these keys into the top-level event.
+| `dataset_from_key` | string | no | Look for this key in each event, and use its value as the destination dataset. If an event doesn't contain the key, it'll be sent to the dataset given by the `dataset` parameter.
 
 ### Buffering options
 
@@ -55,8 +54,13 @@ A note about naming: This gem must be named `fluent-plugin-xxx` in order to auto
 
 ## Releasing a new version
 Travis will automatically upload tagged releases to Rubygems. To release a new
-version, run
-```
-bump patch --tag   # Or bump minor --tag, etc.
-git push --follow-tags
-```
+version:
+
+1. Update the value of `HONEYCOMB_PLUGIN_VERSION` in
+   lib/plugin/out_honeycomb_version.rb`
+
+2. Run
+    ```
+    bump patch --tag   # Or bump minor --tag, etc.
+    git push --follow-tags
+    ```

data/fluent-plugin-honeycomb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name        = 'fluent-plugin-honeycomb'
-  spec.version     = '0.4.2'
+  spec.version     = '0.5.0'
 
   spec.summary     = "Fluentd output plugin for Honeycomb.io"
   spec.description = "Fluentd output plugin for Honeycomb.io"

data/lib/fluent/plugin/out_honeycomb.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'http'
 require 'fluent/output'
+require_relative 'out_honeycomb_version'
 
 module Fluent
   class HoneycombOutput < BufferedOutput
@@ -17,6 +18,7 @@ module Fluent
     config_param :tag_key, :string, :default => "fluentd_tag"
     config_param :api_host, :string, :default => "https://api.honeycomb.io"
     config_param :flatten_keys, :array, value_type: :string, :default => []
+    config_param :dataset_from_key, :string, :default => ""
 
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
@@ -55,7 +57,7 @@ module Fluent
     #
     # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
     def write(chunk)
-      batch = []
+      batches =  Hash.new{ |h, k| h[k] = [] }
       chunk.msgpack_each do |(tag, time, record)|
         if !record.is_a? Hash
           log.debug "Skipping record #{record}"
@@ -72,6 +74,14 @@ module Fluent
           record.merge!(flatten(record[k], k))
           record.delete(k)
         end
+
+        if (@dataset_from_key != "" && record.has_key?(@dataset_from_key))
+          dataset = record[@dataset_from_key]
+          record.delete @dataset_from_key
+        else
+          dataset = @dataset
+        end
+        batch = batches[dataset]
         batch.push({
             "data" => record,
             "samplerate" => @sample_rate,
@@ -79,27 +89,29 @@ module Fluent
         })
       end
 
-      publish_batch(batch, 0)
+      batches.each do |dataset, batch|
+        publish_batch(dataset, batch, 0)
+      end
     end
 
-    def publish_batch(batch, retry_count)
+    def publish_batch(dataset, batch, retry_count)
       if batch.length == 0
         return
       end
-      log.info "publishing #{batch.length} records"
+      log.info "publishing #{batch.length} records to dataset #{dataset}"
       body = JSON.dump(batch)
       resp = HTTP.headers(
-          "User-Agent" => "fluent-plugin-honeycomb",
+          "User-Agent" => "fluent-plugin-honeycomb/#{HONEYCOMB_PLUGIN_VERSION}",
           "Content-Type" => "application/json",
           "X-Honeycomb-Team" => @writekey)
-          .post(URI.join(@api_host, "/1/batch/#{@dataset}"), {
+          .post(URI.join(@api_host, "/1/batch/#{dataset}"), {
               :body => body,
           })
       failures = parse_response(batch, resp)
       if failures.size > 0 && retry_count < @retry_limit
         # sleep and retry with the set of failed events
         sleep 1
-        publish_batch(failures, retry_count + 1)
+        publish_batch(dataset, failures, retry_count + 1)
       end
     end
 

data/lib/fluent/plugin/out_honeycomb_version.rb

@@ -0,0 +1,2 @@
+# Don't just call this VERSION, conflicts with global fluentd version constant
+HONEYCOMB_PLUGIN_VERSION = "0.5.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-honeycomb
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - The Honeycomb.io Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-30 00:00:00.000000000 Z
+date: 2017-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -164,6 +164,7 @@ files:
 - Rakefile
 - fluent-plugin-honeycomb.gemspec
 - lib/fluent/plugin/out_honeycomb.rb
+- lib/fluent/plugin/out_honeycomb_version.rb
 homepage: https://github.com/honeycombio/fluent-plugin-honeycomb
 licenses:
 - Apache-2.0
@@ -184,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin for Honeycomb.io