logstash-output-honeycomb_json_batch 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bf59c27f44d60efa7c924e9b3371133a718e938a
+  data.tar.gz: '09aead3e7779e92feff278e6bfb91c4a5ec09be6'
+SHA512:
+  metadata.gz: c578d3531ba2d72197eab44b471939fc8dccb5ef1e9f99155915fbc937c937262061bab610d19f15a832903ab80623400741004b4c44068aa46e6ffb20fc3e6b
+  data.tar.gz: cc4fba63f1ca522bc4d744b37bad638848a86fe2f1d7721bd652f1b33e18cad757979ed44d603aaac2d3bb2066593005e339057761e61d5805bb2b95657b87aa

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2017 Hound Technology, Inc <https://honeycomb.io>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

data/README.md

@@ -0,0 +1,51 @@
+# Honeycomb Batch JSON Logstash Plugin
+
+This plugin is a heavily modified version of the standard logstash [http output](https://github.com/logstash-plugins/logstash-output-http) plugin and the [Lucidworks JSON batch plugin](https://github.com/lucidworks/logstash-output-json_batch).
+
+# Usage
+
+Please note that the name of the plugin when used is `json_batch`, since it only supports JSON in its current form.
+
+The default batch size is 50, ...
+
+    output {
+      honeycomb_json_batch {
+        write_key => "YOUR_TEAM_KEY"
+        dataset => "Logstash Batch Test"
+      }
+    }
+
+# Installation
+
+The easiest way to use this plugin is by installing it through rubygems like any other logstash plugin. To get the latest version installed, you should run the following command: `bin/logstash-plugin install logstash-output-honeycomb_json_batch`
+
+TODO: publish to rubygems
+
+
+# Running locally
+
+Install dependencies
+
+```
+bundle install
+```
+
+Run tests
+
+```
+bundle exec rspec
+```
+
+## Run in an installed Logstash
+
+You can build the gem and install it using:
+
+```
+gem build logstash-output-honeycomb_json_batch.gemspec
+```
+
+And install it into your local logstash instance:
+
+```
+logstash-plugin install ./path/to/logstash-output-honeycomb_json_batch-VERSION.gem
+```

data/lib/logstash/outputs/honeycomb_json_batch.rb

@@ -0,0 +1,164 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/json"
+require "uri"
+require "stud/buffer"
+require "logstash/plugin_mixins/http_client"
+
+class LogStash::Outputs::HoneycombJSONBatch < LogStash::Outputs::Base
+  include LogStash::PluginMixins::HttpClient
+  include Stud::Buffer
+
+  config_name "honeycomb_json_batch"
+
+  # URL host to use, defaults to api.honeycomb.io
+  config :api_host, :validate => :string
+
+  config :write_key, :validate => :string, :required => true
+
+  config :dataset, :validate => :string, :required => true
+
+  config :flush_size, :validate => :number, :default => 50
+
+  config :idle_flush_time, :validate => :number, :default => 5
+
+  config :retry_individual, :validate => :boolean, :default => true
+
+  config :pool_max, :validate => :number, :default => 50
+
+  def register
+    # We count outstanding requests with this queue
+    # This queue tracks the requests to create backpressure
+    # When this queue is empty no new requests may be sent,
+    # tokens must be added back by the client on success
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times {|t| @request_tokens << true }
+    @total = 0
+    @total_failed = 0
+    @requests = Array.new
+    if !@api_host.start_with? "http"
+      @api_host = "http://#{ @api_host }"
+    end
+    @api_host = @api_host.chomp
+
+    buffer_initialize(
+      :max_items => @flush_size,
+      :max_interval => @idle_flush_time,
+      :logger => @logger
+    )
+    logger.info("Initialized honeycomb_json_batch with settings",
+      :flush_size => @flush_size,
+      :idle_flush_time => @idle_flush_time,
+      :request_tokens => @pool_max,
+      :api_host => @api_host,
+      :headers => request_headers,
+      :retry_individual => @retry_individual)
+
+  end
+
+  # This module currently does not support parallel requests as that would circumvent the batching
+  def receive(event, async_type=:background)
+    buffer_receive(event)
+  end
+
+  public
+  def flush(events, close=false)
+    documents = []  #this is the array of hashes that we push to Fusion as documents
+
+    events.each do |event|
+      data = event.to_hash()
+      timestamp = data.delete("@timestamp")
+      documents.push({ "time" => timestamp, "data" => data })
+    end
+
+    make_request(documents)
+  end
+
+  def multi_receive(events)
+    events.each {|event| buffer_receive(event)}
+  end
+
+  private
+
+  def make_request(documents)
+    body = LogStash::Json.dump({ @dataset => documents })
+    # Block waiting for a token
+    token = @request_tokens.pop
+    @logger.debug("Got token", :tokens => @request_tokens.length)
+
+
+    # Create an async request
+    begin
+      request = client.post("#{@api_host}/1/batch", {
+        :body => body,
+        :headers => request_headers,
+        :async => true
+      })
+    rescue Exception => e
+      @logger.warn("An error occurred while indexing: #{e.message}")
+    end
+
+    # attach handlers before performing request
+    request.on_complete do
+      # Make sure we return the token to the pool
+      @request_tokens << token
+    end
+
+    request.on_success do |response|
+      if response.code >= 200 && response.code < 300
+        @total = @total + documents.length
+        @logger.debug("Successfully submitted", 
+          :docs => documents.length,
+          :response_code => response.code,
+          :total => @total)
+      else
+        if documents.length > 1 && @retry_individual
+          documents.each do |doc| 
+            make_request([doc])
+          end
+        else 
+          @total_failed += documents.length
+          log_failure(
+              "Encountered non-200 HTTP code #{response.code}",
+              :response_code => response.code,
+              :url => url,
+              :response_body => response.body,
+              :num_docs => documents.length,
+              :retry_individual => @retry_individual,
+              :total_failed => @total_failed)
+        end
+      end
+    end
+
+    request.on_failure do |exception|
+      @total_failed += documents.length
+      log_failure("Could not access URL",
+        :url => url,
+        :method => @http_method,
+        :body => body,
+        :headers => headers,
+        :message => exception.message,
+        :class => exception.class.name,
+        :backtrace => exception.backtrace,
+        :total_failed => @total_failed
+      )
+    end
+
+    Thread.new do 
+      client.execute!
+    end
+  end
+
+  # This is split into a separate method mostly to help testing
+  def log_failure(message, opts)
+    @logger.error("[HTTP Output Failure] #{message}", opts)
+  end
+
+  def request_headers()
+    {
+      "Content-Type" => "application/json",
+      "X-Honeycomb-Team" => @write_key
+    }
+  end
+end

data/logstash-output-honeycomb_json_batch.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-output-honeycomb_json_batch'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache-2.0']
+  s.summary         = "This output lets you `POST` batches of events to the Honeycomb.io API endpoint"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Honeycomb"]
+  s.email           = 'support@honeycomb.io'
+  s.homepage        = "https://honeycomb.io"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 2.2.1", "< 5.0.0"
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
+end

data/spec/outputs/honeycomb_json_batch_spec.rb

@@ -0,0 +1,99 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/honeycomb_json_batch"
+
+PORT = rand(65535-1024) + 1025
+WRITE_KEY = "YOUR_WRITE_KEY"
+DATASET = "YOUR_DATASET"
+
+describe LogStash::Outputs::HoneycombJSONBatch do
+  let(:port) { PORT }
+  let(:event) { LogStash::Event.new("message" => "hi") }
+  let(:api_host) { "http://localhost:#{port}"}
+  let(:flush_size) { 15 }
+  let(:client) { @honeycomb.client }
+
+  before do
+    @honeycomb = LogStash::Outputs::HoneycombJSONBatch.new(
+      "write_key" => WRITE_KEY,
+      "dataset" => DATASET,
+      "api_host" => api_host,
+      "flush_size" => flush_size
+    )
+  end
+
+  before do
+    allow(@honeycomb).to receive(:client).and_return(client)
+    @honeycomb.register
+    allow(client).to receive(:post).and_call_original
+  end
+
+  after do
+    @honeycomb.close
+  end
+
+  it "should receive a single post request" do
+    expect(client).to receive(:post).
+                        with("#{ api_host }/1/batch", hash_including(:body, :headers, :async)).
+                        once.
+                        and_call_original
+
+    5.times {|t| @honeycomb.receive(event)}
+    @honeycomb.buffer_flush(:force => true)
+  end
+
+  it "should send batches based on the specified flush_size" do
+    expect(client).to receive(:post).
+                        with("#{ api_host }/1/batch", hash_including(:body, :headers, :async)).
+                        twice.
+                        and_call_original
+
+    (flush_size + 1).times {|t| @honeycomb.receive(event)}
+    @honeycomb.buffer_flush(:force => true)
+  end
+
+  it "should attach the right headers for Honeycomb ingestion" do
+    expect(client).to receive(:post).
+                        with("#{ api_host }/1/batch", hash_including(:headers => {
+                          "Content-Type" => "application/json",
+                          "X-Honeycomb-Team" => WRITE_KEY
+                        })).once.
+                        and_call_original
+
+    @honeycomb.receive(event)
+    @honeycomb.buffer_flush(:force => true)
+  end
+
+  it "should wrap events in the right structure Honeycomb ingestion" do
+    data = event.to_hash()
+    data.delete("@timestamp")
+    expect(client).to receive(:post).
+                        with("#{ api_host }/1/batch", hash_including(:body => LogStash::Json.dump({
+                          DATASET => [ { "time" => event.timestamp.to_s, "data" => data } ]
+                        }))).once.
+                        and_call_original
+
+    @honeycomb.receive(event)
+    @honeycomb.buffer_flush(:force => true)
+  end
+
+  it "should wrap multiple events up in the right structure" do
+    event1 = LogStash::Event.new("alpha" => 1.0)
+    event2 = LogStash::Event.new("beta" => 2.0)
+    event3 = LogStash::Event.new("gamma" => 3.0)
+
+    expect(client).to receive(:post).
+                        with("#{ api_host }/1/batch", hash_including(:body => LogStash::Json.dump({
+                          DATASET => [
+                            { "time" => event1.timestamp.to_s, "data" => { "alpha" => 1.0, "@version" => "1" } },
+                            { "time" => event2.timestamp.to_s, "data" => { "@version" => "1", "beta" => 2.0 } },
+                            { "time" => event3.timestamp.to_s, "data" => { "@version" => "1", "gamma" => 3.0 } }
+                          ]
+                        }))).once.
+                        and_call_original
+
+    @honeycomb.receive(event1)
+    @honeycomb.receive(event2)
+    @honeycomb.receive(event3)
+    @honeycomb.buffer_flush(:force => true)
+  end
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-honeycomb_json_batch
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Honeycomb
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-01-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  name: logstash-mixin-http_client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: sinatra
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+email: support@honeycomb.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/honeycomb_json_batch.rb
+- logstash-output-honeycomb_json_batch.gemspec
+- spec/outputs/honeycomb_json_batch_spec.rb
+homepage: https://honeycomb.io
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: This output lets you `POST` batches of events to the Honeycomb.io API endpoint
+test_files:
+- spec/outputs/honeycomb_json_batch_spec.rb