logstash-output-json_batch 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ee4f16aca346581c664061bb54a8277e221e5202
+  data.tar.gz: 12cba94b51e576e5f255b2cb3b6bd7aa80784325
+SHA512:
+  metadata.gz: 4558c603e114ef0b404ff6e92074aede6a0c646a713f78890f5e65b9cc8df533a51dd9fdcb10a0184dfeb3ed112a5ccd6119ca7334b70984befbfd0a0fb78455
+  data.tar.gz: 3a6771048a1366e45acc1c252f4ca6cb66481539d1c75ce6fa2c468e771623dada1abd2deac0ba354ca52101c4ee3a6f22f40b71c6e7bde6cd53d83eb45dac6d

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2016 Lucidworks
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,26 @@
+# Logstash Plugin
+
+This plugin is a heavily butchered and modified version of the standard logstash http output plugin. That plugin is available [here](https://github.com/logstash-plugins/logstash-output-http). 
+
+It has been modified to only support JSON, but also supports batching of messages into JSON lists.
+
+# Usage
+
+Please note that the name of the plugin when used is `json_batch`, since it only supports json in its current form. If further output formats are added in the future, this might change back to http_batch.
+
+    output {
+      json_batch {
+        headers => ["Authorization", "Basic YWRtaW46cGFzc3dvcmQxMjM="]
+        url => "http://your.site/your/json/update/endpoint"
+      }
+    }
+    
+Default batch size is 50, with a wait of at most 5 seconds per send. These can be tweaked with the parameters `flush_size` and `idle_flush_time` respectively.
+
+# Installation & build
+
+To build the gem yourself, use `gem build logstash-output-json_batch.gemspec` in the root of this repository. Alternatively, you can download a built version of the gem from the `dist` branch of this repository. 
+
+To install, run the following command, assuming the gem is in the local directory: `$LOGSTASH_HOME/bin/plugin install logstash-output-json_batch-X.Y.Z.gem`
+
+Direct links to the built versions of the gems are available on the [releases page](https://github.com/jwestberg/logstash-output-http_batch/releases)

data/lib/logstash/outputs/json_batch.rb

@@ -0,0 +1,158 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/json"
+require "uri"
+require "stud/buffer"
+require "logstash/plugin_mixins/http_client"
+
+class LogStash::Outputs::JSONBatch < LogStash::Outputs::Base
+  include LogStash::PluginMixins::HttpClient
+  include Stud::Buffer
+
+  config_name "json_batch"
+
+  # URL to use
+  config :url, :validate => :string, :required => true 
+
+  # Custom headers to use
+  # format is `headers => ["X-My-Header", "%{host}"]`
+  config :headers, :validate => :hash
+
+  config :flush_size, :validate => :number, :default => 50
+
+  config :idle_flush_time, :validate => :number, :default => 5
+
+  config :retry_individual, :validate => :boolean, :default => true
+
+  config :pool_max, :validate => :number, :default => 50
+
+  def register
+    # Handle this deprecated option. TODO: remove the option
+    #@ssl_certificate_validation = @verify_ssl if @verify_ssl
+
+    # We count outstanding requests with this queue
+    # This queue tracks the requests to create backpressure
+    # When this queue is empty no new requests may be sent,
+    # tokens must be added back by the client on success
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times {|t| @request_tokens << true }
+    @total = 0
+    @total_failed = 0
+    @requests = Array.new
+
+    buffer_initialize(
+      :max_items => @flush_size,
+      :max_interval => @idle_flush_time,
+      :logger => @logger
+    )
+    logger.info("Initialized json_batch with settings", 
+      :flush_size => @flush_size,
+      :idle_flush_time => @idle_flush_time,
+      :request_tokens => @pool_max,
+      :url => @url,
+      :headers => request_headers,
+      :retry_individual => @retry_individual)
+
+  end # def register
+
+  # This module currently does not support parallel requests as that would circumvent the batching
+  def receive(event, async_type=:background)
+    buffer_receive(event)
+  end #def event
+
+  public
+  def flush(events, close=false)
+    documents = []  #this is the array of hashes that we push to Fusion as documents
+
+    events.each do |event|
+        document = event.to_hash()
+        documents.push(document)
+    end
+
+    make_request(documents)
+  end
+
+  def multi_receive(events)
+    events.each {|event| buffer_receive(event)}
+  end
+
+  private
+
+  def make_request(documents)
+    body = LogStash::Json.dump(documents)
+    # Block waiting for a token
+    #@logger.info("Requesting token ", :tokens => request_tokens.length())
+    token = @request_tokens.pop
+    @logger.debug("Got token", :tokens => @request_tokens.length)
+
+    # Create an async request
+    begin
+      request = client.send(:post, @url, :body => body, :headers => request_headers, :async => true)
+    rescue Exception => e
+      @logger.warn("An error occurred while indexing: #{e.message}")
+    end
+
+    # attach handlers before performing request
+    request.on_complete do
+      # Make sure we return the token to the pool
+      @request_tokens << token
+    end
+
+    request.on_success do |response|
+      if response.code >= 200 && response.code < 300
+        @total = @total + documents.length
+        @logger.debug("Successfully submitted", 
+          :docs => documents.length,
+          :response_code => response.code,
+          :total => @total)
+      else
+        if documents.length > 1 && @retry_individual
+          documents.each do |doc| 
+            make_request([doc])
+          end
+        else 
+          @total_failed += documents.length
+          log_failure(
+              "Encountered non-200 HTTP code #{response.code}",
+              :response_code => response.code,
+              :url => url,
+              :response_body => response.body,
+              :num_docs => documents.length,
+              :retry_individual => @retry_individual,
+              :total_failed => @total_failed)
+        end
+      end
+    end
+
+    request.on_failure do |exception|
+      @total_failed += documents.length
+      log_failure("Could not access URL",
+        :url => url,
+        :method => @http_method,
+        :body => body,
+        :headers => headers,
+        :message => exception.message,
+        :class => exception.class.name,
+        :backtrace => exception.backtrace,
+        :total_failed => @total_failed
+      )
+    end
+
+    Thread.new do 
+      client.execute!
+    end
+  end
+
+  # This is split into a separate method mostly to help testing
+  def log_failure(message, opts)
+    @logger.error("[HTTP Output Failure] #{message}", opts)
+  end
+
+  def request_headers()
+    headers = @headers || {}
+    headers["Content-Type"] ||= "application/json"
+    headers
+  end
+
+end

data/logstash-output-json_batch.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-output-json_batch'
+  s.version         = '0.2.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "This output lets you `POST` messages as JSON in a batched fashion to a HTTP endpoint"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["jwestberg"]
+  s.email           = 'joel.westberg@lucidworks.com'
+  s.homepage        = "http://lucidworks.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','Gemfile','LICENSE' ]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 2.2.1", "< 3.0.0"
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
+end
+

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-json_batch
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- jwestberg
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-07-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-mixin-http_client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: sinatra
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: joel.westberg@lucidworks.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/json_batch.rb
+- logstash-output-json_batch.gemspec
+homepage: http://lucidworks.com
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: This output lets you `POST` messages as JSON in a batched fashion to a HTTP endpoint
+test_files: []