logstash-output-jut 0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a9eb2e11117af72725830240a0c80dd9ac967620
+  data.tar.gz: e0302a18404130b4c21ced3836e7bb91c8d8c03f
+SHA512:
+  metadata.gz: d9a292c278f0430a56e0f0f1aa10a4605a357bbe2ef2f24672095bd25213edb94a9d85144c0534fd3a3b192542d52ca218a442b5f592238e17db20602e23345f
+  data.tar.gz: d389e4ddb0ca8af5da51525ffbe9e680996130a685eea176dd8f629a4ee716f1c01b423bcacddcf9a7d16d86ee4dab21ad9de58ab6d3862209ad92f6f9925786

data/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+*.gem
+Gemfile.lock
+Gemfile.bak
+.bundle
+vendor

data/CONTRIBUTORS

@@ -0,0 +1,11 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped this plugin along its way.
+
+Contributors:
+* Stephan Liu (stephan-x-liu)
+* Jut, Inc.
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2015 Jut, Inc. <www.jut.io>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,80 @@
+# Batch HTTP Plugin
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+## Documentation
+
+This plugin sends events to a [Jut](https://jut.io) data engine.
+Events are formatted to be suitable for
+[direct import via http](http://docs.jut.io/user-guide/#data_ingest_http)
+and are grouped into batches to reduce the total number of http
+transactions required to import many records.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone 
+
+##### 2.1.1 Version 1.5 and above
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-output-jut", :path => "/your/local/logstash-output-jut"
+```
+- Install plugin
+```sh
+bin/plugin install --no-verify
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'input { stdin {} } output { jut { url => URL }}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+##### 2.1.1 Version 1.4 and below
+
+- Copy the files jut.rb and HTTPBatcher.rb into /path/to/logstash-1.4.2/lib/logstash/outputs/
+
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'input { stdin {} } output { jut { url => URL }}'
+```
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-output-jut.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/plugin install /your/local/plugin/logstash-output-jut.gem
+```
+- Start Logstash and proceed to test the plugin

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/outputs/HTTPBatcher.rb

@@ -0,0 +1,134 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "json"
+require "thread"
+require "work_queue"
+require "net/https"
+require "openssl"
+require "uri"
+
+class LogStash::Outputs::HTTPBatcher
+  def initialize(url, idle_flush, logger, headers, limit, nthreads, verbose)
+    @url = URI.parse(url)
+    @idle_flush = idle_flush
+    @logger = logger
+    @headers = headers
+    @limit = limit
+    @verbose = verbose
+
+    @content_type = "application/json"
+    @stopped = false
+
+    @queue_mutex = Mutex.new
+    @queue = []
+
+    @flush_time = nil
+    @flush_thread = create_flush_thread()
+    @work_queue = WorkQueue.new nthreads, nil
+  end # def initialize
+
+  def stop
+    if @verbose
+      puts "stopping batcher (have #{@queue.size()} queued message)"
+    end
+
+    @stopped = true
+    while @queue.size() > 0 do
+      enqueue_batch
+    end
+
+    @flush_thread.join
+    @work_queue.join
+  end
+
+  def receive(event)
+    size = 0
+    @queue_mutex.synchronize do
+      @queue << event
+      size = @queue.size
+    end
+
+    if size >= @limit
+      enqueue_batch
+    end
+
+    @flush_time = Time.now + @idle_flush
+  end # def receive
+
+  def create_flush_thread
+    return Thread.new do
+      while !@stopped do
+        now = Time.now
+        if @flush_time != nil && @flush_time <= now
+          enqueue_batch
+          @flush_time = nil
+        end
+
+        sleep(@flush_time == nil ? @idle_flush : @flush_time - now)
+      end
+    end
+  end
+
+  def enqueue_batch
+    tosend = []
+    @queue_mutex.synchronize do
+      tosend = @queue.shift(@limit)
+    end
+
+    if tosend.size > 0
+      @work_queue.enqueue_b do
+        send_batch tosend
+      end
+    end
+  end
+
+  def send_batch(tosend)
+    connection = Thread.current["connection"]
+    if connection == nil
+      if @verbose
+        puts "creating new https connection"
+      end
+
+      connection = Net::HTTP.new(@url.host, @url.port)
+      connection.use_ssl = true
+      connection.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      Thread.current["connection"] = connection
+    end
+
+    beginning = Time.now
+    request = Net::HTTP::Post.new(@url.request_uri)
+
+    request["Content-Type"] = @content_type
+    if @headers
+      @headers.each do |k,v|
+        request.headers[k] = v
+      end
+    end
+    
+    if @verbose
+      puts "posting #{tosend.size} records"
+    end
+
+    request.body = tosend.to_json
+    response = connection.request request
+
+    status = response.code
+    rbody = response.read_body
+
+    if status != "200"
+      raise "POST failed with status #{status} (#{rbody})"
+    end
+
+    if @verbose
+      time = Time.now - beginning
+      puts "POST response in #{time.to_s} #{status} #{rbody}"
+    end
+
+  rescue Exception => e
+    if @verbose
+      @logger.warn("Unhandled exception", :request => request, :exception => e, :stacktrace => e.backtrace)
+    else
+      @logger.warn("Unhandled exception", :host => request["host"], :exception => e, :stacktrace => e.backtrace)
+    end
+  end
+end

data/lib/logstash/outputs/jut.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/outputs/HTTPBatcher"
+
+class LogStash::Outputs::Jut < LogStash::Outputs::Base
+  # This output lets you batch JSON objects and POSTs them to
+  # generic HTTP endpoints. This supports only sending JSON
+  # and sends a request every X seconds.
+  #
+  # Additionally, you are given the option to customize
+  # the headers.
+
+  config_name "jut"
+  milestone 1
+
+  # URL to use
+  config :url, :validate => :string, :required => :true
+
+  # Minimum interval at which requests are made per thread
+  config :interval, :validate => :number, :default => 1
+
+  # Maximum number of events per request
+  config :limit, :validate => :number, :default => 200
+
+  # Number of concurrent threads making requests
+  config :threads, :validate => :number, :default => 5
+
+  # Headers, if necessary, in the form of a hash
+  config :headers, :validate => :hash
+
+  # Verbose error messages
+  config :verbose, :validate => :boolean, :default => false
+
+  public
+  def register
+    @batcher = LogStash::Outputs::HTTPBatcher.new(@url, @interval, @logger, @headers, @limit, @threads, @verbose)
+  end # def register
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    # logstash wants to create "@timestamp", but jut wants "time"
+    # you could of course do this with a mutate filter but lets make
+    # the common case easy...
+    if event.include?('@timestamp')
+      event['time'] = event.remove('@timestamp')
+    end
+
+    if event.include?('@version')
+      event.remove('@version')
+    end
+
+    evt = event.to_hash
+    @batcher.receive(evt)
+
+  rescue Exception => e
+    @logger.warn("Unhandled exception", :exception => e, :stacktrace => e.backtrace)
+  end
+
+  public
+  def teardown
+    if @verbose
+      puts "tearing down jut output plugin"
+    end
+
+    @batcher.stop
+
+    if @verbose
+      puts "finished"
+    end
+  end
+end

data/logstash-output-jut.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-jut'
+  s.version = "0.3"
+  s.licenses = ["Apache License (2.0)"]
+  s.summary = "sends records to a Jut data engine"
+  s.description = "logstash output plugin that sends records to a Jut data engine"
+  s.authors = ["Jut, Inc."]
+  s.email = "josa@jut.io"
+  s.homepage = "https://github.com/jut-io/logstash-output-jut"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  # This metadata attribute blew up for me on Ubuntu 14.04 with the latest rubgems installed
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 2.0.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "work_queue"
+
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/outputs/jut_spec.rb

@@ -0,0 +1,29 @@
+require "spec_helper"
+require "logstash/outputs/jut"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::Jut do
+  let(:jut_config) {
+    {
+      'url' => 'http://localhost:9000/'
+    }
+  }
+  let(:sample_event) { LogStash::Event.new({'message' => 'hello', '@timestamp'=>LogStash::Timestamp.now}) }
+
+  context 'when intializing' do
+    it 'should register' do
+      output = LogStash::Outputs::Jut.new(jut_config)
+      expect {output.register}.to_not raise_error
+    end
+
+    it 'should populate jut config with default values' do
+      cfg = LogStash::Outputs::Jut.new(jut_config)
+      insist {cfg.url} == 'http://localhost:9000/'
+      insist {cfg.interval} == 1
+      insist {cfg.limit} == 200
+      insist {cfg.threads} == 5
+      insist {cfg.verbose} == false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require "logstash/devutils/rspec/spec_helper"
+require "webmock/rspec"
+WebMock.disable_net_connect!(allow_localhost: true)

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-jut
+version: !ruby/object:Gem::Version
+  version: '0.3'
+platform: ruby
+authors:
+- Jut, Inc.
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: work_queue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: logstash output plugin that sends records to a Jut data engine
+email: josa@jut.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash/outputs/HTTPBatcher.rb
+- lib/logstash/outputs/jut.rb
+- logstash-output-jut.gemspec
+- spec/outputs/jut_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/jut-io/logstash-output-jut
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: sends records to a Jut data engine
+test_files:
+- spec/outputs/jut_spec.rb
+- spec/spec_helper.rb