logstash-input-kinesis 1.0.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 39f76c083f83979183452b77aa688f68b3205715
+  data.tar.gz: 9f953db2fd1761396b64802444d92a791ddb84a7
+SHA512:
+  metadata.gz: 2f7a691a7b5acf4c4f425b1e55fd0bb9535ea018ac0d01436af4fbbce9dc72174b17096c485f75b754b9f8bd9f53ac8f96e968f903d7d1fc98f1e744b94b1569
+  data.tar.gz: 033ba37be51948030ba19ebfc962d609667ee6fb1550005d2ed9f149704b15cae470818a79c2d91d484ff752d5b16f3e4af53cb36b888507829f2b16719fc2b5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in logstash-input-kinesis.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,14 @@
+Copyright (c) 2015 Brian Palmer
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

data/README.md

@@ -0,0 +1,33 @@
+# Logstash AWS Kinesis Input Plugin
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+## Installation
+
+This plugin requires Logstash 1.5, and can be installed by Logstash
+itself.
+
+```sh
+$(LOGSTASH_DIR)/bin/plugin install logstash-input-kinesis
+```
+
+## Usage
+
+```
+input {
+  kinesis {
+    kinesis_stream_name => "my-logging-stream"
+    codec => json { }
+  }
+}
+```
+
+At least `kinesis_stream_name` is requred.
+
+## Contributing
+
+1. Fork it ( https://github.com/codekitchen/logstash-input-kinesis/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/logstash/inputs/kinesis.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/errors"
+require "logstash/environment"
+require "logstash/namespace"
+
+require "logstash/inputs/kinesis/version"
+require 'logstash-input-kinesis_jars'
+
+class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
+  KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
+
+  config_name 'kinesis'
+  milestone 1
+
+  # The application name used for the dynamodb coordination table. Must be
+  # unique for this kinesis stream.
+  config :application_name, :validate => :string, :default => "logstash"
+
+  # The kinesis stream name.
+  config :kinesis_stream_name, :validate => :string, :required => true
+
+  def register
+    # the INFO log level is extremely noisy in KCL
+    org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").
+      logger.setLevel(java.util.logging::Level::WARNING)
+
+    worker_id = java.util::UUID.randomUUID.to_s
+    creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new()
+    @config = KCL::KinesisClientLibConfiguration.new(
+      @application_name,
+      @kinesis_stream_name,
+      creds,
+      worker_id).withInitialPositionInStream(KCL::InitialPositionInStream::TRIM_HORIZON)
+  end
+
+  def run(output_queue)
+    worker_factory = WorkerFactory.new(@codec, output_queue, method(:decorate))
+    @worker = KCL::Worker.new(
+      worker_factory,
+      @config,
+      com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory.new)
+    @worker.run()
+  end
+
+  def teardown
+    @worker.shutdown if @worker
+  end
+
+  class WorkerFactory
+    include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessorFactory
+    def initialize(codec, output_queue, decorator)
+      @codec = codec
+      @output_queue = output_queue
+      @decorator = decorator
+    end
+
+    def createProcessor
+      Worker.new(@codec.clone, @output_queue, @decorator)
+    end
+  end
+
+  class Worker
+    include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
+
+    def initialize(*args)
+      # nasty hack, because this is the name of a method on IRecordProcessor, but also ruby's constructor
+      if !@constructed
+        @codec, @output_queue, @decorator = args
+        @constructed = true
+      else
+        _shard_id, _ = args
+        @decoder = java.nio.charset::Charset.forName("UTF-8").newDecoder()
+      end
+    end
+
+    def processRecords(records, checkpointer)
+      records.each { |record| process_record(record) }
+      checkpointer.checkpoint()
+    end
+
+    def shutdown(checkpointer, reason)
+    end
+
+    def process_record(record)
+      raw = @decoder.decode(record.getData).to_s
+      @codec.decode(raw) do |event|
+        @decorator.call(event)
+        @output_queue << event
+      end
+    end
+  end
+end

data/lib/logstash/inputs/kinesis/version.rb

@@ -0,0 +1,7 @@
+module Logstash
+  module Input
+    module Kinesis
+      VERSION = "1.0.0"
+    end
+  end
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-kinesis
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: java
+authors:
+- Brian Palmer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.7
+  name: jar-dependencies
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.7
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.1.1.0.8
+  name: ruby-maven
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.1.1.0.8
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.7
+  name: maven-tools
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.7
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email:
+- brian@codekitchen.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/logstash/inputs/kinesis.rb
+- lib/logstash/inputs/kinesis/version.rb
+homepage: https://github.com/codekitchen/logstash-input-kinesis
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'com.amazonaws:amazon-kinesis-client', '1.2.1'
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Logstash plugin for Kinesis input
+test_files: []