logstash-input-kinesis 1.1.0-java → 1.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb43b802dabbacadd3bb00a085e4b5916c6751d5
-  data.tar.gz: 03d11ef552c46c7022c37f38cb0a379363bc9e02
+  metadata.gz: f080dee0175be6a16a2a124590f0e92a20ccf45e
+  data.tar.gz: 7e3bf28cf181418701f9ac5041512c1b7c2000a7
 SHA512:
-  metadata.gz: 13eac6fc73411ce0c353557cbfba14b5ff0dffa7e3994c4277db3fcb65b43c70b013538d840ab2d619ccf54244af217f68d32e4c33fe75cb9980bd229e025c57
-  data.tar.gz: c5abf46b322cbb4c84d681178db838782aa06bd7e0936ddfbcc7fd7fda91b581479b28b4697be4d6814968f4f8043c04c3e2e14ca36cf0d13d2f7b3873ea14c3
+  metadata.gz: c79c16f7b23a7607c852aa6570f8ab807812cf85087d0aab152ce3fc73b6a2e0f365e606670e8c22393aa173c56df26c7c7ea05afc2b1d758049494dfc64d49a
+  data.tar.gz: 562275486d2548074591e98063f5e54ba5f8a39fd0c45a3f2011aec6e009465afd05bc2bf81fcc7eccc4c400ccebd2e883ae6ca549167dd7b798421639a12b6b

data/Rakefile

@@ -1,2 +1,9 @@
 require "bundler/gem_tasks"
 
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+end
+
+task default: "spec"

data/lib/logstash/inputs/kinesis.rb

@@ -4,15 +4,34 @@ require "logstash/errors"
 require "logstash/environment"
 require "logstash/namespace"
 
-require "logstash/inputs/kinesis/version"
 require 'logstash-input-kinesis_jars'
+require "logstash/inputs/kinesis/version"
 
+# Receive events through an AWS Kinesis stream.
+#
+# This input plugin uses the Java Kinesis Client Library underneath, so the
+# documentation at https://github.com/awslabs/amazon-kinesis-client will be
+# useful.
+#
+# AWS credentials can be specified either through environment variables, or an
+# IAM instance role. The library uses a DynamoDB table for worker coordination,
+# so you'll need to grant access to that as well as to the Kinesis stream. The
+# DynamoDB table has the same name as the `application_name` configuration
+# option, which defaults to "logstash".
+#
+# The library can optionally also send worker statistics to CloudWatch.
 class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
   KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
+  require "logstash/inputs/kinesis/worker"
 
   config_name 'kinesis'
   milestone 1
 
+  attr_reader(
+    :kcl_config,
+    :kcl_worker,
+  )
+
   # The application name used for the dynamodb coordination table. Must be
   # unique for this kinesis stream.
   config :application_name, :validate => :string, :default => "logstash"
@@ -23,6 +42,15 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
   # How many seconds between worker checkpoints to dynamodb.
   config :checkpoint_interval_seconds, :validate => :number, :default => 60
 
+  # Worker metric tracking. By default this is disabled, set it to "cloudwatch"
+  # to enable the cloudwatch integration in the Kinesis Client Library.
+  config :metrics, :validate => [nil, "cloudwatch"], :default => nil
+
+  def initialize(params = {}, kcl_class = KCL::Worker)
+    @kcl_class = kcl_class
+    super(params)
+  end
+
   def register
     # the INFO log level is extremely noisy in KCL
     org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").
@@ -30,7 +58,7 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
 
     worker_id = java.util::UUID.randomUUID.to_s
     creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new()
-    @config = KCL::KinesisClientLibConfiguration.new(
+    @kcl_config = KCL::KinesisClientLibConfiguration.new(
       @application_name,
       @kinesis_stream_name,
       creds,
@@ -38,62 +66,33 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
   end
 
   def run(output_queue)
-    @worker = KCL::Worker.new(
-      proc { Worker.new(@codec, output_queue, method(:decorate), @checkpoint_interval_seconds) },
-      @config,
-      com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory.new)
-    @worker.run()
+    worker_factory = proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
+    if metrics_factory
+      @kcl_worker = @kcl_class.new(
+        worker_factory,
+        @kcl_config,
+        metrics_factory)
+    else
+      @kcl_worker = @kcl_class.new(
+        worker_factory,
+        @kcl_config)
+    end
+
+    @kcl_worker.run()
   end
 
   def teardown
-    @worker.shutdown if @worker
+    @kcl_worker.shutdown if @kcl_worker
   end
 
-  class Worker
-    include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
-
-    def initialize(*args)
-      # nasty hack, because this is the name of a method on IRecordProcessor, but also ruby's constructor
-      if !@constructed
-        @codec, @output_queue, @decorator, @checkpoint_interval = args
-        @next_checkpoint = Time.now - 600
-        @constructed = true
-      else
-        _shard_id, _ = args
-        @decoder = java.nio.charset::Charset.forName("UTF-8").newDecoder()
-      end
-    end
-
-    def processRecords(records, checkpointer)
-      records.each { |record| process_record(record) }
-      if Time.now >= @next_checkpoint
-        checkpoint(checkpointer)
-        @next_checkpoint = Time.now + @checkpoint_interval
-      end
-    end
-
-    def shutdown(checkpointer, reason)
-      if reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
-        checkpoint(checkpointer)
-      end
-    end
-
-    protected
-
-    def checkpoint(checkpointer)
-      checkpointer.checkpoint()
-    rescue => error
-      @logger.error("Kinesis worker failed checkpointing: #{error}")
-    end
+  protected
 
-    def process_record(record)
-      raw = @decoder.decode(record.getData).to_s
-      @codec.decode(raw) do |event|
-        @decorator.call(event)
-        @output_queue << event
-      end
-    rescue => error
-      @logger.error("Error processing record: #{error}")
+  def metrics_factory
+    case @metrics
+    when nil
+      com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory.new
+    when 'cloudwatch'
+      nil # default in the underlying library
     end
   end
 end

data/lib/logstash/inputs/kinesis/version.rb

@@ -1,7 +1,7 @@
 module Logstash
   module Input
     module Kinesis
-      VERSION = "1.1.0"
+      VERSION = "1.2.0"
     end
   end
 end

data/lib/logstash/inputs/kinesis/worker.rb

@@ -0,0 +1,56 @@
+class LogStash::Inputs::Kinesis::Worker
+  include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
+
+  attr_reader(
+    :checkpoint_interval,
+    :codec,
+    :decorator,
+    :logger,
+    :output_queue,
+  )
+
+  def initialize(*args)
+    # nasty hack, because this is the name of a method on IRecordProcessor, but also ruby's constructor
+    if !@constructed
+      @codec, @output_queue, @decorator, @checkpoint_interval, @logger = args
+      @next_checkpoint = Time.now - 600
+      @constructed = true
+    else
+      _shard_id, _ = args
+      @decoder = java.nio.charset::Charset.forName("UTF-8").newDecoder()
+    end
+  end
+  public :initialize
+
+  def processRecords(records, checkpointer)
+    records.each { |record| process_record(record) }
+    if Time.now >= @next_checkpoint
+      checkpoint(checkpointer)
+      @next_checkpoint = Time.now + @checkpoint_interval
+    end
+  end
+
+  def shutdown(checkpointer, reason)
+    if reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
+      checkpoint(checkpointer)
+    end
+  end
+
+  protected
+
+  def checkpoint(checkpointer)
+    checkpointer.checkpoint()
+  rescue => error
+    @logger.error("Kinesis worker failed checkpointing: #{error}")
+  end
+
+  def process_record(record)
+    raw = @decoder.decode(record.getData).to_s
+    @codec.decode(raw) do |event|
+      @decorator.call(event)
+      @output_queue << event
+    end
+  rescue => error
+    @logger.error("Error processing record: #{error}")
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: java
 authors:
 - Brian Palmer
@@ -52,6 +52,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.0.7
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -80,6 +94,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  name: rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email:
 - brian@codekitchen.net
@@ -93,6 +135,7 @@ files:
 - Rakefile
 - lib/logstash/inputs/kinesis.rb
 - lib/logstash/inputs/kinesis/version.rb
+- lib/logstash/inputs/kinesis/worker.rb
 homepage: https://github.com/codekitchen/logstash-input-kinesis
 licenses:
 - Apache License (2.0)