logstash-input-dynamodb 1.0.4-java → 2.0.0-java

data/Rakefile

@@ -1,7 +0,0 @@
-@files=[]
-
-task :default do
-    system("rake -T")
-end
-
-require "logstash/devutils/rake"

data/lib/logstash-input-dynamodb_jars.rb

@@ -1,25 +0,0 @@
-# this is a generated file, to avoid over-writing it just delete this comment
-require 'jar_dependencies'
-
-require_jar( 'com.amazonaws', 'aws-java-sdk-kinesis', '1.10.60' )
-require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.5.3' )
-require_jar( 'com.amazonaws', 'aws-java-sdk-core', '1.10.60' )
-require_jar( 'log4j', 'log4j', '1.2.17' )
-require_jar( 'commons-codec', 'commons-codec', '1.9' )
-require_jar( 'com.amazonaws', 'aws-java-sdk-cloudwatch', '1.10.20' )
-require_jar( 'com.amazonaws', 'aws-java-sdk-s3', '1.10.60' )
-require_jar( 'com.beust', 'jcommander', '1.48' )
-require_jar( 'com.amazonaws', 'dynamodb-import-export-tool', '1.0.0' )
-require_jar( 'commons-lang', 'commons-lang', '2.6' )
-require_jar( 'joda-time', 'joda-time', '2.8.1' )
-require_jar( 'commons-logging', 'commons-logging', '1.1.3' )
-require_jar( 'com.amazonaws', 'dynamodb-streams-kinesis-adapter', '1.0.2' )
-require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.5.3' )
-require_jar( 'com.amazonaws', 'amazon-kinesis-client', '1.6.1' )
-require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.5.0' )
-require_jar( 'com.amazonaws', 'aws-java-sdk-dynamodb', '1.10.60' )
-require_jar( 'com.amazonaws', 'aws-java-sdk-kms', '1.10.60' )
-require_jar( 'org.apache.httpcomponents', 'httpclient', '4.5.2' )
-require_jar( 'org.apache.httpcomponents', 'httpcore', '4.4.4' )
-require_jar( 'com.google.protobuf', 'protobuf-java', '2.6.1' )
-require_jar( 'com.google.guava', 'guava', '15.0' )

data/lib/logstash/inputs/DynamoDBLogParser.rb

@@ -1,166 +0,0 @@
-# encoding: utf-8
-#
-#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-#
-require 'java'
-require 'json'
-require 'bigdecimal'
-require 'activesupport/json_encoder'
-require 'base64'
-
-require "logstash-input-dynamodb_jars"
-java_import "com.fasterxml.jackson.databind.ObjectMapper"
-java_import "com.amazonaws.services.dynamodbv2.model.AttributeValue"
-java_import "com.amazonaws.dynamodb.bootstrap.AttributeValueMixIn"
-
-module Logstash
-  module Inputs
-    module DynamoDB
-      class DynamoDBLogParser
-
-        MAX_NUMBER_OF_BYTES_FOR_NUMBER = 21;
-
-        def initialize(view_type, log_format, key_schema, region)
-          @view_type = view_type
-          @log_format = log_format
-          @mapper ||= ObjectMapper.new()
-          @mapper.setSerializationInclusion(JsonInclude::Include::NON_NULL)
-          @mapper.addMixInAnnotations(AttributeValue, AttributeValueMixIn);
-          @key_schema = key_schema
-          ActiveSupport.encode_big_decimal_as_string = false
-          @hash_template = Hash.new
-          @hash_template["eventID"] = "0"
-          @hash_template["eventName"] = "INSERT"
-          @hash_template["eventVersion"] = "1.0"
-          @hash_template["eventSource"] = "aws:dynamodb"
-          @hash_template["awsRegion"] = region
-        end
-
-        public
-        def parse_scan(log, new_image_size)
-          data_hash = JSON.parse(@mapper.writeValueAsString(log))
-
-          @hash_template["dynamodb"] = Hash.new
-          @hash_template["dynamodb"]["keys"] = Hash.new
-          size_bytes = calculate_key_size_in_bytes(log)
-          @key_schema.each { |x|
-            @hash_template["dynamodb"]["keys"][x] = data_hash[x]
-          }
-          unless @view_type == "keys_only"
-            size_bytes += new_image_size
-            @hash_template["dynamodb"]["newImage"] = data_hash
-          end
-          @hash_template["dynamodb"]["sequenceNumber"] = "0"
-          @hash_template["dynamodb"]["sizeBytes"] = size_bytes
-          @hash_template["dynamodb"]["streamViewType"] = @view_type.upcase
-
-          return parse_view_type(@hash_template)
-        end
-
-        public
-        def parse_stream(log)
-          return parse_view_type(JSON.parse(@mapper.writeValueAsString(log))["internalObject"])
-        end
-
-        private
-        def calculate_key_size_in_bytes(record)
-          key_size = 0
-          @key_schema.each { |x|
-            key_size += x.length
-            value = record.get(x)
-            if !(value.getB().nil?)
-              b = value.getB();
-              key_size += Base64.decode64(b).length
-            elsif !(value.getS().nil?)
-              s = value.getS();
-              key_size += s.length;
-            elsif !(value.getN().nil?)
-              key_size += MAX_NUMBER_OF_BYTES_FOR_NUMBER;
-            end
-          }
-          return key_size
-        end
-
-        private
-        def parse_view_type(hash)
-          if @log_format == LogStash::Inputs::DynamoDB::LF_PLAIN
-            return hash.to_json
-          end
-          case @view_type
-          when LogStash::Inputs::DynamoDB::VT_KEYS_ONLY
-            return parse_format(hash["dynamodb"]["keys"])
-          when LogStash::Inputs::DynamoDB::VT_OLD_IMAGE
-            return parse_format(hash["dynamodb"]["oldImage"])
-          when LogStash::Inputs::DynamoDB::VT_NEW_IMAGE
-            return parse_format(hash["dynamodb"]["newImage"]) #check new and old, dynamodb.
-          end
-        end
-
-        private
-        def parse_format(hash)
-          if @log_format == LogStash::Inputs::DynamoDB::LF_DYNAMODB
-            return hash.to_json
-          else
-            return dynamodb_to_json(hash)
-          end
-        end
-
-        private
-        def dynamodb_to_json(hash)
-          return formatAttributeValueMap(hash).to_json
-        end
-
-        private
-        def formatAttributeValueMap(hash)
-          keys_to_delete = []
-          hash.each do |k, v|
-            dynamodb_key = v.keys.first
-            dynamodb_value = v.values.first
-            if @log_format == LogStash::Inputs::DynamoDB::LF_JSON_NO_BIN and (dynamodb_key == "BS" or dynamodb_key == "B")
-              keys_to_delete.push(k) # remove binary values and binary sets
-              next
-            end
-            hash[k] = formatAttributeValue(v.keys.first, v.values.first)
-          end
-          keys_to_delete.each {|key| hash.delete(key)}
-          return hash
-        end
-
-        private
-        def formatAttributeValue(key, value)
-          case key
-          when "M"
-            formatAttributeValueMap(value)
-          when "L"
-            value.map! do |v|
-              v = formatAttributeValue(v.keys.first, v.values.first)
-            end
-          when "NS","SS","BS"
-            value.map! do |v|
-              v = formatAttributeValue(key[0], v)
-            end
-          when "N"
-            BigDecimal.new(value)
-          when "NULL"
-            nil
-          else
-            value
-          end
-        end
-
-      end
-    end
-  end
-end

data/lib/logstash/inputs/LogStashRecordProcessor.rb

@@ -1,68 +0,0 @@
-# encoding: utf-8
-#
-#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-#
-require "java"
-
-require "logstash-input-dynamodb_jars"
-java_import "com.amazonaws.services.kinesis.clientlibrary.types.ShutdownReason"
-java_import "java.lang.IllegalStateException"
-java_import "org.apache.log4j.LogManager"
-
-module Logstash
-  module Inputs
-    module DynamoDB
-      class LogStashRecordProcessor
-        include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
-
-        attr_accessor :queue, :shard_id
-
-        def initialize(queue)
-          # Workaround for IRecordProcessor.initialize(String shardId) interfering with constructor.
-          # No good way to overload methods in JRuby, so deciding which was supposed to be called here.
-          if (queue.is_a? String)
-            @shard_id  = queue
-            return
-          else
-            @queue ||= queue
-            @logger ||= LogStash::Inputs::DynamoDB.logger
-          end
-        end
-
-        def process_records(records, checkpointer)
-          @logger.debug("Processing batch of " + records.size().to_s + " records")
-          records.each do |record|
-            @queue.push(record)
-          end
-          #checkpoint once all of the records have been consumed
-          checkpointer.checkpoint()
-        end
-
-        def shutdown(checkpointer, reason)
-          case reason
-          when ShutdownReason::TERMINATE
-            checkpointer.checkpoint()
-          when ShutdownReason::ZOMBIE
-          else
-            raise RuntimeError, "Invalid shutdown reason."
-          end
-          unless @shard_id.nil?
-            @logger.info("shutting down record processor with shardId: " + @shard_id + " with reason " + reason.to_s)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/logstash/inputs/LogStashRecordProcessorFactory.rb

@@ -1,43 +0,0 @@
-# encoding: utf-8
-#
-#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-#
-require 'java'
-require_relative "LogStashRecordProcessor"
-
-require "logstash-input-dynamodb_jars"
-
-module KCL
-  include_package "com.amazonaws.services.kinesis.clientlibrary.interfaces"
-end
-
-module Logstash
-  module Inputs
-    module DynamoDB
-      class LogStashRecordProcessorFactory
-        include KCL::IRecordProcessorFactory
-
-        def initialize(queue)
-          @queue ||= queue
-        end
-
-        def create_processor
-          return Logstash::Inputs::DynamoDB::LogStashRecordProcessor.new(@queue)
-        end
-
-      end
-    end
-  end
-end

data/lib/logstash/inputs/dynamodb.rb

@@ -1,341 +0,0 @@
-# encoding: utf-8
-#
-#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-#
-require "logstash/inputs/base"
-require "logstash/namespace"
-require "securerandom"
-require "thread"
-require "socket"
-require_relative "LogStashRecordProcessorFactory"
-require_relative "DynamoDBLogParser"
-
-require "logstash-input-dynamodb_jars"
-
-require 'java'
-java_import "com.amazonaws.AmazonClientException"
-java_import "org.apache.log4j.LogManager"
-java_import "org.apache.log4j.Level"
-java_import "com.fasterxml.jackson.annotation.JsonInclude"
-java_import "com.amazonaws.regions.RegionUtils"
-
-module AmazonDynamoDB
-  include_package "com.amazonaws"
-  include_package "com.amazonaws.services.dynamodbv2"
-  include_package "com.amazonaws.services.dynamodbv2.streamsadapter"
-  include_package "com.amazonaws.services.dynamodbv2.model"
-end
-module AmazonCredentials
-  include_package "com.amazonaws.auth"
-  include_package "com.amazonaws.internal"
-end
-
-module DynamoDBBootstrap
-  include_package "com.amazonaws.dynamodb.bootstrap"
-end
-
-module CloudWatch
-  include_package "com.amazonaws.services.cloudwatch"
-end
-
-module KCL
-  include_package "com.amazonaws.services.kinesis.clientlibrary.lib.worker"
-end
-
-#DynamoDBStreams plugin that will first scan the DynamoDB table
-#and then consume streams and push those records into Logstash
-class LogStash::Inputs::DynamoDB < LogStash::Inputs::Base
-  config_name "dynamodb"
-
-  USER_AGENT = " logstash-input-dynamodb/1.0.0".freeze
-
-  LF_DYNAMODB = "dymamodb".freeze
-  LF_JSON_NO_BIN = "json_drop_binary".freeze
-  LF_PLAIN = "plain".freeze
-  LF_JSON_BIN_AS_TEXT = "json_binary_as_text".freeze
-  VT_KEYS_ONLY = "keys_only".freeze
-  VT_OLD_IMAGE = "old_image".freeze
-  VT_NEW_IMAGE = "new_image".freeze
-  VT_ALL_IMAGES = "new_and_old_images".freeze
-
-  default :codec, 'json'
-
-  # The name of the table to copy and stream through Logstash
-  config :table_name, :validate => :string, :required => true
-
-  # Configuration for what information from the scan and streams to include in the log.
-  # keys_only will return the hash and range keys along with the values for each entry
-  # new_image will return the entire new entry and keys
-  # old_image will return the entire entry before modification and keys (NOTE: Cannot perform scan when using this option)
-  # new_and_old_images will return the old entry before modification along with the new entry and keys
-  config :view_type, :validate => [VT_KEYS_ONLY, VT_OLD_IMAGE, VT_NEW_IMAGE, VT_ALL_IMAGES], :required => true
-
-  # Endpoint from which the table is located. Example: dynamodb.us-east-1.amazonaws.com
-  config :endpoint, :validate => :string, :required => true
-
-  # Endpoint from which streams should read. Example: streams.dynamodb.us-east-1.amazonaws.com
-  config :streams_endpoint, :validate => :string
-
-  # AWS credentials access key.
-  config :aws_access_key_id, :validate => :string, :default => ""
-
-  # AWS credentials secret access key.
-  config :aws_secret_access_key, :validate => :string, :default => ""
-
-  # A flag to indicate whether or not the plugin should scan the entire table before streaming new records.
-  # Streams will only push records that are less than 24 hours old, so in order to get the entire table
-  # an initial scan must be done.
-  config :perform_scan, :validate => :boolean, :default => true
-
-  # A string that uniquely identifies the KCL checkpointer name and cloudwatch metrics name.
-  # This is used when one worker leaves a shard so that another worker knows where to start again.
-  config :checkpointer, :validate => :string, :default => "logstash_input_dynamodb_cptr"
-
-  # Option to publish metrics to Cloudwatch using the checkpointer name.
-  config :publish_metrics, :validate => :boolean, :default => false
-
-  # Option to not automatically stream new data into logstash from DynamoDB streams.
-  config :perform_stream, :validate => :boolean, :default => true
-
-  # Number of read operations per second to perform when scanning the specified table.
-  config :read_ops, :validate => :number, :default => 1
-
-  # Number of threads to use when scanning the specified table
-  config :number_of_scan_threads, :validate => :number, :default => 1
-
-  # Number of threads to write to the logstash queue when scanning the table
-  config :number_of_write_threads, :validate => :number, :default => 1
-
-  # Configuation for how the logs will be transferred.
-  # plain is simply pass the message along without editing it.
-  # dynamodb will return just the data specified in the view_format in dynamodb format.
-    # For more information see: docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataFormat.html
-  # json_drop_binary will return just the data specified in the view_format in JSON while not including any binary values that were present.
-  # json_binary_as_text will return just the data specified in the view_format in JSON while including binary values as base64-encoded text.
-  config :log_format, :validate => [LF_PLAIN, LF_DYNAMODB, LF_JSON_NO_BIN, LF_JSON_BIN_AS_TEXT], :default => "plain"
-
-  public
-  def build_credentials
-    if !@aws_access_key_id.to_s.empty? and !@aws_secret_access_key.to_s.empty?
-      @logger.info("Using static credentials: " + @aws_access_key_id + ", " + @aws_secret_access_key)
-      basic = AmazonCredentials::BasicAWSCredentials.new(@aws_access_key_id, @aws_secret_access_key)
-      return AmazonCredentials::StaticCredentialsProvider.new(basic)
-    else
-      @logger.info("Using default provider chain")
-      return AmazonCredentials::DefaultAWSCredentialsProviderChain.new()
-    end # if neither aws access keys
-  end # def build_credentials
-
-  public
-  def register
-    LogStash::Logger.setup_log4j(@logger)
-
-    @host = Socket.gethostname
-    @logger.info("Tablename: " + @table_name)
-    @queue = SizedQueue.new(20)
-    @credentials = build_credentials()
-    @logger.info("Checkpointer: " + @checkpointer)
-
-    if @perform_scan and @view_type == VT_OLD_IMAGE
-      raise(LogStash::ConfigurationError, "Cannot perform scan with view type: " + @view_type + " configuration")
-    end
-    if @view_type == VT_ALL_IMAGES and !(@log_format == LF_PLAIN)
-      raise(LogStash::ConfigurationError, "Cannot show view_type: " + @view_type + ", with log_format: " + @log_format)
-    end
-
-    #Create DynamoDB Client
-    @client_configuration = AmazonDynamoDB::ClientConfiguration.new()
-    @client_configuration.setUserAgent(@client_configuration.getUserAgent() + USER_AGENT)
-    @dynamodb_client = AmazonDynamoDB::AmazonDynamoDBClient.new(@credentials, @client_configuration)
-
-    @logger.info(@dynamodb_client.to_s)
-
-    @dynamodb_client.setEndpoint(@endpoint)
-    @logger.info("DynamoDB endpoint: " + @endpoint)
-
-    @key_schema = Array.new
-    @table_description = @dynamodb_client.describeTable(@table_name).getTable()
-    key_iterator = @table_description.getKeySchema().iterator()
-    while(key_iterator.hasNext())
-      @key_schema.push(key_iterator.next().getAttributeName().to_s)
-    end
-    region = RegionUtils.getRegionByEndpoint(@endpoint)
-
-    @parser ||= Logstash::Inputs::DynamoDB::DynamoDBLogParser.new(@view_type, @log_format, @key_schema, region)
-
-    if @perform_stream
-      setup_stream
-    end # unless @perform_stream
-  end # def register
-
-  public
-  def run(logstash_queue)
-    begin
-      run_with_catch(logstash_queue)
-    rescue LogStash::ShutdownSignal
-      exit_threads
-      until @queue.empty?
-        @logger.info("Flushing rest of events in logstash queue")
-        event = @queue.pop()
-        queue_event(@parser.parse_stream(event), logstash_queue, @host)
-      end # until !@queue.empty?
-    end # begin
-  end # def run(logstash_queue)
-
-  # Starts KCL app in a background thread
-  # Starts parallel scan if need be in a background thread
-  private
-  def run_with_catch(logstash_queue)
-    if @perform_scan
-      scan(logstash_queue)
-    end # if @perform_scan
-
-    # Once scan is finished, start kcl thread to read from streams
-    if @perform_stream
-      stream(logstash_queue)
-    end # unless @perform_stream
-  end # def run
-
-  private
-  def setup_stream
-    worker_id = SecureRandom.uuid()
-    @logger.info("WorkerId: " + worker_id)
-
-    dynamodb_streams_client = AmazonDynamoDB::AmazonDynamoDBStreamsClient.new(@credentials, @client_configuration)
-    adapter = Java::ComAmazonawsServicesDynamodbv2Streamsadapter::AmazonDynamoDBStreamsAdapterClient.new(@credentials)
-    if !@streams_endpoint.nil?
-      adapter.setEndpoint(@streams_endpoint)
-      dynamodb_streams_client.setEndpoint(@streams_endpoint)
-      @logger.info("DynamoDB Streams endpoint: " + @streams_endpoint)
-    else
-      raise(LogStash::ConfigurationError, "Cannot stream without a configured streams endpoint")
-    end # if not @streams_endpoint.to_s.empty?
-
-    stream_arn = nil
-    begin
-      stream_arn = @table_description.getLatestStreamArn()
-      stream_description = dynamodb_streams_client.describeStream(AmazonDynamoDB::DescribeStreamRequest.new() \
-        .withStreamArn(stream_arn)).getStreamDescription()
-
-      stream_status = stream_description.getStreamStatus()
-
-      stream_view_type = stream_description.getStreamViewType().to_s.downcase
-      unless (stream_view_type == @view_type or @view_type == VT_KEYS_ONLY or stream_view_type == VT_ALL_IMAGES)
-        raise(LogStash::ConfigurationError, "Cannot stream " + @view_type + " when stream is setup for " + stream_view_type)
-      end
-
-      while stream_status == "ENABLING"
-        if(stream_status == "ENABLING")
-          @logger.info("Sleeping until stream is enabled")
-          sleep(1)
-        end # if stream_status == "ENABLING"
-        stream_description = dynamodb_streams_client.describeStream(AmazonDynamoDB::DescribeStreamRequest.new() \
-          .withStreamArn(stream_arn)).getStreamDescription()
-        stream_status = stream_description.getStreamStatus()
-      end # while not active
-
-      if !(stream_status == "ENABLED")
-        raise(LogStash::PluginLoadingError, "No streams are enabled")
-      end # if not active
-      @logger.info("Stream Id: " + stream_arn)
-    rescue AmazonDynamoDB::ResourceNotFoundException => rnfe
-      raise(LogStash::PluginLoadingError, rnfe.message)
-    rescue AmazonClientException => ace
-      raise(LogStash::ConfigurationError, "AWS credentials invalid or not found in the provider chain\n" + ace.message)
-    end # begin
-
-    kcl_config = KCL::KinesisClientLibConfiguration.new(@checkpointer, stream_arn, @credentials, worker_id) \
-      .withInitialPositionInStream(KCL::InitialPositionInStream::TRIM_HORIZON)
-		cloudwatch_client = nil
-    if @publish_metrics
-      cloudwatch_client = CloudWatch::AmazonCloudWatchClient.new(@credentials)
-    else
-      kclMetricsLogger = LogManager.getLogger("com.amazonaws.services.kinesis.metrics")
-      kclMetricsLogger.setAdditivity(false)
-      kclMetricsLogger.setLevel(Level::OFF)
-    end # if @publish_metrics
-    @worker = KCL::Worker.new(Logstash::Inputs::DynamoDB::LogStashRecordProcessorFactory.new(@queue), kcl_config, adapter, @dynamodb_client, cloudwatch_client)
-  end # def setup_stream
-
-  private
-  def scan(logstash_queue)
-    @logger.info("Starting scan...")
-    @logstash_writer = DynamoDBBootstrap::BlockingQueueConsumer.new(@number_of_write_threads)
-
-    @connector = DynamoDBBootstrap::DynamoDBBootstrapWorker.new(@dynamodb_client, @read_ops, @table_name, @number_of_scan_threads)
-    start_table_copy_thread
-
-    scan_queue = @logstash_writer.getQueue()
-    while true
-      event = scan_queue.take()
-      if event.getEntry().nil? and event.getSize() == -1
-        break
-      end # if event.isEmpty()
-      queue_event(@parser.parse_scan(event.getEntry(), event.getSize()), logstash_queue, @host)
-    end # while true
-  end
-
-  private
-  def stream(logstash_queue)
-    @logger.info("Starting stream...")
-    start_kcl_thread
-
-    while true
-      event = @queue.pop()
-      queue_event(@parser.parse_stream(event), logstash_queue, @host)
-    end # while true
-  end
-
-  private
-  def exit_threads
-    unless @dynamodb_scan_thread.nil?
-      @dynamodb_scan_thread.exit
-    end # unless @dynamodb_scan_thread.nil?
-
-    unless @kcl_thread.nil?
-      @kcl_thread.exit
-    end # unless @kcl_thread.nil?
-  end # def exit_threads
-
-  public
-  def queue_event(event, logstash_queue, event_host)
-    logstash_event = LogStash::Event.new("message" => event, "host" => event_host)
-    decorate(logstash_event)
-    logstash_queue << logstash_event
-  end # def queue_event
-
-  private
-  def start_table_copy_thread
-    @dynamodb_scan_thread = Thread.new(@connector, @logstash_writer) {
-      begin
-        @connector.pipe(@logstash_writer)
-      rescue Exception => e
-        abort("Scanning the table caused an error.\n" + e.message)
-      end # begin
-    }
-  end # def start_table_copy_thread()
-
-  private
-  def start_kcl_thread
-    @kcl_thread = Thread.new(@worker) {
-      begin
-        @worker.run()
-      rescue Exception => e
-        abort("KCL worker encountered an error.\n" + e.message)
-      end # begin
-    }
-  end # def start_kcl_thread
-
-end # class Logstash::Inputs::DynamoDB