logstash-input-dynamodb 1.0.0-java

data/lib/logstash/inputs/DynamoDBLogParser.rb

@@ -0,0 +1,166 @@
+# encoding: utf-8
+#
+#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#
+require 'java'
+require 'json'
+require 'bigdecimal'
+require 'activesupport/json_encoder'
+require 'base64'
+
+require "logstash-input-dynamodb_jars"
+java_import "com.fasterxml.jackson.databind.ObjectMapper"
+java_import "com.amazonaws.services.dynamodbv2.model.AttributeValue"
+java_import "com.amazonaws.dynamodb.bootstrap.AttributeValueMixIn"
+
+module Logstash
+  module Inputs
+    module DynamoDB
+      class DynamoDBLogParser
+
+        MAX_NUMBER_OF_BYTES_FOR_NUMBER = 21;
+
+        def initialize(view_type, log_format, key_schema, region)
+          @view_type = view_type
+          @log_format = log_format
+          @mapper ||= ObjectMapper.new()
+          @mapper.setSerializationInclusion(JsonInclude::Include::NON_NULL)
+          @mapper.addMixInAnnotations(AttributeValue, AttributeValueMixIn);
+          @key_schema = key_schema
+          ActiveSupport.encode_big_decimal_as_string = false
+          @hash_template = Hash.new
+          @hash_template["eventID"] = "0"
+          @hash_template["eventName"] = "INSERT"
+          @hash_template["eventVersion"] = "1.0"
+          @hash_template["eventSource"] = "aws:dynamodb"
+          @hash_template["awsRegion"] = region
+        end
+
+        public
+        def parse_scan(log, new_image_size)
+          data_hash = JSON.parse(@mapper.writeValueAsString(log))
+
+          @hash_template["dynamodb"] = Hash.new
+          @hash_template["dynamodb"]["keys"] = Hash.new
+          size_bytes = calculate_key_size_in_bytes(log)
+          @key_schema.each { |x|
+            @hash_template["dynamodb"]["keys"][x] = data_hash[x]
+          }
+          unless @view_type == "keys_only"
+            size_bytes += new_image_size
+            @hash_template["dynamodb"]["newImage"] = data_hash
+          end
+          @hash_template["dynamodb"]["sequenceNumber"] = "0"
+          @hash_template["dynamodb"]["sizeBytes"] = size_bytes
+          @hash_template["dynamodb"]["streamViewType"] = @view_type.upcase
+
+          return parse_view_type(@hash_template)
+        end
+
+        public
+        def parse_stream(log)
+          return parse_view_type(JSON.parse(@mapper.writeValueAsString(log))["internalObject"])
+        end
+
+        private
+        def calculate_key_size_in_bytes(record)
+          key_size = 0
+          @key_schema.each { |x|
+            key_size += x.length
+            value = record.get(x)
+            if !(value.getB().nil?)
+              b = value.getB();
+              key_size += Base64.decode64(b).length
+            elsif !(value.getS().nil?)
+              s = value.getS();
+              key_size += s.length;
+            elsif !(value.getN().nil?)
+              key_size += MAX_NUMBER_OF_BYTES_FOR_NUMBER;
+            end
+          }
+          return key_size
+        end
+
+        private
+        def parse_view_type(hash)
+          if @log_format == LogStash::Inputs::DynamoDB::LF_PLAIN
+            return hash.to_json
+          end
+          case @view_type
+          when LogStash::Inputs::DynamoDB::VT_KEYS_ONLY
+            return parse_format(hash["dynamodb"]["keys"])
+          when LogStash::Inputs::DynamoDB::VT_OLD_IMAGE
+            return parse_format(hash["dynamodb"]["oldImage"])
+          when LogStash::Inputs::DynamoDB::VT_NEW_IMAGE
+            return parse_format(hash["dynamodb"]["newImage"]) #check new and old, dynamodb.
+          end
+        end
+
+        private
+        def parse_format(hash)
+          if @log_format == LogStash::Inputs::DynamoDB::LF_DYNAMODB
+            return hash.to_json
+          else
+            return dynamodb_to_json(hash)
+          end
+        end
+
+        private
+        def dynamodb_to_json(hash)
+          return formatAttributeValueMap(hash).to_json
+        end
+
+        private
+        def formatAttributeValueMap(hash)
+          keys_to_delete = []
+          hash.each do |k, v|
+            dynamodb_key = v.keys.first
+            dynamodb_value = v.values.first
+            if @log_format == LogStash::Inputs::DynamoDB::LF_JSON_NO_BIN and (dynamodb_key == "BS" or dynamodb_key == "B")
+              keys_to_delete.push(k) # remove binary values and binary sets
+              next
+            end
+            hash[k] = formatAttributeValue(v.keys.first, v.values.first)
+          end
+          keys_to_delete.each {|key| hash.delete(key)}
+          return hash
+        end
+
+        private
+        def formatAttributeValue(key, value)
+          case key
+          when "M"
+            formatAttributeValueMap(value)
+          when "L"
+            value.map! do |v|
+              v = formatAttributeValue(v.keys.first, v.values.first)
+            end
+          when "NS","SS","BS"
+            value.map! do |v|
+              v = formatAttributeValue(key[0], v)
+            end
+          when "N"
+            BigDecimal.new(value)
+          when "NULL"
+            nil
+          else
+            value
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/logstash/inputs/LogStashRecordProcessor.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+#
+#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#
+require "java"
+
+require "logstash-input-dynamodb_jars"
+java_import "com.amazonaws.services.kinesis.clientlibrary.types.ShutdownReason"
+java_import "java.lang.IllegalStateException"
+java_import "org.apache.log4j.LogManager"
+
+module Logstash
+  module Inputs
+    module DynamoDB
+      class LogStashRecordProcessor
+        include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
+
+        attr_accessor :queue, :shard_id
+
+        def initialize(queue)
+          # Workaround for IRecordProcessor.initialize(String shardId) interfering with constructor.
+          # No good way to overload methods in JRuby, so deciding which was supposed to be called here.
+          if (queue.is_a? String)
+            @shard_id  = queue
+            return
+          else
+            @queue ||= queue
+            @logger ||= LogStash::Inputs::DynamoDB.logger
+          end
+        end
+
+        def process_records(records, checkpointer)
+          @logger.debug("Processing batch of " + records.size().to_s + " records")
+          records.each do |record|
+            @queue.push(record)
+          end
+          #checkpoint once all of the records have been consumed
+          checkpointer.checkpoint()
+        end
+
+        def shutdown(checkpointer, reason)
+          case reason
+          when ShutdownReason::TERMINATE
+            checkpointer.checkpoint()
+          when ShutdownReason::ZOMBIE
+          else
+            raise RuntimeError, "Invalid shutdown reason."
+          end
+          unless @shard_id.nil?
+            @logger.info("shutting down record processor with shardId: " + @shard_id + " with reason " + reason.to_s)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/inputs/LogStashRecordProcessorFactory.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+#
+#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#
+require 'java'
+require_relative "LogStashRecordProcessor"
+
+require "logstash-input-dynamodb_jars"
+
+module KCL
+  include_package "com.amazonaws.services.kinesis.clientlibrary.interfaces"
+end
+
+module Logstash
+  module Inputs
+    module DynamoDB
+      class LogStashRecordProcessorFactory
+        include KCL::IRecordProcessorFactory
+
+        def initialize(queue)
+          @queue ||= queue
+        end
+
+        def create_processor
+          return Logstash::Inputs::DynamoDB::LogStashRecordProcessor.new(@queue)
+        end
+
+      end
+    end
+  end
+end

data/lib/logstash/inputs/dynamodb.rb

@@ -0,0 +1,341 @@
+# encoding: utf-8
+#
+#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "securerandom"
+require "thread"
+require "socket"
+require_relative "LogStashRecordProcessorFactory"
+require_relative "DynamoDBLogParser"
+
+require "logstash-input-dynamodb_jars"
+
+require 'java'
+java_import "com.amazonaws.AmazonClientException"
+java_import "org.apache.log4j.LogManager"
+java_import "org.apache.log4j.Level"
+java_import "com.fasterxml.jackson.annotation.JsonInclude"
+java_import "com.amazonaws.regions.RegionUtils"
+
+module AmazonDynamoDB
+  include_package "com.amazonaws"
+  include_package "com.amazonaws.services.dynamodbv2"
+  include_package "com.amazonaws.services.dynamodbv2.streamsadapter"
+  include_package "com.amazonaws.services.dynamodbv2.model"
+end
+module AmazonCredentials
+  include_package "com.amazonaws.auth"
+  include_package "com.amazonaws.internal"
+end
+
+module DynamoDBBootstrap
+  include_package "com.amazonaws.dynamodb.bootstrap"
+end
+
+module CloudWatch
+  include_package "com.amazonaws.services.cloudwatch"
+end
+
+module KCL
+  include_package "com.amazonaws.services.kinesis.clientlibrary.lib.worker"
+end
+
+#DynamoDBStreams plugin that will first scan the DynamoDB table
+#and then consume streams and push those records into Logstash
+class LogStash::Inputs::DynamoDB < LogStash::Inputs::Base
+  config_name "dynamodb"
+
+  USER_AGENT = " logstash-input-dynamodb/1.0.0".freeze
+
+  LF_DYNAMODB = "dymamodb".freeze
+  LF_JSON_NO_BIN = "json_drop_binary".freeze
+  LF_PLAIN = "plain".freeze
+  LF_JSON_BIN_AS_TEXT = "json_binary_as_text".freeze
+  VT_KEYS_ONLY = "keys_only".freeze
+  VT_OLD_IMAGE = "old_image".freeze
+  VT_NEW_IMAGE = "new_image".freeze
+  VT_ALL_IMAGES = "new_and_old_images".freeze
+
+  default :codec, 'json'
+
+  # The name of the table to copy and stream through Logstash
+  config :table_name, :validate => :string, :required => true
+
+  # Configuration for what information from the scan and streams to include in the log.
+  # keys_only will return the hash and range keys along with the values for each entry
+  # new_image will return the entire new entry and keys
+  # old_image will return the entire entry before modification and keys (NOTE: Cannot perform scan when using this option)
+  # new_and_old_images will return the old entry before modification along with the new entry and keys
+  config :view_type, :validate => [VT_KEYS_ONLY, VT_OLD_IMAGE, VT_NEW_IMAGE, VT_ALL_IMAGES], :required => true
+
+  # Endpoint from which the table is located. Example: dynamodb.us-east-1.amazonaws.com
+  config :endpoint, :validate => :string, :required => true
+
+  # Endpoint from which streams should read. Example: streams.dynamodb.us-east-1.amazonaws.com
+  config :streams_endpoint, :validate => :string
+
+  # AWS credentials access key.
+  config :aws_access_key_id, :validate => :string, :default => ""
+
+  # AWS credentials secret access key.
+  config :aws_secret_access_key, :validate => :string, :default => ""
+
+  # A flag to indicate whether or not the plugin should scan the entire table before streaming new records.
+  # Streams will only push records that are less than 24 hours old, so in order to get the entire table
+  # an initial scan must be done.
+  config :perform_scan, :validate => :boolean, :default => true
+
+  # A string that uniquely identifies the KCL checkpointer name and cloudwatch metrics name.
+  # This is used when one worker leaves a shard so that another worker knows where to start again.
+  config :checkpointer, :validate => :string, :default => "logstash_input_dynamodb_cptr"
+
+  # Option to publish metrics to Cloudwatch using the checkpointer name.
+  config :publish_metrics, :validate => :boolean, :default => false
+
+  # Option to not automatically stream new data into logstash from DynamoDB streams.
+  config :perform_stream, :validate => :boolean, :default => true
+
+  # Number of read operations per second to perform when scanning the specified table.
+  config :read_ops, :validate => :number, :default => 1
+
+  # Number of threads to use when scanning the specified table
+  config :number_of_scan_threads, :validate => :number, :default => 1
+
+  # Number of threads to write to the logstash queue when scanning the table
+  config :number_of_write_threads, :validate => :number, :default => 1
+
+  # Configuation for how the logs will be transferred.
+  # plain is simply pass the message along without editing it.
+  # dynamodb will return just the data specified in the view_format in dynamodb format.
+    # For more information see: docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataFormat.html
+  # json_drop_binary will return just the data specified in the view_format in JSON while not including any binary values that were present.
+  # json_binary_as_text will return just the data specified in the view_format in JSON while including binary values as base64-encoded text.
+  config :log_format, :validate => [LF_PLAIN, LF_DYNAMODB, LF_JSON_NO_BIN, LF_JSON_BIN_AS_TEXT], :default => "plain"
+
+  public
+  def build_credentials
+    if !@aws_access_key_id.to_s.empty? and !@aws_secret_access_key.to_s.empty?
+      @logger.info("Using static credentials: " + @aws_access_key_id + ", " + @aws_secret_access_key)
+      basic = AmazonCredentials::BasicAWSCredentials.new(@aws_access_key_id, @aws_secret_access_key)
+      return AmazonCredentials::StaticCredentialsProvider.new(basic)
+    else
+      @logger.info("Using default provider chain")
+      return AmazonCredentials::DefaultAWSCredentialsProviderChain.new()
+    end # if neither aws access keys
+  end # def build_credentials
+
+  public
+  def register
+    LogStash::Logger.setup_log4j(@logger)
+
+    @host = Socket.gethostname
+    @logger.info("Tablename: " + @table_name)
+    @queue = SizedQueue.new(20)
+    @credentials = build_credentials()
+    @logger.info("Checkpointer: " + @checkpointer)
+
+    if @perform_scan and @view_type == VT_OLD_IMAGE
+      raise(LogStash::ConfigurationError, "Cannot perform scan with view type: " + @view_type + " configuration")
+    end
+    if @view_type == VT_ALL_IMAGES and !(@log_format == LF_PLAIN)
+      raise(LogStash::ConfigurationError, "Cannot show view_type: " + @view_type + ", with log_format: " + @log_format)
+    end
+
+    #Create DynamoDB Client
+    @client_configuration = AmazonDynamoDB::ClientConfiguration.new()
+    @client_configuration.setUserAgent(@client_configuration.getUserAgent() + USER_AGENT)
+    @dynamodb_client = AmazonDynamoDB::AmazonDynamoDBClient.new(@credentials, @client_configuration)
+
+    @logger.info(@dynamodb_client.to_s)
+
+    @dynamodb_client.setEndpoint(@endpoint)
+    @logger.info("DynamoDB endpoint: " + @endpoint)
+
+    @key_schema = Array.new
+    @table_description = @dynamodb_client.describeTable(@table_name).getTable()
+    key_iterator = @table_description.getKeySchema().iterator()
+    while(key_iterator.hasNext())
+      @key_schema.push(key_iterator.next().getAttributeName().to_s)
+    end
+    region = RegionUtils.getRegionByEndpoint(@endpoint)
+
+    @parser ||= Logstash::Inputs::DynamoDB::DynamoDBLogParser.new(@view_type, @log_format, @key_schema, region)
+
+    if @perform_stream
+      setup_stream
+    end # unless @perform_stream
+  end # def register
+
+  public
+  def run(logstash_queue)
+    begin
+      run_with_catch(logstash_queue)
+    rescue LogStash::ShutdownSignal
+      exit_threads
+      until @queue.empty?
+        @logger.info("Flushing rest of events in logstash queue")
+        event = @queue.pop()
+        queue_event(@parser.parse_stream(event), logstash_queue, @host)
+      end # until !@queue.empty?
+    end # begin
+  end # def run(logstash_queue)
+
+  # Starts KCL app in a background thread
+  # Starts parallel scan if need be in a background thread
+  private
+  def run_with_catch(logstash_queue)
+    if @perform_scan
+      scan(logstash_queue)
+    end # if @perform_scan
+
+    # Once scan is finished, start kcl thread to read from streams
+    if @perform_stream
+      stream(logstash_queue)
+    end # unless @perform_stream
+  end # def run
+
+  private
+  def setup_stream
+    worker_id = SecureRandom.uuid()
+    @logger.info("WorkerId: " + worker_id)
+
+    dynamodb_streams_client = AmazonDynamoDB::AmazonDynamoDBStreamsClient.new(@credentials, @client_configuration)
+    adapter = Java::ComAmazonawsServicesDynamodbv2Streamsadapter::AmazonDynamoDBStreamsAdapterClient.new(@credentials)
+    if !@streams_endpoint.nil?
+      adapter.setEndpoint(@streams_endpoint)
+      dynamodb_streams_client.setEndpoint(@streams_endpoint)
+      @logger.info("DynamoDB Streams endpoint: " + @streams_endpoint)
+    else
+      raise(LogStash::ConfigurationError, "Cannot stream without a configured streams endpoint")
+    end # if not @streams_endpoint.to_s.empty?
+
+    stream_arn = nil
+    begin
+      stream_arn = @table_description.getLatestStreamArn()
+      stream_description = dynamodb_streams_client.describeStream(AmazonDynamoDB::DescribeStreamRequest.new() \
+        .withStreamArn(stream_arn)).getStreamDescription()
+
+      stream_status = stream_description.getStreamStatus()
+
+      stream_view_type = stream_description.getStreamViewType().to_s.downcase
+      unless (stream_view_type == @view_type or @view_type == VT_KEYS_ONLY or stream_view_type == VT_ALL_IMAGES)
+        raise(LogStash::ConfigurationError, "Cannot stream " + @view_type + " when stream is setup for " + stream_view_type)
+      end
+
+      while stream_status == "ENABLING"
+        if(stream_status == "ENABLING")
+          @logger.info("Sleeping until stream is enabled")
+          sleep(1)
+        end # if stream_status == "ENABLING"
+        stream_description = dynamodb_streams_client.describeStream(AmazonDynamoDB::DescribeStreamRequest.new() \
+          .withStreamArn(stream_arn)).getStreamDescription()
+        stream_status = stream_description.getStreamStatus()
+      end # while not active
+
+      if !(stream_status == "ENABLED")
+        raise(LogStash::PluginLoadingError, "No streams are enabled")
+      end # if not active
+      @logger.info("Stream Id: " + stream_arn)
+    rescue AmazonDynamoDB::ResourceNotFoundException => rnfe
+      raise(LogStash::PluginLoadingError, rnfe.message)
+    rescue AmazonClientException => ace
+      raise(LogStash::ConfigurationError, "AWS credentials invalid or not found in the provider chain\n" + ace.message)
+    end # begin
+
+    kcl_config = KCL::KinesisClientLibConfiguration.new(@checkpointer, stream_arn, @credentials, worker_id) \
+      .withInitialPositionInStream(KCL::InitialPositionInStream::TRIM_HORIZON)
+		cloudwatch_client = nil
+    if @publish_metrics
+      cloudwatch_client = CloudWatch::AmazonCloudWatchClient.new(@credentials)
+    else
+      kclMetricsLogger = LogManager.getLogger("com.amazonaws.services.kinesis.metrics")
+      kclMetricsLogger.setAdditivity(false)
+      kclMetricsLogger.setLevel(Level::OFF)
+    end # if @publish_metrics
+    @worker = KCL::Worker.new(Logstash::Inputs::DynamoDB::LogStashRecordProcessorFactory.new(@queue), kcl_config, adapter, @dynamodb_client, cloudwatch_client)
+  end # def setup_stream
+
+  private
+  def scan(logstash_queue)
+    @logger.info("Starting scan...")
+    @logstash_writer = DynamoDBBootstrap::BlockingQueueConsumer.new(@number_of_write_threads)
+
+    @connector = DynamoDBBootstrap::DynamoDBBootstrapWorker.new(@dynamodb_client, @read_ops, @table_name, @number_of_scan_threads)
+    start_table_copy_thread
+
+    scan_queue = @logstash_writer.getQueue()
+    while true
+      event = scan_queue.take()
+      if event.getEntry().nil? and event.getSize() == -1
+        break
+      end # if event.isEmpty()
+      queue_event(@parser.parse_scan(event.getEntry(), event.getSize()), logstash_queue, @host)
+    end # while true
+  end
+
+  private
+  def stream(logstash_queue)
+    @logger.info("Starting stream...")
+    start_kcl_thread
+
+    while true
+      event = @queue.pop()
+      queue_event(@parser.parse_stream(event), logstash_queue, @host)
+    end # while true
+  end
+
+  private
+  def exit_threads
+    unless @dynamodb_scan_thread.nil?
+      @dynamodb_scan_thread.exit
+    end # unless @dynamodb_scan_thread.nil?
+
+    unless @kcl_thread.nil?
+      @kcl_thread.exit
+    end # unless @kcl_thread.nil?
+  end # def exit_threads
+
+  public
+  def queue_event(event, logstash_queue, event_host)
+    logstash_event = LogStash::Event.new("message" => event, "host" => event_host)
+    decorate(logstash_event)
+    logstash_queue << logstash_event
+  end # def queue_event
+
+  private
+  def start_table_copy_thread
+    @dynamodb_scan_thread = Thread.new(@connector, @logstash_writer) {
+      begin
+        @connector.pipe(@logstash_writer)
+      rescue Exception => e
+        abort("Scanning the table caused an error.\n" + e.message)
+      end # begin
+    }
+  end # def start_table_copy_thread()
+
+  private
+  def start_kcl_thread
+    @kcl_thread = Thread.new(@worker) {
+      begin
+        @worker.run()
+      rescue Exception => e
+        abort("KCL worker encountered an error.\n" + e.message)
+      end # begin
+    }
+  end # def start_kcl_thread
+
+end # class Logstash::Inputs::DynamoDB