logstash-input-dynamodb 1.0.0-java

data/logstash-input-dynamodb.gemspec

@@ -0,0 +1,38 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-input-dynamodb'
+  s.version = '1.0.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "This input plugin scans a specified DynamoDB table and then reads changes to a DynamoDB table from the associated DynamoDB Stream."
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Amazon"]
+  s.email = 'dynamodb-interest@amazon.com'
+  s.homepage = "https://github.com/logstash-plugins/logstash-input-dynamodb"
+  s.require_paths = ["lib"]
+  s.platform = 'java'
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-codec-json"
+  s.add_runtime_dependency "activesupport-json_encoder"
+  # Jar dependencies
+  s.requirements << "jar 'com.amazonaws:amazon-kinesis-client', '1.6.0'"
+  s.requirements << "jar 'log4j:log4j', '1.2.17'"
+  s.requirements << "jar 'com.amazonaws:aws-java-sdk-dynamodb', '1.10.10'"
+  s.requirements << "jar 'com.amazonaws:aws-java-sdk-core', '1.10.10'"
+  s.requirements << "jar 'com.amazonaws:dynamodb-import-export-tool', '1.0.0'"
+  s.requirements << "jar 'commons-logging:commons-logging', '1.1.3'"
+  s.requirements << "jar 'com.amazonaws:dynamodb-streams-kinesis-adapter', '1.0.0'"
+  s.requirements << "jar 'com.google.guava:guava', '15.0'"
+  s.add_runtime_dependency 'jar-dependencies'
+  # Development dependencies
+  s.add_development_dependency "logstash-devutils"
+  s.add_development_dependency "mocha"
+end

data/spec/inputs/dynamodb_spec.rb

@@ -0,0 +1,198 @@
+# encoding: utf-8
+#
+#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#
+require "spec/spec_helper"
+
+class LogStash::Inputs::TestDynamoDB < LogStash::Inputs::DynamoDB
+  default :codec, 'json'
+
+  private
+  def shutdown_count
+    @shutdown_count ||= 0
+  end
+
+  def queue_event(event, logstash_queue, host)
+    super(event, logstash_queue, host)
+    # Add additional item to plugin's queue to ensure run() flushes queue before shutting down.
+    # Queue the event and then shutdown, otherwise the threads would run forever
+    if shutdown_count == 0
+      @shutdown_count += 1
+      @queue << "additional event stuck in queue during shutdown"
+      raise LogStash::ShutdownSignal
+    end
+  end
+
+  def start_kcl_thread()
+    @queue << "some message from kcl thread calling process"
+  end
+end
+
+class TestParser
+
+  def parse_scan(msg)
+    return msg
+  end
+
+  def parse_stream(msg)
+    return msg
+  end
+
+end
+
+describe 'inputs/dynamodb' do
+  let (:dynamodb_client) {mock("AmazonDynamoDB::AmazonDynamoDBClient")}
+  let (:dynamodb_streams_client) {mock("AmazonDynamoDB::AmazonDynamoDBStreamsClient")}
+  let (:adapter) {mock("AmazonDynamoDB::AmazonDynamoDBStreamsAdapterClient")}
+  let (:parser) {mock("DynamoDBLogParser")}
+  let (:region_utils) {mock("RegionUtils")}
+
+  def allow_invalid_credentials(stream_status = "ENABLED", error_to_raise = nil)
+    AmazonDynamoDB::AmazonDynamoDBClient.expects(:new).returns(dynamodb_client)
+    AmazonDynamoDB::AmazonDynamoDBStreamsClient.expects(:new).returns(dynamodb_streams_client)
+    AmazonDynamoDB::AmazonDynamoDBStreamsAdapterClient.expects(:new).returns(adapter)
+    Logstash::Inputs::DynamoDB::DynamoDBLogParser.expects(:new).returns(TestParser.new())
+    RegionUtils.expects(:getRegionByEndpoint).with("some endpoint").returns("some region")
+
+    mock_table_description = stub
+    mock_table = stub
+    mock_key_schema = stub
+    mock_iterator = stub
+    mock_describe_stream = stub
+    mock_stream_description = stub
+    unless error_to_raise.nil?
+      dynamodb_client.expects(:describeTable).raises(error_to_raise)
+      return
+    end
+
+    adapter.expects(:setEndpoint).with("some streams endpoint")
+    dynamodb_streams_client.expects(:setEndpoint).with("some streams endpoint")
+    dynamodb_streams_client.expects(:describeStream).returns(mock_describe_stream)
+    mock_describe_stream.expects(:getStreamDescription).returns(mock_stream_description)
+    mock_stream_description.expects(:getStreamStatus).returns(stream_status)
+    mock_stream_description.expects(:getStreamViewType).returns("new_and_old_images")
+    mock_table.expects(:getLatestStreamArn).returns("test streamId")
+    dynamodb_client.expects(:setEndpoint)
+    dynamodb_client.expects(:describeTable).returns(mock_table_description)
+    mock_table_description.expects(:getTable).returns(mock_table)
+    mock_table.expects(:getKeySchema).returns(mock_key_schema)
+    mock_key_schema.expects(:iterator).returns(mock_iterator)
+    mock_iterator.expects(:hasNext).returns(false)
+
+  end
+
+	it "should not allow empty config" do
+    expect {LogStash::Plugin.lookup("input", "dynamodb").new(empty_config)}.to raise_error(LogStash::ConfigurationError)
+  end
+
+	it "should need endpoint" do
+    config = tablename
+    config.delete("endpoint")
+    expect {LogStash::Plugin.lookup("input", "dynamodb").new(config)}.to raise_error(LogStash::ConfigurationError)
+  end
+
+	it "should need table_name config" do
+    config = tablename
+    config.delete("table_name")
+    expect {LogStash::Plugin.lookup("input", "dynamodb").new(config)}.to raise_error(LogStash::ConfigurationError)
+  end
+
+	it "should need view_type config" do
+    config = tablename
+    config.delete("view_type")
+    expect {LogStash::Plugin.lookup("input", "dynamodb").new(config)}.to raise_error(LogStash::ConfigurationError)
+  end
+
+  it "should use default AWS credentials " do
+    input = LogStash::Plugin.lookup("input", "dynamodb").new(tablename)
+    expect(input.build_credentials()).to be_an_instance_of(Java::ComAmazonawsAuth::DefaultAWSCredentialsProviderChain)
+  end
+
+  it "should register correctly" do
+    input = LogStash::Plugin.lookup("input", "dynamodb").new(invalid_aws_credentials_config)
+    allow_invalid_credentials()
+    expect {input.register}.not_to raise_error
+  end
+
+  it "should create new logstash event with metadata and add to queue" do
+    input = LogStash::Plugin.lookup("input", "dynamodb").new(invalid_aws_credentials_config)
+    queue = SizedQueue.new(20)
+    input.queue_event("some message", queue, "some host")
+    expect(queue.size()).to eq(1)
+    event = queue.pop()
+    expect(event["message"]).to eq("some message")
+    expect(event["host"]).to eq("some host")
+  end
+
+  it "should start mock kcl worker thread and receive event from it, then flush additional events stuck in queue before shutting down" do
+    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({'perform_scan' => false}))
+    allow_invalid_credentials()
+    input.register
+    queue = SizedQueue.new(20)
+    input.run queue
+    expect(queue.size()).to eq(2)
+    event = queue.pop()
+    expect(event["message"]).to eq("some message from kcl thread calling process")
+    event = queue.pop()
+    expect(event["message"]).to eq("additional event stuck in queue during shutdown")
+  end
+
+  it "should raise error since no active streams" do
+    input = LogStash::Plugin.lookup("input", "dynamodb").new(invalid_aws_credentials_config)
+    allow_invalid_credentials(stream_status="DISABLED")
+    expect {input.register}.to raise_error(LogStash::PluginLoadingError, "No streams are enabled")
+  end
+
+  it "should handle error for nonexistent table" do
+    input = LogStash::Plugin.lookup("input", "dynamodb").new(invalid_aws_credentials_config)
+    allow_invalid_credentials(error_to_raise=Java::ComAmazonawsServicesDynamodbv2Model::ResourceNotFoundException.new("table does not exist"))
+    expect {input.register}.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should allow cloudwatch metrics when specified by user" do
+    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"publish_metrics" => true}))
+    allow_invalid_credentials()
+    cloudwatch_mock = mock("Java::ComAmazonawsServicesCloudwatch::AmazonCloudWatchClient")
+    Java::ComAmazonawsServicesCloudwatch::AmazonCloudWatchClient.expects(:new).returns(cloudwatch_mock)
+    
+    input.register
+  end
+
+  it "should throw error trying to perform scan with old images" do
+    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"view_type" => LogStash::Inputs::DynamoDB::VT_OLD_IMAGE, \
+      "perform_scan" => true}))
+    expect {input.register}.to raise_error(LogStash::ConfigurationError)
+  end
+
+  it "should throw error when view type all images and dynamodb format" do
+    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"view_type" => LogStash::Inputs::DynamoDB::VT_ALL_IMAGES, \
+      "log_format" => LogStash::Inputs::DynamoDB::LF_DYNAMODB}))
+    expect {input.register}.to raise_error(LogStash::ConfigurationError)
+  end
+
+  it "should throw error when view type all images and json_drop_binary format" do
+    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"view_type" => LogStash::Inputs::DynamoDB::VT_ALL_IMAGES, \
+      "log_format" => LogStash::Inputs::DynamoDB::LF_JSON_NO_BIN}))
+    expect {input.register}.to raise_error(LogStash::ConfigurationError)
+  end
+
+  it "should throw error when view type all images and json_binary_as_text format" do
+    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"view_type" => LogStash::Inputs::DynamoDB::VT_ALL_IMAGES, \
+      "log_format" => LogStash::Inputs::DynamoDB::LF_JSON_BIN_AS_TEXT}))
+    expect {input.register}.to raise_error(LogStash::ConfigurationError)
+  end
+
+
+end

data/spec/log_parser_spec.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+#
+#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#
+require "spec/spec_helper"
+
+class Logstash::Inputs::DynamoDB::DynamoDBLogParserTest < Logstash::Inputs::DynamoDB::DynamoDBLogParser
+
+  private
+  def calculate_key_size_in_bytes(record)
+    return 10
+  end
+
+end
+
+describe "inputs/LogParser" do
+  let (:object_mapper) {mock("ObjectMapper")}
+
+  before(:each) do
+    Java::comFasterxmlJacksonDatabind::ObjectMapper.expects(:new).returns(object_mapper)
+    object_mapper.expects(:setSerializationInclusion)
+    object_mapper.expects(:addMixInAnnotations)
+  end
+
+  def expect_parse_stream()
+    object_mapper.expects(:writeValueAsString).with(sample_stream_result).returns(sample_stream_result)
+    JSON.expects(:parse).with(sample_stream_result).returns(sample_stream_result)
+  end
+
+  def expect_parse_scan()
+    object_mapper.expects(:writeValueAsString).with(sample_scan_result).returns(sample_scan_result)
+    JSON.expects(:parse).with(sample_scan_result).returns(sample_scan_result)
+  end
+
+  it "should parse a scan and parse a stream the same way" do
+    expect_parse_stream
+    expect_parse_scan
+    parser = Logstash::Inputs::DynamoDB::DynamoDBLogParserTest.new(LogStash::Inputs::DynamoDB::VT_ALL_IMAGES, LogStash::Inputs::DynamoDB::LF_PLAIN, key_schema, "us-west-1")
+    scan_after_parse = parser.parse_scan(sample_scan_result, 38)
+    stream_after_parse = parser.parse_stream(sample_stream_result)
+    expect(scan_after_parse).to eq(stream_after_parse)
+  end
+
+  it "should drop binary values when parsing into a json with the correct configuration" do
+    expect_parse_scan
+    parser = Logstash::Inputs::DynamoDB::DynamoDBLogParserTest.new(LogStash::Inputs::DynamoDB::VT_NEW_IMAGE, LogStash::Inputs::DynamoDB::LF_JSON_NO_BIN, key_schema, "us-west-1")
+    result = parser.parse_scan(sample_scan_result, 38)
+    expect(result).to eq({"TBCZDPHPXUTOTYGP" => "sampleString"}.to_json)
+  end
+
+end

data/spec/record_processor_and_factory_spec.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+#
+#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#
+require "spec/spec_helper"
+
+describe 'inputs/LogStashRecordProcessor' do
+  before(:each) do
+    @queue = SizedQueue.new(20)
+    @processor = Logstash::Inputs::DynamoDB::LogStashRecordProcessor.new(@queue)
+  end
+
+  it "should call setShardId when being called with a String" do
+    processor_with_shard = Logstash::Inputs::DynamoDB::LogStashRecordProcessor.new("test shardId")
+    expect(processor_with_shard.shard_id).to eq("test shardId")
+  end
+
+  it "should not call setShardId when being called with a queue" do
+    expect(@processor.queue).to eq(@queue)
+    expect(@processor.shard_id).to be_nil
+  end
+
+  it "should checkpoint when shutdown is called with reason TERMINATE" do
+    checkpointer = mock("checkpointer")
+    checkpointer.expects(:checkpoint).once
+    @processor.shutdown(checkpointer, ShutdownReason::TERMINATE)
+  end
+
+  it "should not checkpoint when shutdown is called with reason ZOMBIE" do
+    checkpointer = mock("checkpointer")
+    checkpointer.expects(:checkpoint).never
+    @processor.shutdown(checkpointer, ShutdownReason::ZOMBIE)
+  end
+
+  it "should raise error when shutdown is called with unknown reason" do
+    expect {@processor.shutdown("some checkpointer", "unknown reason")}.to raise_error(RuntimeError)
+  end
+
+  it "should translate each record into String, push them onto queue, and then checkpoint when process_records is called" do
+    checkpointer = mock("checkpointer")
+    checkpointer.expects(:checkpoint).once
+
+    records = [{"a records data" => "a records value"}, {"another records data" => "another records value"}]
+    @processor.process_records(records, checkpointer)
+  end
+
+end
+
+describe 'inputs/LogStashRecordProcessorFactory' do
+
+  it "should create a new factory correctly and create a new LogStashRecordProcessor when called upon" do
+    queue = SizedQueue.new(20)
+    factory = Logstash::Inputs::DynamoDB::LogStashRecordProcessorFactory.new(queue)
+    processor = factory.create_processor
+    expect(processor).to be_an_instance_of(Logstash::Inputs::DynamoDB::LogStashRecordProcessor)
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,134 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/DynamoDBLogParser"
+require "logstash/inputs/dynamodb"
+require "rspec/expectations"
+require "rspec/mocks"
+require "mocha"
+require "java"
+
+RSpec.configure do |config|
+  config.mock_with :mocha
+end
+
+def empty_config
+  {}
+end
+
+def tablename
+  {'table_name' => 'test tablename', 'view_type' => "new_and_old_images", "endpoint" => "some endpoint"}
+end
+def invalid_aws_credentials_config
+  {'table_name' => 'test tablename', "endpoint" => "some endpoint", 'aws_access_key_id' => 'invalid', 'aws_secret_access_key' => 'invalid_also', 'view_type' => "new_and_old_images", "streams_endpoint" => "some streams endpoint"}
+end
+def invalid_aws_credentials_config_no_endpoints
+  {'table_name' => 'test tablename', 'aws_access_key_id' => 'invalid', 'aws_secret_access_key' => 'invalid_also', 'view_type' => "new_and_old_images"}
+end
+def key_schema
+  ["TBCZDPHPXUTOTYGP", "some bin key"]
+end
+def sample_scan_result
+  {"TBCZDPHPXUTOTYGP" => {"S" => "sampleString"}, "some bin key" => {"B" => "actualbinval"}}
+end
+def sample_stream_result
+  {"internalObject" => {"eventID" => "0","eventName" => "INSERT","eventVersion" => "1.0", \
+    "eventSource" => "aws:dynamodb","awsRegion" => "us-west-1","dynamodb" => {"keys" => {"TBCZDPHPXUTOTYGP" => {"S" => "sampleString"}, \
+    "some bin key" => {"B" => "actualbinval"}}, "newImage" => {"TBCZDPHPXUTOTYGP" => {"S" => "sampleString"}, \
+    "some bin key" => {"B" => "actualbinval"}},"sequenceNumber" => "0","sizeBytes" => 48,"streamViewType" => LogStash::Inputs::DynamoDB::VT_ALL_IMAGES.upcase}}}
+end