logstash-input-dynamodb 1.0.4-java → 2.0.0-java

data/logstash-input-dynamodb.gemspec

@@ -1,40 +0,0 @@
-Gem::Specification.new do |s|
-  s.name = 'logstash-input-dynamodb'
-  s.version = '1.0.4'
-  s.licenses = ['Apache License (2.0)']
-  s.summary = "This input plugin scans a specified DynamoDB table and then reads changes to a DynamoDB table from the associated DynamoDB Stream."
-  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
-  s.authors = ["Amazon"]
-  s.email = 'dynamodb-interest@amazon.com'
-  s.homepage = "https://github.com/logstash-plugins/logstash-input-dynamodb"
-  s.require_paths = ["lib"]
-  s.platform = 'java'
-
-  # Files
-  s.files = `git ls-files`.split($\)
-   # Tests
-  s.test_files = s.files.grep(%r{^(test|spec|features)/})
-
-  # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
-
-  # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '< 2.0.0'
-  s.add_runtime_dependency "logstash-codec-json", '< 2.0.0'
-  s.add_runtime_dependency "activesupport-json_encoder"
-  # Jar dependencies
-  s.requirements << "jar 'org.apache.httpcomponents:httpclient', '4.5.2'"
-  s.requirements << "jar 'com.amazonaws:amazon-kinesis-client', '1.6.1'"
-  s.requirements << "jar 'log4j:log4j', '1.2.17'"
-  s.requirements << "jar 'com.amazonaws:aws-java-sdk-dynamodb', '1.10.60'"
-  s.requirements << "jar 'com.amazonaws:aws-java-sdk-core', '1.10.60'"
-  s.requirements << "jar 'com.amazonaws:aws-java-sdk-kinesis', '1.10.60'"
-  s.requirements << "jar 'com.amazonaws:dynamodb-import-export-tool', '1.0.0'"
-  s.requirements << "jar 'commons-logging:commons-logging', '1.1.3'"
-  s.requirements << "jar 'com.amazonaws:dynamodb-streams-kinesis-adapter', '1.0.2'"
-  s.requirements << "jar 'com.google.guava:guava', '15.0'"
-  s.add_runtime_dependency 'jar-dependencies'
-  # Development dependencies
-  s.add_development_dependency "logstash-devutils"
-  s.add_development_dependency "mocha"
-end

data/spec/inputs/dynamodb_spec.rb

@@ -1,198 +0,0 @@
-# encoding: utf-8
-#
-#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-#
-require "spec/spec_helper"
-
-class LogStash::Inputs::TestDynamoDB < LogStash::Inputs::DynamoDB
-  default :codec, 'json'
-
-  private
-  def shutdown_count
-    @shutdown_count ||= 0
-  end
-
-  def queue_event(event, logstash_queue, host)
-    super(event, logstash_queue, host)
-    # Add additional item to plugin's queue to ensure run() flushes queue before shutting down.
-    # Queue the event and then shutdown, otherwise the threads would run forever
-    if shutdown_count == 0
-      @shutdown_count += 1
-      @queue << "additional event stuck in queue during shutdown"
-      raise LogStash::ShutdownSignal
-    end
-  end
-
-  def start_kcl_thread()
-    @queue << "some message from kcl thread calling process"
-  end
-end
-
-class TestParser
-
-  def parse_scan(msg)
-    return msg
-  end
-
-  def parse_stream(msg)
-    return msg
-  end
-
-end
-
-describe 'inputs/dynamodb' do
-  let (:dynamodb_client) {mock("AmazonDynamoDB::AmazonDynamoDBClient")}
-  let (:dynamodb_streams_client) {mock("AmazonDynamoDB::AmazonDynamoDBStreamsClient")}
-  let (:adapter) {mock("AmazonDynamoDB::AmazonDynamoDBStreamsAdapterClient")}
-  let (:parser) {mock("DynamoDBLogParser")}
-  let (:region_utils) {mock("RegionUtils")}
-
-  def allow_invalid_credentials(stream_status = "ENABLED", error_to_raise = nil)
-    AmazonDynamoDB::AmazonDynamoDBClient.expects(:new).returns(dynamodb_client)
-    AmazonDynamoDB::AmazonDynamoDBStreamsClient.expects(:new).returns(dynamodb_streams_client)
-    AmazonDynamoDB::AmazonDynamoDBStreamsAdapterClient.expects(:new).returns(adapter)
-    Logstash::Inputs::DynamoDB::DynamoDBLogParser.expects(:new).returns(TestParser.new())
-    RegionUtils.expects(:getRegionByEndpoint).with("some endpoint").returns("some region")
-
-    mock_table_description = stub
-    mock_table = stub
-    mock_key_schema = stub
-    mock_iterator = stub
-    mock_describe_stream = stub
-    mock_stream_description = stub
-    unless error_to_raise.nil?
-      dynamodb_client.expects(:describeTable).raises(error_to_raise)
-      return
-    end
-
-    adapter.expects(:setEndpoint).with("some streams endpoint")
-    dynamodb_streams_client.expects(:setEndpoint).with("some streams endpoint")
-    dynamodb_streams_client.expects(:describeStream).returns(mock_describe_stream)
-    mock_describe_stream.expects(:getStreamDescription).returns(mock_stream_description)
-    mock_stream_description.expects(:getStreamStatus).returns(stream_status)
-    mock_stream_description.expects(:getStreamViewType).returns("new_and_old_images")
-    mock_table.expects(:getLatestStreamArn).returns("test streamId")
-    dynamodb_client.expects(:setEndpoint)
-    dynamodb_client.expects(:describeTable).returns(mock_table_description)
-    mock_table_description.expects(:getTable).returns(mock_table)
-    mock_table.expects(:getKeySchema).returns(mock_key_schema)
-    mock_key_schema.expects(:iterator).returns(mock_iterator)
-    mock_iterator.expects(:hasNext).returns(false)
-
-  end
-
-	it "should not allow empty config" do
-    expect {LogStash::Plugin.lookup("input", "dynamodb").new(empty_config)}.to raise_error(LogStash::ConfigurationError)
-  end
-
-	it "should need endpoint" do
-    config = tablename
-    config.delete("endpoint")
-    expect {LogStash::Plugin.lookup("input", "dynamodb").new(config)}.to raise_error(LogStash::ConfigurationError)
-  end
-
-	it "should need table_name config" do
-    config = tablename
-    config.delete("table_name")
-    expect {LogStash::Plugin.lookup("input", "dynamodb").new(config)}.to raise_error(LogStash::ConfigurationError)
-  end
-
-	it "should need view_type config" do
-    config = tablename
-    config.delete("view_type")
-    expect {LogStash::Plugin.lookup("input", "dynamodb").new(config)}.to raise_error(LogStash::ConfigurationError)
-  end
-
-  it "should use default AWS credentials " do
-    input = LogStash::Plugin.lookup("input", "dynamodb").new(tablename)
-    expect(input.build_credentials()).to be_an_instance_of(Java::ComAmazonawsAuth::DefaultAWSCredentialsProviderChain)
-  end
-
-  it "should register correctly" do
-    input = LogStash::Plugin.lookup("input", "dynamodb").new(invalid_aws_credentials_config)
-    allow_invalid_credentials()
-    expect {input.register}.not_to raise_error
-  end
-
-  it "should create new logstash event with metadata and add to queue" do
-    input = LogStash::Plugin.lookup("input", "dynamodb").new(invalid_aws_credentials_config)
-    queue = SizedQueue.new(20)
-    input.queue_event("some message", queue, "some host")
-    expect(queue.size()).to eq(1)
-    event = queue.pop()
-    expect(event["message"]).to eq("some message")
-    expect(event["host"]).to eq("some host")
-  end
-
-  it "should start mock kcl worker thread and receive event from it, then flush additional events stuck in queue before shutting down" do
-    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({'perform_scan' => false}))
-    allow_invalid_credentials()
-    input.register
-    queue = SizedQueue.new(20)
-    input.run queue
-    expect(queue.size()).to eq(2)
-    event = queue.pop()
-    expect(event["message"]).to eq("some message from kcl thread calling process")
-    event = queue.pop()
-    expect(event["message"]).to eq("additional event stuck in queue during shutdown")
-  end
-
-  it "should raise error since no active streams" do
-    input = LogStash::Plugin.lookup("input", "dynamodb").new(invalid_aws_credentials_config)
-    allow_invalid_credentials(stream_status="DISABLED")
-    expect {input.register}.to raise_error(LogStash::PluginLoadingError, "No streams are enabled")
-  end
-
-  it "should handle error for nonexistent table" do
-    input = LogStash::Plugin.lookup("input", "dynamodb").new(invalid_aws_credentials_config)
-    allow_invalid_credentials(error_to_raise=Java::ComAmazonawsServicesDynamodbv2Model::ResourceNotFoundException.new("table does not exist"))
-    expect {input.register}.to raise_error(LogStash::PluginLoadingError)
-  end
-
-  it "should allow cloudwatch metrics when specified by user" do
-    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"publish_metrics" => true}))
-    allow_invalid_credentials()
-    cloudwatch_mock = mock("Java::ComAmazonawsServicesCloudwatch::AmazonCloudWatchClient")
-    Java::ComAmazonawsServicesCloudwatch::AmazonCloudWatchClient.expects(:new).returns(cloudwatch_mock)
-    
-    input.register
-  end
-
-  it "should throw error trying to perform scan with old images" do
-    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"view_type" => LogStash::Inputs::DynamoDB::VT_OLD_IMAGE, \
-      "perform_scan" => true}))
-    expect {input.register}.to raise_error(LogStash::ConfigurationError)
-  end
-
-  it "should throw error when view type all images and dynamodb format" do
-    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"view_type" => LogStash::Inputs::DynamoDB::VT_ALL_IMAGES, \
-      "log_format" => LogStash::Inputs::DynamoDB::LF_DYNAMODB}))
-    expect {input.register}.to raise_error(LogStash::ConfigurationError)
-  end
-
-  it "should throw error when view type all images and json_drop_binary format" do
-    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"view_type" => LogStash::Inputs::DynamoDB::VT_ALL_IMAGES, \
-      "log_format" => LogStash::Inputs::DynamoDB::LF_JSON_NO_BIN}))
-    expect {input.register}.to raise_error(LogStash::ConfigurationError)
-  end
-
-  it "should throw error when view type all images and json_binary_as_text format" do
-    input = LogStash::Inputs::TestDynamoDB.new(invalid_aws_credentials_config.merge({"view_type" => LogStash::Inputs::DynamoDB::VT_ALL_IMAGES, \
-      "log_format" => LogStash::Inputs::DynamoDB::LF_JSON_BIN_AS_TEXT}))
-    expect {input.register}.to raise_error(LogStash::ConfigurationError)
-  end
-
-
-end

data/spec/log_parser_spec.rb

@@ -1,63 +0,0 @@
-# encoding: utf-8
-#
-#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-#
-require "spec/spec_helper"
-
-class Logstash::Inputs::DynamoDB::DynamoDBLogParserTest < Logstash::Inputs::DynamoDB::DynamoDBLogParser
-
-  private
-  def calculate_key_size_in_bytes(record)
-    return 10
-  end
-
-end
-
-describe "inputs/LogParser" do
-  let (:object_mapper) {mock("ObjectMapper")}
-
-  before(:each) do
-    Java::comFasterxmlJacksonDatabind::ObjectMapper.expects(:new).returns(object_mapper)
-    object_mapper.expects(:setSerializationInclusion)
-    object_mapper.expects(:addMixInAnnotations)
-  end
-
-  def expect_parse_stream()
-    object_mapper.expects(:writeValueAsString).with(sample_stream_result).returns(sample_stream_result)
-    JSON.expects(:parse).with(sample_stream_result).returns(sample_stream_result)
-  end
-
-  def expect_parse_scan()
-    object_mapper.expects(:writeValueAsString).with(sample_scan_result).returns(sample_scan_result)
-    JSON.expects(:parse).with(sample_scan_result).returns(sample_scan_result)
-  end
-
-  it "should parse a scan and parse a stream the same way" do
-    expect_parse_stream
-    expect_parse_scan
-    parser = Logstash::Inputs::DynamoDB::DynamoDBLogParserTest.new(LogStash::Inputs::DynamoDB::VT_ALL_IMAGES, LogStash::Inputs::DynamoDB::LF_PLAIN, key_schema, "us-west-1")
-    scan_after_parse = parser.parse_scan(sample_scan_result, 38)
-    stream_after_parse = parser.parse_stream(sample_stream_result)
-    expect(scan_after_parse).to eq(stream_after_parse)
-  end
-
-  it "should drop binary values when parsing into a json with the correct configuration" do
-    expect_parse_scan
-    parser = Logstash::Inputs::DynamoDB::DynamoDBLogParserTest.new(LogStash::Inputs::DynamoDB::VT_NEW_IMAGE, LogStash::Inputs::DynamoDB::LF_JSON_NO_BIN, key_schema, "us-west-1")
-    result = parser.parse_scan(sample_scan_result, 38)
-    expect(result).to eq({"TBCZDPHPXUTOTYGP" => "sampleString"}.to_json)
-  end
-
-end

data/spec/record_processor_and_factory_spec.rb

@@ -1,70 +0,0 @@
-# encoding: utf-8
-#
-#Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-#
-require "spec/spec_helper"
-
-describe 'inputs/LogStashRecordProcessor' do
-  before(:each) do
-    @queue = SizedQueue.new(20)
-    @processor = Logstash::Inputs::DynamoDB::LogStashRecordProcessor.new(@queue)
-  end
-
-  it "should call setShardId when being called with a String" do
-    processor_with_shard = Logstash::Inputs::DynamoDB::LogStashRecordProcessor.new("test shardId")
-    expect(processor_with_shard.shard_id).to eq("test shardId")
-  end
-
-  it "should not call setShardId when being called with a queue" do
-    expect(@processor.queue).to eq(@queue)
-    expect(@processor.shard_id).to be_nil
-  end
-
-  it "should checkpoint when shutdown is called with reason TERMINATE" do
-    checkpointer = mock("checkpointer")
-    checkpointer.expects(:checkpoint).once
-    @processor.shutdown(checkpointer, ShutdownReason::TERMINATE)
-  end
-
-  it "should not checkpoint when shutdown is called with reason ZOMBIE" do
-    checkpointer = mock("checkpointer")
-    checkpointer.expects(:checkpoint).never
-    @processor.shutdown(checkpointer, ShutdownReason::ZOMBIE)
-  end
-
-  it "should raise error when shutdown is called with unknown reason" do
-    expect {@processor.shutdown("some checkpointer", "unknown reason")}.to raise_error(RuntimeError)
-  end
-
-  it "should translate each record into String, push them onto queue, and then checkpoint when process_records is called" do
-    checkpointer = mock("checkpointer")
-    checkpointer.expects(:checkpoint).once
-
-    records = [{"a records data" => "a records value"}, {"another records data" => "another records value"}]
-    @processor.process_records(records, checkpointer)
-  end
-
-end
-
-describe 'inputs/LogStashRecordProcessorFactory' do
-
-  it "should create a new factory correctly and create a new LogStashRecordProcessor when called upon" do
-    queue = SizedQueue.new(20)
-    factory = Logstash::Inputs::DynamoDB::LogStashRecordProcessorFactory.new(queue)
-    processor = factory.create_processor
-    expect(processor).to be_an_instance_of(Logstash::Inputs::DynamoDB::LogStashRecordProcessor)
-  end
-
-end

data/spec/spec_helper.rb

@@ -1,134 +0,0 @@
-# This file was generated by the `rspec --init` command. Conventionally, all
-# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
-# The generated `.rspec` file contains `--require spec_helper` which will cause
-# this file to always be loaded, without a need to explicitly require it in any
-# files.
-#
-# Given that it is always loaded, you are encouraged to keep this file as
-# light-weight as possible. Requiring heavyweight dependencies from this file
-# will add to the boot time of your test suite on EVERY test run, even for an
-# individual file that may not need all of that loaded. Instead, consider making
-# a separate helper file that requires the additional dependencies and performs
-# the additional setup, and require it from the spec files that actually need
-# it.
-#
-# The `.rspec` file also contains a few flags that are not defaults but that
-# users commonly want.
-#
-# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
-RSpec.configure do |config|
-  # rspec-expectations config goes here. You can use an alternate
-  # assertion/expectation library such as wrong or the stdlib/minitest
-  # assertions if you prefer.
-  config.expect_with :rspec do |expectations|
-    # This option will default to `true` in RSpec 4. It makes the `description`
-    # and `failure_message` of custom matchers include text for helper methods
-    # defined using `chain`, e.g.:
-    #     be_bigger_than(2).and_smaller_than(4).description
-    #     # => "be bigger than 2 and smaller than 4"
-    # ...rather than:
-    #     # => "be bigger than 2"
-    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
-  end
-
-  # rspec-mocks config goes here. You can use an alternate test double
-  # library (such as bogus or mocha) by changing the `mock_with` option here.
-  config.mock_with :rspec do |mocks|
-    # Prevents you from mocking or stubbing a method that does not exist on
-    # a real object. This is generally recommended, and will default to
-    # `true` in RSpec 4.
-    mocks.verify_partial_doubles = true
-  end
-
-# The settings below are suggested to provide a good initial experience
-# with RSpec, but feel free to customize to your heart's content.
-=begin
-  # These two settings work together to allow you to limit a spec run
-  # to individual examples or groups you care about by tagging them with
-  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
-  # get run.
-  config.filter_run :focus
-  config.run_all_when_everything_filtered = true
-
-  # Allows RSpec to persist some state between runs in order to support
-  # the `--only-failures` and `--next-failure` CLI options. We recommend
-  # you configure your source control system to ignore this file.
-  config.example_status_persistence_file_path = "spec/examples.txt"
-
-  # Limits the available syntax to the non-monkey patched syntax that is
-  # recommended. For more details, see:
-  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
-  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
-  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
-  config.disable_monkey_patching!
-
-  # This setting enables warnings. It's recommended, but in some cases may
-  # be too noisy due to issues in dependencies.
-  config.warnings = true
-
-  # Many RSpec users commonly either run the entire suite or an individual
-  # file, and it's useful to allow more verbose output when running an
-  # individual spec file.
-  if config.files_to_run.one?
-    # Use the documentation formatter for detailed output,
-    # unless a formatter has already been configured
-    # (e.g. via a command-line flag).
-    config.default_formatter = 'doc'
-  end
-
-  # Print the 10 slowest examples and example groups at the
-  # end of the spec run, to help surface which specs are running
-  # particularly slow.
-  config.profile_examples = 10
-
-  # Run specs in random order to surface order dependencies. If you find an
-  # order dependency and want to debug it, you can fix the order by providing
-  # the seed, which is printed after each run.
-  #     --seed 1234
-  config.order = :random
-
-  # Seed global randomization in this process using the `--seed` CLI option.
-  # Setting this allows you to use `--seed` to deterministically reproduce
-  # test failures related to randomization by passing the same `--seed` value
-  # as the one that triggered the failure.
-  Kernel.srand config.seed
-=end
-end
-
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/inputs/DynamoDBLogParser"
-require "logstash/inputs/dynamodb"
-require "rspec/expectations"
-require "rspec/mocks"
-require "mocha"
-require "java"
-
-RSpec.configure do |config|
-  config.mock_with :mocha
-end
-
-def empty_config
-  {}
-end
-
-def tablename
-  {'table_name' => 'test tablename', 'view_type' => "new_and_old_images", "endpoint" => "some endpoint"}
-end
-def invalid_aws_credentials_config
-  {'table_name' => 'test tablename', "endpoint" => "some endpoint", 'aws_access_key_id' => 'invalid', 'aws_secret_access_key' => 'invalid_also', 'view_type' => "new_and_old_images", "streams_endpoint" => "some streams endpoint"}
-end
-def invalid_aws_credentials_config_no_endpoints
-  {'table_name' => 'test tablename', 'aws_access_key_id' => 'invalid', 'aws_secret_access_key' => 'invalid_also', 'view_type' => "new_and_old_images"}
-end
-def key_schema
-  ["TBCZDPHPXUTOTYGP", "some bin key"]
-end
-def sample_scan_result
-  {"TBCZDPHPXUTOTYGP" => {"S" => "sampleString"}, "some bin key" => {"B" => "actualbinval"}}
-end
-def sample_stream_result
-  {"internalObject" => {"eventID" => "0","eventName" => "INSERT","eventVersion" => "1.0", \
-    "eventSource" => "aws:dynamodb","awsRegion" => "us-west-1","dynamodb" => {"keys" => {"TBCZDPHPXUTOTYGP" => {"S" => "sampleString"}, \
-    "some bin key" => {"B" => "actualbinval"}}, "newImage" => {"TBCZDPHPXUTOTYGP" => {"S" => "sampleString"}, \
-    "some bin key" => {"B" => "actualbinval"}},"sequenceNumber" => "0","sizeBytes" => 48,"streamViewType" => LogStash::Inputs::DynamoDB::VT_ALL_IMAGES.upcase}}}
-end