logstash-input-kinesis 1.3.1-java → 1.4.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +36 -3
  3. data/lib/logstash/inputs/kinesis.rb +19 -15
  4. data/lib/logstash/inputs/kinesis/version.rb +1 -1
  5. data/lib/logstash/inputs/kinesis/worker.rb +8 -8
  6. data/pom.xml +1 -1
  7. data/spec/inputs/kinesis/worker_spec.rb +24 -9
  8. data/spec/inputs/kinesis_spec.rb +41 -25
  9. data/vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.6.1.jar +0 -0
  10. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.10.20.jar +0 -0
  11. data/vendor/jar-dependencies/runtime-jars/{aws-java-sdk-core-1.9.16.jar → aws-java-sdk-core-1.10.20.jar} +0 -0
  12. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.10.20.jar +0 -0
  13. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.10.20.jar +0 -0
  14. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.10.20.jar +0 -0
  15. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.10.20.jar +0 -0
  16. data/vendor/jar-dependencies/runtime-jars/commons-lang-2.6.jar +0 -0
  17. data/vendor/jar-dependencies/runtime-jars/guava-18.0.jar +0 -0
  18. data/vendor/jar-dependencies/runtime-jars/{httpclient-4.3.4.jar → httpclient-4.3.6.jar} +0 -0
  19. data/vendor/jar-dependencies/runtime-jars/{httpcore-4.3.2.jar → httpcore-4.3.3.jar} +0 -0
  20. data/vendor/jar-dependencies/runtime-jars/jackson-annotations-2.5.0.jar +0 -0
  21. data/vendor/jar-dependencies/runtime-jars/jackson-core-2.5.3.jar +0 -0
  22. data/vendor/jar-dependencies/runtime-jars/jackson-databind-2.5.3.jar +0 -0
  23. data/vendor/jar-dependencies/runtime-jars/protobuf-java-2.6.1.jar +0 -0
  24. metadata +17 -53
  25. data/vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.2.1.jar +0 -0
  26. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-1.9.16.jar +0 -0
  27. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-autoscaling-1.9.16.jar +0 -0
  28. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudformation-1.9.16.jar +0 -0
  29. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudfront-1.9.16.jar +0 -0
  30. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudhsm-1.9.16.jar +0 -0
  31. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudsearch-1.9.16.jar +0 -0
  32. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudtrail-1.9.16.jar +0 -0
  33. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.9.16.jar +0 -0
  34. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatchmetrics-1.9.16.jar +0 -0
  35. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-codedeploy-1.9.16.jar +0 -0
  36. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitoidentity-1.9.16.jar +0 -0
  37. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitosync-1.9.16.jar +0 -0
  38. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-config-1.9.16.jar +0 -0
  39. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-datapipeline-1.9.16.jar +0 -0
  40. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-directconnect-1.9.16.jar +0 -0
  41. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.9.16.jar +0 -0
  42. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-ec2-1.9.16.jar +0 -0
  43. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-ecs-1.9.16.jar +0 -0
  44. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticache-1.9.16.jar +0 -0
  45. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticbeanstalk-1.9.16.jar +0 -0
  46. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticloadbalancing-1.9.16.jar +0 -0
  47. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-elastictranscoder-1.9.16.jar +0 -0
  48. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-emr-1.9.16.jar +0 -0
  49. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-glacier-1.9.16.jar +0 -0
  50. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-iam-1.9.16.jar +0 -0
  51. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-importexport-1.9.16.jar +0 -0
  52. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.9.16.jar +0 -0
  53. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.9.16.jar +0 -0
  54. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-lambda-1.9.16.jar +0 -0
  55. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-logs-1.9.16.jar +0 -0
  56. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-opsworks-1.9.16.jar +0 -0
  57. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-rds-1.9.16.jar +0 -0
  58. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-redshift-1.9.16.jar +0 -0
  59. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-route53-1.9.16.jar +0 -0
  60. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.9.16.jar +0 -0
  61. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-ses-1.9.16.jar +0 -0
  62. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpledb-1.9.16.jar +0 -0
  63. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpleworkflow-1.9.16.jar +0 -0
  64. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-sns-1.9.16.jar +0 -0
  65. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-sqs-1.9.16.jar +0 -0
  66. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-storagegateway-1.9.16.jar +0 -0
  67. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-sts-1.9.16.jar +0 -0
  68. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-support-1.9.16.jar +0 -0
  69. data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-swf-libraries-1.9.16.jar +0 -0
  70. data/vendor/jar-dependencies/runtime-jars/jackson-annotations-2.3.0.jar +0 -0
  71. data/vendor/jar-dependencies/runtime-jars/jackson-core-2.3.2.jar +0 -0
  72. data/vendor/jar-dependencies/runtime-jars/jackson-databind-2.3.2.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 83fd102abe45c289fa1a52e9bff99ce35d6fc07b
4
- data.tar.gz: ab03d504c9c2e4448425fdad4f41218214960a63
3
+ metadata.gz: e0660cef254c9970b671583df6a7c95489ae888f
4
+ data.tar.gz: d1a24cc652091346a8f0102ac532bf462f00b310
5
5
  SHA512:
6
- metadata.gz: 2269a2acf2d07448b222a9020044b949166286a68b0683b72abecabfb780d6d37e71e50d3e87de74cc17cb2da33782af5dfee5a8e4f35f7fc844a8d8434c402c
7
- data.tar.gz: 22470cf8c07ef38db0b2036853732ca527212c5abe03e2613021a497e1f320be534272ddafcb8b3c9c740005bf42a18b3edd22c63b7bcb026205865ee392ea4d
6
+ metadata.gz: 945f30cbd64354c94addfbeb96be414c8a9d1fb0bb96efa70fac0b403242ddcd04c44864d396d7859a35395ca92861e3e7453590156acdb20688a7c03202288b
7
+ data.tar.gz: e3ccec93560deb6bd871517cc6238c663e49ac9e63fb519757c0c1c70389f3a3732944c858c8a0f9483af87086fc170372911e39f3377c2fbad30ab22afab5d5
data/README.md CHANGED
@@ -1,9 +1,9 @@
1
1
  # Logstash AWS Kinesis Input Plugin
2
2
 
3
- This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
4
-
5
3
  [![Build Status](https://travis-ci.org/codekitchen/logstash-input-kinesis.svg)](https://travis-ci.org/codekitchen/logstash-input-kinesis)
6
4
 
5
+ This is a [AWS Kinesis](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html) input plugin for [Logstash](https://github.com/elasticsearch/logstash). Under the hood uses the [Kinesis Client Library](http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-implementation-app-java.html).
6
+
7
7
  ## Installation
8
8
 
9
9
  This plugin requires Logstash 1.5, and can be installed by Logstash
@@ -24,7 +24,40 @@ input {
24
24
  }
25
25
  ```
26
26
 
27
- At least `kinesis_stream_name` is requred.
27
+ ## Configuration
28
+
29
+ This are the properties you can configure and what are the default values:
30
+
31
+ * `application_name`: The name of the application used in DynamoDB for coordination. Only one worker per unique stream partition and application will be actively consuming messages.
32
+ * **required**: false
33
+ * **default value**: `logstash`
34
+ * `kinesis_stream_name`: The Kinesis stream name.
35
+ * **required**: true
36
+ * `region`: The AWS region name for Kinesis, DynamoDB and Cloudwatch (if enabled)
37
+ * **required**: false
38
+ * **default value**: `us-east-1`
39
+ * `checkpoint_interval_seconds`: How many seconds between worker checkpoints to DynamoDB. A low value ussually means lower message replay in case of node failure/restart but it increases CPU+network ussage (which increases the AWS costs).
40
+ * **required**: false
41
+ * **default value**: `60`
42
+ * `metrics`: Worker metric tracking. By default this is disabled, set it to "cloudwatch" to enable the cloudwatch integration in the Kinesis Client Library.
43
+ * **required**: false
44
+ * **default value**: `nil`
45
+
46
+ ## Authentication
47
+
48
+ This plugin uses the default AWS SDK auth chain, [DefaultAWSCredentialsProviderChain](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html), to determine which credentials the client will use.
49
+
50
+ The default chain follows this order trying to read the credentials:
51
+ * `AWS_ACCESS_KEY_ID` / `AWS_SECRET_KEY` environment variables
52
+ * `~/.aws/credentials` credentials file
53
+ * EC2 instance profile
54
+
55
+ The credentials will need access to the following services:
56
+ * AWS Kinesis
57
+ * AWS DynamoDB: the client library stores information for worker coordination in DynamoDB (offsets and active worker per partition)
58
+ * AWS CloudWatch: if the metrics are enabled the credentials need CloudWatch update permisions granted.
59
+
60
+ Look at the [documentation](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) for deeper information.
28
61
 
29
62
  ## Contributing
30
63
 
@@ -22,6 +22,7 @@ require "logstash/inputs/kinesis/version"
22
22
  # The library can optionally also send worker statistics to CloudWatch.
23
23
  class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
24
24
  KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
25
+ KCL_PROCESSOR_FACTORY_CLASS = com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory
25
26
  require "logstash/inputs/kinesis/worker"
26
27
 
27
28
  config_name 'kinesis'
@@ -49,8 +50,7 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
49
50
  # to enable the cloudwatch integration in the Kinesis Client Library.
50
51
  config :metrics, :validate => [nil, "cloudwatch"], :default => nil
51
52
 
52
- def initialize(params = {}, kcl_class = KCL::Worker)
53
- @kcl_class = kcl_class
53
+ def initialize(params = {})
54
54
  super(params)
55
55
  end
56
56
 
@@ -60,7 +60,7 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
60
60
  logger.setLevel(java.util.logging::Level::WARNING)
61
61
 
62
62
  worker_id = java.util::UUID.randomUUID.to_s
63
- creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new()
63
+ creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new
64
64
  @kcl_config = KCL::KinesisClientLibConfiguration.new(
65
65
  @application_name,
66
66
  @kinesis_stream_name,
@@ -71,25 +71,29 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
71
71
  end
72
72
 
73
73
  def run(output_queue)
74
- worker_factory = proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
75
- if metrics_factory
76
- @kcl_worker = @kcl_class.new(
77
- worker_factory,
78
- @kcl_config,
79
- metrics_factory)
80
- else
81
- @kcl_worker = @kcl_class.new(
82
- worker_factory,
83
- @kcl_config)
84
- end
74
+ @kcl_worker = kcl_builder(output_queue).build
75
+ @kcl_worker.run
76
+ end
77
+
78
+ def kcl_builder(output_queue)
79
+ KCL::Worker::Builder.new.tap do |builder|
80
+ builder.java_send(:recordProcessorFactory, [KCL_PROCESSOR_FACTORY_CLASS.java_class], worker_factory(output_queue))
81
+ builder.config(@kcl_config)
85
82
 
86
- @kcl_worker.run()
83
+ if metrics_factory
84
+ builder.metricsFactory(metrics_factory)
85
+ end
86
+ end
87
87
  end
88
88
 
89
89
  def teardown
90
90
  @kcl_worker.shutdown if @kcl_worker
91
91
  end
92
92
 
93
+ def worker_factory(output_queue)
94
+ proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
95
+ end
96
+
93
97
  protected
94
98
 
95
99
  def metrics_factory
@@ -1,7 +1,7 @@
1
1
  module Logstash
2
2
  module Input
3
3
  module Kinesis
4
- VERSION = "1.3.1"
4
+ VERSION = "1.4.0"
5
5
  end
6
6
  end
7
7
  end
@@ -1,5 +1,5 @@
1
1
  class LogStash::Inputs::Kinesis::Worker
2
- include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
2
+ include com.amazonaws.services.kinesis.clientlibrary.interfaces.v2::IRecordProcessor
3
3
 
4
4
  attr_reader(
5
5
  :checkpoint_interval,
@@ -16,23 +16,23 @@ class LogStash::Inputs::Kinesis::Worker
16
16
  @next_checkpoint = Time.now - 600
17
17
  @constructed = true
18
18
  else
19
- _shard_id, _ = args
19
+ _shard_id = args[0].shardId
20
20
  @decoder = java.nio.charset::Charset.forName("UTF-8").newDecoder()
21
21
  end
22
22
  end
23
23
  public :initialize
24
24
 
25
- def processRecords(records, checkpointer)
26
- records.each { |record| process_record(record) }
25
+ def processRecords(records_input)
26
+ records_input.records.each { |record| process_record(record) }
27
27
  if Time.now >= @next_checkpoint
28
- checkpoint(checkpointer)
28
+ checkpoint(records_input.checkpointer)
29
29
  @next_checkpoint = Time.now + @checkpoint_interval
30
30
  end
31
31
  end
32
32
 
33
- def shutdown(checkpointer, reason)
34
- if reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
35
- checkpoint(checkpointer)
33
+ def shutdown(shutdown_input)
34
+ if shutdown_input.reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
35
+ checkpoint(shutdown_input.checkpointer)
36
36
  end
37
37
  end
38
38
 
data/pom.xml CHANGED
@@ -8,7 +8,7 @@
8
8
  <dependency>
9
9
  <groupId>com.amazonaws</groupId>
10
10
  <artifactId>amazon-kinesis-client</artifactId>
11
- <version>1.2.1</version>
11
+ <version>1.6.1</version>
12
12
  </dependency>
13
13
  </dependencies>
14
14
  <build>
@@ -1,19 +1,22 @@
1
+ require 'logstash-input-kinesis_jars'
1
2
  require "logstash/plugin"
2
3
  require "logstash/inputs/kinesis"
3
4
  require "logstash/codecs/json"
4
5
  require "json"
5
6
 
6
7
  RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
8
+ KCL_TYPES = com.amazonaws.services.kinesis.clientlibrary.types
9
+
7
10
  subject!(:worker) { LogStash::Inputs::Kinesis::Worker.new(codec, queue, decorator, checkpoint_interval) }
8
11
  let(:codec) { LogStash::Codecs::JSON.new() }
9
12
  let(:queue) { Queue.new }
10
13
  let(:decorator) { proc { |x| x["decorated"] = true; x } }
11
14
  let(:checkpoint_interval) { 120 }
12
15
  let(:checkpointer) { double('checkpointer', checkpoint: nil) }
13
- let(:shard_id) { "xyz" }
16
+ let(:init_input) { KCL_TYPES::InitializationInput.new().withShardId("xyz") }
14
17
 
15
18
  it "honors the initialize java interface method contract" do
16
- expect { worker.initialize(shard_id) }.to_not raise_error
19
+ expect { worker.initialize(init_input) }.to_not raise_error
17
20
  end
18
21
 
19
22
  def record(hash = { "message" => "test" })
@@ -22,17 +25,29 @@ RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
22
25
  double(getData: data)
23
26
  end
24
27
 
25
- let(:record1) { record(id: "record1", message: "test1") }
26
- let(:record2) { record(id: "record2", message: "test2") }
28
+ let(:process_input) {
29
+ KCL_TYPES::ProcessRecordsInput.new()
30
+ .withRecords(java.util.Arrays.asList([
31
+ record(id: "record1", message: "test1"),
32
+ record(id: "record2", message: "test2")
33
+ ].to_java)
34
+ )
35
+ .withCheckpointer(checkpointer)
36
+ }
37
+ let(:empty_process_input) {
38
+ KCL_TYPES::ProcessRecordsInput.new()
39
+ .withRecords(java.util.Arrays.asList([].to_java))
40
+ .withCheckpointer(checkpointer)
41
+ }
27
42
 
28
43
  context "initialized" do
29
44
  before do
30
- worker.initialize(shard_id)
45
+ worker.initialize(init_input)
31
46
  end
32
47
 
33
48
  describe "#processRecords" do
34
49
  it "decodes and queues each record with decoration" do
35
- worker.processRecords([record1, record2], checkpointer)
50
+ worker.processRecords(process_input)
36
51
  m1 = queue.pop
37
52
  m2 = queue.pop
38
53
  expect(m1).to be_kind_of(LogStash::Event)
@@ -44,14 +59,14 @@ RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
44
59
 
45
60
  it "checkpoints on interval" do
46
61
  expect(checkpointer).to receive(:checkpoint).once
47
- worker.processRecords([], checkpointer)
62
+ worker.processRecords(empty_process_input)
48
63
 
49
64
  # not this time
50
- worker.processRecords([], checkpointer)
65
+ worker.processRecords(empty_process_input)
51
66
 
52
67
  allow(Time).to receive(:now).and_return(Time.now + 125)
53
68
  expect(checkpointer).to receive(:checkpoint).once
54
- worker.processRecords([], checkpointer)
69
+ worker.processRecords(empty_process_input)
55
70
  end
56
71
  end
57
72
  end
@@ -2,17 +2,10 @@ require "logstash/plugin"
2
2
  require "logstash/inputs/kinesis"
3
3
  require "logstash/codecs/json"
4
4
 
5
+
5
6
  RSpec.describe "inputs/kinesis" do
6
7
  KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
7
8
 
8
- # It's very difficult to directly test the Java class, with all its private members,
9
- # so we subclass it here.
10
- class TestKCLWorker < KCL::Worker
11
- field_reader :metricsFactory, :recordProcessorFactory
12
- def run
13
- end
14
- end
15
-
16
9
  let(:config) {{
17
10
  "application_name" => "my-processor",
18
11
  "kinesis_stream_name" => "run-specs",
@@ -22,10 +15,12 @@ RSpec.describe "inputs/kinesis" do
22
15
  "region" => "ap-southeast-1",
23
16
  }}
24
17
 
25
- subject!(:kinesis) { LogStash::Inputs::Kinesis.new(config, kcl_class) }
26
- let(:kcl_class) { TestKCLWorker }
18
+ subject!(:kinesis) { LogStash::Inputs::Kinesis.new(config) }
19
+ let(:kcl_worker) { double('kcl_worker') }
20
+ let(:stub_builder) { double('kcl_builder', build: kcl_worker) }
27
21
  let(:metrics) { nil }
28
22
  let(:codec) { LogStash::Codecs::JSON.new() }
23
+ let(:queue) { Queue.new }
29
24
 
30
25
  it "registers without error" do
31
26
  input = LogStash::Plugin.lookup("input", "kinesis").new("kinesis_stream_name" => "specs", "codec" => codec)
@@ -41,42 +36,63 @@ RSpec.describe "inputs/kinesis" do
41
36
  end
42
37
 
43
38
  context "#run" do
44
- let(:queue) { Queue.new }
45
-
46
- before do
47
- kinesis.register
39
+ it "runs the KCL worker" do
40
+ expect(kinesis).to receive(:kcl_builder).with(queue).and_return(stub_builder)
41
+ expect(kcl_worker).to receive(:run).with(no_args)
42
+ builder = kinesis.run(queue)
48
43
  end
44
+ end
49
45
 
46
+ context "#worker_factory" do
50
47
  it "clones the codec for each worker" do
51
48
  expect(codec).to receive(:clone).once
52
- kinesis.run(queue)
53
- worker = kinesis.kcl_worker.recordProcessorFactory.call()
49
+ worker = kinesis.worker_factory(queue).call()
54
50
  expect(worker).to be_kind_of(LogStash::Inputs::Kinesis::Worker)
55
51
  end
56
52
 
57
- it "generates a valid worker via the factory proc" do
58
- kinesis.run(queue)
59
- worker = kinesis.kcl_worker.recordProcessorFactory.call()
53
+ it "generates a valid worker" do
54
+ worker = kinesis.worker_factory(queue).call()
55
+
60
56
  expect(worker.codec).to be_kind_of(codec.class)
61
57
  expect(worker.checkpoint_interval).to eq(120)
62
58
  expect(worker.output_queue).to eq(queue)
63
59
  expect(worker.decorator).to eq(kinesis.method(:decorate))
64
60
  expect(worker.logger).to eq(kinesis.logger)
65
61
  end
62
+ end
63
+
64
+ # these tests are heavily dependent on the current Worker::Builder
65
+ # implementation because its state is all private
66
+ context "#kcl_builder" do
67
+ let(:builder) { kinesis.kcl_builder(queue) }
68
+
69
+ it "sets the worker factory" do
70
+ expect(field(builder, "recordProcessorFactory")).to_not eq(nil)
71
+ end
72
+
73
+ it "sets the config" do
74
+ kinesis.register
75
+ config = field(builder, "config")
76
+ expect(config).to eq(kinesis.kcl_config)
77
+ end
66
78
 
67
79
  it "disables metric tracking by default" do
68
- kinesis.run(queue)
69
- expect(kinesis.kcl_worker.metricsFactory).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory)
80
+ expect(field(builder, "metricsFactory")).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory)
70
81
  end
71
82
 
72
83
  context "cloudwatch" do
73
84
  let(:metrics) { "cloudwatch" }
74
85
  it "uses cloudwatch metrics if specified" do
75
- kinesis.run(queue)
76
- expect(kinesis.kcl_worker.metricsFactory).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::CWMetricsFactory)
77
- # the process hangs otherwise
78
- kinesis.kcl_worker.metricsFactory.shutdown
86
+ # since the behaviour is enclosed on private methods it is not testable. So here
87
+ # the expected value can be tested, not the result associated to set this value
88
+ expect(field(builder, "metricsFactory")).to eq(nil)
79
89
  end
80
90
  end
81
91
  end
92
+
93
+ def field(obj, name)
94
+ field = obj.java_class.declared_field(name)
95
+ field.accessible = true
96
+ field.value(obj)
97
+ end
82
98
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.1
4
+ version: 1.4.0
5
5
  platform: java
6
6
  authors:
7
7
  - Brian Palmer
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-06-17 00:00:00.000000000 Z
11
+ date: 2015-09-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -103,60 +103,24 @@ files:
103
103
  - spec/inputs/kinesis/worker_spec.rb
104
104
  - spec/inputs/kinesis_spec.rb
105
105
  - spec/spec_helper.rb
106
- - vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.2.1.jar
107
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-1.9.16.jar
108
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-autoscaling-1.9.16.jar
109
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudformation-1.9.16.jar
110
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudfront-1.9.16.jar
111
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudhsm-1.9.16.jar
112
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudsearch-1.9.16.jar
113
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudtrail-1.9.16.jar
114
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.9.16.jar
115
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatchmetrics-1.9.16.jar
116
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-codedeploy-1.9.16.jar
117
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitoidentity-1.9.16.jar
118
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitosync-1.9.16.jar
119
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-config-1.9.16.jar
120
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.9.16.jar
121
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-datapipeline-1.9.16.jar
122
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-directconnect-1.9.16.jar
123
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.9.16.jar
124
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-ec2-1.9.16.jar
125
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-ecs-1.9.16.jar
126
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticache-1.9.16.jar
127
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticbeanstalk-1.9.16.jar
128
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticloadbalancing-1.9.16.jar
129
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-elastictranscoder-1.9.16.jar
130
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-emr-1.9.16.jar
131
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-glacier-1.9.16.jar
132
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-iam-1.9.16.jar
133
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-importexport-1.9.16.jar
134
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.9.16.jar
135
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.9.16.jar
136
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-lambda-1.9.16.jar
137
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-logs-1.9.16.jar
138
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-opsworks-1.9.16.jar
139
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-rds-1.9.16.jar
140
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-redshift-1.9.16.jar
141
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-route53-1.9.16.jar
142
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.9.16.jar
143
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-ses-1.9.16.jar
144
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpledb-1.9.16.jar
145
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpleworkflow-1.9.16.jar
146
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-sns-1.9.16.jar
147
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-sqs-1.9.16.jar
148
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-storagegateway-1.9.16.jar
149
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-sts-1.9.16.jar
150
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-support-1.9.16.jar
151
- - vendor/jar-dependencies/runtime-jars/aws-java-sdk-swf-libraries-1.9.16.jar
106
+ - vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.6.1.jar
107
+ - vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.10.20.jar
108
+ - vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.10.20.jar
109
+ - vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.10.20.jar
110
+ - vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.10.20.jar
111
+ - vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.10.20.jar
112
+ - vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.10.20.jar
152
113
  - vendor/jar-dependencies/runtime-jars/commons-codec-1.6.jar
114
+ - vendor/jar-dependencies/runtime-jars/commons-lang-2.6.jar
153
115
  - vendor/jar-dependencies/runtime-jars/commons-logging-1.1.3.jar
154
- - vendor/jar-dependencies/runtime-jars/httpclient-4.3.4.jar
155
- - vendor/jar-dependencies/runtime-jars/httpcore-4.3.2.jar
156
- - vendor/jar-dependencies/runtime-jars/jackson-annotations-2.3.0.jar
157
- - vendor/jar-dependencies/runtime-jars/jackson-core-2.3.2.jar
158
- - vendor/jar-dependencies/runtime-jars/jackson-databind-2.3.2.jar
116
+ - vendor/jar-dependencies/runtime-jars/guava-18.0.jar
117
+ - vendor/jar-dependencies/runtime-jars/httpclient-4.3.6.jar
118
+ - vendor/jar-dependencies/runtime-jars/httpcore-4.3.3.jar
119
+ - vendor/jar-dependencies/runtime-jars/jackson-annotations-2.5.0.jar
120
+ - vendor/jar-dependencies/runtime-jars/jackson-core-2.5.3.jar
121
+ - vendor/jar-dependencies/runtime-jars/jackson-databind-2.5.3.jar
159
122
  - vendor/jar-dependencies/runtime-jars/joda-time-2.8.1.jar
123
+ - vendor/jar-dependencies/runtime-jars/protobuf-java-2.6.1.jar
160
124
  homepage: https://github.com/codekitchen/logstash-input-kinesis
161
125
  licenses:
162
126
  - Apache License (2.0)