logstash-input-kinesis 1.3.1-java → 1.4.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +36 -3
- data/lib/logstash/inputs/kinesis.rb +19 -15
- data/lib/logstash/inputs/kinesis/version.rb +1 -1
- data/lib/logstash/inputs/kinesis/worker.rb +8 -8
- data/pom.xml +1 -1
- data/spec/inputs/kinesis/worker_spec.rb +24 -9
- data/spec/inputs/kinesis_spec.rb +41 -25
- data/vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.6.1.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.10.20.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/{aws-java-sdk-core-1.9.16.jar → aws-java-sdk-core-1.10.20.jar} +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.10.20.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.10.20.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.10.20.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.10.20.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/commons-lang-2.6.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/guava-18.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/{httpclient-4.3.4.jar → httpclient-4.3.6.jar} +0 -0
- data/vendor/jar-dependencies/runtime-jars/{httpcore-4.3.2.jar → httpcore-4.3.3.jar} +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-annotations-2.5.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-core-2.5.3.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-databind-2.5.3.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/protobuf-java-2.6.1.jar +0 -0
- metadata +17 -53
- data/vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.2.1.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-autoscaling-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudformation-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudfront-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudhsm-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudsearch-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudtrail-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatchmetrics-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-codedeploy-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitoidentity-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitosync-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-config-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-datapipeline-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-directconnect-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-ec2-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-ecs-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticache-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticbeanstalk-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticloadbalancing-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-elastictranscoder-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-emr-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-glacier-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-iam-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-importexport-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-lambda-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-logs-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-opsworks-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-rds-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-redshift-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-route53-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-ses-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpledb-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpleworkflow-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-sns-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-sqs-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-storagegateway-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-sts-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-support-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/aws-java-sdk-swf-libraries-1.9.16.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-annotations-2.3.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-core-2.3.2.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/jackson-databind-2.3.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e0660cef254c9970b671583df6a7c95489ae888f
|
4
|
+
data.tar.gz: d1a24cc652091346a8f0102ac532bf462f00b310
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 945f30cbd64354c94addfbeb96be414c8a9d1fb0bb96efa70fac0b403242ddcd04c44864d396d7859a35395ca92861e3e7453590156acdb20688a7c03202288b
|
7
|
+
data.tar.gz: e3ccec93560deb6bd871517cc6238c663e49ac9e63fb519757c0c1c70389f3a3732944c858c8a0f9483af87086fc170372911e39f3377c2fbad30ab22afab5d5
|
data/README.md
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
# Logstash AWS Kinesis Input Plugin
|
2
2
|
|
3
|
-
This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
|
4
|
-
|
5
3
|
[](https://travis-ci.org/codekitchen/logstash-input-kinesis)
|
6
4
|
|
5
|
+
This is a [AWS Kinesis](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html) input plugin for [Logstash](https://github.com/elasticsearch/logstash). Under the hood uses the [Kinesis Client Library](http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-implementation-app-java.html).
|
6
|
+
|
7
7
|
## Installation
|
8
8
|
|
9
9
|
This plugin requires Logstash 1.5, and can be installed by Logstash
|
@@ -24,7 +24,40 @@ input {
|
|
24
24
|
}
|
25
25
|
```
|
26
26
|
|
27
|
-
|
27
|
+
## Configuration
|
28
|
+
|
29
|
+
This are the properties you can configure and what are the default values:
|
30
|
+
|
31
|
+
* `application_name`: The name of the application used in DynamoDB for coordination. Only one worker per unique stream partition and application will be actively consuming messages.
|
32
|
+
* **required**: false
|
33
|
+
* **default value**: `logstash`
|
34
|
+
* `kinesis_stream_name`: The Kinesis stream name.
|
35
|
+
* **required**: true
|
36
|
+
* `region`: The AWS region name for Kinesis, DynamoDB and Cloudwatch (if enabled)
|
37
|
+
* **required**: false
|
38
|
+
* **default value**: `us-east-1`
|
39
|
+
* `checkpoint_interval_seconds`: How many seconds between worker checkpoints to DynamoDB. A low value ussually means lower message replay in case of node failure/restart but it increases CPU+network ussage (which increases the AWS costs).
|
40
|
+
* **required**: false
|
41
|
+
* **default value**: `60`
|
42
|
+
* `metrics`: Worker metric tracking. By default this is disabled, set it to "cloudwatch" to enable the cloudwatch integration in the Kinesis Client Library.
|
43
|
+
* **required**: false
|
44
|
+
* **default value**: `nil`
|
45
|
+
|
46
|
+
## Authentication
|
47
|
+
|
48
|
+
This plugin uses the default AWS SDK auth chain, [DefaultAWSCredentialsProviderChain](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html), to determine which credentials the client will use.
|
49
|
+
|
50
|
+
The default chain follows this order trying to read the credentials:
|
51
|
+
* `AWS_ACCESS_KEY_ID` / `AWS_SECRET_KEY` environment variables
|
52
|
+
* `~/.aws/credentials` credentials file
|
53
|
+
* EC2 instance profile
|
54
|
+
|
55
|
+
The credentials will need access to the following services:
|
56
|
+
* AWS Kinesis
|
57
|
+
* AWS DynamoDB: the client library stores information for worker coordination in DynamoDB (offsets and active worker per partition)
|
58
|
+
* AWS CloudWatch: if the metrics are enabled the credentials need CloudWatch update permisions granted.
|
59
|
+
|
60
|
+
Look at the [documentation](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) for deeper information.
|
28
61
|
|
29
62
|
## Contributing
|
30
63
|
|
@@ -22,6 +22,7 @@ require "logstash/inputs/kinesis/version"
|
|
22
22
|
# The library can optionally also send worker statistics to CloudWatch.
|
23
23
|
class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
24
24
|
KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
|
25
|
+
KCL_PROCESSOR_FACTORY_CLASS = com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory
|
25
26
|
require "logstash/inputs/kinesis/worker"
|
26
27
|
|
27
28
|
config_name 'kinesis'
|
@@ -49,8 +50,7 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
49
50
|
# to enable the cloudwatch integration in the Kinesis Client Library.
|
50
51
|
config :metrics, :validate => [nil, "cloudwatch"], :default => nil
|
51
52
|
|
52
|
-
def initialize(params = {}
|
53
|
-
@kcl_class = kcl_class
|
53
|
+
def initialize(params = {})
|
54
54
|
super(params)
|
55
55
|
end
|
56
56
|
|
@@ -60,7 +60,7 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
60
60
|
logger.setLevel(java.util.logging::Level::WARNING)
|
61
61
|
|
62
62
|
worker_id = java.util::UUID.randomUUID.to_s
|
63
|
-
creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new
|
63
|
+
creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new
|
64
64
|
@kcl_config = KCL::KinesisClientLibConfiguration.new(
|
65
65
|
@application_name,
|
66
66
|
@kinesis_stream_name,
|
@@ -71,25 +71,29 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
71
71
|
end
|
72
72
|
|
73
73
|
def run(output_queue)
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
worker_factory,
|
83
|
-
@kcl_config)
|
84
|
-
end
|
74
|
+
@kcl_worker = kcl_builder(output_queue).build
|
75
|
+
@kcl_worker.run
|
76
|
+
end
|
77
|
+
|
78
|
+
def kcl_builder(output_queue)
|
79
|
+
KCL::Worker::Builder.new.tap do |builder|
|
80
|
+
builder.java_send(:recordProcessorFactory, [KCL_PROCESSOR_FACTORY_CLASS.java_class], worker_factory(output_queue))
|
81
|
+
builder.config(@kcl_config)
|
85
82
|
|
86
|
-
|
83
|
+
if metrics_factory
|
84
|
+
builder.metricsFactory(metrics_factory)
|
85
|
+
end
|
86
|
+
end
|
87
87
|
end
|
88
88
|
|
89
89
|
def teardown
|
90
90
|
@kcl_worker.shutdown if @kcl_worker
|
91
91
|
end
|
92
92
|
|
93
|
+
def worker_factory(output_queue)
|
94
|
+
proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
|
95
|
+
end
|
96
|
+
|
93
97
|
protected
|
94
98
|
|
95
99
|
def metrics_factory
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class LogStash::Inputs::Kinesis::Worker
|
2
|
-
include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
|
2
|
+
include com.amazonaws.services.kinesis.clientlibrary.interfaces.v2::IRecordProcessor
|
3
3
|
|
4
4
|
attr_reader(
|
5
5
|
:checkpoint_interval,
|
@@ -16,23 +16,23 @@ class LogStash::Inputs::Kinesis::Worker
|
|
16
16
|
@next_checkpoint = Time.now - 600
|
17
17
|
@constructed = true
|
18
18
|
else
|
19
|
-
_shard_id
|
19
|
+
_shard_id = args[0].shardId
|
20
20
|
@decoder = java.nio.charset::Charset.forName("UTF-8").newDecoder()
|
21
21
|
end
|
22
22
|
end
|
23
23
|
public :initialize
|
24
24
|
|
25
|
-
def processRecords(
|
26
|
-
records.each { |record| process_record(record) }
|
25
|
+
def processRecords(records_input)
|
26
|
+
records_input.records.each { |record| process_record(record) }
|
27
27
|
if Time.now >= @next_checkpoint
|
28
|
-
checkpoint(checkpointer)
|
28
|
+
checkpoint(records_input.checkpointer)
|
29
29
|
@next_checkpoint = Time.now + @checkpoint_interval
|
30
30
|
end
|
31
31
|
end
|
32
32
|
|
33
|
-
def shutdown(
|
34
|
-
if reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
|
35
|
-
checkpoint(checkpointer)
|
33
|
+
def shutdown(shutdown_input)
|
34
|
+
if shutdown_input.reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
|
35
|
+
checkpoint(shutdown_input.checkpointer)
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
data/pom.xml
CHANGED
@@ -1,19 +1,22 @@
|
|
1
|
+
require 'logstash-input-kinesis_jars'
|
1
2
|
require "logstash/plugin"
|
2
3
|
require "logstash/inputs/kinesis"
|
3
4
|
require "logstash/codecs/json"
|
4
5
|
require "json"
|
5
6
|
|
6
7
|
RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
|
8
|
+
KCL_TYPES = com.amazonaws.services.kinesis.clientlibrary.types
|
9
|
+
|
7
10
|
subject!(:worker) { LogStash::Inputs::Kinesis::Worker.new(codec, queue, decorator, checkpoint_interval) }
|
8
11
|
let(:codec) { LogStash::Codecs::JSON.new() }
|
9
12
|
let(:queue) { Queue.new }
|
10
13
|
let(:decorator) { proc { |x| x["decorated"] = true; x } }
|
11
14
|
let(:checkpoint_interval) { 120 }
|
12
15
|
let(:checkpointer) { double('checkpointer', checkpoint: nil) }
|
13
|
-
let(:
|
16
|
+
let(:init_input) { KCL_TYPES::InitializationInput.new().withShardId("xyz") }
|
14
17
|
|
15
18
|
it "honors the initialize java interface method contract" do
|
16
|
-
expect { worker.initialize(
|
19
|
+
expect { worker.initialize(init_input) }.to_not raise_error
|
17
20
|
end
|
18
21
|
|
19
22
|
def record(hash = { "message" => "test" })
|
@@ -22,17 +25,29 @@ RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
|
|
22
25
|
double(getData: data)
|
23
26
|
end
|
24
27
|
|
25
|
-
let(:
|
26
|
-
|
28
|
+
let(:process_input) {
|
29
|
+
KCL_TYPES::ProcessRecordsInput.new()
|
30
|
+
.withRecords(java.util.Arrays.asList([
|
31
|
+
record(id: "record1", message: "test1"),
|
32
|
+
record(id: "record2", message: "test2")
|
33
|
+
].to_java)
|
34
|
+
)
|
35
|
+
.withCheckpointer(checkpointer)
|
36
|
+
}
|
37
|
+
let(:empty_process_input) {
|
38
|
+
KCL_TYPES::ProcessRecordsInput.new()
|
39
|
+
.withRecords(java.util.Arrays.asList([].to_java))
|
40
|
+
.withCheckpointer(checkpointer)
|
41
|
+
}
|
27
42
|
|
28
43
|
context "initialized" do
|
29
44
|
before do
|
30
|
-
worker.initialize(
|
45
|
+
worker.initialize(init_input)
|
31
46
|
end
|
32
47
|
|
33
48
|
describe "#processRecords" do
|
34
49
|
it "decodes and queues each record with decoration" do
|
35
|
-
worker.processRecords(
|
50
|
+
worker.processRecords(process_input)
|
36
51
|
m1 = queue.pop
|
37
52
|
m2 = queue.pop
|
38
53
|
expect(m1).to be_kind_of(LogStash::Event)
|
@@ -44,14 +59,14 @@ RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
|
|
44
59
|
|
45
60
|
it "checkpoints on interval" do
|
46
61
|
expect(checkpointer).to receive(:checkpoint).once
|
47
|
-
worker.processRecords(
|
62
|
+
worker.processRecords(empty_process_input)
|
48
63
|
|
49
64
|
# not this time
|
50
|
-
worker.processRecords(
|
65
|
+
worker.processRecords(empty_process_input)
|
51
66
|
|
52
67
|
allow(Time).to receive(:now).and_return(Time.now + 125)
|
53
68
|
expect(checkpointer).to receive(:checkpoint).once
|
54
|
-
worker.processRecords(
|
69
|
+
worker.processRecords(empty_process_input)
|
55
70
|
end
|
56
71
|
end
|
57
72
|
end
|
data/spec/inputs/kinesis_spec.rb
CHANGED
@@ -2,17 +2,10 @@ require "logstash/plugin"
|
|
2
2
|
require "logstash/inputs/kinesis"
|
3
3
|
require "logstash/codecs/json"
|
4
4
|
|
5
|
+
|
5
6
|
RSpec.describe "inputs/kinesis" do
|
6
7
|
KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
|
7
8
|
|
8
|
-
# It's very difficult to directly test the Java class, with all its private members,
|
9
|
-
# so we subclass it here.
|
10
|
-
class TestKCLWorker < KCL::Worker
|
11
|
-
field_reader :metricsFactory, :recordProcessorFactory
|
12
|
-
def run
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
9
|
let(:config) {{
|
17
10
|
"application_name" => "my-processor",
|
18
11
|
"kinesis_stream_name" => "run-specs",
|
@@ -22,10 +15,12 @@ RSpec.describe "inputs/kinesis" do
|
|
22
15
|
"region" => "ap-southeast-1",
|
23
16
|
}}
|
24
17
|
|
25
|
-
subject!(:kinesis) { LogStash::Inputs::Kinesis.new(config
|
26
|
-
let(:
|
18
|
+
subject!(:kinesis) { LogStash::Inputs::Kinesis.new(config) }
|
19
|
+
let(:kcl_worker) { double('kcl_worker') }
|
20
|
+
let(:stub_builder) { double('kcl_builder', build: kcl_worker) }
|
27
21
|
let(:metrics) { nil }
|
28
22
|
let(:codec) { LogStash::Codecs::JSON.new() }
|
23
|
+
let(:queue) { Queue.new }
|
29
24
|
|
30
25
|
it "registers without error" do
|
31
26
|
input = LogStash::Plugin.lookup("input", "kinesis").new("kinesis_stream_name" => "specs", "codec" => codec)
|
@@ -41,42 +36,63 @@ RSpec.describe "inputs/kinesis" do
|
|
41
36
|
end
|
42
37
|
|
43
38
|
context "#run" do
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
kinesis.
|
39
|
+
it "runs the KCL worker" do
|
40
|
+
expect(kinesis).to receive(:kcl_builder).with(queue).and_return(stub_builder)
|
41
|
+
expect(kcl_worker).to receive(:run).with(no_args)
|
42
|
+
builder = kinesis.run(queue)
|
48
43
|
end
|
44
|
+
end
|
49
45
|
|
46
|
+
context "#worker_factory" do
|
50
47
|
it "clones the codec for each worker" do
|
51
48
|
expect(codec).to receive(:clone).once
|
52
|
-
kinesis.
|
53
|
-
worker = kinesis.kcl_worker.recordProcessorFactory.call()
|
49
|
+
worker = kinesis.worker_factory(queue).call()
|
54
50
|
expect(worker).to be_kind_of(LogStash::Inputs::Kinesis::Worker)
|
55
51
|
end
|
56
52
|
|
57
|
-
it "generates a valid worker
|
58
|
-
kinesis.
|
59
|
-
|
53
|
+
it "generates a valid worker" do
|
54
|
+
worker = kinesis.worker_factory(queue).call()
|
55
|
+
|
60
56
|
expect(worker.codec).to be_kind_of(codec.class)
|
61
57
|
expect(worker.checkpoint_interval).to eq(120)
|
62
58
|
expect(worker.output_queue).to eq(queue)
|
63
59
|
expect(worker.decorator).to eq(kinesis.method(:decorate))
|
64
60
|
expect(worker.logger).to eq(kinesis.logger)
|
65
61
|
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# these tests are heavily dependent on the current Worker::Builder
|
65
|
+
# implementation because its state is all private
|
66
|
+
context "#kcl_builder" do
|
67
|
+
let(:builder) { kinesis.kcl_builder(queue) }
|
68
|
+
|
69
|
+
it "sets the worker factory" do
|
70
|
+
expect(field(builder, "recordProcessorFactory")).to_not eq(nil)
|
71
|
+
end
|
72
|
+
|
73
|
+
it "sets the config" do
|
74
|
+
kinesis.register
|
75
|
+
config = field(builder, "config")
|
76
|
+
expect(config).to eq(kinesis.kcl_config)
|
77
|
+
end
|
66
78
|
|
67
79
|
it "disables metric tracking by default" do
|
68
|
-
kinesis.
|
69
|
-
expect(kinesis.kcl_worker.metricsFactory).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory)
|
80
|
+
expect(field(builder, "metricsFactory")).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory)
|
70
81
|
end
|
71
82
|
|
72
83
|
context "cloudwatch" do
|
73
84
|
let(:metrics) { "cloudwatch" }
|
74
85
|
it "uses cloudwatch metrics if specified" do
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
kinesis.kcl_worker.metricsFactory.shutdown
|
86
|
+
# since the behaviour is enclosed on private methods it is not testable. So here
|
87
|
+
# the expected value can be tested, not the result associated to set this value
|
88
|
+
expect(field(builder, "metricsFactory")).to eq(nil)
|
79
89
|
end
|
80
90
|
end
|
81
91
|
end
|
92
|
+
|
93
|
+
def field(obj, name)
|
94
|
+
field = obj.java_class.declared_field(name)
|
95
|
+
field.accessible = true
|
96
|
+
field.value(obj)
|
97
|
+
end
|
82
98
|
end
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-kinesis
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Brian Palmer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -103,60 +103,24 @@ files:
|
|
103
103
|
- spec/inputs/kinesis/worker_spec.rb
|
104
104
|
- spec/inputs/kinesis_spec.rb
|
105
105
|
- spec/spec_helper.rb
|
106
|
-
- vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.
|
107
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-1.
|
108
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-
|
109
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-
|
110
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-
|
111
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-
|
112
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-
|
113
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudtrail-1.9.16.jar
|
114
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.9.16.jar
|
115
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatchmetrics-1.9.16.jar
|
116
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-codedeploy-1.9.16.jar
|
117
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitoidentity-1.9.16.jar
|
118
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitosync-1.9.16.jar
|
119
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-config-1.9.16.jar
|
120
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.9.16.jar
|
121
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-datapipeline-1.9.16.jar
|
122
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-directconnect-1.9.16.jar
|
123
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.9.16.jar
|
124
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ec2-1.9.16.jar
|
125
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ecs-1.9.16.jar
|
126
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticache-1.9.16.jar
|
127
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticbeanstalk-1.9.16.jar
|
128
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticloadbalancing-1.9.16.jar
|
129
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elastictranscoder-1.9.16.jar
|
130
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-emr-1.9.16.jar
|
131
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-glacier-1.9.16.jar
|
132
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-iam-1.9.16.jar
|
133
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-importexport-1.9.16.jar
|
134
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.9.16.jar
|
135
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.9.16.jar
|
136
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-lambda-1.9.16.jar
|
137
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-logs-1.9.16.jar
|
138
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-opsworks-1.9.16.jar
|
139
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-rds-1.9.16.jar
|
140
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-redshift-1.9.16.jar
|
141
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-route53-1.9.16.jar
|
142
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.9.16.jar
|
143
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ses-1.9.16.jar
|
144
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpledb-1.9.16.jar
|
145
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpleworkflow-1.9.16.jar
|
146
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sns-1.9.16.jar
|
147
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sqs-1.9.16.jar
|
148
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-storagegateway-1.9.16.jar
|
149
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sts-1.9.16.jar
|
150
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-support-1.9.16.jar
|
151
|
-
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-swf-libraries-1.9.16.jar
|
106
|
+
- vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.6.1.jar
|
107
|
+
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.10.20.jar
|
108
|
+
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.10.20.jar
|
109
|
+
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.10.20.jar
|
110
|
+
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.10.20.jar
|
111
|
+
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.10.20.jar
|
112
|
+
- vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.10.20.jar
|
152
113
|
- vendor/jar-dependencies/runtime-jars/commons-codec-1.6.jar
|
114
|
+
- vendor/jar-dependencies/runtime-jars/commons-lang-2.6.jar
|
153
115
|
- vendor/jar-dependencies/runtime-jars/commons-logging-1.1.3.jar
|
154
|
-
- vendor/jar-dependencies/runtime-jars/
|
155
|
-
- vendor/jar-dependencies/runtime-jars/
|
156
|
-
- vendor/jar-dependencies/runtime-jars/
|
157
|
-
- vendor/jar-dependencies/runtime-jars/jackson-
|
158
|
-
- vendor/jar-dependencies/runtime-jars/jackson-
|
116
|
+
- vendor/jar-dependencies/runtime-jars/guava-18.0.jar
|
117
|
+
- vendor/jar-dependencies/runtime-jars/httpclient-4.3.6.jar
|
118
|
+
- vendor/jar-dependencies/runtime-jars/httpcore-4.3.3.jar
|
119
|
+
- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.5.0.jar
|
120
|
+
- vendor/jar-dependencies/runtime-jars/jackson-core-2.5.3.jar
|
121
|
+
- vendor/jar-dependencies/runtime-jars/jackson-databind-2.5.3.jar
|
159
122
|
- vendor/jar-dependencies/runtime-jars/joda-time-2.8.1.jar
|
123
|
+
- vendor/jar-dependencies/runtime-jars/protobuf-java-2.6.1.jar
|
160
124
|
homepage: https://github.com/codekitchen/logstash-input-kinesis
|
161
125
|
licenses:
|
162
126
|
- Apache License (2.0)
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|