logstash-input-kinesis 1.0.0-java → 1.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/inputs/kinesis.rb +23 -17
- data/lib/logstash/inputs/kinesis/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bb43b802dabbacadd3bb00a085e4b5916c6751d5
|
|
4
|
+
data.tar.gz: 03d11ef552c46c7022c37f38cb0a379363bc9e02
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 13eac6fc73411ce0c353557cbfba14b5ff0dffa7e3994c4277db3fcb65b43c70b013538d840ab2d619ccf54244af217f68d32e4c33fe75cb9980bd229e025c57
|
|
7
|
+
data.tar.gz: c5abf46b322cbb4c84d681178db838782aa06bd7e0936ddfbcc7fd7fda91b581479b28b4697be4d6814968f4f8043c04c3e2e14ca36cf0d13d2f7b3873ea14c3
|
|
@@ -20,6 +20,9 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
20
20
|
# The kinesis stream name.
|
|
21
21
|
config :kinesis_stream_name, :validate => :string, :required => true
|
|
22
22
|
|
|
23
|
+
# How many seconds between worker checkpoints to dynamodb.
|
|
24
|
+
config :checkpoint_interval_seconds, :validate => :number, :default => 60
|
|
25
|
+
|
|
23
26
|
def register
|
|
24
27
|
# the INFO log level is extremely noisy in KCL
|
|
25
28
|
org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").
|
|
@@ -35,9 +38,8 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
35
38
|
end
|
|
36
39
|
|
|
37
40
|
def run(output_queue)
|
|
38
|
-
worker_factory = WorkerFactory.new(@codec, output_queue, method(:decorate))
|
|
39
41
|
@worker = KCL::Worker.new(
|
|
40
|
-
|
|
42
|
+
proc { Worker.new(@codec, output_queue, method(:decorate), @checkpoint_interval_seconds) },
|
|
41
43
|
@config,
|
|
42
44
|
com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory.new)
|
|
43
45
|
@worker.run()
|
|
@@ -47,26 +49,14 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
47
49
|
@worker.shutdown if @worker
|
|
48
50
|
end
|
|
49
51
|
|
|
50
|
-
class WorkerFactory
|
|
51
|
-
include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessorFactory
|
|
52
|
-
def initialize(codec, output_queue, decorator)
|
|
53
|
-
@codec = codec
|
|
54
|
-
@output_queue = output_queue
|
|
55
|
-
@decorator = decorator
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def createProcessor
|
|
59
|
-
Worker.new(@codec.clone, @output_queue, @decorator)
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
|
|
63
52
|
class Worker
|
|
64
53
|
include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
|
|
65
54
|
|
|
66
55
|
def initialize(*args)
|
|
67
56
|
# nasty hack, because this is the name of a method on IRecordProcessor, but also ruby's constructor
|
|
68
57
|
if !@constructed
|
|
69
|
-
@codec, @output_queue, @decorator = args
|
|
58
|
+
@codec, @output_queue, @decorator, @checkpoint_interval = args
|
|
59
|
+
@next_checkpoint = Time.now - 600
|
|
70
60
|
@constructed = true
|
|
71
61
|
else
|
|
72
62
|
_shard_id, _ = args
|
|
@@ -76,10 +66,24 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
76
66
|
|
|
77
67
|
def processRecords(records, checkpointer)
|
|
78
68
|
records.each { |record| process_record(record) }
|
|
79
|
-
|
|
69
|
+
if Time.now >= @next_checkpoint
|
|
70
|
+
checkpoint(checkpointer)
|
|
71
|
+
@next_checkpoint = Time.now + @checkpoint_interval
|
|
72
|
+
end
|
|
80
73
|
end
|
|
81
74
|
|
|
82
75
|
def shutdown(checkpointer, reason)
|
|
76
|
+
if reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
|
|
77
|
+
checkpoint(checkpointer)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
protected
|
|
82
|
+
|
|
83
|
+
def checkpoint(checkpointer)
|
|
84
|
+
checkpointer.checkpoint()
|
|
85
|
+
rescue => error
|
|
86
|
+
@logger.error("Kinesis worker failed checkpointing: #{error}")
|
|
83
87
|
end
|
|
84
88
|
|
|
85
89
|
def process_record(record)
|
|
@@ -88,6 +92,8 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
88
92
|
@decorator.call(event)
|
|
89
93
|
@output_queue << event
|
|
90
94
|
end
|
|
95
|
+
rescue => error
|
|
96
|
+
@logger.error("Error processing record: #{error}")
|
|
91
97
|
end
|
|
92
98
|
end
|
|
93
99
|
end
|