logstash-input-kinesis 1.0.0-java → 1.1.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/logstash/inputs/kinesis.rb +23 -17
- data/lib/logstash/inputs/kinesis/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bb43b802dabbacadd3bb00a085e4b5916c6751d5
|
|
4
|
+
data.tar.gz: 03d11ef552c46c7022c37f38cb0a379363bc9e02
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 13eac6fc73411ce0c353557cbfba14b5ff0dffa7e3994c4277db3fcb65b43c70b013538d840ab2d619ccf54244af217f68d32e4c33fe75cb9980bd229e025c57
|
|
7
|
+
data.tar.gz: c5abf46b322cbb4c84d681178db838782aa06bd7e0936ddfbcc7fd7fda91b581479b28b4697be4d6814968f4f8043c04c3e2e14ca36cf0d13d2f7b3873ea14c3
|
|
@@ -20,6 +20,9 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
20
20
|
# The kinesis stream name.
|
|
21
21
|
config :kinesis_stream_name, :validate => :string, :required => true
|
|
22
22
|
|
|
23
|
+
# How many seconds between worker checkpoints to dynamodb.
|
|
24
|
+
config :checkpoint_interval_seconds, :validate => :number, :default => 60
|
|
25
|
+
|
|
23
26
|
def register
|
|
24
27
|
# the INFO log level is extremely noisy in KCL
|
|
25
28
|
org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").
|
|
@@ -35,9 +38,8 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
35
38
|
end
|
|
36
39
|
|
|
37
40
|
def run(output_queue)
|
|
38
|
-
worker_factory = WorkerFactory.new(@codec, output_queue, method(:decorate))
|
|
39
41
|
@worker = KCL::Worker.new(
|
|
40
|
-
|
|
42
|
+
proc { Worker.new(@codec, output_queue, method(:decorate), @checkpoint_interval_seconds) },
|
|
41
43
|
@config,
|
|
42
44
|
com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory.new)
|
|
43
45
|
@worker.run()
|
|
@@ -47,26 +49,14 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
47
49
|
@worker.shutdown if @worker
|
|
48
50
|
end
|
|
49
51
|
|
|
50
|
-
class WorkerFactory
|
|
51
|
-
include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessorFactory
|
|
52
|
-
def initialize(codec, output_queue, decorator)
|
|
53
|
-
@codec = codec
|
|
54
|
-
@output_queue = output_queue
|
|
55
|
-
@decorator = decorator
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def createProcessor
|
|
59
|
-
Worker.new(@codec.clone, @output_queue, @decorator)
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
|
|
63
52
|
class Worker
|
|
64
53
|
include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
|
|
65
54
|
|
|
66
55
|
def initialize(*args)
|
|
67
56
|
# nasty hack, because this is the name of a method on IRecordProcessor, but also ruby's constructor
|
|
68
57
|
if !@constructed
|
|
69
|
-
@codec, @output_queue, @decorator = args
|
|
58
|
+
@codec, @output_queue, @decorator, @checkpoint_interval = args
|
|
59
|
+
@next_checkpoint = Time.now - 600
|
|
70
60
|
@constructed = true
|
|
71
61
|
else
|
|
72
62
|
_shard_id, _ = args
|
|
@@ -76,10 +66,24 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
76
66
|
|
|
77
67
|
def processRecords(records, checkpointer)
|
|
78
68
|
records.each { |record| process_record(record) }
|
|
79
|
-
|
|
69
|
+
if Time.now >= @next_checkpoint
|
|
70
|
+
checkpoint(checkpointer)
|
|
71
|
+
@next_checkpoint = Time.now + @checkpoint_interval
|
|
72
|
+
end
|
|
80
73
|
end
|
|
81
74
|
|
|
82
75
|
def shutdown(checkpointer, reason)
|
|
76
|
+
if reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
|
|
77
|
+
checkpoint(checkpointer)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
protected
|
|
82
|
+
|
|
83
|
+
def checkpoint(checkpointer)
|
|
84
|
+
checkpointer.checkpoint()
|
|
85
|
+
rescue => error
|
|
86
|
+
@logger.error("Kinesis worker failed checkpointing: #{error}")
|
|
83
87
|
end
|
|
84
88
|
|
|
85
89
|
def process_record(record)
|
|
@@ -88,6 +92,8 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
|
|
|
88
92
|
@decorator.call(event)
|
|
89
93
|
@output_queue << event
|
|
90
94
|
end
|
|
95
|
+
rescue => error
|
|
96
|
+
@logger.error("Error processing record: #{error}")
|
|
91
97
|
end
|
|
92
98
|
end
|
|
93
99
|
end
|