fluent-plugin-dynamodb-streams 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/fluent-plugin-dynamodb-streams.gemspec +1 -1
- data/lib/fluent/plugin/in_dynamodb_streams.rb +27 -15
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1e5741297ac53b959692b1f5fb61617797f2b537
|
|
4
|
+
data.tar.gz: f62d4ed375bb57d37562a1808d556f8fce5b233a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 44da5152460a042ef52a3c82908bf97402ac8ab532828df4166b44a3cf0b586fdfab1a69a5be67df9820a4c62efc9615cb83b971fbe3728c18ca27ba7ac0cb88
|
|
7
|
+
data.tar.gz: fd3c4ab09f1d0bdd6dd18e1cdcb14f0fc7c77995251852167055f8dfb16288d68b4cfe15d04237623e073d0e325d9e346a9f9f36d0e2afe6ab9856fdcb21943c
|
data/README.md
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# fluent-plugin-dynamodb-streams
|
|
2
2
|
|
|
3
|
+
[](http://badge.fury.io/rb/fluent-plugin-dynamodb-streams)
|
|
3
4
|
[](https://travis-ci.org/takus/fluent-plugin-dynamodb-streams)
|
|
4
5
|
[](https://codeclimate.com/github/takus/fluent-plugin-dynamodb-streams)
|
|
5
6
|
[](https://codeclimate.com/github/takus/fluent-plugin-dynamodb-streams/coverage)
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "fluent-plugin-dynamodb-streams"
|
|
7
|
-
spec.version = "0.0.
|
|
7
|
+
spec.version = "0.0.2"
|
|
8
8
|
spec.authors = ["Takumi Sakamoto"]
|
|
9
9
|
spec.email = ["takumi.saka@gmail.com"]
|
|
10
10
|
spec.summary = %q{Amazon DynamoDB Streams input plugin for Fluentd}
|
|
@@ -51,6 +51,8 @@ module Fluent
|
|
|
51
51
|
options[:endpoint] = @stream_endpoint
|
|
52
52
|
@client = Aws::DynamoDBStreams::Client.new(options)
|
|
53
53
|
|
|
54
|
+
@iterator = {}
|
|
55
|
+
|
|
54
56
|
@running = true
|
|
55
57
|
@thread = Thread.new(&method(:run))
|
|
56
58
|
end
|
|
@@ -73,23 +75,10 @@ module Fluent
|
|
|
73
75
|
next
|
|
74
76
|
end
|
|
75
77
|
|
|
76
|
-
|
|
77
|
-
iterator = @client.get_shard_iterator({
|
|
78
|
-
stream_arn: @stream_arn,
|
|
79
|
-
shard_id: s.shard_id,
|
|
80
|
-
shard_iterator_type: "AFTER_SEQUENCE_NUMBER",
|
|
81
|
-
sequence_number: load_sequence(s.shard_id),
|
|
82
|
-
}).shard_iterator
|
|
83
|
-
else
|
|
84
|
-
iterator = @client.get_shard_iterator({
|
|
85
|
-
stream_arn: @stream_arn,
|
|
86
|
-
shard_id: s.shard_id,
|
|
87
|
-
shard_iterator_type: "TRIM_HORIZON",
|
|
88
|
-
}).shard_iterator
|
|
89
|
-
end
|
|
78
|
+
set_iterator(s.shard_id) unless @iterator.key? s.shard_id
|
|
90
79
|
|
|
91
80
|
resp = @client.get_records({
|
|
92
|
-
shard_iterator: iterator,
|
|
81
|
+
shard_iterator: @iterator[s.shard_id],
|
|
93
82
|
limit: @fetch_size,
|
|
94
83
|
})
|
|
95
84
|
|
|
@@ -101,10 +90,33 @@ module Fluent
|
|
|
101
90
|
end
|
|
102
91
|
save_sequence(s.shard_id, r.dynamodb.sequence_number)
|
|
103
92
|
end
|
|
93
|
+
|
|
94
|
+
if resp.next_shard_iterator
|
|
95
|
+
@iterator[s.shard_id] = resp.next_shard_iterator
|
|
96
|
+
else
|
|
97
|
+
@iterator.delete s.shard_id
|
|
98
|
+
end
|
|
104
99
|
end
|
|
105
100
|
end
|
|
106
101
|
end
|
|
107
102
|
|
|
103
|
+
def set_iterator(shard_id)
|
|
104
|
+
if load_sequence(shard_id)
|
|
105
|
+
@iterator[shard_id] = @client.get_shard_iterator({
|
|
106
|
+
stream_arn: @stream_arn,
|
|
107
|
+
shard_id: shard_id,
|
|
108
|
+
shard_iterator_type: "AFTER_SEQUENCE_NUMBER",
|
|
109
|
+
sequence_number: load_sequence(shard_id),
|
|
110
|
+
}).shard_iterator
|
|
111
|
+
else
|
|
112
|
+
@iterator[shard_id] = @client.get_shard_iterator({
|
|
113
|
+
stream_arn: @stream_arn,
|
|
114
|
+
shard_id: shard_id,
|
|
115
|
+
shard_iterator_type: "TRIM_HORIZON",
|
|
116
|
+
}).shard_iterator
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
108
120
|
def load_sequence(shard_id)
|
|
109
121
|
if @pos_file
|
|
110
122
|
return nil unless File.exist?("#{@pos_file}.#{shard_id}")
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-dynamodb-streams
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Takumi Sakamoto
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-
|
|
11
|
+
date: 2015-11-06 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|