fluent-plugin-dynamodb 0.1.12 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -5
- data/VERSION +1 -1
- data/fluent-plugin-dynamodb.gemspec +1 -1
- data/lib/fluent/plugin/out_dynamodb.rb +32 -23
- metadata +11 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6dfdedc9ebf1e4d94fa6061553cfe31279489b03
|
4
|
+
data.tar.gz: c5f9b73d35c3f2beb45f0736e37862a97a39f522
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2a8f774500e08a187d68d41d055f693bceab0448d9751099d35d7b5c6900d65d0f424815d9f7ee603bc012f85341066db7ab8afa10f5167b6631fe9072d6af0b
|
7
|
+
data.tar.gz: 5a8d97c7e0f3d35d07dde216f1a681429e901407804f98717d80baf7439d02633bf57b29a4cf2dcd93a67414ad25440da44b93e580b30c8c30b9dde070ebb217
|
data/README.md
CHANGED
@@ -17,7 +17,7 @@ Specify table name, hash attribute name and throughput as you like. fluent-plugi
|
|
17
17
|
### Fluentd
|
18
18
|
|
19
19
|
<match dynamodb.**>
|
20
|
-
type dynamodb
|
20
|
+
@type dynamodb
|
21
21
|
aws_key_id AWS_ACCESS_KEY
|
22
22
|
aws_sec_key AWS_SECRET_ACCESS_KEY
|
23
23
|
proxy_uri http://user:password@192.168.0.250:3128/
|
@@ -83,7 +83,7 @@ By the way, you can write scan-filter with AWS SDK like [this](https://gist.gith
|
|
83
83
|
If you need high throughput and if you have much provisioned throughput and abudant buffer, you can setup multiprocessing. fluent-plugin-dynamo inherits **DetachMultiProcessMixin**, so you can launch 6 processes as follows.
|
84
84
|
|
85
85
|
<match dynamodb.**>
|
86
|
-
type dynamodb
|
86
|
+
@type dynamodb
|
87
87
|
aws_key_id AWS_ACCESS_KEY
|
88
88
|
aws_sec_key AWS_SECRET_ACCESS_KEY
|
89
89
|
proxy_uri http://user:password@192.168.0.250:3128/
|
@@ -98,16 +98,16 @@ As you know fluentd has **copy** output plugin.
|
|
98
98
|
So you can easily setup multi-region redundancy as follows.
|
99
99
|
|
100
100
|
<match dynamo.**>
|
101
|
-
type copy
|
101
|
+
@type copy
|
102
102
|
<store>
|
103
|
-
type dynamodb
|
103
|
+
@type dynamodb
|
104
104
|
aws_key_id AWS_ACCESS_KEY
|
105
105
|
aws_sec_key AWS_SECRET_ACCESS_KEY
|
106
106
|
dynamo_db_table test
|
107
107
|
dynamo_db_endpoint dynamodb.ap-northeast-1.amazonaws.com
|
108
108
|
</store>
|
109
109
|
<store>
|
110
|
-
type dynamodb
|
110
|
+
@type dynamodb
|
111
111
|
aws_key_id AWS_ACCESS_KEY
|
112
112
|
aws_sec_key AWS_SECRET_ACCESS_KEY
|
113
113
|
dynamo_db_table test
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
|
|
18
18
|
gem.require_paths = ['lib']
|
19
19
|
|
20
20
|
gem.add_dependency "fluentd", [">= 0.10.0", "< 2"]
|
21
|
-
gem.add_dependency "aws-sdk
|
21
|
+
gem.add_dependency "aws-sdk", [">= 2.3.22", "< 3"]
|
22
22
|
gem.add_dependency "uuidtools", "~> 2.1.0"
|
23
23
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
24
24
|
gem.add_development_dependency "test-unit", ">= 3.1.0"
|
@@ -5,6 +5,11 @@ module Fluent
|
|
5
5
|
class DynamoDBOutput < Fluent::BufferedOutput
|
6
6
|
Fluent::Plugin.register_output('dynamodb', self)
|
7
7
|
|
8
|
+
# To support log_level option implemented by Fluentd v0.10.43
|
9
|
+
unless method_defined?(:log)
|
10
|
+
define_method("log") { $log }
|
11
|
+
end
|
12
|
+
|
8
13
|
include DetachMultiProcessMixin
|
9
14
|
|
10
15
|
BATCHWRITE_ITEM_LIMIT = 25
|
@@ -12,7 +17,7 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
12
17
|
|
13
18
|
def initialize
|
14
19
|
super
|
15
|
-
require 'aws-sdk
|
20
|
+
require 'aws-sdk'
|
16
21
|
require 'msgpack'
|
17
22
|
require 'time'
|
18
23
|
require 'uuidtools'
|
@@ -21,6 +26,7 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
21
26
|
config_param :aws_key_id, :string, :default => nil, :secret => true
|
22
27
|
config_param :aws_sec_key, :string, :default => nil, :secret => true
|
23
28
|
config_param :proxy_uri, :string, :default => nil
|
29
|
+
config_param :dynamo_db_region, :string, default: ENV["AWS_REGION"] || "us-east-1"
|
24
30
|
config_param :dynamo_db_table, :string
|
25
31
|
config_param :dynamo_db_endpoint, :string, :default => nil
|
26
32
|
config_param :time_format, :string, :default => nil
|
@@ -38,7 +44,8 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
38
44
|
options[:access_key_id] = @aws_key_id
|
39
45
|
options[:secret_access_key] = @aws_sec_key
|
40
46
|
end
|
41
|
-
options[:
|
47
|
+
options[:region] = @dynamo_db_region if @dynamo_db_region
|
48
|
+
options[:endpoint] = @dynamo_db_endpoint
|
42
49
|
options[:proxy_uri] = @proxy_uri if @proxy_uri
|
43
50
|
|
44
51
|
detach_multi_process do
|
@@ -48,48 +55,48 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
48
55
|
restart_session(options)
|
49
56
|
valid_table(@dynamo_db_table)
|
50
57
|
rescue ConfigError => e
|
51
|
-
|
58
|
+
log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
|
52
59
|
exit!
|
53
60
|
rescue Exception => e
|
54
|
-
|
61
|
+
log.fatal "UnknownError: '#{e}'"
|
55
62
|
exit!
|
56
63
|
end
|
57
64
|
end
|
58
65
|
end
|
59
66
|
|
60
67
|
def restart_session(options)
|
61
|
-
|
62
|
-
@
|
63
|
-
|
68
|
+
@dynamo_db = Aws::DynamoDB::Client.new(options)
|
69
|
+
@resource = Aws::DynamoDB::Resource.new(client: @dynamo_db)
|
70
|
+
|
64
71
|
end
|
65
72
|
|
66
73
|
def valid_table(table_name)
|
67
|
-
table = @
|
68
|
-
table.
|
69
|
-
|
70
|
-
@range_key =
|
74
|
+
table = @resource.table(table_name)
|
75
|
+
@hash_key = table.key_schema.select{|e| e.key_type == "HASH" }.first
|
76
|
+
range_key_candidate = table.key_schema.select{|e| e.key_type == "RANGE" }
|
77
|
+
@range_key = range_key_candidate.first if range_key_candidate
|
71
78
|
end
|
72
79
|
|
73
80
|
def match_type!(key, record)
|
74
|
-
if key.
|
75
|
-
potential_value = record[key.
|
81
|
+
if key.key_type == "NUMBER"
|
82
|
+
potential_value = record[key.attribute_name].to_i
|
76
83
|
if potential_value == 0
|
77
|
-
|
84
|
+
log.fatal "Failed attempt to cast hash_key to Integer."
|
78
85
|
end
|
79
|
-
record[key.
|
86
|
+
record[key.attribute_name] = potential_value
|
80
87
|
end
|
81
88
|
end
|
82
89
|
|
83
90
|
def format(tag, time, record)
|
84
|
-
if !record.key?(@hash_key.
|
85
|
-
record[@hash_key.
|
91
|
+
if !record.key?(@hash_key.attribute_name)
|
92
|
+
record[@hash_key.attribute_name] = UUIDTools::UUID.timestamp_create.to_s
|
86
93
|
end
|
87
94
|
match_type!(@hash_key, record)
|
88
95
|
|
89
96
|
formatted_time = @timef.format(time)
|
90
97
|
if @range_key
|
91
|
-
if !record.key?(@range_key.
|
92
|
-
record[@range_key.
|
98
|
+
if !record.key?(@range_key.attribute_name)
|
99
|
+
record[@range_key.attribute_name] = formatted_time
|
93
100
|
end
|
94
101
|
match_type!(@range_key, record)
|
95
102
|
end
|
@@ -102,7 +109,11 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
102
109
|
batch_size = 0
|
103
110
|
batch_records = []
|
104
111
|
chunk.msgpack_each {|record|
|
105
|
-
batch_records <<
|
112
|
+
batch_records << {
|
113
|
+
put_request: {
|
114
|
+
item: record
|
115
|
+
}
|
116
|
+
}
|
106
117
|
batch_size += record.to_json.length # FIXME: heuristic
|
107
118
|
if batch_records.size >= BATCHWRITE_ITEM_LIMIT || batch_size >= BATCHWRITE_CONTENT_SIZE_LIMIT
|
108
119
|
batch_put_records(batch_records)
|
@@ -116,12 +127,10 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
116
127
|
end
|
117
128
|
|
118
129
|
def batch_put_records(records)
|
119
|
-
@
|
120
|
-
@batch.process!
|
130
|
+
@dynamo_db.batch_write_item(request_items: { @dynamo_db_table => records })
|
121
131
|
end
|
122
132
|
|
123
133
|
end
|
124
134
|
|
125
135
|
|
126
136
|
end
|
127
|
-
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-dynamodb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takashi Matsuno
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-05-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -31,19 +31,25 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name: aws-sdk
|
34
|
+
name: aws-sdk
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
37
|
- - ">="
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
39
|
+
version: 2.3.22
|
40
|
+
- - "<"
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '3'
|
40
43
|
type: :runtime
|
41
44
|
prerelease: false
|
42
45
|
version_requirements: !ruby/object:Gem::Requirement
|
43
46
|
requirements:
|
44
47
|
- - ">="
|
45
48
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
49
|
+
version: 2.3.22
|
50
|
+
- - "<"
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: '3'
|
47
53
|
- !ruby/object:Gem::Dependency
|
48
54
|
name: uuidtools
|
49
55
|
requirement: !ruby/object:Gem::Requirement
|