fluent-plugin-dynamodb 0.1.6 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +8 -0
- data/README.md +15 -2
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_dynamodb.rb +16 -10
- metadata +53 -45
data/ChangeLog
CHANGED
data/README.md
CHANGED
@@ -75,10 +75,24 @@ For example if you read apache's access log via fluentd, structure of the table
|
|
75
75
|
</tr>
|
76
76
|
</table>
|
77
77
|
|
78
|
-
Item can be retrieved by the key, but fluent-plugin-dynamo
|
78
|
+
Item can be retrieved by the key, but fluent-plugin-dynamo uses UUID as a primary key.
|
79
79
|
There is no simple way to retrieve logs you want.
|
80
80
|
By the way, you can write scan-filter with AWS SDK like [this](https://gist.github.com/2906291), but Hive on EMR is the best practice I think.
|
81
81
|
|
82
|
+
###multiprocessing
|
83
|
+
|
84
|
+
If you need high throughput and if you have much provisioned throughput and abudant buffer, you can setup multiprocessing. fluent-plugin-dynamo inherits **DetachMultiProcessMixin**, so you can launch 6 processes as follows.
|
85
|
+
|
86
|
+
<match dynamodb.**>
|
87
|
+
type dynamodb
|
88
|
+
aws_key_id AWS_ACCESS_KEY
|
89
|
+
aws_sec_key AWS_SECRET_ACCESS_KEY
|
90
|
+
proxy_uri http://user:password@192.168.0.250:3128/
|
91
|
+
detach_process 6
|
92
|
+
dynamo_db_endpoint dynamodb.ap-northeast-1.amazonaws.com
|
93
|
+
dynamo_db_table access_log
|
94
|
+
</match>
|
95
|
+
|
82
96
|
###multi-region redundancy
|
83
97
|
|
84
98
|
As you know fluentd has **copy** output plugin.
|
@@ -106,7 +120,6 @@ So you can easily setup multi-region redundancy as follows.
|
|
106
120
|
|
107
121
|
* auto-create table
|
108
122
|
* tag_mapped
|
109
|
-
* Multiprocessing
|
110
123
|
|
111
124
|
##Copyright
|
112
125
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.7
|
@@ -5,6 +5,8 @@ module Fluent
|
|
5
5
|
class DynamoDBOutput < Fluent::BufferedOutput
|
6
6
|
Fluent::Plugin.register_output('dynamodb', self)
|
7
7
|
|
8
|
+
include DetachMultiProcessMixin
|
9
|
+
|
8
10
|
BATCHWRITE_ITEM_LIMIT = 25
|
9
11
|
BATCHWRITE_CONTENT_SIZE_LIMIT = 1024*1024
|
10
12
|
|
@@ -22,6 +24,7 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
22
24
|
config_param :dynamo_db_table, :string
|
23
25
|
config_param :dynamo_db_endpoint, :string, :default => nil
|
24
26
|
config_param :time_format, :string, :default => nil
|
27
|
+
config_param :detach_process, :integer, :default => 2
|
25
28
|
|
26
29
|
def configure(conf)
|
27
30
|
super
|
@@ -30,7 +33,6 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
30
33
|
end
|
31
34
|
|
32
35
|
def start
|
33
|
-
super
|
34
36
|
options = {
|
35
37
|
:access_key_id => @aws_key_id,
|
36
38
|
:secret_access_key => @aws_sec_key,
|
@@ -38,15 +40,19 @@ class DynamoDBOutput < Fluent::BufferedOutput
|
|
38
40
|
}
|
39
41
|
options[:proxy_uri] = @proxy_uri if @proxy_uri
|
40
42
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
43
|
+
detach_multi_process do
|
44
|
+
super
|
45
|
+
|
46
|
+
begin
|
47
|
+
restart_session(options)
|
48
|
+
valid_table(@dynamo_db_table)
|
49
|
+
rescue ConfigError => e
|
50
|
+
$log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
|
51
|
+
exit!
|
52
|
+
rescue Exception => e
|
53
|
+
$log.fatal "UnknownError: '#{e}'"
|
54
|
+
exit!
|
55
|
+
end
|
50
56
|
end
|
51
57
|
end
|
52
58
|
|
metadata
CHANGED
@@ -1,66 +1,70 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-dynamodb
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.6
|
3
|
+
version: !ruby/object:Gem::Version
|
5
4
|
prerelease:
|
5
|
+
version: 0.1.7
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Takashi Matsuno
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
|
13
|
+
date: 2012-06-17 00:00:00 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
15
16
|
name: fluentd
|
16
|
-
|
17
|
+
prerelease: false
|
18
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
17
19
|
none: false
|
18
|
-
requirements:
|
20
|
+
requirements:
|
19
21
|
- - ~>
|
20
|
-
- !ruby/object:Gem::Version
|
22
|
+
- !ruby/object:Gem::Version
|
21
23
|
version: 0.10.0
|
22
24
|
type: :runtime
|
23
|
-
|
24
|
-
|
25
|
-
- !ruby/object:Gem::Dependency
|
25
|
+
version_requirements: *id001
|
26
|
+
- !ruby/object:Gem::Dependency
|
26
27
|
name: aws-sdk
|
27
|
-
|
28
|
+
prerelease: false
|
29
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
28
30
|
none: false
|
29
|
-
requirements:
|
31
|
+
requirements:
|
30
32
|
- - ~>
|
31
|
-
- !ruby/object:Gem::Version
|
33
|
+
- !ruby/object:Gem::Version
|
32
34
|
version: 1.5.2
|
33
35
|
type: :runtime
|
34
|
-
|
35
|
-
|
36
|
-
- !ruby/object:Gem::Dependency
|
36
|
+
version_requirements: *id002
|
37
|
+
- !ruby/object:Gem::Dependency
|
37
38
|
name: uuidtools
|
38
|
-
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
39
41
|
none: false
|
40
|
-
requirements:
|
42
|
+
requirements:
|
41
43
|
- - ~>
|
42
|
-
- !ruby/object:Gem::Version
|
44
|
+
- !ruby/object:Gem::Version
|
43
45
|
version: 2.1.0
|
44
46
|
type: :runtime
|
45
|
-
|
46
|
-
|
47
|
-
- !ruby/object:Gem::Dependency
|
47
|
+
version_requirements: *id003
|
48
|
+
- !ruby/object:Gem::Dependency
|
48
49
|
name: rake
|
49
|
-
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
50
52
|
none: false
|
51
|
-
requirements:
|
52
|
-
- -
|
53
|
-
- !ruby/object:Gem::Version
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
54
56
|
version: 0.9.2
|
55
57
|
type: :development
|
56
|
-
|
57
|
-
version_requirements: *16492000
|
58
|
+
version_requirements: *id004
|
58
59
|
description: Amazon DynamoDB output plugin for Fluent event collector
|
59
60
|
email: g0n5uk3@gmail.com
|
60
61
|
executables: []
|
62
|
+
|
61
63
|
extensions: []
|
64
|
+
|
62
65
|
extra_rdoc_files: []
|
63
|
-
|
66
|
+
|
67
|
+
files:
|
64
68
|
- AUTHORS
|
65
69
|
- ChangeLog
|
66
70
|
- Gemfile
|
@@ -72,26 +76,30 @@ files:
|
|
72
76
|
- test/out_dynamodb.rb
|
73
77
|
homepage: https://github.com/fluent/fluent-plugin-dynamodb
|
74
78
|
licenses: []
|
79
|
+
|
75
80
|
post_install_message:
|
76
81
|
rdoc_options: []
|
77
|
-
|
82
|
+
|
83
|
+
require_paths:
|
78
84
|
- lib
|
79
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
80
86
|
none: false
|
81
|
-
requirements:
|
82
|
-
- -
|
83
|
-
- !ruby/object:Gem::Version
|
84
|
-
version:
|
85
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: "0"
|
91
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
86
92
|
none: false
|
87
|
-
requirements:
|
88
|
-
- -
|
89
|
-
- !ruby/object:Gem::Version
|
90
|
-
version:
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: "0"
|
91
97
|
requirements: []
|
98
|
+
|
92
99
|
rubyforge_project:
|
93
|
-
rubygems_version: 1.8.
|
100
|
+
rubygems_version: 1.8.10
|
94
101
|
signing_key:
|
95
102
|
specification_version: 3
|
96
103
|
summary: Amazon DynamoDB output plugin for Fluent event collector
|
97
|
-
test_files:
|
104
|
+
test_files:
|
105
|
+
- test/out_dynamodb.rb
|