fluent-plugin-dynamodb 0.1.12 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 77192a0f199ba2601b2380899db1cc7a22d2e3d3
4
- data.tar.gz: 0ff2cda375bf297f48e8bb4f516ec5fa42f0fd49
3
+ metadata.gz: 6dfdedc9ebf1e4d94fa6061553cfe31279489b03
4
+ data.tar.gz: c5f9b73d35c3f2beb45f0736e37862a97a39f522
5
5
  SHA512:
6
- metadata.gz: 9f2438eacc665f6864982a5ff837b296e41dfd014880c73712d87bdac23b0f3d12ebe1b691bf0d538ad085cd1034c2e3c81cbc4f84ac0f4d5bfeb32c1bacdf10
7
- data.tar.gz: cc409b173af0ce73517413b8b771e82c26d213c5da18f73a424b1215ba17d614c80e5e1249e310403d52975d27ed9bfd7620d472e725f8958740a09adaec7aab
6
+ metadata.gz: 2a8f774500e08a187d68d41d055f693bceab0448d9751099d35d7b5c6900d65d0f424815d9f7ee603bc012f85341066db7ab8afa10f5167b6631fe9072d6af0b
7
+ data.tar.gz: 5a8d97c7e0f3d35d07dde216f1a681429e901407804f98717d80baf7439d02633bf57b29a4cf2dcd93a67414ad25440da44b93e580b30c8c30b9dde070ebb217
data/README.md CHANGED
@@ -17,7 +17,7 @@ Specify table name, hash attribute name and throughput as you like. fluent-plugi
17
17
  ### Fluentd
18
18
 
19
19
  <match dynamodb.**>
20
- type dynamodb
20
+ @type dynamodb
21
21
  aws_key_id AWS_ACCESS_KEY
22
22
  aws_sec_key AWS_SECRET_ACCESS_KEY
23
23
  proxy_uri http://user:password@192.168.0.250:3128/
@@ -83,7 +83,7 @@ By the way, you can write scan-filter with AWS SDK like [this](https://gist.gith
83
83
  If you need high throughput and if you have much provisioned throughput and abudant buffer, you can setup multiprocessing. fluent-plugin-dynamo inherits **DetachMultiProcessMixin**, so you can launch 6 processes as follows.
84
84
 
85
85
  <match dynamodb.**>
86
- type dynamodb
86
+ @type dynamodb
87
87
  aws_key_id AWS_ACCESS_KEY
88
88
  aws_sec_key AWS_SECRET_ACCESS_KEY
89
89
  proxy_uri http://user:password@192.168.0.250:3128/
@@ -98,16 +98,16 @@ As you know fluentd has **copy** output plugin.
98
98
  So you can easily setup multi-region redundancy as follows.
99
99
 
100
100
  <match dynamo.**>
101
- type copy
101
+ @type copy
102
102
  <store>
103
- type dynamodb
103
+ @type dynamodb
104
104
  aws_key_id AWS_ACCESS_KEY
105
105
  aws_sec_key AWS_SECRET_ACCESS_KEY
106
106
  dynamo_db_table test
107
107
  dynamo_db_endpoint dynamodb.ap-northeast-1.amazonaws.com
108
108
  </store>
109
109
  <store>
110
- type dynamodb
110
+ @type dynamodb
111
111
  aws_key_id AWS_ACCESS_KEY
112
112
  aws_sec_key AWS_SECRET_ACCESS_KEY
113
113
  dynamo_db_table test
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.12
1
+ 0.2.0
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
18
18
  gem.require_paths = ['lib']
19
19
 
20
20
  gem.add_dependency "fluentd", [">= 0.10.0", "< 2"]
21
- gem.add_dependency "aws-sdk-v1", ">= 1.5.2"
21
+ gem.add_dependency "aws-sdk", [">= 2.3.22", "< 3"]
22
22
  gem.add_dependency "uuidtools", "~> 2.1.0"
23
23
  gem.add_development_dependency "rake", ">= 0.9.2"
24
24
  gem.add_development_dependency "test-unit", ">= 3.1.0"
@@ -5,6 +5,11 @@ module Fluent
5
5
  class DynamoDBOutput < Fluent::BufferedOutput
6
6
  Fluent::Plugin.register_output('dynamodb', self)
7
7
 
8
+ # To support log_level option implemented by Fluentd v0.10.43
9
+ unless method_defined?(:log)
10
+ define_method("log") { $log }
11
+ end
12
+
8
13
  include DetachMultiProcessMixin
9
14
 
10
15
  BATCHWRITE_ITEM_LIMIT = 25
@@ -12,7 +17,7 @@ class DynamoDBOutput < Fluent::BufferedOutput
12
17
 
13
18
  def initialize
14
19
  super
15
- require 'aws-sdk-v1'
20
+ require 'aws-sdk'
16
21
  require 'msgpack'
17
22
  require 'time'
18
23
  require 'uuidtools'
@@ -21,6 +26,7 @@ class DynamoDBOutput < Fluent::BufferedOutput
21
26
  config_param :aws_key_id, :string, :default => nil, :secret => true
22
27
  config_param :aws_sec_key, :string, :default => nil, :secret => true
23
28
  config_param :proxy_uri, :string, :default => nil
29
+ config_param :dynamo_db_region, :string, default: ENV["AWS_REGION"] || "us-east-1"
24
30
  config_param :dynamo_db_table, :string
25
31
  config_param :dynamo_db_endpoint, :string, :default => nil
26
32
  config_param :time_format, :string, :default => nil
@@ -38,7 +44,8 @@ class DynamoDBOutput < Fluent::BufferedOutput
38
44
  options[:access_key_id] = @aws_key_id
39
45
  options[:secret_access_key] = @aws_sec_key
40
46
  end
41
- options[:dynamo_db_endpoint] = @dynamo_db_endpoint
47
+ options[:region] = @dynamo_db_region if @dynamo_db_region
48
+ options[:endpoint] = @dynamo_db_endpoint
42
49
  options[:proxy_uri] = @proxy_uri if @proxy_uri
43
50
 
44
51
  detach_multi_process do
@@ -48,48 +55,48 @@ class DynamoDBOutput < Fluent::BufferedOutput
48
55
  restart_session(options)
49
56
  valid_table(@dynamo_db_table)
50
57
  rescue ConfigError => e
51
- $log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
58
+ log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
52
59
  exit!
53
60
  rescue Exception => e
54
- $log.fatal "UnknownError: '#{e}'"
61
+ log.fatal "UnknownError: '#{e}'"
55
62
  exit!
56
63
  end
57
64
  end
58
65
  end
59
66
 
60
67
  def restart_session(options)
61
- config = AWS.config(options)
62
- @batch = AWS::DynamoDB::BatchWrite.new(config)
63
- @dynamo_db = AWS::DynamoDB.new(options)
68
+ @dynamo_db = Aws::DynamoDB::Client.new(options)
69
+ @resource = Aws::DynamoDB::Resource.new(client: @dynamo_db)
70
+
64
71
  end
65
72
 
66
73
  def valid_table(table_name)
67
- table = @dynamo_db.tables[table_name]
68
- table.load_schema
69
- @hash_key = table.hash_key
70
- @range_key = table.range_key unless table.simple_key?
74
+ table = @resource.table(table_name)
75
+ @hash_key = table.key_schema.select{|e| e.key_type == "HASH" }.first
76
+ range_key_candidate = table.key_schema.select{|e| e.key_type == "RANGE" }
77
+ @range_key = range_key_candidate.first if range_key_candidate
71
78
  end
72
79
 
73
80
  def match_type!(key, record)
74
- if key.type == :number
75
- potential_value = record[key.name].to_i
81
+ if key.key_type == "NUMBER"
82
+ potential_value = record[key.attribute_name].to_i
76
83
  if potential_value == 0
77
- $log.fatal "Failed attempt to cast hash_key to Integer."
84
+ log.fatal "Failed attempt to cast hash_key to Integer."
78
85
  end
79
- record[key.name] = potential_value
86
+ record[key.attribute_name] = potential_value
80
87
  end
81
88
  end
82
89
 
83
90
  def format(tag, time, record)
84
- if !record.key?(@hash_key.name)
85
- record[@hash_key.name] = UUIDTools::UUID.timestamp_create.to_s
91
+ if !record.key?(@hash_key.attribute_name)
92
+ record[@hash_key.attribute_name] = UUIDTools::UUID.timestamp_create.to_s
86
93
  end
87
94
  match_type!(@hash_key, record)
88
95
 
89
96
  formatted_time = @timef.format(time)
90
97
  if @range_key
91
- if !record.key?(@range_key.name)
92
- record[@range_key.name] = formatted_time
98
+ if !record.key?(@range_key.attribute_name)
99
+ record[@range_key.attribute_name] = formatted_time
93
100
  end
94
101
  match_type!(@range_key, record)
95
102
  end
@@ -102,7 +109,11 @@ class DynamoDBOutput < Fluent::BufferedOutput
102
109
  batch_size = 0
103
110
  batch_records = []
104
111
  chunk.msgpack_each {|record|
105
- batch_records << record
112
+ batch_records << {
113
+ put_request: {
114
+ item: record
115
+ }
116
+ }
106
117
  batch_size += record.to_json.length # FIXME: heuristic
107
118
  if batch_records.size >= BATCHWRITE_ITEM_LIMIT || batch_size >= BATCHWRITE_CONTENT_SIZE_LIMIT
108
119
  batch_put_records(batch_records)
@@ -116,12 +127,10 @@ class DynamoDBOutput < Fluent::BufferedOutput
116
127
  end
117
128
 
118
129
  def batch_put_records(records)
119
- @batch.put(@dynamo_db_table, records)
120
- @batch.process!
130
+ @dynamo_db.batch_write_item(request_items: { @dynamo_db_table => records })
121
131
  end
122
132
 
123
133
  end
124
134
 
125
135
 
126
136
  end
127
-
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-dynamodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.12
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takashi Matsuno
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-03 00:00:00.000000000 Z
11
+ date: 2017-05-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -31,19 +31,25 @@ dependencies:
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
33
33
  - !ruby/object:Gem::Dependency
34
- name: aws-sdk-v1
34
+ name: aws-sdk
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
37
  - - ">="
38
38
  - !ruby/object:Gem::Version
39
- version: 1.5.2
39
+ version: 2.3.22
40
+ - - "<"
41
+ - !ruby/object:Gem::Version
42
+ version: '3'
40
43
  type: :runtime
41
44
  prerelease: false
42
45
  version_requirements: !ruby/object:Gem::Requirement
43
46
  requirements:
44
47
  - - ">="
45
48
  - !ruby/object:Gem::Version
46
- version: 1.5.2
49
+ version: 2.3.22
50
+ - - "<"
51
+ - !ruby/object:Gem::Version
52
+ version: '3'
47
53
  - !ruby/object:Gem::Dependency
48
54
  name: uuidtools
49
55
  requirement: !ruby/object:Gem::Requirement