fluent-plugin-dynamodb 0.1.12 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77192a0f199ba2601b2380899db1cc7a22d2e3d3
-  data.tar.gz: 0ff2cda375bf297f48e8bb4f516ec5fa42f0fd49
+  metadata.gz: 6dfdedc9ebf1e4d94fa6061553cfe31279489b03
+  data.tar.gz: c5f9b73d35c3f2beb45f0736e37862a97a39f522
 SHA512:
-  metadata.gz: 9f2438eacc665f6864982a5ff837b296e41dfd014880c73712d87bdac23b0f3d12ebe1b691bf0d538ad085cd1034c2e3c81cbc4f84ac0f4d5bfeb32c1bacdf10
-  data.tar.gz: cc409b173af0ce73517413b8b771e82c26d213c5da18f73a424b1215ba17d614c80e5e1249e310403d52975d27ed9bfd7620d472e725f8958740a09adaec7aab
+  metadata.gz: 2a8f774500e08a187d68d41d055f693bceab0448d9751099d35d7b5c6900d65d0f424815d9f7ee603bc012f85341066db7ab8afa10f5167b6631fe9072d6af0b
+  data.tar.gz: 5a8d97c7e0f3d35d07dde216f1a681429e901407804f98717d80baf7439d02633bf57b29a4cf2dcd93a67414ad25440da44b93e580b30c8c30b9dde070ebb217

data/README.md

@@ -17,7 +17,7 @@ Specify table name, hash attribute name and throughput as you like. fluent-plugi
 ### Fluentd
 
     <match dynamodb.**>
-      type dynamodb
+      @type dynamodb
       aws_key_id AWS_ACCESS_KEY
       aws_sec_key AWS_SECRET_ACCESS_KEY
       proxy_uri http://user:password@192.168.0.250:3128/
@@ -83,7 +83,7 @@ By the way, you can write scan-filter with AWS SDK like [this](https://gist.gith
 If you need high throughput and if you have much provisioned throughput and abudant buffer, you can setup multiprocessing. fluent-plugin-dynamo inherits **DetachMultiProcessMixin**, so you can launch 6 processes as follows.
 
     <match dynamodb.**>
-      type dynamodb
+      @type dynamodb
       aws_key_id AWS_ACCESS_KEY
       aws_sec_key AWS_SECRET_ACCESS_KEY
       proxy_uri http://user:password@192.168.0.250:3128/
@@ -98,16 +98,16 @@ As you know fluentd has **copy** output plugin.
 So you can easily setup multi-region redundancy as follows.
 
     <match dynamo.**>
-      type copy
+      @type copy
       <store>
-        type dynamodb
+        @type dynamodb
         aws_key_id AWS_ACCESS_KEY
         aws_sec_key AWS_SECRET_ACCESS_KEY
         dynamo_db_table test
         dynamo_db_endpoint dynamodb.ap-northeast-1.amazonaws.com
       </store>
       <store>
-        type dynamodb
+        @type dynamodb
         aws_key_id AWS_ACCESS_KEY
         aws_sec_key AWS_SECRET_ACCESS_KEY
         dynamo_db_table test

data/VERSION

@@ -1 +1 @@
-0.1.12
+0.2.0

data/fluent-plugin-dynamodb.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", [">= 0.10.0", "< 2"]
-  gem.add_dependency "aws-sdk-v1", ">= 1.5.2"
+  gem.add_dependency "aws-sdk", [">= 2.3.22", "< 3"]
   gem.add_dependency "uuidtools", "~> 2.1.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.1.0"

data/lib/fluent/plugin/out_dynamodb.rb

@@ -5,6 +5,11 @@ module Fluent
 class DynamoDBOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('dynamodb', self)
 
+  # To support log_level option implemented by Fluentd v0.10.43
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
   include DetachMultiProcessMixin
 
   BATCHWRITE_ITEM_LIMIT = 25
@@ -12,7 +17,7 @@ class DynamoDBOutput < Fluent::BufferedOutput
 
   def initialize
     super
-    require 'aws-sdk-v1'
+    require 'aws-sdk'
     require 'msgpack'
     require 'time'
     require 'uuidtools'
@@ -21,6 +26,7 @@ class DynamoDBOutput < Fluent::BufferedOutput
   config_param :aws_key_id, :string, :default => nil, :secret => true
   config_param :aws_sec_key, :string, :default => nil, :secret => true
   config_param :proxy_uri, :string, :default => nil
+  config_param :dynamo_db_region, :string, default: ENV["AWS_REGION"] || "us-east-1"
   config_param :dynamo_db_table, :string
   config_param :dynamo_db_endpoint, :string, :default => nil
   config_param :time_format, :string, :default => nil
@@ -38,7 +44,8 @@ class DynamoDBOutput < Fluent::BufferedOutput
       options[:access_key_id] = @aws_key_id
       options[:secret_access_key] = @aws_sec_key
     end
-    options[:dynamo_db_endpoint] = @dynamo_db_endpoint
+    options[:region] = @dynamo_db_region if @dynamo_db_region
+    options[:endpoint] = @dynamo_db_endpoint
     options[:proxy_uri] = @proxy_uri if @proxy_uri
 
     detach_multi_process do
@@ -48,48 +55,48 @@ class DynamoDBOutput < Fluent::BufferedOutput
         restart_session(options)
         valid_table(@dynamo_db_table)
       rescue ConfigError => e
-        $log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
+        log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
         exit!
       rescue Exception => e
-        $log.fatal "UnknownError: '#{e}'"
+        log.fatal "UnknownError: '#{e}'"
         exit!
       end
     end
   end
 
   def restart_session(options)
-    config = AWS.config(options)
-    @batch = AWS::DynamoDB::BatchWrite.new(config)
-    @dynamo_db = AWS::DynamoDB.new(options)
+    @dynamo_db = Aws::DynamoDB::Client.new(options)
+    @resource = Aws::DynamoDB::Resource.new(client: @dynamo_db)
+
   end
 
   def valid_table(table_name)
-    table = @dynamo_db.tables[table_name]
-    table.load_schema
-    @hash_key = table.hash_key
-    @range_key = table.range_key unless table.simple_key?
+    table = @resource.table(table_name)
+    @hash_key = table.key_schema.select{|e| e.key_type == "HASH" }.first
+    range_key_candidate = table.key_schema.select{|e| e.key_type == "RANGE" }
+    @range_key = range_key_candidate.first if range_key_candidate
   end
 
   def match_type!(key, record)
-    if key.type == :number
-      potential_value = record[key.name].to_i
+    if key.key_type == "NUMBER"
+      potential_value = record[key.attribute_name].to_i
       if potential_value == 0
-        $log.fatal "Failed attempt to cast hash_key to Integer."
+        log.fatal "Failed attempt to cast hash_key to Integer."
       end
-      record[key.name] = potential_value
+      record[key.attribute_name] = potential_value
     end
   end
 
   def format(tag, time, record)
-    if !record.key?(@hash_key.name)
-      record[@hash_key.name] = UUIDTools::UUID.timestamp_create.to_s
+    if !record.key?(@hash_key.attribute_name)
+      record[@hash_key.attribute_name] = UUIDTools::UUID.timestamp_create.to_s
     end
     match_type!(@hash_key, record)
 
     formatted_time = @timef.format(time)
     if @range_key
-      if !record.key?(@range_key.name)
-        record[@range_key.name] = formatted_time
+      if !record.key?(@range_key.attribute_name)
+        record[@range_key.attribute_name] = formatted_time
       end
       match_type!(@range_key, record)
     end
@@ -102,7 +109,11 @@ class DynamoDBOutput < Fluent::BufferedOutput
     batch_size = 0
     batch_records = []
     chunk.msgpack_each {|record|
-      batch_records << record
+      batch_records << {
+        put_request: {
+          item: record
+        }
+      }
       batch_size += record.to_json.length # FIXME: heuristic
       if batch_records.size >= BATCHWRITE_ITEM_LIMIT || batch_size >= BATCHWRITE_CONTENT_SIZE_LIMIT
         batch_put_records(batch_records)
@@ -116,12 +127,10 @@ class DynamoDBOutput < Fluent::BufferedOutput
   end
 
   def batch_put_records(records)
-    @batch.put(@dynamo_db_table, records)
-    @batch.process!
+    @dynamo_db.batch_write_item(request_items: { @dynamo_db_table => records })
   end
 
 end
 
 
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dynamodb
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.2.0
 platform: ruby
 authors:
 - Takashi Matsuno
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-03 00:00:00.000000000 Z
+date: 2017-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -31,19 +31,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-v1
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: 2.3.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: 2.3.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: uuidtools
   requirement: !ruby/object:Gem::Requirement