fluent-plugin-dynamodb 0.1.6 → 0.1.7

data/ChangeLog

@@ -1,3 +1,11 @@
+Release 0.1.7 - 2012/06/17
+
+  * Inherits DetachMultiProcessMixin
+
+Release 0.1.6 - 2012/06/12
+
+ * Optimized write(chunk) method not to collect all records in memory
+
 Release 0.1.5 - 2012/06/10
 
   * First release

data/README.md

@@ -75,10 +75,24 @@ For example if you read apache's access log via fluentd, structure of the table
   </tr>
 </table>
 
-Item can be retrieved by the key, but fluent-plugin-dynamo using UUID as a primary key.
+Item can be retrieved by the key, but fluent-plugin-dynamo uses UUID as a primary key.
 There is no simple way to retrieve logs you want.
 By the way, you can write scan-filter with AWS SDK like [this](https://gist.github.com/2906291), but Hive on EMR is the best practice I think.
 
+###multiprocessing
+
+If you need high throughput and if you have much provisioned throughput and abudant buffer, you can setup multiprocessing. fluent-plugin-dynamo inherits **DetachMultiProcessMixin**, so you can launch 6 processes as follows.
+
+    <match dynamodb.**>
+      type dynamodb
+      aws_key_id AWS_ACCESS_KEY
+      aws_sec_key AWS_SECRET_ACCESS_KEY
+      proxy_uri http://user:password@192.168.0.250:3128/
+      detach_process 6
+      dynamo_db_endpoint dynamodb.ap-northeast-1.amazonaws.com
+      dynamo_db_table access_log
+    </match>
+
 ###multi-region redundancy
 
 As you know fluentd has **copy** output plugin.
@@ -106,7 +120,6 @@ So you can easily setup multi-region redundancy as follows.
 
  * auto-create table
  * tag_mapped
- * Multiprocessing 
 
 ##Copyright
 

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.1.7

data/lib/fluent/plugin/out_dynamodb.rb

@@ -5,6 +5,8 @@ module Fluent
 class DynamoDBOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('dynamodb', self)
 
+  include DetachMultiProcessMixin
+
   BATCHWRITE_ITEM_LIMIT = 25
   BATCHWRITE_CONTENT_SIZE_LIMIT = 1024*1024
 
@@ -22,6 +24,7 @@ class DynamoDBOutput < Fluent::BufferedOutput
   config_param :dynamo_db_table, :string
   config_param :dynamo_db_endpoint, :string, :default => nil
   config_param :time_format, :string, :default => nil
+  config_param :detach_process, :integer, :default => 2
 
   def configure(conf)
     super
@@ -30,7 +33,6 @@ class DynamoDBOutput < Fluent::BufferedOutput
   end
 
   def start
-    super
     options = {
       :access_key_id      => @aws_key_id,
       :secret_access_key  => @aws_sec_key,
@@ -38,15 +40,19 @@ class DynamoDBOutput < Fluent::BufferedOutput
     }
     options[:proxy_uri] = @proxy_uri if @proxy_uri
 
-    begin
-      restart_session(options)
-      valid_table(@dynamo_db_table)
-    rescue ConfigError => e
-      $log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
-      exit!
-    rescue Exception => e
-      $log.fatal "UnknownError: '#{e}'"
-      exit!
+    detach_multi_process do
+      super
+
+      begin
+        restart_session(options)
+        valid_table(@dynamo_db_table)
+      rescue ConfigError => e
+        $log.fatal "ConfigError: Please check your configuration, then restart fluentd. '#{e}'"
+        exit!
+      rescue Exception => e
+        $log.fatal "UnknownError: '#{e}'"
+        exit!
+      end
     end
   end
 

metadata

@@ -1,66 +1,70 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: fluent-plugin-dynamodb
-version: !ruby/object:Gem::Version
-  version: 0.1.6
+version: !ruby/object:Gem::Version 
   prerelease: 
+  version: 0.1.7
 platform: ruby
-authors:
+authors: 
 - Takashi Matsuno
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-12 00:00:00.000000000Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2012-06-17 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: fluentd
-  requirement: &16493400 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 0.10.0
   type: :runtime
-  prerelease: false
-  version_requirements: *16493400
-- !ruby/object:Gem::Dependency
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
   name: aws-sdk
-  requirement: &16492920 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 1.5.2
   type: :runtime
-  prerelease: false
-  version_requirements: *16492920
-- !ruby/object:Gem::Dependency
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
   name: uuidtools
-  requirement: &16492460 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
         version: 2.1.0
   type: :runtime
-  prerelease: false
-  version_requirements: *16492460
-- !ruby/object:Gem::Dependency
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
   name: rake
-  requirement: &16492000 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
         version: 0.9.2
   type: :development
-  prerelease: false
-  version_requirements: *16492000
+  version_requirements: *id004
 description: Amazon DynamoDB output plugin for Fluent event collector
 email: g0n5uk3@gmail.com
 executables: []
+
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - AUTHORS
 - ChangeLog
 - Gemfile
@@ -72,26 +76,30 @@ files:
 - test/out_dynamodb.rb
 homepage: https://github.com/fluent/fluent-plugin-dynamodb
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 1.8.17
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Amazon DynamoDB output plugin for Fluent event collector
-test_files: []
+test_files: 
+- test/out_dynamodb.rb