fluent-plugin-dynamodb 0.1.5 → 0.1.6

data/AUTHORS

@@ -0,0 +1,3 @@
+Takashi Matsuno
+Sadayuki Furuhashi
+

data/README.md

@@ -32,6 +32,76 @@ Specify table name, hash attribute name and throughput as you like. fluent-plugi
  * **dynamo\_db\_endpoint (required)** - end point of dynamodb. see  [Regions and Endpoints](http://docs.amazonwebservices.com/general/latest/gr/rande.html#ddb_region)
  * **dynamo\_db\_table (required)** - table name of dynamodb.
 
+##TIPS
+
+###retrieving data
+
+fluent-plugin-dynamo will add **time** attribute and any other attributes of record automatically.
+For example if you read apache's access log via fluentd, structure of the table will have been like this.
+
+<table>
+  <tr>
+    <th>id (Hash Key)</th>
+    <th>time</th>
+    <th>host</th>
+    <th>path</th>
+    <th>method</th>
+    <th>referer</th>
+    <th>code</th>
+    <th>agent</th>
+    <th>size</th>
+  </tr>
+  <tr>
+    <td>"a937f980-b304-11e1-bc96-c82a14fffef2"</td>
+    <td>"2012-06-10T05:26:46Z"</td>
+    <td>"192.168.0.3"</td>
+    <td>"/index.html"</td>
+    <td>"GET"</td>
+    <td>"-"</td>
+    <td>"200"</td>
+    <td>"Mozilla/5.0"</td>
+    <td>"4286"</td>
+  </tr>
+  <tr>
+    <td>"a87fc51e-b308-11e1-ba0f-5855caf50759"</td>
+    <td>"2012-06-10T05:28:23Z"</td>
+    <td>"192.168.0.4"</td>
+    <td>"/sample.html"</td>
+    <td>"GET"</td>
+    <td>"-"</td>
+    <td>"200"</td>
+    <td>"Mozilla/5.0"</td>
+    <td>"8933"</td>
+  </tr>
+</table>
+
+Item can be retrieved by the key, but fluent-plugin-dynamo using UUID as a primary key.
+There is no simple way to retrieve logs you want.
+By the way, you can write scan-filter with AWS SDK like [this](https://gist.github.com/2906291), but Hive on EMR is the best practice I think.
+
+###multi-region redundancy
+
+As you know fluentd has **copy** output plugin.
+So you can easily setup multi-region redundancy as follows.
+
+    <match dynamo.**>
+      type copy
+      <store>
+        type dynamodb
+        aws_key_id AWS_ACCESS_KEY
+        aws_sec_key AWS_SECRET_ACCESS_KEY
+        dynamo_db_table test
+        dynamo_db_endpoint dynamodb.ap-northeast-1.amazonaws.com
+      </store>
+      <store>
+        type dynamodb
+        aws_key_id AWS_ACCESS_KEY
+        aws_sec_key AWS_SECRET_ACCESS_KEY
+        dynamo_db_table test
+        dynamo_db_endpoint dynamodb.ap-southeast-1.amazonaws.com
+      </store>
+    </match>
+
 ##TODO
 
  * auto-create table

data/VERSION

@@ -1 +1 @@
-0.1.5
+0.1.6

data/lib/fluent/plugin/out_dynamodb.rb

@@ -68,31 +68,31 @@ class DynamoDBOutput < Fluent::BufferedOutput
       record[@hash_key_value] = UUIDTools::UUID.timestamp_create.to_s
     end
 
-    [time, record].to_msgpack
+    record['time'] = @timef.format(time)
+    
+    record.to_msgpack
   end
 
   def write(chunk)
-    records = collect_records(chunk)
-    item_num = item_size = 0
-    records.each {|record|
-      item_num += 1
-      item_size += record.to_json.length # FIXME: heuristic
-      if item_num >= BATCHWRITE_ITEM_LIMIT || item_size >= BATCHWRITE_CONTENT_SIZE_LIMIT
-        @batch.process!
-        item_num = item_size = 0
+    batch_size = 0
+    batch_records = []
+    chunk.msgpack_each {|record|
+      batch_records << record
+      batch_size += record.to_json.length # FIXME: heuristic
+      if batch_records.size >= BATCHWRITE_ITEM_LIMIT || batch_size >= BATCHWRITE_CONTENT_SIZE_LIMIT
+        batch_put_records(batch_records)
+        batch_records.clear
+        batch_size = 0
       end
-      @batch.put(@dynamo_db_table, [record])
     }
-    @batch.process!
+    unless batch_records.empty?
+      batch_put_records(batch_records)
+    end
   end
 
-  def collect_records(chunk)
-    records = []
-    chunk.msgpack_each { |time, record|
-      record['time'] = @timef.format(time)
-      records << record
-    }
-    records
+  def batch_put_records(records)
+    @batch.put(@dynamo_db_table, records)
+    @batch.process!
   end
 
 end

metadata

@@ -1,70 +1,67 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: fluent-plugin-dynamodb
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 0.1.6
   prerelease: 
-  version: 0.1.5
 platform: ruby
-authors: 
+authors:
 - Takashi Matsuno
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-06-10 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-06-12 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: fluentd
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &16493400 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 0.10.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: aws-sdk
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *16493400
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: &16492920 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 1.5.2
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: uuidtools
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *16492920
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: &16492460 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 2.1.0
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *16492460
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &16492000 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
-  version_requirements: *id004
+  prerelease: false
+  version_requirements: *16492000
 description: Amazon DynamoDB output plugin for Fluent event collector
 email: g0n5uk3@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
+- AUTHORS
 - ChangeLog
 - Gemfile
 - README.md
@@ -75,30 +72,26 @@ files:
 - test/out_dynamodb.rb
 homepage: https://github.com/fluent/fluent-plugin-dynamodb
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: Amazon DynamoDB output plugin for Fluent event collector
-test_files: 
-- test/out_dynamodb.rb
+test_files: []