fluent-plugin-dynamodb-alt 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 19e9db9d6797b1ac0b232a1190fe09498ea4502b
+  data.tar.gz: 21a3ce6f08bbcb44aa19f35bda31ecac38af0662
+SHA512:
+  metadata.gz: de30d92832b0c478a585fb0aa8eff15cc2b640ddb10f74e920fd6788e98415e0ddc9f33888ef4c94b36a16e7cc4de50f06932e4b9300e00dfb8c4c0a74fc20b4
+  data.tar.gz: 9cdcde14528e63ac12d402a214d2843d108077a936ae824ecba828805fa7a71920e8ba3237d7c3377f2e4a8627869500c04f7edce5bbabf89461ff5bdafafe28

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+test.rb
+node_modules

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--colour

data/.travis.yml

@@ -0,0 +1,11 @@
+
+language: ruby
+rvm:
+- 2.0.0
+script:
+- bundle install
+- bundle exec rake
+env:
+  global:
+  - secure: "ESwtSv6459PcZVWzwPbxIFxKVkm7F0BE30JlEgE2N4yWaMaWZfE9i2tsjh6cCB03I89sUuysHjpDtV/Hf7nUQC58s6q5VFPEex0RBBcbAC8ALLL+Ai6go6vPX2tqp0NP8v1OUdNDu7TtPQW2EFyBG75OJ1ga9UT1Q+hGnd2Xdmg="
+  - secure: "hHZGXwrDVpY7DtY6nSbkiBcIQlT4uvIlA0Xx/+AHHsEBgeAakSPVO8pVI3JrkpYNRbmYOXoj3TWs0Z0ROjPZdbOwKIaH3xzoFS61zk/oJbHume8wjhvWp8QGNukcbas+/3FuTyBc2CUDcy2zpuEx+p/PEYwccdS3LrTTSp4nYf0="

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-dynamodb-session.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Genki Sugawara
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# fluent-plugin-dynamodb-alt
+
+Alternative fluent plugin to output to DynamoDB.
+
+## Features
+
+* Use [PutItem](http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html) Acrion
+* Sort the records in the timestamp key.
+* Aggregate the records in the primary key.
+* Support [Expected constraint](http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-Expected)
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-dynamodb-alt.png)](http://badge.fury.io/rb/fluent-plugin-dynamodb-alt)
+[![Build Status](https://travis-ci.org/winebarrel/fluent-plugin-dynamodb-alt.svg)](https://travis-ci.org/winebarrel/fluent-plugin-dynamodb-alt)
+
+## Installation
+
+```sh
+bundle install
+bundle exec rake install
+```
+
+## Configuration
+
+```
+<match tag>
+  type dynamodb_alt
+  aws_key_id AKIAIOSFODNN7EXAMPLE
+  aws_sec_key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+  region ap-northeast-1
+  table_name my_table
+  timestamp_key timestamp
+  #concurrency 1
+
+  # see http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-Expected
+  #expected id NULL,timestamp LT ${timestamp},key EQ "val"
+  #conditional_operator OR
+
+  #include_time_key false
+  #include_tag_key false
+</match>
+```

data/Rakefile

@@ -0,0 +1,5 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+task :default => :spec

data/fluent-plugin-dynamodb-alt.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'fluent-plugin-dynamodb-alt'
+  spec.version       = '0.1.0'
+  spec.authors       = ['Genki Sugawara']
+  spec.email         = ['sgwr_dts@yahoo.co.jp']
+  spec.summary       = %q{Fluent plugin to output to DynamoDB.}
+  spec.description   = %q{Fluent plugin to output to DynamoDB.}
+  spec.homepage      = 'https://github.com/winebarrel/fluent-plugin-dynamodb-alt'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'fluentd'
+  spec.add_dependency 'aws-sdk-core', '>= 2.0.0.rc15'
+  spec.add_dependency 'parallel'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '>= 3.0.0'
+  spec.add_development_dependency 'hashie'
+  spec.add_development_dependency 'ddbcli'
+end

data/lib/fluent/plugin/out_dynamodb_alt.rb

@@ -0,0 +1,202 @@
+class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('dynamodb_alt', self)
+
+  include Fluent::SetTimeKeyMixin
+  include Fluent::SetTagKeyMixin
+
+  unless method_defined?(:log)
+    define_method('log') { $log }
+  end
+
+  config_param :profile,              :string,  :default => nil
+  config_param :credentials_path,     :string,  :default => nil
+  config_param :aws_key_id,           :string,  :default => nil
+  config_param :aws_sec_key,          :string,  :default => nil
+  config_param :region,               :string,  :default => nil
+  config_param :table_name,           :string
+  config_param :timestamp_key,        :string
+  config_param :concurrency,          :integer, :default => 1
+  config_param :expected,             :string,  :default => nil
+  config_param :conditional_operator, :string,  :default => 'AND'
+
+  config_set_default :include_time_key, true
+  config_set_default :include_tag_key,  true
+
+  def initialize
+    super
+    require 'aws-sdk-core'
+    require 'parallel'
+  end
+
+  def configure(conf)
+    super
+
+    aws_opts = {}
+
+    if @profile
+      credentials_opts = {:profile_name => @profile}
+      credentials_opts[:path] = @credentials_path if @credentials_path
+      credentials = Aws::SharedCredentials.new(credentials_opts)
+      aws_opts[:credentials] = credentials
+    end
+
+    aws_opts[:access_key_id] = @aws_key_id if @aws_key_id
+    aws_opts[:secret_access_key] = @aws_sec_key if @aws_sec_key
+    aws_opts[:region] = @region if @region
+
+    configure_aws(aws_opts)
+
+    client = create_client
+    table = client.describe_table(:table_name => @table_name)
+
+    table.table.key_schema.each do |attribute|
+      case attribute.key_type
+      when 'HASH'
+        @hash_key = attribute.attribute_name
+      when 'RANGE'
+        @range_key = attribute.attribute_name
+      else
+        raise 'must not happen'
+      end
+    end
+
+    if @expected
+      @expected = parse_expected(@expected)
+      log.info("dynamodb_alt expected: #{@expected.inspect}")
+    end
+  end
+
+  def start
+    super
+
+    @client = create_client
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def write(chunk)
+    chunk = aggregate_records(chunk)
+    block = proc do |tag, time, record|
+      put_record(record)
+    end
+
+    if @concurrency > 1
+      Parallel.each(chunk, :in_threads => @concurrency, &block)
+    else
+      chunk.each(&block)
+    end
+  end
+
+  private
+
+  def configure_aws(options)
+    Aws.config.update(options)
+  end
+
+  def create_client
+    Aws::DynamoDB::Client.new
+  end
+
+  def put_record(record)
+    item = {
+      :table_name => @table_name,
+      :item => record
+    }
+
+    begin
+      if @expected
+        expected = create_expected(record)
+        return unless expected
+        item[:expected] = expected
+        item[:conditional_operator] = @conditional_operator if expected.length > 1
+      end
+
+      @client.put_item(item)
+    rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException, Aws::DynamoDB::Errors::ValidationException => e
+      log.warn("#{e.message}: #{item.inspect}")
+    end
+  end
+
+  def validate_record(record)
+    if not record[@hash_key]
+      log.warn("Hash Key '#{@hash_key}' does not exist in the record: #{record.inspect}")
+      return false
+    end
+
+    if @range_key and not record[@range_key]
+      log.warn("Range Key '#{@range_key}' does not exist in the record: #{record.inspect}")
+      return false
+    end
+
+    if not record[@timestamp_key]
+      log.warn("Timestamp Key '#{@timestamp_key}' does not exist in the record: #{record.inspect}")
+      return false
+    end
+
+    return true
+  end
+
+  def parse_expected(expected)
+    expected.split(',').map do |expr|
+      key, op, val = expr.strip.split(/\s+/)
+
+      if val
+        if val =~ /\A\$\{(.+)\}\z/
+          record_key = $1.inspect
+          val = eval("proc {|record| record[#{record_key}] }")
+        else
+          begin
+            val = JSON.parse("[#{val}]").first if val
+          rescue JSON::ParserError => e
+            raise "Cannot parse the expected expression (#{expr}): #{e.message}"
+          end
+        end
+      end
+
+      [key, op, val]
+    end
+  end
+
+  def create_expected(record)
+    attrs = {}
+
+    @expected.map do |key, op, val|
+      attrs[key] = {:comparison_operator => op}
+
+      if val
+        if val.kind_of?(Proc)
+          record_val = val.call(record)
+
+          unless record_val
+            log.warn("Expected value does not exist in the record: #{record.inspect}")
+            return nil
+          end
+
+          attrs[key][:attribute_value_list] = [record_val]
+        else
+          attrs[key][:attribute_value_list] = [val]
+        end
+      end
+    end
+
+    return attrs
+  end
+
+  def aggregate_records(chunk)
+    chunk.enum_for(:msgpack_each).select {|tag, time, record|
+      validate_record(record)
+    }.chunk {|tag, time, record|
+      if @range_key
+        record.values_at(@hash_key, @range_key)
+      else
+        record[@hash_key]
+      end
+    }.map {|primary_key, records|
+      records.sort_by {|tag, time, record|
+        record[@timestamp_key]
+      }.last
+    }
+  end
+end

data/spec/out_dynamodb_alt_spec.rb

@@ -0,0 +1,430 @@
+describe Fluent::DynamodbAltOutput do
+  let(:time) {
+    Time.parse('2014-09-01 01:23:45 UTC').to_i
+  }
+
+  context('configure') {
+    it do
+      driver = create_driver
+      expect(driver.instance).to receive(:configure_aws).with(
+        :access_key_id     => "AKIAIOSFODNN7EXAMPLE",
+        :secret_access_key => "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        :region            => "ap-northeast-1")
+
+      expect(driver.instance).to receive(:create_client) {
+        client = double('client')
+        allow(client).to receive(:describe_table).with(:table_name => 'my_table') {
+          Hashie::Mash.new(:table => {
+            :key_schema => [
+              {:key_type => 'HASH',  :attribute_name => 'hash_key'},
+              {:key_type => 'RANGE', :attribute_name => 'range_key'}
+            ]})
+        }
+        client
+      }
+
+      driver.configure(<<-EOS)
+        type dynamodb_alt
+        aws_key_id AKIAIOSFODNN7EXAMPLE
+        aws_sec_key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+        region ap-northeast-1
+        table_name my_table
+        timestamp_key timestamp
+        concurrency 2
+        conditional_operator OR
+      EOS
+
+      expect(driver.instance.aws_key_id          ).to eq 'AKIAIOSFODNN7EXAMPLE'
+      expect(driver.instance.aws_sec_key         ).to eq 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
+      expect(driver.instance.table_name          ).to eq 'my_table'
+      expect(driver.instance.timestamp_key       ).to eq 'timestamp'
+      expect(driver.instance.concurrency         ).to eq 2
+      expect(driver.instance.conditional_operator).to eq 'OR'
+      expect(driver.instance.instance_variable_get(:@hash_key) ).to eq 'hash_key'
+      expect(driver.instance.instance_variable_get(:@range_key)).to eq 'range_key'
+    end
+
+    it do
+      driver = create_driver
+      allow(driver.instance).to receive(:configure_aws)
+
+      allow(driver.instance).to receive(:create_client) {
+        client = double('client')
+        allow(client).to receive(:describe_table) {
+          Hashie::Mash.new(:table => {
+            :key_schema => [
+              {:key_type => 'HASH',  :attribute_name => 'hash_key'},
+            ]})
+        }
+        client
+      }
+
+      driver.configure(<<-EOS)
+        type dynamodb_alt
+        table_name my_table
+        timestamp_key timestamp
+        expected timestamp GE 0,key LT 100
+      EOS
+
+      expected = driver.instance.instance_variable_get(:@expected)
+      expect(expected).to eq [["timestamp", "GE", 0],["key", "LT", 100]]
+    end
+
+    it do
+      driver = create_driver
+      allow(driver.instance).to receive(:configure_aws)
+
+      allow(driver.instance).to receive(:create_client) {
+        client = double('client')
+        allow(client).to receive(:describe_table) {
+          Hashie::Mash.new(:table => {
+            :key_schema => [
+              {:key_type => 'HASH',  :attribute_name => 'hash_key'},
+            ]})
+        }
+        client
+      }
+
+      driver.configure(<<-EOS)
+        type dynamodb_alt
+        table_name my_table
+        timestamp_key timestamp
+        expected id NULL,timestamp LT ${ts},key EQ ${k}
+      EOS
+
+      expected = driver.instance.instance_variable_get(:@expected)
+
+      expect(expected[0]).to eq ["id", "NULL", nil]
+
+      col1, op1, val1 = expected[1]
+      expect(col1).to eq 'timestamp'
+      expect(op1 ).to eq 'LT'
+      expect(val1.call('ts' => 1)).to eq 1
+
+      col2, op2, val2 = expected[2]
+      expect(col2).to eq 'key'
+      expect(op2 ).to eq 'EQ'
+      expect(val2.call('k' => '1')).to eq '1'
+    end
+
+    it do
+      driver = create_driver
+      allow(driver.instance).to receive(:configure_aws)
+
+      allow(driver.instance).to receive(:create_client) {
+        client = double('client')
+        allow(client).to receive(:describe_table) {
+          Hashie::Mash.new(:table => {
+            :key_schema => [
+              {:key_type => 'HASH',  :attribute_name => 'hash_key'},
+            ]})
+        }
+        client
+      }
+
+      expect {
+        driver.configure(<<-EOS)
+          type dynamodb_alt
+          table_name my_table
+          timestamp_key timestamp
+          expected key EQ val
+        EOS
+      }.to raise_error("Cannot parse the expected expression (key EQ val): 399: unexpected token at 'val]'")
+    end
+
+    it do
+      driver = create_driver
+      allow(driver.instance).to receive(:configure_aws)
+
+      allow(driver.instance).to receive(:create_client) {
+        client = double('client')
+        allow(client).to receive(:describe_table) {
+          Hashie::Mash.new(:table => {
+            :key_schema => [
+              {:key_type => 'HASH',  :attribute_name => 'hash_key'},
+            ]})
+        }
+        client
+      }
+
+      driver.configure(<<-EOS)
+        type dynamodb_alt
+        table_name my_table
+        timestamp_key timestamp
+        expected key1 EQ "str",key2 EQ 1
+      EOS
+
+      expected = driver.instance.instance_variable_get(:@expected)
+      expect(expected).to eq [["key1", "EQ", "str"], ["key2", "EQ", 1]]
+    end
+  }
+
+  context('emit') {
+    before(:all) do
+      drop_table
+      create_table('(id STRING HASH) READ = 20 WRITE = 20')
+    end
+
+    before(:each) do
+      truncate_table
+    end
+
+    after(:all) do
+      drop_table
+    end
+
+    context('without condition') {
+      it do
+        run_driver do |d|
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+      end
+    }
+
+    context('concurrency > 1') {
+      it do
+        run_driver(:concurrency => 2) do |d|
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+      end
+    }
+
+    context('with condition (1)') {
+      it do
+        run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR') do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+
+        run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR') do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625004, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'key' => 'val'}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625004, 'key' => 'val'},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625006, 'key' => 'val'},
+        ]
+      end
+
+      it do
+        run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR') do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+
+        run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR') do |d|
+          expect(d.instance.log).to receive(:warn)
+            .with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :item=>{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001, "key"=>"val"}, :expected=>{"id"=>{:comparison_operator=>"NULL"}, "timestamp"=>{:comparison_operator=>"LT", :attribute_value_list=>[1409534625001]}}, :conditional_operator=>"OR"}!)
+          expect(d.instance.log).to receive(:warn)
+            .with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :item=>{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002, "key"=>"val"}, :expected=>{"id"=>{:comparison_operator=>"NULL"}, "timestamp"=>{:comparison_operator=>"LT", :attribute_value_list=>[1409534625002]}}, :conditional_operator=>"OR"}!)
+          expect(d.instance.log).to receive(:warn)
+            .with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :item=>{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003, "key"=>"val"}, :expected=>{"id"=>{:comparison_operator=>"NULL"}, "timestamp"=>{:comparison_operator=>"LT", :attribute_value_list=>[1409534625003]}}, :conditional_operator=>"OR"}!)
+
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003, 'key' => 'val'}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+      end
+    }
+
+    context('with condition (2)') {
+      it do
+        run_driver do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003, 'key' => 'val'}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001, 'key' => 'val'},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002, 'key' => 'val'},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003, 'key' => 'val'},
+        ]
+
+        run_driver(:expected => 'timestamp LE ${timestamp},key EQ "val"') do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625004, 'key' => 'val2'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val2'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'key' => 'val2'}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625004, 'key' => 'val2'},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val2'},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625006, 'key' => 'val2'},
+        ]
+      end
+
+      it do
+        run_driver do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'key' => 'val3'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002, 'key' => 'val3'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003, 'key' => 'val3'}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001, 'key' => 'val3'},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002, 'key' => 'val3'},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003, 'key' => 'val3'},
+        ]
+
+        run_driver(:expected => 'timestamp LE ${timestamp},key EQ "val"') do |d|
+          expect(d.instance.log).to receive(:warn)
+            .with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :item=>{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625004, "key"=>"val2"}, :expected=>{"timestamp"=>{:comparison_operator=>"LE", :attribute_value_list=>[1409534625004]}, "key"=>{:comparison_operator=>"EQ", :attribute_value_list=>["val"]}}, :conditional_operator=>"AND"}!)
+          expect(d.instance.log).to receive(:warn)
+            .with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :item=>{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, "key"=>"val2"}, :expected=>{"timestamp"=>{:comparison_operator=>"LE", :attribute_value_list=>[1409534625005]}, "key"=>{:comparison_operator=>"EQ", :attribute_value_list=>["val"]}}, :conditional_operator=>"AND"}!)
+          expect(d.instance.log).to receive(:warn)
+            .with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :item=>{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625006, "key"=>"val2"}, :expected=>{"timestamp"=>{:comparison_operator=>"LE", :attribute_value_list=>[1409534625006]}, "key"=>{:comparison_operator=>"EQ", :attribute_value_list=>["val"]}}, :conditional_operator=>"AND"}!)
+
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625004, 'key' => 'val2'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val2'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'key' => 'val2'}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001, 'key' => 'val3'},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002, 'key' => 'val3'},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003, 'key' => 'val3'},
+        ]
+      end
+
+      it do
+        run_driver do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'key' => 'val3'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002, 'key' => 'val3'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003, 'key' => 'val3'}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001, 'key' => 'val3'},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002, 'key' => 'val3'},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003, 'key' => 'val3'},
+        ]
+
+        run_driver(:expected => 'timestamp LE ${timestamp},key EQ "val"', :conditional_operator => 'OR') do |d|
+          expect(d.instance.log).to_not receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625004, 'key' => 'val2'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val2'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'key' => 'val2'}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625004, 'key' => 'val2'},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val2'},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625006, 'key' => 'val2'},
+        ]
+      end
+    }
+
+    context('key dose not exist') {
+      it do
+        run_driver do |d|
+          expect(d.instance.log).to receive(:warn)
+            .with(%!Hash Key 'id' does not exist in the record: {"timestamp"=>1409534625001, "key"=>"val"}!)
+          d.emit({'timestamp' => 1409534625001, 'key' => 'val'}, time)
+        end
+
+        expect(select_all).to match_array []
+      end
+
+      it do
+        run_driver do |d|
+          expect(d.instance.log).to receive(:warn)
+            .with(%!Timestamp Key 'timestamp' does not exist in the record: {"id"=>"12345678-1234-1234-1234-123456789001", "key"=>"val"}!)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'key' => 'val'}, time)
+        end
+
+        expect(select_all).to match_array []
+      end
+    }
+
+    context('aggregate') {
+      it do
+        run_driver(:flush_interval => 3) do |d|
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625003}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625003}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625001}, time)
+
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625001}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625003},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625003},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+      end
+
+      it do
+        run_driver(:flush_interval => 3) do |d|
+          expect(d.instance).to receive(:put_record).with('id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625003)
+          expect(d.instance).to receive(:put_record).with('id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625003)
+          expect(d.instance).to receive(:put_record).with('id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003)
+
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625003}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625003}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625001}, time)
+
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625001}, time)
+        end
+      end
+    }
+  }
+end

data/spec/spec_helper.rb

@@ -0,0 +1,99 @@
+require 'fluent/test'
+require 'fluent/plugin/out_dynamodb_alt'
+require 'hashie'
+require 'securerandom'
+
+DRIVER_DEFAULT_TAG = 'test.default'
+TEST_TABLE_NAME = 'my_table-' + SecureRandom.uuid
+TEST_AWS_KEY_ID = ENV['OUT_DYNAMODB_ALT_SPEC_AWS_KEY_ID'] || 'scott'
+TEST_AWS_SEC_KEY = ENV['OUT_DYNAMODB_ALT_SPEC_AWS_SEC_KEY'] || 'tiger'
+TEST_REGION = ENV['OUT_DYNAMODB_ALT_SPEC_REGION'] || 'us-west-1'
+
+# Disable Test::Unit
+module Test::Unit::RunCount; def run(*); end; end
+
+RSpec.configure do |config|
+  config.before(:all) do
+    Fluent::Test.setup
+  end
+end
+
+def query(sql)
+  out = `ddbcli -k '#{TEST_AWS_KEY_ID}' -s '#{TEST_AWS_SEC_KEY}' -r #{TEST_REGION} -e "#{sql.gsub(/\n/, ' ')}"`
+  raise out unless $?.success?
+  return out
+end
+
+def select_all
+  JSON.parse(query("SELECT ALL * FROM #{TEST_TABLE_NAME}"))
+end
+
+def create_driver(tag = DRIVER_DEFAULT_TAG)
+  Fluent::Test::OutputTestDriver.new(Fluent::DynamodbAltOutput, tag)
+end
+
+def run_driver(options = {})
+  options = {
+    :flush_interval => 0
+  }.merge(options.dup)
+
+  additional_options = options.map {|key, value|
+    "#{key} #{value}"
+  }.join("\n")
+
+  fluentd_conf = <<-EOS
+    type dynamodb_alt
+    aws_key_id #{TEST_AWS_KEY_ID}
+    aws_sec_key #{TEST_AWS_SEC_KEY}
+    region #{TEST_REGION}
+    table_name #{TEST_TABLE_NAME}
+    timestamp_key timestamp
+    #{additional_options}
+  EOS
+
+  tag = options[:tag] || DRIVER_DEFAULT_TAG
+  driver = create_driver(tag)
+
+  driver.configure(fluentd_conf)
+
+  driver.run do
+    yield(driver)
+  end
+end
+
+def create_table(attrs)
+  query("CREATE TABLE #{TEST_TABLE_NAME} #{attrs}")
+
+  loop do
+    status = query("SHOW TABLE STATUS LIKE '#{TEST_TABLE_NAME}'")
+    status = JSON.parse(status)
+    break if (not status[TEST_TABLE_NAME] or status[TEST_TABLE_NAME]['TableStatus'] == 'ACTIVE')
+  end
+end
+
+def drop_table
+  table_is_exist = proc do
+    tables = query("SHOW TABLES LIKE '#{TEST_TABLE_NAME}'")
+    tables = JSON.parse(tables)
+    not tables.empty?
+  end
+
+  return unless table_is_exist.call
+
+  query("DROP TABLE #{TEST_TABLE_NAME}")
+
+  loop do
+    tables = query("SHOW TABLES LIKE '#{TEST_TABLE_NAME}'")
+    tables = JSON.parse(tables)
+    break unless table_is_exist.call
+  end
+end
+
+def truncate_table
+  query("DELETE ALL FROM #{TEST_TABLE_NAME}")
+
+  loop do
+    count = query("SELECT ALL COUNT(*) FROM #{TEST_TABLE_NAME}").strip
+    break if count == '0'
+  end
+end

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-dynamodb-alt
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Genki Sugawara
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc15
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc15
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ddbcli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent plugin to output to DynamoDB.
+email:
+- sgwr_dts@yahoo.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-dynamodb-alt.gemspec
+- lib/fluent/plugin/out_dynamodb_alt.rb
+- spec/out_dynamodb_alt_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/winebarrel/fluent-plugin-dynamodb-alt
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Fluent plugin to output to DynamoDB.
+test_files:
+- spec/out_dynamodb_alt_spec.rb
+- spec/spec_helper.rb