fluent-plugin-dynamodb-alt 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7a92c9fe9a41c6c1c7360d796a22cf7fd235259
-  data.tar.gz: 090c2a1de0e08e5730119dcc49e896556493f8b1
+  metadata.gz: 9f647d44ff98f3042dd62d8eb86456a117be83db
+  data.tar.gz: a66fa211bd4fe96e91b074541de9934f23ac252d
 SHA512:
-  metadata.gz: 84e9d9b2747a0bf6570efaa56e1ec49892032572cfa1380cc424bd30e5e9622ec3b9f1d11eca669a0189d2dc5d96ee3e1578b35b8a997af544dcb043d01e6be7
-  data.tar.gz: e73846a99a68e9033ee3a92a9300b7c987d148b5464e8655e9ecbcf6fadabe843356641d845483015ea591f59938c2cc12bebc528045f6ec596cf44020db59f5
+  metadata.gz: d3becb79c07015ba503af5c99c0492427c8f7d77358544874685a16b09a322cf72de87388a1416f2a99bdaca7007ceca0d73e715488ffa524ab917a96c6872bc
+  data.tar.gz: 13c874044ed23568ce0262a5c5867240fe80fae3839c13a311e36b0807a793fd826f88f96e78b69aa9e1b5117f72ef1e31aa09ab34bb30d5856a016382be2a30

data/README.md

@@ -32,6 +32,7 @@ bundle exec rake install
   #binary_keys data1,data2
   #endpoint http:://localhost:4567
   #concurrency 1
+  #delete_key delete
 
   # see http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-Expected
   #expected id NULL,timestamp LT ${timestamp},key EQ "val"

data/fluent-plugin-dynamodb-alt.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'fluent-plugin-dynamodb-alt'
-  spec.version       = '0.1.3'
+  spec.version       = '0.1.4'
   spec.authors       = ['Genki Sugawara']
   spec.email         = ['sgwr_dts@yahoo.co.jp']
   spec.summary       = %q{Fluent plugin to output to DynamoDB.}

data/lib/fluent/plugin/out_dynamodb_alt.rb

@@ -17,6 +17,7 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
   config_param :table_name,           :string
   config_param :timestamp_key,        :string
   config_param :binary_keys,          :string,  :default => nil
+  config_param :delete_key,           :string,  :default => nil
   config_param :concurrency,          :integer, :default => 1
   config_param :expected,             :string,  :default => nil
   config_param :conditional_operator, :string,  :default => 'AND'
@@ -89,8 +90,13 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
 
   def write(chunk)
     chunk = aggregate_records(chunk)
+
     block = proc do |tag, time, record|
-      put_record(record)
+      if @delete_key and record[@delete_key]
+        delete_record(record)
+      else
+        put_record(record)
+      end
     end
 
     if @concurrency > 1
@@ -133,6 +139,29 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
     end
   end
 
+  def delete_record(record)
+    key = {@hash_key => record[@hash_key]}
+    key[@range_key] = record[@range_key] if @range_key
+
+    item = {
+      :table_name => @table_name,
+      :key => key
+    }
+
+    begin
+      if @expected
+        expected = create_expected(record)
+        return unless expected
+        item[:expected] = expected
+        item[:conditional_operator] = @conditional_operator if expected.length > 1
+      end
+
+      @client.delete_item(item)
+    rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException, Aws::DynamoDB::Errors::ValidationException => e
+      log.warn("#{e.message}: #{item.inspect}")
+    end
+  end
+
   def validate_record(record)
     if not record[@hash_key]
       log.warn("Hash Key '#{@hash_key}' does not exist in the record: #{record.inspect}")

data/spec/out_dynamodb_alt_spec.rb

@@ -33,6 +33,7 @@ describe Fluent::DynamodbAltOutput do
         table_name my_table
         timestamp_key timestamp
         binary_keys aaa,bbb
+        delete_key delete
         concurrency 2
         conditional_operator OR
       EOS
@@ -44,6 +45,7 @@ describe Fluent::DynamodbAltOutput do
       expect(driver.instance.table_name          ).to eq 'my_table'
       expect(driver.instance.timestamp_key       ).to eq 'timestamp'
       expect(driver.instance.binary_keys         ).to eq ['aaa', 'bbb']
+      expect(driver.instance.delete_key          ).to eq 'delete'
       expect(driver.instance.concurrency         ).to eq 2
       expect(driver.instance.conditional_operator).to eq 'OR'
       expect(driver.instance.instance_variable_get(:@hash_key) ).to eq 'hash_key'
@@ -500,5 +502,90 @@ describe Fluent::DynamodbAltOutput do
         ]
       end
     }
+
+    context('delete record (1)') {
+      it do
+        run_driver do |d|
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+
+        run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625004, 'delete' => true}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'delete' => true}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
+        ]
+      end
+    }
+
+    context('delete record (2)') {
+      it do
+        run_driver do |d|
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+
+        run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
+          expect(d.instance.log).to receive(:warn)
+            .with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :key=>{"id"=>"12345678-1234-1234-1234-123456789001"}, :expected=>{"id"=>{:comparison_operator=>"NULL"}, "timestamp"=>{:comparison_operator=>"LT", :attribute_value_list=>[1409534625001]}}, :conditional_operator=>"OR"}!)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'delete' => true}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'delete' => true}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
+        ]
+      end
+    }
+
+    context('delete empty record') {
+      it do
+        run_driver do |d|
+          d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+
+        run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
+          expect(d.instance.log).not_to receive(:warn)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789011', 'timestamp' => 1409534625004, 'delete' => true}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
+          d.emit({'id' => '12345678-1234-1234-1234-123456789013', 'timestamp' => 1409534625006, 'delete' => true}, time)
+        end
+
+        expect(select_all).to match_array [
+          {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
+          {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
+          {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
+        ]
+      end
+    }
   }
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dynamodb-alt
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Genki Sugawara