fluent-plugin-dynamodb-alt 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c7a92c9fe9a41c6c1c7360d796a22cf7fd235259
4
- data.tar.gz: 090c2a1de0e08e5730119dcc49e896556493f8b1
3
+ metadata.gz: 9f647d44ff98f3042dd62d8eb86456a117be83db
4
+ data.tar.gz: a66fa211bd4fe96e91b074541de9934f23ac252d
5
5
  SHA512:
6
- metadata.gz: 84e9d9b2747a0bf6570efaa56e1ec49892032572cfa1380cc424bd30e5e9622ec3b9f1d11eca669a0189d2dc5d96ee3e1578b35b8a997af544dcb043d01e6be7
7
- data.tar.gz: e73846a99a68e9033ee3a92a9300b7c987d148b5464e8655e9ecbcf6fadabe843356641d845483015ea591f59938c2cc12bebc528045f6ec596cf44020db59f5
6
+ metadata.gz: d3becb79c07015ba503af5c99c0492427c8f7d77358544874685a16b09a322cf72de87388a1416f2a99bdaca7007ceca0d73e715488ffa524ab917a96c6872bc
7
+ data.tar.gz: 13c874044ed23568ce0262a5c5867240fe80fae3839c13a311e36b0807a793fd826f88f96e78b69aa9e1b5117f72ef1e31aa09ab34bb30d5856a016382be2a30
data/README.md CHANGED
@@ -32,6 +32,7 @@ bundle exec rake install
32
32
  #binary_keys data1,data2
33
33
  #endpoint http:://localhost:4567
34
34
  #concurrency 1
35
+ #delete_key delete
35
36
 
36
37
  # see http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-Expected
37
38
  #expected id NULL,timestamp LT ${timestamp},key EQ "val"
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = 'fluent-plugin-dynamodb-alt'
7
- spec.version = '0.1.3'
7
+ spec.version = '0.1.4'
8
8
  spec.authors = ['Genki Sugawara']
9
9
  spec.email = ['sgwr_dts@yahoo.co.jp']
10
10
  spec.summary = %q{Fluent plugin to output to DynamoDB.}
@@ -17,6 +17,7 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
17
17
  config_param :table_name, :string
18
18
  config_param :timestamp_key, :string
19
19
  config_param :binary_keys, :string, :default => nil
20
+ config_param :delete_key, :string, :default => nil
20
21
  config_param :concurrency, :integer, :default => 1
21
22
  config_param :expected, :string, :default => nil
22
23
  config_param :conditional_operator, :string, :default => 'AND'
@@ -89,8 +90,13 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
89
90
 
90
91
  def write(chunk)
91
92
  chunk = aggregate_records(chunk)
93
+
92
94
  block = proc do |tag, time, record|
93
- put_record(record)
95
+ if @delete_key and record[@delete_key]
96
+ delete_record(record)
97
+ else
98
+ put_record(record)
99
+ end
94
100
  end
95
101
 
96
102
  if @concurrency > 1
@@ -133,6 +139,29 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
133
139
  end
134
140
  end
135
141
 
142
+ def delete_record(record)
143
+ key = {@hash_key => record[@hash_key]}
144
+ key[@range_key] = record[@range_key] if @range_key
145
+
146
+ item = {
147
+ :table_name => @table_name,
148
+ :key => key
149
+ }
150
+
151
+ begin
152
+ if @expected
153
+ expected = create_expected(record)
154
+ return unless expected
155
+ item[:expected] = expected
156
+ item[:conditional_operator] = @conditional_operator if expected.length > 1
157
+ end
158
+
159
+ @client.delete_item(item)
160
+ rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException, Aws::DynamoDB::Errors::ValidationException => e
161
+ log.warn("#{e.message}: #{item.inspect}")
162
+ end
163
+ end
164
+
136
165
  def validate_record(record)
137
166
  if not record[@hash_key]
138
167
  log.warn("Hash Key '#{@hash_key}' does not exist in the record: #{record.inspect}")
@@ -33,6 +33,7 @@ describe Fluent::DynamodbAltOutput do
33
33
  table_name my_table
34
34
  timestamp_key timestamp
35
35
  binary_keys aaa,bbb
36
+ delete_key delete
36
37
  concurrency 2
37
38
  conditional_operator OR
38
39
  EOS
@@ -44,6 +45,7 @@ describe Fluent::DynamodbAltOutput do
44
45
  expect(driver.instance.table_name ).to eq 'my_table'
45
46
  expect(driver.instance.timestamp_key ).to eq 'timestamp'
46
47
  expect(driver.instance.binary_keys ).to eq ['aaa', 'bbb']
48
+ expect(driver.instance.delete_key ).to eq 'delete'
47
49
  expect(driver.instance.concurrency ).to eq 2
48
50
  expect(driver.instance.conditional_operator).to eq 'OR'
49
51
  expect(driver.instance.instance_variable_get(:@hash_key) ).to eq 'hash_key'
@@ -500,5 +502,90 @@ describe Fluent::DynamodbAltOutput do
500
502
  ]
501
503
  end
502
504
  }
505
+
506
+ context('delete record (1)') {
507
+ it do
508
+ run_driver do |d|
509
+ d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
510
+ d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
511
+ d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
512
+ end
513
+
514
+ expect(select_all).to match_array [
515
+ {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
516
+ {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
517
+ {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
518
+ ]
519
+
520
+ run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
521
+ expect(d.instance.log).not_to receive(:warn)
522
+ d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625004, 'delete' => true}, time)
523
+ d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
524
+ d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'delete' => true}, time)
525
+ end
526
+
527
+ expect(select_all).to match_array [
528
+ {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
529
+ ]
530
+ end
531
+ }
532
+
533
+ context('delete record (2)') {
534
+ it do
535
+ run_driver do |d|
536
+ d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
537
+ d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
538
+ d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
539
+ end
540
+
541
+ expect(select_all).to match_array [
542
+ {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
543
+ {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
544
+ {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
545
+ ]
546
+
547
+ run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
548
+ expect(d.instance.log).to receive(:warn)
549
+ .with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :key=>{"id"=>"12345678-1234-1234-1234-123456789001"}, :expected=>{"id"=>{:comparison_operator=>"NULL"}, "timestamp"=>{:comparison_operator=>"LT", :attribute_value_list=>[1409534625001]}}, :conditional_operator=>"OR"}!)
550
+ d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'delete' => true}, time)
551
+ d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
552
+ d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'delete' => true}, time)
553
+ end
554
+
555
+ expect(select_all).to match_array [
556
+ {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
557
+ {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
558
+ ]
559
+ end
560
+ }
561
+
562
+ context('delete empty record') {
563
+ it do
564
+ run_driver do |d|
565
+ d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
566
+ d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
567
+ d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
568
+ end
569
+
570
+ expect(select_all).to match_array [
571
+ {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
572
+ {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
573
+ {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
574
+ ]
575
+
576
+ run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
577
+ expect(d.instance.log).not_to receive(:warn)
578
+ d.emit({'id' => '12345678-1234-1234-1234-123456789011', 'timestamp' => 1409534625004, 'delete' => true}, time)
579
+ d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
580
+ d.emit({'id' => '12345678-1234-1234-1234-123456789013', 'timestamp' => 1409534625006, 'delete' => true}, time)
581
+ end
582
+
583
+ expect(select_all).to match_array [
584
+ {"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
585
+ {"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
586
+ {"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
587
+ ]
588
+ end
589
+ }
503
590
  }
504
591
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-dynamodb-alt
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Genki Sugawara