fluent-plugin-dynamodb-alt 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/fluent-plugin-dynamodb-alt.gemspec +1 -1
- data/lib/fluent/plugin/out_dynamodb_alt.rb +30 -1
- data/spec/out_dynamodb_alt_spec.rb +87 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f647d44ff98f3042dd62d8eb86456a117be83db
|
4
|
+
data.tar.gz: a66fa211bd4fe96e91b074541de9934f23ac252d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d3becb79c07015ba503af5c99c0492427c8f7d77358544874685a16b09a322cf72de87388a1416f2a99bdaca7007ceca0d73e715488ffa524ab917a96c6872bc
|
7
|
+
data.tar.gz: 13c874044ed23568ce0262a5c5867240fe80fae3839c13a311e36b0807a793fd826f88f96e78b69aa9e1b5117f72ef1e31aa09ab34bb30d5856a016382be2a30
|
data/README.md
CHANGED
@@ -32,6 +32,7 @@ bundle exec rake install
|
|
32
32
|
#binary_keys data1,data2
|
33
33
|
#endpoint http:://localhost:4567
|
34
34
|
#concurrency 1
|
35
|
+
#delete_key delete
|
35
36
|
|
36
37
|
# see http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-Expected
|
37
38
|
#expected id NULL,timestamp LT ${timestamp},key EQ "val"
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = 'fluent-plugin-dynamodb-alt'
|
7
|
-
spec.version = '0.1.
|
7
|
+
spec.version = '0.1.4'
|
8
8
|
spec.authors = ['Genki Sugawara']
|
9
9
|
spec.email = ['sgwr_dts@yahoo.co.jp']
|
10
10
|
spec.summary = %q{Fluent plugin to output to DynamoDB.}
|
@@ -17,6 +17,7 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
|
|
17
17
|
config_param :table_name, :string
|
18
18
|
config_param :timestamp_key, :string
|
19
19
|
config_param :binary_keys, :string, :default => nil
|
20
|
+
config_param :delete_key, :string, :default => nil
|
20
21
|
config_param :concurrency, :integer, :default => 1
|
21
22
|
config_param :expected, :string, :default => nil
|
22
23
|
config_param :conditional_operator, :string, :default => 'AND'
|
@@ -89,8 +90,13 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
|
|
89
90
|
|
90
91
|
def write(chunk)
|
91
92
|
chunk = aggregate_records(chunk)
|
93
|
+
|
92
94
|
block = proc do |tag, time, record|
|
93
|
-
|
95
|
+
if @delete_key and record[@delete_key]
|
96
|
+
delete_record(record)
|
97
|
+
else
|
98
|
+
put_record(record)
|
99
|
+
end
|
94
100
|
end
|
95
101
|
|
96
102
|
if @concurrency > 1
|
@@ -133,6 +139,29 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
|
|
133
139
|
end
|
134
140
|
end
|
135
141
|
|
142
|
+
def delete_record(record)
|
143
|
+
key = {@hash_key => record[@hash_key]}
|
144
|
+
key[@range_key] = record[@range_key] if @range_key
|
145
|
+
|
146
|
+
item = {
|
147
|
+
:table_name => @table_name,
|
148
|
+
:key => key
|
149
|
+
}
|
150
|
+
|
151
|
+
begin
|
152
|
+
if @expected
|
153
|
+
expected = create_expected(record)
|
154
|
+
return unless expected
|
155
|
+
item[:expected] = expected
|
156
|
+
item[:conditional_operator] = @conditional_operator if expected.length > 1
|
157
|
+
end
|
158
|
+
|
159
|
+
@client.delete_item(item)
|
160
|
+
rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException, Aws::DynamoDB::Errors::ValidationException => e
|
161
|
+
log.warn("#{e.message}: #{item.inspect}")
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
136
165
|
def validate_record(record)
|
137
166
|
if not record[@hash_key]
|
138
167
|
log.warn("Hash Key '#{@hash_key}' does not exist in the record: #{record.inspect}")
|
@@ -33,6 +33,7 @@ describe Fluent::DynamodbAltOutput do
|
|
33
33
|
table_name my_table
|
34
34
|
timestamp_key timestamp
|
35
35
|
binary_keys aaa,bbb
|
36
|
+
delete_key delete
|
36
37
|
concurrency 2
|
37
38
|
conditional_operator OR
|
38
39
|
EOS
|
@@ -44,6 +45,7 @@ describe Fluent::DynamodbAltOutput do
|
|
44
45
|
expect(driver.instance.table_name ).to eq 'my_table'
|
45
46
|
expect(driver.instance.timestamp_key ).to eq 'timestamp'
|
46
47
|
expect(driver.instance.binary_keys ).to eq ['aaa', 'bbb']
|
48
|
+
expect(driver.instance.delete_key ).to eq 'delete'
|
47
49
|
expect(driver.instance.concurrency ).to eq 2
|
48
50
|
expect(driver.instance.conditional_operator).to eq 'OR'
|
49
51
|
expect(driver.instance.instance_variable_get(:@hash_key) ).to eq 'hash_key'
|
@@ -500,5 +502,90 @@ describe Fluent::DynamodbAltOutput do
|
|
500
502
|
]
|
501
503
|
end
|
502
504
|
}
|
505
|
+
|
506
|
+
context('delete record (1)') {
|
507
|
+
it do
|
508
|
+
run_driver do |d|
|
509
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
|
510
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
|
511
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
|
512
|
+
end
|
513
|
+
|
514
|
+
expect(select_all).to match_array [
|
515
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
516
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
|
517
|
+
{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
|
518
|
+
]
|
519
|
+
|
520
|
+
run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
|
521
|
+
expect(d.instance.log).not_to receive(:warn)
|
522
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625004, 'delete' => true}, time)
|
523
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
|
524
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'delete' => true}, time)
|
525
|
+
end
|
526
|
+
|
527
|
+
expect(select_all).to match_array [
|
528
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
|
529
|
+
]
|
530
|
+
end
|
531
|
+
}
|
532
|
+
|
533
|
+
context('delete record (2)') {
|
534
|
+
it do
|
535
|
+
run_driver do |d|
|
536
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
|
537
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
|
538
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
|
539
|
+
end
|
540
|
+
|
541
|
+
expect(select_all).to match_array [
|
542
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
543
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
|
544
|
+
{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
|
545
|
+
]
|
546
|
+
|
547
|
+
run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
|
548
|
+
expect(d.instance.log).to receive(:warn)
|
549
|
+
.with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :key=>{"id"=>"12345678-1234-1234-1234-123456789001"}, :expected=>{"id"=>{:comparison_operator=>"NULL"}, "timestamp"=>{:comparison_operator=>"LT", :attribute_value_list=>[1409534625001]}}, :conditional_operator=>"OR"}!)
|
550
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'delete' => true}, time)
|
551
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
|
552
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'delete' => true}, time)
|
553
|
+
end
|
554
|
+
|
555
|
+
expect(select_all).to match_array [
|
556
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
557
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
|
558
|
+
]
|
559
|
+
end
|
560
|
+
}
|
561
|
+
|
562
|
+
context('delete empty record') {
|
563
|
+
it do
|
564
|
+
run_driver do |d|
|
565
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
|
566
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
|
567
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
|
568
|
+
end
|
569
|
+
|
570
|
+
expect(select_all).to match_array [
|
571
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
572
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
|
573
|
+
{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
|
574
|
+
]
|
575
|
+
|
576
|
+
run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
|
577
|
+
expect(d.instance.log).not_to receive(:warn)
|
578
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789011', 'timestamp' => 1409534625004, 'delete' => true}, time)
|
579
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
|
580
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789013', 'timestamp' => 1409534625006, 'delete' => true}, time)
|
581
|
+
end
|
582
|
+
|
583
|
+
expect(select_all).to match_array [
|
584
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
585
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
|
586
|
+
{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
|
587
|
+
]
|
588
|
+
end
|
589
|
+
}
|
503
590
|
}
|
504
591
|
end
|