fluent-plugin-dynamodb-alt 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/fluent-plugin-dynamodb-alt.gemspec +1 -1
- data/lib/fluent/plugin/out_dynamodb_alt.rb +30 -1
- data/spec/out_dynamodb_alt_spec.rb +87 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f647d44ff98f3042dd62d8eb86456a117be83db
|
4
|
+
data.tar.gz: a66fa211bd4fe96e91b074541de9934f23ac252d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d3becb79c07015ba503af5c99c0492427c8f7d77358544874685a16b09a322cf72de87388a1416f2a99bdaca7007ceca0d73e715488ffa524ab917a96c6872bc
|
7
|
+
data.tar.gz: 13c874044ed23568ce0262a5c5867240fe80fae3839c13a311e36b0807a793fd826f88f96e78b69aa9e1b5117f72ef1e31aa09ab34bb30d5856a016382be2a30
|
data/README.md
CHANGED
@@ -32,6 +32,7 @@ bundle exec rake install
|
|
32
32
|
#binary_keys data1,data2
|
33
33
|
#endpoint http:://localhost:4567
|
34
34
|
#concurrency 1
|
35
|
+
#delete_key delete
|
35
36
|
|
36
37
|
# see http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-Expected
|
37
38
|
#expected id NULL,timestamp LT ${timestamp},key EQ "val"
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = 'fluent-plugin-dynamodb-alt'
|
7
|
-
spec.version = '0.1.
|
7
|
+
spec.version = '0.1.4'
|
8
8
|
spec.authors = ['Genki Sugawara']
|
9
9
|
spec.email = ['sgwr_dts@yahoo.co.jp']
|
10
10
|
spec.summary = %q{Fluent plugin to output to DynamoDB.}
|
@@ -17,6 +17,7 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
|
|
17
17
|
config_param :table_name, :string
|
18
18
|
config_param :timestamp_key, :string
|
19
19
|
config_param :binary_keys, :string, :default => nil
|
20
|
+
config_param :delete_key, :string, :default => nil
|
20
21
|
config_param :concurrency, :integer, :default => 1
|
21
22
|
config_param :expected, :string, :default => nil
|
22
23
|
config_param :conditional_operator, :string, :default => 'AND'
|
@@ -89,8 +90,13 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
|
|
89
90
|
|
90
91
|
def write(chunk)
|
91
92
|
chunk = aggregate_records(chunk)
|
93
|
+
|
92
94
|
block = proc do |tag, time, record|
|
93
|
-
|
95
|
+
if @delete_key and record[@delete_key]
|
96
|
+
delete_record(record)
|
97
|
+
else
|
98
|
+
put_record(record)
|
99
|
+
end
|
94
100
|
end
|
95
101
|
|
96
102
|
if @concurrency > 1
|
@@ -133,6 +139,29 @@ class Fluent::DynamodbAltOutput < Fluent::BufferedOutput
|
|
133
139
|
end
|
134
140
|
end
|
135
141
|
|
142
|
+
def delete_record(record)
|
143
|
+
key = {@hash_key => record[@hash_key]}
|
144
|
+
key[@range_key] = record[@range_key] if @range_key
|
145
|
+
|
146
|
+
item = {
|
147
|
+
:table_name => @table_name,
|
148
|
+
:key => key
|
149
|
+
}
|
150
|
+
|
151
|
+
begin
|
152
|
+
if @expected
|
153
|
+
expected = create_expected(record)
|
154
|
+
return unless expected
|
155
|
+
item[:expected] = expected
|
156
|
+
item[:conditional_operator] = @conditional_operator if expected.length > 1
|
157
|
+
end
|
158
|
+
|
159
|
+
@client.delete_item(item)
|
160
|
+
rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException, Aws::DynamoDB::Errors::ValidationException => e
|
161
|
+
log.warn("#{e.message}: #{item.inspect}")
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
136
165
|
def validate_record(record)
|
137
166
|
if not record[@hash_key]
|
138
167
|
log.warn("Hash Key '#{@hash_key}' does not exist in the record: #{record.inspect}")
|
@@ -33,6 +33,7 @@ describe Fluent::DynamodbAltOutput do
|
|
33
33
|
table_name my_table
|
34
34
|
timestamp_key timestamp
|
35
35
|
binary_keys aaa,bbb
|
36
|
+
delete_key delete
|
36
37
|
concurrency 2
|
37
38
|
conditional_operator OR
|
38
39
|
EOS
|
@@ -44,6 +45,7 @@ describe Fluent::DynamodbAltOutput do
|
|
44
45
|
expect(driver.instance.table_name ).to eq 'my_table'
|
45
46
|
expect(driver.instance.timestamp_key ).to eq 'timestamp'
|
46
47
|
expect(driver.instance.binary_keys ).to eq ['aaa', 'bbb']
|
48
|
+
expect(driver.instance.delete_key ).to eq 'delete'
|
47
49
|
expect(driver.instance.concurrency ).to eq 2
|
48
50
|
expect(driver.instance.conditional_operator).to eq 'OR'
|
49
51
|
expect(driver.instance.instance_variable_get(:@hash_key) ).to eq 'hash_key'
|
@@ -500,5 +502,90 @@ describe Fluent::DynamodbAltOutput do
|
|
500
502
|
]
|
501
503
|
end
|
502
504
|
}
|
505
|
+
|
506
|
+
context('delete record (1)') {
|
507
|
+
it do
|
508
|
+
run_driver do |d|
|
509
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
|
510
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
|
511
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
|
512
|
+
end
|
513
|
+
|
514
|
+
expect(select_all).to match_array [
|
515
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
516
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
|
517
|
+
{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
|
518
|
+
]
|
519
|
+
|
520
|
+
run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
|
521
|
+
expect(d.instance.log).not_to receive(:warn)
|
522
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625004, 'delete' => true}, time)
|
523
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
|
524
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'delete' => true}, time)
|
525
|
+
end
|
526
|
+
|
527
|
+
expect(select_all).to match_array [
|
528
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
|
529
|
+
]
|
530
|
+
end
|
531
|
+
}
|
532
|
+
|
533
|
+
context('delete record (2)') {
|
534
|
+
it do
|
535
|
+
run_driver do |d|
|
536
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
|
537
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
|
538
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
|
539
|
+
end
|
540
|
+
|
541
|
+
expect(select_all).to match_array [
|
542
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
543
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
|
544
|
+
{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
|
545
|
+
]
|
546
|
+
|
547
|
+
run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
|
548
|
+
expect(d.instance.log).to receive(:warn)
|
549
|
+
.with(%!The conditional request failed: {:table_name=>"#{TEST_TABLE_NAME}", :key=>{"id"=>"12345678-1234-1234-1234-123456789001"}, :expected=>{"id"=>{:comparison_operator=>"NULL"}, "timestamp"=>{:comparison_operator=>"LT", :attribute_value_list=>[1409534625001]}}, :conditional_operator=>"OR"}!)
|
550
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001, 'delete' => true}, time)
|
551
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
|
552
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625006, 'delete' => true}, time)
|
553
|
+
end
|
554
|
+
|
555
|
+
expect(select_all).to match_array [
|
556
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
557
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
|
558
|
+
]
|
559
|
+
end
|
560
|
+
}
|
561
|
+
|
562
|
+
context('delete empty record') {
|
563
|
+
it do
|
564
|
+
run_driver do |d|
|
565
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789001', 'timestamp' => 1409534625001}, time)
|
566
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625002}, time)
|
567
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789003', 'timestamp' => 1409534625003}, time)
|
568
|
+
end
|
569
|
+
|
570
|
+
expect(select_all).to match_array [
|
571
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
572
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625002},
|
573
|
+
{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
|
574
|
+
]
|
575
|
+
|
576
|
+
run_driver(:expected => 'id NULL,timestamp LT ${timestamp}', :conditional_operator => 'OR', :delete_key => 'delete') do |d|
|
577
|
+
expect(d.instance.log).not_to receive(:warn)
|
578
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789011', 'timestamp' => 1409534625004, 'delete' => true}, time)
|
579
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789002', 'timestamp' => 1409534625005, 'key' => 'val'}, time)
|
580
|
+
d.emit({'id' => '12345678-1234-1234-1234-123456789013', 'timestamp' => 1409534625006, 'delete' => true}, time)
|
581
|
+
end
|
582
|
+
|
583
|
+
expect(select_all).to match_array [
|
584
|
+
{"id"=>"12345678-1234-1234-1234-123456789001", "timestamp"=>1409534625001},
|
585
|
+
{"id"=>"12345678-1234-1234-1234-123456789002", "timestamp"=>1409534625005, 'key' => 'val'},
|
586
|
+
{"id"=>"12345678-1234-1234-1234-123456789003", "timestamp"=>1409534625003},
|
587
|
+
]
|
588
|
+
end
|
589
|
+
}
|
503
590
|
}
|
504
591
|
end
|