bricolage-streamingload 0.15.0 → 0.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +5 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +61 -0
- data/LICENSES +21 -0
- data/RELEASE.md +135 -0
- data/Rakefile +3 -0
- data/bin/send-data-event +1 -1
- data/bricolage-streamingload.gemspec +27 -0
- data/config/production/database.yml +66 -0
- data/config/production/password.yml +5 -0
- data/config/production/streamingload.yml +20 -0
- data/config/production/variable.yml +5 -0
- data/lib/bricolage/streamingload/dispatcher.rb +8 -1
- data/lib/bricolage/streamingload/job.rb +14 -3
- data/lib/bricolage/streamingload/jobparams.rb +1 -1
- data/lib/bricolage/streamingload/taskhandler.rb +5 -2
- data/lib/bricolage/streamingload/version.rb +1 -1
- data/sample/sqs-message.txt +38 -0
- data/sample/sqs-result.txt +18 -0
- data/strload_load_logs.ct +13 -0
- data/testschema/strload_test.ct +11 -0
- data/testschema/testlog.json.gz +0 -0
- data/testschema/with_work_table.job +4 -0
- data/testschema/with_work_table.sql +1 -0
- data/utils/init_strload_tables.sql +13 -0
- data/utils/strload-stat.sql +36 -0
- metadata +46 -30
- data/test/all.rb +0 -3
- data/test/streamingload/test_dispatcher.rb +0 -241
- data/test/streamingload/test_dispatchermessage.rb +0 -31
- data/test/streamingload/test_job.rb +0 -620
- data/test/test_sqsdatasource.rb +0 -55
data/test/test_sqsdatasource.rb
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
require 'test/unit'
|
|
2
|
-
require 'bricolage/sqsmock'
|
|
3
|
-
require 'bricolage/logger'
|
|
4
|
-
|
|
5
|
-
module Bricolage
|
|
6
|
-
|
|
7
|
-
class TestSQSDataSource < Test::Unit::TestCase
|
|
8
|
-
|
|
9
|
-
test "#delete_message_async" do
|
|
10
|
-
messages = (0..2).map {|seq|
|
|
11
|
-
SQSMock::Message.new(message_id: "message_id_#{seq}", receipt_handle: "receipt_handle_#{seq}")
|
|
12
|
-
}
|
|
13
|
-
ds = SQSDataSource.new_mock(
|
|
14
|
-
delete_message_batch: -> (queue_url:, entries:) {
|
|
15
|
-
if entries.size == 3
|
|
16
|
-
# first time
|
|
17
|
-
assert_equal messages[0].receipt_handle, entries[0][:receipt_handle]
|
|
18
|
-
assert_equal messages[1].receipt_handle, entries[1][:receipt_handle]
|
|
19
|
-
assert_equal messages[2].receipt_handle, entries[2][:receipt_handle]
|
|
20
|
-
SQSMock::DeleteMessageBatchResponse.new.tap {|res|
|
|
21
|
-
res.add_success_for(entries[0])
|
|
22
|
-
res.add_failure_for(entries[1])
|
|
23
|
-
res.add_success_for(entries[2])
|
|
24
|
-
}
|
|
25
|
-
else
|
|
26
|
-
# second time
|
|
27
|
-
SQSMock::DeleteMessageBatchResponse.new.tap {|res|
|
|
28
|
-
res.add_success_for(entries[0])
|
|
29
|
-
}
|
|
30
|
-
end
|
|
31
|
-
}
|
|
32
|
-
)
|
|
33
|
-
|
|
34
|
-
ds.delete_message_async(messages[0])
|
|
35
|
-
ds.delete_message_async(messages[1])
|
|
36
|
-
ds.delete_message_async(messages[2])
|
|
37
|
-
|
|
38
|
-
# first flush
|
|
39
|
-
flush_time = Time.now
|
|
40
|
-
ds.process_async_delete(flush_time)
|
|
41
|
-
delete_buf = ds.__send__(:delete_message_buffer)
|
|
42
|
-
bufent = delete_buf.instance_variable_get(:@buf).values.first
|
|
43
|
-
assert_equal 'receipt_handle_1', bufent.message.receipt_handle
|
|
44
|
-
assert_equal 1, bufent.n_failure
|
|
45
|
-
assert_false bufent.issuable?(flush_time)
|
|
46
|
-
assert_true bufent.issuable?(flush_time + 180)
|
|
47
|
-
|
|
48
|
-
# second flush
|
|
49
|
-
ds.process_async_delete(flush_time + 180)
|
|
50
|
-
assert_true delete_buf.empty?
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
end
|