txgh-queue 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/txgh-queue/application.rb +35 -0
- data/lib/txgh-queue/backends/null.rb +19 -0
- data/lib/txgh-queue/backends/sqs/config.rb +29 -0
- data/lib/txgh-queue/backends/sqs/consumer.rb +30 -0
- data/lib/txgh-queue/backends/sqs/history_sequence.rb +62 -0
- data/lib/txgh-queue/backends/sqs/job.rb +115 -0
- data/lib/txgh-queue/backends/sqs/message_attributes.rb +33 -0
- data/lib/txgh-queue/backends/sqs/producer.rb +33 -0
- data/lib/txgh-queue/backends/sqs/queue.rb +45 -0
- data/lib/txgh-queue/backends/sqs/retry_logic.rb +118 -0
- data/lib/txgh-queue/backends/sqs.rb +36 -0
- data/lib/txgh-queue/backends.rb +22 -0
- data/lib/txgh-queue/config.rb +48 -0
- data/lib/txgh-queue/error_handlers/github.rb +47 -0
- data/lib/txgh-queue/error_handlers/server_response.rb +22 -0
- data/lib/txgh-queue/error_handlers/standard_errors.rb +15 -0
- data/lib/txgh-queue/error_handlers/transifex.rb +23 -0
- data/lib/txgh-queue/error_handlers/txgh_errors.rb +27 -0
- data/lib/txgh-queue/error_handlers.rb +9 -0
- data/lib/txgh-queue/job.rb +77 -0
- data/lib/txgh-queue/result.rb +26 -0
- data/lib/txgh-queue/status.rb +51 -0
- data/lib/txgh-queue/supervisor.rb +76 -0
- data/lib/txgh-queue/version.rb +3 -0
- data/lib/txgh-queue/webhooks/github/request_handler.rb +34 -0
- data/lib/txgh-queue/webhooks/github.rb +7 -0
- data/lib/txgh-queue/webhooks/transifex/request_handler.rb +21 -0
- data/lib/txgh-queue/webhooks/transifex.rb +7 -0
- data/lib/txgh-queue/webhooks.rb +6 -0
- data/lib/txgh-queue.rb +14 -0
- data/spec/application_spec.rb +97 -0
- data/spec/backends/sqs/config_spec.rb +30 -0
- data/spec/backends/sqs/consumer_spec.rb +34 -0
- data/spec/backends/sqs/history_sequence_spec.rb +75 -0
- data/spec/backends/sqs/job_spec.rb +189 -0
- data/spec/backends/sqs/message_attributes_spec.rb +64 -0
- data/spec/backends/sqs/producer_spec.rb +32 -0
- data/spec/backends/sqs/queue_spec.rb +65 -0
- data/spec/backends/sqs/retry_logic_spec.rb +157 -0
- data/spec/backends/sqs_spec.rb +57 -0
- data/spec/backends_spec.rb +31 -0
- data/spec/config_spec.rb +15 -0
- data/spec/error_handlers/github_spec.rb +30 -0
- data/spec/error_handlers/server_response_spec.rb +36 -0
- data/spec/error_handlers/standard_errors_spec.rb +22 -0
- data/spec/error_handlers/transifex_spec.rb +43 -0
- data/spec/error_handlers/txgh_errors_spec.rb +25 -0
- data/spec/helpers/env_helpers.rb +13 -0
- data/spec/helpers/nil_logger.rb +10 -0
- data/spec/helpers/sqs/sqs_test_message.rb +47 -0
- data/spec/helpers/test_backend.rb +54 -0
- data/spec/job_spec.rb +111 -0
- data/spec/result_spec.rb +63 -0
- data/spec/spec_helper.rb +66 -0
- data/spec/status_spec.rb +68 -0
- data/spec/supervisor_spec.rb +40 -0
- data/spec/webhooks/github/request_handler_spec.rb +145 -0
- data/spec/webhooks/transifex/request_handler_spec.rb +87 -0
- data/txgh-queue.gemspec +24 -0
- metadata +172 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ff7b50a82408cdac666a7921a2f9f011bc8d5226
|
4
|
+
data.tar.gz: dbe3735109b47773c7bb8175544273f2edffd86c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8c0b6b954cf71c340094acf5de1f34d33796b3eb92bcd6ac4b13ec5758f3c0f38b97f2a35617e1d224e2fd93881cd554d5cb3ce8bb26e3326b22f47480bd5af8
|
7
|
+
data.tar.gz: d3113158ce321050b04ebe1c6abdb8260039a8cfe00ac6764d45f46d7d4cd2c1da7be04b101674aa9bc4bd234b96465f5e59c5a2c78e096b661784d4fadaed6d
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'sinatra'
|
2
|
+
require 'sinatra/json'
|
3
|
+
|
4
|
+
module TxghQueue
|
5
|
+
module RespondWith
|
6
|
+
def respond_with(resp)
|
7
|
+
env['txgh.response'] = resp
|
8
|
+
status resp.status
|
9
|
+
json resp.body
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class WebhookEndpoints < Sinatra::Base
|
14
|
+
include TxghQueue::Webhooks
|
15
|
+
helpers RespondWith
|
16
|
+
|
17
|
+
configure do
|
18
|
+
set :logging, nil
|
19
|
+
logger = Txgh::TxLogger.logger
|
20
|
+
set :logger, logger
|
21
|
+
end
|
22
|
+
|
23
|
+
post '/transifex/enqueue' do
|
24
|
+
respond_with(
|
25
|
+
Transifex::RequestHandler.handle_request(request, settings.logger)
|
26
|
+
)
|
27
|
+
end
|
28
|
+
|
29
|
+
post '/github/enqueue' do
|
30
|
+
respond_with(
|
31
|
+
Github::RequestHandler.handle_request(request, settings.logger)
|
32
|
+
)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'txgh'
|
2
|
+
|
3
|
+
module TxghQueue
|
4
|
+
module Backends
|
5
|
+
module Null
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def producer_for(*args)
|
9
|
+
raise BackendNotConfiguredError, 'No queue backend has been configured'
|
10
|
+
end
|
11
|
+
|
12
|
+
def consumer_for(*args)
|
13
|
+
raise BackendNotConfiguredError, 'No queue backend has been configured'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module TxghQueue
|
2
|
+
module Backends
|
3
|
+
module Sqs
|
4
|
+
class Config
|
5
|
+
class << self
|
6
|
+
def queues
|
7
|
+
@queues ||= TxghQueue::Config.options[:queues].map do |queue_options|
|
8
|
+
Queue.new(queue_options)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def failure_queue
|
13
|
+
@failure_queue ||= Queue.new(
|
14
|
+
TxghQueue::Config.options[:failure_queue]
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
def get_queue(queue_name)
|
19
|
+
queues.find { |q| q.name == queue_name }
|
20
|
+
end
|
21
|
+
|
22
|
+
def reset!
|
23
|
+
@queues = nil
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'txgh'
|
3
|
+
|
4
|
+
module TxghQueue
|
5
|
+
module Backends
|
6
|
+
module Sqs
|
7
|
+
class Consumer
|
8
|
+
RECEIVE_PARAMS = { message_attribute_names: %w(history_sequence) }
|
9
|
+
|
10
|
+
attr_reader :queues, :logger
|
11
|
+
|
12
|
+
def initialize(queues, logger)
|
13
|
+
@queues = queues
|
14
|
+
@logger = logger
|
15
|
+
end
|
16
|
+
|
17
|
+
def work
|
18
|
+
queues.each do |queue|
|
19
|
+
queue.receive_message(RECEIVE_PARAMS).messages.each do |message|
|
20
|
+
logger.info("Received message from #{queue.name}, id: #{message.message_id}")
|
21
|
+
Job.new(message, queue, logger).complete
|
22
|
+
end
|
23
|
+
end
|
24
|
+
rescue => e
|
25
|
+
Txgh.events.publish_error!(e)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'txgh'
|
3
|
+
|
4
|
+
module TxghQueue
|
5
|
+
module Backends
|
6
|
+
module Sqs
|
7
|
+
class HistorySequence
|
8
|
+
extend Forwardable
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
class << self
|
12
|
+
def from_message(message)
|
13
|
+
if attribute = message.message_attributes['history_sequence']
|
14
|
+
new(JSON.parse(attribute.string_value))
|
15
|
+
else
|
16
|
+
new([])
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def from_h(hash)
|
21
|
+
new(
|
22
|
+
JSON.parse(
|
23
|
+
Txgh::Utils.deep_symbolize_keys(hash)
|
24
|
+
.fetch(:history_sequence, {})
|
25
|
+
.fetch(:string_value, nil)
|
26
|
+
)
|
27
|
+
)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
attr_reader :sequence
|
32
|
+
|
33
|
+
def_delegators :sequence, :[], :<<, :first, :last, :size, :length
|
34
|
+
|
35
|
+
def initialize(sequence)
|
36
|
+
@sequence = Txgh::Utils.deep_symbolize_keys(sequence)
|
37
|
+
end
|
38
|
+
|
39
|
+
def add(obj)
|
40
|
+
sequence << obj
|
41
|
+
end
|
42
|
+
|
43
|
+
def to_h
|
44
|
+
{ string_value: sequence.to_json, data_type: 'String' }
|
45
|
+
end
|
46
|
+
|
47
|
+
def dup
|
48
|
+
# use marshal serialization to deep copy the sequence
|
49
|
+
self.class.new(Marshal.load(Marshal.dump(sequence)))
|
50
|
+
end
|
51
|
+
|
52
|
+
def current
|
53
|
+
sequence.last
|
54
|
+
end
|
55
|
+
|
56
|
+
def each(&block)
|
57
|
+
sequence.each(&block)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module TxghQueue
|
4
|
+
module Backends
|
5
|
+
module Sqs
|
6
|
+
class Job < TxghQueue::Job
|
7
|
+
attr_reader :message, :message_attributes, :queue, :logger
|
8
|
+
|
9
|
+
def initialize(message, queue, logger)
|
10
|
+
@message = message
|
11
|
+
@queue = queue
|
12
|
+
@message_attributes = MessageAttributes.from_message(message)
|
13
|
+
|
14
|
+
# add empty retry attributes hash to sequence - will be populated when
|
15
|
+
# the complete method is called
|
16
|
+
message_attributes.history_sequence.add({})
|
17
|
+
|
18
|
+
super(logger)
|
19
|
+
end
|
20
|
+
|
21
|
+
def complete
|
22
|
+
result = process(payload)
|
23
|
+
logger.info("Finished processing #{message.message_id}, result: #{result.status}")
|
24
|
+
|
25
|
+
message_attributes.history_sequence.current.merge!(attributes_for(result))
|
26
|
+
|
27
|
+
return do_ok(result) if result.status.ok?
|
28
|
+
return do_retry(result) if result.status.retry?
|
29
|
+
return do_fail(result) if result.status.fail?
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def attributes_for(result)
|
35
|
+
{ status: result.status.to_s }.merge(
|
36
|
+
if result.has_error?
|
37
|
+
error_attributes_for(result)
|
38
|
+
else
|
39
|
+
response_attributes_for(result)
|
40
|
+
end
|
41
|
+
)
|
42
|
+
end
|
43
|
+
|
44
|
+
def error_attributes_for(result)
|
45
|
+
{
|
46
|
+
response_type: 'error',
|
47
|
+
class: result.error.class,
|
48
|
+
message: result.error.message,
|
49
|
+
backtrace: (result.error.backtrace || []).first,
|
50
|
+
error_tracking: publish_error_for(result)
|
51
|
+
}
|
52
|
+
end
|
53
|
+
|
54
|
+
def response_attributes_for(result)
|
55
|
+
{
|
56
|
+
response_type: 'response',
|
57
|
+
code: result.response.status,
|
58
|
+
body: result.response.body
|
59
|
+
}
|
60
|
+
end
|
61
|
+
|
62
|
+
def payload
|
63
|
+
@payload ||= begin
|
64
|
+
logger.info("Processing #{message.message_id}")
|
65
|
+
|
66
|
+
JSON.parse(message.body).tap do |payload|
|
67
|
+
logger.info("Payload: #{payload.inspect}")
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def do_ok(result)
|
73
|
+
delete(message)
|
74
|
+
logger.info("Deleted #{message.message_id}")
|
75
|
+
end
|
76
|
+
|
77
|
+
def do_retry(result)
|
78
|
+
retry_logic = RetryLogic.new(message_attributes, result.status)
|
79
|
+
|
80
|
+
if retry_logic.retry?
|
81
|
+
logger.info("Retrying #{message.message_id} with #{retry_logic.next_delay_seconds} second delay")
|
82
|
+
new_message = queue.send_message(message.body, retry_logic.sqs_retry_params)
|
83
|
+
logger.info("Re-enqueued as #{new_message.message_id}")
|
84
|
+
delete(message)
|
85
|
+
logger.info("Deleted original #{message.message_id}")
|
86
|
+
elsif retry_logic.retries_exceeded?
|
87
|
+
logger.info("Message #{message.message_id} has exceeded allowed retries.")
|
88
|
+
do_fail(result)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def do_fail(result)
|
93
|
+
# send to failure queue
|
94
|
+
new_message = Config.failure_queue.send_message(
|
95
|
+
message.body, message_attributes: message_attributes.to_h
|
96
|
+
)
|
97
|
+
|
98
|
+
delete(message)
|
99
|
+
end
|
100
|
+
|
101
|
+
def publish_error_for(result)
|
102
|
+
Txgh.events.publish_error(result.error, {
|
103
|
+
payload: payload,
|
104
|
+
message_id: message.message_id,
|
105
|
+
queue: queue.name
|
106
|
+
})
|
107
|
+
end
|
108
|
+
|
109
|
+
def delete(message)
|
110
|
+
queue.delete_message(message.receipt_handle)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module TxghQueue
|
2
|
+
module Backends
|
3
|
+
module Sqs
|
4
|
+
class MessageAttributes
|
5
|
+
class << self
|
6
|
+
def from_message(message)
|
7
|
+
history_sequence = HistorySequence.from_message(message)
|
8
|
+
new(history_sequence)
|
9
|
+
end
|
10
|
+
|
11
|
+
def from_h(hash)
|
12
|
+
history_sequence = HistorySequence.from_h(hash)
|
13
|
+
new(history_sequence)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :history_sequence
|
18
|
+
|
19
|
+
def initialize(history_sequence)
|
20
|
+
@history_sequence = history_sequence
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_h
|
24
|
+
{ history_sequence: history_sequence.to_h }
|
25
|
+
end
|
26
|
+
|
27
|
+
def dup
|
28
|
+
self.class.new(history_sequence.dup)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module TxghQueue
|
4
|
+
module Backends
|
5
|
+
module Sqs
|
6
|
+
class Producer
|
7
|
+
attr_reader :queues, :logger
|
8
|
+
|
9
|
+
def initialize(queues, logger)
|
10
|
+
@queues = queues
|
11
|
+
@logger = logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def enqueue(payload, options = {})
|
15
|
+
payload_json = payload.to_json
|
16
|
+
|
17
|
+
message_ids = queues.map do |queue|
|
18
|
+
new_message = queue.send_message(payload_json, options)
|
19
|
+
|
20
|
+
logger.info(
|
21
|
+
"Enqueued new message with id #{new_message.message_id} and params "\
|
22
|
+
"#{payload_json}"
|
23
|
+
)
|
24
|
+
|
25
|
+
new_message.message_id
|
26
|
+
end
|
27
|
+
|
28
|
+
{ message_ids: message_ids }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module TxghQueue
|
4
|
+
module Backends
|
5
|
+
module Sqs
|
6
|
+
class Queue
|
7
|
+
attr_reader :name, :region, :events
|
8
|
+
|
9
|
+
def initialize(options = {})
|
10
|
+
@name = options.fetch(:name)
|
11
|
+
@region = options.fetch(:region)
|
12
|
+
@events = options.fetch(:events, [])
|
13
|
+
end
|
14
|
+
|
15
|
+
def client
|
16
|
+
@client ||= Aws::SQS::Client.new(region: region)
|
17
|
+
end
|
18
|
+
|
19
|
+
def url
|
20
|
+
@url ||= client.get_queue_url(queue_name: name).queue_url
|
21
|
+
end
|
22
|
+
|
23
|
+
def receive_message(options = {})
|
24
|
+
client.receive_message(options.merge(queue_url: url))
|
25
|
+
end
|
26
|
+
|
27
|
+
def send_message(body, options = {})
|
28
|
+
params = options.merge(message_body: body, queue_url: url)
|
29
|
+
client.send_message(params)
|
30
|
+
end
|
31
|
+
|
32
|
+
def delete_message(receipt_handle)
|
33
|
+
params = { queue_url: url, receipt_handle: receipt_handle }
|
34
|
+
client.delete_message(params)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def sqs_attributes
|
40
|
+
@sqs_attributes ||= client.get_queue_attributes(queue_url: url)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module TxghQueue
|
4
|
+
module Backends
|
5
|
+
module Sqs
|
6
|
+
class RetriesExceededError < StandardError; end
|
7
|
+
|
8
|
+
class RetryLogic
|
9
|
+
SEQUENTIAL_MAX_RETRIES = 5
|
10
|
+
OVERALL_MAX_RETRIES = 15
|
11
|
+
|
12
|
+
# SQS max is 15 minutes, or 900 seconds
|
13
|
+
DELAY_INTERVALS = [16, 32, 64, 128, 256, 512, 900]
|
14
|
+
|
15
|
+
attr_reader :message_attributes, :current_status
|
16
|
+
|
17
|
+
def initialize(message_attributes, current_status)
|
18
|
+
@message_attributes = message_attributes.dup
|
19
|
+
@current_status = current_status
|
20
|
+
end
|
21
|
+
|
22
|
+
def retry?
|
23
|
+
retry_with_delay? || retry_without_delay?
|
24
|
+
end
|
25
|
+
|
26
|
+
def retries_exceeded?
|
27
|
+
max_overall_retries_exceeded? ||
|
28
|
+
max_sequential_retries_exceeded? ||
|
29
|
+
max_sequential_delays_exceeded?
|
30
|
+
end
|
31
|
+
|
32
|
+
def next_delay_seconds
|
33
|
+
raise RetriesExceededError unless retry?
|
34
|
+
return 0 unless retry_with_delay?
|
35
|
+
DELAY_INTERVALS[[current_sequence.size - 1, 0].max]
|
36
|
+
end
|
37
|
+
|
38
|
+
def sqs_retry_params
|
39
|
+
if retry_without_delay?
|
40
|
+
sqs_params_without_delay
|
41
|
+
elsif retry_with_delay?
|
42
|
+
sqs_params_with_delay
|
43
|
+
else
|
44
|
+
raise RetriesExceededError
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def retry_with_delay?
|
51
|
+
return false if max_overall_retries_exceeded?
|
52
|
+
return false unless current_status.retry_with_delay?
|
53
|
+
!max_sequential_delays_exceeded?
|
54
|
+
end
|
55
|
+
|
56
|
+
def retry_without_delay?
|
57
|
+
return false if max_overall_retries_exceeded?
|
58
|
+
return false unless current_status.retry_without_delay?
|
59
|
+
!max_sequential_retries_exceeded?
|
60
|
+
end
|
61
|
+
|
62
|
+
def sqs_params_without_delay
|
63
|
+
{ message_attributes: message_attributes.to_h }
|
64
|
+
end
|
65
|
+
|
66
|
+
def sqs_params_with_delay
|
67
|
+
sqs_params_without_delay.merge(delay_seconds: next_delay_seconds)
|
68
|
+
end
|
69
|
+
|
70
|
+
def max_overall_retries_exceeded?
|
71
|
+
history_sequence.size >= OVERALL_MAX_RETRIES
|
72
|
+
end
|
73
|
+
|
74
|
+
def max_sequential_delays_exceeded?
|
75
|
+
delay_sequence_will_continue? && current_sequence.size >= DELAY_INTERVALS.size
|
76
|
+
end
|
77
|
+
|
78
|
+
def max_sequential_retries_exceeded?
|
79
|
+
retry_sequence_will_continue? && current_sequence.size >= SEQUENTIAL_MAX_RETRIES
|
80
|
+
end
|
81
|
+
|
82
|
+
def retry_sequence_will_continue?
|
83
|
+
current_status.status == last_status &&
|
84
|
+
current_status.retry_without_delay?
|
85
|
+
end
|
86
|
+
|
87
|
+
def delay_sequence_will_continue?
|
88
|
+
current_status.status == last_status &&
|
89
|
+
current_status.retry_with_delay?
|
90
|
+
end
|
91
|
+
|
92
|
+
def last_status
|
93
|
+
rt = current_sequence.last
|
94
|
+
rt.to_sym if rt
|
95
|
+
end
|
96
|
+
|
97
|
+
def current_sequence
|
98
|
+
return [] unless sequence = chunked_history_sequence.last
|
99
|
+
return [] unless last_elem = sequence.last
|
100
|
+
return [] unless current_status.status.to_s == last_elem
|
101
|
+
sequence
|
102
|
+
end
|
103
|
+
|
104
|
+
def chunked_history_sequence
|
105
|
+
@chunked_history_sequence ||= history_sequence.lazy
|
106
|
+
.map { |elem| elem[:status] }
|
107
|
+
.chunk(&:itself)
|
108
|
+
.map(&:last)
|
109
|
+
.to_a
|
110
|
+
end
|
111
|
+
|
112
|
+
def history_sequence
|
113
|
+
message_attributes.history_sequence
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'txgh'
|
2
|
+
|
3
|
+
module TxghQueue
|
4
|
+
module Backends
|
5
|
+
module Sqs
|
6
|
+
autoload :Config, 'txgh-queue/backends/sqs/config'
|
7
|
+
autoload :Consumer, 'txgh-queue/backends/sqs/consumer'
|
8
|
+
autoload :HistorySequence, 'txgh-queue/backends/sqs/history_sequence'
|
9
|
+
autoload :Job, 'txgh-queue/backends/sqs/job'
|
10
|
+
autoload :MessageAttributes, 'txgh-queue/backends/sqs/message_attributes'
|
11
|
+
autoload :Producer, 'txgh-queue/backends/sqs/producer'
|
12
|
+
autoload :Queue, 'txgh-queue/backends/sqs/queue'
|
13
|
+
autoload :RetryLogic, 'txgh-queue/backends/sqs/retry_logic'
|
14
|
+
|
15
|
+
class << self
|
16
|
+
def producer_for(events, logger = Txgh::TxLogger.logger)
|
17
|
+
Producer.new(find_queues_for(Array(events)), logger)
|
18
|
+
end
|
19
|
+
|
20
|
+
def consumer_for(events, logger = Txgh::TxLogger.logger)
|
21
|
+
Consumer.new(find_queues_for(Array(events)), logger)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def find_queues_for(events)
|
27
|
+
queues = events.flat_map do |event|
|
28
|
+
Config.queues.select { |queue| queue.events.include?(event) }
|
29
|
+
end
|
30
|
+
|
31
|
+
queues.uniq
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module TxghQueue
|
2
|
+
module Backends
|
3
|
+
autoload :Null, 'txgh-queue/backends/null'
|
4
|
+
autoload :Sqs, 'txgh-queue/backends/sqs'
|
5
|
+
|
6
|
+
class BackendNotConfiguredError < StandardError; end
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def register(name, klass)
|
10
|
+
all[name] = klass
|
11
|
+
end
|
12
|
+
|
13
|
+
def get(name)
|
14
|
+
all[name]
|
15
|
+
end
|
16
|
+
|
17
|
+
def all
|
18
|
+
@all ||= {}
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module TxghQueue
|
4
|
+
class Config
|
5
|
+
DEFAULT_BACKEND = 'null'
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def backend
|
9
|
+
TxghQueue::Backends.get(
|
10
|
+
raw_config.fetch(:backend, DEFAULT_BACKEND)
|
11
|
+
)
|
12
|
+
end
|
13
|
+
|
14
|
+
def processing_enabled?
|
15
|
+
raw_config.fetch(:processing_enabled, true)
|
16
|
+
end
|
17
|
+
|
18
|
+
def options
|
19
|
+
raw_config.fetch(:options, {})
|
20
|
+
end
|
21
|
+
|
22
|
+
def reset!
|
23
|
+
@raw_config = nil
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def raw_config
|
29
|
+
@raw_config ||= begin
|
30
|
+
if ENV['TXGH_QUEUE_CONFIG']
|
31
|
+
scheme, payload = ENV['TXGH_QUEUE_CONFIG'].split('://')
|
32
|
+
send(:"load_#{scheme}", payload)
|
33
|
+
else
|
34
|
+
{}
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def load_file(payload)
|
40
|
+
Txgh::Utils.deep_symbolize_keys(YAML.load_file(payload))
|
41
|
+
end
|
42
|
+
|
43
|
+
def load_raw(payload)
|
44
|
+
Txgh::Utils.deep_symbolize_keys(YAML.load(payload))
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'octokit'
|
2
|
+
|
3
|
+
module TxghQueue
|
4
|
+
module ErrorHandlers
|
5
|
+
class Github
|
6
|
+
ERROR_CLASSES = {
|
7
|
+
Octokit::AbuseDetected => Status.fail,
|
8
|
+
Octokit::BadGateway => Status.retry_without_delay,
|
9
|
+
Octokit::BadRequest => Status.fail,
|
10
|
+
Octokit::ClientError => Status.fail,
|
11
|
+
Octokit::Conflict => Status.fail,
|
12
|
+
Octokit::Forbidden => Status.fail,
|
13
|
+
Octokit::InternalServerError => Status.retry_with_delay,
|
14
|
+
Octokit::MethodNotAllowed => Status.fail,
|
15
|
+
Octokit::NotAcceptable => Status.fail,
|
16
|
+
Octokit::NotFound => Status.fail,
|
17
|
+
Octokit::NotImplemented => Status.fail,
|
18
|
+
Octokit::OneTimePasswordRequired => Status.fail,
|
19
|
+
Octokit::RepositoryUnavailable => Status.retry_with_delay,
|
20
|
+
Octokit::ServerError => Status.retry_with_delay,
|
21
|
+
Octokit::ServiceUnavailable => Status.retry_with_delay,
|
22
|
+
Octokit::TooManyLoginAttempts => Status.retry_with_delay,
|
23
|
+
Octokit::TooManyRequests => Status.retry_with_delay,
|
24
|
+
Octokit::Unauthorized => Status.fail,
|
25
|
+
Octokit::UnprocessableEntity => Status.fail,
|
26
|
+
Octokit::UnsupportedMediaType => Status.fail,
|
27
|
+
Octokit::UnverifiedEmail => Status.fail
|
28
|
+
}
|
29
|
+
|
30
|
+
class << self
|
31
|
+
def can_handle?(error_or_response)
|
32
|
+
error_or_response.is_a?(Octokit::Error)
|
33
|
+
end
|
34
|
+
|
35
|
+
def status_for(error)
|
36
|
+
ERROR_CLASSES.fetch(error.class) { handle_other(error) }
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def handle_other(error)
|
42
|
+
Status.fail
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|