icalia-sdk-event-notification 0.1.13 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/icalia-sdk-event-notification.rb +0 -24
- data/lib/icalia-sdk-event-notification/publisher.rb +40 -26
- data/lib/icalia-sdk-event-notification/version.rb +1 -1
- metadata +7 -28
- data/lib/icalia-sdk-event-notification/shoryuken.rb +0 -9
- data/lib/icalia-sdk-event-notification/shoryuken/cli.rb +0 -9
- data/lib/icalia-sdk-event-notification/shoryuken/cli/base.rb +0 -39
- data/lib/icalia-sdk-event-notification/shoryuken/cli/runner.rb +0 -49
- data/lib/icalia-sdk-event-notification/shoryuken/cli/sqs.rb +0 -214
- data/lib/icalia-sdk-event-notification/shoryuken/worker.rb +0 -63
- data/lib/icalia-sdk-event-notification/shoryuken/worker_registry.rb +0 -44
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef20f61096dca725e5d61ebc6cc7629b33b694ea61142e4a4bdf331dacff2a2c
|
4
|
+
data.tar.gz: 4fb4d685249caf1dd80ed0322a2a412c0c3053e3876f769b770d56f9a749b9b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6ba2954b5171714bfb90a1e21e80efc3fb3af118ac8d48d42f775af1028d6e292432a197447af9e847e432c8109cade762c4ec367f94cea36359ff1c55a44d6d
|
7
|
+
data.tar.gz: 7b0b343f021ae429163c58d17545495572a3aa6acba53570981663a6d1a983cd5b07d6d1007595a963d2cac529562591e00f7a54dd2089bfa81f1ca671b809a8
|
@@ -11,33 +11,9 @@ module Icalia
|
|
11
11
|
autoload :Publisher, 'icalia-sdk-event-notification/publisher'
|
12
12
|
autoload :Notification, 'icalia-sdk-event-notification/notification'
|
13
13
|
|
14
|
-
autoload :Shoryuken, 'icalia-sdk-event-notification/shoryuken'
|
15
|
-
|
16
14
|
autoload :TopicMessageProcessing,
|
17
15
|
'icalia-sdk-event-notification/topic_message_processing'
|
18
16
|
|
19
|
-
def self.generic_aws_client_options
|
20
|
-
{
|
21
|
-
region: ENV.fetch('AWS_REGION', 'us-west-2'),
|
22
|
-
access_key_id: ENV.fetch('AWS_ACCESS_KEY_ID'),
|
23
|
-
secret_access_key: ENV.fetch('AWS_SECRET_ACCESS_KEY')
|
24
|
-
}
|
25
|
-
end
|
26
|
-
|
27
|
-
def self.sqs_client_options
|
28
|
-
options = Icalia::Event.generic_aws_client_options
|
29
|
-
return options if ENV['AWS_SQS_ENDPOINT'].blank?
|
30
|
-
|
31
|
-
options.merge endpoint: ENV['AWS_SQS_ENDPOINT'], verify_checksums: false
|
32
|
-
end
|
33
|
-
|
34
|
-
def self.sns_client_options
|
35
|
-
options = Icalia::Event.generic_aws_client_options
|
36
|
-
return options if ENV['AWS_SNS_ENDPOINT'].blank?
|
37
|
-
|
38
|
-
options.merge endpoint: ENV['AWS_SQS_ENDPOINT']
|
39
|
-
end
|
40
|
-
|
41
17
|
mattr_reader :deferred_publishing, default: true
|
42
18
|
end
|
43
19
|
end
|
@@ -1,16 +1,21 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'google/cloud/pubsub'
|
4
4
|
|
5
5
|
require 'active_support'
|
6
6
|
require 'active_support/logger'
|
7
7
|
|
8
8
|
module Icalia::Event
|
9
|
-
|
9
|
+
module GRPCRailsLogger
|
10
|
+
def logger; Rails.logger; end
|
11
|
+
end
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
+
module GRPCStdoutLogger
|
14
|
+
LOGGER = ActiveSupport::Logger.new(STDOUT)
|
15
|
+
def logger; LOGGER; end
|
16
|
+
end
|
13
17
|
|
18
|
+
class Publisher
|
14
19
|
class << self
|
15
20
|
def instance
|
16
21
|
@instance ||= new
|
@@ -19,41 +24,48 @@ module Icalia::Event
|
|
19
24
|
delegate :publish, to: :instance
|
20
25
|
end
|
21
26
|
|
22
|
-
attr_reader :
|
27
|
+
attr_reader :client, :logger
|
28
|
+
|
29
|
+
delegate :encode, to: ActiveSupport::JSON, prefix: :json
|
23
30
|
|
24
31
|
def initialize
|
25
|
-
|
26
|
-
|
32
|
+
initialize_pubsub_client
|
33
|
+
initialize_logger
|
27
34
|
end
|
28
35
|
|
29
|
-
def publish(topic_name, data, json_data: true)
|
30
|
-
raise UnknownTopicError, "Topic '#{topic_name}' does not exist" \
|
31
|
-
unless topic_exists?(topic_name)
|
32
|
-
|
33
|
-
topic = get_topic topic_name
|
36
|
+
def publish(topic_name, data, attributes = { json_data: true })
|
34
37
|
logger.debug "Publishing to '#{topic_name}': #{data.inspect}"
|
35
|
-
|
38
|
+
encoded_data = attributes[:json_data] ? json_encode(data) : data
|
39
|
+
get_or_create_topic(topic_name).publish(encoded_data, attributes)
|
36
40
|
end
|
37
41
|
|
38
|
-
delegate :encode, to: ActiveSupport::JSON, prefix: :json
|
39
|
-
|
40
42
|
protected
|
41
43
|
|
42
|
-
def
|
43
|
-
|
44
|
-
topic.arn.split(':').last == topic_name
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def topic_exists?(topic_name)
|
49
|
-
get_topic(topic_name).present?
|
44
|
+
def get_or_create_topic(topic_name)
|
45
|
+
client.topic(topic_name) || client.new_topic(topic_name)
|
50
46
|
end
|
51
47
|
|
52
|
-
|
53
|
-
|
48
|
+
private
|
49
|
+
|
50
|
+
# See https://googleapis.dev/ruby/google-cloud-pubsub/latest/file.AUTHENTICATION.html#Environment_Variables
|
51
|
+
#
|
52
|
+
# The environment variables that google-cloud-pubsub checks for project ID are:
|
53
|
+
# 1. PUBSUB_PROJECT
|
54
|
+
# 2. GOOGLE_CLOUD_PROJECT
|
55
|
+
#
|
56
|
+
# The environment variables that google-cloud-pubsub checks for credentials
|
57
|
+
# are configured on Google::Cloud::PubSub::V1::Credentials:
|
58
|
+
#
|
59
|
+
# 1. PUBSUB_CREDENTIALS - Path to JSON file, or JSON contents
|
60
|
+
# 2. PUBSUB_KEYFILE - Path to JSON file, or JSON contents
|
61
|
+
# 3. GOOGLE_CLOUD_CREDENTIALS - Path to JSON file, or JSON contents
|
62
|
+
# 4. GOOGLE_CLOUD_KEYFILE - Path to JSON file, or JSON contents
|
63
|
+
# 5. GOOGLE_APPLICATION_CREDENTIALS - Path to JSON file
|
64
|
+
def initialize_pubsub_client
|
65
|
+
@client = Google::Cloud::Pubsub.new
|
54
66
|
end
|
55
67
|
|
56
|
-
def
|
68
|
+
def initialize_logger
|
57
69
|
return use_rails_logger if rails_logger_exist?
|
58
70
|
use_default_logger
|
59
71
|
end
|
@@ -64,10 +76,12 @@ module Icalia::Event
|
|
64
76
|
|
65
77
|
def use_rails_logger
|
66
78
|
@logger = Rails.logger
|
79
|
+
GRPCRailsLogger.send :extend, GRPCRailsLogger
|
67
80
|
end
|
68
81
|
|
69
82
|
def use_default_logger
|
70
83
|
@logger = ActiveSupport::Logger.new(STDOUT)
|
84
|
+
GRPCStdoutLogger.send :extend, GRPCStdoutLogger
|
71
85
|
end
|
72
86
|
end
|
73
87
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: icalia-sdk-event-notification
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Roberto Quintanilla
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: icalia-sdk-event-core
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - '='
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.2.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.2.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: activesupport
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -39,33 +39,19 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 5.2.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: google-cloud-pubsub
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '0.31'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
|
-
name: aws-sdk-sqs
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
58
|
-
requirements:
|
59
|
-
- - "~>"
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: '1.20'
|
62
|
-
type: :runtime
|
63
|
-
prerelease: false
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
65
|
-
requirements:
|
66
|
-
- - "~>"
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: '1.20'
|
54
|
+
version: '0.31'
|
69
55
|
- !ruby/object:Gem::Dependency
|
70
56
|
name: bundler
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -119,13 +105,6 @@ files:
|
|
119
105
|
- lib/icalia-sdk-event-notification.rb
|
120
106
|
- lib/icalia-sdk-event-notification/notification.rb
|
121
107
|
- lib/icalia-sdk-event-notification/publisher.rb
|
122
|
-
- lib/icalia-sdk-event-notification/shoryuken.rb
|
123
|
-
- lib/icalia-sdk-event-notification/shoryuken/cli.rb
|
124
|
-
- lib/icalia-sdk-event-notification/shoryuken/cli/base.rb
|
125
|
-
- lib/icalia-sdk-event-notification/shoryuken/cli/runner.rb
|
126
|
-
- lib/icalia-sdk-event-notification/shoryuken/cli/sqs.rb
|
127
|
-
- lib/icalia-sdk-event-notification/shoryuken/worker.rb
|
128
|
-
- lib/icalia-sdk-event-notification/shoryuken/worker_registry.rb
|
129
108
|
- lib/icalia-sdk-event-notification/topic_message_processing.rb
|
130
109
|
- lib/icalia-sdk-event-notification/version.rb
|
131
110
|
homepage: https://github.com/IcaliaLabs/icalia-sdk-ruby
|
@@ -1,9 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event
|
4
|
-
module Shoryuken
|
5
|
-
autoload :Worker, 'icalia-sdk-event-notification/shoryuken/worker'
|
6
|
-
autoload :WorkerRegistry, 'icalia-sdk-event-notification/shoryuken/worker_registry'
|
7
|
-
autoload :CLI, 'icalia-sdk-event-notification/shoryuken/cli'
|
8
|
-
end
|
9
|
-
end
|
@@ -1,9 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event::Shoryuken
|
4
|
-
module CLI
|
5
|
-
autoload :Base, 'icalia-sdk-event-notification/shoryuken/cli/base'
|
6
|
-
autoload :Runner, 'icalia-sdk-event-notification/shoryuken/cli/runner'
|
7
|
-
autoload :SQS, 'icalia-sdk-event-notification/shoryuken/cli/sqs'
|
8
|
-
end
|
9
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event::Shoryuken::CLI
|
4
|
-
class Base < Thor
|
5
|
-
no_commands do
|
6
|
-
def print_table(entries)
|
7
|
-
column_sizes = print_columns_size(entries)
|
8
|
-
|
9
|
-
entries.map do |entry|
|
10
|
-
puts entry.map.with_index { |e, i| print_format_column(e, column_sizes[i]) }.join
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
def print_columns_size(entries)
|
15
|
-
column_sizes = Hash.new(0)
|
16
|
-
|
17
|
-
entries.each do |entry|
|
18
|
-
entry.each_with_index do |e, i|
|
19
|
-
e = e.to_s
|
20
|
-
column_sizes[i] = e.size if column_sizes[i] < e.size
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
column_sizes
|
25
|
-
end
|
26
|
-
|
27
|
-
def print_format_column(column, size)
|
28
|
-
size_with_padding = size + 4
|
29
|
-
column = column.to_s.ljust(size_with_padding)
|
30
|
-
column
|
31
|
-
end
|
32
|
-
|
33
|
-
def fail_task(msg, quit = true)
|
34
|
-
say "[FAIL] #{msg}", :red
|
35
|
-
exit(1) if quit
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
@@ -1,49 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'thor'
|
4
|
-
require 'aws-sdk-core'
|
5
|
-
|
6
|
-
module Icalia::Event::Shoryuken::CLI
|
7
|
-
class Runner < Base
|
8
|
-
default_task :start
|
9
|
-
|
10
|
-
register Icalia::Event::Shoryuken::CLI::SQS,
|
11
|
-
'sqs',
|
12
|
-
'sqs COMMAND',
|
13
|
-
'SQS commands'
|
14
|
-
|
15
|
-
desc 'start', 'Starts shoryuken'
|
16
|
-
method_option :concurrency, aliases: '-c', type: :numeric, desc: 'Processor threads to use'
|
17
|
-
method_option :daemon, aliases: '-d', type: :boolean, desc: 'Daemonize process'
|
18
|
-
method_option :queues, aliases: '-q', type: :array, desc: 'Queues to process with optional weights'
|
19
|
-
method_option :require, aliases: '-r', type: :string, desc: 'Dir or path of the workers'
|
20
|
-
method_option :timeout, aliases: '-t', type: :numeric, desc: 'Hard shutdown timeout'
|
21
|
-
method_option :config, aliases: '-C', type: :string, desc: 'Path to config file'
|
22
|
-
method_option :config_file, type: :string, desc: 'Path to config file (backwards compatibility)'
|
23
|
-
method_option :rails, aliases: '-R', type: :boolean, desc: 'Load Rails'
|
24
|
-
method_option :logfile, aliases: '-L', type: :string, desc: 'Path to logfile'
|
25
|
-
method_option :pidfile, aliases: '-P', type: :string, desc: 'Path to pidfile'
|
26
|
-
method_option :verbose, aliases: '-v', type: :boolean, desc: 'Print more verbose output'
|
27
|
-
method_option :delay, aliases: '-D', type: :numeric, desc: 'Number of seconds to pause fetching from an empty queue'
|
28
|
-
|
29
|
-
def start
|
30
|
-
opts = options.to_h.symbolize_keys
|
31
|
-
|
32
|
-
say '[DEPRECATED] Please use --config instead of --config-file', :yellow if opts[:config_file]
|
33
|
-
|
34
|
-
opts[:config_file] = opts.delete(:config) if opts[:config]
|
35
|
-
|
36
|
-
# Keep compatibility with old CLI queue format
|
37
|
-
opts[:queues] = opts[:queues].reject(&:empty?).map { |q| q.split(',') } if opts[:queues]
|
38
|
-
|
39
|
-
fail_task "You should set a logfile if you're going to daemonize" if opts[:daemon] && opts[:logfile].nil?
|
40
|
-
|
41
|
-
Shoryuken::Runner.instance.run(opts.freeze)
|
42
|
-
end
|
43
|
-
|
44
|
-
desc 'version', 'Prints version'
|
45
|
-
def version
|
46
|
-
say "Shoryuken #{Shoryuken::VERSION}"
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
@@ -1,214 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'date'
|
4
|
-
require 'aws-sdk-sqs'
|
5
|
-
|
6
|
-
# rubocop:disable Metrics/BlockLength
|
7
|
-
module Icalia::Event::Shoryuken::CLI
|
8
|
-
class SQS < Base
|
9
|
-
namespace :sqs
|
10
|
-
|
11
|
-
no_commands do
|
12
|
-
def normalize_dump_message(message)
|
13
|
-
# symbolize_keys is needed for keeping it compatible with `requeue`
|
14
|
-
attributes = message[:attributes].symbolize_keys
|
15
|
-
{
|
16
|
-
id: message[:message_id],
|
17
|
-
message_body: message[:body],
|
18
|
-
message_attributes: message[:message_attributes],
|
19
|
-
message_deduplication_id: attributes[:MessageDeduplicationId],
|
20
|
-
message_group_id: attributes[:MessageGroupId]
|
21
|
-
}
|
22
|
-
end
|
23
|
-
|
24
|
-
def sqs
|
25
|
-
@_sqs ||= Aws::SQS::Client.new Icalia::Event.sqs_client_options
|
26
|
-
end
|
27
|
-
|
28
|
-
def find_queue_url(queue_name)
|
29
|
-
sqs.get_queue_url(queue_name: queue_name).queue_url
|
30
|
-
rescue Aws::SQS::Errors::NonExistentQueue
|
31
|
-
fail_task "The specified queue #{queue_name} does not exist"
|
32
|
-
end
|
33
|
-
|
34
|
-
def batch_delete(url, messages)
|
35
|
-
messages.to_a.flatten.each_slice(10) do |batch|
|
36
|
-
sqs.delete_message_batch(
|
37
|
-
queue_url: url,
|
38
|
-
entries: batch.map { |message| { id: message.message_id, receipt_handle: message.receipt_handle } }
|
39
|
-
).failed.any? do |failure|
|
40
|
-
say(
|
41
|
-
"Could not delete #{failure.id}, code: #{failure.code}, message: #{failure.message}, sender_fault: #{failure.sender_fault}",
|
42
|
-
:yellow
|
43
|
-
)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def batch_send(url, messages, messages_per_batch = 10)
|
49
|
-
messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch|
|
50
|
-
sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
|
51
|
-
say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def find_all(url, limit)
|
57
|
-
count = 0
|
58
|
-
batch_size = limit > 10 ? 10 : limit
|
59
|
-
|
60
|
-
loop do
|
61
|
-
n = limit - count
|
62
|
-
batch_size = n if n < batch_size
|
63
|
-
|
64
|
-
messages = sqs.receive_message(
|
65
|
-
queue_url: url,
|
66
|
-
max_number_of_messages: batch_size,
|
67
|
-
attribute_names: ['All'],
|
68
|
-
message_attribute_names: ['All']
|
69
|
-
).messages
|
70
|
-
|
71
|
-
messages.each { |m| yield m }
|
72
|
-
|
73
|
-
count += messages.size
|
74
|
-
|
75
|
-
break if count >= limit
|
76
|
-
break if messages.empty?
|
77
|
-
end
|
78
|
-
|
79
|
-
count
|
80
|
-
end
|
81
|
-
|
82
|
-
def list_and_print_queues(urls)
|
83
|
-
attrs = %w[QueueArn ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible LastModifiedTimestamp]
|
84
|
-
|
85
|
-
entries = urls.map { |u| sqs.get_queue_attributes(queue_url: u, attribute_names: attrs).attributes }.map do |q|
|
86
|
-
[
|
87
|
-
q['QueueArn'].split(':').last,
|
88
|
-
q['ApproximateNumberOfMessages'],
|
89
|
-
q['ApproximateNumberOfMessagesNotVisible'],
|
90
|
-
Time.at(q['LastModifiedTimestamp'].to_i)
|
91
|
-
]
|
92
|
-
end
|
93
|
-
|
94
|
-
entries.unshift(['Queue', 'Messages Available', 'Messages Inflight', 'Last Modified'])
|
95
|
-
|
96
|
-
print_table(entries)
|
97
|
-
end
|
98
|
-
|
99
|
-
def dump_file(path, queue_name)
|
100
|
-
File.join(path, "#{queue_name}-#{Date.today}.jsonl")
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
desc 'ls [QUEUE-NAME-PREFIX]', 'Lists queues'
|
105
|
-
method_option :watch, aliases: '-w', type: :boolean, desc: 'watch queues'
|
106
|
-
method_option :interval, aliases: '-n', type: :numeric, default: 2, desc: 'watch interval in seconds'
|
107
|
-
def ls(queue_name_prefix = '')
|
108
|
-
trap('SIGINT', 'EXIT') # expect ctrl-c from loop
|
109
|
-
|
110
|
-
urls = sqs.list_queues(queue_name_prefix: queue_name_prefix).queue_urls
|
111
|
-
|
112
|
-
loop do
|
113
|
-
list_and_print_queues(urls)
|
114
|
-
|
115
|
-
break unless options[:watch]
|
116
|
-
|
117
|
-
sleep options[:interval]
|
118
|
-
puts
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
|
-
desc 'dump QUEUE-NAME', 'Dumps messages from a queue into a JSON lines file'
|
123
|
-
method_option :number, aliases: '-n', type: :numeric, default: Float::INFINITY, desc: 'number of messages to dump'
|
124
|
-
method_option :path, aliases: '-p', type: :string, default: './', desc: 'path to save the dump file'
|
125
|
-
method_option :delete, aliases: '-d', type: :boolean, default: true, desc: 'delete from the queue'
|
126
|
-
def dump(queue_name)
|
127
|
-
path = dump_file(options[:path], queue_name)
|
128
|
-
|
129
|
-
fail_task "File #{path} already exists" if File.exist?(path)
|
130
|
-
|
131
|
-
url = find_queue_url(queue_name)
|
132
|
-
|
133
|
-
messages = []
|
134
|
-
|
135
|
-
file = nil
|
136
|
-
|
137
|
-
count = find_all(url, options[:number]) do |m|
|
138
|
-
file ||= File.open(path, 'w')
|
139
|
-
|
140
|
-
file.puts(JSON.dump(m.to_h))
|
141
|
-
|
142
|
-
messages << m if options[:delete]
|
143
|
-
end
|
144
|
-
|
145
|
-
batch_delete(url, messages) if options[:delete]
|
146
|
-
|
147
|
-
if count.zero?
|
148
|
-
say "Queue #{queue_name} is empty", :yellow
|
149
|
-
else
|
150
|
-
say "Dump saved in #{path} with #{count} messages", :green
|
151
|
-
end
|
152
|
-
ensure
|
153
|
-
file.close if file
|
154
|
-
end
|
155
|
-
|
156
|
-
desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
|
157
|
-
method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
|
158
|
-
def requeue(queue_name, path)
|
159
|
-
fail_task "Path #{path} not found" unless File.exist?(path)
|
160
|
-
|
161
|
-
messages = File.readlines(path).map { |line| JSON.parse(line, symbolize_names: true) }
|
162
|
-
|
163
|
-
batch_send(find_queue_url(queue_name), messages, options[:batch_size])
|
164
|
-
|
165
|
-
say "Requeued #{messages.size} messages from #{path} to #{queue_name}", :green
|
166
|
-
end
|
167
|
-
|
168
|
-
desc 'mv QUEUE-NAME-SOURCE QUEUE-NAME-TARGET', 'Moves messages from one queue (source) to another (target)'
|
169
|
-
method_option :number, aliases: '-n', type: :numeric, default: Float::INFINITY, desc: 'number of messages to move'
|
170
|
-
method_option :delete, aliases: '-d', type: :boolean, default: true, desc: 'delete from the queue'
|
171
|
-
def mv(queue_name_source, queue_name_target)
|
172
|
-
url_source = find_queue_url(queue_name_source)
|
173
|
-
messages = []
|
174
|
-
|
175
|
-
count = find_all(url_source, options[:number]) do |m|
|
176
|
-
messages << m
|
177
|
-
end
|
178
|
-
|
179
|
-
batch_send(find_queue_url(queue_name_target), messages.map(&:to_h))
|
180
|
-
batch_delete(url_source, messages) if options[:delete]
|
181
|
-
|
182
|
-
if count.zero?
|
183
|
-
say "Queue #{queue_name_source} is empty", :yellow
|
184
|
-
else
|
185
|
-
say "Moved #{count} messages from #{queue_name_source} to #{queue_name_target}", :green
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
desc 'purge QUEUE-NAME', 'Deletes the messages in a queue'
|
190
|
-
def purge(queue_name)
|
191
|
-
sqs.purge_queue(queue_url: find_queue_url(queue_name))
|
192
|
-
|
193
|
-
say "Purge request sent for #{queue_name}. The message deletion process takes up to 60 seconds", :yellow
|
194
|
-
end
|
195
|
-
|
196
|
-
desc 'create QUEUE-NAME', 'Create a queue'
|
197
|
-
method_option :attributes, aliases: '-a', type: :hash, default: {}, desc: 'queue attributes'
|
198
|
-
def create(queue_name)
|
199
|
-
attributes = options[:attributes]
|
200
|
-
attributes['FifoQueue'] ||= 'true' if queue_name.end_with?('.fifo')
|
201
|
-
|
202
|
-
queue_url = sqs.create_queue(queue_name: queue_name, attributes: attributes).queue_url
|
203
|
-
|
204
|
-
say "Queue #{queue_name} was successfully created. Queue URL #{queue_url}", :green
|
205
|
-
end
|
206
|
-
|
207
|
-
desc 'delete QUEUE-NAME', 'delete a queue'
|
208
|
-
def delete(queue_name)
|
209
|
-
sqs.delete_queue(queue_url: find_queue_url(queue_name))
|
210
|
-
|
211
|
-
say "Queue #{queue_name} was successfully delete", :green
|
212
|
-
end
|
213
|
-
end
|
214
|
-
end
|
@@ -1,63 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event::Shoryuken
|
4
|
-
class Worker
|
5
|
-
include Shoryuken::Worker
|
6
|
-
include Icalia::Event::TopicMessageProcessing
|
7
|
-
|
8
|
-
shoryuken_options queue: 'icalia_events', auto_delete: true
|
9
|
-
|
10
|
-
delegate :logger, to: Shoryuken
|
11
|
-
|
12
|
-
# By default, process all notifications:
|
13
|
-
def process_notification?(notification); true; end
|
14
|
-
|
15
|
-
def perform(_sqs_msg, raw_notification)
|
16
|
-
notification = Icalia::Event::Notification.new raw_notification
|
17
|
-
|
18
|
-
return unless process_notification?(notification)
|
19
|
-
|
20
|
-
logger.info "Processing notification from topic"\
|
21
|
-
" '#{notification.topic}'" #: #{notification.inspect}"
|
22
|
-
|
23
|
-
# If there was no action (some events may not have an action!), we'll
|
24
|
-
# run the `process_message` method instead:
|
25
|
-
action_name = notification.action || 'default'
|
26
|
-
send "process_#{action_name.underscore}".to_sym, notification
|
27
|
-
end
|
28
|
-
|
29
|
-
delegate :define_noop_action_processor, to: :class
|
30
|
-
|
31
|
-
class ActionProcessorPatternTest
|
32
|
-
attr_reader :action_name
|
33
|
-
|
34
|
-
def initialize(method_sym)
|
35
|
-
if method_sym.to_s =~ /\Aprocess_(\w+)\z/
|
36
|
-
@action_name = $1.to_sym
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def match?; @action_name != nil; end
|
41
|
-
end
|
42
|
-
|
43
|
-
def method_missing(method_sym, *arguments, &block)
|
44
|
-
processor_test = ActionProcessorPatternTest.new(method_sym)
|
45
|
-
|
46
|
-
if processor_test.match?
|
47
|
-
define_noop_action_processor(method_sym, processor_test.action_name)
|
48
|
-
return send(method_sym, *arguments)
|
49
|
-
end
|
50
|
-
|
51
|
-
super
|
52
|
-
end
|
53
|
-
|
54
|
-
def self.define_noop_action_processor(action_processor_method, action_name)
|
55
|
-
return if method_defined? action_processor_method
|
56
|
-
|
57
|
-
define_method action_processor_method do |*args|
|
58
|
-
logger.info "#{self.class.name} does not know how to process " \
|
59
|
-
"#{action_name} - Ignoring message"
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
@@ -1,44 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event::Shoryuken
|
4
|
-
class WorkerRegistry < Shoryuken::DefaultWorkerRegistry
|
5
|
-
def fetch_worker(queue, message)
|
6
|
-
worker = super
|
7
|
-
return worker if worker.present?
|
8
|
-
|
9
|
-
require_jobs if empty_mappings?
|
10
|
-
|
11
|
-
topic_name = extract_topic_name(message)
|
12
|
-
|
13
|
-
worker_class = Icalia::Event::TopicMessageProcessing
|
14
|
-
.worker_mappings[topic_name]
|
15
|
-
|
16
|
-
return worker_class.new if worker_class.present?
|
17
|
-
end
|
18
|
-
|
19
|
-
protected
|
20
|
-
|
21
|
-
delegate :extract_topic_name, :require_jobs, to: :class
|
22
|
-
delegate :empty_mappings?, to: Icalia::Event::TopicMessageProcessing
|
23
|
-
|
24
|
-
def self.require_jobs
|
25
|
-
path = Rails.root.join('app', 'jobs', '**', '*_job.rb')
|
26
|
-
Dir[path].each {|file| require file }
|
27
|
-
end
|
28
|
-
|
29
|
-
def self.extract_topic_name(message)
|
30
|
-
extract_metadata(message)['topic_arn'].split(':')[5..-1].join(':')
|
31
|
-
end
|
32
|
-
|
33
|
-
def self.extract_metadata(message)
|
34
|
-
extract_body(message)
|
35
|
-
.except('Message')
|
36
|
-
.transform_keys(&:underscore)
|
37
|
-
.with_indifferent_access
|
38
|
-
end
|
39
|
-
|
40
|
-
def self.extract_body(message)
|
41
|
-
ActiveSupport::JSON.decode message.body
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|