icalia-sdk-event-notification 0.1.13 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/icalia-sdk-event-notification.rb +0 -24
- data/lib/icalia-sdk-event-notification/publisher.rb +40 -26
- data/lib/icalia-sdk-event-notification/version.rb +1 -1
- metadata +7 -28
- data/lib/icalia-sdk-event-notification/shoryuken.rb +0 -9
- data/lib/icalia-sdk-event-notification/shoryuken/cli.rb +0 -9
- data/lib/icalia-sdk-event-notification/shoryuken/cli/base.rb +0 -39
- data/lib/icalia-sdk-event-notification/shoryuken/cli/runner.rb +0 -49
- data/lib/icalia-sdk-event-notification/shoryuken/cli/sqs.rb +0 -214
- data/lib/icalia-sdk-event-notification/shoryuken/worker.rb +0 -63
- data/lib/icalia-sdk-event-notification/shoryuken/worker_registry.rb +0 -44
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef20f61096dca725e5d61ebc6cc7629b33b694ea61142e4a4bdf331dacff2a2c
|
4
|
+
data.tar.gz: 4fb4d685249caf1dd80ed0322a2a412c0c3053e3876f769b770d56f9a749b9b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6ba2954b5171714bfb90a1e21e80efc3fb3af118ac8d48d42f775af1028d6e292432a197447af9e847e432c8109cade762c4ec367f94cea36359ff1c55a44d6d
|
7
|
+
data.tar.gz: 7b0b343f021ae429163c58d17545495572a3aa6acba53570981663a6d1a983cd5b07d6d1007595a963d2cac529562591e00f7a54dd2089bfa81f1ca671b809a8
|
@@ -11,33 +11,9 @@ module Icalia
|
|
11
11
|
autoload :Publisher, 'icalia-sdk-event-notification/publisher'
|
12
12
|
autoload :Notification, 'icalia-sdk-event-notification/notification'
|
13
13
|
|
14
|
-
autoload :Shoryuken, 'icalia-sdk-event-notification/shoryuken'
|
15
|
-
|
16
14
|
autoload :TopicMessageProcessing,
|
17
15
|
'icalia-sdk-event-notification/topic_message_processing'
|
18
16
|
|
19
|
-
def self.generic_aws_client_options
|
20
|
-
{
|
21
|
-
region: ENV.fetch('AWS_REGION', 'us-west-2'),
|
22
|
-
access_key_id: ENV.fetch('AWS_ACCESS_KEY_ID'),
|
23
|
-
secret_access_key: ENV.fetch('AWS_SECRET_ACCESS_KEY')
|
24
|
-
}
|
25
|
-
end
|
26
|
-
|
27
|
-
def self.sqs_client_options
|
28
|
-
options = Icalia::Event.generic_aws_client_options
|
29
|
-
return options if ENV['AWS_SQS_ENDPOINT'].blank?
|
30
|
-
|
31
|
-
options.merge endpoint: ENV['AWS_SQS_ENDPOINT'], verify_checksums: false
|
32
|
-
end
|
33
|
-
|
34
|
-
def self.sns_client_options
|
35
|
-
options = Icalia::Event.generic_aws_client_options
|
36
|
-
return options if ENV['AWS_SNS_ENDPOINT'].blank?
|
37
|
-
|
38
|
-
options.merge endpoint: ENV['AWS_SQS_ENDPOINT']
|
39
|
-
end
|
40
|
-
|
41
17
|
mattr_reader :deferred_publishing, default: true
|
42
18
|
end
|
43
19
|
end
|
@@ -1,16 +1,21 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'google/cloud/pubsub'
|
4
4
|
|
5
5
|
require 'active_support'
|
6
6
|
require 'active_support/logger'
|
7
7
|
|
8
8
|
module Icalia::Event
|
9
|
-
|
9
|
+
module GRPCRailsLogger
|
10
|
+
def logger; Rails.logger; end
|
11
|
+
end
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
+
module GRPCStdoutLogger
|
14
|
+
LOGGER = ActiveSupport::Logger.new(STDOUT)
|
15
|
+
def logger; LOGGER; end
|
16
|
+
end
|
13
17
|
|
18
|
+
class Publisher
|
14
19
|
class << self
|
15
20
|
def instance
|
16
21
|
@instance ||= new
|
@@ -19,41 +24,48 @@ module Icalia::Event
|
|
19
24
|
delegate :publish, to: :instance
|
20
25
|
end
|
21
26
|
|
22
|
-
attr_reader :
|
27
|
+
attr_reader :client, :logger
|
28
|
+
|
29
|
+
delegate :encode, to: ActiveSupport::JSON, prefix: :json
|
23
30
|
|
24
31
|
def initialize
|
25
|
-
|
26
|
-
|
32
|
+
initialize_pubsub_client
|
33
|
+
initialize_logger
|
27
34
|
end
|
28
35
|
|
29
|
-
def publish(topic_name, data, json_data: true)
|
30
|
-
raise UnknownTopicError, "Topic '#{topic_name}' does not exist" \
|
31
|
-
unless topic_exists?(topic_name)
|
32
|
-
|
33
|
-
topic = get_topic topic_name
|
36
|
+
def publish(topic_name, data, attributes = { json_data: true })
|
34
37
|
logger.debug "Publishing to '#{topic_name}': #{data.inspect}"
|
35
|
-
|
38
|
+
encoded_data = attributes[:json_data] ? json_encode(data) : data
|
39
|
+
get_or_create_topic(topic_name).publish(encoded_data, attributes)
|
36
40
|
end
|
37
41
|
|
38
|
-
delegate :encode, to: ActiveSupport::JSON, prefix: :json
|
39
|
-
|
40
42
|
protected
|
41
43
|
|
42
|
-
def
|
43
|
-
|
44
|
-
topic.arn.split(':').last == topic_name
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def topic_exists?(topic_name)
|
49
|
-
get_topic(topic_name).present?
|
44
|
+
def get_or_create_topic(topic_name)
|
45
|
+
client.topic(topic_name) || client.new_topic(topic_name)
|
50
46
|
end
|
51
47
|
|
52
|
-
|
53
|
-
|
48
|
+
private
|
49
|
+
|
50
|
+
# See https://googleapis.dev/ruby/google-cloud-pubsub/latest/file.AUTHENTICATION.html#Environment_Variables
|
51
|
+
#
|
52
|
+
# The environment variables that google-cloud-pubsub checks for project ID are:
|
53
|
+
# 1. PUBSUB_PROJECT
|
54
|
+
# 2. GOOGLE_CLOUD_PROJECT
|
55
|
+
#
|
56
|
+
# The environment variables that google-cloud-pubsub checks for credentials
|
57
|
+
# are configured on Google::Cloud::PubSub::V1::Credentials:
|
58
|
+
#
|
59
|
+
# 1. PUBSUB_CREDENTIALS - Path to JSON file, or JSON contents
|
60
|
+
# 2. PUBSUB_KEYFILE - Path to JSON file, or JSON contents
|
61
|
+
# 3. GOOGLE_CLOUD_CREDENTIALS - Path to JSON file, or JSON contents
|
62
|
+
# 4. GOOGLE_CLOUD_KEYFILE - Path to JSON file, or JSON contents
|
63
|
+
# 5. GOOGLE_APPLICATION_CREDENTIALS - Path to JSON file
|
64
|
+
def initialize_pubsub_client
|
65
|
+
@client = Google::Cloud::Pubsub.new
|
54
66
|
end
|
55
67
|
|
56
|
-
def
|
68
|
+
def initialize_logger
|
57
69
|
return use_rails_logger if rails_logger_exist?
|
58
70
|
use_default_logger
|
59
71
|
end
|
@@ -64,10 +76,12 @@ module Icalia::Event
|
|
64
76
|
|
65
77
|
def use_rails_logger
|
66
78
|
@logger = Rails.logger
|
79
|
+
GRPCRailsLogger.send :extend, GRPCRailsLogger
|
67
80
|
end
|
68
81
|
|
69
82
|
def use_default_logger
|
70
83
|
@logger = ActiveSupport::Logger.new(STDOUT)
|
84
|
+
GRPCStdoutLogger.send :extend, GRPCStdoutLogger
|
71
85
|
end
|
72
86
|
end
|
73
87
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: icalia-sdk-event-notification
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Roberto Quintanilla
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: icalia-sdk-event-core
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - '='
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.2.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.2.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: activesupport
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -39,33 +39,19 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 5.2.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: google-cloud-pubsub
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '0.31'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
|
-
name: aws-sdk-sqs
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
58
|
-
requirements:
|
59
|
-
- - "~>"
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: '1.20'
|
62
|
-
type: :runtime
|
63
|
-
prerelease: false
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
65
|
-
requirements:
|
66
|
-
- - "~>"
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: '1.20'
|
54
|
+
version: '0.31'
|
69
55
|
- !ruby/object:Gem::Dependency
|
70
56
|
name: bundler
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -119,13 +105,6 @@ files:
|
|
119
105
|
- lib/icalia-sdk-event-notification.rb
|
120
106
|
- lib/icalia-sdk-event-notification/notification.rb
|
121
107
|
- lib/icalia-sdk-event-notification/publisher.rb
|
122
|
-
- lib/icalia-sdk-event-notification/shoryuken.rb
|
123
|
-
- lib/icalia-sdk-event-notification/shoryuken/cli.rb
|
124
|
-
- lib/icalia-sdk-event-notification/shoryuken/cli/base.rb
|
125
|
-
- lib/icalia-sdk-event-notification/shoryuken/cli/runner.rb
|
126
|
-
- lib/icalia-sdk-event-notification/shoryuken/cli/sqs.rb
|
127
|
-
- lib/icalia-sdk-event-notification/shoryuken/worker.rb
|
128
|
-
- lib/icalia-sdk-event-notification/shoryuken/worker_registry.rb
|
129
108
|
- lib/icalia-sdk-event-notification/topic_message_processing.rb
|
130
109
|
- lib/icalia-sdk-event-notification/version.rb
|
131
110
|
homepage: https://github.com/IcaliaLabs/icalia-sdk-ruby
|
@@ -1,9 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event
|
4
|
-
module Shoryuken
|
5
|
-
autoload :Worker, 'icalia-sdk-event-notification/shoryuken/worker'
|
6
|
-
autoload :WorkerRegistry, 'icalia-sdk-event-notification/shoryuken/worker_registry'
|
7
|
-
autoload :CLI, 'icalia-sdk-event-notification/shoryuken/cli'
|
8
|
-
end
|
9
|
-
end
|
@@ -1,9 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event::Shoryuken
|
4
|
-
module CLI
|
5
|
-
autoload :Base, 'icalia-sdk-event-notification/shoryuken/cli/base'
|
6
|
-
autoload :Runner, 'icalia-sdk-event-notification/shoryuken/cli/runner'
|
7
|
-
autoload :SQS, 'icalia-sdk-event-notification/shoryuken/cli/sqs'
|
8
|
-
end
|
9
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event::Shoryuken::CLI
|
4
|
-
class Base < Thor
|
5
|
-
no_commands do
|
6
|
-
def print_table(entries)
|
7
|
-
column_sizes = print_columns_size(entries)
|
8
|
-
|
9
|
-
entries.map do |entry|
|
10
|
-
puts entry.map.with_index { |e, i| print_format_column(e, column_sizes[i]) }.join
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
def print_columns_size(entries)
|
15
|
-
column_sizes = Hash.new(0)
|
16
|
-
|
17
|
-
entries.each do |entry|
|
18
|
-
entry.each_with_index do |e, i|
|
19
|
-
e = e.to_s
|
20
|
-
column_sizes[i] = e.size if column_sizes[i] < e.size
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
column_sizes
|
25
|
-
end
|
26
|
-
|
27
|
-
def print_format_column(column, size)
|
28
|
-
size_with_padding = size + 4
|
29
|
-
column = column.to_s.ljust(size_with_padding)
|
30
|
-
column
|
31
|
-
end
|
32
|
-
|
33
|
-
def fail_task(msg, quit = true)
|
34
|
-
say "[FAIL] #{msg}", :red
|
35
|
-
exit(1) if quit
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
@@ -1,49 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'thor'
|
4
|
-
require 'aws-sdk-core'
|
5
|
-
|
6
|
-
module Icalia::Event::Shoryuken::CLI
|
7
|
-
class Runner < Base
|
8
|
-
default_task :start
|
9
|
-
|
10
|
-
register Icalia::Event::Shoryuken::CLI::SQS,
|
11
|
-
'sqs',
|
12
|
-
'sqs COMMAND',
|
13
|
-
'SQS commands'
|
14
|
-
|
15
|
-
desc 'start', 'Starts shoryuken'
|
16
|
-
method_option :concurrency, aliases: '-c', type: :numeric, desc: 'Processor threads to use'
|
17
|
-
method_option :daemon, aliases: '-d', type: :boolean, desc: 'Daemonize process'
|
18
|
-
method_option :queues, aliases: '-q', type: :array, desc: 'Queues to process with optional weights'
|
19
|
-
method_option :require, aliases: '-r', type: :string, desc: 'Dir or path of the workers'
|
20
|
-
method_option :timeout, aliases: '-t', type: :numeric, desc: 'Hard shutdown timeout'
|
21
|
-
method_option :config, aliases: '-C', type: :string, desc: 'Path to config file'
|
22
|
-
method_option :config_file, type: :string, desc: 'Path to config file (backwards compatibility)'
|
23
|
-
method_option :rails, aliases: '-R', type: :boolean, desc: 'Load Rails'
|
24
|
-
method_option :logfile, aliases: '-L', type: :string, desc: 'Path to logfile'
|
25
|
-
method_option :pidfile, aliases: '-P', type: :string, desc: 'Path to pidfile'
|
26
|
-
method_option :verbose, aliases: '-v', type: :boolean, desc: 'Print more verbose output'
|
27
|
-
method_option :delay, aliases: '-D', type: :numeric, desc: 'Number of seconds to pause fetching from an empty queue'
|
28
|
-
|
29
|
-
def start
|
30
|
-
opts = options.to_h.symbolize_keys
|
31
|
-
|
32
|
-
say '[DEPRECATED] Please use --config instead of --config-file', :yellow if opts[:config_file]
|
33
|
-
|
34
|
-
opts[:config_file] = opts.delete(:config) if opts[:config]
|
35
|
-
|
36
|
-
# Keep compatibility with old CLI queue format
|
37
|
-
opts[:queues] = opts[:queues].reject(&:empty?).map { |q| q.split(',') } if opts[:queues]
|
38
|
-
|
39
|
-
fail_task "You should set a logfile if you're going to daemonize" if opts[:daemon] && opts[:logfile].nil?
|
40
|
-
|
41
|
-
Shoryuken::Runner.instance.run(opts.freeze)
|
42
|
-
end
|
43
|
-
|
44
|
-
desc 'version', 'Prints version'
|
45
|
-
def version
|
46
|
-
say "Shoryuken #{Shoryuken::VERSION}"
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
@@ -1,214 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'date'
|
4
|
-
require 'aws-sdk-sqs'
|
5
|
-
|
6
|
-
# rubocop:disable Metrics/BlockLength
|
7
|
-
module Icalia::Event::Shoryuken::CLI
|
8
|
-
class SQS < Base
|
9
|
-
namespace :sqs
|
10
|
-
|
11
|
-
no_commands do
|
12
|
-
def normalize_dump_message(message)
|
13
|
-
# symbolize_keys is needed for keeping it compatible with `requeue`
|
14
|
-
attributes = message[:attributes].symbolize_keys
|
15
|
-
{
|
16
|
-
id: message[:message_id],
|
17
|
-
message_body: message[:body],
|
18
|
-
message_attributes: message[:message_attributes],
|
19
|
-
message_deduplication_id: attributes[:MessageDeduplicationId],
|
20
|
-
message_group_id: attributes[:MessageGroupId]
|
21
|
-
}
|
22
|
-
end
|
23
|
-
|
24
|
-
def sqs
|
25
|
-
@_sqs ||= Aws::SQS::Client.new Icalia::Event.sqs_client_options
|
26
|
-
end
|
27
|
-
|
28
|
-
def find_queue_url(queue_name)
|
29
|
-
sqs.get_queue_url(queue_name: queue_name).queue_url
|
30
|
-
rescue Aws::SQS::Errors::NonExistentQueue
|
31
|
-
fail_task "The specified queue #{queue_name} does not exist"
|
32
|
-
end
|
33
|
-
|
34
|
-
def batch_delete(url, messages)
|
35
|
-
messages.to_a.flatten.each_slice(10) do |batch|
|
36
|
-
sqs.delete_message_batch(
|
37
|
-
queue_url: url,
|
38
|
-
entries: batch.map { |message| { id: message.message_id, receipt_handle: message.receipt_handle } }
|
39
|
-
).failed.any? do |failure|
|
40
|
-
say(
|
41
|
-
"Could not delete #{failure.id}, code: #{failure.code}, message: #{failure.message}, sender_fault: #{failure.sender_fault}",
|
42
|
-
:yellow
|
43
|
-
)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def batch_send(url, messages, messages_per_batch = 10)
|
49
|
-
messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch|
|
50
|
-
sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
|
51
|
-
say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def find_all(url, limit)
|
57
|
-
count = 0
|
58
|
-
batch_size = limit > 10 ? 10 : limit
|
59
|
-
|
60
|
-
loop do
|
61
|
-
n = limit - count
|
62
|
-
batch_size = n if n < batch_size
|
63
|
-
|
64
|
-
messages = sqs.receive_message(
|
65
|
-
queue_url: url,
|
66
|
-
max_number_of_messages: batch_size,
|
67
|
-
attribute_names: ['All'],
|
68
|
-
message_attribute_names: ['All']
|
69
|
-
).messages
|
70
|
-
|
71
|
-
messages.each { |m| yield m }
|
72
|
-
|
73
|
-
count += messages.size
|
74
|
-
|
75
|
-
break if count >= limit
|
76
|
-
break if messages.empty?
|
77
|
-
end
|
78
|
-
|
79
|
-
count
|
80
|
-
end
|
81
|
-
|
82
|
-
def list_and_print_queues(urls)
|
83
|
-
attrs = %w[QueueArn ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible LastModifiedTimestamp]
|
84
|
-
|
85
|
-
entries = urls.map { |u| sqs.get_queue_attributes(queue_url: u, attribute_names: attrs).attributes }.map do |q|
|
86
|
-
[
|
87
|
-
q['QueueArn'].split(':').last,
|
88
|
-
q['ApproximateNumberOfMessages'],
|
89
|
-
q['ApproximateNumberOfMessagesNotVisible'],
|
90
|
-
Time.at(q['LastModifiedTimestamp'].to_i)
|
91
|
-
]
|
92
|
-
end
|
93
|
-
|
94
|
-
entries.unshift(['Queue', 'Messages Available', 'Messages Inflight', 'Last Modified'])
|
95
|
-
|
96
|
-
print_table(entries)
|
97
|
-
end
|
98
|
-
|
99
|
-
def dump_file(path, queue_name)
|
100
|
-
File.join(path, "#{queue_name}-#{Date.today}.jsonl")
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
desc 'ls [QUEUE-NAME-PREFIX]', 'Lists queues'
|
105
|
-
method_option :watch, aliases: '-w', type: :boolean, desc: 'watch queues'
|
106
|
-
method_option :interval, aliases: '-n', type: :numeric, default: 2, desc: 'watch interval in seconds'
|
107
|
-
def ls(queue_name_prefix = '')
|
108
|
-
trap('SIGINT', 'EXIT') # expect ctrl-c from loop
|
109
|
-
|
110
|
-
urls = sqs.list_queues(queue_name_prefix: queue_name_prefix).queue_urls
|
111
|
-
|
112
|
-
loop do
|
113
|
-
list_and_print_queues(urls)
|
114
|
-
|
115
|
-
break unless options[:watch]
|
116
|
-
|
117
|
-
sleep options[:interval]
|
118
|
-
puts
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
|
-
desc 'dump QUEUE-NAME', 'Dumps messages from a queue into a JSON lines file'
|
123
|
-
method_option :number, aliases: '-n', type: :numeric, default: Float::INFINITY, desc: 'number of messages to dump'
|
124
|
-
method_option :path, aliases: '-p', type: :string, default: './', desc: 'path to save the dump file'
|
125
|
-
method_option :delete, aliases: '-d', type: :boolean, default: true, desc: 'delete from the queue'
|
126
|
-
def dump(queue_name)
|
127
|
-
path = dump_file(options[:path], queue_name)
|
128
|
-
|
129
|
-
fail_task "File #{path} already exists" if File.exist?(path)
|
130
|
-
|
131
|
-
url = find_queue_url(queue_name)
|
132
|
-
|
133
|
-
messages = []
|
134
|
-
|
135
|
-
file = nil
|
136
|
-
|
137
|
-
count = find_all(url, options[:number]) do |m|
|
138
|
-
file ||= File.open(path, 'w')
|
139
|
-
|
140
|
-
file.puts(JSON.dump(m.to_h))
|
141
|
-
|
142
|
-
messages << m if options[:delete]
|
143
|
-
end
|
144
|
-
|
145
|
-
batch_delete(url, messages) if options[:delete]
|
146
|
-
|
147
|
-
if count.zero?
|
148
|
-
say "Queue #{queue_name} is empty", :yellow
|
149
|
-
else
|
150
|
-
say "Dump saved in #{path} with #{count} messages", :green
|
151
|
-
end
|
152
|
-
ensure
|
153
|
-
file.close if file
|
154
|
-
end
|
155
|
-
|
156
|
-
desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
|
157
|
-
method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
|
158
|
-
def requeue(queue_name, path)
|
159
|
-
fail_task "Path #{path} not found" unless File.exist?(path)
|
160
|
-
|
161
|
-
messages = File.readlines(path).map { |line| JSON.parse(line, symbolize_names: true) }
|
162
|
-
|
163
|
-
batch_send(find_queue_url(queue_name), messages, options[:batch_size])
|
164
|
-
|
165
|
-
say "Requeued #{messages.size} messages from #{path} to #{queue_name}", :green
|
166
|
-
end
|
167
|
-
|
168
|
-
desc 'mv QUEUE-NAME-SOURCE QUEUE-NAME-TARGET', 'Moves messages from one queue (source) to another (target)'
|
169
|
-
method_option :number, aliases: '-n', type: :numeric, default: Float::INFINITY, desc: 'number of messages to move'
|
170
|
-
method_option :delete, aliases: '-d', type: :boolean, default: true, desc: 'delete from the queue'
|
171
|
-
def mv(queue_name_source, queue_name_target)
|
172
|
-
url_source = find_queue_url(queue_name_source)
|
173
|
-
messages = []
|
174
|
-
|
175
|
-
count = find_all(url_source, options[:number]) do |m|
|
176
|
-
messages << m
|
177
|
-
end
|
178
|
-
|
179
|
-
batch_send(find_queue_url(queue_name_target), messages.map(&:to_h))
|
180
|
-
batch_delete(url_source, messages) if options[:delete]
|
181
|
-
|
182
|
-
if count.zero?
|
183
|
-
say "Queue #{queue_name_source} is empty", :yellow
|
184
|
-
else
|
185
|
-
say "Moved #{count} messages from #{queue_name_source} to #{queue_name_target}", :green
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
desc 'purge QUEUE-NAME', 'Deletes the messages in a queue'
|
190
|
-
def purge(queue_name)
|
191
|
-
sqs.purge_queue(queue_url: find_queue_url(queue_name))
|
192
|
-
|
193
|
-
say "Purge request sent for #{queue_name}. The message deletion process takes up to 60 seconds", :yellow
|
194
|
-
end
|
195
|
-
|
196
|
-
desc 'create QUEUE-NAME', 'Create a queue'
|
197
|
-
method_option :attributes, aliases: '-a', type: :hash, default: {}, desc: 'queue attributes'
|
198
|
-
def create(queue_name)
|
199
|
-
attributes = options[:attributes]
|
200
|
-
attributes['FifoQueue'] ||= 'true' if queue_name.end_with?('.fifo')
|
201
|
-
|
202
|
-
queue_url = sqs.create_queue(queue_name: queue_name, attributes: attributes).queue_url
|
203
|
-
|
204
|
-
say "Queue #{queue_name} was successfully created. Queue URL #{queue_url}", :green
|
205
|
-
end
|
206
|
-
|
207
|
-
desc 'delete QUEUE-NAME', 'delete a queue'
|
208
|
-
def delete(queue_name)
|
209
|
-
sqs.delete_queue(queue_url: find_queue_url(queue_name))
|
210
|
-
|
211
|
-
say "Queue #{queue_name} was successfully delete", :green
|
212
|
-
end
|
213
|
-
end
|
214
|
-
end
|
@@ -1,63 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event::Shoryuken
|
4
|
-
class Worker
|
5
|
-
include Shoryuken::Worker
|
6
|
-
include Icalia::Event::TopicMessageProcessing
|
7
|
-
|
8
|
-
shoryuken_options queue: 'icalia_events', auto_delete: true
|
9
|
-
|
10
|
-
delegate :logger, to: Shoryuken
|
11
|
-
|
12
|
-
# By default, process all notifications:
|
13
|
-
def process_notification?(notification); true; end
|
14
|
-
|
15
|
-
def perform(_sqs_msg, raw_notification)
|
16
|
-
notification = Icalia::Event::Notification.new raw_notification
|
17
|
-
|
18
|
-
return unless process_notification?(notification)
|
19
|
-
|
20
|
-
logger.info "Processing notification from topic"\
|
21
|
-
" '#{notification.topic}'" #: #{notification.inspect}"
|
22
|
-
|
23
|
-
# If there was no action (some events may not have an action!), we'll
|
24
|
-
# run the `process_message` method instead:
|
25
|
-
action_name = notification.action || 'default'
|
26
|
-
send "process_#{action_name.underscore}".to_sym, notification
|
27
|
-
end
|
28
|
-
|
29
|
-
delegate :define_noop_action_processor, to: :class
|
30
|
-
|
31
|
-
class ActionProcessorPatternTest
|
32
|
-
attr_reader :action_name
|
33
|
-
|
34
|
-
def initialize(method_sym)
|
35
|
-
if method_sym.to_s =~ /\Aprocess_(\w+)\z/
|
36
|
-
@action_name = $1.to_sym
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def match?; @action_name != nil; end
|
41
|
-
end
|
42
|
-
|
43
|
-
def method_missing(method_sym, *arguments, &block)
|
44
|
-
processor_test = ActionProcessorPatternTest.new(method_sym)
|
45
|
-
|
46
|
-
if processor_test.match?
|
47
|
-
define_noop_action_processor(method_sym, processor_test.action_name)
|
48
|
-
return send(method_sym, *arguments)
|
49
|
-
end
|
50
|
-
|
51
|
-
super
|
52
|
-
end
|
53
|
-
|
54
|
-
def self.define_noop_action_processor(action_processor_method, action_name)
|
55
|
-
return if method_defined? action_processor_method
|
56
|
-
|
57
|
-
define_method action_processor_method do |*args|
|
58
|
-
logger.info "#{self.class.name} does not know how to process " \
|
59
|
-
"#{action_name} - Ignoring message"
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
@@ -1,44 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Icalia::Event::Shoryuken
|
4
|
-
class WorkerRegistry < Shoryuken::DefaultWorkerRegistry
|
5
|
-
def fetch_worker(queue, message)
|
6
|
-
worker = super
|
7
|
-
return worker if worker.present?
|
8
|
-
|
9
|
-
require_jobs if empty_mappings?
|
10
|
-
|
11
|
-
topic_name = extract_topic_name(message)
|
12
|
-
|
13
|
-
worker_class = Icalia::Event::TopicMessageProcessing
|
14
|
-
.worker_mappings[topic_name]
|
15
|
-
|
16
|
-
return worker_class.new if worker_class.present?
|
17
|
-
end
|
18
|
-
|
19
|
-
protected
|
20
|
-
|
21
|
-
delegate :extract_topic_name, :require_jobs, to: :class
|
22
|
-
delegate :empty_mappings?, to: Icalia::Event::TopicMessageProcessing
|
23
|
-
|
24
|
-
def self.require_jobs
|
25
|
-
path = Rails.root.join('app', 'jobs', '**', '*_job.rb')
|
26
|
-
Dir[path].each {|file| require file }
|
27
|
-
end
|
28
|
-
|
29
|
-
def self.extract_topic_name(message)
|
30
|
-
extract_metadata(message)['topic_arn'].split(':')[5..-1].join(':')
|
31
|
-
end
|
32
|
-
|
33
|
-
def self.extract_metadata(message)
|
34
|
-
extract_body(message)
|
35
|
-
.except('Message')
|
36
|
-
.transform_keys(&:underscore)
|
37
|
-
.with_indifferent_access
|
38
|
-
end
|
39
|
-
|
40
|
-
def self.extract_body(message)
|
41
|
-
ActiveSupport::JSON.decode message.body
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|