cyclone_lariat 0.4.0 → 1.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/gem-push.yml +4 -4
- data/.rubocop.yml +9 -5
- data/Gemfile.lock +123 -21
- data/Guardfile +42 -0
- data/README.md +417 -220
- data/bin/cyclone_lariat +75 -43
- data/cyclone_lariat.gemspec +10 -3
- data/lib/cyclone_lariat/clients/abstract.rb +40 -0
- data/lib/cyclone_lariat/clients/sns.rb +163 -0
- data/lib/cyclone_lariat/clients/sqs.rb +114 -0
- data/lib/cyclone_lariat/core.rb +21 -0
- data/lib/cyclone_lariat/errors.rb +16 -0
- data/lib/cyclone_lariat/fake.rb +19 -0
- data/lib/cyclone_lariat/generators/command.rb +53 -0
- data/lib/cyclone_lariat/generators/event.rb +52 -0
- data/lib/cyclone_lariat/generators/queue.rb +30 -0
- data/lib/cyclone_lariat/generators/topic.rb +29 -0
- data/lib/cyclone_lariat/messages/v1/abstract.rb +139 -0
- data/lib/cyclone_lariat/messages/v1/command.rb +20 -0
- data/lib/cyclone_lariat/messages/v1/event.rb +20 -0
- data/lib/cyclone_lariat/messages/v1/validator.rb +31 -0
- data/lib/cyclone_lariat/messages/v2/abstract.rb +149 -0
- data/lib/cyclone_lariat/messages/v2/command.rb +20 -0
- data/lib/cyclone_lariat/messages/v2/event.rb +20 -0
- data/lib/cyclone_lariat/messages/v2/validator.rb +39 -0
- data/lib/cyclone_lariat/middleware.rb +9 -6
- data/lib/cyclone_lariat/migration.rb +54 -117
- data/lib/cyclone_lariat/options.rb +52 -0
- data/lib/cyclone_lariat/presenters/graph.rb +54 -0
- data/lib/cyclone_lariat/presenters/queues.rb +41 -0
- data/lib/cyclone_lariat/presenters/subscriptions.rb +34 -0
- data/lib/cyclone_lariat/presenters/topics.rb +40 -0
- data/lib/cyclone_lariat/publisher.rb +25 -0
- data/lib/cyclone_lariat/repo/active_record/messages.rb +92 -0
- data/lib/cyclone_lariat/repo/active_record/versions.rb +28 -0
- data/lib/cyclone_lariat/repo/messages.rb +43 -0
- data/lib/cyclone_lariat/repo/messages_mapper.rb +49 -0
- data/lib/cyclone_lariat/repo/sequel/messages.rb +73 -0
- data/lib/cyclone_lariat/repo/sequel/versions.rb +28 -0
- data/lib/cyclone_lariat/repo/versions.rb +42 -0
- data/lib/cyclone_lariat/resources/queue.rb +167 -0
- data/lib/cyclone_lariat/resources/topic.rb +132 -0
- data/lib/cyclone_lariat/services/migrate.rb +51 -0
- data/lib/cyclone_lariat/services/rollback.rb +51 -0
- data/lib/cyclone_lariat/version.rb +1 -1
- data/lib/cyclone_lariat.rb +4 -11
- data/lib/tasks/console.rake +1 -1
- data/lib/tasks/cyclone_lariat.rake +10 -12
- data/lib/tasks/db.rake +0 -15
- metadata +127 -27
- data/config/db.example.rb +0 -9
- data/config/initializers/sequel.rb +0 -7
- data/db/migrate/01_add_uuid_extensions.rb +0 -15
- data/db/migrate/02_add_events.rb +0 -19
- data/db/migrate/03_add_versions.rb +0 -9
- data/docs/_imgs/graphviz_01.png +0 -0
- data/docs/_imgs/graphviz_02.png +0 -0
- data/docs/_imgs/graphviz_03.png +0 -0
- data/docs/_imgs/lariat.jpg +0 -0
- data/docs/_imgs/logic.png +0 -0
- data/docs/_imgs/sqs_sns_diagram.png +0 -0
- data/lib/cyclone_lariat/abstract/client.rb +0 -112
- data/lib/cyclone_lariat/abstract/message.rb +0 -98
- data/lib/cyclone_lariat/command.rb +0 -13
- data/lib/cyclone_lariat/configure.rb +0 -15
- data/lib/cyclone_lariat/event.rb +0 -13
- data/lib/cyclone_lariat/messages_mapper.rb +0 -46
- data/lib/cyclone_lariat/messages_repo.rb +0 -60
- data/lib/cyclone_lariat/queue.rb +0 -147
- data/lib/cyclone_lariat/sns_client.rb +0 -149
- data/lib/cyclone_lariat/sqs_client.rb +0 -93
- data/lib/cyclone_lariat/topic.rb +0 -113
data/bin/cyclone_lariat
CHANGED
@@ -24,13 +24,25 @@ module CycloneLariat
|
|
24
24
|
|
25
25
|
class Install < Dry::CLI::Command
|
26
26
|
desc 'Install cyclone lariat to current directory'
|
27
|
+
option :adapter,
|
28
|
+
default: 'sequel',
|
29
|
+
values: %w[sequel active_record],
|
30
|
+
desc: 'adapter for store events and versions'
|
27
31
|
|
28
|
-
def call(
|
29
|
-
create_config
|
32
|
+
def call(adapter: 'sequel', **)
|
33
|
+
create_config(adapter)
|
30
34
|
create_rake_task
|
31
35
|
end
|
32
36
|
|
33
|
-
def create_config
|
37
|
+
def create_config(adapter)
|
38
|
+
FileUtils.mkdir_p INITIALIZERS_DIR unless Dir.exist? INITIALIZERS_DIR
|
39
|
+
config_path = "#{INITIALIZERS_DIR}/cyclone_lariat.rb"
|
40
|
+
config_file = File.open(config_path, 'w')
|
41
|
+
config_file.puts config_contents(adapter)
|
42
|
+
puts "Created config: #{config_path}"
|
43
|
+
end
|
44
|
+
|
45
|
+
def create_rake_task
|
34
46
|
FileUtils.mkdir_p RAKE_TASKS_DIR unless Dir.exist? RAKE_TASKS_DIR
|
35
47
|
config_path = "#{RAKE_TASKS_DIR}/cyclone_lariat.rake"
|
36
48
|
config_file = File.open(config_path, 'w')
|
@@ -38,28 +50,49 @@ module CycloneLariat
|
|
38
50
|
puts "Created rake task: #{config_path}"
|
39
51
|
end
|
40
52
|
|
41
|
-
def
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
puts "Created config: #{config_path}"
|
53
|
+
def config_contents(adapter)
|
54
|
+
return config_active_record_contents if adapter == 'active_record'
|
55
|
+
return config_sequel_contents if adapter == 'sequel'
|
56
|
+
|
57
|
+
raise ArgumentError, "Unknown adapter #{adapter}"
|
47
58
|
end
|
48
59
|
|
49
|
-
def
|
60
|
+
def config_sequel_contents
|
50
61
|
<<~CONFIG
|
51
62
|
# frozen_string_literal: true
|
52
63
|
|
53
|
-
CycloneLariat.
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
64
|
+
CycloneLariat.configure do |c|
|
65
|
+
c.version = 1 # messages version
|
66
|
+
c.aws_key = ENV['AWS_KEY'] # aws key
|
67
|
+
c.aws_account_id = ENV['AWS_ACCOUNT_ID'] # aws account id
|
68
|
+
c.aws_secret_key = ENV['AWS_SECRET_KEY'] # aws secret
|
69
|
+
c.aws_region = ENV['AWS_REGION'] # aws default region
|
70
|
+
c.publisher = ENV['APP_NAME'] # name of your publishers, usually name of your application
|
71
|
+
c.instance = ENV['INSTANCE'] # stage, production, test
|
72
|
+
c.driver = :sequel # :sequel or :active_record
|
73
|
+
c.messages_dataset = DB[:messages] # Sequel dataset / ActiveRecord model for store income messages (on receiver)
|
74
|
+
c.versions_dataset = DB[:lariat_versions] # Sequel dataset / ActiveRecord model for publisher migrations
|
75
|
+
c.fake_publish = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
|
76
|
+
end
|
77
|
+
CONFIG
|
78
|
+
end
|
79
|
+
|
80
|
+
def config_active_record_contents
|
81
|
+
<<~CONFIG
|
82
|
+
# frozen_string_literal: true
|
83
|
+
|
84
|
+
CycloneLariat.configure do |c|
|
85
|
+
c.version = 1 # messages version
|
86
|
+
c.aws_key = ENV['AWS_KEY'] # aws key
|
87
|
+
c.aws_account_id = ENV['AWS_ACCOUNT_ID'] # aws account id
|
88
|
+
c.aws_secret_key = ENV['AWS_SECRET_KEY'] # aws secret
|
89
|
+
c.aws_region = ENV['AWS_REGION'] # aws default region
|
90
|
+
c.publisher = ENV['APP_NAME'] # name of your publishers, usually name of your application
|
91
|
+
c.instance = ENV['INSTANCE'] # stage, production, test
|
92
|
+
c.driver = :active_record # :sequel or :active_record
|
93
|
+
c.messages_dataset = CycloneLariatMessage # Sequel dataset / ActiveRecord model for store income messages (on receiver)
|
94
|
+
c.versions_dataset = CycloneLariatVersion # Sequel dataset / ActiveRecord model for publisher migrations
|
95
|
+
c.fake_publish = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
|
63
96
|
end
|
64
97
|
CONFIG
|
65
98
|
end
|
@@ -69,46 +102,44 @@ module CycloneLariat
|
|
69
102
|
# frozen_string_literal: true
|
70
103
|
|
71
104
|
require 'cyclone_lariat'
|
72
|
-
|
105
|
+
|
73
106
|
namespace :cyclone_lariat do
|
74
107
|
desc 'Migrate topics for SQS/SNS'
|
75
|
-
task migrate: :
|
76
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
108
|
+
task migrate: :cyclone_lariat_config do
|
77
109
|
CycloneLariat::Migration.migrate
|
78
110
|
end
|
79
|
-
|
111
|
+
|
80
112
|
desc 'Rollback topics for SQS/SNS'
|
81
|
-
task :rollback, [:version] => :
|
82
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
113
|
+
task :rollback, [:version] => :cyclone_lariat_config do |_, args|
|
83
114
|
target_version = args[:version] ? args[:version].to_i : nil
|
84
115
|
CycloneLariat::Migration.rollback(target_version)
|
85
116
|
end
|
86
|
-
|
117
|
+
|
87
118
|
namespace :list do
|
88
119
|
desc 'List all topics'
|
89
|
-
task :
|
90
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
120
|
+
task topics: :cyclone_lariat_config do
|
91
121
|
CycloneLariat::Migration.list_topics
|
92
122
|
end
|
93
|
-
|
123
|
+
|
94
124
|
desc 'List all queues'
|
95
|
-
task :
|
96
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
125
|
+
task queues: :cyclone_lariat_config do
|
97
126
|
CycloneLariat::Migration.list_queues
|
98
127
|
end
|
99
|
-
|
128
|
+
|
100
129
|
desc 'List all subscriptions'
|
101
|
-
task :
|
102
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
130
|
+
task subscriptions: :cyclone_lariat_config do
|
103
131
|
CycloneLariat::Migration.list_subscriptions
|
104
132
|
end
|
105
133
|
end
|
106
|
-
|
134
|
+
|
107
135
|
desc 'Build graphviz graph for whole system'
|
108
|
-
task :
|
109
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
136
|
+
task graph: :cyclone_lariat_config do
|
110
137
|
CycloneLariat::Migration.build_graph
|
111
138
|
end
|
139
|
+
|
140
|
+
task :cyclone_lariat_config do
|
141
|
+
require_relative '../../config/initializers/cyclone_lariat'
|
142
|
+
end
|
112
143
|
end
|
113
144
|
TASKS
|
114
145
|
end
|
@@ -125,10 +156,11 @@ module CycloneLariat
|
|
125
156
|
|
126
157
|
FileUtils.mkdir_p CycloneLariat::Migration::DIR unless Dir.exist? CycloneLariat::Migration::DIR
|
127
158
|
|
128
|
-
file_name
|
129
|
-
class_name = generate_class_name
|
130
|
-
|
131
|
-
file.
|
159
|
+
file_name = generate_filename(title)
|
160
|
+
class_name = generate_class_name(title)
|
161
|
+
|
162
|
+
file = File.open(file_name, 'w')
|
163
|
+
file.puts(file_contents(class_name))
|
132
164
|
puts "Migration successful created:\n\t#{file_name}"
|
133
165
|
end
|
134
166
|
|
@@ -153,7 +185,7 @@ module CycloneLariat
|
|
153
185
|
class #{klass_name} < CycloneLariat::Migration
|
154
186
|
def up
|
155
187
|
end
|
156
|
-
|
188
|
+
|
157
189
|
def down
|
158
190
|
end
|
159
191
|
end
|
data/cyclone_lariat.gemspec
CHANGED
@@ -7,7 +7,7 @@ require 'cyclone_lariat/version'
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
8
|
spec.name = 'cyclone_lariat'
|
9
9
|
spec.version = CycloneLariat::VERSION
|
10
|
-
spec.authors = ['Alexander Kudrin', 'Philip Sorokin']
|
10
|
+
spec.authors = ['Alexander Kudrin', 'Philip Sorokin', 'Kirill Drozdov', 'Vitaly Perminov']
|
11
11
|
spec.email = ['kudrin.alexander@gmail.com']
|
12
12
|
|
13
13
|
spec.summary = 'Shoryuken middleware for LunaPark based application.'
|
@@ -28,21 +28,28 @@ Gem::Specification.new do |spec|
|
|
28
28
|
# Specify which files should be added to the gem when it is released.
|
29
29
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
30
30
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
31
|
-
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
31
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|docs|examples|config)/}) }
|
32
32
|
end
|
33
33
|
|
34
34
|
spec.require_paths = ['lib']
|
35
|
-
spec.
|
35
|
+
spec.bindir = 'bin'
|
36
|
+
spec.executables = ['cyclone_lariat']
|
36
37
|
|
37
38
|
spec.add_dependency 'aws-sdk-sns'
|
38
39
|
spec.add_dependency 'aws-sdk-sqs'
|
39
40
|
spec.add_dependency 'dry-cli', '~> 0.6'
|
41
|
+
spec.add_dependency 'dry-validation', '~> 1.5'
|
40
42
|
spec.add_dependency 'luna_park', '~> 0.11'
|
41
43
|
spec.add_dependency 'terminal-table', '~> 3.0'
|
42
44
|
|
43
45
|
spec.add_development_dependency 'bundler', '~> 1.17'
|
44
46
|
spec.add_development_dependency 'byebug', '~> 11.1'
|
47
|
+
spec.add_development_dependency 'database_cleaner-active_record'
|
45
48
|
spec.add_development_dependency 'database_cleaner-sequel', '~> 2.0'
|
49
|
+
spec.add_development_dependency 'guard'
|
50
|
+
spec.add_development_dependency 'guard-bundler'
|
51
|
+
spec.add_development_dependency 'guard-rspec'
|
52
|
+
spec.add_development_dependency 'guard-rubocop'
|
46
53
|
spec.add_development_dependency 'pg', '~> 1.2'
|
47
54
|
spec.add_development_dependency 'pry', '~> 0.13'
|
48
55
|
spec.add_development_dependency 'pry-byebug', '~> 3.9'
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'luna_park/extensions/injector'
|
4
|
+
require 'cyclone_lariat/generators/event'
|
5
|
+
require 'cyclone_lariat/generators/command'
|
6
|
+
require 'cyclone_lariat/errors'
|
7
|
+
require 'cyclone_lariat/core'
|
8
|
+
|
9
|
+
module CycloneLariat
|
10
|
+
module Clients
|
11
|
+
class Abstract
|
12
|
+
include LunaPark::Extensions::Injector
|
13
|
+
include Generators::Event
|
14
|
+
include Generators::Command
|
15
|
+
|
16
|
+
dependency(:aws_client_class) { raise ArgumentError, 'Client class should be defined' }
|
17
|
+
dependency(:aws_credentials_class) { Aws::Credentials }
|
18
|
+
|
19
|
+
def initialize(**options)
|
20
|
+
@config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
|
21
|
+
end
|
22
|
+
|
23
|
+
attr_reader :config
|
24
|
+
|
25
|
+
def publish
|
26
|
+
raise LunaPark::Errors::AbstractMethod, 'Publish method should be defined'
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def aws_client
|
32
|
+
@aws_client ||= aws_client_class.new(credentials: aws_credentials, region: config.aws_region)
|
33
|
+
end
|
34
|
+
|
35
|
+
def aws_credentials
|
36
|
+
@aws_credentials ||= aws_credentials_class.new(config.aws_key, config.aws_secret_key)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,163 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aws-sdk-sns'
|
4
|
+
require 'cyclone_lariat/fake'
|
5
|
+
require 'cyclone_lariat/clients/abstract'
|
6
|
+
require 'cyclone_lariat/resources/topic'
|
7
|
+
require 'cyclone_lariat/resources/queue'
|
8
|
+
|
9
|
+
module CycloneLariat
|
10
|
+
module Clients
|
11
|
+
class Sns < Abstract
|
12
|
+
include LunaPark::Extensions::Injector
|
13
|
+
include Generators::Topic
|
14
|
+
|
15
|
+
dependency(:aws_client_class) { Aws::SNS::Client }
|
16
|
+
|
17
|
+
def publish(msg, fifo:, topic: nil, skip_validation: false)
|
18
|
+
return Fake.sns_publish_response(msg) if config.fake_publish
|
19
|
+
|
20
|
+
topic = topic ? custom_topic(topic) : topic(msg.type, kind: msg.kind, fifo: fifo)
|
21
|
+
|
22
|
+
raise Errors::GroupIdUndefined.new(resource: topic) if fifo && msg.group_id.nil?
|
23
|
+
raise Errors::GroupDefined.new(resource: topic) if !fifo && msg.group_id
|
24
|
+
raise Errors::DeduplicationIdDefined.new(resource: topic) if !fifo && msg.deduplication_id
|
25
|
+
|
26
|
+
msg.validation.check! unless skip_validation
|
27
|
+
|
28
|
+
params = {
|
29
|
+
topic_arn: topic.arn,
|
30
|
+
message: msg.to_json,
|
31
|
+
message_group_id: msg.group_id,
|
32
|
+
message_deduplication_id: msg.deduplication_id
|
33
|
+
}.compact
|
34
|
+
|
35
|
+
aws_client.publish(**params)
|
36
|
+
end
|
37
|
+
|
38
|
+
def exists?(topic)
|
39
|
+
raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
|
40
|
+
|
41
|
+
aws_client.get_topic_attributes({ topic_arn: topic.arn }) && true
|
42
|
+
rescue Aws::SNS::Errors::NotFound
|
43
|
+
false
|
44
|
+
end
|
45
|
+
|
46
|
+
def publish_event(type, fifo:, topic: nil, **options)
|
47
|
+
options[:version] ||= config.version
|
48
|
+
options[:data] ||= {}
|
49
|
+
options[:uuid] ||= SecureRandom.uuid
|
50
|
+
|
51
|
+
publish event(type, **options), fifo: fifo, topic: topic
|
52
|
+
end
|
53
|
+
|
54
|
+
def publish_command(type, fifo:, topic: nil, **options)
|
55
|
+
options[:version] ||= config.version
|
56
|
+
options[:data] ||= {}
|
57
|
+
options[:uuid] ||= SecureRandom.uuid
|
58
|
+
|
59
|
+
publish command(type, **options), fifo: fifo, topic: topic
|
60
|
+
end
|
61
|
+
|
62
|
+
def create(topic)
|
63
|
+
raise ArgumentError, 'Should be Resources::Topic' unless topic.is_a? Resources::Topic
|
64
|
+
raise Errors::TopicAlreadyExists.new(expected_topic: topic.name) if exists?(topic)
|
65
|
+
|
66
|
+
aws_client.create_topic(name: topic.name, attributes: topic.attributes, tags: topic.tags)
|
67
|
+
topic
|
68
|
+
end
|
69
|
+
|
70
|
+
def delete(topic)
|
71
|
+
raise ArgumentError, 'Should be Resources::Topic' unless topic.is_a? Resources::Topic
|
72
|
+
raise Errors::TopicDoesNotExists.new(expected_topic: topic.name) unless exists?(topic)
|
73
|
+
|
74
|
+
aws_client.delete_topic topic_arn: topic.arn
|
75
|
+
topic
|
76
|
+
end
|
77
|
+
|
78
|
+
def subscribe(topic:, endpoint:)
|
79
|
+
subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
|
80
|
+
raise Errors::SubscriptionAlreadyExists.new(topic: topic, endpoint: endpoint) if subscription_arn
|
81
|
+
|
82
|
+
aws_client.subscribe(
|
83
|
+
{
|
84
|
+
topic_arn: topic.arn,
|
85
|
+
protocol: endpoint.protocol,
|
86
|
+
endpoint: endpoint.arn
|
87
|
+
}
|
88
|
+
)
|
89
|
+
end
|
90
|
+
|
91
|
+
def unsubscribe(topic:, endpoint:)
|
92
|
+
subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
|
93
|
+
raise Errors::SubscriptionDoesNotExists.new(topic: topic, endpoint: endpoint) unless subscription_arn
|
94
|
+
|
95
|
+
aws_client.unsubscribe(subscription_arn: subscription_arn)
|
96
|
+
end
|
97
|
+
|
98
|
+
def list_all
|
99
|
+
topics = []
|
100
|
+
resp = aws_client.list_topics
|
101
|
+
|
102
|
+
loop do
|
103
|
+
resp[:topics].map do |t|
|
104
|
+
topics << Resources::Topic.from_arn(t[:topic_arn])
|
105
|
+
end
|
106
|
+
|
107
|
+
break if resp[:next_token].nil?
|
108
|
+
|
109
|
+
resp = aws_client.list_topics(next_token: resp[:next_token])
|
110
|
+
end
|
111
|
+
topics
|
112
|
+
end
|
113
|
+
|
114
|
+
def list_subscriptions
|
115
|
+
subscriptions = []
|
116
|
+
resp = aws_client.list_subscriptions
|
117
|
+
|
118
|
+
loop do
|
119
|
+
resp[:subscriptions].each do |s|
|
120
|
+
endpoint = s.endpoint.split(':')[2] == 'sqs' ? Resources::Queue.from_arn(s.endpoint) : Resources::Topic.from_arn(s.endpoint)
|
121
|
+
subscriptions << { topic: Resources::Topic.from_arn(s.topic_arn), endpoint: endpoint, arn: s.subscription_arn }
|
122
|
+
end
|
123
|
+
|
124
|
+
break if resp[:next_token].nil?
|
125
|
+
|
126
|
+
resp = aws_client.list_subscriptions(next_token: resp[:next_token])
|
127
|
+
end
|
128
|
+
subscriptions
|
129
|
+
end
|
130
|
+
|
131
|
+
def topic_subscriptions(topic)
|
132
|
+
raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
|
133
|
+
|
134
|
+
subscriptions = []
|
135
|
+
|
136
|
+
resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn)
|
137
|
+
|
138
|
+
loop do
|
139
|
+
next_token = resp[:next_token]
|
140
|
+
subscriptions += resp[:subscriptions]
|
141
|
+
|
142
|
+
break if next_token.nil?
|
143
|
+
|
144
|
+
resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn, next_token: next_token)
|
145
|
+
end
|
146
|
+
subscriptions
|
147
|
+
end
|
148
|
+
|
149
|
+
def find_subscription_arn(topic:, endpoint:)
|
150
|
+
raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
|
151
|
+
unless [Resources::Topic, Resources::Queue].include? endpoint.class
|
152
|
+
raise ArgumentError, 'Endpoint should be Topic or Queue'
|
153
|
+
end
|
154
|
+
|
155
|
+
found_subscription = topic_subscriptions(topic).select do |subscription|
|
156
|
+
subscription.endpoint == endpoint.arn
|
157
|
+
end.first
|
158
|
+
|
159
|
+
found_subscription ? found_subscription.subscription_arn : nil
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aws-sdk-sqs'
|
4
|
+
require 'cyclone_lariat/fake'
|
5
|
+
require 'cyclone_lariat/clients/abstract'
|
6
|
+
require 'cyclone_lariat/resources/queue'
|
7
|
+
require 'cyclone_lariat/generators/queue'
|
8
|
+
|
9
|
+
module CycloneLariat
|
10
|
+
module Clients
|
11
|
+
class Sqs < Abstract
|
12
|
+
include LunaPark::Extensions::Injector
|
13
|
+
include Generators::Queue
|
14
|
+
|
15
|
+
dependency(:aws_client_class) { Aws::SQS::Client }
|
16
|
+
|
17
|
+
def exists?(queue)
|
18
|
+
raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
|
19
|
+
|
20
|
+
aws_client.get_queue_url(queue_name: queue.to_s) && true
|
21
|
+
rescue Aws::SQS::Errors::NonExistentQueue
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def add_policy(queue:, policy:)
|
26
|
+
current_policy_json = aws_client.get_queue_attributes({
|
27
|
+
queue_url: queue.url,
|
28
|
+
attribute_names: ['Policy']
|
29
|
+
}).attributes['Policy']
|
30
|
+
|
31
|
+
current_policy = JSON.parse(current_policy_json) if current_policy_json
|
32
|
+
|
33
|
+
return if current_policy && current_policy['Statement'].find { |s| s['Sid'] == policy['Sid'] }
|
34
|
+
|
35
|
+
new_policy = current_policy || { 'Statement' => [] }
|
36
|
+
new_policy['Statement'] << policy
|
37
|
+
|
38
|
+
aws_client.set_queue_attributes({ queue_url: queue.url, attributes: { 'Policy' => new_policy.to_json } })
|
39
|
+
end
|
40
|
+
|
41
|
+
def publish(msg, fifo:, dest: nil, queue: nil, skip_validation: false)
|
42
|
+
return Fake.sqs_send_message_result(msg) if config.fake_publish
|
43
|
+
|
44
|
+
queue = queue ? custom_queue(queue) : queue(msg.type, kind: msg.kind, fifo: fifo, dest: dest)
|
45
|
+
|
46
|
+
raise Errors::GroupIdUndefined.new(resource: queue) if fifo && msg.group_id.nil?
|
47
|
+
raise Errors::GroupDefined.new(resource: queue) if !fifo && msg.group_id
|
48
|
+
raise Errors::DeduplicationIdDefined.new(resource: queue) if !fifo && msg.deduplication_id
|
49
|
+
|
50
|
+
msg.validation.check! unless skip_validation
|
51
|
+
|
52
|
+
params = {
|
53
|
+
queue_url: queue.url,
|
54
|
+
message_body: msg.to_json,
|
55
|
+
message_group_id: msg.group_id,
|
56
|
+
message_deduplication_id: msg.deduplication_id
|
57
|
+
}.compact
|
58
|
+
|
59
|
+
aws_client.send_message(**params)
|
60
|
+
end
|
61
|
+
|
62
|
+
def publish_event(type, fifo:, dest: nil, queue: nil, **options)
|
63
|
+
options[:version] ||= self.config.version
|
64
|
+
options[:data] ||= {}
|
65
|
+
options[:uuid] ||= SecureRandom.uuid
|
66
|
+
|
67
|
+
publish event(type, data: data, **options), fifo: fifo, dest: dest, queue: queue
|
68
|
+
end
|
69
|
+
|
70
|
+
def publish_command(type, fifo:, dest: nil, queue: nil, **options)
|
71
|
+
options[:version] ||= self.config.version
|
72
|
+
options[:data] ||= {}
|
73
|
+
options[:uuid] ||= SecureRandom.uuid
|
74
|
+
|
75
|
+
publish event(type, data: data, **options), fifo: fifo, dest: dest, queue: queue
|
76
|
+
end
|
77
|
+
|
78
|
+
def create(queue)
|
79
|
+
raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
|
80
|
+
raise Errors::QueueAlreadyExists.new(expected_queue: queue.name) if exists?(queue)
|
81
|
+
|
82
|
+
aws_client.create_queue(queue_name: queue.name, attributes: queue.attributes, tags: queue.tags)
|
83
|
+
queue
|
84
|
+
end
|
85
|
+
|
86
|
+
def delete(queue)
|
87
|
+
raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
|
88
|
+
raise Errors::QueueDoesNotExists.new(expected_queue: queue.name) unless exists?(queue)
|
89
|
+
|
90
|
+
aws_client.delete_queue queue_url: queue.url
|
91
|
+
queue
|
92
|
+
end
|
93
|
+
|
94
|
+
def list_all
|
95
|
+
queues = []
|
96
|
+
resp = aws_client.list_queues
|
97
|
+
|
98
|
+
loop do
|
99
|
+
next_token = resp[:next_token]
|
100
|
+
|
101
|
+
resp[:queue_urls].map do |url|
|
102
|
+
queues << Resources::Queue.from_url(url)
|
103
|
+
end
|
104
|
+
|
105
|
+
break if next_token.nil?
|
106
|
+
|
107
|
+
resp = aws_client.list_queues(next_token: next_token)
|
108
|
+
end
|
109
|
+
|
110
|
+
queues
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'cyclone_lariat/generators/queue'
|
4
|
+
require 'cyclone_lariat/generators/topic'
|
5
|
+
require 'cyclone_lariat/options'
|
6
|
+
|
7
|
+
module CycloneLariat
|
8
|
+
module CycloneLariatMethods
|
9
|
+
def config
|
10
|
+
@config ||= Options.new
|
11
|
+
end
|
12
|
+
|
13
|
+
def configure
|
14
|
+
yield(config)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
extend Generators::Topic
|
19
|
+
extend Generators::Queue
|
20
|
+
extend CycloneLariatMethods
|
21
|
+
end
|
@@ -40,5 +40,21 @@ module CycloneLariat
|
|
40
40
|
class SubscriptionDoesNotExists < LunaPark::Errors::System
|
41
41
|
message { |d| "Subscription for topic `#{d[:topic].name}`, on endpoint `#{d[:endpoint].name}` does not exists" }
|
42
42
|
end
|
43
|
+
|
44
|
+
class InvalidMessage < LunaPark::Errors::Business
|
45
|
+
message 'Message is not valid'
|
46
|
+
end
|
47
|
+
|
48
|
+
class GroupIdUndefined < LunaPark::Errors::System
|
49
|
+
message { |d| "Group id must be defined for FIFO resources: `#{d[:resource].name}`" }
|
50
|
+
end
|
51
|
+
|
52
|
+
class GroupDefined < LunaPark::Errors::System
|
53
|
+
message { |d| "Group id must be nil for non-FIFO resources: `#{d[:resource].name}`" }
|
54
|
+
end
|
55
|
+
|
56
|
+
class DeduplicationIdDefined < LunaPark::Errors::System
|
57
|
+
message { |d| "Deduplication id must be nil for non-FIFO resources: `#{d[:resource].name}`" }
|
58
|
+
end
|
43
59
|
end
|
44
60
|
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module CycloneLariat
|
4
|
+
class Fake
|
5
|
+
def self.sns_publish_response(message)
|
6
|
+
Aws::SNS::Types::PublishResponse.new.tap do |resp|
|
7
|
+
resp.message_id = SecureRandom.uuid
|
8
|
+
resp.sequence_number = rand(10).to_s if message.fifo?
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.sqs_send_message_result(message)
|
13
|
+
Aws::SQS::Types::SendMessageResult.new.tap do |res|
|
14
|
+
res.message_id = SecureRandom.uuid
|
15
|
+
res.sequence_number = rand(10).to_s if message.fifo?
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'securerandom'
|
4
|
+
require 'cyclone_lariat/messages/v1/command'
|
5
|
+
require 'cyclone_lariat/messages/v2/command'
|
6
|
+
|
7
|
+
module CycloneLariat
|
8
|
+
module Generators
|
9
|
+
module Command
|
10
|
+
def command(type, version: config.version, **options)
|
11
|
+
case version.to_i
|
12
|
+
when 1 then command_v1(type, **options)
|
13
|
+
when 2 then command_v2(type, **options)
|
14
|
+
else raise ArgumentError, "Unknown version #{version}"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def command_v1(type, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
|
19
|
+
params = {
|
20
|
+
uuid: uuid,
|
21
|
+
type: type,
|
22
|
+
sent_at: Time.now.iso8601(3),
|
23
|
+
version: 1,
|
24
|
+
publisher: config.publisher,
|
25
|
+
data: data,
|
26
|
+
request_id: request_id,
|
27
|
+
group_id: group_id,
|
28
|
+
deduplication_id: deduplication_id
|
29
|
+
}
|
30
|
+
|
31
|
+
Messages::V1::Command.wrap(params.compact)
|
32
|
+
end
|
33
|
+
|
34
|
+
def command_v2(type, subject:, object:, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
|
35
|
+
params = {
|
36
|
+
uuid: uuid,
|
37
|
+
type: type,
|
38
|
+
subject: subject,
|
39
|
+
object: object,
|
40
|
+
sent_at: Time.now.iso8601(3),
|
41
|
+
version: 2,
|
42
|
+
publisher: config.publisher,
|
43
|
+
data: data,
|
44
|
+
request_id: request_id,
|
45
|
+
group_id: group_id,
|
46
|
+
deduplication_id: deduplication_id
|
47
|
+
}
|
48
|
+
|
49
|
+
Messages::V2::Command.wrap(params.compact)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|