cyclone_lariat 0.4.0 → 1.0.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/gem-push.yml +4 -4
- data/.rubocop.yml +9 -5
- data/Gemfile.lock +123 -21
- data/Guardfile +42 -0
- data/README.md +417 -220
- data/bin/cyclone_lariat +75 -43
- data/cyclone_lariat.gemspec +10 -3
- data/lib/cyclone_lariat/clients/abstract.rb +40 -0
- data/lib/cyclone_lariat/clients/sns.rb +163 -0
- data/lib/cyclone_lariat/clients/sqs.rb +114 -0
- data/lib/cyclone_lariat/core.rb +21 -0
- data/lib/cyclone_lariat/errors.rb +16 -0
- data/lib/cyclone_lariat/fake.rb +19 -0
- data/lib/cyclone_lariat/generators/command.rb +53 -0
- data/lib/cyclone_lariat/generators/event.rb +52 -0
- data/lib/cyclone_lariat/generators/queue.rb +30 -0
- data/lib/cyclone_lariat/generators/topic.rb +29 -0
- data/lib/cyclone_lariat/messages/v1/abstract.rb +139 -0
- data/lib/cyclone_lariat/messages/v1/command.rb +20 -0
- data/lib/cyclone_lariat/messages/v1/event.rb +20 -0
- data/lib/cyclone_lariat/messages/v1/validator.rb +31 -0
- data/lib/cyclone_lariat/messages/v2/abstract.rb +149 -0
- data/lib/cyclone_lariat/messages/v2/command.rb +20 -0
- data/lib/cyclone_lariat/messages/v2/event.rb +20 -0
- data/lib/cyclone_lariat/messages/v2/validator.rb +39 -0
- data/lib/cyclone_lariat/middleware.rb +9 -6
- data/lib/cyclone_lariat/migration.rb +54 -117
- data/lib/cyclone_lariat/options.rb +52 -0
- data/lib/cyclone_lariat/presenters/graph.rb +54 -0
- data/lib/cyclone_lariat/presenters/queues.rb +41 -0
- data/lib/cyclone_lariat/presenters/subscriptions.rb +34 -0
- data/lib/cyclone_lariat/presenters/topics.rb +40 -0
- data/lib/cyclone_lariat/publisher.rb +25 -0
- data/lib/cyclone_lariat/repo/active_record/messages.rb +92 -0
- data/lib/cyclone_lariat/repo/active_record/versions.rb +28 -0
- data/lib/cyclone_lariat/repo/messages.rb +43 -0
- data/lib/cyclone_lariat/repo/messages_mapper.rb +49 -0
- data/lib/cyclone_lariat/repo/sequel/messages.rb +73 -0
- data/lib/cyclone_lariat/repo/sequel/versions.rb +28 -0
- data/lib/cyclone_lariat/repo/versions.rb +42 -0
- data/lib/cyclone_lariat/resources/queue.rb +167 -0
- data/lib/cyclone_lariat/resources/topic.rb +132 -0
- data/lib/cyclone_lariat/services/migrate.rb +51 -0
- data/lib/cyclone_lariat/services/rollback.rb +51 -0
- data/lib/cyclone_lariat/version.rb +1 -1
- data/lib/cyclone_lariat.rb +4 -11
- data/lib/tasks/console.rake +1 -1
- data/lib/tasks/cyclone_lariat.rake +10 -12
- data/lib/tasks/db.rake +0 -15
- metadata +127 -27
- data/config/db.example.rb +0 -9
- data/config/initializers/sequel.rb +0 -7
- data/db/migrate/01_add_uuid_extensions.rb +0 -15
- data/db/migrate/02_add_events.rb +0 -19
- data/db/migrate/03_add_versions.rb +0 -9
- data/docs/_imgs/graphviz_01.png +0 -0
- data/docs/_imgs/graphviz_02.png +0 -0
- data/docs/_imgs/graphviz_03.png +0 -0
- data/docs/_imgs/lariat.jpg +0 -0
- data/docs/_imgs/logic.png +0 -0
- data/docs/_imgs/sqs_sns_diagram.png +0 -0
- data/lib/cyclone_lariat/abstract/client.rb +0 -112
- data/lib/cyclone_lariat/abstract/message.rb +0 -98
- data/lib/cyclone_lariat/command.rb +0 -13
- data/lib/cyclone_lariat/configure.rb +0 -15
- data/lib/cyclone_lariat/event.rb +0 -13
- data/lib/cyclone_lariat/messages_mapper.rb +0 -46
- data/lib/cyclone_lariat/messages_repo.rb +0 -60
- data/lib/cyclone_lariat/queue.rb +0 -147
- data/lib/cyclone_lariat/sns_client.rb +0 -149
- data/lib/cyclone_lariat/sqs_client.rb +0 -93
- data/lib/cyclone_lariat/topic.rb +0 -113
data/bin/cyclone_lariat
CHANGED
@@ -24,13 +24,25 @@ module CycloneLariat
|
|
24
24
|
|
25
25
|
class Install < Dry::CLI::Command
|
26
26
|
desc 'Install cyclone lariat to current directory'
|
27
|
+
option :adapter,
|
28
|
+
default: 'sequel',
|
29
|
+
values: %w[sequel active_record],
|
30
|
+
desc: 'adapter for store events and versions'
|
27
31
|
|
28
|
-
def call(
|
29
|
-
create_config
|
32
|
+
def call(adapter: 'sequel', **)
|
33
|
+
create_config(adapter)
|
30
34
|
create_rake_task
|
31
35
|
end
|
32
36
|
|
33
|
-
def create_config
|
37
|
+
def create_config(adapter)
|
38
|
+
FileUtils.mkdir_p INITIALIZERS_DIR unless Dir.exist? INITIALIZERS_DIR
|
39
|
+
config_path = "#{INITIALIZERS_DIR}/cyclone_lariat.rb"
|
40
|
+
config_file = File.open(config_path, 'w')
|
41
|
+
config_file.puts config_contents(adapter)
|
42
|
+
puts "Created config: #{config_path}"
|
43
|
+
end
|
44
|
+
|
45
|
+
def create_rake_task
|
34
46
|
FileUtils.mkdir_p RAKE_TASKS_DIR unless Dir.exist? RAKE_TASKS_DIR
|
35
47
|
config_path = "#{RAKE_TASKS_DIR}/cyclone_lariat.rake"
|
36
48
|
config_file = File.open(config_path, 'w')
|
@@ -38,28 +50,49 @@ module CycloneLariat
|
|
38
50
|
puts "Created rake task: #{config_path}"
|
39
51
|
end
|
40
52
|
|
41
|
-
def
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
puts "Created config: #{config_path}"
|
53
|
+
def config_contents(adapter)
|
54
|
+
return config_active_record_contents if adapter == 'active_record'
|
55
|
+
return config_sequel_contents if adapter == 'sequel'
|
56
|
+
|
57
|
+
raise ArgumentError, "Unknown adapter #{adapter}"
|
47
58
|
end
|
48
59
|
|
49
|
-
def
|
60
|
+
def config_sequel_contents
|
50
61
|
<<~CONFIG
|
51
62
|
# frozen_string_literal: true
|
52
63
|
|
53
|
-
CycloneLariat.
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
64
|
+
CycloneLariat.configure do |c|
|
65
|
+
c.version = 1 # messages version
|
66
|
+
c.aws_key = ENV['AWS_KEY'] # aws key
|
67
|
+
c.aws_account_id = ENV['AWS_ACCOUNT_ID'] # aws account id
|
68
|
+
c.aws_secret_key = ENV['AWS_SECRET_KEY'] # aws secret
|
69
|
+
c.aws_region = ENV['AWS_REGION'] # aws default region
|
70
|
+
c.publisher = ENV['APP_NAME'] # name of your publishers, usually name of your application
|
71
|
+
c.instance = ENV['INSTANCE'] # stage, production, test
|
72
|
+
c.driver = :sequel # :sequel or :active_record
|
73
|
+
c.messages_dataset = DB[:messages] # Sequel dataset / ActiveRecord model for store income messages (on receiver)
|
74
|
+
c.versions_dataset = DB[:lariat_versions] # Sequel dataset / ActiveRecord model for publisher migrations
|
75
|
+
c.fake_publish = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
|
76
|
+
end
|
77
|
+
CONFIG
|
78
|
+
end
|
79
|
+
|
80
|
+
def config_active_record_contents
|
81
|
+
<<~CONFIG
|
82
|
+
# frozen_string_literal: true
|
83
|
+
|
84
|
+
CycloneLariat.configure do |c|
|
85
|
+
c.version = 1 # messages version
|
86
|
+
c.aws_key = ENV['AWS_KEY'] # aws key
|
87
|
+
c.aws_account_id = ENV['AWS_ACCOUNT_ID'] # aws account id
|
88
|
+
c.aws_secret_key = ENV['AWS_SECRET_KEY'] # aws secret
|
89
|
+
c.aws_region = ENV['AWS_REGION'] # aws default region
|
90
|
+
c.publisher = ENV['APP_NAME'] # name of your publishers, usually name of your application
|
91
|
+
c.instance = ENV['INSTANCE'] # stage, production, test
|
92
|
+
c.driver = :active_record # :sequel or :active_record
|
93
|
+
c.messages_dataset = CycloneLariatMessage # Sequel dataset / ActiveRecord model for store income messages (on receiver)
|
94
|
+
c.versions_dataset = CycloneLariatVersion # Sequel dataset / ActiveRecord model for publisher migrations
|
95
|
+
c.fake_publish = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
|
63
96
|
end
|
64
97
|
CONFIG
|
65
98
|
end
|
@@ -69,46 +102,44 @@ module CycloneLariat
|
|
69
102
|
# frozen_string_literal: true
|
70
103
|
|
71
104
|
require 'cyclone_lariat'
|
72
|
-
|
105
|
+
|
73
106
|
namespace :cyclone_lariat do
|
74
107
|
desc 'Migrate topics for SQS/SNS'
|
75
|
-
task migrate: :
|
76
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
108
|
+
task migrate: :cyclone_lariat_config do
|
77
109
|
CycloneLariat::Migration.migrate
|
78
110
|
end
|
79
|
-
|
111
|
+
|
80
112
|
desc 'Rollback topics for SQS/SNS'
|
81
|
-
task :rollback, [:version] => :
|
82
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
113
|
+
task :rollback, [:version] => :cyclone_lariat_config do |_, args|
|
83
114
|
target_version = args[:version] ? args[:version].to_i : nil
|
84
115
|
CycloneLariat::Migration.rollback(target_version)
|
85
116
|
end
|
86
|
-
|
117
|
+
|
87
118
|
namespace :list do
|
88
119
|
desc 'List all topics'
|
89
|
-
task :
|
90
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
120
|
+
task topics: :cyclone_lariat_config do
|
91
121
|
CycloneLariat::Migration.list_topics
|
92
122
|
end
|
93
|
-
|
123
|
+
|
94
124
|
desc 'List all queues'
|
95
|
-
task :
|
96
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
125
|
+
task queues: :cyclone_lariat_config do
|
97
126
|
CycloneLariat::Migration.list_queues
|
98
127
|
end
|
99
|
-
|
128
|
+
|
100
129
|
desc 'List all subscriptions'
|
101
|
-
task :
|
102
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
130
|
+
task subscriptions: :cyclone_lariat_config do
|
103
131
|
CycloneLariat::Migration.list_subscriptions
|
104
132
|
end
|
105
133
|
end
|
106
|
-
|
134
|
+
|
107
135
|
desc 'Build graphviz graph for whole system'
|
108
|
-
task :
|
109
|
-
require_relative '../../config/initializers/cyclone_lariat'
|
136
|
+
task graph: :cyclone_lariat_config do
|
110
137
|
CycloneLariat::Migration.build_graph
|
111
138
|
end
|
139
|
+
|
140
|
+
task :cyclone_lariat_config do
|
141
|
+
require_relative '../../config/initializers/cyclone_lariat'
|
142
|
+
end
|
112
143
|
end
|
113
144
|
TASKS
|
114
145
|
end
|
@@ -125,10 +156,11 @@ module CycloneLariat
|
|
125
156
|
|
126
157
|
FileUtils.mkdir_p CycloneLariat::Migration::DIR unless Dir.exist? CycloneLariat::Migration::DIR
|
127
158
|
|
128
|
-
file_name
|
129
|
-
class_name = generate_class_name
|
130
|
-
|
131
|
-
file.
|
159
|
+
file_name = generate_filename(title)
|
160
|
+
class_name = generate_class_name(title)
|
161
|
+
|
162
|
+
file = File.open(file_name, 'w')
|
163
|
+
file.puts(file_contents(class_name))
|
132
164
|
puts "Migration successful created:\n\t#{file_name}"
|
133
165
|
end
|
134
166
|
|
@@ -153,7 +185,7 @@ module CycloneLariat
|
|
153
185
|
class #{klass_name} < CycloneLariat::Migration
|
154
186
|
def up
|
155
187
|
end
|
156
|
-
|
188
|
+
|
157
189
|
def down
|
158
190
|
end
|
159
191
|
end
|
data/cyclone_lariat.gemspec
CHANGED
@@ -7,7 +7,7 @@ require 'cyclone_lariat/version'
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
8
|
spec.name = 'cyclone_lariat'
|
9
9
|
spec.version = CycloneLariat::VERSION
|
10
|
-
spec.authors = ['Alexander Kudrin', 'Philip Sorokin']
|
10
|
+
spec.authors = ['Alexander Kudrin', 'Philip Sorokin', 'Kirill Drozdov', 'Vitaly Perminov']
|
11
11
|
spec.email = ['kudrin.alexander@gmail.com']
|
12
12
|
|
13
13
|
spec.summary = 'Shoryuken middleware for LunaPark based application.'
|
@@ -28,21 +28,28 @@ Gem::Specification.new do |spec|
|
|
28
28
|
# Specify which files should be added to the gem when it is released.
|
29
29
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
30
30
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
31
|
-
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
31
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|docs|examples|config)/}) }
|
32
32
|
end
|
33
33
|
|
34
34
|
spec.require_paths = ['lib']
|
35
|
-
spec.
|
35
|
+
spec.bindir = 'bin'
|
36
|
+
spec.executables = ['cyclone_lariat']
|
36
37
|
|
37
38
|
spec.add_dependency 'aws-sdk-sns'
|
38
39
|
spec.add_dependency 'aws-sdk-sqs'
|
39
40
|
spec.add_dependency 'dry-cli', '~> 0.6'
|
41
|
+
spec.add_dependency 'dry-validation', '~> 1.5'
|
40
42
|
spec.add_dependency 'luna_park', '~> 0.11'
|
41
43
|
spec.add_dependency 'terminal-table', '~> 3.0'
|
42
44
|
|
43
45
|
spec.add_development_dependency 'bundler', '~> 1.17'
|
44
46
|
spec.add_development_dependency 'byebug', '~> 11.1'
|
47
|
+
spec.add_development_dependency 'database_cleaner-active_record'
|
45
48
|
spec.add_development_dependency 'database_cleaner-sequel', '~> 2.0'
|
49
|
+
spec.add_development_dependency 'guard'
|
50
|
+
spec.add_development_dependency 'guard-bundler'
|
51
|
+
spec.add_development_dependency 'guard-rspec'
|
52
|
+
spec.add_development_dependency 'guard-rubocop'
|
46
53
|
spec.add_development_dependency 'pg', '~> 1.2'
|
47
54
|
spec.add_development_dependency 'pry', '~> 0.13'
|
48
55
|
spec.add_development_dependency 'pry-byebug', '~> 3.9'
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'luna_park/extensions/injector'
|
4
|
+
require 'cyclone_lariat/generators/event'
|
5
|
+
require 'cyclone_lariat/generators/command'
|
6
|
+
require 'cyclone_lariat/errors'
|
7
|
+
require 'cyclone_lariat/core'
|
8
|
+
|
9
|
+
module CycloneLariat
|
10
|
+
module Clients
|
11
|
+
class Abstract
|
12
|
+
include LunaPark::Extensions::Injector
|
13
|
+
include Generators::Event
|
14
|
+
include Generators::Command
|
15
|
+
|
16
|
+
dependency(:aws_client_class) { raise ArgumentError, 'Client class should be defined' }
|
17
|
+
dependency(:aws_credentials_class) { Aws::Credentials }
|
18
|
+
|
19
|
+
def initialize(**options)
|
20
|
+
@config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
|
21
|
+
end
|
22
|
+
|
23
|
+
attr_reader :config
|
24
|
+
|
25
|
+
def publish
|
26
|
+
raise LunaPark::Errors::AbstractMethod, 'Publish method should be defined'
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def aws_client
|
32
|
+
@aws_client ||= aws_client_class.new(credentials: aws_credentials, region: config.aws_region)
|
33
|
+
end
|
34
|
+
|
35
|
+
def aws_credentials
|
36
|
+
@aws_credentials ||= aws_credentials_class.new(config.aws_key, config.aws_secret_key)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,163 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aws-sdk-sns'
|
4
|
+
require 'cyclone_lariat/fake'
|
5
|
+
require 'cyclone_lariat/clients/abstract'
|
6
|
+
require 'cyclone_lariat/resources/topic'
|
7
|
+
require 'cyclone_lariat/resources/queue'
|
8
|
+
|
9
|
+
module CycloneLariat
|
10
|
+
module Clients
|
11
|
+
class Sns < Abstract
|
12
|
+
include LunaPark::Extensions::Injector
|
13
|
+
include Generators::Topic
|
14
|
+
|
15
|
+
dependency(:aws_client_class) { Aws::SNS::Client }
|
16
|
+
|
17
|
+
def publish(msg, fifo:, topic: nil, skip_validation: false)
|
18
|
+
return Fake.sns_publish_response(msg) if config.fake_publish
|
19
|
+
|
20
|
+
topic = topic ? custom_topic(topic) : topic(msg.type, kind: msg.kind, fifo: fifo)
|
21
|
+
|
22
|
+
raise Errors::GroupIdUndefined.new(resource: topic) if fifo && msg.group_id.nil?
|
23
|
+
raise Errors::GroupDefined.new(resource: topic) if !fifo && msg.group_id
|
24
|
+
raise Errors::DeduplicationIdDefined.new(resource: topic) if !fifo && msg.deduplication_id
|
25
|
+
|
26
|
+
msg.validation.check! unless skip_validation
|
27
|
+
|
28
|
+
params = {
|
29
|
+
topic_arn: topic.arn,
|
30
|
+
message: msg.to_json,
|
31
|
+
message_group_id: msg.group_id,
|
32
|
+
message_deduplication_id: msg.deduplication_id
|
33
|
+
}.compact
|
34
|
+
|
35
|
+
aws_client.publish(**params)
|
36
|
+
end
|
37
|
+
|
38
|
+
def exists?(topic)
|
39
|
+
raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
|
40
|
+
|
41
|
+
aws_client.get_topic_attributes({ topic_arn: topic.arn }) && true
|
42
|
+
rescue Aws::SNS::Errors::NotFound
|
43
|
+
false
|
44
|
+
end
|
45
|
+
|
46
|
+
def publish_event(type, fifo:, topic: nil, **options)
|
47
|
+
options[:version] ||= config.version
|
48
|
+
options[:data] ||= {}
|
49
|
+
options[:uuid] ||= SecureRandom.uuid
|
50
|
+
|
51
|
+
publish event(type, **options), fifo: fifo, topic: topic
|
52
|
+
end
|
53
|
+
|
54
|
+
def publish_command(type, fifo:, topic: nil, **options)
|
55
|
+
options[:version] ||= config.version
|
56
|
+
options[:data] ||= {}
|
57
|
+
options[:uuid] ||= SecureRandom.uuid
|
58
|
+
|
59
|
+
publish command(type, **options), fifo: fifo, topic: topic
|
60
|
+
end
|
61
|
+
|
62
|
+
def create(topic)
|
63
|
+
raise ArgumentError, 'Should be Resources::Topic' unless topic.is_a? Resources::Topic
|
64
|
+
raise Errors::TopicAlreadyExists.new(expected_topic: topic.name) if exists?(topic)
|
65
|
+
|
66
|
+
aws_client.create_topic(name: topic.name, attributes: topic.attributes, tags: topic.tags)
|
67
|
+
topic
|
68
|
+
end
|
69
|
+
|
70
|
+
def delete(topic)
|
71
|
+
raise ArgumentError, 'Should be Resources::Topic' unless topic.is_a? Resources::Topic
|
72
|
+
raise Errors::TopicDoesNotExists.new(expected_topic: topic.name) unless exists?(topic)
|
73
|
+
|
74
|
+
aws_client.delete_topic topic_arn: topic.arn
|
75
|
+
topic
|
76
|
+
end
|
77
|
+
|
78
|
+
def subscribe(topic:, endpoint:)
|
79
|
+
subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
|
80
|
+
raise Errors::SubscriptionAlreadyExists.new(topic: topic, endpoint: endpoint) if subscription_arn
|
81
|
+
|
82
|
+
aws_client.subscribe(
|
83
|
+
{
|
84
|
+
topic_arn: topic.arn,
|
85
|
+
protocol: endpoint.protocol,
|
86
|
+
endpoint: endpoint.arn
|
87
|
+
}
|
88
|
+
)
|
89
|
+
end
|
90
|
+
|
91
|
+
def unsubscribe(topic:, endpoint:)
|
92
|
+
subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
|
93
|
+
raise Errors::SubscriptionDoesNotExists.new(topic: topic, endpoint: endpoint) unless subscription_arn
|
94
|
+
|
95
|
+
aws_client.unsubscribe(subscription_arn: subscription_arn)
|
96
|
+
end
|
97
|
+
|
98
|
+
def list_all
|
99
|
+
topics = []
|
100
|
+
resp = aws_client.list_topics
|
101
|
+
|
102
|
+
loop do
|
103
|
+
resp[:topics].map do |t|
|
104
|
+
topics << Resources::Topic.from_arn(t[:topic_arn])
|
105
|
+
end
|
106
|
+
|
107
|
+
break if resp[:next_token].nil?
|
108
|
+
|
109
|
+
resp = aws_client.list_topics(next_token: resp[:next_token])
|
110
|
+
end
|
111
|
+
topics
|
112
|
+
end
|
113
|
+
|
114
|
+
def list_subscriptions
|
115
|
+
subscriptions = []
|
116
|
+
resp = aws_client.list_subscriptions
|
117
|
+
|
118
|
+
loop do
|
119
|
+
resp[:subscriptions].each do |s|
|
120
|
+
endpoint = s.endpoint.split(':')[2] == 'sqs' ? Resources::Queue.from_arn(s.endpoint) : Resources::Topic.from_arn(s.endpoint)
|
121
|
+
subscriptions << { topic: Resources::Topic.from_arn(s.topic_arn), endpoint: endpoint, arn: s.subscription_arn }
|
122
|
+
end
|
123
|
+
|
124
|
+
break if resp[:next_token].nil?
|
125
|
+
|
126
|
+
resp = aws_client.list_subscriptions(next_token: resp[:next_token])
|
127
|
+
end
|
128
|
+
subscriptions
|
129
|
+
end
|
130
|
+
|
131
|
+
def topic_subscriptions(topic)
|
132
|
+
raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
|
133
|
+
|
134
|
+
subscriptions = []
|
135
|
+
|
136
|
+
resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn)
|
137
|
+
|
138
|
+
loop do
|
139
|
+
next_token = resp[:next_token]
|
140
|
+
subscriptions += resp[:subscriptions]
|
141
|
+
|
142
|
+
break if next_token.nil?
|
143
|
+
|
144
|
+
resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn, next_token: next_token)
|
145
|
+
end
|
146
|
+
subscriptions
|
147
|
+
end
|
148
|
+
|
149
|
+
def find_subscription_arn(topic:, endpoint:)
|
150
|
+
raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
|
151
|
+
unless [Resources::Topic, Resources::Queue].include? endpoint.class
|
152
|
+
raise ArgumentError, 'Endpoint should be Topic or Queue'
|
153
|
+
end
|
154
|
+
|
155
|
+
found_subscription = topic_subscriptions(topic).select do |subscription|
|
156
|
+
subscription.endpoint == endpoint.arn
|
157
|
+
end.first
|
158
|
+
|
159
|
+
found_subscription ? found_subscription.subscription_arn : nil
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aws-sdk-sqs'
|
4
|
+
require 'cyclone_lariat/fake'
|
5
|
+
require 'cyclone_lariat/clients/abstract'
|
6
|
+
require 'cyclone_lariat/resources/queue'
|
7
|
+
require 'cyclone_lariat/generators/queue'
|
8
|
+
|
9
|
+
module CycloneLariat
|
10
|
+
module Clients
|
11
|
+
class Sqs < Abstract
|
12
|
+
include LunaPark::Extensions::Injector
|
13
|
+
include Generators::Queue
|
14
|
+
|
15
|
+
dependency(:aws_client_class) { Aws::SQS::Client }
|
16
|
+
|
17
|
+
def exists?(queue)
|
18
|
+
raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
|
19
|
+
|
20
|
+
aws_client.get_queue_url(queue_name: queue.to_s) && true
|
21
|
+
rescue Aws::SQS::Errors::NonExistentQueue
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def add_policy(queue:, policy:)
|
26
|
+
current_policy_json = aws_client.get_queue_attributes({
|
27
|
+
queue_url: queue.url,
|
28
|
+
attribute_names: ['Policy']
|
29
|
+
}).attributes['Policy']
|
30
|
+
|
31
|
+
current_policy = JSON.parse(current_policy_json) if current_policy_json
|
32
|
+
|
33
|
+
return if current_policy && current_policy['Statement'].find { |s| s['Sid'] == policy['Sid'] }
|
34
|
+
|
35
|
+
new_policy = current_policy || { 'Statement' => [] }
|
36
|
+
new_policy['Statement'] << policy
|
37
|
+
|
38
|
+
aws_client.set_queue_attributes({ queue_url: queue.url, attributes: { 'Policy' => new_policy.to_json } })
|
39
|
+
end
|
40
|
+
|
41
|
+
def publish(msg, fifo:, dest: nil, queue: nil, skip_validation: false)
|
42
|
+
return Fake.sqs_send_message_result(msg) if config.fake_publish
|
43
|
+
|
44
|
+
queue = queue ? custom_queue(queue) : queue(msg.type, kind: msg.kind, fifo: fifo, dest: dest)
|
45
|
+
|
46
|
+
raise Errors::GroupIdUndefined.new(resource: queue) if fifo && msg.group_id.nil?
|
47
|
+
raise Errors::GroupDefined.new(resource: queue) if !fifo && msg.group_id
|
48
|
+
raise Errors::DeduplicationIdDefined.new(resource: queue) if !fifo && msg.deduplication_id
|
49
|
+
|
50
|
+
msg.validation.check! unless skip_validation
|
51
|
+
|
52
|
+
params = {
|
53
|
+
queue_url: queue.url,
|
54
|
+
message_body: msg.to_json,
|
55
|
+
message_group_id: msg.group_id,
|
56
|
+
message_deduplication_id: msg.deduplication_id
|
57
|
+
}.compact
|
58
|
+
|
59
|
+
aws_client.send_message(**params)
|
60
|
+
end
|
61
|
+
|
62
|
+
def publish_event(type, fifo:, dest: nil, queue: nil, **options)
|
63
|
+
options[:version] ||= self.config.version
|
64
|
+
options[:data] ||= {}
|
65
|
+
options[:uuid] ||= SecureRandom.uuid
|
66
|
+
|
67
|
+
publish event(type, data: data, **options), fifo: fifo, dest: dest, queue: queue
|
68
|
+
end
|
69
|
+
|
70
|
+
def publish_command(type, fifo:, dest: nil, queue: nil, **options)
|
71
|
+
options[:version] ||= self.config.version
|
72
|
+
options[:data] ||= {}
|
73
|
+
options[:uuid] ||= SecureRandom.uuid
|
74
|
+
|
75
|
+
publish event(type, data: data, **options), fifo: fifo, dest: dest, queue: queue
|
76
|
+
end
|
77
|
+
|
78
|
+
def create(queue)
|
79
|
+
raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
|
80
|
+
raise Errors::QueueAlreadyExists.new(expected_queue: queue.name) if exists?(queue)
|
81
|
+
|
82
|
+
aws_client.create_queue(queue_name: queue.name, attributes: queue.attributes, tags: queue.tags)
|
83
|
+
queue
|
84
|
+
end
|
85
|
+
|
86
|
+
def delete(queue)
|
87
|
+
raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
|
88
|
+
raise Errors::QueueDoesNotExists.new(expected_queue: queue.name) unless exists?(queue)
|
89
|
+
|
90
|
+
aws_client.delete_queue queue_url: queue.url
|
91
|
+
queue
|
92
|
+
end
|
93
|
+
|
94
|
+
def list_all
|
95
|
+
queues = []
|
96
|
+
resp = aws_client.list_queues
|
97
|
+
|
98
|
+
loop do
|
99
|
+
next_token = resp[:next_token]
|
100
|
+
|
101
|
+
resp[:queue_urls].map do |url|
|
102
|
+
queues << Resources::Queue.from_url(url)
|
103
|
+
end
|
104
|
+
|
105
|
+
break if next_token.nil?
|
106
|
+
|
107
|
+
resp = aws_client.list_queues(next_token: next_token)
|
108
|
+
end
|
109
|
+
|
110
|
+
queues
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'cyclone_lariat/generators/queue'
|
4
|
+
require 'cyclone_lariat/generators/topic'
|
5
|
+
require 'cyclone_lariat/options'
|
6
|
+
|
7
|
+
module CycloneLariat
|
8
|
+
module CycloneLariatMethods
|
9
|
+
def config
|
10
|
+
@config ||= Options.new
|
11
|
+
end
|
12
|
+
|
13
|
+
def configure
|
14
|
+
yield(config)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
extend Generators::Topic
|
19
|
+
extend Generators::Queue
|
20
|
+
extend CycloneLariatMethods
|
21
|
+
end
|
@@ -40,5 +40,21 @@ module CycloneLariat
|
|
40
40
|
class SubscriptionDoesNotExists < LunaPark::Errors::System
|
41
41
|
message { |d| "Subscription for topic `#{d[:topic].name}`, on endpoint `#{d[:endpoint].name}` does not exists" }
|
42
42
|
end
|
43
|
+
|
44
|
+
class InvalidMessage < LunaPark::Errors::Business
|
45
|
+
message 'Message is not valid'
|
46
|
+
end
|
47
|
+
|
48
|
+
class GroupIdUndefined < LunaPark::Errors::System
|
49
|
+
message { |d| "Group id must be defined for FIFO resources: `#{d[:resource].name}`" }
|
50
|
+
end
|
51
|
+
|
52
|
+
class GroupDefined < LunaPark::Errors::System
|
53
|
+
message { |d| "Group id must be nil for non-FIFO resources: `#{d[:resource].name}`" }
|
54
|
+
end
|
55
|
+
|
56
|
+
class DeduplicationIdDefined < LunaPark::Errors::System
|
57
|
+
message { |d| "Deduplication id must be nil for non-FIFO resources: `#{d[:resource].name}`" }
|
58
|
+
end
|
43
59
|
end
|
44
60
|
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module CycloneLariat
|
4
|
+
class Fake
|
5
|
+
def self.sns_publish_response(message)
|
6
|
+
Aws::SNS::Types::PublishResponse.new.tap do |resp|
|
7
|
+
resp.message_id = SecureRandom.uuid
|
8
|
+
resp.sequence_number = rand(10).to_s if message.fifo?
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.sqs_send_message_result(message)
|
13
|
+
Aws::SQS::Types::SendMessageResult.new.tap do |res|
|
14
|
+
res.message_id = SecureRandom.uuid
|
15
|
+
res.sequence_number = rand(10).to_s if message.fifo?
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'securerandom'
|
4
|
+
require 'cyclone_lariat/messages/v1/command'
|
5
|
+
require 'cyclone_lariat/messages/v2/command'
|
6
|
+
|
7
|
+
module CycloneLariat
|
8
|
+
module Generators
|
9
|
+
module Command
|
10
|
+
def command(type, version: config.version, **options)
|
11
|
+
case version.to_i
|
12
|
+
when 1 then command_v1(type, **options)
|
13
|
+
when 2 then command_v2(type, **options)
|
14
|
+
else raise ArgumentError, "Unknown version #{version}"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def command_v1(type, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
|
19
|
+
params = {
|
20
|
+
uuid: uuid,
|
21
|
+
type: type,
|
22
|
+
sent_at: Time.now.iso8601(3),
|
23
|
+
version: 1,
|
24
|
+
publisher: config.publisher,
|
25
|
+
data: data,
|
26
|
+
request_id: request_id,
|
27
|
+
group_id: group_id,
|
28
|
+
deduplication_id: deduplication_id
|
29
|
+
}
|
30
|
+
|
31
|
+
Messages::V1::Command.wrap(params.compact)
|
32
|
+
end
|
33
|
+
|
34
|
+
def command_v2(type, subject:, object:, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
|
35
|
+
params = {
|
36
|
+
uuid: uuid,
|
37
|
+
type: type,
|
38
|
+
subject: subject,
|
39
|
+
object: object,
|
40
|
+
sent_at: Time.now.iso8601(3),
|
41
|
+
version: 2,
|
42
|
+
publisher: config.publisher,
|
43
|
+
data: data,
|
44
|
+
request_id: request_id,
|
45
|
+
group_id: group_id,
|
46
|
+
deduplication_id: deduplication_id
|
47
|
+
}
|
48
|
+
|
49
|
+
Messages::V2::Command.wrap(params.compact)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|