cyclone_lariat 0.4.0 → 1.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/gem-push.yml +4 -4
  3. data/.rubocop.yml +9 -5
  4. data/Gemfile.lock +123 -21
  5. data/Guardfile +42 -0
  6. data/README.md +417 -220
  7. data/bin/cyclone_lariat +75 -43
  8. data/cyclone_lariat.gemspec +10 -3
  9. data/lib/cyclone_lariat/clients/abstract.rb +40 -0
  10. data/lib/cyclone_lariat/clients/sns.rb +163 -0
  11. data/lib/cyclone_lariat/clients/sqs.rb +114 -0
  12. data/lib/cyclone_lariat/core.rb +21 -0
  13. data/lib/cyclone_lariat/errors.rb +16 -0
  14. data/lib/cyclone_lariat/fake.rb +19 -0
  15. data/lib/cyclone_lariat/generators/command.rb +53 -0
  16. data/lib/cyclone_lariat/generators/event.rb +52 -0
  17. data/lib/cyclone_lariat/generators/queue.rb +30 -0
  18. data/lib/cyclone_lariat/generators/topic.rb +29 -0
  19. data/lib/cyclone_lariat/messages/v1/abstract.rb +139 -0
  20. data/lib/cyclone_lariat/messages/v1/command.rb +20 -0
  21. data/lib/cyclone_lariat/messages/v1/event.rb +20 -0
  22. data/lib/cyclone_lariat/messages/v1/validator.rb +31 -0
  23. data/lib/cyclone_lariat/messages/v2/abstract.rb +149 -0
  24. data/lib/cyclone_lariat/messages/v2/command.rb +20 -0
  25. data/lib/cyclone_lariat/messages/v2/event.rb +20 -0
  26. data/lib/cyclone_lariat/messages/v2/validator.rb +39 -0
  27. data/lib/cyclone_lariat/middleware.rb +9 -6
  28. data/lib/cyclone_lariat/migration.rb +54 -117
  29. data/lib/cyclone_lariat/options.rb +52 -0
  30. data/lib/cyclone_lariat/presenters/graph.rb +54 -0
  31. data/lib/cyclone_lariat/presenters/queues.rb +41 -0
  32. data/lib/cyclone_lariat/presenters/subscriptions.rb +34 -0
  33. data/lib/cyclone_lariat/presenters/topics.rb +40 -0
  34. data/lib/cyclone_lariat/publisher.rb +25 -0
  35. data/lib/cyclone_lariat/repo/active_record/messages.rb +92 -0
  36. data/lib/cyclone_lariat/repo/active_record/versions.rb +28 -0
  37. data/lib/cyclone_lariat/repo/messages.rb +43 -0
  38. data/lib/cyclone_lariat/repo/messages_mapper.rb +49 -0
  39. data/lib/cyclone_lariat/repo/sequel/messages.rb +73 -0
  40. data/lib/cyclone_lariat/repo/sequel/versions.rb +28 -0
  41. data/lib/cyclone_lariat/repo/versions.rb +42 -0
  42. data/lib/cyclone_lariat/resources/queue.rb +167 -0
  43. data/lib/cyclone_lariat/resources/topic.rb +132 -0
  44. data/lib/cyclone_lariat/services/migrate.rb +51 -0
  45. data/lib/cyclone_lariat/services/rollback.rb +51 -0
  46. data/lib/cyclone_lariat/version.rb +1 -1
  47. data/lib/cyclone_lariat.rb +4 -11
  48. data/lib/tasks/console.rake +1 -1
  49. data/lib/tasks/cyclone_lariat.rake +10 -12
  50. data/lib/tasks/db.rake +0 -15
  51. metadata +127 -27
  52. data/config/db.example.rb +0 -9
  53. data/config/initializers/sequel.rb +0 -7
  54. data/db/migrate/01_add_uuid_extensions.rb +0 -15
  55. data/db/migrate/02_add_events.rb +0 -19
  56. data/db/migrate/03_add_versions.rb +0 -9
  57. data/docs/_imgs/graphviz_01.png +0 -0
  58. data/docs/_imgs/graphviz_02.png +0 -0
  59. data/docs/_imgs/graphviz_03.png +0 -0
  60. data/docs/_imgs/lariat.jpg +0 -0
  61. data/docs/_imgs/logic.png +0 -0
  62. data/docs/_imgs/sqs_sns_diagram.png +0 -0
  63. data/lib/cyclone_lariat/abstract/client.rb +0 -112
  64. data/lib/cyclone_lariat/abstract/message.rb +0 -98
  65. data/lib/cyclone_lariat/command.rb +0 -13
  66. data/lib/cyclone_lariat/configure.rb +0 -15
  67. data/lib/cyclone_lariat/event.rb +0 -13
  68. data/lib/cyclone_lariat/messages_mapper.rb +0 -46
  69. data/lib/cyclone_lariat/messages_repo.rb +0 -60
  70. data/lib/cyclone_lariat/queue.rb +0 -147
  71. data/lib/cyclone_lariat/sns_client.rb +0 -149
  72. data/lib/cyclone_lariat/sqs_client.rb +0 -93
  73. data/lib/cyclone_lariat/topic.rb +0 -113
@@ -1,147 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'uri'
4
-
5
- module CycloneLariat
6
- class Queue
7
- SNS_SUFFIX = :queue
8
-
9
- attr_reader :instance, :kind, :region, :dest, :account_id, :publisher, :type, :fifo, :tags
10
-
11
- def initialize(instance:, kind:, region:, dest:, account_id:, publisher:, type:, fifo:, tags: nil, name: nil)
12
- @instance = instance
13
- @kind = kind
14
- @region = region
15
- @dest = dest
16
- @account_id = account_id
17
- @publisher = publisher
18
- @type = type
19
- @fifo = fifo
20
- @tags = tags || default_tags(instance, kind, publisher, type, dest, fifo)
21
- @name = name
22
- end
23
-
24
- def arn
25
- ['arn', 'aws', 'sqs', region, account_id, name].join ':'
26
- end
27
-
28
- ##
29
- # Url example:
30
- # https://sqs.eu-west-1.amazonaws.com/247606935658/stage-event-queue
31
- def url
32
- "https://sqs.#{region}.amazonaws.com/#{account_id}/#{name}"
33
- end
34
-
35
- def custom?
36
- !standard?
37
- end
38
-
39
- def standard?
40
- instance && kind && publisher && type && true
41
- end
42
-
43
- def name
44
- @name ||= begin
45
- name = [instance, kind, SNS_SUFFIX, publisher, type, dest].compact.join '-'
46
- name += '.fifo' if fifo
47
- name
48
- end
49
- end
50
-
51
- alias to_s name
52
-
53
-
54
- def topic?
55
- false
56
- end
57
-
58
- def queue?
59
- true
60
- end
61
-
62
- def protocol
63
- 'sqs'
64
- end
65
-
66
- class << self
67
- ##
68
- # Name example: test-event-queue-cyclone_lariat-note_added.fifo
69
- # instance: teste
70
- # kind: event
71
- # publisher: cyclone_lariat
72
- # type: note_added
73
- # dest: nil
74
- # fifo: true
75
- def from_name(name, region:, account_id:)
76
- is_fifo_array = name.split('.')
77
- full_name = is_fifo_array[0]
78
- fifo_suffix = is_fifo_array[-1]
79
- suffix_exists = fifo_suffix != full_name
80
-
81
- if suffix_exists && fifo_suffix != 'fifo'
82
- raise ArgumentError, "Queue name #{name} consists unexpected suffix #{fifo_suffix}"
83
- end
84
-
85
- fifo = suffix_exists
86
- queue_array = full_name.split('-')
87
-
88
- raise ArgumentError, "Topic name should consists `#{SNS_SUFFIX}`" unless queue_array[2] != SNS_SUFFIX
89
-
90
- new(
91
- instance: queue_array[0], kind: queue_array[1], region: region, dest: queue_array[5],
92
- account_id: account_id, publisher: queue_array[3], type: queue_array[4], fifo: fifo, name: name
93
- )
94
- end
95
-
96
- ##
97
- # URL example: https://sqs.eu-west-1.amazonaws.com/247606935658/test-event-queue-cyclone_lariat-note_added.fifo
98
- # url_array[0] => https
99
- # host_array[0] => sqs
100
- # host_array[1] => eu-west-1
101
- # url_array[3] => 247606935658 # account_id
102
- # url_array[4] => test-event-queue-cyclone_lariat-note_added.fifo # name
103
- def from_url(url)
104
- raise ArgumentError, 'Url is not http format' unless url =~ URI::DEFAULT_PARSER.make_regexp
105
-
106
- url_array = url.split('/')
107
- raise ArgumentError, 'Url should start from https' unless url_array[0] == 'https:'
108
-
109
- host_array = url_array[2].split('.')
110
- raise ArgumentError, 'It is not queue url' unless host_array[0] == 'sqs'
111
-
112
- from_name(url_array[4], region: host_array[1], account_id: url_array[3])
113
- end
114
-
115
- ##
116
- # Arn example: "arn:aws:sqs:eu-west-1:247606935658:custom_queue"
117
- # arn_array[0] => 'arn'
118
- # arn_array[1] => 'aws'
119
- # arn_array[2] => 'sqs'
120
- # arn_array[3] => 'eu-west-1' # region
121
- # arn_array[4] => '247606935658' # account_id
122
- # arn_array[5] => 'alexey_test2' # name
123
- def from_arn(arn)
124
- arn_array = arn.split(':')
125
-
126
- raise ArgumentError, "Arn `#{arn}` should consists `arn`" unless arn_array[0] == 'arn'
127
- raise ArgumentError, "Arn `#{arn}` should consists `aws`" unless arn_array[1] == 'aws'
128
- raise ArgumentError, "Arn `#{arn}` should consists `sqs`" unless arn_array[2] == 'sqs'
129
-
130
- from_name(arn_array[5], region: arn_array[3], account_id: arn_array[4])
131
- end
132
- end
133
-
134
- private
135
-
136
- def default_tags(instance, kind, publisher, type, dest, fifo)
137
- {
138
- instance: String(instance),
139
- kind: String(kind),
140
- publisher: String(publisher),
141
- type: String(type),
142
- dest: dest ? String(dest) : 'undefined',
143
- fifo: fifo ? 'true' : 'false'
144
- }
145
- end
146
- end
147
- end
@@ -1,149 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'aws-sdk-sns'
4
- require_relative 'abstract/client'
5
- require_relative 'topic'
6
- require_relative 'queue'
7
-
8
- module CycloneLariat
9
- class SnsClient < Abstract::Client
10
- include LunaPark::Extensions::Injector
11
-
12
- dependency(:aws_client_class) { Aws::SNS::Client }
13
-
14
- def custom_topic(name)
15
- Topic.from_name(name, account_id: account_id, region: region)
16
- end
17
-
18
- def topic(type, fifo:, publisher: nil, kind: :event)
19
- publisher ||= self.publisher
20
-
21
- Topic.new(
22
- instance: instance,
23
- publisher: publisher,
24
- region: region,
25
- account_id: account_id,
26
- kind: kind,
27
- type: type, fifo: fifo
28
- )
29
- end
30
-
31
- def publish(msg, fifo:, topic: nil)
32
- topic = topic ? custom_topic(topic) : topic(msg.type, kind: msg.kind, fifo: fifo)
33
- aws_client.publish(topic_arn: topic.arn, message: msg.to_json)
34
- end
35
-
36
- def exists?(topic)
37
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
38
-
39
- aws_client.get_topic_attributes({ topic_arn: topic.arn }) && true
40
- rescue Aws::SNS::Errors::NotFound
41
- false
42
- end
43
-
44
- def publish_event(type, fifo:, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, topic: nil)
45
- publish event(type, data: data, version: version, uuid: uuid, request_id: request_id), topic: topic, fifo: fifo
46
- end
47
-
48
- def publish_command(type, fifo:, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, topic: nil)
49
- publish command(type, data: data, version: version, uuid: uuid, request_id: request_id), topic: topic, fifo: fifo
50
- end
51
-
52
- def create(topic)
53
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
54
- raise Errors::TopicAlreadyExists.new(expected_topic: topic.name) if exists?(topic)
55
-
56
- aws_client.create_topic(name: topic.name, attributes: topic.attributes, tags: topic.tags)
57
- topic
58
- end
59
-
60
- def delete(topic)
61
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
62
- raise Errors::TopicDoesNotExists.new(expected_topic: topic.name) unless exists?(topic)
63
-
64
- aws_client.delete_topic topic_arn: topic.arn
65
- topic
66
- end
67
-
68
- def subscribe(topic:, endpoint:)
69
- subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
70
- raise Errors::SubscriptionAlreadyExists.new(topic: topic, endpoint: endpoint) if subscription_arn
71
-
72
- aws_client.subscribe(
73
- {
74
- topic_arn: topic.arn,
75
- protocol: endpoint.protocol,
76
- endpoint: endpoint.arn
77
- }
78
- )
79
- end
80
-
81
- def unsubscribe(topic:, endpoint:)
82
- subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
83
- raise Errors::SubscriptionDoesNotExists.new(topic: topic, endpoint: endpoint) unless subscription_arn
84
-
85
- aws_client.unsubscribe(subscription_arn: subscription_arn)
86
- end
87
-
88
- def list_all
89
- topics = []
90
- resp = aws_client.list_topics
91
-
92
- loop do
93
- resp[:topics].map do |t|
94
- topics << Topic.from_arn(t[:topic_arn])
95
- end
96
-
97
- break if resp[:next_token].nil?
98
-
99
- resp = aws_client.list_topics(next_token: resp[:next_token])
100
- end
101
- topics
102
- end
103
-
104
- def list_subscriptions
105
- subscriptions = []
106
- resp = aws_client.list_subscriptions
107
-
108
- loop do
109
- resp[:subscriptions].each do |s|
110
- endpoint = s.endpoint.split(':')[2] == 'sqs' ? Queue.from_arn(s.endpoint) : Topic.from_arn(s.endpoint)
111
- subscriptions << { topic: Topic.from_arn(s.topic_arn), endpoint: endpoint, arn: s.subscription_arn }
112
- end
113
-
114
- break if resp[:next_token].nil?
115
-
116
- resp = aws_client.list_subscriptions(next_token: resp[:next_token])
117
- end
118
- subscriptions
119
- end
120
-
121
- def topic_subscriptions(topic)
122
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
123
-
124
- subscriptions = []
125
- resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn)
126
-
127
- loop do
128
- next_token = resp[:next_token]
129
- subscriptions += resp[:subscriptions]
130
-
131
- break if next_token.nil?
132
-
133
- resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn, next_token: next_token)
134
- end
135
- subscriptions
136
- end
137
-
138
- def find_subscription_arn(topic:, endpoint:)
139
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
140
- raise ArgumentError, 'Endpoint should be Topic or Queue' unless [Topic, Queue].include? endpoint.class
141
-
142
- found_subscription = topic_subscriptions(topic).select do |subscription|
143
- subscription.endpoint == endpoint.arn
144
- end.first
145
-
146
- found_subscription ? found_subscription.subscription_arn : nil
147
- end
148
- end
149
- end
@@ -1,93 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'aws-sdk-sqs'
4
- require_relative 'abstract/client'
5
- require_relative 'queue'
6
-
7
- module CycloneLariat
8
- class SqsClient < Abstract::Client
9
- include LunaPark::Extensions::Injector
10
-
11
- dependency(:aws_client_class) { Aws::SQS::Client }
12
-
13
- def custom_queue(name)
14
- Queue.from_name(name, account_id: account_id, region: region)
15
- end
16
-
17
- def queue(type = :all, fifo:, dest: nil, publisher: nil, kind: :event)
18
- publisher ||= self.publisher
19
-
20
- Queue.new(
21
- instance: instance, publisher: publisher, region: region,
22
- account_id: account_id, kind: kind, type: type, fifo: fifo, dest: dest
23
- )
24
- end
25
-
26
- def get_url(queue)
27
- raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
28
-
29
- aws_client.get_queue_url(queue_name: queue.to_s).queue_url
30
- end
31
-
32
- def exists?(queue)
33
- raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
34
-
35
- get_url(queue) && true
36
- rescue Aws::SQS::Errors::NonExistentQueue
37
- false
38
- end
39
-
40
- def publish(msg, fifo:, dest: nil, queue: nil)
41
- queue = queue ? custom_queue(queue) : queue(msg.type, kind: msg.kind, fifo: fifo, dest: dest)
42
- aws_client.send_message(queue_url: get_url(queue), message_body: msg.to_json)
43
- end
44
-
45
- def publish_event(type, fifo:, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, queue: nil)
46
- publish event(type, data: data, version: version, uuid: uuid, request_id: request_id),
47
- fifo: fifo, dest: dest, queue: queue
48
- end
49
-
50
- def publish_command(type, fifo:, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, queue: nil)
51
- publish command(type, data: data, version: version, uuid: uuid, request_id: request_id),
52
- fifo: fifo, dest: dest, queue: queue
53
- end
54
-
55
- def create(queue)
56
- raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
57
- raise Errors::QueueAlreadyExists.new(expected_queue: queue.name) if exists?(queue)
58
-
59
- attrs = {}
60
- attrs['FifoQueue'] = 'true' if queue.fifo
61
-
62
- aws_client.create_queue(queue_name: queue.name, attributes: attrs, tags: queue.tags)
63
- queue
64
- end
65
-
66
- def delete(queue)
67
- raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
68
- raise Errors::QueueDoesNotExists.new(expected_queue: queue.name) unless exists?(queue)
69
-
70
- aws_client.delete_queue queue_url: queue.url
71
- queue
72
- end
73
-
74
- def list_all
75
- queues = []
76
- resp = aws_client.list_queues
77
-
78
- loop do
79
- next_token = resp[:next_token]
80
-
81
- resp[:queue_urls].map do |url|
82
- queues << Queue.from_url(url)
83
- end
84
-
85
- break if next_token.nil?
86
-
87
- resp = aws_client.list_queues(next_token: next_token)
88
- end
89
-
90
- queues
91
- end
92
- end
93
- end
@@ -1,113 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module CycloneLariat
4
- class Topic
5
- SNS_SUFFIX = :fanout
6
-
7
- attr_reader :instance, :kind, :region, :account_id, :publisher, :type, :fifo, :tags
8
-
9
- def initialize(instance:, kind:, region:, account_id:, publisher:, type:, fifo:, tags: nil, name: nil)
10
- @instance = instance
11
- @kind = kind
12
- @region = region
13
- @account_id = account_id
14
- @publisher = publisher
15
- @type = type
16
- @fifo = fifo
17
- @tags = tags || default_tags(instance, kind, publisher, type, fifo)
18
- @name = name
19
- end
20
-
21
- def arn
22
- ['arn', 'aws', 'sns', region, account_id, to_s].join ':'
23
- end
24
-
25
- def custom?
26
- !standard?
27
- end
28
-
29
- def standard?
30
- instance && kind && publisher && type && true
31
- end
32
-
33
- def name
34
- @name ||= begin
35
- name = [instance, kind, SNS_SUFFIX, publisher, type].join '-'
36
- name += '.fifo' if fifo
37
- name
38
- end
39
- end
40
-
41
- def attributes
42
- fifo ? { 'FifoTopic' => 'true' } : {}
43
- end
44
-
45
- def topic?
46
- true
47
- end
48
-
49
- def queue?
50
- false
51
- end
52
-
53
- def protocol
54
- 'sns'
55
- end
56
-
57
- alias to_s name
58
-
59
- def ==(other)
60
- arn == other.arn
61
- end
62
-
63
- class << self
64
- def from_name(name, region:, account_id:)
65
- is_fifo_array = name.split('.')
66
- full_name = is_fifo_array[0]
67
- fifo_suffix = is_fifo_array[-1]
68
- suffix_exists = fifo_suffix != full_name
69
-
70
- if suffix_exists && fifo_suffix != 'fifo'
71
- raise ArgumentError, "Topic name #{name} consists unexpected suffix #{fifo_suffix}"
72
- end
73
-
74
- fifo = suffix_exists
75
- topic_array = full_name.split('-')
76
-
77
- raise ArgumentError, "Topic name should consists `#{SNS_SUFFIX}`" unless topic_array[2] != SNS_SUFFIX
78
-
79
- new(
80
- instance: topic_array[0],
81
- kind: topic_array[1],
82
- publisher: topic_array[3],
83
- type: topic_array[4],
84
- region: region,
85
- account_id: account_id,
86
- fifo: fifo,
87
- name: name
88
- )
89
- end
90
-
91
- def from_arn(arn)
92
- arn_array = arn.split(':')
93
- raise ArgumentError, 'Arn should consists `arn`' unless arn_array[0] == 'arn'
94
- raise ArgumentError, 'Arn should consists `aws`' unless arn_array[1] == 'aws'
95
- raise ArgumentError, 'Arn should consists `sns`' unless arn_array[2] == 'sns'
96
-
97
- from_name(arn_array[5], region: arn_array[3], account_id: arn_array[4])
98
- end
99
- end
100
-
101
- private
102
-
103
- def default_tags(instance, kind, publisher, type, fifo)
104
- [
105
- { key: 'instance', value: String(instance) },
106
- { key: 'kind', value: String(kind) },
107
- { key: 'publisher', value: String(publisher) },
108
- { key: 'type', value: String(type) },
109
- { key: 'fifo', value: fifo ? 'true' : 'false' }
110
- ]
111
- end
112
- end
113
- end