cyclone_lariat 0.4.0 → 1.0.0.rc2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (74) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/gem-push.yml +4 -4
  3. data/.rubocop.yml +9 -5
  4. data/CHANGELOG.md +7 -1
  5. data/Gemfile.lock +123 -21
  6. data/Guardfile +42 -0
  7. data/README.md +420 -223
  8. data/bin/cyclone_lariat +75 -43
  9. data/cyclone_lariat.gemspec +10 -3
  10. data/lib/cyclone_lariat/clients/abstract.rb +40 -0
  11. data/lib/cyclone_lariat/clients/sns.rb +163 -0
  12. data/lib/cyclone_lariat/clients/sqs.rb +114 -0
  13. data/lib/cyclone_lariat/core.rb +21 -0
  14. data/lib/cyclone_lariat/errors.rb +16 -0
  15. data/lib/cyclone_lariat/fake.rb +19 -0
  16. data/lib/cyclone_lariat/generators/command.rb +53 -0
  17. data/lib/cyclone_lariat/generators/event.rb +52 -0
  18. data/lib/cyclone_lariat/generators/queue.rb +30 -0
  19. data/lib/cyclone_lariat/generators/topic.rb +29 -0
  20. data/lib/cyclone_lariat/messages/v1/abstract.rb +139 -0
  21. data/lib/cyclone_lariat/messages/v1/command.rb +20 -0
  22. data/lib/cyclone_lariat/messages/v1/event.rb +20 -0
  23. data/lib/cyclone_lariat/messages/v1/validator.rb +31 -0
  24. data/lib/cyclone_lariat/messages/v2/abstract.rb +149 -0
  25. data/lib/cyclone_lariat/messages/v2/command.rb +20 -0
  26. data/lib/cyclone_lariat/messages/v2/event.rb +20 -0
  27. data/lib/cyclone_lariat/messages/v2/validator.rb +39 -0
  28. data/lib/cyclone_lariat/middleware.rb +9 -6
  29. data/lib/cyclone_lariat/migration.rb +54 -117
  30. data/lib/cyclone_lariat/options.rb +52 -0
  31. data/lib/cyclone_lariat/presenters/graph.rb +54 -0
  32. data/lib/cyclone_lariat/presenters/queues.rb +41 -0
  33. data/lib/cyclone_lariat/presenters/subscriptions.rb +34 -0
  34. data/lib/cyclone_lariat/presenters/topics.rb +40 -0
  35. data/lib/cyclone_lariat/publisher.rb +25 -0
  36. data/lib/cyclone_lariat/repo/active_record/messages.rb +92 -0
  37. data/lib/cyclone_lariat/repo/active_record/versions.rb +28 -0
  38. data/lib/cyclone_lariat/repo/messages.rb +43 -0
  39. data/lib/cyclone_lariat/repo/messages_mapper.rb +49 -0
  40. data/lib/cyclone_lariat/repo/sequel/messages.rb +73 -0
  41. data/lib/cyclone_lariat/repo/sequel/versions.rb +28 -0
  42. data/lib/cyclone_lariat/repo/versions.rb +42 -0
  43. data/lib/cyclone_lariat/resources/queue.rb +167 -0
  44. data/lib/cyclone_lariat/resources/topic.rb +132 -0
  45. data/lib/cyclone_lariat/services/migrate.rb +51 -0
  46. data/lib/cyclone_lariat/services/rollback.rb +51 -0
  47. data/lib/cyclone_lariat/version.rb +1 -1
  48. data/lib/cyclone_lariat.rb +5 -11
  49. data/lib/tasks/console.rake +1 -1
  50. data/lib/tasks/cyclone_lariat.rake +10 -12
  51. data/lib/tasks/db.rake +0 -15
  52. metadata +127 -27
  53. data/config/db.example.rb +0 -9
  54. data/config/initializers/sequel.rb +0 -7
  55. data/db/migrate/01_add_uuid_extensions.rb +0 -15
  56. data/db/migrate/02_add_events.rb +0 -19
  57. data/db/migrate/03_add_versions.rb +0 -9
  58. data/docs/_imgs/graphviz_01.png +0 -0
  59. data/docs/_imgs/graphviz_02.png +0 -0
  60. data/docs/_imgs/graphviz_03.png +0 -0
  61. data/docs/_imgs/lariat.jpg +0 -0
  62. data/docs/_imgs/logic.png +0 -0
  63. data/docs/_imgs/sqs_sns_diagram.png +0 -0
  64. data/lib/cyclone_lariat/abstract/client.rb +0 -112
  65. data/lib/cyclone_lariat/abstract/message.rb +0 -98
  66. data/lib/cyclone_lariat/command.rb +0 -13
  67. data/lib/cyclone_lariat/configure.rb +0 -15
  68. data/lib/cyclone_lariat/event.rb +0 -13
  69. data/lib/cyclone_lariat/messages_mapper.rb +0 -46
  70. data/lib/cyclone_lariat/messages_repo.rb +0 -60
  71. data/lib/cyclone_lariat/queue.rb +0 -147
  72. data/lib/cyclone_lariat/sns_client.rb +0 -149
  73. data/lib/cyclone_lariat/sqs_client.rb +0 -93
  74. data/lib/cyclone_lariat/topic.rb +0 -113
@@ -1,147 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'uri'
4
-
5
- module CycloneLariat
6
- class Queue
7
- SNS_SUFFIX = :queue
8
-
9
- attr_reader :instance, :kind, :region, :dest, :account_id, :publisher, :type, :fifo, :tags
10
-
11
- def initialize(instance:, kind:, region:, dest:, account_id:, publisher:, type:, fifo:, tags: nil, name: nil)
12
- @instance = instance
13
- @kind = kind
14
- @region = region
15
- @dest = dest
16
- @account_id = account_id
17
- @publisher = publisher
18
- @type = type
19
- @fifo = fifo
20
- @tags = tags || default_tags(instance, kind, publisher, type, dest, fifo)
21
- @name = name
22
- end
23
-
24
- def arn
25
- ['arn', 'aws', 'sqs', region, account_id, name].join ':'
26
- end
27
-
28
- ##
29
- # Url example:
30
- # https://sqs.eu-west-1.amazonaws.com/247606935658/stage-event-queue
31
- def url
32
- "https://sqs.#{region}.amazonaws.com/#{account_id}/#{name}"
33
- end
34
-
35
- def custom?
36
- !standard?
37
- end
38
-
39
- def standard?
40
- instance && kind && publisher && type && true
41
- end
42
-
43
- def name
44
- @name ||= begin
45
- name = [instance, kind, SNS_SUFFIX, publisher, type, dest].compact.join '-'
46
- name += '.fifo' if fifo
47
- name
48
- end
49
- end
50
-
51
- alias to_s name
52
-
53
-
54
- def topic?
55
- false
56
- end
57
-
58
- def queue?
59
- true
60
- end
61
-
62
- def protocol
63
- 'sqs'
64
- end
65
-
66
- class << self
67
- ##
68
- # Name example: test-event-queue-cyclone_lariat-note_added.fifo
69
- # instance: teste
70
- # kind: event
71
- # publisher: cyclone_lariat
72
- # type: note_added
73
- # dest: nil
74
- # fifo: true
75
- def from_name(name, region:, account_id:)
76
- is_fifo_array = name.split('.')
77
- full_name = is_fifo_array[0]
78
- fifo_suffix = is_fifo_array[-1]
79
- suffix_exists = fifo_suffix != full_name
80
-
81
- if suffix_exists && fifo_suffix != 'fifo'
82
- raise ArgumentError, "Queue name #{name} consists unexpected suffix #{fifo_suffix}"
83
- end
84
-
85
- fifo = suffix_exists
86
- queue_array = full_name.split('-')
87
-
88
- raise ArgumentError, "Topic name should consists `#{SNS_SUFFIX}`" unless queue_array[2] != SNS_SUFFIX
89
-
90
- new(
91
- instance: queue_array[0], kind: queue_array[1], region: region, dest: queue_array[5],
92
- account_id: account_id, publisher: queue_array[3], type: queue_array[4], fifo: fifo, name: name
93
- )
94
- end
95
-
96
- ##
97
- # URL example: https://sqs.eu-west-1.amazonaws.com/247606935658/test-event-queue-cyclone_lariat-note_added.fifo
98
- # url_array[0] => https
99
- # host_array[0] => sqs
100
- # host_array[1] => eu-west-1
101
- # url_array[3] => 247606935658 # account_id
102
- # url_array[4] => test-event-queue-cyclone_lariat-note_added.fifo # name
103
- def from_url(url)
104
- raise ArgumentError, 'Url is not http format' unless url =~ URI::DEFAULT_PARSER.make_regexp
105
-
106
- url_array = url.split('/')
107
- raise ArgumentError, 'Url should start from https' unless url_array[0] == 'https:'
108
-
109
- host_array = url_array[2].split('.')
110
- raise ArgumentError, 'It is not queue url' unless host_array[0] == 'sqs'
111
-
112
- from_name(url_array[4], region: host_array[1], account_id: url_array[3])
113
- end
114
-
115
- ##
116
- # Arn example: "arn:aws:sqs:eu-west-1:247606935658:custom_queue"
117
- # arn_array[0] => 'arn'
118
- # arn_array[1] => 'aws'
119
- # arn_array[2] => 'sqs'
120
- # arn_array[3] => 'eu-west-1' # region
121
- # arn_array[4] => '247606935658' # account_id
122
- # arn_array[5] => 'alexey_test2' # name
123
- def from_arn(arn)
124
- arn_array = arn.split(':')
125
-
126
- raise ArgumentError, "Arn `#{arn}` should consists `arn`" unless arn_array[0] == 'arn'
127
- raise ArgumentError, "Arn `#{arn}` should consists `aws`" unless arn_array[1] == 'aws'
128
- raise ArgumentError, "Arn `#{arn}` should consists `sqs`" unless arn_array[2] == 'sqs'
129
-
130
- from_name(arn_array[5], region: arn_array[3], account_id: arn_array[4])
131
- end
132
- end
133
-
134
- private
135
-
136
- def default_tags(instance, kind, publisher, type, dest, fifo)
137
- {
138
- instance: String(instance),
139
- kind: String(kind),
140
- publisher: String(publisher),
141
- type: String(type),
142
- dest: dest ? String(dest) : 'undefined',
143
- fifo: fifo ? 'true' : 'false'
144
- }
145
- end
146
- end
147
- end
@@ -1,149 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'aws-sdk-sns'
4
- require_relative 'abstract/client'
5
- require_relative 'topic'
6
- require_relative 'queue'
7
-
8
- module CycloneLariat
9
- class SnsClient < Abstract::Client
10
- include LunaPark::Extensions::Injector
11
-
12
- dependency(:aws_client_class) { Aws::SNS::Client }
13
-
14
- def custom_topic(name)
15
- Topic.from_name(name, account_id: account_id, region: region)
16
- end
17
-
18
- def topic(type, fifo:, publisher: nil, kind: :event)
19
- publisher ||= self.publisher
20
-
21
- Topic.new(
22
- instance: instance,
23
- publisher: publisher,
24
- region: region,
25
- account_id: account_id,
26
- kind: kind,
27
- type: type, fifo: fifo
28
- )
29
- end
30
-
31
- def publish(msg, fifo:, topic: nil)
32
- topic = topic ? custom_topic(topic) : topic(msg.type, kind: msg.kind, fifo: fifo)
33
- aws_client.publish(topic_arn: topic.arn, message: msg.to_json)
34
- end
35
-
36
- def exists?(topic)
37
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
38
-
39
- aws_client.get_topic_attributes({ topic_arn: topic.arn }) && true
40
- rescue Aws::SNS::Errors::NotFound
41
- false
42
- end
43
-
44
- def publish_event(type, fifo:, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, topic: nil)
45
- publish event(type, data: data, version: version, uuid: uuid, request_id: request_id), topic: topic, fifo: fifo
46
- end
47
-
48
- def publish_command(type, fifo:, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, topic: nil)
49
- publish command(type, data: data, version: version, uuid: uuid, request_id: request_id), topic: topic, fifo: fifo
50
- end
51
-
52
- def create(topic)
53
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
54
- raise Errors::TopicAlreadyExists.new(expected_topic: topic.name) if exists?(topic)
55
-
56
- aws_client.create_topic(name: topic.name, attributes: topic.attributes, tags: topic.tags)
57
- topic
58
- end
59
-
60
- def delete(topic)
61
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
62
- raise Errors::TopicDoesNotExists.new(expected_topic: topic.name) unless exists?(topic)
63
-
64
- aws_client.delete_topic topic_arn: topic.arn
65
- topic
66
- end
67
-
68
- def subscribe(topic:, endpoint:)
69
- subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
70
- raise Errors::SubscriptionAlreadyExists.new(topic: topic, endpoint: endpoint) if subscription_arn
71
-
72
- aws_client.subscribe(
73
- {
74
- topic_arn: topic.arn,
75
- protocol: endpoint.protocol,
76
- endpoint: endpoint.arn
77
- }
78
- )
79
- end
80
-
81
- def unsubscribe(topic:, endpoint:)
82
- subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
83
- raise Errors::SubscriptionDoesNotExists.new(topic: topic, endpoint: endpoint) unless subscription_arn
84
-
85
- aws_client.unsubscribe(subscription_arn: subscription_arn)
86
- end
87
-
88
- def list_all
89
- topics = []
90
- resp = aws_client.list_topics
91
-
92
- loop do
93
- resp[:topics].map do |t|
94
- topics << Topic.from_arn(t[:topic_arn])
95
- end
96
-
97
- break if resp[:next_token].nil?
98
-
99
- resp = aws_client.list_topics(next_token: resp[:next_token])
100
- end
101
- topics
102
- end
103
-
104
- def list_subscriptions
105
- subscriptions = []
106
- resp = aws_client.list_subscriptions
107
-
108
- loop do
109
- resp[:subscriptions].each do |s|
110
- endpoint = s.endpoint.split(':')[2] == 'sqs' ? Queue.from_arn(s.endpoint) : Topic.from_arn(s.endpoint)
111
- subscriptions << { topic: Topic.from_arn(s.topic_arn), endpoint: endpoint, arn: s.subscription_arn }
112
- end
113
-
114
- break if resp[:next_token].nil?
115
-
116
- resp = aws_client.list_subscriptions(next_token: resp[:next_token])
117
- end
118
- subscriptions
119
- end
120
-
121
- def topic_subscriptions(topic)
122
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
123
-
124
- subscriptions = []
125
- resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn)
126
-
127
- loop do
128
- next_token = resp[:next_token]
129
- subscriptions += resp[:subscriptions]
130
-
131
- break if next_token.nil?
132
-
133
- resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn, next_token: next_token)
134
- end
135
- subscriptions
136
- end
137
-
138
- def find_subscription_arn(topic:, endpoint:)
139
- raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
140
- raise ArgumentError, 'Endpoint should be Topic or Queue' unless [Topic, Queue].include? endpoint.class
141
-
142
- found_subscription = topic_subscriptions(topic).select do |subscription|
143
- subscription.endpoint == endpoint.arn
144
- end.first
145
-
146
- found_subscription ? found_subscription.subscription_arn : nil
147
- end
148
- end
149
- end
@@ -1,93 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'aws-sdk-sqs'
4
- require_relative 'abstract/client'
5
- require_relative 'queue'
6
-
7
- module CycloneLariat
8
- class SqsClient < Abstract::Client
9
- include LunaPark::Extensions::Injector
10
-
11
- dependency(:aws_client_class) { Aws::SQS::Client }
12
-
13
- def custom_queue(name)
14
- Queue.from_name(name, account_id: account_id, region: region)
15
- end
16
-
17
- def queue(type = :all, fifo:, dest: nil, publisher: nil, kind: :event)
18
- publisher ||= self.publisher
19
-
20
- Queue.new(
21
- instance: instance, publisher: publisher, region: region,
22
- account_id: account_id, kind: kind, type: type, fifo: fifo, dest: dest
23
- )
24
- end
25
-
26
- def get_url(queue)
27
- raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
28
-
29
- aws_client.get_queue_url(queue_name: queue.to_s).queue_url
30
- end
31
-
32
- def exists?(queue)
33
- raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
34
-
35
- get_url(queue) && true
36
- rescue Aws::SQS::Errors::NonExistentQueue
37
- false
38
- end
39
-
40
- def publish(msg, fifo:, dest: nil, queue: nil)
41
- queue = queue ? custom_queue(queue) : queue(msg.type, kind: msg.kind, fifo: fifo, dest: dest)
42
- aws_client.send_message(queue_url: get_url(queue), message_body: msg.to_json)
43
- end
44
-
45
- def publish_event(type, fifo:, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, queue: nil)
46
- publish event(type, data: data, version: version, uuid: uuid, request_id: request_id),
47
- fifo: fifo, dest: dest, queue: queue
48
- end
49
-
50
- def publish_command(type, fifo:, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, queue: nil)
51
- publish command(type, data: data, version: version, uuid: uuid, request_id: request_id),
52
- fifo: fifo, dest: dest, queue: queue
53
- end
54
-
55
- def create(queue)
56
- raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
57
- raise Errors::QueueAlreadyExists.new(expected_queue: queue.name) if exists?(queue)
58
-
59
- attrs = {}
60
- attrs['FifoQueue'] = 'true' if queue.fifo
61
-
62
- aws_client.create_queue(queue_name: queue.name, attributes: attrs, tags: queue.tags)
63
- queue
64
- end
65
-
66
- def delete(queue)
67
- raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
68
- raise Errors::QueueDoesNotExists.new(expected_queue: queue.name) unless exists?(queue)
69
-
70
- aws_client.delete_queue queue_url: queue.url
71
- queue
72
- end
73
-
74
- def list_all
75
- queues = []
76
- resp = aws_client.list_queues
77
-
78
- loop do
79
- next_token = resp[:next_token]
80
-
81
- resp[:queue_urls].map do |url|
82
- queues << Queue.from_url(url)
83
- end
84
-
85
- break if next_token.nil?
86
-
87
- resp = aws_client.list_queues(next_token: next_token)
88
- end
89
-
90
- queues
91
- end
92
- end
93
- end
@@ -1,113 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module CycloneLariat
4
- class Topic
5
- SNS_SUFFIX = :fanout
6
-
7
- attr_reader :instance, :kind, :region, :account_id, :publisher, :type, :fifo, :tags
8
-
9
- def initialize(instance:, kind:, region:, account_id:, publisher:, type:, fifo:, tags: nil, name: nil)
10
- @instance = instance
11
- @kind = kind
12
- @region = region
13
- @account_id = account_id
14
- @publisher = publisher
15
- @type = type
16
- @fifo = fifo
17
- @tags = tags || default_tags(instance, kind, publisher, type, fifo)
18
- @name = name
19
- end
20
-
21
- def arn
22
- ['arn', 'aws', 'sns', region, account_id, to_s].join ':'
23
- end
24
-
25
- def custom?
26
- !standard?
27
- end
28
-
29
- def standard?
30
- instance && kind && publisher && type && true
31
- end
32
-
33
- def name
34
- @name ||= begin
35
- name = [instance, kind, SNS_SUFFIX, publisher, type].join '-'
36
- name += '.fifo' if fifo
37
- name
38
- end
39
- end
40
-
41
- def attributes
42
- fifo ? { 'FifoTopic' => 'true' } : {}
43
- end
44
-
45
- def topic?
46
- true
47
- end
48
-
49
- def queue?
50
- false
51
- end
52
-
53
- def protocol
54
- 'sns'
55
- end
56
-
57
- alias to_s name
58
-
59
- def ==(other)
60
- arn == other.arn
61
- end
62
-
63
- class << self
64
- def from_name(name, region:, account_id:)
65
- is_fifo_array = name.split('.')
66
- full_name = is_fifo_array[0]
67
- fifo_suffix = is_fifo_array[-1]
68
- suffix_exists = fifo_suffix != full_name
69
-
70
- if suffix_exists && fifo_suffix != 'fifo'
71
- raise ArgumentError, "Topic name #{name} consists unexpected suffix #{fifo_suffix}"
72
- end
73
-
74
- fifo = suffix_exists
75
- topic_array = full_name.split('-')
76
-
77
- raise ArgumentError, "Topic name should consists `#{SNS_SUFFIX}`" unless topic_array[2] != SNS_SUFFIX
78
-
79
- new(
80
- instance: topic_array[0],
81
- kind: topic_array[1],
82
- publisher: topic_array[3],
83
- type: topic_array[4],
84
- region: region,
85
- account_id: account_id,
86
- fifo: fifo,
87
- name: name
88
- )
89
- end
90
-
91
- def from_arn(arn)
92
- arn_array = arn.split(':')
93
- raise ArgumentError, 'Arn should consists `arn`' unless arn_array[0] == 'arn'
94
- raise ArgumentError, 'Arn should consists `aws`' unless arn_array[1] == 'aws'
95
- raise ArgumentError, 'Arn should consists `sns`' unless arn_array[2] == 'sns'
96
-
97
- from_name(arn_array[5], region: arn_array[3], account_id: arn_array[4])
98
- end
99
- end
100
-
101
- private
102
-
103
- def default_tags(instance, kind, publisher, type, fifo)
104
- [
105
- { key: 'instance', value: String(instance) },
106
- { key: 'kind', value: String(kind) },
107
- { key: 'publisher', value: String(publisher) },
108
- { key: 'type', value: String(type) },
109
- { key: 'fifo', value: fifo ? 'true' : 'false' }
110
- ]
111
- end
112
- end
113
- end