cyclone_lariat 0.4.0 → 1.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/gem-push.yml +4 -4
  3. data/.rubocop.yml +9 -5
  4. data/Gemfile.lock +123 -21
  5. data/Guardfile +42 -0
  6. data/README.md +417 -220
  7. data/bin/cyclone_lariat +75 -43
  8. data/cyclone_lariat.gemspec +10 -3
  9. data/lib/cyclone_lariat/clients/abstract.rb +40 -0
  10. data/lib/cyclone_lariat/clients/sns.rb +163 -0
  11. data/lib/cyclone_lariat/clients/sqs.rb +114 -0
  12. data/lib/cyclone_lariat/core.rb +21 -0
  13. data/lib/cyclone_lariat/errors.rb +16 -0
  14. data/lib/cyclone_lariat/fake.rb +19 -0
  15. data/lib/cyclone_lariat/generators/command.rb +53 -0
  16. data/lib/cyclone_lariat/generators/event.rb +52 -0
  17. data/lib/cyclone_lariat/generators/queue.rb +30 -0
  18. data/lib/cyclone_lariat/generators/topic.rb +29 -0
  19. data/lib/cyclone_lariat/messages/v1/abstract.rb +139 -0
  20. data/lib/cyclone_lariat/messages/v1/command.rb +20 -0
  21. data/lib/cyclone_lariat/messages/v1/event.rb +20 -0
  22. data/lib/cyclone_lariat/messages/v1/validator.rb +31 -0
  23. data/lib/cyclone_lariat/messages/v2/abstract.rb +149 -0
  24. data/lib/cyclone_lariat/messages/v2/command.rb +20 -0
  25. data/lib/cyclone_lariat/messages/v2/event.rb +20 -0
  26. data/lib/cyclone_lariat/messages/v2/validator.rb +39 -0
  27. data/lib/cyclone_lariat/middleware.rb +9 -6
  28. data/lib/cyclone_lariat/migration.rb +54 -117
  29. data/lib/cyclone_lariat/options.rb +52 -0
  30. data/lib/cyclone_lariat/presenters/graph.rb +54 -0
  31. data/lib/cyclone_lariat/presenters/queues.rb +41 -0
  32. data/lib/cyclone_lariat/presenters/subscriptions.rb +34 -0
  33. data/lib/cyclone_lariat/presenters/topics.rb +40 -0
  34. data/lib/cyclone_lariat/publisher.rb +25 -0
  35. data/lib/cyclone_lariat/repo/active_record/messages.rb +92 -0
  36. data/lib/cyclone_lariat/repo/active_record/versions.rb +28 -0
  37. data/lib/cyclone_lariat/repo/messages.rb +43 -0
  38. data/lib/cyclone_lariat/repo/messages_mapper.rb +49 -0
  39. data/lib/cyclone_lariat/repo/sequel/messages.rb +73 -0
  40. data/lib/cyclone_lariat/repo/sequel/versions.rb +28 -0
  41. data/lib/cyclone_lariat/repo/versions.rb +42 -0
  42. data/lib/cyclone_lariat/resources/queue.rb +167 -0
  43. data/lib/cyclone_lariat/resources/topic.rb +132 -0
  44. data/lib/cyclone_lariat/services/migrate.rb +51 -0
  45. data/lib/cyclone_lariat/services/rollback.rb +51 -0
  46. data/lib/cyclone_lariat/version.rb +1 -1
  47. data/lib/cyclone_lariat.rb +4 -11
  48. data/lib/tasks/console.rake +1 -1
  49. data/lib/tasks/cyclone_lariat.rake +10 -12
  50. data/lib/tasks/db.rake +0 -15
  51. metadata +127 -27
  52. data/config/db.example.rb +0 -9
  53. data/config/initializers/sequel.rb +0 -7
  54. data/db/migrate/01_add_uuid_extensions.rb +0 -15
  55. data/db/migrate/02_add_events.rb +0 -19
  56. data/db/migrate/03_add_versions.rb +0 -9
  57. data/docs/_imgs/graphviz_01.png +0 -0
  58. data/docs/_imgs/graphviz_02.png +0 -0
  59. data/docs/_imgs/graphviz_03.png +0 -0
  60. data/docs/_imgs/lariat.jpg +0 -0
  61. data/docs/_imgs/logic.png +0 -0
  62. data/docs/_imgs/sqs_sns_diagram.png +0 -0
  63. data/lib/cyclone_lariat/abstract/client.rb +0 -112
  64. data/lib/cyclone_lariat/abstract/message.rb +0 -98
  65. data/lib/cyclone_lariat/command.rb +0 -13
  66. data/lib/cyclone_lariat/configure.rb +0 -15
  67. data/lib/cyclone_lariat/event.rb +0 -13
  68. data/lib/cyclone_lariat/messages_mapper.rb +0 -46
  69. data/lib/cyclone_lariat/messages_repo.rb +0 -60
  70. data/lib/cyclone_lariat/queue.rb +0 -147
  71. data/lib/cyclone_lariat/sns_client.rb +0 -149
  72. data/lib/cyclone_lariat/sqs_client.rb +0 -93
  73. data/lib/cyclone_lariat/topic.rb +0 -113
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'securerandom'
4
+ require 'cyclone_lariat/messages/v1/event'
5
+ require 'cyclone_lariat/messages/v2/event'
6
+
7
+ module CycloneLariat
8
+ module Generators
9
+ module Event
10
+ def event(type, version: config.version, **options)
11
+ case version.to_i
12
+ when 1 then event_v1(type, **options)
13
+ when 2 then event_v2(type, **options)
14
+ else raise ArgumentError, "Unknown version #{version}"
15
+ end
16
+ end
17
+
18
+ def event_v1(type, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
19
+ params = {
20
+ uuid: uuid,
21
+ type: type,
22
+ sent_at: Time.now.iso8601(3),
23
+ version: 1,
24
+ publisher: config.publisher,
25
+ data: data,
26
+ request_id: request_id,
27
+ group_id: group_id,
28
+ deduplication_id: deduplication_id
29
+ }
30
+
31
+ Messages::V1::Event.wrap(params.compact)
32
+ end
33
+ def event_v2(type, subject:, object:, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
34
+ params = {
35
+ uuid: uuid,
36
+ type: type,
37
+ subject: subject,
38
+ object: object,
39
+ sent_at: Time.now.iso8601(3),
40
+ version: 2,
41
+ publisher: config.publisher,
42
+ data: data,
43
+ request_id: request_id,
44
+ group_id: group_id,
45
+ deduplication_id: deduplication_id
46
+ }
47
+
48
+ Messages::V2::Event.wrap(params.compact)
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cyclone_lariat/resources/queue'
4
+
5
+ module CycloneLariat
6
+ module Generators
7
+ module Queue
8
+ def queue(type = :all, fifo:, dest: nil, content_based_deduplication: nil, kind: :event, **options)
9
+ options = CycloneLariat::Options.wrap(options)
10
+ options.merge!(config)
11
+
12
+ Resources::Queue.new(
13
+ instance: options.instance,
14
+ publisher: options.publisher,
15
+ region: options.aws_region,
16
+ account_id: options.aws_account_id,
17
+ kind: kind,
18
+ type: type,
19
+ fifo: fifo,
20
+ dest: dest,
21
+ content_based_deduplication: content_based_deduplication
22
+ )
23
+ end
24
+
25
+ def custom_queue(name)
26
+ Resources::Queue.from_name(name, account_id: config.aws_account_id, region: config.aws_region)
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cyclone_lariat/resources/topic'
4
+
5
+ module CycloneLariat
6
+ module Generators
7
+ module Topic
8
+ def topic(type, fifo:, kind: :event, content_based_deduplication: nil, **options)
9
+ options = CycloneLariat::Options.wrap(options)
10
+ options.merge!(config)
11
+
12
+ Resources::Topic.new(
13
+ instance: options.instance,
14
+ publisher: options.publisher,
15
+ region: options.aws_region,
16
+ account_id: options.aws_account_id,
17
+ kind: kind,
18
+ type: type,
19
+ fifo: fifo,
20
+ content_based_deduplication: content_based_deduplication
21
+ )
22
+ end
23
+
24
+ def custom_topic(name)
25
+ Resources::Topic.from_name(name, account_id: config.aws_account_id, region: config.aws_region)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'luna_park/entities/attributable'
4
+ require 'luna_park/extensions/validatable'
5
+ require 'cyclone_lariat/messages/v1/validator'
6
+ require 'cyclone_lariat/errors'
7
+
8
+ module CycloneLariat
9
+ module Messages
10
+ module V1
11
+ class Abstract < LunaPark::Entities::Attributable
12
+ include LunaPark::Extensions::Validatable
13
+
14
+ attr :uuid, String, :new
15
+ attr :publisher, String, :new
16
+ attr :type, String, :new
17
+
18
+ attrs :client_error, :version, :data, :request_id, :sent_at,
19
+ :deduplication_id, :group_id, :processed_at, :received_at
20
+
21
+ validator Validator
22
+
23
+ # Make validation public
24
+ def validation
25
+ super
26
+ end
27
+
28
+ def serialize
29
+ {
30
+ uuid: uuid,
31
+ publisher: publisher,
32
+ type: [kind, type].join('_'),
33
+ version: version,
34
+ data: data,
35
+ request_id: request_id,
36
+ sent_at: sent_at&.iso8601(3)
37
+ }.compact
38
+ end
39
+
40
+ def to_json(*args)
41
+ serialize.to_json(*args)
42
+ end
43
+
44
+ alias params serialize
45
+
46
+ def kind
47
+ raise LunaPark::Errors::AbstractMethod
48
+ end
49
+
50
+ def data
51
+ @data ||= {}
52
+ end
53
+
54
+ def version=(value)
55
+ @version = Integer(value)
56
+ end
57
+
58
+ def sent_at=(value)
59
+ @sent_at = wrap_time(value)
60
+ end
61
+
62
+ def received_at=(value)
63
+ @received_at = wrap_time(value)
64
+ end
65
+
66
+ def processed_at=(value)
67
+ @processed_at = wrap_time(value)
68
+ end
69
+
70
+ def request_id=(value)
71
+ @request_id = wrap_string(value)
72
+ end
73
+
74
+ def group_id=(value)
75
+ @group_id = wrap_string(value)
76
+ end
77
+
78
+ def deduplication_id=(value)
79
+ @deduplication_id = wrap_string(value)
80
+ end
81
+
82
+ def processed?
83
+ !@processed_at.nil?
84
+ end
85
+
86
+ def client_error_message=(txt)
87
+ return unless txt
88
+
89
+ @client_error ||= Errors::ClientError.new
90
+ @client_error.message = txt
91
+ end
92
+
93
+ def client_error_details=(details)
94
+ return unless details
95
+
96
+ @client_error ||= Errors::ClientError.new
97
+ @client_error.details = details
98
+ end
99
+
100
+ def fifo?
101
+ !@group_id.nil?
102
+ end
103
+
104
+ def ==(other)
105
+ kind == other.kind &&
106
+ uuid == other.uuid &&
107
+ publisher == other.publisher &&
108
+ type == other.type &&
109
+ client_error&.message == other.client_error&.message &&
110
+ version == other.version &&
111
+ sent_at.to_i == other.sent_at.to_i &&
112
+ received_at.to_i == other.received_at.to_i &&
113
+ processed_at.to_i == other.processed_at.to_i
114
+ end
115
+
116
+ private
117
+
118
+ def wrap_time(value)
119
+ case value
120
+ when String then Time.parse(value)
121
+ when Time then value
122
+ when NilClass then nil
123
+ else raise ArgumentError, "Unknown type `#{value.class}`"
124
+ end
125
+ end
126
+
127
+ def wrap_string(value)
128
+ case value
129
+ when String then String(value)
130
+ when Integer then String(value)
131
+ when NilClass then nil
132
+ when FalseClass then nil
133
+ else raise ArgumentError, "Unknown type `#{value.class}`"
134
+ end
135
+ end
136
+ end
137
+ end
138
+ end
139
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cyclone_lariat/messages/v1/abstract'
4
+
5
+ module CycloneLariat
6
+ module Messages
7
+ module V1
8
+ class Command < Abstract
9
+ include LunaPark::Extensions::Validatable
10
+ validator Messages::V1::Validator
11
+
12
+ KIND = 'command'
13
+
14
+ def kind
15
+ KIND
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cyclone_lariat/messages/v1/abstract'
4
+
5
+ module CycloneLariat
6
+ module Messages
7
+ module V1
8
+ class Event < Abstract
9
+ include LunaPark::Extensions::Validatable
10
+ validator Messages::V1::Validator
11
+
12
+ KIND = 'event'
13
+
14
+ def kind
15
+ KIND
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'luna_park/validators/dry'
4
+ require 'cyclone_lariat/errors'
5
+
6
+ module CycloneLariat
7
+ module Messages
8
+ module V1
9
+ class Validator < LunaPark::Validators::Dry
10
+ UUID_MATCHER = /^\h{8}-\h{4}-(\h{4})-\h{4}-\h{12}$/.freeze
11
+ ISO8601_MATCHER = /^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])T(2[0-3]|[0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[0-1][0-9]):[0-5][0-9])?$/.freeze
12
+
13
+ validation_schema do
14
+ params do
15
+ required(:uuid).value(format?: UUID_MATCHER)
16
+ required(:publisher).filled(:string)
17
+ required(:type).filled(:string)
18
+ required(:version).filled(:integer).value(eql?: 1)
19
+ required(:data).value(:hash?)
20
+ optional(:request_id).value(format?: UUID_MATCHER)
21
+ required(:sent_at).value(format?: ISO8601_MATCHER)
22
+ end
23
+ end
24
+
25
+ def check!
26
+ raise Errors::InvalidMessage.new(message: params, validation_errors: errors) unless success?
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,149 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'luna_park/entities/attributable'
4
+ require 'luna_park/extensions/validatable'
5
+ require 'cyclone_lariat/messages/v2/validator'
6
+ require 'cyclone_lariat/errors'
7
+
8
+ module CycloneLariat
9
+ module Messages
10
+ module V2
11
+ class Abstract < LunaPark::Entities::Attributable
12
+ include LunaPark::Extensions::Validatable
13
+
14
+ attr :uuid, String, :new
15
+ attr :publisher, String, :new
16
+ attr :type, String, :new
17
+ attrs :client_error, :version, :data, :request_id, :sent_at,
18
+ :deduplication_id, :group_id, :processed_at, :received_at,
19
+ :subject, :object
20
+
21
+ validator Validator
22
+
23
+ # Make validation public
24
+ def validation
25
+ super
26
+ end
27
+
28
+ def serialize
29
+ {
30
+ uuid: uuid,
31
+ publisher: publisher,
32
+ type: [kind, type].join('_'),
33
+ version: version,
34
+ data: data,
35
+ request_id: request_id,
36
+ sent_at: sent_at&.iso8601(3),
37
+ subject: subject,
38
+ object: object
39
+ }.compact
40
+ end
41
+
42
+ def to_json(*args)
43
+ serialize.to_json(*args)
44
+ end
45
+
46
+ alias params serialize
47
+
48
+ def kind
49
+ raise LunaPark::Errors::AbstractMethod
50
+ end
51
+
52
+ def data
53
+ @data ||= {}
54
+ end
55
+
56
+ def subject
57
+ @subject ||= {}
58
+ end
59
+
60
+ def object
61
+ @object ||= {}
62
+ end
63
+
64
+ def version=(value)
65
+ @version = Integer(value)
66
+ end
67
+
68
+ def sent_at=(value)
69
+ @sent_at = wrap_time(value)
70
+ end
71
+
72
+ def received_at=(value)
73
+ @received_at = wrap_time(value)
74
+ end
75
+
76
+ def processed_at=(value)
77
+ @processed_at = wrap_time(value)
78
+ end
79
+
80
+ def request_id=(value)
81
+ @request_id = wrap_string(value)
82
+ end
83
+
84
+ def group_id=(value)
85
+ @group_id = wrap_string(value)
86
+ end
87
+
88
+ def deduplication_id=(value)
89
+ @deduplication_id = wrap_string(value)
90
+ end
91
+
92
+ def processed?
93
+ !@processed_at.nil?
94
+ end
95
+
96
+ def client_error_message=(txt)
97
+ return unless txt
98
+
99
+ @client_error ||= Errors::ClientError.new
100
+ @client_error.message = txt
101
+ end
102
+
103
+ def client_error_details=(details)
104
+ return unless details
105
+
106
+ @client_error ||= Errors::ClientError.new
107
+ @client_error.details = details
108
+ end
109
+
110
+ def fifo?
111
+ !@group_id.nil?
112
+ end
113
+
114
+ def ==(other)
115
+ kind == other.kind &&
116
+ uuid == other.uuid &&
117
+ publisher == other.publisher &&
118
+ type == other.type &&
119
+ client_error&.message == other.client_error&.message &&
120
+ version == other.version &&
121
+ sent_at.to_i == other.sent_at.to_i &&
122
+ received_at.to_i == other.received_at.to_i &&
123
+ processed_at.to_i == other.processed_at.to_i
124
+ end
125
+
126
+ private
127
+
128
+ def wrap_time(value)
129
+ case value
130
+ when String then Time.parse(value)
131
+ when Time then value
132
+ when NilClass then nil
133
+ else raise ArgumentError, "Unknown type `#{value.class}`"
134
+ end
135
+ end
136
+
137
+ def wrap_string(value)
138
+ case value
139
+ when String then String(value)
140
+ when Integer then String(value)
141
+ when NilClass then nil
142
+ when FalseClass then nil
143
+ else raise ArgumentError, "Unknown type `#{value.class}`"
144
+ end
145
+ end
146
+ end
147
+ end
148
+ end
149
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cyclone_lariat/messages/v2/abstract'
4
+
5
+ module CycloneLariat
6
+ module Messages
7
+ module V2
8
+ class Command < Abstract
9
+ include LunaPark::Extensions::Validatable
10
+ validator Messages::V2::Validator
11
+
12
+ KIND = 'command'
13
+
14
+ def kind
15
+ KIND
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cyclone_lariat/messages/v2/abstract'
4
+
5
+ module CycloneLariat
6
+ module Messages
7
+ module V2
8
+ class Event < Abstract
9
+ include LunaPark::Extensions::Validatable
10
+ validator Messages::V2::Validator
11
+
12
+ KIND = 'event'
13
+
14
+ def kind
15
+ KIND
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'luna_park/validators/dry'
4
+ require 'cyclone_lariat/errors'
5
+
6
+ module CycloneLariat
7
+ module Messages
8
+ module V2
9
+ class Validator < LunaPark::Validators::Dry
10
+ UUID_MATCHER = /^\h{8}-\h{4}-(\h{4})-\h{4}-\h{12}$/.freeze
11
+ ISO8601_MATCHER = /^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])T(2[0-3]|[0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[0-1][0-9]):[0-5][0-9])?$/.freeze
12
+
13
+ validation_schema do
14
+ params do
15
+ required(:uuid).value(format?: UUID_MATCHER)
16
+ required(:publisher).filled(:string)
17
+ required(:type).filled(:string)
18
+ required(:version).filled(:integer).value(eql?: 2)
19
+ required(:data).value(:hash?)
20
+ optional(:request_id).value(format?: UUID_MATCHER)
21
+ required(:sent_at).value(format?: ISO8601_MATCHER)
22
+ required(:subject).hash do
23
+ required(:type).filled(:string)
24
+ required(:uuid).value(format?: UUID_MATCHER)
25
+ end
26
+ required(:object).hash do
27
+ required(:type).filled(:string)
28
+ required(:uuid).value(format?: UUID_MATCHER)
29
+ end
30
+ end
31
+ end
32
+
33
+ def check!
34
+ raise Errors::InvalidMessage.new(message: params, validation_errors: errors) unless success?
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -1,14 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative 'messages_repo'
3
+ require 'cyclone_lariat/repo/messages'
4
+ require 'cyclone_lariat/core'
4
5
  require 'luna_park/errors'
5
6
  require 'json'
6
7
 
7
8
  module CycloneLariat
8
9
  class Middleware
9
- def initialize(dataset: nil, errors_notifier: nil, message_notifier: nil, repo: MessagesRepo)
10
- events_dataset = dataset || CycloneLariat.events_dataset
11
- @events_repo = repo.new(events_dataset) if events_dataset
10
+ attr_reader :config
11
+
12
+ def initialize(errors_notifier: nil, message_notifier: nil, repo: Repo::Messages, **options)
13
+ @config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
14
+ @events_repo = repo.new(**@config.to_h)
12
15
  @message_notifier = message_notifier
13
16
  @errors_notifier = errors_notifier
14
17
  end
@@ -20,7 +23,7 @@ module CycloneLariat
20
23
  return if msg.is_a? String
21
24
 
22
25
  catch_standard_error(queue, msg) do
23
- event = Event.wrap(msg)
26
+ event = Messages::V1::Event.wrap(msg)
24
27
 
25
28
  store_in_dataset(event) do
26
29
  catch_business_error(event, &block)
@@ -40,7 +43,7 @@ module CycloneLariat
40
43
  end
41
44
 
42
45
  def store_in_dataset(event)
43
- return yield if events_repo.nil?
46
+ return yield if events_repo.disabled?
44
47
 
45
48
  existed = events_repo.find(uuid: event.uuid)
46
49
  return true if existed&.processed?