cyclone_lariat 1.0.0.rc5 → 1.0.0.rc6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e85d9e3115b5cba0ff233723389a20a5eef9f3c
-  data.tar.gz: d87d6e81b0fba3f4aa78e432b7aab2bf993f5a82
+  metadata.gz: fab5b60b7bc9091b050be9f19f409dbb108b868c
+  data.tar.gz: 4c2458fe43a0353b5d2a8e1977858889220a9cb8
 SHA512:
-  metadata.gz: 285ef03ab2d275f51401d547b3ce6d43c7dbfcc2e16cb53cbbe134dde906d043ec2eb8ea427286b413910a85e70b0b320a8a857ce2f03f7a021db6c47b72e7f4
-  data.tar.gz: 8fed12ef46bc40ac995b80a377ab9702fc526d2b12ced8750b6e84ca5a44cfef49b14498d45c84ba421cf1a7f3ddaaa71088abdcd98e28a329057369af194b7c
+  metadata.gz: '090c611cb5e0bd49c4654042fbd285472b369448887789ff2392fd279dc68f12ebbb82f734254da9ae11750f38423749461f8a794c09370d85b06e11b097c73c'
+  data.tar.gz: 1b66103e5350f18335b70789246be6379f21c92ea6c1e92d181e3439e1039f96558ef2fd9c97eb6d95b77bcb21bff61f8c8880cfbd8a8c342a02a3620032f50c

data/CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0.rc6]
+Changed
+- Rename `messages_dataset` to `inbox_dataset`
+Added
+- `CycloneLariat::Outbox` - implementation of the transactional outbox pattern
+
 ## [1.0.0.rc5]
 Changed
 - Update Gemfile.lock

data/README.md

@@ -67,7 +67,7 @@ Last install command will create 2 files:
     c.publisher = ENV['APP_NAME']               # name of your publishers, usually name of your application
     c.instance = ENV['INSTANCE']                # stage, production, test
     c.driver = :sequel                          # driver Sequel
-    c.messages_dataset = DB[:async_messages]    # Sequel dataset for store income messages (on receiver)
+    c.inbox_dataset = DB[:inbox_messages]       # Sequel dataset for store incoming messages (on receiver)
     c.versions_dataset = DB[:lariat_versions]   # Sequel dataset for versions of publisher migrations
     c.fake_publish = ENV['INSTANCE'] == 'test'  # when true, prevents messages from being published
   end
@@ -93,7 +93,7 @@ Last install command will create 2 files:
 
   Sequel.migration do
     change do
-      create_table :async_messages do
+      create_table :inbox_messages do
         column   :uuid, :uuid, primary_key: true
         String   :type,                         null: false
         Integer  :version,                      null: false
@@ -124,19 +124,19 @@ Last install command will create 2 files:
   # frozen_string_literal: true
 
   CycloneLariat.configure do |c|
-    c.version = 1                               # api version
+    c.version = 1                                 # api version
 
-    c.aws_key = ENV['AWS_KEY']                  # aws key
-    c.aws_secret_key = ENV['AWS_SECRET_KEY']    # aws secret
-    c.aws_account_id = ENV['AWS_ACCOUNT_ID']    # aws account id
-    c.aws_region = ENV['AWS_REGION']            # aws region
+    c.aws_key = ENV['AWS_KEY']                    # aws key
+    c.aws_secret_key = ENV['AWS_SECRET_KEY']      # aws secret
+    c.aws_account_id = ENV['AWS_ACCOUNT_ID']      # aws account id
+    c.aws_region = ENV['AWS_REGION']              # aws region
 
-    c.publisher = ENV['APP_NAME']               # name of your publishers, usually name of your application
-    c.instance = ENV['INSTANCE']                # stage, production, test
-    c.driver = :active_record                   # driver ActiveRecord
-    c.messages_dataset = CycloneLariatMessage   # ActiveRecord model for store income messages (on receiver)
-    c.versions_dataset = CycloneLariatVersion   # ActiveRecord model for versions of publisher migrations
-    c.fake_publish = ENV['INSTANCE'] == 'test'  # when true, prevents messages from being published
+    c.publisher = ENV['APP_NAME']                 # name of your publishers, usually name of your application
+    c.instance = ENV['INSTANCE']                  # stage, production, test
+    c.driver = :active_record                     # driver ActiveRecord
+    c.inbox_dataset = CycloneLariatInboxMessage   # ActiveRecord model for store income messages (on receiver)
+    c.versions_dataset = CycloneLariatVersion     # ActiveRecord model for versions of publisher migrations
+    c.fake_publish = ENV['INSTANCE'] == 'test'    # when true, prevents messages from being published
   end
   ```
 
@@ -734,7 +734,7 @@ $ rake cyclone_lariat:graph               # Make graph
 Graph generated in [grpahviz](https://graphviz.org/) format for the entry scheme. You should install
 it on your system. For convert it in png use:
 ```bash
-$ rake cyclone_lariat:list:subscriptions | dot -Tpng -o foo.png
+$ rake cyclone_lariat:graph | dot -Tpng -o foo.png
 ```
 
 ## Subscriber
@@ -801,20 +801,91 @@ class Receiver
 end
 ```
 
+## Transactional outbox
+
+This extension allows you to save messages to a database inside a transaction. It prevents messages from being lost when publishing fails. After the transaction is copmpleted, publishing will be perfromed and successfully published messages will be deleted from the database. For more information, see [Transactional outbox pattern](https://microservices.io/patterns/data/transactional-outbox.html)
+
+
+### Configuration
+
+```ruby
+OutboxErrorLogger = LunaPark::Notifiers::Log.new
+CycloneLariat::Outbox.configure do |config|
+  config.dataset = DB[:outbox_messages] # Outbox messages dataset. Sequel dataset or ActiveRecord model
+  config.on_sending_error = lambda do |event, error|
+    OutboxErrorLogger.error(error, details: event.to_h)
+  end
+end
+
+CycloneLariat::Outbox.load
+```
+
+Before using the outbox, add and apply this migration:
+
+```ruby
+# Sequel
+DB.create_table :outbox_messages do
+  column :uuid, :uuid, primary_key: true
+  column :deduplication_id, String, null: true
+  column :group_id, String, null: true
+  column :serialized_message, :json, null: false
+  column :sending_error, String, null: true
+  DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+end
+
+# ActiveRecord
+create_table(:outbox_messages, id: :uuid, primary_key: :uuid, default: -> { 'public.uuid_generate_v4()' }) do |t|
+  t.string :deduplication_id, null: true
+  t.string :group_id, null: true
+  t.string :sending_error, null: true
+  t.jsonb :serialized_message, null: false
+  t.datetime :created_at, null: false, default: -> { 'CURRENT_TIMESTAMP' }
+end
+```
+
+### Usage example
+
+```ruby
+# Sequel
+DB.transaction(with_outbox: true) do |outbox|
+  some_action
+  outbox << CycloneLariat::Messages::V1::Event.new(...)
+  ...
+end
+
+# ActiveRecord
+ActiveRecord::Base.transaction(with_outbox: true) do |outbox|
+  some_action
+  outbox << CycloneLariat::Messages::V1::Event.new(...)
+  ...
+end
+```
+
+### Resending
+
+To resend messages you can use the following service:
+
+```ruby
+CycloneLariat::Outbox::Services::Resend.call
+```
+
+This service tries to publish messages from the outbox table with `sending_error != nil`.
+Successfully published messages will be removed.
+
 ## Rake tasks
 
-For simplify write some Rake tasks you can use `CycloneLariat::Repo::Messages`.
+For simplify write some Rake tasks you can use `CycloneLariat::Repo::InboxMessages`.
 
 ```ruby
 # For retry all unprocessed
 
-CycloneLariat::Repo::Messages.new.each_unprocessed do |event|
+CycloneLariat::Repo::InboxMessages.new.each_unprocessed do |event|
   # Your logic here
 end
 
 # For retry all events with client errors
 
-CycloneLariat::Repo::Messages.new.each_with_client_errors do |event|
+CycloneLariat::Repo::InboxMessages.new.each_with_client_errors do |event|
   # Your logic here
 end
 ```

data/bin/cyclone_lariat

@@ -70,7 +70,7 @@ module CycloneLariat
               c.publisher          = ENV['APP_NAME']           # name of your publishers, usually name of your application
               c.instance           = ENV['INSTANCE']           # stage, production, test
               c.driver             = :sequel                   # :sequel or :active_record
-              c.messages_dataset   = DB[:messages]             # Sequel dataset / ActiveRecord model for store income messages (on receiver)
+              c.inbox_dataset      = DB[:inbox_messages]       # Sequel dataset / ActiveRecord model for store income messages (on receiver)
               c.versions_dataset   = DB[:lariat_versions]      # Sequel dataset / ActiveRecord model for publisher migrations
               c.fake_publish       = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
             end
@@ -82,17 +82,17 @@ module CycloneLariat
             # frozen_string_literal: true
 
             CycloneLariat.configure do |c|
-              c.version            = 1                         # messages version
-              c.aws_key            = ENV['AWS_KEY']            # aws key
-              c.aws_account_id     = ENV['AWS_ACCOUNT_ID']     # aws account id
-              c.aws_secret_key     = ENV['AWS_SECRET_KEY']     # aws secret
-              c.aws_region         = ENV['AWS_REGION']         # aws default region
-              c.publisher          = ENV['APP_NAME']           # name of your publishers, usually name of your application
-              c.instance           = ENV['INSTANCE']           # stage, production, test
-              c.driver             = :active_record            # :sequel or :active_record
-              c.messages_dataset   = CycloneLariatMessage      # Sequel dataset / ActiveRecord model for store income messages (on receiver)
-              c.versions_dataset   = CycloneLariatVersion      # Sequel dataset / ActiveRecord model for publisher migrations
-              c.fake_publish       = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
+              c.version            = 1                          # messages version
+              c.aws_key            = ENV['AWS_KEY']             # aws key
+              c.aws_account_id     = ENV['AWS_ACCOUNT_ID']      # aws account id
+              c.aws_secret_key     = ENV['AWS_SECRET_KEY']      # aws secret
+              c.aws_region         = ENV['AWS_REGION']          # aws default region
+              c.publisher          = ENV['APP_NAME']            # name of your publishers, usually name of your application
+              c.instance           = ENV['INSTANCE']            # stage, production, test
+              c.driver             = :active_record             # :sequel or :active_record
+              c.inbox_dataset      = CycloneLariatInboxMessage  # Sequel dataset / ActiveRecord model for store incoming messages (on receiver)
+              c.versions_dataset   = CycloneLariatVersion       # Sequel dataset / ActiveRecord model for publisher migrations
+              c.fake_publish       = ENV['INSTANCE'] == 'test'  # when true, prevents messages from being published
             end
           CONFIG
         end

data/lib/cyclone_lariat/generators/event.rb

@@ -30,6 +30,7 @@ module CycloneLariat
 
         Messages::V1::Event.wrap(params.compact)
       end
+
       def event_v2(type, subject:, object:, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
         params = {
           uuid: uuid,

data/lib/cyclone_lariat/messages/abstract.rb

@@ -15,7 +15,7 @@ module CycloneLariat
       attr :publisher, String, :new
       attr :type,      String, :new
 
-      attrs :client_error, :version, :data, :request_id, :sent_at,
+      attrs :client_error, :sending_error, :version, :data, :request_id, :sent_at,
             :deduplication_id, :group_id, :processed_at, :received_at
 
       # Make validation public

data/lib/cyclone_lariat/middleware.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'cyclone_lariat/repo/messages'
+require 'cyclone_lariat/repo/inbox_messages'
 require 'cyclone_lariat/core'
 require 'luna_park/errors'
 require 'cyclone_lariat/messages/builder'
@@ -10,7 +10,7 @@ module CycloneLariat
   class Middleware
     attr_reader :config
 
-    def initialize(errors_notifier: nil, message_notifier: nil, repo: Repo::Messages, **options)
+    def initialize(errors_notifier: nil, message_notifier: nil, repo: Repo::InboxMessages, **options)
       @config           = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
       @events_repo      = repo.new(**@config.to_h)
       @message_notifier = message_notifier

data/lib/cyclone_lariat/options.rb

@@ -6,7 +6,7 @@ module CycloneLariat
   class Options < LunaPark::Values::Compound
     attr_accessor :aws_key, :aws_secret_key, :publisher,
                   :aws_region, :instance, :aws_account_id,
-                  :messages_dataset, :version, :versions_dataset,
+                  :inbox_dataset, :version, :versions_dataset,
                   :driver, :fake_publish
 
     # @param [CycloneLariat::Options, Hash] other
@@ -14,17 +14,17 @@ module CycloneLariat
     def merge!(other)
       other = self.class.wrap(other)
 
-      self.aws_key          ||= other.aws_key
-      self.aws_secret_key   ||= other.aws_secret_key
-      self.publisher        ||= other.publisher
-      self.aws_region       ||= other.aws_region
-      self.instance         ||= other.instance
-      self.aws_account_id   ||= other.aws_account_id
-      self.messages_dataset ||= other.messages_dataset
-      self.version          ||= other.version
-      self.versions_dataset ||= other.versions_dataset
-      self.driver           ||= other.driver
-      self.fake_publish     ||= other.fake_publish
+      self.aws_key                       ||= other.aws_key
+      self.aws_secret_key                ||= other.aws_secret_key
+      self.publisher                     ||= other.publisher
+      self.aws_region                    ||= other.aws_region
+      self.instance                      ||= other.instance
+      self.aws_account_id                ||= other.aws_account_id
+      self.inbox_dataset                 ||= other.inbox_dataset
+      self.version                       ||= other.version
+      self.versions_dataset              ||= other.versions_dataset
+      self.driver                        ||= other.driver
+      self.fake_publish                  ||= other.fake_publish
 
       self
     end
@@ -41,7 +41,7 @@ module CycloneLariat
         aws_region: aws_region,
         instance: instance,
         aws_account_id: aws_account_id,
-        messages_dataset: messages_dataset,
+        inbox_dataset: inbox_dataset,
         version: version,
         versions_dataset: versions_dataset,
         driver: driver,

data/lib/cyclone_lariat/plugins/outbox/configurable.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  class Outbox
+    module Configurable
+      CONFIG_ATTRS = %i[dataset on_sending_error].freeze
+
+      def config
+        @config ||= Struct.new(*CONFIG_ATTRS).new
+      end
+
+      def configure
+        yield(config) if block_given?
+        config
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox/extensions/active_record_outbox.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  class Outbox
+    module Extensions
+      module ActiveRecordOutbox
+        def transaction(opts = {}, &block)
+          opts = opts.dup
+          return super unless opts.delete(:with_outbox)
+
+          outbox = CycloneLariat::Outbox.new
+          result = super(opts) do
+            block.call(outbox)
+          end
+
+          outbox.publish
+          result
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox/extensions/sequel_outbox.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  class Outbox
+    module Extensions
+      module SequelOutbox
+        def transaction(opts = {}, &block)
+          opts = Sequel::OPTS.dup.merge(opts)
+          return super unless opts.delete(:with_outbox)
+
+          outbox = CycloneLariat::Outbox.new
+          result = super(opts) do |conn|
+            block.call(outbox, conn)
+          end
+
+          outbox.publish
+          result
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox/loadable.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  class Outbox
+    module Loadable
+      def load
+        extend_driver_transaction
+      end
+
+      private
+
+      def extend_driver_transaction
+        case CycloneLariat.config.driver
+        when :sequel
+          Sequel::Database.prepend(Outbox::Extensions::SequelOutbox)
+        when :active_record
+          ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(Outbox::Extensions::ActiveRecordOutbox)
+        else
+          raise ArgumentError, "Undefined driver `#{driver}`"
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox/mappers/messages.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/repo/mappers/base'
+
+module CycloneLariat
+  class Outbox
+    module Mappers
+      class Messages < CycloneLariat::Repo::Mappers::Base
+        class << self
+          def from_row(row)
+            return if row.nil?
+
+            attrs = hash_from_json_column(row[:serialized_message]).symbolize_keys
+            attrs[:uuid]             = row[:uuid]
+            attrs[:deduplication_id] = row[:deduplication_id]
+            attrs[:group_id]         = row[:group_id]
+            attrs[:sending_error]    = row[:sending_error]
+
+            attrs
+          end
+
+          def to_row(input)
+            {}.tap do |row|
+              row[:uuid] = input.uuid if input.uuid
+              row[:deduplication_id] = input.deduplication_id
+              row[:group_id] = input.group_id
+              row[:serialized_message] = input.to_json
+              row[:sending_error] = input.sending_error
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox/repo/active_record/messages.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/messages/v1/event'
+require 'cyclone_lariat/messages/v1/command'
+require 'cyclone_lariat/messages/builder'
+require 'cyclone_lariat/plugins/outbox/mappers/messages'
+
+module CycloneLariat
+  class Outbox
+    module Repo
+      module ActiveRecord
+        class Messages
+          LIMIT = 1000
+
+          attr_reader :dataset
+
+          def initialize(dataset)
+            @dataset = dataset
+          end
+
+          def create(msg)
+            dataset.create(Outbox::Mappers::Messages.to_row(msg)).uuid
+          end
+
+          def delete(uuid)
+            dataset.where(uuid: uuid).delete_all
+          end
+
+          def update_error(uuid, error_message)
+            dataset.where(uuid: uuid).update(sending_error: error_message)
+          end
+
+          def each_with_error
+            dataset
+              .where('sending_error IS NOT NULL')
+              .order(created_at: :asc)
+              .limit(LIMIT)
+              .each do |row|
+                msg = build_message_from_ar_row(row)
+                yield(msg)
+              end
+          end
+
+          def transaction(&block)
+            dataset.transaction(&block)
+          end
+
+          def lock(uuid)
+            dataset.lock('FOR UPDATE NOWAIT').where(uuid: uuid)
+          end
+
+          private
+
+          def build_message_from_ar_row(row)
+            build Outbox::Mappers::Messages.from_row(row.attributes.symbolize_keys)
+          end
+
+          def build(raw)
+            CycloneLariat::Messages::Builder.new(raw_message: raw).call
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox/repo/messages.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'luna_park/extensions/injector'
+require 'cyclone_lariat/plugins/outbox/repo/active_record/messages'
+require 'cyclone_lariat/plugins/outbox/repo/sequel/messages'
+
+module CycloneLariat
+  class Outbox
+    module Repo
+      class Messages
+        include LunaPark::Extensions::Injector
+
+        dependency(:sequel_messages_class)        { Repo::Sequel::Messages }
+        dependency(:active_record_messages_class) { Repo::ActiveRecord::Messages }
+        dependency(:general_config)               { CycloneLariat.config }
+
+        extend Forwardable
+
+        def_delegators :driver, :transaction, :lock, :update_error, :create, :delete, :each_with_error
+
+        def driver
+          @driver ||= select_driver
+        end
+
+        private
+
+        def select_driver
+          case general_config.driver
+          when :sequel        then sequel_messages_class.new(config.dataset)
+          when :active_record then active_record_messages_class.new(config.dataset)
+          else raise ArgumentError, "Undefined driver `#{general_config.driver}`"
+          end
+        end
+
+        def config
+          @config ||= CycloneLariat::Outbox.config
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox/repo/sequel/messages.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/messages/v1/event'
+require 'cyclone_lariat/messages/v1/command'
+require 'cyclone_lariat/messages/builder'
+require 'cyclone_lariat/plugins/outbox/mappers/messages'
+
+module CycloneLariat
+  class Outbox
+    module Repo
+      module Sequel
+        class Messages
+          LIMIT = 1000
+
+          attr_reader :dataset
+
+          def initialize(dataset)
+            @dataset = dataset
+          end
+
+          def create(msg)
+            dataset.returning.insert(Outbox::Mappers::Messages.to_row(msg)).first[:uuid]
+          end
+
+          def delete(uuid)
+            dataset.where(uuid: uuid).delete
+          end
+
+          def update_error(uuid, error_message)
+            dataset.where(uuid: uuid).update(sending_error: error_message)
+          end
+
+          def each_with_error
+            dataset
+              .where { sending_error !~ nil }
+              .order(::Sequel.asc(:created_at))
+              .limit(LIMIT)
+              .each do |row|
+                msg = build Outbox::Mappers::Messages.from_row(row)
+                yield(msg)
+              end
+          end
+
+          def transaction(&block)
+            dataset.db.transaction(&block)
+          end
+
+          def lock(uuid)
+            dataset.where(uuid: uuid).for_update.nowait
+          end
+
+          private
+
+          def build(raw)
+            CycloneLariat::Messages::Builder.new(raw_message: raw).call
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox/services/resend.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'luna_park/extensions/callable'
+require 'luna_park/extensions/injector'
+require 'cyclone_lariat/clients/sns'
+require 'cyclone_lariat/plugins/outbox/repo/messages'
+
+module CycloneLariat
+  class Outbox
+    module Services
+      class Resend
+        extend LunaPark::Extensions::Callable
+        include LunaPark::Extensions::Injector
+
+        dependency(:messages_repo)    { CycloneLariat::Outbox::Repo::Messages.new }
+        dependency(:sns_client)       { CycloneLariat::Clients::Sns.new }
+        dependency(:on_sending_error) { CycloneLariat::Outbox.config.on_sending_error }
+
+        def call
+          messages_repo.each_with_error do |message|
+            messages_repo.transaction do
+              begin
+                messages_repo.lock(message.uuid)
+                sns_client.publish message, fifo: message.fifo?
+                messages_repo.delete(message.uuid)
+              rescue StandardError => e
+                messages_repo.update_error(message.uuid, e.message)
+                on_sending_error&.call(message, e)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/plugins/outbox.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/core'
+require 'cyclone_lariat/clients/sns'
+require 'cyclone_lariat/plugins/outbox/configurable'
+require 'cyclone_lariat/plugins/outbox/loadable'
+require 'cyclone_lariat/plugins/outbox/extensions/active_record_outbox'
+require 'cyclone_lariat/plugins/outbox/extensions/sequel_outbox'
+require 'cyclone_lariat/plugins/outbox/repo/messages'
+
+module CycloneLariat
+  class Outbox
+    extend CycloneLariat::Outbox::Configurable
+    extend CycloneLariat::Outbox::Loadable
+    include LunaPark::Extensions::Injector
+
+    dependency(:sns_client) { CycloneLariat::Clients::Sns.new }
+    dependency(:repo)       { CycloneLariat::Outbox::Repo::Messages.new }
+
+    attr_reader :messages
+
+    def initialize
+      @messages = []
+    end
+
+    def publish
+      sent_message_uids = messages.each_with_object([]) do |message, sent_message_uuids|
+        begin
+          sns_client.publish message, fifo: message.fifo?
+          sent_message_uuids << message.uuid
+        rescue StandardError => e
+          repo.update_error(message.uuid, e.message)
+          config.on_sending_error&.call(message, e)
+          next
+        end
+      end
+      repo.delete(sent_message_uids) unless sent_message_uids.empty?
+    end
+
+    def <<(message)
+      message.uuid = repo.create(message)
+      messages << message
+    end
+
+    def push(message)
+      self << message
+    end
+
+    private
+
+    def config
+      self.class.config
+    end
+  end
+end

data/lib/cyclone_lariat/repo/active_record/{messages.rb → inbox_messages.rb}

@@ -2,13 +2,13 @@
 
 require 'cyclone_lariat/messages/v1/event'
 require 'cyclone_lariat/messages/v1/command'
-require 'cyclone_lariat/repo/messages_mapper'
+require 'cyclone_lariat/repo/mappers/inbox_messages'
 require 'cyclone_lariat/messages/builder'
 
 module CycloneLariat
   module Repo
     module ActiveRecord
-      class Messages
+      class InboxMessages
         attr_reader :dataset
 
         def initialize(dataset)
@@ -24,7 +24,7 @@ module CycloneLariat
         end
 
         def create(msg)
-          dataset.create(MessagesMapper.to_row(msg)).uuid
+          dataset.create(Mappers::InboxMessages.to_row(msg)).uuid
         end
 
         def exists?(uuid:)
@@ -68,7 +68,7 @@ module CycloneLariat
         private
 
         def build_message_from_ar_row(row)
-          build MessagesMapper.from_row(row.attributes.symbolize_keys)
+          build Mappers::InboxMessages.from_row(row.attributes.symbolize_keys)
         end
 
         def current_timestamp_from_db

data/lib/cyclone_lariat/repo/{messages.rb → inbox_messages.rb}

@@ -3,18 +3,18 @@
 require 'forwardable'
 require 'luna_park/extensions/injector'
 require 'cyclone_lariat/core'
-require 'cyclone_lariat/repo/sequel/messages'
-require 'cyclone_lariat/repo/active_record/messages'
+require 'cyclone_lariat/repo/sequel/inbox_messages'
+require 'cyclone_lariat/repo/active_record/inbox_messages'
 
 module CycloneLariat
   module Repo
-    class Messages
+    class InboxMessages
       include LunaPark::Extensions::Injector
 
       attr_reader :config
 
-      dependency(:sequel_messages_class) { Repo::Sequel::Messages }
-      dependency(:active_record_messages_class) { Repo::ActiveRecord::Messages }
+      dependency(:sequel_messages_class) { Repo::Sequel::InboxMessages }
+      dependency(:active_record_messages_class) { Repo::ActiveRecord::InboxMessages }
 
       extend Forwardable
 
@@ -33,8 +33,8 @@ module CycloneLariat
 
       def select(driver:)
         case driver
-        when :sequel then sequel_messages_class.new(config.messages_dataset)
-        when :active_record then active_record_messages_class.new(config.messages_dataset)
+        when :sequel then sequel_messages_class.new(config.inbox_dataset)
+        when :active_record then active_record_messages_class.new(config.inbox_dataset)
         else raise ArgumentError, "Undefined driver `#{driver}`"
         end
       end

data/lib/cyclone_lariat/repo/mappers/base.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Repo
+    module Mappers
+      class Base
+        class << self
+          private
+
+          def hash_from_json_column(data)
+            return data if data.is_a?(Hash)
+            return JSON.parse(data) if data.is_a?(String)
+
+            if pg_json_extension_enabled?
+              return data.to_h             if data.is_a?(::Sequel::Postgres::JSONHash)
+              return JSON.parse(data.to_s) if data.is_a?(::Sequel::Postgres::JSONString)
+            end
+
+            raise ArgumentError, "Unknown type of `#{data}`"
+          end
+
+          def pg_json_extension_enabled?
+            Object.const_defined?('Sequel::Postgres::JSONHash')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/mappers/inbox_messages.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/repo/mappers/base'
+
+module CycloneLariat
+  module Repo
+    module Mappers
+      class InboxMessages < Base
+        class << self
+          def from_row(row)
+            return if row.nil?
+
+            row[:data] = hash_from_json_column(row[:data])
+            row[:client_error_details] = hash_from_json_column(row[:client_error_details]) if row[:client_error_details]
+            row
+          end
+
+          def to_row(input)
+            {
+              uuid: input.uuid,
+              kind: input.kind,
+              type: input.type,
+              publisher: input.publisher,
+              data: JSON.generate(input.data),
+              client_error_message: input.client_error&.message,
+              client_error_details: JSON.generate(input.client_error&.details),
+              version: input.version,
+              sent_at: input.sent_at
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/sequel/{messages.rb → inbox_messages.rb}

@@ -2,13 +2,13 @@
 
 require 'cyclone_lariat/messages/v1/event'
 require 'cyclone_lariat/messages/v1/command'
-require 'cyclone_lariat/repo/messages_mapper'
+require 'cyclone_lariat/repo/mappers/inbox_messages'
 require 'cyclone_lariat/messages/builder'
 
 module CycloneLariat
   module Repo
     module Sequel
-      class Messages
+      class InboxMessages
         attr_reader :dataset
 
         def initialize(dataset)
@@ -24,7 +24,7 @@ module CycloneLariat
         end
 
         def create(msg)
-          dataset.insert MessagesMapper.to_row(msg)
+          dataset.insert Mappers::InboxMessages.to_row(msg)
         end
 
         def exists?(uuid:)
@@ -42,19 +42,19 @@ module CycloneLariat
           row = dataset.where(uuid: uuid).first
           return if row.nil?
 
-          build MessagesMapper.from_row(row)
+          build Mappers::InboxMessages.from_row(row)
         end
 
         def each_unprocessed
           dataset.where(processed_at: nil).each do |row|
-            msg = build MessagesMapper.from_row(row)
+            msg = build Mappers::InboxMessages.from_row(row)
             yield(msg)
           end
         end
 
         def each_with_client_errors
           dataset.where { (processed_at !~ nil) & (client_error_message !~ nil) }.each do |row|
-            msg = build MessagesMapper.from_row(row)
+            msg = build Mappers::InboxMessages.from_row(row)
             yield(msg)
           end
         end

data/lib/cyclone_lariat/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CycloneLariat
-  VERSION = '1.0.0.rc5'
+  VERSION = '1.0.0.rc6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cyclone_lariat
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc5
+  version: 1.0.0.rc6
 platform: ruby
 authors:
 - Alexander Kudrin
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-30 00:00:00.000000000 Z
+date: 2023-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
@@ -400,16 +400,27 @@ files:
 - lib/cyclone_lariat/middleware.rb
 - lib/cyclone_lariat/migration.rb
 - lib/cyclone_lariat/options.rb
+- lib/cyclone_lariat/plugins/outbox.rb
+- lib/cyclone_lariat/plugins/outbox/configurable.rb
+- lib/cyclone_lariat/plugins/outbox/extensions/active_record_outbox.rb
+- lib/cyclone_lariat/plugins/outbox/extensions/sequel_outbox.rb
+- lib/cyclone_lariat/plugins/outbox/loadable.rb
+- lib/cyclone_lariat/plugins/outbox/mappers/messages.rb
+- lib/cyclone_lariat/plugins/outbox/repo/active_record/messages.rb
+- lib/cyclone_lariat/plugins/outbox/repo/messages.rb
+- lib/cyclone_lariat/plugins/outbox/repo/sequel/messages.rb
+- lib/cyclone_lariat/plugins/outbox/services/resend.rb
 - lib/cyclone_lariat/presenters/graph.rb
 - lib/cyclone_lariat/presenters/queues.rb
 - lib/cyclone_lariat/presenters/subscriptions.rb
 - lib/cyclone_lariat/presenters/topics.rb
 - lib/cyclone_lariat/publisher.rb
-- lib/cyclone_lariat/repo/active_record/messages.rb
+- lib/cyclone_lariat/repo/active_record/inbox_messages.rb
 - lib/cyclone_lariat/repo/active_record/versions.rb
-- lib/cyclone_lariat/repo/messages.rb
-- lib/cyclone_lariat/repo/messages_mapper.rb
-- lib/cyclone_lariat/repo/sequel/messages.rb
+- lib/cyclone_lariat/repo/inbox_messages.rb
+- lib/cyclone_lariat/repo/mappers/base.rb
+- lib/cyclone_lariat/repo/mappers/inbox_messages.rb
+- lib/cyclone_lariat/repo/sequel/inbox_messages.rb
 - lib/cyclone_lariat/repo/sequel/versions.rb
 - lib/cyclone_lariat/repo/versions.rb
 - lib/cyclone_lariat/resources/queue.rb

data/lib/cyclone_lariat/repo/messages_mapper.rb

@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-module CycloneLariat
-  module Repo
-    class MessagesMapper
-      class << self
-        def from_row(row)
-          return if row.nil?
-
-          row[:data] = hash_from_json_column(row[:data])
-          row[:client_error_details] = hash_from_json_column(row[:client_error_details]) if row[:client_error_details]
-          row
-        end
-
-        def to_row(input)
-          {
-            uuid: input.uuid,
-            kind: input.kind,
-            type: input.type,
-            publisher: input.publisher,
-            data: JSON.generate(input.data),
-            client_error_message: input.client_error&.message,
-            client_error_details: JSON.generate(input.client_error&.details),
-            version: input.version,
-            sent_at: input.sent_at
-          }
-        end
-
-        private
-
-        def hash_from_json_column(data)
-          return data if data.is_a?(Hash)
-          return JSON.parse(data) if data.is_a?(String)
-
-          if pg_json_extension_enabled?
-            return data.to_h             if data.is_a?(::Sequel::Postgres::JSONHash)
-            return JSON.parse(data.to_s) if data.is_a?(::Sequel::Postgres::JSONString)
-          end
-
-          raise ArgumentError, "Unknown type of `#{data}`"
-        end
-
-        def pg_json_extension_enabled?
-          Object.const_defined?('Sequel::Postgres::JSONHash')
-        end
-      end
-    end
-  end
-end