cyclone_lariat 0.2.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b29f194e1ee444b5621fee3ad9be7547ad612c34acf8cf451d5be0f20976cc4e
-  data.tar.gz: efdedb4530a004672b9c671c02b899fddedaa1d45b9876d28bce672f86f88812
+  metadata.gz: 0cebf79d1deb7a107446827a619897bf59364b89e8745fd767ed001bea3d6272
+  data.tar.gz: 6ff840519d31a46ab2f2ae6615176489041154213093052d5f924495dd537a20
 SHA512:
-  metadata.gz: 4c784a8c6ec54fef60a529a09d6a45f9dd32bab493e2acabdabed7e2039e743cb547e132a38522ee1b8473a1e2f21644b8d7a25d8bd1daca498e2bef5cd390ee
-  data.tar.gz: 85628b6886b4a82b639aa0f271b199e9cd85db4cbb316d4621637ca062ad996892f8c171ad32367a64f213b1ebcdd5e495e6f7cdb00f518923e87ecfe41a2610
+  metadata.gz: 45eab91fdef2c3c97d6638bf2ec0bf5d83dbaa54b18037a8245d25f2142b4582d60cd771c758f6d9300b2ed51977ce430864606f9155b2a235592470e92db539
+  data.tar.gz: 2f58eef796c7e0c52b6f3b8e4341466f7c8baf77ca3a6ff482e4435bd4aad73c04e1a667f39a11d820e774421c58b0519ce8401bdd1984ad872178a85d92a034

data/.github/workflows/gem-push.yml

@@ -0,0 +1,43 @@
+name: Ruby Gem
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+
+    - name: Publish to GPR
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+      env:
+        GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+        OWNER: ${{ github.repository_owner }}
+
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.gitignore

@@ -9,7 +9,7 @@
 /tmp/
 /.tmp/
 .byebug_history
-config/database.yml
+config/db.rb
 
 # rspec failure tracking
 .rspec_status

data/CHANGELOG.md

@@ -4,11 +4,37 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.2.0] - 2021-06-2
+## [0.3.2] - 2021-06-11
 Changed
+- Bugfix
+
+## [0.3.1] - 2021-06-11
+Changed
+- Command
+- SqsClient
+
+## [0.3.0] - 2021-06-09
+Added
+- Command
+- SqsClient
+
+Changed:
+- Client renamed to SnsClient
+- `to:` renamed to `topic:`
+
+## [0.2.3] - 2021-06-09
+Added
+- Skip on empty message with error notify
+
+## [0.2.2] - 2021-06-08
+Changed
+- Fix save to database 
+- Rename error to client error 
+
+## [0.2.1] - 2021-06-02
 - Fix can load from database if error_details is nil
 
-## [0.2.0] - 2021-06-2
+## [0.2.0] - 2021-06-02
 Added
 - Complete tests
 - Production ready

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    cyclone_lariat (0.2.1)
+    cyclone_lariat (0.3.2)
       aws-sdk-sns
+      aws-sdk-sqs
       luna_park (~> 0.11)
 
 GEM
@@ -21,6 +22,9 @@ GEM
     aws-sdk-sns (1.40.0)
       aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
+    aws-sdk-sqs (1.39.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
+      aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.3)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)

data/README.md

@@ -2,7 +2,7 @@
 
 This is gem work like middleware for [shoryuken](https://github.com/ruby-shoryuken/shoryuken). It save all events to database. And catch and produce all exceptions.  
 
-![Luna Park](docs/_imgs/lariat.jpg)
+![Cyclone lariat](docs/_imgs/lariat.jpg)
 
 
 ```ruby
@@ -15,59 +15,86 @@ gem 'cyclone_lariat', require: false
 gem 'cyclone_lariat'
 ```
 
+## Logic
 
-## Client
+![diagram](docs/_imgs/diagram.png)
 
+## Command vs Event
+Commands and events are both simple domain structures that contain solely data for reading. That means they contain no 
+behaviour or business logic.
+
+A command is an object that is sent to the domain for a state change which is handled by a command handler. They should 
+be named with a verb in an imperative mood plus the aggregate name which it operates on. Such request can be rejected 
+due to the data the command holds being invalid/inconsistent. There should be exactly 1 handler for each command. 
+Once the command has been executed, the consumer can then carry out whatever the task is depending on the output of the 
+command.
+
+An event is a statement of fact about what change has been made to the domain state. They are named with the aggregate 
+name where the change took place plus the verb past-participle. An event happens off the back of a command. 
+A command can emit any number of events. The sender of the event does not care who receives it or whether it has been 
+received at all.
+
+## SnsClient
 You can use client directly
 
 ```ruby
-require 'cyclone_lariat/client' # If require: false in Gemfile 
+require 'cyclone_lariat/sns_client' # If require: false in Gemfile 
 
-client = CycloneLariat::Client.new(
-  key:        APP_CONF.aws.key,
+client = CycloneLariat::SnsClient.new(
+  key: APP_CONF.aws.key,
   secret_key: APP_CONF.aws.secret_key,
-  region:     APP_CONF.aws.region,
-  version:    1,                          # at default 1
-  publisher:  'pilot',
-  instance:   INSTANCE                    # at default :prod
+  region: APP_CONF.aws.region,
+  version: 1, # at default 1
+  publisher: 'pilot',
+  instance: INSTANCE # at default :prod
 )
 ```
 
 You can don't define topic, and it's name will be defined automatically 
 ```ruby
                      # event_type        data                                    topic
-client.publish_event 'email_is_created', data: { mail: 'john.doe@example.com' } # prod-event-fanout-pilot-email_is_created
-client.publish_event 'email_is_removed', data: { mail: 'john.doe@example.com' } # prod-event-fanout-pilot-email_is_removed
+client.publish_event   'email_is_created', data: { mail: 'john.doe@example.com' } # prod-event-fanout-pilot-email_is_created
+client.publish_event   'email_is_removed', data: { mail: 'john.doe@example.com' } # prod-event-fanout-pilot-email_is_removed
+client.publish_command 'delete_user',      data: { mail: 'john.doe@example.com' } # prod-command-fanout-pilot-delete_user
 ```
 Or you can define it by handle. For example, if you want to send different events to same channel.
 ```ruby
                      # event_type        data                                    topic
-client.publish_event 'email_is_created', data: { mail: 'john.doe@example.com' }, to: 'prod-event-fanout-pilot-emails'
-client.publish_event 'email_is_removed', data: { mail: 'john.doe@example.com' }, to: 'prod-event-fanout-pilot-emails'
+client.publish_event   'email_is_created', data: { mail: 'john.doe@example.com' }, topic: 'prod-event-fanout-pilot-emails'
+client.publish_event   'email_is_removed', data: { mail: 'john.doe@example.com' }, topic: 'prod-event-fanout-pilot-emails'
+client.publish_command 'delete_user',      data: { mail: 'john.doe@example.com' }, topic: 'prod-command-fanout-pilot-emails'
 ```
 
 Or you can use client as Repo.
 
 ```ruby
-require 'cyclone_lariat/client' # If require: false in Gemfile
+require 'cyclone_lariat/sns_client' # If require: false in Gemfile
 
-class YourClient < CycloneLariat::Client
-  version   1
+class YourClient < CycloneLariat::SnsClient
+  version 1
   publisher 'pilot'
-  instance  'stage'
-  
+  instance 'stage'
+
   def email_is_created(mail)
-    publish event( 'email_is_created', 
-      data: { mail: mail }
-    ), 
-    to: APP_CONF.aws.fanout.emails
+    publish event('email_is_created',
+                  data: { mail: mail }
+            ),
+            to: APP_CONF.aws.fanout.emails
   end
-  
+
   def email_is_removed(mail)
-    publish event( 'email_is_removed', 
-      data: { mail: mail }
-    ), 
-    to: APP_CONF.aws.fanout.email
+    publish event('email_is_removed',
+                  data: { mail: mail }
+            ),
+            to: APP_CONF.aws.fanout.email
+  end
+
+
+  def delete_user(mail)
+    publish command('delete_user',
+                  data: { mail: mail }
+            ),
+            to: APP_CONF.aws.fanout.email
   end
 end
 
@@ -77,8 +104,40 @@ client = YourClient.new(key: APP_CONF.aws.key, secret_key: APP_CONF.aws.secret_k
 # And send topics
 client.email_is_created 'john.doe@example.com'
 client.email_is_removed 'john.doe@example.com'
+client.delete_user      'john.doe@example.com'
+```
+
+
+# SqsClient
+SqsClient is really similar to SnsClient. It can be initialized in same way:
+
+```ruby
+require 'cyclone_lariat/sns_client' # If require: false in Gemfile 
+
+client = CycloneLariat::SqsClient.new(
+  key: APP_CONF.aws.key,
+  secret_key: APP_CONF.aws.secret_key,
+  region: APP_CONF.aws.region,
+  version: 1, # at default 1
+  publisher: 'pilot',
+  instance: INSTANCE # at default :prod
+)
 ```
 
+As you see all params identity. And you can easily change your sqs-queue to sns-topic when you start work with more
+subscribes. But you should define destination.
+
+```ruby
+client.publish_event 'email_is_created', data: { mail: 'john.doe@example.com' }, dest: 'notify_service'  
+# prod-event-queue-pilot-email_is_created-notify_service
+```
+
+Or you can define topic directly:
+```ruby
+client.publish_event 'email_is_created', data: { mail: 'john.doe@example.com' }, topic: 'prod-event-fanout-pilot-emails'
+```
+
+
 # Middleware
 If you use middleware:
 - Store all events to dataset
@@ -100,7 +159,6 @@ class Receiver
                     queue: 'your_sqs_queue_name'
 
   server_middleware do |chain|
-    
     # Options dataset, errors_notifier and message_notifier is optionals.
     # If you dont define notifiers - middleware does not notify
     # If you dont define dataset - middleware does store events in db
@@ -140,18 +198,36 @@ end
 # The second one:
 Sequel.migration do
   change do
-    create_table :events do
+    create_table :async_messages do
       column   :uuid, :uuid, primary_key: true
-      String   :type,                     null: false
-      Integer  :version,                  null: false
-      String   :publisher,                null: false
-      column   :data, :json,              null: false
-      String   :error_message,            null: true,  default: nil
-      column   :error_details, :json,     null: true,  default: nil
-      DateTime :sent_at,                  null: true,  default: nil
-      DateTime :received_at,              null: false, default: Sequel::CURRENT_TIMESTAMP
-      DateTime :processed_at,             null: true,  default: nil
+      String   :type,                         null: false
+      Integer  :version,                      null: false
+      String   :publisher,                    null: false
+      column   :data, :json,                  null: false
+      String   :client_error_message,         null: true,  default: nil
+      column   :client_error_details, :json,  null: true,  default: nil
+      DateTime :sent_at,                      null: true,  default: nil
+      DateTime :received_at,                  null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :processed_at,                 null: true,  default: nil
     end
   end
 end
 ```
+
+### Rake tasks
+
+For simplify write some Rake tasks you can use CycloneLariat::Repo.
+
+```ruby
+# For retry all unprocessed
+
+CycloneLariat.new(DB[:events]).each_unprocessed do |event|
+  # Your logic here
+end
+
+# For retry all events with client errors
+
+CycloneLariat.new(DB[:events]).each_with_client_errors do |event|
+  # Your logic here
+end
+```

data/cyclone_lariat.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.
   if spec.respond_to?(:metadata)
-    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+    # spec.metadata['allowed_push_host'] = 'https://rubygems.org'
   else
     raise 'RubyGems 2.0 or newer is required to protect against ' \
       'public gem pushes.'
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'aws-sdk-sns'
+  spec.add_dependency 'aws-sdk-sqs'
   spec.add_dependency 'luna_park', '~> 0.11'
 
   spec.add_development_dependency 'bundler', '~> 2.1'

data/db/migrate/02_add_events.rb

@@ -2,17 +2,18 @@
 
 Sequel.migration do
   change do
-    create_table :events do
+    create_table :async_messages do
       column   :uuid, :uuid, primary_key: true
-      String   :type,                     null: false
-      Integer  :version,                  null: false
-      String   :publisher,                null: false
-      column   :data, :json,              null: false
-      String   :error_message,            null: true,  default: nil
-      column   :error_details, :json,     null: true,  default: nil
-      DateTime :sent_at,                  null: true,  default: nil
-      DateTime :received_at,              null: false, default: Sequel::CURRENT_TIMESTAMP
-      DateTime :processed_at,             null: true,  default: nil
+      String   :kind,                        null: false
+      String   :type,                        null: false
+      Integer  :version,                     null: false
+      String   :publisher,                   null: false
+      column   :data, :json,                 null: false
+      String   :client_error_message,        null: true,  default: nil
+      column   :client_error_details, :json, null: true,  default: nil
+      DateTime :sent_at,                     null: true,  default: nil
+      DateTime :received_at,                 null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :processed_at,                null: true,  default: nil
     end
   end
 end

data/lib/cyclone_lariat.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative 'cyclone_lariat/client'
+require_relative 'cyclone_lariat/sns_client'
 require_relative 'cyclone_lariat/errors'
 require_relative 'cyclone_lariat/event'
-require_relative 'cyclone_lariat/events_repo'
+require_relative 'cyclone_lariat/messages_repo'
 require_relative 'cyclone_lariat/middleware'
 require_relative 'cyclone_lariat/version'
 

data/lib/cyclone_lariat/abstract/client.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# require 'aws-sdk-sns'
+require 'luna_park/extensions/injector'
+require_relative '../event'
+require_relative '../command'
+require_relative '../errors'
+
+module CycloneLariat
+  module Abstract
+    class Client
+      include LunaPark::Extensions::Injector
+
+      dependency(:aws_client_class)      { raise ArgumentError, 'Client class should be defined' }
+      dependency(:aws_credentials_class) { Aws::Credentials }
+
+      DEFAULT_VERSION  = 1
+      DEFAULT_INSTANCE = :prod
+
+      def initialize(key:, secret_key:, region:, version: nil, publisher: nil, instance: nil)
+        @key = key
+        @secret_key = secret_key
+        @region = region
+        @version = version
+        @publisher = publisher
+        @instance = instance
+      end
+
+      def event(type, data: {}, version: self.version, uuid: SecureRandom.uuid)
+        Event.wrap(
+          uuid: uuid,
+          type: type,
+          sent_at: Time.now.iso8601,
+          version: version,
+          publisher: publisher,
+          data: data
+        )
+      end
+
+      def command(type, data: {}, version: self.version, uuid: SecureRandom.uuid)
+        Command.wrap(
+          uuid: uuid,
+          type: type,
+          sent_at: Time.now.iso8601,
+          version: version,
+          publisher: publisher,
+          data: data
+        )
+      end
+
+      def publish
+        raise LunaPark::Errors::AbstractMethod, 'Publish method should be defined'
+      end
+
+      class << self
+        def version(version = nil)
+          version.nil? ? @version || DEFAULT_VERSION : @version = version
+        end
+
+        def instance(instance = nil)
+          instance.nil? ? @instance || DEFAULT_INSTANCE : @instance = instance
+        end
+
+        def publisher(publisher = nil)
+          publisher.nil? ? @publisher || (raise 'You should define publisher') : @publisher = publisher
+        end
+      end
+
+      def version
+        @version ||= self.class.version
+      end
+
+      def publisher
+        @publisher ||= self.class.publisher
+      end
+
+      def instance
+        @instance ||= self.class.instance
+      end
+
+      private
+
+      attr_reader :key, :secret_key, :region
+
+      def aws_client
+        @aws_client ||= aws_client_class.new(credentials: aws_credentials, region: region)
+      end
+
+      def aws_credentials
+        @aws_credentials ||= aws_credentials_class.new(key, secret_key)
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/abstract/message.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'luna_park/entities/attributable'
+require_relative '../errors'
+
+module CycloneLariat
+  module Abstract
+    class Message < LunaPark::Entities::Attributable
+      attr :uuid,      String, :new
+      attr :publisher, String, :new
+      attr :type,      String, :new
+      attrs :client_error, :version, :data,
+            :sent_at, :processed_at, :received_at
+
+      def kind
+        raise LunaPark::Errors::AbstractMethod
+      end
+
+      def version=(value)
+        @version = Integer(value)
+      end
+
+      def sent_at=(value)
+        @sent_at = wrap_time(value)
+      end
+
+      def received_at=(value)
+        @received_at = wrap_time(value)
+      end
+
+      def processed_at=(value)
+        @processed_at = wrap_time(value)
+      end
+
+      def processed?
+        !@processed_at.nil?
+      end
+
+      def client_error_message=(txt)
+        return unless txt
+
+        @client_error ||= Errors::ClientError.new
+        @client_error.message = txt
+      end
+
+      def client_error_details=(details)
+        return unless details
+
+        @client_error ||= Errors::ClientError.new
+        @client_error.details = details
+      end
+
+      def ==(other)
+        kind == other.kind &&
+          uuid == other.uuid &&
+          publisher == other.publisher &&
+          type == other.type &&
+          client_error&.message == other.client_error&.message &&
+          client_error&.details == other.client_error&.details &&
+          version == other.version &&
+          sent_at.to_i == other.sent_at.to_i &&
+          received_at.to_i == other.received_at.to_i
+        processed_at.to_i == other.processed_at.to_i
+      end
+
+      def to_json(*args)
+        hash = serialize
+        hash[:type] = [kind, hash[:type]].join '_'
+        hash.to_json(*args)
+      end
+
+      private
+
+      def wrap_time(value)
+        case value
+        when String   then Time.parse(value)
+        when Time     then value
+        when NilClass then nil
+        else raise ArgumentError, "Unknown type `#{value.class}`"
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/command.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative 'abstract/message'
+
+module CycloneLariat
+  class Command < Abstract::Message
+    KIND = 'command'
+
+    def kind
+      KIND
+    end
+  end
+end

data/lib/cyclone_lariat/errors.rb

@@ -5,11 +5,15 @@ require 'luna_park/errors/business'
 
 module CycloneLariat
   module Errors
+    class EmptyMessage < LunaPark::Errors::System
+      message 'Received message is empty'
+    end
+
     class TopicNotFound < LunaPark::Errors::System
       message { |d| "Could not found topic: `#{d[:expected_topic]}`" }
     end
 
-    class ProcessingEventLogic < LunaPark::Errors::Business
+    class ClientError < LunaPark::Errors::Business
       attr_writer :message, :details
 
       def ==(other)

data/lib/cyclone_lariat/event.rb

@@ -1,81 +1,13 @@
 # frozen_string_literal: true
 
-require 'luna_park/entities/attributable'
-require_relative 'errors'
+require_relative 'abstract/message'
 
 module CycloneLariat
-  class Event < LunaPark::Entities::Attributable
+  class Event < Abstract::Message
     KIND = 'event'
 
-    attr :uuid,      String, :new
-    attr :publisher, String, :new
-    attr :type,      String, :new
-    attr :error
-    attr :version
-    attr :data
-
-    attr_reader :sent_at,
-                :processed_at,
-                :received_at
-
     def kind
       KIND
     end
-
-    def version=(value)
-      @version = Integer(value)
-    end
-
-    def sent_at=(value)
-      @sent_at = wrap_time(value)
-    end
-
-    def received_at=(value)
-      @received_at = wrap_time(value)
-    end
-
-    def processed_at=(value)
-      @processed_at = wrap_time(value)
-    end
-
-    def error_message=(txt)
-      @error ||= Errors::ProcessingEventLogic.new
-      @error.message = txt
-    end
-
-    def error_details=(details)
-      @error ||= Errors::ProcessingEventLogic.new
-      @error.details = details
-    end
-
-    def ==(other)
-      kind == other.kind &&
-        uuid == other.uuid &&
-        publisher == other.publisher &&
-        type == other.type &&
-        error&.message == other.error&.message &&
-        error&.details == other.error&.details &&
-        version == other.version &&
-        sent_at.to_i == other.sent_at.to_i &&
-        received_at.to_i == other.received_at.to_i
-      processed_at.to_i == other.processed_at.to_i
-    end
-
-    def to_json(*args)
-      hash = serialize
-      hash[:type] = [kind, hash[:type]].join '_'
-      hash.to_json(*args)
-    end
-
-    private
-
-    def wrap_time(value)
-      case value
-      when String   then Time.parse(value)
-      when Time     then value
-      when NilClass then nil
-      else raise ArgumentError, "Unknown type `#{value.class}`"
-      end
-    end
   end
 end

data/lib/cyclone_lariat/messages_repo.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+
+module CycloneLariat
+  class MessagesRepo
+    attr_reader :dataset
+
+    def initialize(dataset)
+      @dataset = dataset
+    end
+
+    def create(msg)
+      dataset.insert(
+        uuid: msg.uuid,
+        kind: msg.kind,
+        type: msg.type,
+        publisher: msg.publisher,
+        data: JSON.generate(msg.data),
+        client_error_message: msg.client_error&.message,
+        client_error_details: JSON.generate(msg.client_error&.details),
+        version: msg.version,
+        sent_at: msg.sent_at
+      )
+    end
+
+    def exists?(uuid:)
+      dataset.where(uuid: uuid).limit(1).any?
+    end
+
+    def processed!(uuid:, error: nil)
+      data = { processed_at: Sequel.function(:NOW) }
+      data.merge!(client_error_message: error.message, client_error_details: JSON.generate(error.details)) if error
+
+      !dataset.where(uuid: uuid).update(data).zero?
+    end
+
+    def find(uuid:)
+      raw = dataset.where(uuid: uuid).first
+      return nil unless raw
+
+      raw[:data] = JSON.parse(raw[:data], symbolize_names: true)
+      if raw[:client_error_details]
+        raw[:client_error_details] = JSON.parse(raw[:client_error_details], symbolize_names: true)
+      end
+      build raw
+    end
+
+    def each_unprocessed
+      dataset.where(processed_at: nil).each do |raw|
+        raw[:data]                 = JSON.parse(raw[:data], symbolize_names: true)
+        if raw[:client_error_details]
+          raw[:client_error_details] = JSON.parse(raw[:client_error_details], symbolize_names: true)
+        end
+        msg = build raw
+        yield(msg)
+      end
+    end
+
+    def each_with_client_errors
+      dataset.where { (processed_at !~ nil) & (client_error_message !~ nil) }.each do |raw|
+        raw[:data] = JSON.parse(raw[:data], symbolize_names: true)
+        if raw[:client_error_details]
+          raw[:client_error_details] = JSON.parse(raw[:client_error_details], symbolize_names: true)
+        end
+        msg = build raw
+        yield(msg)
+      end
+    end
+
+    private
+
+    def build(raw)
+      case kind = raw.delete(:kind)
+      when 'event'   then Event.wrap raw
+      when 'command' then Command.wrap raw
+      else raise ArgumentError, "Unknown kind `#{kind}` of message"
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/middleware.rb

@@ -1,25 +1,27 @@
 # frozen_string_literal: true
 
-require_relative 'events_repo'
+require_relative 'messages_repo'
 require 'luna_park/errors'
 require 'json'
 
 module CycloneLariat
   class Middleware
-    def initialize(dataset: nil, errors_notifier: nil, message_notifier: nil, repo: EventsRepo)
+    def initialize(dataset: nil, errors_notifier: nil, message_notifier: nil, repo: MessagesRepo)
       @events_repo      = repo.new(dataset) if dataset
       @message_notifier = message_notifier
       @errors_notifier  = errors_notifier
     end
 
     def call(_worker_instance, queue, _sqs_msg, body, &block)
-      log_received_message queue, body
+      msg = receive_message(body)
 
-      catch_standard_error(queue, body) do
-        event = Event.wrap(JSON.parse(body[:Message]))
+      message_notifier&.info 'Receive message', message: msg, queue: queue
 
-        catch_business_error(event) do
-          store_in_dataset(event, &block)
+      catch_standard_error(queue, msg) do
+        event = Event.wrap(msg)
+
+        store_in_dataset(event) do
+          catch_business_error(event, &block)
         end
       end
     end
@@ -28,29 +30,32 @@ module CycloneLariat
 
     attr_reader :errors_notifier, :message_notifier, :events_repo
 
-    def log_received_message(queue, body)
-      message_notifier&.info 'Receive message', queue: queue, aws_message_id: body[:MessageId], message: body[:Message]
+    def receive_message(body)
+      body[:Message] ? JSON.parse(body[:Message], symbolize_names: true ) : body
     end
 
     def store_in_dataset(event)
       return yield if events_repo.nil?
-      return true  if events_repo.exists?(uuid: event.uuid)
 
-      events_repo.create(event)
+      existed = events_repo.find(uuid: event.uuid)
+      return true if existed&.processed?
+
+      events_repo.create(event) unless existed
       yield
-      events_repo.processed! uuid: event.uuid
+      events_repo.processed! uuid: event.uuid, error: event.client_error
     end
 
     def catch_business_error(event)
       yield
     rescue LunaPark::Errors::Business => e
-      errors_notifier&.warning(e, event: event)
+      errors_notifier&.error(e, event: event)
+      event.client_error = e
     end
 
-    def catch_standard_error(queue, body)
+    def catch_standard_error(queue, msg)
       yield
-    rescue StandardError => e
-      errors_notifier&.error(e, queue: queue, aws_message_id: body[:MessageId], message: body[:Message])
+    rescue Exception => e
+      errors_notifier&.error(e, queue: queue, message: msg)
       raise e
     end
   end

data/lib/cyclone_lariat/sns_client.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sns'
+require_relative 'abstract/client'
+
+module CycloneLariat
+  class SnsClient < Abstract::Client
+    include LunaPark::Extensions::Injector
+
+    dependency(:aws_client_class) { Aws::SNS::Client }
+
+    SNS_SUFFIX = :fanout
+
+    def publish(msg, topic: nil)
+      topic ||= [instance, msg.kind, SNS_SUFFIX, publisher, msg.type].join('-')
+
+      aws_client.publish(
+        topic_arn: topic_arn(topic),
+        message: msg.to_json
+      )
+    end
+
+    def publish_event(type, data: {}, version: self.version, uuid: SecureRandom.uuid, topic: nil)
+      publish event(type, data: data, version: version, uuid: uuid), topic: topic
+    end
+
+    def publish_command(type, data: {}, version: self.version, uuid: SecureRandom.uuid, topic: nil)
+      publish command(type, data: data, version: version, uuid: uuid), topic: topic
+    end
+
+    private
+
+    def topic_arn(topic_name)
+      list  = aws_client.list_topics.topics
+      topic = list.find { |t| t.topic_arn.match?(topic_name) }
+      raise Errors::TopicNotFound.new(expected_topic: topic_name, existed_topics: list.map(&:topic_arn)) if topic.nil?
+
+      topic.topic_arn
+    end
+  end
+end

data/lib/cyclone_lariat/sqs_client.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require_relative 'abstract/client'
+
+module CycloneLariat
+  class SqsClient < Abstract::Client
+    include LunaPark::Extensions::Injector
+
+    dependency(:aws_client_class) { Aws::SQS::Client }
+
+    SQS_SUFFIX = :queue
+
+    def publish(msg, dest: nil, topic: nil)
+      raise ArgumentError, 'You should define dest or topic' if dest.nil? && topic.nil?
+
+      topic ||= [instance, msg.kind, SQS_SUFFIX, publisher, msg.type, dest].join('-')
+
+      aws_client.send_message(
+        queue_url: url(topic),
+        message_body: msg.to_json
+      )
+    end
+
+    def publish_event(type, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, topic: nil)
+      publish event(type, data: data, version: version, uuid: uuid), dest: dest, topic: topic
+    end
+
+    def publish_command(type, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, topic: nil)
+      publish command(type, data: data, version: version, uuid: uuid), dest: dest, topic: topic
+    end
+
+    private
+
+    def url(topic_name)
+      aws_client.get_queue_url(queue_name: topic_name).queue_url
+    rescue Aws::SQS::Errors::NonExistentQueue => _e
+      raise Errors::TopicNotFound.new(expected_topic: topic_name, existed_topics: aws_client.list_queues)
+    end
+  end
+end

data/lib/cyclone_lariat/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CycloneLariat
-  VERSION = '0.2.1'
+  VERSION = '0.3.2'
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: cyclone_lariat
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Alexander Kudrin
 - Philip Sorokin
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-02 00:00:00.000000000 Z
+date: 2021-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
@@ -25,6 +25,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: luna_park
   requirement: !ruby/object:Gem::Requirement
@@ -235,13 +249,14 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-description:
+description: 
 email:
 - kudrin.alexander@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/gem-push.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
@@ -253,26 +268,29 @@ files:
 - README.md
 - Rakefile
 - config/db.example.rb
-- config/db.rb
 - cyclone_lariat.gemspec
 - db/migrate/01_add_uuid_extensions.rb
 - db/migrate/02_add_events.rb
+- docs/_imgs/diagram.png
 - docs/_imgs/lariat.jpg
 - lib/cyclone_lariat.rb
-- lib/cyclone_lariat/client.rb
+- lib/cyclone_lariat/abstract/client.rb
+- lib/cyclone_lariat/abstract/message.rb
+- lib/cyclone_lariat/command.rb
 - lib/cyclone_lariat/errors.rb
 - lib/cyclone_lariat/event.rb
-- lib/cyclone_lariat/events_repo.rb
+- lib/cyclone_lariat/messages_repo.rb
 - lib/cyclone_lariat/middleware.rb
+- lib/cyclone_lariat/sns_client.rb
+- lib/cyclone_lariat/sqs_client.rb
 - lib/cyclone_lariat/version.rb
 - lib/tasks/db.rake
 homepage: https://am-team.github.io/cyclone_lariat/#/
 licenses:
 - MIT
 metadata:
-  allowed_push_host: https://rubygems.org
   yard.run: yri
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -287,8 +305,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: Shoryuken middleware for LunaPark based application.
 test_files: []

data/config/db.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-DB_CONF = {
-  adapter: 'postgresql',
-  host: 'localhost',
-  username: 'ruby',
-  password: 'ruby',
-  database: 'cyclone-lariat-test'
-}.freeze

data/lib/cyclone_lariat/client.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-require 'aws-sdk-sns'
-require 'luna_park/extensions/injector'
-require_relative 'event'
-require_relative 'errors'
-
-module CycloneLariat
-  class Client
-    include LunaPark::Extensions::Injector
-
-    dependency(:aws_sns_client_class)  { Aws::SNS::Client }
-    dependency(:aws_credentials_class) { Aws::Credentials }
-
-    DEFAULT_VERSION  = 1
-    DEFAULT_INSTANCE = :prod
-    SNS_SUFFIX       = :fanout
-
-    def initialize(key:, secret_key:, region:, version: nil, publisher: nil, instance: nil)
-      @key = key
-      @secret_key = secret_key
-      @region = region
-      @version = version
-      @publisher = publisher
-      @instance = instance
-    end
-
-    def event(type, data: {}, version: self.version, uuid: SecureRandom.uuid)
-      Event.wrap(
-        uuid: uuid,
-        type: type,
-        sent_at: Time.now.iso8601,
-        version: version,
-        publisher: publisher,
-        data: data
-      )
-    end
-
-    def publish(msg, to: nil)
-      topic = to || [instance, msg.kind, SNS_SUFFIX, publisher, msg.type].join('-')
-
-      aws_client.publish(
-        topic_arn: topic_arn(topic),
-        message: msg.to_json
-      )
-    end
-
-    def publish_event(type, data: {}, version: self.version, uuid: SecureRandom.uuid, to: nil)
-      publish event(type, data: data, version: version, uuid: uuid), to: to
-    end
-
-    class << self
-      def version(version = nil)
-        version.nil? ? @version || DEFAULT_VERSION : @version = version
-      end
-
-      def instance(instance = nil)
-        instance.nil? ? @instance || DEFAULT_INSTANCE : @instance = instance
-      end
-
-      def publisher(publisher = nil)
-        publisher.nil? ? @publisher || (raise 'You should define publisher') : @publisher = publisher
-      end
-    end
-
-    def version
-      @version ||= self.class.version
-    end
-
-    def publisher
-      @publisher ||= self.class.publisher
-    end
-
-    def instance
-      @instance ||= self.class.instance
-    end
-
-    private
-
-    attr_reader :key, :secret_key, :region
-
-    def topic_arn(topic_name)
-      list  = aws_client.list_topics.topics
-      topic = list.find { |t| t.topic_arn.match?(topic_name) }
-      raise Errors::TopicNotFound.new(expected_topic: topic_name, existed_topics: list.map(&:topic_arn)) if topic.nil?
-
-      topic.topic_arn
-    end
-
-    def aws_client
-      @aws_client ||= aws_sns_client_class.new(credentials: aws_credentials, region: region)
-    end
-
-    def aws_credentials
-      @aws_credentials ||= aws_credentials_class.new(key, secret_key)
-    end
-  end
-end

data/lib/cyclone_lariat/events_repo.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'event'
-
-module CycloneLariat
-  class EventsRepo
-    attr_reader :dataset
-
-    def initialize(dataset)
-      @dataset = dataset
-    end
-
-    def create(event)
-      dataset.insert(
-        uuid: event.uuid,
-        type: event.type,
-        publisher: event.publisher,
-        data: JSON.generate(event.data),
-        error_message: event.error&.message,
-        error_details: JSON.generate(event.error&.details),
-        version: event.version,
-        sent_at: event.sent_at
-      )
-    end
-
-    def exists?(uuid:)
-      dataset.where(uuid: uuid).limit(1).any?
-    end
-
-    def processed!(uuid:)
-      !dataset.where(uuid: uuid).update(processed_at: Sequel.function(:NOW)).zero?
-    end
-
-    def find(uuid:)
-      raw = dataset.where(uuid: uuid).first
-      raw[:data]          = JSON.parse(raw[:data], symbolize_names: true)
-      raw[:error_details] = JSON.parse(raw[:error_details], symbolize_names: true) if raw[:error_details]
-      Event.wrap raw
-    end
-
-    def each_unprocessed
-      dataset.where(processed_at: nil).each do |raw|
-        raw[:data]          = JSON.parse(raw[:data], symbolize_names: true)
-        raw[:error_details] = JSON.parse(raw[:error_details], symbolize_names: true) if raw[:error_details]
-        event = Event.wrap(raw)
-        yield(event)
-      end
-    end
-  end
-end