cyclone_lariat 0.4.0 → 1.0.0.rc2

data/bin/cyclone_lariat

@@ -24,13 +24,25 @@ module CycloneLariat
 
       class Install < Dry::CLI::Command
         desc 'Install cyclone lariat to current directory'
+        option :adapter,
+          default: 'sequel',
+          values: %w[sequel active_record],
+          desc: 'adapter for store events and versions'
 
-        def call(*)
-          create_config
+        def call(adapter: 'sequel', **)
+          create_config(adapter)
           create_rake_task
         end
 
-        def create_config
+        def create_config(adapter)
+          FileUtils.mkdir_p INITIALIZERS_DIR unless Dir.exist? INITIALIZERS_DIR
+          config_path = "#{INITIALIZERS_DIR}/cyclone_lariat.rb"
+          config_file = File.open(config_path, 'w')
+          config_file.puts config_contents(adapter)
+          puts "Created config: #{config_path}"
+        end
+
+        def create_rake_task
           FileUtils.mkdir_p RAKE_TASKS_DIR unless Dir.exist? RAKE_TASKS_DIR
           config_path = "#{RAKE_TASKS_DIR}/cyclone_lariat.rake"
           config_file = File.open(config_path, 'w')
@@ -38,28 +50,49 @@ module CycloneLariat
           puts "Created rake task: #{config_path}"
         end
 
-        def create_rake_task
-          FileUtils.mkdir_p INITIALIZERS_DIR unless Dir.exist? INITIALIZERS_DIR
-          config_path = "#{INITIALIZERS_DIR}/cyclone_lariat.rb"
-          config_file = File.open(config_path, 'w')
-          config_file.puts config_contents
-          puts "Created config: #{config_path}"
+        def config_contents(adapter)
+          return config_active_record_contents if adapter == 'active_record'
+          return config_sequel_contents if adapter == 'sequel'
+
+          raise ArgumentError, "Unknown adapter #{adapter}"
         end
 
-        def config_contents
+        def config_sequel_contents
           <<~CONFIG
             # frozen_string_literal: true
 
-            CycloneLariat.tap do |cl|
-              cl.default_version    = 1                       # api version
-              cl.aws_key            = ENV['AWS_KEY']          # aws key
-              cl.aws_account_id     = ENV['AWS_ACCOUNT_ID']   # aws account id
-              cl.aws_secret_key     = ENV['AWS_SECRET_KEY']   # aws secret
-              cl.aws_default_region = ENV['AWS_REGION']       # aws default region
-              cl.publisher          = ENV['APP_NAME']         # name of your publishers, usually name of your application 
-              cl.default_instance   = ENV['INSTANCE']         # stage, production, test
-              cl.events_dataset     = DB[:events]             # sequel dataset for store income messages, for receiver
-              cl.versions_dataset   = DB[:lariat_versions]    # sequel dataset for migrations, for publisher
+            CycloneLariat.configure do |c|
+              c.version            = 1                         # messages version
+              c.aws_key            = ENV['AWS_KEY']            # aws key
+              c.aws_account_id     = ENV['AWS_ACCOUNT_ID']     # aws account id
+              c.aws_secret_key     = ENV['AWS_SECRET_KEY']     # aws secret
+              c.aws_region         = ENV['AWS_REGION']         # aws default region
+              c.publisher          = ENV['APP_NAME']           # name of your publishers, usually name of your application
+              c.instance           = ENV['INSTANCE']           # stage, production, test
+              c.driver             = :sequel                   # :sequel or :active_record
+              c.messages_dataset   = DB[:messages]             # Sequel dataset / ActiveRecord model for store income messages (on receiver)
+              c.versions_dataset   = DB[:lariat_versions]      # Sequel dataset / ActiveRecord model for publisher migrations
+              c.fake_publish       = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
+            end
+          CONFIG
+        end
+
+        def config_active_record_contents
+          <<~CONFIG
+            # frozen_string_literal: true
+
+            CycloneLariat.configure do |c|
+              c.version            = 1                         # messages version
+              c.aws_key            = ENV['AWS_KEY']            # aws key
+              c.aws_account_id     = ENV['AWS_ACCOUNT_ID']     # aws account id
+              c.aws_secret_key     = ENV['AWS_SECRET_KEY']     # aws secret
+              c.aws_region         = ENV['AWS_REGION']         # aws default region
+              c.publisher          = ENV['APP_NAME']           # name of your publishers, usually name of your application
+              c.instance           = ENV['INSTANCE']           # stage, production, test
+              c.driver             = :active_record            # :sequel or :active_record
+              c.messages_dataset   = CycloneLariatMessage      # Sequel dataset / ActiveRecord model for store income messages (on receiver)
+              c.versions_dataset   = CycloneLariatVersion      # Sequel dataset / ActiveRecord model for publisher migrations
+              c.fake_publish       = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
             end
           CONFIG
         end
@@ -69,46 +102,44 @@ module CycloneLariat
             # frozen_string_literal: true
 
             require 'cyclone_lariat'
-            
+
             namespace :cyclone_lariat do
               desc 'Migrate topics for SQS/SNS'
-              task migrate: :config do
-                require_relative '../../config/initializers/cyclone_lariat'
+              task migrate: :cyclone_lariat_config do
                 CycloneLariat::Migration.migrate
               end
-            
+
               desc 'Rollback topics for SQS/SNS'
-              task :rollback, [:version] => :config do |_, args|
-                require_relative '../../config/initializers/cyclone_lariat'
+              task :rollback, [:version] => :cyclone_lariat_config do |_, args|
                 target_version = args[:version] ? args[:version].to_i : nil
                 CycloneLariat::Migration.rollback(target_version)
               end
-            
+
               namespace :list do
                 desc 'List all topics'
-                task :topics do
-                  require_relative '../../config/initializers/cyclone_lariat'
+                task topics: :cyclone_lariat_config do
                   CycloneLariat::Migration.list_topics
                 end
-            
+
                 desc 'List all queues'
-                task :queues do
-                  require_relative '../../config/initializers/cyclone_lariat'
+                task queues: :cyclone_lariat_config do
                   CycloneLariat::Migration.list_queues
                 end
-            
+
                 desc 'List all subscriptions'
-                task :subscriptions do
-                  require_relative '../../config/initializers/cyclone_lariat'
+                task subscriptions: :cyclone_lariat_config do
                   CycloneLariat::Migration.list_subscriptions
                 end
               end
-            
+
               desc 'Build graphviz graph for whole system'
-              task :graph do
-                require_relative '../../config/initializers/cyclone_lariat'
+              task graph: :cyclone_lariat_config do
                 CycloneLariat::Migration.build_graph
               end
+
+              task :cyclone_lariat_config do
+                require_relative '../../config/initializers/cyclone_lariat'
+              end
             end
           TASKS
         end
@@ -125,10 +156,11 @@ module CycloneLariat
 
             FileUtils.mkdir_p CycloneLariat::Migration::DIR unless Dir.exist? CycloneLariat::Migration::DIR
 
-            file_name  = generate_filename title
-            class_name = generate_class_name title
-            file       = File.open(file_name, 'w')
-            file.puts file_contents(class_name)
+            file_name = generate_filename(title)
+            class_name = generate_class_name(title)
+
+            file = File.open(file_name, 'w')
+            file.puts(file_contents(class_name))
             puts "Migration successful created:\n\t#{file_name}"
           end
 
@@ -153,7 +185,7 @@ module CycloneLariat
               class #{klass_name} < CycloneLariat::Migration
                 def up
                 end
-              
+
                 def down
                 end
               end

data/cyclone_lariat.gemspec

@@ -7,7 +7,7 @@ require 'cyclone_lariat/version'
 Gem::Specification.new do |spec|
   spec.name          = 'cyclone_lariat'
   spec.version       = CycloneLariat::VERSION
-  spec.authors       = ['Alexander Kudrin', 'Philip Sorokin']
+  spec.authors       = ['Alexander Kudrin', 'Philip Sorokin', 'Kirill Drozdov', 'Vitaly Perminov']
   spec.email         = ['kudrin.alexander@gmail.com']
 
   spec.summary       = 'Shoryuken middleware for LunaPark based application.'
@@ -28,21 +28,28 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|docs|examples|config)/}) }
   end
 
   spec.require_paths = ['lib']
-  spec.executables   = ['cyclone_lariat']
+  spec.bindir = 'bin'
+  spec.executables = ['cyclone_lariat']
 
   spec.add_dependency 'aws-sdk-sns'
   spec.add_dependency 'aws-sdk-sqs'
   spec.add_dependency 'dry-cli', '~> 0.6'
+  spec.add_dependency 'dry-validation', '~> 1.5'
   spec.add_dependency 'luna_park', '~> 0.11'
   spec.add_dependency 'terminal-table', '~> 3.0'
 
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'byebug', '~> 11.1'
+  spec.add_development_dependency 'database_cleaner-active_record'
   spec.add_development_dependency 'database_cleaner-sequel', '~> 2.0'
+  spec.add_development_dependency 'guard'
+  spec.add_development_dependency 'guard-bundler'
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'guard-rubocop'
   spec.add_development_dependency 'pg', '~> 1.2'
   spec.add_development_dependency 'pry', '~> 0.13'
   spec.add_development_dependency 'pry-byebug', '~> 3.9'

data/lib/cyclone_lariat/clients/abstract.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'luna_park/extensions/injector'
+require 'cyclone_lariat/generators/event'
+require 'cyclone_lariat/generators/command'
+require 'cyclone_lariat/errors'
+require 'cyclone_lariat/core'
+
+module CycloneLariat
+  module Clients
+    class Abstract
+      include LunaPark::Extensions::Injector
+      include Generators::Event
+      include Generators::Command
+
+      dependency(:aws_client_class)      { raise ArgumentError, 'Client class should be defined' }
+      dependency(:aws_credentials_class) { Aws::Credentials }
+
+      def initialize(**options)
+        @config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
+      end
+
+      attr_reader :config
+
+      def publish
+        raise LunaPark::Errors::AbstractMethod, 'Publish method should be defined'
+      end
+
+      private
+
+      def aws_client
+        @aws_client ||= aws_client_class.new(credentials: aws_credentials, region: config.aws_region)
+      end
+
+      def aws_credentials
+        @aws_credentials ||= aws_credentials_class.new(config.aws_key, config.aws_secret_key)
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/clients/sns.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sns'
+require 'cyclone_lariat/fake'
+require 'cyclone_lariat/clients/abstract'
+require 'cyclone_lariat/resources/topic'
+require 'cyclone_lariat/resources/queue'
+
+module CycloneLariat
+  module Clients
+    class Sns < Abstract
+      include LunaPark::Extensions::Injector
+      include Generators::Topic
+
+      dependency(:aws_client_class) { Aws::SNS::Client }
+
+      def publish(msg, fifo:, topic: nil, skip_validation: false)
+        return Fake.sns_publish_response(msg) if config.fake_publish
+
+        topic = topic ? custom_topic(topic) : topic(msg.type, kind: msg.kind, fifo: fifo)
+
+        raise Errors::GroupIdUndefined.new(resource: topic)       if fifo && msg.group_id.nil?
+        raise Errors::GroupDefined.new(resource: topic)           if !fifo && msg.group_id
+        raise Errors::DeduplicationIdDefined.new(resource: topic) if !fifo && msg.deduplication_id
+
+        msg.validation.check! unless skip_validation
+
+        params = {
+          topic_arn: topic.arn,
+          message: msg.to_json,
+          message_group_id: msg.group_id,
+          message_deduplication_id: msg.deduplication_id
+        }.compact
+
+        aws_client.publish(**params)
+      end
+
+      def exists?(topic)
+        raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
+
+        aws_client.get_topic_attributes({ topic_arn: topic.arn }) && true
+      rescue Aws::SNS::Errors::NotFound
+        false
+      end
+
+      def publish_event(type, fifo:, topic: nil, **options)
+        options[:version] ||= config.version
+        options[:data]    ||= {}
+        options[:uuid]    ||= SecureRandom.uuid
+
+        publish event(type, **options), fifo: fifo, topic: topic
+      end
+
+      def publish_command(type, fifo:, topic: nil, **options)
+        options[:version] ||= config.version
+        options[:data]    ||= {}
+        options[:uuid]    ||= SecureRandom.uuid
+
+        publish command(type, **options), fifo: fifo, topic: topic
+      end
+
+      def create(topic)
+        raise ArgumentError, 'Should be Resources::Topic' unless topic.is_a? Resources::Topic
+        raise Errors::TopicAlreadyExists.new(expected_topic: topic.name) if exists?(topic)
+
+        aws_client.create_topic(name: topic.name, attributes: topic.attributes, tags: topic.tags)
+        topic
+      end
+
+      def delete(topic)
+        raise ArgumentError, 'Should be Resources::Topic' unless topic.is_a? Resources::Topic
+        raise Errors::TopicDoesNotExists.new(expected_topic: topic.name) unless exists?(topic)
+
+        aws_client.delete_topic topic_arn: topic.arn
+        topic
+      end
+
+      def subscribe(topic:, endpoint:)
+        subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
+        raise Errors::SubscriptionAlreadyExists.new(topic: topic, endpoint: endpoint) if subscription_arn
+
+        aws_client.subscribe(
+          {
+            topic_arn: topic.arn,
+            protocol: endpoint.protocol,
+            endpoint: endpoint.arn
+          }
+        )
+      end
+
+      def unsubscribe(topic:, endpoint:)
+        subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
+        raise Errors::SubscriptionDoesNotExists.new(topic: topic, endpoint: endpoint) unless subscription_arn
+
+        aws_client.unsubscribe(subscription_arn: subscription_arn)
+      end
+
+      def list_all
+        topics = []
+        resp = aws_client.list_topics
+
+        loop do
+          resp[:topics].map do |t|
+            topics << Resources::Topic.from_arn(t[:topic_arn])
+          end
+
+          break if resp[:next_token].nil?
+
+          resp = aws_client.list_topics(next_token: resp[:next_token])
+        end
+        topics
+      end
+
+      def list_subscriptions
+        subscriptions = []
+        resp = aws_client.list_subscriptions
+
+        loop do
+          resp[:subscriptions].each do |s|
+            endpoint = s.endpoint.split(':')[2] == 'sqs' ? Resources::Queue.from_arn(s.endpoint) : Resources::Topic.from_arn(s.endpoint)
+            subscriptions << { topic: Resources::Topic.from_arn(s.topic_arn), endpoint: endpoint, arn: s.subscription_arn }
+          end
+
+          break if resp[:next_token].nil?
+
+          resp = aws_client.list_subscriptions(next_token: resp[:next_token])
+        end
+        subscriptions
+      end
+
+      def topic_subscriptions(topic)
+        raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
+
+        subscriptions = []
+
+        resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn)
+
+        loop do
+          next_token = resp[:next_token]
+          subscriptions += resp[:subscriptions]
+
+          break if next_token.nil?
+
+          resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn, next_token: next_token)
+        end
+        subscriptions
+      end
+
+      def find_subscription_arn(topic:, endpoint:)
+        raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
+        unless [Resources::Topic, Resources::Queue].include? endpoint.class
+          raise ArgumentError, 'Endpoint should be Topic or Queue'
+        end
+
+        found_subscription = topic_subscriptions(topic).select do |subscription|
+          subscription.endpoint == endpoint.arn
+        end.first
+
+        found_subscription ? found_subscription.subscription_arn : nil
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/clients/sqs.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'cyclone_lariat/fake'
+require 'cyclone_lariat/clients/abstract'
+require 'cyclone_lariat/resources/queue'
+require 'cyclone_lariat/generators/queue'
+
+module CycloneLariat
+  module Clients
+    class Sqs < Abstract
+      include LunaPark::Extensions::Injector
+      include Generators::Queue
+
+      dependency(:aws_client_class) { Aws::SQS::Client }
+
+      def exists?(queue)
+        raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
+
+        aws_client.get_queue_url(queue_name: queue.to_s) && true
+      rescue Aws::SQS::Errors::NonExistentQueue
+        false
+      end
+
+      def add_policy(queue:, policy:)
+        current_policy_json = aws_client.get_queue_attributes({
+          queue_url: queue.url,
+          attribute_names: ['Policy']
+        }).attributes['Policy']
+
+        current_policy = JSON.parse(current_policy_json) if current_policy_json
+
+        return if current_policy && current_policy['Statement'].find { |s| s['Sid'] == policy['Sid'] }
+
+        new_policy = current_policy || { 'Statement' => [] }
+        new_policy['Statement'] << policy
+
+        aws_client.set_queue_attributes({ queue_url: queue.url, attributes: { 'Policy' => new_policy.to_json } })
+      end
+
+      def publish(msg, fifo:, dest: nil, queue: nil, skip_validation: false)
+        return Fake.sqs_send_message_result(msg) if config.fake_publish
+
+        queue = queue ? custom_queue(queue) : queue(msg.type, kind: msg.kind, fifo: fifo, dest: dest)
+
+        raise Errors::GroupIdUndefined.new(resource: queue)       if fifo && msg.group_id.nil?
+        raise Errors::GroupDefined.new(resource: queue)           if !fifo && msg.group_id
+        raise Errors::DeduplicationIdDefined.new(resource: queue) if !fifo && msg.deduplication_id
+
+        msg.validation.check! unless skip_validation
+
+        params = {
+          queue_url: queue.url,
+          message_body: msg.to_json,
+          message_group_id: msg.group_id,
+          message_deduplication_id: msg.deduplication_id
+        }.compact
+
+        aws_client.send_message(**params)
+      end
+
+      def publish_event(type, fifo:, dest: nil, queue: nil, **options)
+        options[:version] ||= self.config.version
+        options[:data]    ||= {}
+        options[:uuid]    ||= SecureRandom.uuid
+
+        publish event(type, data: data, **options), fifo: fifo, dest: dest, queue: queue
+      end
+
+      def publish_command(type, fifo:, dest: nil, queue: nil, **options)
+        options[:version] ||= self.config.version
+        options[:data]    ||= {}
+        options[:uuid]    ||= SecureRandom.uuid
+
+        publish event(type, data: data, **options), fifo: fifo, dest: dest, queue: queue
+      end
+
+      def create(queue)
+        raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
+        raise Errors::QueueAlreadyExists.new(expected_queue: queue.name) if exists?(queue)
+
+        aws_client.create_queue(queue_name: queue.name, attributes: queue.attributes, tags: queue.tags)
+        queue
+      end
+
+      def delete(queue)
+        raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
+        raise Errors::QueueDoesNotExists.new(expected_queue: queue.name) unless exists?(queue)
+
+        aws_client.delete_queue queue_url: queue.url
+        queue
+      end
+
+      def list_all
+        queues = []
+        resp = aws_client.list_queues
+
+        loop do
+          next_token = resp[:next_token]
+
+          resp[:queue_urls].map do |url|
+            queues << Resources::Queue.from_url(url)
+          end
+
+          break if next_token.nil?
+
+          resp = aws_client.list_queues(next_token: next_token)
+        end
+
+        queues
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/core.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/generators/queue'
+require 'cyclone_lariat/generators/topic'
+require 'cyclone_lariat/options'
+
+module CycloneLariat
+  module CycloneLariatMethods
+    def config
+      @config ||= Options.new
+    end
+
+    def configure
+      yield(config)
+    end
+  end
+
+  extend Generators::Topic
+  extend Generators::Queue
+  extend CycloneLariatMethods
+end

data/lib/cyclone_lariat/errors.rb

@@ -40,5 +40,21 @@ module CycloneLariat
     class SubscriptionDoesNotExists < LunaPark::Errors::System
       message { |d| "Subscription for topic `#{d[:topic].name}`, on endpoint `#{d[:endpoint].name}` does not exists" }
     end
+
+    class InvalidMessage < LunaPark::Errors::Business
+      message 'Message is not valid'
+    end
+
+    class GroupIdUndefined < LunaPark::Errors::System
+      message { |d| "Group id must be defined for FIFO resources: `#{d[:resource].name}`" }
+    end
+
+    class GroupDefined < LunaPark::Errors::System
+      message { |d| "Group id must be nil for non-FIFO resources: `#{d[:resource].name}`" }
+    end
+
+    class DeduplicationIdDefined < LunaPark::Errors::System
+      message { |d| "Deduplication id must be nil for non-FIFO resources: `#{d[:resource].name}`" }
+    end
   end
 end

data/lib/cyclone_lariat/fake.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  class Fake
+    def self.sns_publish_response(message)
+      Aws::SNS::Types::PublishResponse.new.tap do |resp|
+        resp.message_id = SecureRandom.uuid
+        resp.sequence_number = rand(10).to_s if message.fifo?
+      end
+    end
+
+    def self.sqs_send_message_result(message)
+      Aws::SQS::Types::SendMessageResult.new.tap do |res|
+        res.message_id = SecureRandom.uuid
+        res.sequence_number = rand(10).to_s if message.fifo?
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/generators/command.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'cyclone_lariat/messages/v1/command'
+require 'cyclone_lariat/messages/v2/command'
+
+module CycloneLariat
+  module Generators
+    module Command
+      def command(type, version: config.version, **options)
+        case version.to_i
+        when 1 then command_v1(type, **options)
+        when 2 then command_v2(type, **options)
+        else raise ArgumentError, "Unknown version #{version}"
+        end
+      end
+
+      def command_v1(type, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
+        params = {
+          uuid: uuid,
+          type: type,
+          sent_at: Time.now.iso8601(3),
+          version: 1,
+          publisher: config.publisher,
+          data: data,
+          request_id: request_id,
+          group_id: group_id,
+          deduplication_id: deduplication_id
+        }
+
+        Messages::V1::Command.wrap(params.compact)
+      end
+
+      def command_v2(type, subject:, object:, data: {}, request_id: nil, group_id: nil, deduplication_id: nil, uuid: SecureRandom.uuid)
+        params = {
+          uuid: uuid,
+          type: type,
+          subject: subject,
+          object: object,
+          sent_at: Time.now.iso8601(3),
+          version: 2,
+          publisher: config.publisher,
+          data: data,
+          request_id: request_id,
+          group_id: group_id,
+          deduplication_id: deduplication_id
+        }
+
+        Messages::V2::Command.wrap(params.compact)
+      end
+    end
+  end
+end