cyclone_lariat 0.3.10 → 1.0.0.rc1

data/Rakefile

@@ -7,9 +7,6 @@ require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec)
 
 # tasks from lib directory
-Dir[File.expand_path('lib/tasks/**/*.rake', __dir__)].each do |entity|
-  print "#{entity} : "
-  puts load entity
-end
+Rake.add_rakelib 'lib/tasks'
 
-task default: %i[spec] # rubocop]
+task default: %i[spec]

data/bin/cyclone_lariat

@@ -0,0 +1,206 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/cyclone_lariat'
+require 'bundler/setup'
+require 'dry/cli'
+require 'fileutils'
+
+module CycloneLariat
+  module CLI
+    module Commands
+      extend Dry::CLI::Registry
+
+      INITIALIZERS_DIR = './config/initializers'
+      RAKE_TASKS_DIR = './lib/tasks'
+
+      class Version < Dry::CLI::Command
+        desc 'Print version'
+
+        def call(*)
+          puts CycloneLariat::VERSION
+        end
+      end
+
+      class Install < Dry::CLI::Command
+        desc 'Install cyclone lariat to current directory'
+        option :adapter,
+          default: 'sequel',
+          values: %w[sequel active_record],
+          desc: 'adapter for store events and versions'
+
+        def call(adapter: 'sequel', **)
+          create_config(adapter)
+          create_rake_task
+        end
+
+        def create_config(adapter)
+          FileUtils.mkdir_p INITIALIZERS_DIR unless Dir.exist? INITIALIZERS_DIR
+          config_path = "#{INITIALIZERS_DIR}/cyclone_lariat.rb"
+          config_file = File.open(config_path, 'w')
+          config_file.puts config_contents(adapter)
+          puts "Created config: #{config_path}"
+        end
+
+        def create_rake_task
+          FileUtils.mkdir_p RAKE_TASKS_DIR unless Dir.exist? RAKE_TASKS_DIR
+          config_path = "#{RAKE_TASKS_DIR}/cyclone_lariat.rake"
+          config_file = File.open(config_path, 'w')
+          config_file.puts rake_task_context
+          puts "Created rake task: #{config_path}"
+        end
+
+        def config_contents(adapter)
+          return config_active_record_contents if adapter == 'active_record'
+          return config_sequel_contents if adapter == 'sequel'
+
+          raise ArgumentError, "Unknown adapter #{adapter}"
+        end
+
+        def config_sequel_contents
+          <<~CONFIG
+            # frozen_string_literal: true
+
+            CycloneLariat.configure do |c|
+              c.version            = 1                         # messages version
+              c.aws_key            = ENV['AWS_KEY']            # aws key
+              c.aws_account_id     = ENV['AWS_ACCOUNT_ID']     # aws account id
+              c.aws_secret_key     = ENV['AWS_SECRET_KEY']     # aws secret
+              c.aws_region         = ENV['AWS_REGION']         # aws default region
+              c.publisher          = ENV['APP_NAME']           # name of your publishers, usually name of your application
+              c.instance           = ENV['INSTANCE']           # stage, production, test
+              c.driver             = :sequel                   # :sequel or :active_record
+              c.messages_dataset   = DB[:messages]             # Sequel dataset / ActiveRecord model for store income messages (on receiver)
+              c.versions_dataset   = DB[:lariat_versions]      # Sequel dataset / ActiveRecord model for publisher migrations
+              c.fake_publish       = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
+            end
+          CONFIG
+        end
+
+        def config_active_record_contents
+          <<~CONFIG
+            # frozen_string_literal: true
+
+            CycloneLariat.configure do |c|
+              c.version            = 1                         # messages version
+              c.aws_key            = ENV['AWS_KEY']            # aws key
+              c.aws_account_id     = ENV['AWS_ACCOUNT_ID']     # aws account id
+              c.aws_secret_key     = ENV['AWS_SECRET_KEY']     # aws secret
+              c.aws_region         = ENV['AWS_REGION']         # aws default region
+              c.publisher          = ENV['APP_NAME']           # name of your publishers, usually name of your application
+              c.instance           = ENV['INSTANCE']           # stage, production, test
+              c.driver             = :active_record            # :sequel or :active_record
+              c.messages_dataset   = CycloneLariatMessage      # Sequel dataset / ActiveRecord model for store income messages (on receiver)
+              c.versions_dataset   = CycloneLariatVersion      # Sequel dataset / ActiveRecord model for publisher migrations
+              c.fake_publish       = ENV['INSTANCE'] == 'test' # when true, prevents messages from being published
+            end
+          CONFIG
+        end
+
+        def rake_task_context
+          <<~TASKS
+            # frozen_string_literal: true
+
+            require 'cyclone_lariat'
+
+            namespace :cyclone_lariat do
+              desc 'Migrate topics for SQS/SNS'
+              task migrate: :cyclone_lariat_config do
+                CycloneLariat::Migration.migrate
+              end
+
+              desc 'Rollback topics for SQS/SNS'
+              task :rollback, [:version] => :cyclone_lariat_config do |_, args|
+                target_version = args[:version] ? args[:version].to_i : nil
+                CycloneLariat::Migration.rollback(target_version)
+              end
+
+              namespace :list do
+                desc 'List all topics'
+                task topics: :cyclone_lariat_config do
+                  CycloneLariat::Migration.list_topics
+                end
+
+                desc 'List all queues'
+                task queues: :cyclone_lariat_config do
+                  CycloneLariat::Migration.list_queues
+                end
+
+                desc 'List all subscriptions'
+                task subscriptions: :cyclone_lariat_config do
+                  CycloneLariat::Migration.list_subscriptions
+                end
+              end
+
+              desc 'Build graphviz graph for whole system'
+              task graph: :cyclone_lariat_config do
+                CycloneLariat::Migration.build_graph
+              end
+
+              task :cyclone_lariat_config do
+                require_relative '../../config/initializers/cyclone_lariat'
+              end
+            end
+          TASKS
+        end
+      end
+
+      module Generate
+        class Migration < Dry::CLI::Command
+          desc 'Generate migration'
+
+          argument :title, type: :string, required: true, desc: 'Title of migration use only a-z and _'
+
+          def call(title:, **)
+            abort('Use only a-z and _ in your title') unless title_correct? title
+
+            FileUtils.mkdir_p CycloneLariat::Migration::DIR unless Dir.exist? CycloneLariat::Migration::DIR
+
+            file_name = generate_filename(title)
+            class_name = generate_class_name(title)
+
+            file = File.open(file_name, 'w')
+            file.puts(file_contents(class_name))
+            puts "Migration successful created:\n\t#{file_name}"
+          end
+
+          private
+
+          def title_correct?(title)
+            /^(?!.*__.*)[a-z]?[a-z_]+[a-z]+$/.match? title
+          end
+
+          def generate_filename(title)
+            "#{CycloneLariat::Migration::DIR}/#{Time.now.to_i}_#{title}.rb"
+          end
+
+          def generate_class_name(title)
+            title.split('_').collect(&:capitalize).join
+          end
+
+          def file_contents(klass_name)
+            <<~MIGRATION
+              # frozen_string_literal: true
+
+              class #{klass_name} < CycloneLariat::Migration
+                def up
+                end
+
+                def down
+                end
+              end
+            MIGRATION
+          end
+        end
+      end
+
+      register 'version', Version, aliases: %w[v -v --version]
+      register 'install', Install
+      register 'generate', aliases: %w[g] do |prefix|
+        prefix.register 'migration', Generate::Migration
+      end
+    end
+  end
+end
+
+Dry::CLI.new(CycloneLariat::CLI::Commands).call

data/cyclone_lariat.gemspec

@@ -7,7 +7,7 @@ require 'cyclone_lariat/version'
 Gem::Specification.new do |spec|
   spec.name          = 'cyclone_lariat'
   spec.version       = CycloneLariat::VERSION
-  spec.authors       = ['Alexander Kudrin', 'Philip Sorokin']
+  spec.authors       = ['Alexander Kudrin', 'Philip Sorokin', 'Kirill Drozdov', 'Vitaly Perminov']
   spec.email         = ['kudrin.alexander@gmail.com']
 
   spec.summary       = 'Shoryuken middleware for LunaPark based application.'
@@ -28,17 +28,28 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|docs|examples|config)/}) }
   end
+
   spec.require_paths = ['lib']
+  spec.bindir = 'bin'
+  spec.executables = ['cyclone_lariat']
 
   spec.add_dependency 'aws-sdk-sns'
   spec.add_dependency 'aws-sdk-sqs'
+  spec.add_dependency 'dry-cli', '~> 0.6'
+  spec.add_dependency 'dry-validation', '~> 1.5'
   spec.add_dependency 'luna_park', '~> 0.11'
+  spec.add_dependency 'terminal-table', '~> 3.0'
 
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'byebug', '~> 11.1'
+  spec.add_development_dependency 'database_cleaner-active_record'
   spec.add_development_dependency 'database_cleaner-sequel', '~> 2.0'
+  spec.add_development_dependency 'guard'
+  spec.add_development_dependency 'guard-bundler'
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'guard-rubocop'
   spec.add_development_dependency 'pg', '~> 1.2'
   spec.add_development_dependency 'pry', '~> 0.13'
   spec.add_development_dependency 'pry-byebug', '~> 3.9'

data/lib/cyclone_lariat/clients/abstract.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'luna_park/extensions/injector'
+require 'cyclone_lariat/generators/event'
+require 'cyclone_lariat/generators/command'
+require 'cyclone_lariat/errors'
+require 'cyclone_lariat/core'
+
+module CycloneLariat
+  module Clients
+    class Abstract
+      include LunaPark::Extensions::Injector
+      include Generators::Event
+      include Generators::Command
+
+      dependency(:aws_client_class)      { raise ArgumentError, 'Client class should be defined' }
+      dependency(:aws_credentials_class) { Aws::Credentials }
+
+      def initialize(**options)
+        @config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
+      end
+
+      attr_reader :config
+
+      def publish
+        raise LunaPark::Errors::AbstractMethod, 'Publish method should be defined'
+      end
+
+      private
+
+      def aws_client
+        @aws_client ||= aws_client_class.new(credentials: aws_credentials, region: config.aws_region)
+      end
+
+      def aws_credentials
+        @aws_credentials ||= aws_credentials_class.new(config.aws_key, config.aws_secret_key)
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/clients/sns.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sns'
+require 'cyclone_lariat/fake'
+require 'cyclone_lariat/clients/abstract'
+require 'cyclone_lariat/resources/topic'
+require 'cyclone_lariat/resources/queue'
+
+module CycloneLariat
+  module Clients
+    class Sns < Abstract
+      include LunaPark::Extensions::Injector
+      include Generators::Topic
+
+      dependency(:aws_client_class) { Aws::SNS::Client }
+
+      def publish(msg, fifo:, topic: nil, skip_validation: false)
+        return Fake.sns_publish_response(msg) if config.fake_publish
+
+        topic = topic ? custom_topic(topic) : topic(msg.type, kind: msg.kind, fifo: fifo)
+
+        raise Errors::GroupIdUndefined.new(resource: topic)       if fifo && msg.group_id.nil?
+        raise Errors::GroupDefined.new(resource: topic)           if !fifo && msg.group_id
+        raise Errors::DeduplicationIdDefined.new(resource: topic) if !fifo && msg.deduplication_id
+
+        msg.validation.check! unless skip_validation
+
+        params = {
+          topic_arn: topic.arn,
+          message: msg.to_json,
+          message_group_id: msg.group_id,
+          message_deduplication_id: msg.deduplication_id
+        }.compact
+
+        aws_client.publish(**params)
+      end
+
+      def exists?(topic)
+        raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
+
+        aws_client.get_topic_attributes({ topic_arn: topic.arn }) && true
+      rescue Aws::SNS::Errors::NotFound
+        false
+      end
+
+      def publish_event(type, fifo:, topic: nil, **options)
+        options[:version] ||= config.version
+        options[:data]    ||= {}
+        options[:uuid]    ||= SecureRandom.uuid
+
+        publish event(type, **options), fifo: fifo, topic: topic
+      end
+
+      def publish_command(type, fifo:, topic: nil, **options)
+        options[:version] ||= config.version
+        options[:data]    ||= {}
+        options[:uuid]    ||= SecureRandom.uuid
+
+        publish command(type, **options), fifo: fifo, topic: topic
+      end
+
+      def create(topic)
+        raise ArgumentError, 'Should be Resources::Topic' unless topic.is_a? Resources::Topic
+        raise Errors::TopicAlreadyExists.new(expected_topic: topic.name) if exists?(topic)
+
+        aws_client.create_topic(name: topic.name, attributes: topic.attributes, tags: topic.tags)
+        topic
+      end
+
+      def delete(topic)
+        raise ArgumentError, 'Should be Resources::Topic' unless topic.is_a? Resources::Topic
+        raise Errors::TopicDoesNotExists.new(expected_topic: topic.name) unless exists?(topic)
+
+        aws_client.delete_topic topic_arn: topic.arn
+        topic
+      end
+
+      def subscribe(topic:, endpoint:)
+        subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
+        raise Errors::SubscriptionAlreadyExists.new(topic: topic, endpoint: endpoint) if subscription_arn
+
+        aws_client.subscribe(
+          {
+            topic_arn: topic.arn,
+            protocol: endpoint.protocol,
+            endpoint: endpoint.arn
+          }
+        )
+      end
+
+      def unsubscribe(topic:, endpoint:)
+        subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
+        raise Errors::SubscriptionDoesNotExists.new(topic: topic, endpoint: endpoint) unless subscription_arn
+
+        aws_client.unsubscribe(subscription_arn: subscription_arn)
+      end
+
+      def list_all
+        topics = []
+        resp = aws_client.list_topics
+
+        loop do
+          resp[:topics].map do |t|
+            topics << Resources::Topic.from_arn(t[:topic_arn])
+          end
+
+          break if resp[:next_token].nil?
+
+          resp = aws_client.list_topics(next_token: resp[:next_token])
+        end
+        topics
+      end
+
+      def list_subscriptions
+        subscriptions = []
+        resp = aws_client.list_subscriptions
+
+        loop do
+          resp[:subscriptions].each do |s|
+            endpoint = s.endpoint.split(':')[2] == 'sqs' ? Resources::Queue.from_arn(s.endpoint) : Resources::Topic.from_arn(s.endpoint)
+            subscriptions << { topic: Resources::Topic.from_arn(s.topic_arn), endpoint: endpoint, arn: s.subscription_arn }
+          end
+
+          break if resp[:next_token].nil?
+
+          resp = aws_client.list_subscriptions(next_token: resp[:next_token])
+        end
+        subscriptions
+      end
+
+      def topic_subscriptions(topic)
+        raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
+
+        subscriptions = []
+
+        resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn)
+
+        loop do
+          next_token = resp[:next_token]
+          subscriptions += resp[:subscriptions]
+
+          break if next_token.nil?
+
+          resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn, next_token: next_token)
+        end
+        subscriptions
+      end
+
+      def find_subscription_arn(topic:, endpoint:)
+        raise ArgumentError, 'Should be Topic' unless topic.is_a? Resources::Topic
+        unless [Resources::Topic, Resources::Queue].include? endpoint.class
+          raise ArgumentError, 'Endpoint should be Topic or Queue'
+        end
+
+        found_subscription = topic_subscriptions(topic).select do |subscription|
+          subscription.endpoint == endpoint.arn
+        end.first
+
+        found_subscription ? found_subscription.subscription_arn : nil
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/clients/sqs.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'cyclone_lariat/fake'
+require 'cyclone_lariat/clients/abstract'
+require 'cyclone_lariat/resources/queue'
+require 'cyclone_lariat/generators/queue'
+
+module CycloneLariat
+  module Clients
+    class Sqs < Abstract
+      include LunaPark::Extensions::Injector
+      include Generators::Queue
+
+      dependency(:aws_client_class) { Aws::SQS::Client }
+
+      def exists?(queue)
+        raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
+
+        aws_client.get_queue_url(queue_name: queue.to_s) && true
+      rescue Aws::SQS::Errors::NonExistentQueue
+        false
+      end
+
+      def add_policy(queue:, policy:)
+        current_policy_json = aws_client.get_queue_attributes({
+          queue_url: queue.url,
+          attribute_names: ['Policy']
+        }).attributes['Policy']
+
+        current_policy = JSON.parse(current_policy_json) if current_policy_json
+
+        return if current_policy && current_policy['Statement'].find { |s| s['Sid'] == policy['Sid'] }
+
+        new_policy = current_policy || { 'Statement' => [] }
+        new_policy['Statement'] << policy
+
+        aws_client.set_queue_attributes({ queue_url: queue.url, attributes: { 'Policy' => new_policy.to_json } })
+      end
+
+      def publish(msg, fifo:, dest: nil, queue: nil, skip_validation: false)
+        return Fake.sqs_send_message_result(msg) if config.fake_publish
+
+        queue = queue ? custom_queue(queue) : queue(msg.type, kind: msg.kind, fifo: fifo, dest: dest)
+
+        raise Errors::GroupIdUndefined.new(resource: queue)       if fifo && msg.group_id.nil?
+        raise Errors::GroupDefined.new(resource: queue)           if !fifo && msg.group_id
+        raise Errors::DeduplicationIdDefined.new(resource: queue) if !fifo && msg.deduplication_id
+
+        msg.validation.check! unless skip_validation
+
+        params = {
+          queue_url: queue.url,
+          message_body: msg.to_json,
+          message_group_id: msg.group_id,
+          message_deduplication_id: msg.deduplication_id
+        }.compact
+
+        aws_client.send_message(**params)
+      end
+
+      def publish_event(type, fifo:, dest: nil, queue: nil, **options)
+        options[:version] ||= self.config.version
+        options[:data]    ||= {}
+        options[:uuid]    ||= SecureRandom.uuid
+
+        publish event(type, data: data, **options), fifo: fifo, dest: dest, queue: queue
+      end
+
+      def publish_command(type, fifo:, dest: nil, queue: nil, **options)
+        options[:version] ||= self.config.version
+        options[:data]    ||= {}
+        options[:uuid]    ||= SecureRandom.uuid
+
+        publish event(type, data: data, **options), fifo: fifo, dest: dest, queue: queue
+      end
+
+      def create(queue)
+        raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
+        raise Errors::QueueAlreadyExists.new(expected_queue: queue.name) if exists?(queue)
+
+        aws_client.create_queue(queue_name: queue.name, attributes: queue.attributes, tags: queue.tags)
+        queue
+      end
+
+      def delete(queue)
+        raise ArgumentError, 'Should be queue' unless queue.is_a? Resources::Queue
+        raise Errors::QueueDoesNotExists.new(expected_queue: queue.name) unless exists?(queue)
+
+        aws_client.delete_queue queue_url: queue.url
+        queue
+      end
+
+      def list_all
+        queues = []
+        resp = aws_client.list_queues
+
+        loop do
+          next_token = resp[:next_token]
+
+          resp[:queue_urls].map do |url|
+            queues << Resources::Queue.from_url(url)
+          end
+
+          break if next_token.nil?
+
+          resp = aws_client.list_queues(next_token: next_token)
+        end
+
+        queues
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/core.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/generators/queue'
+require 'cyclone_lariat/generators/topic'
+require 'cyclone_lariat/options'
+
+module CycloneLariat
+  module CycloneLariatMethods
+    def config
+      @config ||= Options.new
+    end
+
+    def configure
+      yield(config)
+    end
+  end
+
+  extend Generators::Topic
+  extend Generators::Queue
+  extend CycloneLariatMethods
+end

data/lib/cyclone_lariat/errors.rb

@@ -18,5 +18,43 @@ module CycloneLariat
           other.details == details
       end
     end
+
+    class TopicAlreadyExists < LunaPark::Errors::System
+      message { |d| "Topic already exists: `#{d[:expected_topic]}`" }
+    end
+
+    class TopicDoesNotExists < LunaPark::Errors::System
+      message { |d| "Topic does not exists: `#{d[:expected_topic]}`" }
+    end
+
+    class QueueAlreadyExists < LunaPark::Errors::System
+      message { |d| "Queue already exists: `#{d[:expected_queue]}`" }
+    end
+
+    class QueueDoesNotExists < LunaPark::Errors::System
+      message { |d| "Queue does not exists: `#{d[:expected_queue]}`" }
+    end
+    class SubscriptionAlreadyExists < LunaPark::Errors::System
+      message { |d| "Subscription for topic `#{d[:topic].name}`, on endpoint `#{d[:endpoint].name}` already exists" }
+    end
+    class SubscriptionDoesNotExists < LunaPark::Errors::System
+      message { |d| "Subscription for topic `#{d[:topic].name}`, on endpoint `#{d[:endpoint].name}` does not exists" }
+    end
+
+    class InvalidMessage < LunaPark::Errors::Business
+      message 'Message is not valid'
+    end
+
+    class GroupIdUndefined < LunaPark::Errors::System
+      message { |d| "Group id must be defined for FIFO resources: `#{d[:resource].name}`" }
+    end
+
+    class GroupDefined < LunaPark::Errors::System
+      message { |d| "Group id must be nil for non-FIFO resources: `#{d[:resource].name}`" }
+    end
+
+    class DeduplicationIdDefined < LunaPark::Errors::System
+      message { |d| "Deduplication id must be nil for non-FIFO resources: `#{d[:resource].name}`" }
+    end
   end
 end

data/lib/cyclone_lariat/fake.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  class Fake
+    def self.sns_publish_response(message)
+      Aws::SNS::Types::PublishResponse.new.tap do |resp|
+        resp.message_id = SecureRandom.uuid
+        resp.sequence_number = rand(10).to_s if message.fifo?
+      end
+    end
+
+    def self.sqs_send_message_result(message)
+      Aws::SQS::Types::SendMessageResult.new.tap do |res|
+        res.message_id = SecureRandom.uuid
+        res.sequence_number = rand(10).to_s if message.fifo?
+      end
+    end
+  end
+end