cyclone_lariat 0.4.0 → 1.0.0.rc1

data/lib/cyclone_lariat/repo/sequel/messages.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/messages/v1/event'
+require 'cyclone_lariat/messages/v1/command'
+require 'cyclone_lariat/repo/messages_mapper'
+
+module CycloneLariat
+  module Repo
+    module Sequel
+      class Messages
+        attr_reader :dataset
+
+        def initialize(dataset)
+          @dataset = dataset
+        end
+
+        def enabled?
+          !dataset.nil?
+        end
+
+        def disabled?
+          dataset.nil?
+        end
+
+        def create(msg)
+          dataset.insert MessagesMapper.to_row(msg)
+        end
+
+        def exists?(uuid:)
+          dataset.where(uuid: uuid).limit(1).any?
+        end
+
+        def processed!(uuid:, error: nil)
+          data = { processed_at: ::Sequel.function(:NOW) }
+          data.merge!(client_error_message: error.message, client_error_details: JSON.generate(error.details)) if error
+
+          !dataset.where(uuid: uuid).update(data).zero?
+        end
+
+        def find(uuid:)
+          row = dataset.where(uuid: uuid).first
+          return if row.nil?
+
+          build MessagesMapper.from_row(row)
+        end
+
+        def each_unprocessed
+          dataset.where(processed_at: nil).each do |row|
+            msg = build MessagesMapper.from_row(row)
+            yield(msg)
+          end
+        end
+
+        def each_with_client_errors
+          dataset.where { (processed_at !~ nil) & (client_error_message !~ nil) }.each do |row|
+            msg = build MessagesMapper.from_row(row)
+            yield(msg)
+          end
+        end
+
+        private
+
+        def build(raw)
+          case kind = raw.delete(:kind)
+          when 'event'   then CycloneLariat::Messages::V1::Event.wrap raw
+          when 'command' then CycloneLariat::Messages::V1::Command.wrap raw
+          else raise ArgumentError, "Unknown kind `#{kind}` of message"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/sequel/versions.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Repo
+    module Sequel
+      class Versions
+        attr_reader :dataset
+
+        def initialize(dataset)
+          @dataset = dataset
+        end
+
+        def add(version)
+          dataset.insert(version: version)
+          true
+        end
+
+        def remove(version)
+          dataset.filter(version: version).delete.positive?
+        end
+
+        def all
+          dataset.all
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/versions.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'luna_park/extensions/injector'
+require 'cyclone_lariat/core'
+require 'cyclone_lariat/repo/sequel/versions'
+require 'cyclone_lariat/repo/active_record/versions'
+
+module CycloneLariat
+  module Repo
+    class Versions
+      include LunaPark::Extensions::Injector
+
+      attr_reader :config
+
+      dependency(:sequel_versions_class) { Repo::Sequel::Versions }
+      dependency(:active_record_versions_class) { Repo::ActiveRecord::Versions }
+
+      extend Forwardable
+
+      def_delegators :driver, :add, :remove, :all
+
+      def initialize(**options)
+        @config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
+      end
+
+      def driver
+        @driver ||= select(driver: config.driver)
+      end
+
+      private
+
+      def select(driver:)
+        case driver
+        when :sequel then sequel_versions_class.new(config.versions_dataset)
+        when :active_record then active_record_versions_class.new(config.versions_dataset)
+        else raise ArgumentError, "Undefined driver `#{driver}`"
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/resources/queue.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module CycloneLariat
+  module Resources
+    class Queue
+      SNS_SUFFIX = :queue
+
+      attr_reader :instance, :kind, :region, :dest, :account_id, :publisher, :type, :fifo, :content_based_deduplication
+
+      def initialize(instance:, kind:, region:, dest:, account_id:, publisher:, type:, fifo:, content_based_deduplication: nil, tags: nil, name: nil)
+        @instance  = instance
+        @kind      = kind
+        @region    = region
+        @dest      = dest
+        @account_id = account_id
+        @publisher = publisher
+        @type      = type
+        @fifo      = fifo
+        @tags      = tags
+        @name      = name
+        @content_based_deduplication = content_based_deduplication
+      end
+
+      def arn
+        ['arn', 'aws', 'sqs', region, account_id, name].join ':'
+      end
+
+      ##
+      # Url example:
+      #  https://sqs.eu-west-1.amazonaws.com/247606935658/stage-event-queue
+      def url
+        "https://sqs.#{region}.amazonaws.com/#{account_id}/#{name}"
+      end
+
+      def custom?
+        !standard?
+      end
+
+      def standard?
+        instance && kind && publisher && type && true
+      end
+
+      def name
+        @name ||= begin
+          name = [instance, kind, SNS_SUFFIX, publisher, type, dest].compact.join '-'
+          name += '.fifo' if fifo
+          name
+        end
+      end
+
+      alias to_s name
+
+      def topic?
+        false
+      end
+
+      def queue?
+        true
+      end
+
+      def protocol
+        'sqs'
+      end
+
+      def attributes
+        attrs = {}
+        attrs['FifoQueue']                 = 'true' if fifo
+        attrs['ContentBasedDeduplication'] = 'true' if content_based_deduplication
+        attrs
+      end
+
+      class << self
+        ##
+        # Name example: test-event-queue-cyclone_lariat-note_added.fifo
+        # instance: teste
+        # kind: event
+        # publisher: cyclone_lariat
+        # type: note_added
+        # dest: nil
+        # fifo: true
+        def from_name(name, region:, account_id:)
+          is_fifo_array  = name.split('.')
+          full_name      = is_fifo_array[0]
+          fifo_suffix    = is_fifo_array[-1]
+          suffix_exists  = fifo_suffix != full_name
+
+          if suffix_exists && fifo_suffix != 'fifo'
+            raise ArgumentError, "Queue name #{name} consists unexpected suffix #{fifo_suffix}"
+          end
+
+          fifo        = suffix_exists
+          queue_array = full_name.split('-')
+
+          raise ArgumentError, "Topic name should consists `#{SNS_SUFFIX}`" unless queue_array[2] != SNS_SUFFIX
+
+          queue_array.clear if queue_array.size < 5
+
+          new(
+            instance: queue_array[0], kind: queue_array[1], region: region, dest: queue_array[5],
+            account_id: account_id, publisher: queue_array[3], type: queue_array[4], fifo: fifo, name: name
+          )
+        end
+
+        ##
+        # URL example: https://sqs.eu-west-1.amazonaws.com/247606935658/test-event-queue-cyclone_lariat-note_added.fifo
+        # url_array[0]  => https
+        # host_array[0] => sqs
+        # host_array[1] => eu-west-1
+        # url_array[3]  => 247606935658 # account_id
+        # url_array[4]  => test-event-queue-cyclone_lariat-note_added.fifo # name
+        def from_url(url)
+          raise ArgumentError, 'Url is not http format' unless url =~ URI::DEFAULT_PARSER.make_regexp
+
+          url_array = url.split('/')
+          raise ArgumentError, 'Url should start from https' unless url_array[0] == 'https:'
+
+          host_array = url_array[2].split('.')
+          raise ArgumentError, 'It is not queue url' unless host_array[0] == 'sqs'
+
+          from_name(url_array[4], region: host_array[1], account_id: url_array[3])
+        end
+
+        ##
+        # Arn example: "arn:aws:sqs:eu-west-1:247606935658:custom_queue"
+        # arn_array[0] => 'arn'
+        # arn_array[1] => 'aws'
+        # arn_array[2] => 'sqs'
+        # arn_array[3] => 'eu-west-1'     # region
+        # arn_array[4] => '247606935658'  # account_id
+        # arn_array[5] => 'alexey_test2'  # name
+        def from_arn(arn)
+          arn_array = arn.split(':')
+
+          raise ArgumentError, "Arn `#{arn}` should consists `arn`" unless arn_array[0] == 'arn'
+          raise ArgumentError, "Arn `#{arn}` should consists `aws`" unless arn_array[1] == 'aws'
+          raise ArgumentError, "Arn `#{arn}` should consists `sqs`" unless arn_array[2] == 'sqs'
+
+          from_name(arn_array[5], region: arn_array[3], account_id: arn_array[4])
+        end
+      end
+
+      def tags
+        @tags ||= begin
+          if standard?
+            {
+              'standard' => 'true',
+              'instance' => String(instance),
+              'kind' => String(kind),
+              'publisher' => String(publisher),
+              'type' => String(type),
+              'dest' => dest ? String(dest) : 'undefined',
+              'fifo' => fifo ? 'true' : 'false'
+            }
+          else
+            {
+              'standard' => 'false',
+              'name' => String(name),
+              'fifo' => fifo ? 'true' : 'false'
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/resources/topic.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Resources
+    class Topic
+      SNS_SUFFIX = :fanout
+
+      attr_reader :instance, :kind, :region, :account_id, :publisher, :type, :fifo, :content_based_deduplication
+
+      def initialize(instance:, kind:, region:, account_id:, publisher:, type:, fifo:, content_based_deduplication: nil, tags: nil, name: nil)
+        @instance  = instance
+        @kind      = kind
+        @region    = region
+        @account_id = account_id
+        @publisher = publisher
+        @type      = type
+        @fifo      = fifo
+        @tags      = tags
+        @name      = name
+        @content_based_deduplication = content_based_deduplication
+      end
+
+      def arn
+        ['arn', 'aws', 'sns', region, account_id, to_s].join ':'
+      end
+
+      def custom?
+        !standard?
+      end
+
+      def standard?
+        return false unless instance && kind && publisher && type
+
+        true
+      end
+
+      def name
+        @name ||= begin
+          name = [instance, kind, SNS_SUFFIX, publisher, type].compact.join '-'
+          name += '.fifo' if fifo
+          name
+        end
+      end
+
+      def attributes
+        attrs = {}
+        attrs['FifoTopic']                 = 'true' if fifo
+        attrs['ContentBasedDeduplication'] = 'true' if content_based_deduplication
+        attrs
+      end
+
+      def topic?
+        true
+      end
+
+      def queue?
+        false
+      end
+
+      def protocol
+        'sns'
+      end
+
+      alias to_s name
+
+      def ==(other)
+        arn == other.arn
+      end
+
+      class << self
+        def from_name(name, region:, account_id:)
+          is_fifo_array  = name.split('.')
+          full_name      = is_fifo_array[0]
+          fifo_suffix    = is_fifo_array[-1]
+          suffix_exists  = fifo_suffix != full_name
+
+          if suffix_exists && fifo_suffix != 'fifo'
+            raise ArgumentError, "Topic name #{name} consists unexpected suffix #{fifo_suffix}"
+          end
+
+          fifo = suffix_exists
+          topic_array = full_name.split('-')
+
+          raise ArgumentError, "Topic name should consists `#{SNS_SUFFIX}`" unless topic_array[2] != SNS_SUFFIX
+
+          topic_array.clear if topic_array.size < 5
+
+          new(
+            instance: topic_array[0],
+            kind: topic_array[1],
+            publisher: topic_array[3],
+            type: topic_array[4],
+            region: region,
+            account_id: account_id,
+            fifo: fifo,
+            name: name
+          )
+        end
+
+        def from_arn(arn)
+          arn_array = arn.split(':')
+          raise ArgumentError, 'Arn should consists `arn`' unless arn_array[0] == 'arn'
+          raise ArgumentError, 'Arn should consists `aws`' unless arn_array[1] == 'aws'
+          raise ArgumentError, 'Arn should consists `sns`' unless arn_array[2] == 'sns'
+
+          from_name(arn_array[5], region: arn_array[3], account_id: arn_array[4])
+        end
+      end
+
+      def tags
+        @tags ||= begin
+          if standard?
+            [
+              { key: 'standard',  value: 'true' },
+              { key: 'instance',  value: String(instance) },
+              { key: 'kind',      value: String(kind) },
+              { key: 'publisher', value: String(publisher) },
+              { key: 'type',      value: String(type) },
+              { key: 'fifo',      value: fifo ? 'true' : 'false' }
+            ]
+          else
+            [
+              { key: 'standard',  value: 'false' },
+              { key: 'name',      value: String(name) },
+              { key: 'fifo',      value: fifo ? 'true' : 'false' }
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/services/migrate.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Services
+    class Migrate
+      attr_reader :repo, :dir
+
+      def initialize(repo:, dir:)
+        @repo = repo
+        @dir = dir
+      end
+
+      def call
+        alert('No one migration exists') if !Dir.exist?(dir) || Dir.empty?(dir)
+        output = []
+
+        migration_paths.each do |path|
+          filename = File.basename(path, '.rb')
+          version, title = filename.split('_', 2)
+
+          next if existed_migrations.include? version.to_i
+
+          class_name = title.split('_').collect(&:capitalize).join
+          output << "Up - #{version} #{class_name} #{path}"
+          require_relative Pathname.new(Dir.pwd) + Pathname.new(path)
+          Object.const_get(class_name).new.up
+          repo.add(version)
+        end
+
+        output
+      end
+
+      private
+
+      # Sorted by timestamp
+      def migration_paths
+        # lariat/migrate/1668161620_many_to_one.rb
+        Dir.glob("#{dir}/*.rb").sort_by do |file_path|
+          # 1668161620_many_to_one.rb
+          file_name = file_path.split('/')[-1]
+          # 1668161620
+          file_name.split('_').first.to_i
+        end
+      end
+
+      def existed_migrations
+        @existed_migrations ||= repo.all.map { |row| row[:version] }
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/services/rollback.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Services
+    class Rollback
+      attr_reader :repo, :dir
+
+      def initialize(repo:, dir:)
+        @repo = repo
+        @dir = dir
+      end
+
+      def call(version = nil)
+        version ||= existed_migrations[-1]
+        output = []
+
+        paths_of_downgrades(version).each do |path|
+          filename       = File.basename(path, '.rb')
+          version, title = filename.split('_', 2)
+          class_name     = title.split('_').collect(&:capitalize).join
+          output << "Down - #{version} #{class_name} #{path}"
+          require_relative Pathname.new(Dir.pwd) + Pathname.new(path)
+          Object.const_get(class_name).new.down
+          repo.remove(version)
+        end
+
+        output
+      end
+
+      def existed_migrations
+        @existed_migrations ||= repo.all.map { |row| row[:version] }.sort
+      end
+
+      def paths_of_downgrades(version)
+        migrations_to_downgrade = existed_migrations.select { |migration| migration >= version }
+
+        paths = []
+        migrations_to_downgrade.each do |migration|
+          path = Pathname.new(Dir.pwd) + Pathname.new(dir)
+          founded = Dir.glob("#{path}/#{migration}_*.rb")
+          raise "Could not found migration: `#{migration}` in #{path}" if founded.empty?
+          raise "Found lot of migration: `#{migration}` in #{path}"    if founded.size > 1
+
+          paths += founded
+        end
+
+        paths
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CycloneLariat
-  VERSION = '0.4.0'
+  VERSION = '1.0.0.rc1'
 end

data/lib/cyclone_lariat.rb

@@ -1,13 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'cyclone_lariat/configure'
-require_relative 'cyclone_lariat/sns_client'
-require_relative 'cyclone_lariat/errors'
-require_relative 'cyclone_lariat/event'
-require_relative 'cyclone_lariat/messages_mapper'
-require_relative 'cyclone_lariat/messages_repo'
-require_relative 'cyclone_lariat/migration'
-require_relative 'cyclone_lariat/middleware'
-require_relative 'cyclone_lariat/version'
-
-module CycloneLariat; end
+require 'cyclone_lariat/core'
+require 'cyclone_lariat/middleware'
+require 'cyclone_lariat/migration'
+require 'cyclone_lariat/publisher'

data/lib/tasks/console.rake

@@ -3,7 +3,7 @@
 desc 'IRB console with required CycloneLariat'
 task :console do
   # require 'cyclone_lariat'
-  require_relative '../../lib/cyclone_lariat'
+  require 'cyclone_lariat'
   require 'irb'
   require_relative '../../config/initializers/cyclone_lariat'
   # require_relative(init_file) if File.exists?(init_file)

data/lib/tasks/cyclone_lariat.rake

@@ -3,42 +3,40 @@
 require 'cyclone_lariat'
 
 namespace :cyclone_lariat do
-  desc 'Migrate topics for SQSSNS'
+  desc 'Migrate topics for SQS/SNS'
   task migrate: :config do
-    require_relative '../../config/initializers/cyclone_lariat'
     CycloneLariat::Migration.migrate
   end
 
-  desc 'Rollback topics for SQSSNS'
+  desc 'Rollback topics for SQS/SNS'
   task :rollback, [:version] => :config do |_, args|
-    require_relative '../../config/initializers/cyclone_lariat'
     target_version = args[:version] ? args[:version].to_i : nil
     CycloneLariat::Migration.rollback(target_version)
   end
 
   namespace :list do
     desc 'List all topics'
-    task :topics do
-      require_relative '../../config/initializers/cyclone_lariat'
+    task topics: :config do
       CycloneLariat::Migration.list_topics
     end
 
     desc 'List all queues'
-    task :queues do
-      require_relative '../../config/initializers/cyclone_lariat'
+    task queues: :config do
       CycloneLariat::Migration.list_queues
     end
 
     desc 'List all subscriptions'
-    task :subscriptions do
-      require_relative '../../config/initializers/cyclone_lariat'
+    task subscriptions: :config do
       CycloneLariat::Migration.list_subscriptions
     end
   end
 
   desc 'Build graphviz graph for whole system'
-  task :graph do
-    require_relative '../../config/initializers/cyclone_lariat'
+  task graph: :config do
     CycloneLariat::Migration.build_graph
   end
+
+  task :config do
+    require_relative '../../config/initializers/cyclone_lariat'
+  end
 end

data/lib/tasks/db.rake

@@ -22,20 +22,6 @@ namespace :db do
     puts "Database `#{DB_CONF[:database]}` successfully dropped" if system(cmd)
   end
 
-  desc 'Apply migrate'
-  task :migrate, [:version] => :config do |_, args|
-    require 'logger'
-    require 'sequel/core'
-
-    Sequel.extension :migration
-    version = args[:version] ? args[:version].to_i : nil
-    migrations_path = "#{__dir__}/../../db/migrate/"
-
-    Sequel.connect(**DB_CONF, logger: Logger.new($stdout)) do |db|
-      Sequel::Migrator.run(db, migrations_path, target: version)
-    end
-  end
-
   desc 'Database console'
   task console: :config do
     cmd = "PGPASSWORD=#{DB_CONF[:password]} psql" \
@@ -50,6 +36,5 @@ namespace :db do
   task :reset do
     Rake::Task['db:drop'].invoke
     Rake::Task['db:create'].invoke
-    Rake::Task['db:migrate'].invoke
   end
 end