cyclone_lariat 0.4.0 → 1.0.0.rc2

data/lib/cyclone_lariat/migration.rb

@@ -2,19 +2,24 @@
 
 require 'fileutils'
 require 'forwardable'
-require_relative 'sns_client'
-require_relative 'sqs_client'
+require 'cyclone_lariat/clients/sns'
+require 'cyclone_lariat/clients/sqs'
+require 'cyclone_lariat/repo/versions'
+require 'cyclone_lariat/services/migrate'
+require 'cyclone_lariat/services/rollback'
+require 'cyclone_lariat/presenters/topics'
+require 'cyclone_lariat/presenters/queues'
+require 'cyclone_lariat/presenters/subscriptions'
+require 'cyclone_lariat/presenters/graph'
 require 'luna_park/errors'
-require 'terminal-table'
-require 'set'
 
 module CycloneLariat
   class Migration
     extend Forwardable
     include LunaPark::Extensions::Injector
 
-    dependency(:sns) { CycloneLariat::SnsClient.new }
-    dependency(:sqs) { CycloneLariat::SqsClient.new }
+    dependency(:sns) { CycloneLariat::Clients::Sns.new }
+    dependency(:sqs) { CycloneLariat::Clients::Sqs.new }
 
     DIR = './lariat/migrate'
 
@@ -56,7 +61,9 @@ module CycloneLariat
       )
     end
 
-    def subscribe(topic:, endpoint:)
+    def subscribe(topic:, endpoint:, policy: nil)
+      policy ||= default_policy(endpoint)
+      sqs.add_policy(queue: endpoint, policy: policy) if endpoint.queue?
       sns.subscribe topic: topic, endpoint: endpoint
       puts "  Subscription was created `#{topic.name} -> #{endpoint.name}`"
     end
@@ -66,6 +73,23 @@ module CycloneLariat
       puts "  Subscription was deleted `#{topic.name} -> #{endpoint.name}`"
     end
 
+    def default_policy(queue)
+      {
+        'Sid' => queue.arn,
+        'Effect' => 'Allow',
+        'Principal' => {
+          'AWS' => '*'
+        },
+        'Action' => 'SQS:*',
+        'Resource' => queue.arn,
+        'Condition' => {
+          'ArnEquals' => {
+            'aws:SourceArn' => fanout_arn_pattern
+          }
+        }
+      }
+    end
+
     def topics
       sns.list_all
     end
@@ -82,132 +106,45 @@ module CycloneLariat
 
     def process(resource:, for_topic:, for_queue:)
       case resource
-      when Topic then for_topic.call(resource)
-      when Queue then for_queue.call(resource)
+      when Resources::Topic then for_topic.call(resource)
+      when Resources::Queue then for_queue.call(resource)
       else
         raise ArgumentError, "Unknown resource class #{resource.class}"
       end
     end
 
+    def fanout_arn_pattern
+      @fanout_arn_pattern ||= [
+        'arn:aws:sns',
+        CycloneLariat.config.aws_region,
+        CycloneLariat.config.aws_account_id,
+        "#{CycloneLariat.config.instance}-*-fanout-*"
+      ].join(':')
+    end
+
     class << self
-      def migrate(dataset: CycloneLariat.versions_dataset, dir: DIR)
-        alert('No one migration exists') if !Dir.exist?(dir) || Dir.empty?(dir)
-
-        Dir.glob("#{dir}/*.rb") do |path|
-          filename = File.basename(path, '.rb')
-          version, title = filename.split('_', 2)
-
-          existed_migrations = dataset.all.map { |row| row[:version] }
-          unless existed_migrations.include? version.to_i
-            class_name = title.split('_').collect(&:capitalize).join
-            puts "Up - #{version} #{class_name} #{path}"
-            require_relative Pathname.new(Dir.pwd) + Pathname.new(path)
-            Object.const_get(class_name).new.up
-            dataset.insert(version: version)
-          end
-        end
+      def migrate(repo: CycloneLariat::Repo::Versions.new, dir: DIR, service: Services::Migrate)
+        puts service.new(repo: repo, dir: dir).call
       end
 
-      def rollback(version = nil, dataset: CycloneLariat.versions_dataset, dir: DIR)
-        existed_migrations = dataset.all.map { |row| row[:version] }.sort
-        version ||= existed_migrations[-1]
-        migrations_to_downgrade = existed_migrations.select { |migration| migration >= version }
-
-        paths = []
-        migrations_to_downgrade.each do |migration|
-          path = Pathname.new(Dir.pwd) + Pathname.new(dir)
-          founded = Dir.glob("#{path}/#{migration}_*.rb")
-          raise "Could not found migration: `#{migration}` in #{path}" if founded.empty?
-          raise "Found lot of migration: `#{migration}` in #{path}"    if founded.size > 1
-
-          paths += founded
-        end
-
-        paths.each do |path|
-          filename       = File.basename(path, '.rb')
-          version, title = filename.split('_', 2)
-          class_name     = title.split('_').collect(&:capitalize).join
-          puts "Down - #{version} #{class_name} #{path}"
-          require_relative Pathname.new(Dir.pwd) + Pathname.new(path)
-          Object.const_get(class_name).new.down
-          dataset.filter(version: version).delete
-        end
+      def rollback(version = nil, repo: CycloneLariat::Repo::Versions.new, dir: DIR, service: Services::Rollback)
+        puts service.new(repo: repo, dir: dir).call(version)
       end
 
-      def list_topics
-        rows = []
-        new.topics.each do |topic|
-          rows << [
-            topic.custom? ? 'custom' : 'standard',
-            topic.region,
-            topic.account_id,
-            topic.name,
-            topic.instance,
-            topic.kind,
-            topic.publisher,
-            topic.type,
-            topic.fifo
-          ]
-        end
-
-        puts Terminal::Table.new rows: rows, headings: %w[valid region account_id name instance kind publisher type fifo]
+      def list_topics(presenter: Presenters::Topics)
+        puts presenter.call(new.topics)
       end
 
-      def list_queues
-        rows = []
-        new.queues.each do |queue|
-          rows << [
-            queue.custom? ? 'custom' : 'standard',
-            queue.region,
-            queue.account_id,
-            queue.name,
-            queue.instance,
-            queue.kind,
-            queue.publisher,
-            queue.type,
-            queue.dest,
-            queue.fifo
-          ]
-        end
-
-        puts Terminal::Table.new rows: rows, headings: %w[valid region account_id name instance kind publisher type destination fifo]
+      def list_queues(presenter: Presenters::Queues)
+        puts presenter.call(new.queues)
       end
 
-      def list_subscriptions
-        rows = []
-        new.subscriptions.each do |subscription|
-          rows << [
-            subscription[:topic].name,
-            subscription[:endpoint].name,
-            subscription[:arn]
-          ]
-        end
-
-        puts Terminal::Table.new rows: rows, headings: %w[topic endpoint subscription_arn]
+      def list_subscriptions(presenter: Presenters::Subscriptions)
+        puts presenter.call(new.subscriptions)
       end
 
-      def build_graph
-        subscriptions = new.subscriptions
-        resources_set = Set.new
-
-        subscriptions.each do |subscription|
-          resources_set << subscription[:topic]
-          resources_set << subscription[:endpoint]
-        end
-
-        puts 'digraph G {'
-        puts '  rankdir=LR;'
-
-        resources_set.each do |resource|
-          color = resource.custom? ? ', fillcolor=grey' : ', fillcolor=white'
-          style = resource.topic? ? "[shape=component style=filled#{color}]" : "[shape=record, style=\"rounded,filled\"#{color}]"
-          puts "  \"#{resource.name}\" #{style};"
-        end
-
-        subscriptions.each do |subscription|
-          puts "  \"#{subscription[:topic].name}\" -> \"#{subscription[:endpoint].name}\";"
-        end
-        puts '}'
+      def build_graph(presenter: Presenters::Graph)
+        puts presenter.call(new.subscriptions)
       end
     end
   end

data/lib/cyclone_lariat/options.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'luna_park/values/compound'
+
+module CycloneLariat
+  class Options < LunaPark::Values::Compound
+    attr_accessor :aws_key, :aws_secret_key, :publisher,
+                  :aws_region, :instance, :aws_account_id,
+                  :messages_dataset, :version, :versions_dataset,
+                  :driver, :fake_publish
+
+    # @param [CycloneLariat::Options, Hash] other
+    # @return [CycloneLariat::Options]
+    def merge!(other)
+      other = self.class.wrap(other)
+
+      self.aws_key          ||= other.aws_key
+      self.aws_secret_key   ||= other.aws_secret_key
+      self.publisher        ||= other.publisher
+      self.aws_region       ||= other.aws_region
+      self.instance         ||= other.instance
+      self.aws_account_id   ||= other.aws_account_id
+      self.messages_dataset ||= other.messages_dataset
+      self.version          ||= other.version
+      self.versions_dataset ||= other.versions_dataset
+      self.driver           ||= other.driver
+      self.fake_publish     ||= other.fake_publish
+
+      self
+    end
+
+    def merge(other)
+      dup.merge!(other)
+    end
+
+    def to_h
+      {
+        aws_key: aws_key,
+        aws_secret_key: aws_secret_key,
+        publisher: publisher,
+        aws_region: aws_region,
+        instance: instance,
+        aws_account_id: aws_account_id,
+        messages_dataset: messages_dataset,
+        version: version,
+        versions_dataset: versions_dataset,
+        driver: driver,
+        fake_publish: fake_publish
+      }
+    end
+  end
+end

data/lib/cyclone_lariat/presenters/graph.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module CycloneLariat
+  module Presenters
+    class Graph
+      HEADS = %w[topic endpoint subscription_arn].freeze
+      def self.call(subscriptions)
+        new.call(subscriptions)
+      end
+
+      def call(subscriptions)
+        return '' if subscriptions.empty?
+
+        resources_set = Set.new
+
+        subscriptions.each do |subscription|
+          resources_set << subscription[:topic]
+          resources_set << subscription[:endpoint]
+        end
+
+        [].tap do |output|
+          output << open_graph
+
+          resources_set.each { |resource| output << present_resource(resource) }
+          subscriptions.each { |subscription| output << present_subscription(subscription) }
+
+          output << close_graph
+        end
+      end
+
+      private
+
+      def present_resource(resource)
+        color = resource.custom? ? ', fillcolor=grey' : ', fillcolor=white'
+        style = resource.topic? ? "[shape=component style=filled#{color}]" : "[shape=record, style=\"rounded,filled\"#{color}]"
+        "  \"#{resource.name}\" #{style};"
+      end
+
+      def present_subscription(subscription)
+        "  \"#{subscription[:topic].name}\" -> \"#{subscription[:endpoint].name}\";"
+      end
+
+      def open_graph
+        "digraph G {\n  rankdir=LR;"
+      end
+
+      def close_graph
+        '}'
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/presenters/queues.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'terminal-table'
+
+module CycloneLariat
+  module Presenters
+    class Queues
+      HEADS = %w[valid region account_id name instance kind publisher type destination fifo].freeze
+
+      def self.call(queues)
+        new.call(queues)
+      end
+
+      def call(queues)
+        rows = []
+        queues.each do |queue|
+          rows << row(queue)
+        end
+
+        Terminal::Table.new rows: rows, headings: HEADS
+      end
+
+      private
+
+      def row(queue)
+        [
+          queue.custom? ? 'custom' : 'standard',
+          queue.region,
+          queue.account_id,
+          queue.name,
+          queue.instance,
+          queue.kind,
+          queue.publisher,
+          queue.type,
+          queue.dest,
+          queue.fifo
+        ]
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/presenters/subscriptions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'terminal-table'
+
+module CycloneLariat
+  module Presenters
+    class Subscriptions
+      HEADS = %w[topic endpoint subscription_arn].freeze
+
+      def self.call(subscriptions)
+        new.call(subscriptions)
+      end
+
+      def call(subscriptions)
+        rows = []
+        subscriptions.each do |subscription|
+          rows << row(subscription)
+        end
+
+        Terminal::Table.new rows: rows, headings: HEADS
+      end
+
+      private
+
+      def row(subscription)
+        [
+          subscription[:topic].name,
+          subscription[:endpoint].name,
+          subscription[:arn]
+        ]
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/presenters/topics.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'terminal-table'
+
+module CycloneLariat
+  module Presenters
+    class Topics
+      HEADS = %w[valid region account_id name instance kind publisher type fifo].freeze
+
+      def self.call(topics)
+        new.call(topics)
+      end
+
+      def call(topics)
+        rows = []
+        topics.each do |topic|
+          rows << row(topic)
+        end
+
+        Terminal::Table.new rows: rows, headings: HEADS
+      end
+
+      private
+
+      def row(topic)
+        [
+          topic.custom? ? 'custom' : 'standard',
+          topic.region,
+          topic.account_id,
+          topic.name,
+          topic.instance,
+          topic.kind,
+          topic.publisher,
+          topic.type,
+          topic.fifo
+        ]
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/publisher.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/clients/sns'
+require 'cyclone_lariat/clients/sqs'
+
+module CycloneLariat
+  class Publisher
+    include Generators::Event
+    include Generators::Command
+
+    attr_reader :config
+
+    def initialize(**options)
+      @config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
+    end
+
+    def sqs
+      @sqs ||= Clients::Sqs.new(**config.to_h)
+    end
+
+    def sns
+      @sns ||= Clients::Sns.new(**config.to_h)
+    end
+  end
+end

data/lib/cyclone_lariat/repo/active_record/messages.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/messages/v1/event'
+require 'cyclone_lariat/messages/v1/command'
+require 'cyclone_lariat/repo/messages_mapper'
+
+module CycloneLariat
+  module Repo
+    module ActiveRecord
+      class Messages
+        attr_reader :dataset
+
+        def initialize(dataset)
+          @dataset = dataset
+        end
+
+        def enabled?
+          !dataset.nil?
+        end
+
+        def disabled?
+          dataset.nil?
+        end
+
+        def create(msg)
+          dataset.create(MessagesMapper.to_row(msg)).uuid
+        end
+
+        def exists?(uuid:)
+          dataset.exists?(uuid: uuid)
+        end
+
+        def processed!(uuid:, error: nil)
+          data = { processed_at: current_timestamp_from_db }
+          data.merge!(client_error_message: error.message, client_error_details: JSON.generate(error.details)) if error
+
+          message = dataset.where(uuid: uuid).first
+          return false unless message
+
+          message.update(data)
+        end
+
+        def find(uuid:)
+          row = dataset.where(uuid: uuid).first
+          return if row.nil?
+
+          build_message_from_ar_row(row)
+        end
+
+        def each_unprocessed
+          dataset.where(processed_at: nil).each do |row|
+            msg = build_message_from_ar_row(row)
+            yield(msg)
+          end
+        end
+
+        def each_with_client_errors
+          dataset
+            .where.not(processed_at: nil)
+            .where.not(client_error_message: nil)
+            .each do |row|
+              msg = build_message_from_ar_row(row)
+              yield(msg)
+            end
+        end
+
+        private
+
+        def build_message_from_ar_row(row)
+          build MessagesMapper.from_row(row.attributes.symbolize_keys)
+        end
+
+        def current_timestamp_from_db
+          time_from_db =
+            ::ActiveRecord::Base
+            .connection.execute('select current_timestamp;')
+            .first
+          time = time_from_db.is_a?(Hash) ? time_from_db['current_timestamp'] : time_from_db[0]
+          time.is_a?(Time) ? time : Time.parse(time)
+        end
+
+        def build(raw)
+          case kind = raw.delete(:kind)
+          when 'event'   then CycloneLariat::Messages::V1::Event.wrap raw
+          when 'command' then CycloneLariat::Messages::V1::Command.wrap raw
+          else raise ArgumentError, "Unknown kind `#{kind}` of message"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/active_record/versions.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Repo
+    module ActiveRecord
+      class Versions
+        attr_reader :dataset
+
+        def initialize(dataset)
+          @dataset = dataset
+        end
+
+        def add(version)
+          dataset.create(version: version)
+          true
+        end
+
+        def remove(version)
+          dataset.where(version: version).delete_all.positive?
+        end
+
+        def all
+          dataset.pluck(:version).map { |version| { version: version } }
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/messages.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'luna_park/extensions/injector'
+require 'cyclone_lariat/core'
+require 'cyclone_lariat/repo/sequel/messages'
+require 'cyclone_lariat/repo/active_record/messages'
+
+module CycloneLariat
+  module Repo
+    class Messages
+      include LunaPark::Extensions::Injector
+
+      attr_reader :config
+
+      dependency(:sequel_messages_class) { Repo::Sequel::Messages }
+      dependency(:active_record_messages_class) { Repo::ActiveRecord::Messages }
+
+      extend Forwardable
+
+      def_delegators :driver, :create, :exists?, :processed!, :find, :each_unprocessed, :each_with_client_errors,
+                     :enabled?, :disabled?
+
+      def initialize(**options)
+        @config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
+      end
+
+      def driver
+        @driver ||= select(driver: config.driver)
+      end
+
+      private
+
+      def select(driver:)
+        case driver
+        when :sequel then sequel_messages_class.new(config.messages_dataset)
+        when :active_record then active_record_messages_class.new(config.messages_dataset)
+        else raise ArgumentError, "Undefined driver `#{driver}`"
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/messages_mapper.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Repo
+    class MessagesMapper
+      class << self
+        def from_row(row)
+          return if row.nil?
+
+          row[:data] = hash_from_json_column(row[:data])
+          row[:client_error_details] = hash_from_json_column(row[:client_error_details]) if row[:client_error_details]
+          row
+        end
+
+        def to_row(input)
+          {
+            uuid: input.uuid,
+            kind: input.kind,
+            type: input.type,
+            publisher: input.publisher,
+            data: JSON.generate(input.data),
+            client_error_message: input.client_error&.message,
+            client_error_details: JSON.generate(input.client_error&.details),
+            version: input.version,
+            sent_at: input.sent_at
+          }
+        end
+
+        private
+
+        def hash_from_json_column(data)
+          return data if data.is_a?(Hash)
+          return JSON.parse(data) if data.is_a?(String)
+
+          if pg_json_extension_enabled?
+            return data.to_h             if data.is_a?(::Sequel::Postgres::JSONHash)
+            return JSON.parse(data.to_s) if data.is_a?(::Sequel::Postgres::JSONString)
+          end
+
+          raise ArgumentError, "Unknown type of `#{data}`"
+        end
+
+        def pg_json_extension_enabled?
+          Object.const_defined?('Sequel::Postgres::JSONHash')
+        end
+      end
+    end
+  end
+end