cyclone_lariat 0.3.10 → 1.0.0.rc1

data/lib/cyclone_lariat/migration.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'forwardable'
+require 'cyclone_lariat/clients/sns'
+require 'cyclone_lariat/clients/sqs'
+require 'cyclone_lariat/repo/versions'
+require 'cyclone_lariat/services/migrate'
+require 'cyclone_lariat/services/rollback'
+require 'cyclone_lariat/presenters/topics'
+require 'cyclone_lariat/presenters/queues'
+require 'cyclone_lariat/presenters/subscriptions'
+require 'cyclone_lariat/presenters/graph'
+require 'luna_park/errors'
+
+module CycloneLariat
+  class Migration
+    extend Forwardable
+    include LunaPark::Extensions::Injector
+
+    dependency(:sns) { CycloneLariat::Clients::Sns.new }
+    dependency(:sqs) { CycloneLariat::Clients::Sqs.new }
+
+    DIR = './lariat/migrate'
+
+    def up
+      raise LunaPark::Errors::Abstract, "Up method should be defined in #{self.class.name}"
+    end
+
+    def down
+      raise LunaPark::Errors::Abstract, "Down method should be defined in #{self.class.name}"
+    end
+
+    def_delegators :sqs, :queue, :custom_queue
+    def_delegators :sns, :topic, :custom_topic
+
+    def create(resource)
+      process(
+        resource: resource,
+        for_topic: ->(topic) { sns.create(topic) },
+        for_queue: ->(queue) { sqs.create(queue) }
+      )
+
+      puts "  #{resource.class.name.split('::').last} was created `#{resource.name}`"
+    end
+
+    def delete(resource)
+      process(
+        resource: resource,
+        for_topic: ->(topic) { sns.delete(topic) },
+        for_queue: ->(queue) { sqs.delete(queue) }
+      )
+      puts "  #{resource.class.name.split('::').last} was deleted `#{resource.name}`"
+    end
+
+    def exists?(resource)
+      process(
+        resource: resource,
+        for_topic: ->(topic) { sns.exists?(topic) },
+        for_queue: ->(queue) { sqs.exists?(queue) }
+      )
+    end
+
+    def subscribe(topic:, endpoint:, policy: nil)
+      policy ||= default_policy(endpoint)
+      sqs.add_policy(queue: endpoint, policy: policy) if endpoint.queue?
+      sns.subscribe topic: topic, endpoint: endpoint
+      puts "  Subscription was created `#{topic.name} -> #{endpoint.name}`"
+    end
+
+    def unsubscribe(topic:, endpoint:)
+      sns.unsubscribe topic: topic, endpoint: endpoint
+      puts "  Subscription was deleted `#{topic.name} -> #{endpoint.name}`"
+    end
+
+    def default_policy(queue)
+      {
+        'Sid' => queue.arn,
+        'Effect' => 'Allow',
+        'Principal' => {
+          'AWS' => '*'
+        },
+        'Action' => 'SQS:*',
+        'Resource' => queue.arn,
+        'Condition' => {
+          'ArnEquals' => {
+            'aws:SourceArn' => fanout_arn_pattern
+          }
+        }
+      }
+    end
+
+    def topics
+      sns.list_all
+    end
+
+    def queues
+      sqs.list_all
+    end
+
+    def subscriptions
+      sns.list_subscriptions
+    end
+
+    private
+
+    def process(resource:, for_topic:, for_queue:)
+      case resource
+      when Resources::Topic then for_topic.call(resource)
+      when Resources::Queue then for_queue.call(resource)
+      else
+        raise ArgumentError, "Unknown resource class #{resource.class}"
+      end
+    end
+
+    def fanout_arn_pattern
+      @fanout_arn_pattern ||= [
+        'arn:aws:sns',
+        CycloneLariat.config.aws_region,
+        CycloneLariat.config.aws_account_id,
+        "#{CycloneLariat.config.instance}-*-fanout-*"
+      ].join(':')
+    end
+
+    class << self
+      def migrate(repo: CycloneLariat::Repo::Versions.new, dir: DIR, service: Services::Migrate)
+        puts service.new(repo: repo, dir: dir).call
+      end
+
+      def rollback(version = nil, repo: CycloneLariat::Repo::Versions.new, dir: DIR, service: Services::Rollback)
+        puts service.new(repo: repo, dir: dir).call(version)
+      end
+
+      def list_topics(presenter: Presenters::Topics)
+        puts presenter.call(new.topics)
+      end
+
+      def list_queues(presenter: Presenters::Queues)
+        puts presenter.call(new.queues)
+      end
+
+      def list_subscriptions(presenter: Presenters::Subscriptions)
+        puts presenter.call(new.subscriptions)
+      end
+
+      def build_graph(presenter: Presenters::Graph)
+        puts presenter.call(new.subscriptions)
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/options.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'luna_park/values/compound'
+
+module CycloneLariat
+  class Options < LunaPark::Values::Compound
+    attr_accessor :aws_key, :aws_secret_key, :publisher,
+                  :aws_region, :instance, :aws_account_id,
+                  :messages_dataset, :version, :versions_dataset,
+                  :driver, :fake_publish
+
+    # @param [CycloneLariat::Options, Hash] other
+    # @return [CycloneLariat::Options]
+    def merge!(other)
+      other = self.class.wrap(other)
+
+      self.aws_key          ||= other.aws_key
+      self.aws_secret_key   ||= other.aws_secret_key
+      self.publisher        ||= other.publisher
+      self.aws_region       ||= other.aws_region
+      self.instance         ||= other.instance
+      self.aws_account_id   ||= other.aws_account_id
+      self.messages_dataset ||= other.messages_dataset
+      self.version          ||= other.version
+      self.versions_dataset ||= other.versions_dataset
+      self.driver           ||= other.driver
+      self.fake_publish     ||= other.fake_publish
+
+      self
+    end
+
+    def merge(other)
+      dup.merge!(other)
+    end
+
+    def to_h
+      {
+        aws_key: aws_key,
+        aws_secret_key: aws_secret_key,
+        publisher: publisher,
+        aws_region: aws_region,
+        instance: instance,
+        aws_account_id: aws_account_id,
+        messages_dataset: messages_dataset,
+        version: version,
+        versions_dataset: versions_dataset,
+        driver: driver,
+        fake_publish: fake_publish
+      }
+    end
+  end
+end

data/lib/cyclone_lariat/presenters/graph.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module CycloneLariat
+  module Presenters
+    class Graph
+      HEADS = %w[topic endpoint subscription_arn].freeze
+      def self.call(subscriptions)
+        new.call(subscriptions)
+      end
+
+      def call(subscriptions)
+        return '' if subscriptions.empty?
+
+        resources_set = Set.new
+
+        subscriptions.each do |subscription|
+          resources_set << subscription[:topic]
+          resources_set << subscription[:endpoint]
+        end
+
+        [].tap do |output|
+          output << open_graph
+
+          resources_set.each { |resource| output << present_resource(resource) }
+          subscriptions.each { |subscription| output << present_subscription(subscription) }
+
+          output << close_graph
+        end
+      end
+
+      private
+
+      def present_resource(resource)
+        color = resource.custom? ? ', fillcolor=grey' : ', fillcolor=white'
+        style = resource.topic? ? "[shape=component style=filled#{color}]" : "[shape=record, style=\"rounded,filled\"#{color}]"
+        "  \"#{resource.name}\" #{style};"
+      end
+
+      def present_subscription(subscription)
+        "  \"#{subscription[:topic].name}\" -> \"#{subscription[:endpoint].name}\";"
+      end
+
+      def open_graph
+        "digraph G {\n  rankdir=LR;"
+      end
+
+      def close_graph
+        '}'
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/presenters/queues.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'terminal-table'
+
+module CycloneLariat
+  module Presenters
+    class Queues
+      HEADS = %w[valid region account_id name instance kind publisher type destination fifo].freeze
+
+      def self.call(queues)
+        new.call(queues)
+      end
+
+      def call(queues)
+        rows = []
+        queues.each do |queue|
+          rows << row(queue)
+        end
+
+        Terminal::Table.new rows: rows, headings: HEADS
+      end
+
+      private
+
+      def row(queue)
+        [
+          queue.custom? ? 'custom' : 'standard',
+          queue.region,
+          queue.account_id,
+          queue.name,
+          queue.instance,
+          queue.kind,
+          queue.publisher,
+          queue.type,
+          queue.dest,
+          queue.fifo
+        ]
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/presenters/subscriptions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'terminal-table'
+
+module CycloneLariat
+  module Presenters
+    class Subscriptions
+      HEADS = %w[topic endpoint subscription_arn].freeze
+
+      def self.call(subscriptions)
+        new.call(subscriptions)
+      end
+
+      def call(subscriptions)
+        rows = []
+        subscriptions.each do |subscription|
+          rows << row(subscription)
+        end
+
+        Terminal::Table.new rows: rows, headings: HEADS
+      end
+
+      private
+
+      def row(subscription)
+        [
+          subscription[:topic].name,
+          subscription[:endpoint].name,
+          subscription[:arn]
+        ]
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/presenters/topics.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'terminal-table'
+
+module CycloneLariat
+  module Presenters
+    class Topics
+      HEADS = %w[valid region account_id name instance kind publisher type fifo].freeze
+
+      def self.call(topics)
+        new.call(topics)
+      end
+
+      def call(topics)
+        rows = []
+        topics.each do |topic|
+          rows << row(topic)
+        end
+
+        Terminal::Table.new rows: rows, headings: HEADS
+      end
+
+      private
+
+      def row(topic)
+        [
+          topic.custom? ? 'custom' : 'standard',
+          topic.region,
+          topic.account_id,
+          topic.name,
+          topic.instance,
+          topic.kind,
+          topic.publisher,
+          topic.type,
+          topic.fifo
+        ]
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/publisher.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/clients/sns'
+require 'cyclone_lariat/clients/sqs'
+
+module CycloneLariat
+  class Publisher
+    include Generators::Event
+    include Generators::Command
+
+    attr_reader :config
+
+    def initialize(**options)
+      @config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
+    end
+
+    def sqs
+      @sqs ||= Clients::Sqs.new(**config.to_h)
+    end
+
+    def sns
+      @sns ||= Clients::Sns.new(**config.to_h)
+    end
+  end
+end

data/lib/cyclone_lariat/repo/active_record/messages.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/messages/v1/event'
+require 'cyclone_lariat/messages/v1/command'
+require 'cyclone_lariat/repo/messages_mapper'
+
+module CycloneLariat
+  module Repo
+    module ActiveRecord
+      class Messages
+        attr_reader :dataset
+
+        def initialize(dataset)
+          @dataset = dataset
+        end
+
+        def enabled?
+          !dataset.nil?
+        end
+
+        def disabled?
+          dataset.nil?
+        end
+
+        def create(msg)
+          dataset.create(MessagesMapper.to_row(msg)).uuid
+        end
+
+        def exists?(uuid:)
+          dataset.exists?(uuid: uuid)
+        end
+
+        def processed!(uuid:, error: nil)
+          data = { processed_at: current_timestamp_from_db }
+          data.merge!(client_error_message: error.message, client_error_details: JSON.generate(error.details)) if error
+
+          message = dataset.where(uuid: uuid).first
+          return false unless message
+
+          message.update(data)
+        end
+
+        def find(uuid:)
+          row = dataset.where(uuid: uuid).first
+          return if row.nil?
+
+          build_message_from_ar_row(row)
+        end
+
+        def each_unprocessed
+          dataset.where(processed_at: nil).each do |row|
+            msg = build_message_from_ar_row(row)
+            yield(msg)
+          end
+        end
+
+        def each_with_client_errors
+          dataset
+            .where.not(processed_at: nil)
+            .where.not(client_error_message: nil)
+            .each do |row|
+              msg = build_message_from_ar_row(row)
+              yield(msg)
+            end
+        end
+
+        private
+
+        def build_message_from_ar_row(row)
+          build MessagesMapper.from_row(row.attributes.symbolize_keys)
+        end
+
+        def current_timestamp_from_db
+          time_from_db =
+            ::ActiveRecord::Base
+            .connection.execute('select current_timestamp;')
+            .first
+          time = time_from_db.is_a?(Hash) ? time_from_db['current_timestamp'] : time_from_db[0]
+          time.is_a?(Time) ? time : Time.parse(time)
+        end
+
+        def build(raw)
+          case kind = raw.delete(:kind)
+          when 'event'   then CycloneLariat::Messages::V1::Event.wrap raw
+          when 'command' then CycloneLariat::Messages::V1::Command.wrap raw
+          else raise ArgumentError, "Unknown kind `#{kind}` of message"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/active_record/versions.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Repo
+    module ActiveRecord
+      class Versions
+        attr_reader :dataset
+
+        def initialize(dataset)
+          @dataset = dataset
+        end
+
+        def add(version)
+          dataset.create(version: version)
+          true
+        end
+
+        def remove(version)
+          dataset.where(version: version).delete_all.positive?
+        end
+
+        def all
+          dataset.pluck(:version).map { |version| { version: version } }
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/messages.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'luna_park/extensions/injector'
+require 'cyclone_lariat/core'
+require 'cyclone_lariat/repo/sequel/messages'
+require 'cyclone_lariat/repo/active_record/messages'
+
+module CycloneLariat
+  module Repo
+    class Messages
+      include LunaPark::Extensions::Injector
+
+      attr_reader :config
+
+      dependency(:sequel_messages_class) { Repo::Sequel::Messages }
+      dependency(:active_record_messages_class) { Repo::ActiveRecord::Messages }
+
+      extend Forwardable
+
+      def_delegators :driver, :create, :exists?, :processed!, :find, :each_unprocessed, :each_with_client_errors,
+                     :enabled?, :disabled?
+
+      def initialize(**options)
+        @config = CycloneLariat::Options.wrap(options).merge!(CycloneLariat.config)
+      end
+
+      def driver
+        @driver ||= select(driver: config.driver)
+      end
+
+      private
+
+      def select(driver:)
+        case driver
+        when :sequel then sequel_messages_class.new(config.messages_dataset)
+        when :active_record then active_record_messages_class.new(config.messages_dataset)
+        else raise ArgumentError, "Undefined driver `#{driver}`"
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/messages_mapper.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  module Repo
+    class MessagesMapper
+      class << self
+        def from_row(row)
+          return if row.nil?
+
+          row[:data] = hash_from_json_column(row[:data])
+          row[:client_error_details] = hash_from_json_column(row[:client_error_details]) if row[:client_error_details]
+          row
+        end
+
+        def to_row(input)
+          {
+            uuid: input.uuid,
+            kind: input.kind,
+            type: input.type,
+            publisher: input.publisher,
+            data: JSON.generate(input.data),
+            client_error_message: input.client_error&.message,
+            client_error_details: JSON.generate(input.client_error&.details),
+            version: input.version,
+            sent_at: input.sent_at
+          }
+        end
+
+        private
+
+        def hash_from_json_column(data)
+          return data if data.is_a?(Hash)
+          return JSON.parse(data) if data.is_a?(String)
+
+          if pg_json_extension_enabled?
+            return data.to_h             if data.is_a?(::Sequel::Postgres::JSONHash)
+            return JSON.parse(data.to_s) if data.is_a?(::Sequel::Postgres::JSONString)
+          end
+
+          raise ArgumentError, "Unknown type of `#{data}`"
+        end
+
+        def pg_json_extension_enabled?
+          Object.const_defined?('Sequel::Postgres::JSONHash')
+        end
+      end
+    end
+  end
+end

data/lib/cyclone_lariat/repo/sequel/messages.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat/messages/v1/event'
+require 'cyclone_lariat/messages/v1/command'
+require 'cyclone_lariat/repo/messages_mapper'
+
+module CycloneLariat
+  module Repo
+    module Sequel
+      class Messages
+        attr_reader :dataset
+
+        def initialize(dataset)
+          @dataset = dataset
+        end
+
+        def enabled?
+          !dataset.nil?
+        end
+
+        def disabled?
+          dataset.nil?
+        end
+
+        def create(msg)
+          dataset.insert MessagesMapper.to_row(msg)
+        end
+
+        def exists?(uuid:)
+          dataset.where(uuid: uuid).limit(1).any?
+        end
+
+        def processed!(uuid:, error: nil)
+          data = { processed_at: ::Sequel.function(:NOW) }
+          data.merge!(client_error_message: error.message, client_error_details: JSON.generate(error.details)) if error
+
+          !dataset.where(uuid: uuid).update(data).zero?
+        end
+
+        def find(uuid:)
+          row = dataset.where(uuid: uuid).first
+          return if row.nil?
+
+          build MessagesMapper.from_row(row)
+        end
+
+        def each_unprocessed
+          dataset.where(processed_at: nil).each do |row|
+            msg = build MessagesMapper.from_row(row)
+            yield(msg)
+          end
+        end
+
+        def each_with_client_errors
+          dataset.where { (processed_at !~ nil) & (client_error_message !~ nil) }.each do |row|
+            msg = build MessagesMapper.from_row(row)
+            yield(msg)
+          end
+        end
+
+        private
+
+        def build(raw)
+          case kind = raw.delete(:kind)
+          when 'event'   then CycloneLariat::Messages::V1::Event.wrap raw
+          when 'command' then CycloneLariat::Messages::V1::Command.wrap raw
+          else raise ArgumentError, "Unknown kind `#{kind}` of message"
+          end
+        end
+      end
+    end
+  end
+end