cyclone_lariat 0.3.10 → 0.4.0

data/lib/cyclone_lariat/queue.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module CycloneLariat
+  class Queue
+    SNS_SUFFIX = :queue
+
+    attr_reader :instance, :kind, :region, :dest, :account_id, :publisher, :type, :fifo, :tags
+
+    def initialize(instance:, kind:, region:, dest:, account_id:, publisher:, type:, fifo:, tags: nil, name: nil)
+      @instance  = instance
+      @kind      = kind
+      @region    = region
+      @dest      = dest
+      @account_id = account_id
+      @publisher = publisher
+      @type      = type
+      @fifo      = fifo
+      @tags      = tags || default_tags(instance, kind, publisher, type, dest, fifo)
+      @name      = name
+    end
+
+    def arn
+      ['arn', 'aws', 'sqs', region, account_id, name].join ':'
+    end
+
+    ##
+    # Url example:
+    #  https://sqs.eu-west-1.amazonaws.com/247606935658/stage-event-queue
+    def url
+      "https://sqs.#{region}.amazonaws.com/#{account_id}/#{name}"
+    end
+
+    def custom?
+      !standard?
+    end
+
+    def standard?
+      instance && kind && publisher && type && true
+    end
+
+    def name
+      @name ||= begin
+        name = [instance, kind, SNS_SUFFIX, publisher, type, dest].compact.join '-'
+        name += '.fifo' if fifo
+        name
+      end
+    end
+
+    alias to_s name
+
+
+    def topic?
+      false
+    end
+
+    def queue?
+      true
+    end
+
+    def protocol
+      'sqs'
+    end
+
+    class << self
+      ##
+      # Name example: test-event-queue-cyclone_lariat-note_added.fifo
+      # instance: teste
+      # kind: event
+      # publisher: cyclone_lariat
+      # type: note_added
+      # dest: nil
+      # fifo: true
+      def from_name(name, region:, account_id:)
+        is_fifo_array  = name.split('.')
+        full_name      = is_fifo_array[0]
+        fifo_suffix    = is_fifo_array[-1]
+        suffix_exists  = fifo_suffix != full_name
+
+        if suffix_exists && fifo_suffix != 'fifo'
+          raise ArgumentError, "Queue name #{name} consists unexpected suffix #{fifo_suffix}"
+        end
+
+        fifo        = suffix_exists
+        queue_array = full_name.split('-')
+
+        raise ArgumentError, "Topic name should consists `#{SNS_SUFFIX}`" unless queue_array[2] != SNS_SUFFIX
+
+        new(
+          instance: queue_array[0], kind: queue_array[1], region: region, dest: queue_array[5],
+          account_id: account_id, publisher: queue_array[3], type: queue_array[4], fifo: fifo, name: name
+        )
+      end
+
+      ##
+      # URL example: https://sqs.eu-west-1.amazonaws.com/247606935658/test-event-queue-cyclone_lariat-note_added.fifo
+      # url_array[0]  => https
+      # host_array[0] => sqs
+      # host_array[1] => eu-west-1
+      # url_array[3]  => 247606935658 # account_id
+      # url_array[4]  => test-event-queue-cyclone_lariat-note_added.fifo # name
+      def from_url(url)
+        raise ArgumentError, 'Url is not http format' unless url =~ URI::DEFAULT_PARSER.make_regexp
+
+        url_array = url.split('/')
+        raise ArgumentError, 'Url should start from https' unless url_array[0] == 'https:'
+
+        host_array = url_array[2].split('.')
+        raise ArgumentError, 'It is not queue url' unless host_array[0] == 'sqs'
+
+        from_name(url_array[4], region: host_array[1], account_id: url_array[3])
+      end
+
+      ##
+      # Arn example: "arn:aws:sqs:eu-west-1:247606935658:custom_queue"
+      # arn_array[0] => 'arn'
+      # arn_array[1] => 'aws'
+      # arn_array[2] => 'sqs'
+      # arn_array[3] => 'eu-west-1'     # region
+      # arn_array[4] => '247606935658'  # account_id
+      # arn_array[5] => 'alexey_test2'  # name
+      def from_arn(arn)
+        arn_array = arn.split(':')
+
+        raise ArgumentError, "Arn `#{arn}` should consists `arn`" unless arn_array[0] == 'arn'
+        raise ArgumentError, "Arn `#{arn}` should consists `aws`" unless arn_array[1] == 'aws'
+        raise ArgumentError, "Arn `#{arn}` should consists `sqs`" unless arn_array[2] == 'sqs'
+
+        from_name(arn_array[5], region: arn_array[3], account_id: arn_array[4])
+      end
+    end
+
+    private
+
+    def default_tags(instance, kind, publisher, type, dest, fifo)
+      {
+        instance: String(instance),
+        kind: String(kind),
+        publisher: String(publisher),
+        type: String(type),
+        dest: dest ? String(dest) : 'undefined',
+        fifo: fifo ? 'true' : 'false'
+      }
+    end
+  end
+end

data/lib/cyclone_lariat/sns_client.rb

@@ -2,6 +2,8 @@
 
 require 'aws-sdk-sns'
 require_relative 'abstract/client'
+require_relative 'topic'
+require_relative 'queue'
 
 module CycloneLariat
   class SnsClient < Abstract::Client
@@ -9,30 +11,139 @@ module CycloneLariat
 
     dependency(:aws_client_class) { Aws::SNS::Client }
 
-    SNS_SUFFIX = :fanout
+    def custom_topic(name)
+      Topic.from_name(name, account_id: account_id, region: region)
+    end
+
+    def topic(type, fifo:, publisher: nil, kind: :event)
+      publisher ||= self.publisher
+
+      Topic.new(
+        instance: instance,
+        publisher: publisher,
+        region: region,
+        account_id: account_id,
+        kind: kind,
+        type: type, fifo: fifo
+      )
+    end
+
+    def publish(msg, fifo:, topic: nil)
+      topic = topic ? custom_topic(topic) : topic(msg.type, kind: msg.kind, fifo: fifo)
+      aws_client.publish(topic_arn: topic.arn, message: msg.to_json)
+    end
+
+    def exists?(topic)
+      raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
+
+      aws_client.get_topic_attributes({ topic_arn: topic.arn }) && true
+    rescue Aws::SNS::Errors::NotFound
+      false
+    end
+
+    def publish_event(type, fifo:, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, topic: nil)
+      publish event(type, data: data, version: version, uuid: uuid, request_id: request_id), topic: topic, fifo: fifo
+    end
+
+    def publish_command(type, fifo:, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, topic: nil)
+      publish command(type, data: data, version: version, uuid: uuid, request_id: request_id), topic: topic, fifo: fifo
+    end
+
+    def create(topic)
+      raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
+      raise Errors::TopicAlreadyExists.new(expected_topic: topic.name) if exists?(topic)
+
+      aws_client.create_topic(name: topic.name, attributes: topic.attributes, tags: topic.tags)
+      topic
+    end
+
+    def delete(topic)
+      raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
+      raise Errors::TopicDoesNotExists.new(expected_topic: topic.name) unless exists?(topic)
 
-    def publish(msg, topic: nil)
-      topic ||= get_topic(msg.kind, msg.type)
-      arn     = get_arn(topic)
-      aws_client.publish(topic_arn: arn, message: msg.to_json)
+      aws_client.delete_topic topic_arn: topic.arn
+      topic
     end
 
-    def publish_event(type, data: {}, version: self.version, uuid: SecureRandom.uuid, topic: nil)
-      publish event(type, data: data, version: version, uuid: uuid), topic: topic
+    def subscribe(topic:, endpoint:)
+      subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
+      raise Errors::SubscriptionAlreadyExists.new(topic: topic, endpoint: endpoint) if subscription_arn
+
+      aws_client.subscribe(
+        {
+          topic_arn: topic.arn,
+          protocol: endpoint.protocol,
+          endpoint: endpoint.arn
+        }
+      )
     end
 
-    def publish_command(type, data: {}, version: self.version, uuid: SecureRandom.uuid, topic: nil)
-      publish command(type, data: data, version: version, uuid: uuid), topic: topic
+    def unsubscribe(topic:, endpoint:)
+      subscription_arn = find_subscription_arn(topic: topic, endpoint: endpoint)
+      raise Errors::SubscriptionDoesNotExists.new(topic: topic, endpoint: endpoint) unless subscription_arn
+
+      aws_client.unsubscribe(subscription_arn: subscription_arn)
     end
 
-    private
+    def list_all
+      topics = []
+      resp = aws_client.list_topics
+
+      loop do
+        resp[:topics].map do |t|
+          topics << Topic.from_arn(t[:topic_arn])
+        end
+
+        break if resp[:next_token].nil?
+
+        resp = aws_client.list_topics(next_token: resp[:next_token])
+      end
+      topics
+    end
+
+    def list_subscriptions
+      subscriptions = []
+      resp = aws_client.list_subscriptions
+
+      loop do
+        resp[:subscriptions].each do |s|
+          endpoint = s.endpoint.split(':')[2] == 'sqs' ? Queue.from_arn(s.endpoint) : Topic.from_arn(s.endpoint)
+          subscriptions << { topic: Topic.from_arn(s.topic_arn), endpoint: endpoint, arn: s.subscription_arn }
+        end
+
+        break if resp[:next_token].nil?
 
-    def get_arn(topic)
-      ['arn', 'aws', 'sns', region, client_id, topic].join ':'
+        resp = aws_client.list_subscriptions(next_token: resp[:next_token])
+      end
+      subscriptions
     end
 
-    def get_topic(kind, type)
-      [instance, kind, SNS_SUFFIX, publisher, type].join '-'
+    def topic_subscriptions(topic)
+      raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
+
+      subscriptions = []
+      resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn)
+
+      loop do
+        next_token = resp[:next_token]
+        subscriptions += resp[:subscriptions]
+
+        break if next_token.nil?
+
+        resp = aws_client.list_subscriptions_by_topic(topic_arn: topic.arn, next_token: next_token)
+      end
+      subscriptions
+    end
+
+    def find_subscription_arn(topic:, endpoint:)
+      raise ArgumentError, 'Should be Topic' unless topic.is_a? Topic
+      raise ArgumentError, 'Endpoint should be Topic or Queue' unless [Topic, Queue].include? endpoint.class
+
+      found_subscription = topic_subscriptions(topic).select do |subscription|
+        subscription.endpoint == endpoint.arn
+      end.first
+
+      found_subscription ? found_subscription.subscription_arn : nil
     end
   end
 end

data/lib/cyclone_lariat/sqs_client.rb

@@ -2,6 +2,7 @@
 
 require 'aws-sdk-sqs'
 require_relative 'abstract/client'
+require_relative 'queue'
 
 module CycloneLariat
   class SqsClient < Abstract::Client
@@ -9,31 +10,84 @@ module CycloneLariat
 
     dependency(:aws_client_class) { Aws::SQS::Client }
 
-    SQS_SUFFIX = :queue
-
-    def publish(msg, dest: nil, topic: nil)
-      raise ArgumentError, 'You should define dest or topic' if dest.nil? && topic.nil?
+    def custom_queue(name)
+      Queue.from_name(name, account_id: account_id, region: region)
+    end
 
-      topic ||= [instance, msg.kind, SQS_SUFFIX, publisher, msg.type, dest].join('-')
+    def queue(type = :all, fifo:, dest: nil, publisher: nil, kind: :event)
+      publisher ||= self.publisher
 
-      aws_client.send_message(
-        queue_url: url(topic),
-        message_body: msg.to_json
+      Queue.new(
+        instance: instance, publisher: publisher, region: region,
+        account_id: account_id, kind: kind, type: type, fifo: fifo, dest: dest
       )
     end
 
-    def publish_event(type, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, topic: nil)
-      publish event(type, data: data, version: version, uuid: uuid), dest: dest, topic: topic
+    def get_url(queue)
+      raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
+
+      aws_client.get_queue_url(queue_name: queue.to_s).queue_url
+    end
+
+    def exists?(queue)
+      raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
+
+      get_url(queue) && true
+    rescue Aws::SQS::Errors::NonExistentQueue
+      false
     end
 
-    def publish_command(type, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, topic: nil)
-      publish command(type, data: data, version: version, uuid: uuid), dest: dest, topic: topic
+    def publish(msg, fifo:, dest: nil, queue: nil)
+      queue = queue ? custom_queue(queue) : queue(msg.type, kind: msg.kind, fifo: fifo, dest: dest)
+      aws_client.send_message(queue_url: get_url(queue), message_body: msg.to_json)
     end
 
-    private
+    def publish_event(type, fifo:, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, queue: nil)
+      publish event(type, data: data, version: version, uuid: uuid, request_id: request_id),
+              fifo: fifo, dest: dest, queue: queue
+    end
+
+    def publish_command(type, fifo:, dest: nil, data: {}, version: self.version, uuid: SecureRandom.uuid, request_id: nil, queue: nil)
+      publish command(type, data: data, version: version, uuid: uuid, request_id: request_id),
+              fifo: fifo, dest: dest, queue: queue
+    end
+
+    def create(queue)
+      raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
+      raise Errors::QueueAlreadyExists.new(expected_queue: queue.name) if exists?(queue)
+
+      attrs = {}
+      attrs['FifoQueue'] = 'true' if queue.fifo
+
+      aws_client.create_queue(queue_name: queue.name, attributes: attrs, tags: queue.tags)
+      queue
+    end
+
+    def delete(queue)
+      raise ArgumentError, 'Should be queue' unless queue.is_a? Queue
+      raise Errors::QueueDoesNotExists.new(expected_queue: queue.name) unless exists?(queue)
+
+      aws_client.delete_queue queue_url: queue.url
+      queue
+    end
+
+    def list_all
+      queues = []
+      resp = aws_client.list_queues
+
+      loop do
+        next_token = resp[:next_token]
+
+        resp[:queue_urls].map do |url|
+          queues << Queue.from_url(url)
+        end
+
+        break if next_token.nil?
+
+        resp = aws_client.list_queues(next_token: next_token)
+      end
 
-    def url(topic_name)
-      aws_client.get_queue_url(queue_name: topic_name).queue_url
+      queues
     end
   end
 end

data/lib/cyclone_lariat/topic.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module CycloneLariat
+  class Topic
+    SNS_SUFFIX = :fanout
+
+    attr_reader :instance, :kind, :region, :account_id, :publisher, :type, :fifo, :tags
+
+    def initialize(instance:, kind:, region:, account_id:, publisher:, type:, fifo:, tags: nil, name: nil)
+      @instance  = instance
+      @kind      = kind
+      @region    = region
+      @account_id = account_id
+      @publisher = publisher
+      @type      = type
+      @fifo      = fifo
+      @tags      = tags || default_tags(instance, kind, publisher, type, fifo)
+      @name      = name
+    end
+
+    def arn
+      ['arn', 'aws', 'sns', region, account_id, to_s].join ':'
+    end
+
+    def custom?
+      !standard?
+    end
+
+    def standard?
+      instance && kind && publisher && type && true
+    end
+
+    def name
+      @name ||= begin
+        name = [instance, kind, SNS_SUFFIX, publisher, type].join '-'
+        name += '.fifo' if fifo
+        name
+      end
+    end
+
+    def attributes
+      fifo ? { 'FifoTopic' => 'true' } : {}
+    end
+
+    def topic?
+      true
+    end
+
+    def queue?
+      false
+    end
+
+    def protocol
+      'sns'
+    end
+
+    alias to_s name
+
+    def ==(other)
+      arn == other.arn
+    end
+
+    class << self
+      def from_name(name, region:, account_id:)
+        is_fifo_array  = name.split('.')
+        full_name      = is_fifo_array[0]
+        fifo_suffix    = is_fifo_array[-1]
+        suffix_exists  = fifo_suffix != full_name
+
+        if suffix_exists && fifo_suffix != 'fifo'
+          raise ArgumentError, "Topic name #{name} consists unexpected suffix #{fifo_suffix}"
+        end
+
+        fifo = suffix_exists
+        topic_array = full_name.split('-')
+
+        raise ArgumentError, "Topic name should consists `#{SNS_SUFFIX}`" unless topic_array[2] != SNS_SUFFIX
+
+        new(
+          instance: topic_array[0],
+          kind: topic_array[1],
+          publisher: topic_array[3],
+          type: topic_array[4],
+          region: region,
+          account_id: account_id,
+          fifo: fifo,
+          name: name
+        )
+      end
+
+      def from_arn(arn)
+        arn_array = arn.split(':')
+        raise ArgumentError, 'Arn should consists `arn`' unless arn_array[0] == 'arn'
+        raise ArgumentError, 'Arn should consists `aws`' unless arn_array[1] == 'aws'
+        raise ArgumentError, 'Arn should consists `sns`' unless arn_array[2] == 'sns'
+
+        from_name(arn_array[5], region: arn_array[3], account_id: arn_array[4])
+      end
+    end
+
+    private
+
+    def default_tags(instance, kind, publisher, type, fifo)
+      [
+        { key: 'instance',  value: String(instance) },
+        { key: 'kind',      value: String(kind) },
+        { key: 'publisher', value: String(publisher) },
+        { key: 'type',      value: String(type) },
+        { key: 'fifo',      value: fifo ? 'true' : 'false' }
+      ]
+    end
+  end
+end

data/lib/cyclone_lariat/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CycloneLariat
-  VERSION = '0.3.10'
+  VERSION = '0.4.0'
 end

data/lib/cyclone_lariat.rb

@@ -6,6 +6,7 @@ require_relative 'cyclone_lariat/errors'
 require_relative 'cyclone_lariat/event'
 require_relative 'cyclone_lariat/messages_mapper'
 require_relative 'cyclone_lariat/messages_repo'
+require_relative 'cyclone_lariat/migration'
 require_relative 'cyclone_lariat/middleware'
 require_relative 'cyclone_lariat/version'
 

data/lib/tasks/console.rake

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+desc 'IRB console with required CycloneLariat'
+task :console do
+  # require 'cyclone_lariat'
+  require_relative '../../lib/cyclone_lariat'
+  require 'irb'
+  require_relative '../../config/initializers/cyclone_lariat'
+  # require_relative(init_file) if File.exists?(init_file)
+
+  ARGV.clear
+  IRB.start
+end

data/lib/tasks/cyclone_lariat.rake

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'cyclone_lariat'
+
+namespace :cyclone_lariat do
+  desc 'Migrate topics for SQSSNS'
+  task migrate: :config do
+    require_relative '../../config/initializers/cyclone_lariat'
+    CycloneLariat::Migration.migrate
+  end
+
+  desc 'Rollback topics for SQSSNS'
+  task :rollback, [:version] => :config do |_, args|
+    require_relative '../../config/initializers/cyclone_lariat'
+    target_version = args[:version] ? args[:version].to_i : nil
+    CycloneLariat::Migration.rollback(target_version)
+  end
+
+  namespace :list do
+    desc 'List all topics'
+    task :topics do
+      require_relative '../../config/initializers/cyclone_lariat'
+      CycloneLariat::Migration.list_topics
+    end
+
+    desc 'List all queues'
+    task :queues do
+      require_relative '../../config/initializers/cyclone_lariat'
+      CycloneLariat::Migration.list_queues
+    end
+
+    desc 'List all subscriptions'
+    task :subscriptions do
+      require_relative '../../config/initializers/cyclone_lariat'
+      CycloneLariat::Migration.list_subscriptions
+    end
+  end
+
+  desc 'Build graphviz graph for whole system'
+  task :graph do
+    require_relative '../../config/initializers/cyclone_lariat'
+    CycloneLariat::Migration.build_graph
+  end
+end

data/lib/tasks/db.rake

@@ -22,7 +22,7 @@ namespace :db do
     puts "Database `#{DB_CONF[:database]}` successfully dropped" if system(cmd)
   end
 
-  desc 'Apply migrations'
+  desc 'Apply migrate'
   task :migrate, [:version] => :config do |_, args|
     require 'logger'
     require 'sequel/core'