cyclone_lariat 0.3.10 → 0.4.0

data/bin/cyclone_lariat

@@ -0,0 +1,174 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/cyclone_lariat'
+require 'bundler/setup'
+require 'dry/cli'
+require 'fileutils'
+
+module CycloneLariat
+  module CLI
+    module Commands
+      extend Dry::CLI::Registry
+
+      INITIALIZERS_DIR = './config/initializers'
+      RAKE_TASKS_DIR = './lib/tasks'
+
+      class Version < Dry::CLI::Command
+        desc 'Print version'
+
+        def call(*)
+          puts CycloneLariat::VERSION
+        end
+      end
+
+      class Install < Dry::CLI::Command
+        desc 'Install cyclone lariat to current directory'
+
+        def call(*)
+          create_config
+          create_rake_task
+        end
+
+        def create_config
+          FileUtils.mkdir_p RAKE_TASKS_DIR unless Dir.exist? RAKE_TASKS_DIR
+          config_path = "#{RAKE_TASKS_DIR}/cyclone_lariat.rake"
+          config_file = File.open(config_path, 'w')
+          config_file.puts rake_task_context
+          puts "Created rake task: #{config_path}"
+        end
+
+        def create_rake_task
+          FileUtils.mkdir_p INITIALIZERS_DIR unless Dir.exist? INITIALIZERS_DIR
+          config_path = "#{INITIALIZERS_DIR}/cyclone_lariat.rb"
+          config_file = File.open(config_path, 'w')
+          config_file.puts config_contents
+          puts "Created config: #{config_path}"
+        end
+
+        def config_contents
+          <<~CONFIG
+            # frozen_string_literal: true
+
+            CycloneLariat.tap do |cl|
+              cl.default_version    = 1                       # api version
+              cl.aws_key            = ENV['AWS_KEY']          # aws key
+              cl.aws_account_id     = ENV['AWS_ACCOUNT_ID']   # aws account id
+              cl.aws_secret_key     = ENV['AWS_SECRET_KEY']   # aws secret
+              cl.aws_default_region = ENV['AWS_REGION']       # aws default region
+              cl.publisher          = ENV['APP_NAME']         # name of your publishers, usually name of your application 
+              cl.default_instance   = ENV['INSTANCE']         # stage, production, test
+              cl.events_dataset     = DB[:events]             # sequel dataset for store income messages, for receiver
+              cl.versions_dataset   = DB[:lariat_versions]    # sequel dataset for migrations, for publisher
+            end
+          CONFIG
+        end
+
+        def rake_task_context
+          <<~TASKS
+            # frozen_string_literal: true
+
+            require 'cyclone_lariat'
+            
+            namespace :cyclone_lariat do
+              desc 'Migrate topics for SQS/SNS'
+              task migrate: :config do
+                require_relative '../../config/initializers/cyclone_lariat'
+                CycloneLariat::Migration.migrate
+              end
+            
+              desc 'Rollback topics for SQS/SNS'
+              task :rollback, [:version] => :config do |_, args|
+                require_relative '../../config/initializers/cyclone_lariat'
+                target_version = args[:version] ? args[:version].to_i : nil
+                CycloneLariat::Migration.rollback(target_version)
+              end
+            
+              namespace :list do
+                desc 'List all topics'
+                task :topics do
+                  require_relative '../../config/initializers/cyclone_lariat'
+                  CycloneLariat::Migration.list_topics
+                end
+            
+                desc 'List all queues'
+                task :queues do
+                  require_relative '../../config/initializers/cyclone_lariat'
+                  CycloneLariat::Migration.list_queues
+                end
+            
+                desc 'List all subscriptions'
+                task :subscriptions do
+                  require_relative '../../config/initializers/cyclone_lariat'
+                  CycloneLariat::Migration.list_subscriptions
+                end
+              end
+            
+              desc 'Build graphviz graph for whole system'
+              task :graph do
+                require_relative '../../config/initializers/cyclone_lariat'
+                CycloneLariat::Migration.build_graph
+              end
+            end
+          TASKS
+        end
+      end
+
+      module Generate
+        class Migration < Dry::CLI::Command
+          desc 'Generate migration'
+
+          argument :title, type: :string, required: true, desc: 'Title of migration use only a-z and _'
+
+          def call(title:, **)
+            abort('Use only a-z and _ in your title') unless title_correct? title
+
+            FileUtils.mkdir_p CycloneLariat::Migration::DIR unless Dir.exist? CycloneLariat::Migration::DIR
+
+            file_name  = generate_filename title
+            class_name = generate_class_name title
+            file       = File.open(file_name, 'w')
+            file.puts file_contents(class_name)
+            puts "Migration successful created:\n\t#{file_name}"
+          end
+
+          private
+
+          def title_correct?(title)
+            /^(?!.*__.*)[a-z]?[a-z_]+[a-z]+$/.match? title
+          end
+
+          def generate_filename(title)
+            "#{CycloneLariat::Migration::DIR}/#{Time.now.to_i}_#{title}.rb"
+          end
+
+          def generate_class_name(title)
+            title.split('_').collect(&:capitalize).join
+          end
+
+          def file_contents(klass_name)
+            <<~MIGRATION
+              # frozen_string_literal: true
+
+              class #{klass_name} < CycloneLariat::Migration
+                def up
+                end
+              
+                def down
+                end
+              end
+            MIGRATION
+          end
+        end
+      end
+
+      register 'version', Version, aliases: %w[v -v --version]
+      register 'install', Install
+      register 'generate', aliases: %w[g] do |prefix|
+        prefix.register 'migration', Generate::Migration
+      end
+    end
+  end
+end
+
+Dry::CLI.new(CycloneLariat::CLI::Commands).call

data/config/initializers/sequel.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative '../db'
+
+require 'sequel'
+
+DB = Sequel.connect(DB_CONF)

data/cyclone_lariat.gemspec

@@ -30,11 +30,15 @@ Gem::Specification.new do |spec|
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
+
   spec.require_paths = ['lib']
+  spec.executables   = ['cyclone_lariat']
 
   spec.add_dependency 'aws-sdk-sns'
   spec.add_dependency 'aws-sdk-sqs'
+  spec.add_dependency 'dry-cli', '~> 0.6'
   spec.add_dependency 'luna_park', '~> 0.11'
+  spec.add_dependency 'terminal-table', '~> 3.0'
 
   spec.add_development_dependency 'bundler', '~> 1.17'
   spec.add_development_dependency 'byebug', '~> 11.1'

data/db/migrate/03_add_versions.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table :lariat_versions do
+      Integer :version, null: false, unique: true
+    end
+  end
+end

data/lib/cyclone_lariat/abstract/client.rb

@@ -14,36 +14,42 @@ module CycloneLariat
       dependency(:aws_client_class)      { raise ArgumentError, 'Client class should be defined' }
       dependency(:aws_credentials_class) { Aws::Credentials }
 
-      def initialize(key: nil, secret_key: nil, region: nil, version: nil, publisher: nil, instance: nil, client_id: nil)
+      def initialize(key: nil, secret_key: nil, region: nil, version: nil, publisher: nil, instance: nil, account_id: nil)
         @key = key
         @secret_key = secret_key
         @region = region
         @version = version
         @publisher = publisher
         @instance = instance
-        @client_id = client_id
+        @account_id = account_id
       end
 
-      def event(type, data: {}, version: self.version, uuid: SecureRandom.uuid)
-        Event.wrap(
+      def event(type, data: {}, version: self.version, request_id: nil, uuid: SecureRandom.uuid)
+        params = {
           uuid: uuid,
           type: type,
-          sent_at: Time.now.iso8601,
+          sent_at: Time.now.iso8601(3),
           version: version,
           publisher: publisher,
-          data: data
-        )
+          data: data,
+          request_id: request_id
+        }
+
+        Event.wrap(params.compact)
       end
 
-      def command(type, data: {}, version: self.version, uuid: SecureRandom.uuid)
-        Command.wrap(
+      def command(type, data: {}, version: self.version, request_id: nil, uuid: SecureRandom.uuid)
+        params = {
           uuid: uuid,
           type: type,
-          sent_at: Time.now.iso8601,
+          sent_at: Time.now.iso8601(3),
           version: version,
           publisher: publisher,
-          data: data
-        )
+          data: data,
+          request_id: request_id
+        }
+
+        Command.wrap(params.compact)
       end
 
       def publish
@@ -88,8 +94,8 @@ module CycloneLariat
         @region ||= CycloneLariat.aws_default_region
       end
 
-      def client_id
-        @client_id ||= CycloneLariat.aws_client_id
+      def account_id
+        @account_id ||= CycloneLariat.aws_account_id
       end
 
       private

data/lib/cyclone_lariat/abstract/message.rb

@@ -6,10 +6,10 @@ require_relative '../errors'
 module CycloneLariat
   module Abstract
     class Message < LunaPark::Entities::Attributable
-      attr :uuid,      String, :new
-      attr :publisher, String, :new
-      attr :type,      String, :new
-      attrs :client_error, :version, :data,
+      attr :uuid,       String, :new
+      attr :publisher,  String, :new
+      attr :type,       String, :new
+      attrs :client_error, :version, :data, :request_id,
             :sent_at, :processed_at, :received_at
 
       def kind
@@ -32,6 +32,10 @@ module CycloneLariat
         @processed_at = wrap_time(value)
       end
 
+      def request_at=(value)
+        @request_id = wrap_string(value)
+      end
+
       def processed?
         !@processed_at.nil?
       end
@@ -64,7 +68,10 @@ module CycloneLariat
 
       def to_json(*args)
         hash = serialize
-        hash[:type] = [kind, hash[:type]].join '_'
+        hash[:type]         = [kind, hash[:type]].join '_'
+        hash[:sent_at]      = hash[:sent_at].iso8601(3)      if hash[:sent_at]
+        hash[:received_at]  = hash[:received_at].iso8601(3)  if hash[:received_at]
+        hash[:processed_at] = hash[:processed_at].iso8601(3) if hash[:processed_at]
         hash.to_json(*args)
       end
 
@@ -78,6 +85,14 @@ module CycloneLariat
         else raise ArgumentError, "Unknown type `#{value.class}`"
         end
       end
+
+      def wrap_string(value)
+        case value
+        when String then String(value)
+        when NilClass then nil
+        else raise ArgumentError, "Unknown type `#{value.class}`"
+        end
+      end
     end
   end
 end

data/lib/cyclone_lariat/configure.rb

@@ -5,7 +5,7 @@ module CycloneLariat
     DEFAULT_VERSION = 1
 
     attr_accessor :aws_key, :aws_secret_key, :publisher, :aws_default_region, :default_instance,
-                  :aws_client_id
+                  :aws_account_id, :events_dataset, :versions_dataset
     attr_writer :default_version
 
     def default_version

data/lib/cyclone_lariat/errors.rb

@@ -18,5 +18,27 @@ module CycloneLariat
           other.details == details
       end
     end
+
+    class TopicAlreadyExists < LunaPark::Errors::System
+      message { |d| "Topic already exists: `#{d[:expected_topic]}`" }
+    end
+
+    class TopicDoesNotExists < LunaPark::Errors::System
+      message { |d| "Topic does not exists: `#{d[:expected_topic]}`" }
+    end
+
+    class QueueAlreadyExists < LunaPark::Errors::System
+      message { |d| "Queue already exists: `#{d[:expected_queue]}`" }
+    end
+
+    class QueueDoesNotExists < LunaPark::Errors::System
+      message { |d| "Queue does not exists: `#{d[:expected_queue]}`" }
+    end
+    class SubscriptionAlreadyExists < LunaPark::Errors::System
+      message { |d| "Subscription for topic `#{d[:topic].name}`, on endpoint `#{d[:endpoint].name}` already exists" }
+    end
+    class SubscriptionDoesNotExists < LunaPark::Errors::System
+      message { |d| "Subscription for topic `#{d[:topic].name}`, on endpoint `#{d[:endpoint].name}` does not exists" }
+    end
   end
 end

data/lib/cyclone_lariat/middleware.rb

@@ -7,7 +7,8 @@ require 'json'
 module CycloneLariat
   class Middleware
     def initialize(dataset: nil, errors_notifier: nil, message_notifier: nil, repo: MessagesRepo)
-      @events_repo      = repo.new(dataset) if dataset
+      events_dataset    = dataset || CycloneLariat.events_dataset
+      @events_repo      = repo.new(events_dataset) if events_dataset
       @message_notifier = message_notifier
       @errors_notifier  = errors_notifier
     end

data/lib/cyclone_lariat/migration.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'forwardable'
+require_relative 'sns_client'
+require_relative 'sqs_client'
+require 'luna_park/errors'
+require 'terminal-table'
+require 'set'
+
+module CycloneLariat
+  class Migration
+    extend Forwardable
+    include LunaPark::Extensions::Injector
+
+    dependency(:sns) { CycloneLariat::SnsClient.new }
+    dependency(:sqs) { CycloneLariat::SqsClient.new }
+
+    DIR = './lariat/migrate'
+
+    def up
+      raise LunaPark::Errors::Abstract, "Up method should be defined in #{self.class.name}"
+    end
+
+    def down
+      raise LunaPark::Errors::Abstract, "Down method should be defined in #{self.class.name}"
+    end
+
+    def_delegators :sqs, :queue, :custom_queue
+    def_delegators :sns, :topic, :custom_topic
+
+    def create(resource)
+      process(
+        resource: resource,
+        for_topic: ->(topic) { sns.create(topic) },
+        for_queue: ->(queue) { sqs.create(queue) }
+      )
+
+      puts "  #{resource.class.name.split('::').last} was created `#{resource.name}`"
+    end
+
+    def delete(resource)
+      process(
+        resource: resource,
+        for_topic: ->(topic) { sns.delete(topic) },
+        for_queue: ->(queue) { sqs.delete(queue) }
+      )
+      puts "  #{resource.class.name.split('::').last} was deleted `#{resource.name}`"
+    end
+
+    def exists?(resource)
+      process(
+        resource: resource,
+        for_topic: ->(topic) { sns.exists?(topic) },
+        for_queue: ->(queue) { sqs.exists?(queue) }
+      )
+    end
+
+    def subscribe(topic:, endpoint:)
+      sns.subscribe topic: topic, endpoint: endpoint
+      puts "  Subscription was created `#{topic.name} -> #{endpoint.name}`"
+    end
+
+    def unsubscribe(topic:, endpoint:)
+      sns.unsubscribe topic: topic, endpoint: endpoint
+      puts "  Subscription was deleted `#{topic.name} -> #{endpoint.name}`"
+    end
+
+    def topics
+      sns.list_all
+    end
+
+    def queues
+      sqs.list_all
+    end
+
+    def subscriptions
+      sns.list_subscriptions
+    end
+
+    private
+
+    def process(resource:, for_topic:, for_queue:)
+      case resource
+      when Topic then for_topic.call(resource)
+      when Queue then for_queue.call(resource)
+      else
+        raise ArgumentError, "Unknown resource class #{resource.class}"
+      end
+    end
+
+    class << self
+      def migrate(dataset: CycloneLariat.versions_dataset, dir: DIR)
+        alert('No one migration exists') if !Dir.exist?(dir) || Dir.empty?(dir)
+
+        Dir.glob("#{dir}/*.rb") do |path|
+          filename = File.basename(path, '.rb')
+          version, title = filename.split('_', 2)
+
+          existed_migrations = dataset.all.map { |row| row[:version] }
+          unless existed_migrations.include? version.to_i
+            class_name = title.split('_').collect(&:capitalize).join
+            puts "Up - #{version} #{class_name} #{path}"
+            require_relative Pathname.new(Dir.pwd) + Pathname.new(path)
+            Object.const_get(class_name).new.up
+            dataset.insert(version: version)
+          end
+        end
+      end
+
+      def rollback(version = nil, dataset: CycloneLariat.versions_dataset, dir: DIR)
+        existed_migrations = dataset.all.map { |row| row[:version] }.sort
+        version ||= existed_migrations[-1]
+        migrations_to_downgrade = existed_migrations.select { |migration| migration >= version }
+
+        paths = []
+        migrations_to_downgrade.each do |migration|
+          path = Pathname.new(Dir.pwd) + Pathname.new(dir)
+          founded = Dir.glob("#{path}/#{migration}_*.rb")
+          raise "Could not found migration: `#{migration}` in #{path}" if founded.empty?
+          raise "Found lot of migration: `#{migration}` in #{path}"    if founded.size > 1
+
+          paths += founded
+        end
+
+        paths.each do |path|
+          filename       = File.basename(path, '.rb')
+          version, title = filename.split('_', 2)
+          class_name     = title.split('_').collect(&:capitalize).join
+          puts "Down - #{version} #{class_name} #{path}"
+          require_relative Pathname.new(Dir.pwd) + Pathname.new(path)
+          Object.const_get(class_name).new.down
+          dataset.filter(version: version).delete
+        end
+      end
+
+      def list_topics
+        rows = []
+        new.topics.each do |topic|
+          rows << [
+            topic.custom? ? 'custom' : 'standard',
+            topic.region,
+            topic.account_id,
+            topic.name,
+            topic.instance,
+            topic.kind,
+            topic.publisher,
+            topic.type,
+            topic.fifo
+          ]
+        end
+
+        puts Terminal::Table.new rows: rows, headings: %w[valid region account_id name instance kind publisher type fifo]
+      end
+
+      def list_queues
+        rows = []
+        new.queues.each do |queue|
+          rows << [
+            queue.custom? ? 'custom' : 'standard',
+            queue.region,
+            queue.account_id,
+            queue.name,
+            queue.instance,
+            queue.kind,
+            queue.publisher,
+            queue.type,
+            queue.dest,
+            queue.fifo
+          ]
+        end
+
+        puts Terminal::Table.new rows: rows, headings: %w[valid region account_id name instance kind publisher type destination fifo]
+      end
+
+      def list_subscriptions
+        rows = []
+        new.subscriptions.each do |subscription|
+          rows << [
+            subscription[:topic].name,
+            subscription[:endpoint].name,
+            subscription[:arn]
+          ]
+        end
+
+        puts Terminal::Table.new rows: rows, headings: %w[topic endpoint subscription_arn]
+      end
+
+      def build_graph
+        subscriptions = new.subscriptions
+        resources_set = Set.new
+
+        subscriptions.each do |subscription|
+          resources_set << subscription[:topic]
+          resources_set << subscription[:endpoint]
+        end
+
+        puts 'digraph G {'
+        puts '  rankdir=LR;'
+
+        resources_set.each do |resource|
+          color = resource.custom? ? ', fillcolor=grey' : ', fillcolor=white'
+          style = resource.topic? ? "[shape=component style=filled#{color}]" : "[shape=record, style=\"rounded,filled\"#{color}]"
+          puts "  \"#{resource.name}\" #{style};"
+        end
+
+        subscriptions.each do |subscription|
+          puts "  \"#{subscription[:topic].name}\" -> \"#{subscription[:endpoint].name}\";"
+        end
+        puts '}'
+      end
+    end
+  end
+end