kybus-cli 0.1.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2787f36a297b9c14e6c9fb0d4b28284721e4065c0c2a7c4f64db8246da1767fc
-  data.tar.gz: 619dcf54613ed6d18aecec46e84ae1b1fecad32c05882ec5e2b6d1d4ca4bc8aa
+  metadata.gz: 3103d3aeaedc50e8bd321cd103bd2f1b2aa42ddddda7dcceba146aa016ea687c
+  data.tar.gz: f1c1f730b4ed14b631e5b8773bfa506859fc60e121c3588bbe835e84dc8a861a
 SHA512:
-  metadata.gz: 4e2cd597770a3f06ef11a3077752d35608f90883def9e89be0b78cc10c4256d1039fa5877dd38d217c1702fbe9b7b82d1026c3cf9a0d79ac90e41429b2b06ed9
-  data.tar.gz: f2fc53893eb88358018a1a3251c0b78d24120735d8ec9e09ce81ef26ab985768800bb9e83de51810b7b66f18931041a6d6eaf67a5cb4a0797ae31cdfdcfb6008
+  metadata.gz: d2901d1d9f717a1838ed670f0eacf24c4d3c0dd96a160ea7f082ae28506d1d399df3de8af5a0b4d41fc7fe3f8a53774a061e54bc53208650134f1efc1eab11b4
+  data.tar.gz: b83a368aa834d44fcd1175756ee147047ed16bedd8d894ab5f39edb42074f5988ab917edf4bacf00b63e9be059a6f75272072936fd73d2d3c4f0fe82c4911aaf

data/lib/kybus/aws/code_packager.rb

@@ -1,47 +1,49 @@
 # frozen_string_literal: true
 
+require 'digest'
+require_relative 'zip_creator'
+
 module Kybus
   module AWS
     class CodePackager < Resource
       def create_or_update!
-        ruby_version = RUBY_VERSION.split('.')[0..1].join('.') + '.0'
-        create_zip('.deps.zip', "vendor/bundle/ruby/#{ruby_version}", zip_root: "ruby/gems/#{ruby_version}")
-        create_zip('.kybuscode.zip', @config['repo_path'], exclude_files: [
-                     'test', 'Gemfile*', 'Rakefile', '.gitignore', '.bundle', '.git', '.deps.zip', '.kybuscode.zip', 'kybusbot.yaml', 'vendor', '.ruby-version'
-                   ], extra_files: { '.bundle/config' => 'BUNDLE_PATH: "/opt/vendor/bundle"' }, zip_root: '.')
+        ruby_version = fetch_ruby_version
+        create_deps_zip(ruby_version)
+        create_code_zip(ruby_version)
       end
 
-      def create_zip(zip_name, directory, exclude_files: [], extra_files: {}, zip_root: '')
-        require 'zip'
-        FileUtils.rm(zip_name, force: true)
-        entries = Dir.entries(directory) - %w[. ..]
-
-        Zip::File.open(zip_name, Zip::File::CREATE) do |zipfile|
-          extra_files.each do |entry, contents|
-            zipfile.get_output_stream(entry) { |f| f.puts(contents) }
-          end
-
-          entries.each do |entry|
-            entry_path = File.join(directory, entry)
-            next if exclude_files.any? { |pattern| File.fnmatch(pattern, entry) || File.fnmatch(pattern, entry_path) }
-
-            puts "Adding #{entry} to #{zip_name}"
-
-            if File.directory?(entry_path)
-              zipfile.mkdir("#{zip_root}#{entry_path.sub(directory, '')}")
-              Dir[File.join(entry_path, '**', '**')].each do |file|
-                next if exclude_files.any? { |pattern| File.fnmatch(pattern, file) }
-
-                zipfile.add("#{zip_root}#{file.sub(directory, '')}", file)
-              end
-            else
-              zipfile.add("#{zip_root}#{entry_path.sub(directory, '')}", entry_path)
-            end
-          end
-        end
+      def destroy!; end
+
+      private
+
+      def fetch_ruby_version
+        "#{RUBY_VERSION.split('.')[0..1].join('.')}.0"
       end
 
-      def destroy!; end
+      def create_deps_zip(ruby_version)
+        ZipCreator.new(
+          '.deps.zip',
+          "vendor/bundle/ruby/#{ruby_version}",
+          zip_root: "ruby/gems/#{ruby_version}"
+        ).create_zip
+      end
+
+      def create_code_zip(_ruby_version)
+        ZipCreator.new(
+          '.kybuscode.zip',
+          @config['repo_path'],
+          exclude_files: excluded_files,
+          extra_files: { '.bundle/config' => 'BUNDLE_PATH: "/opt/vendor/bundle"' },
+          zip_root: '.'
+        ).create_zip
+      end
+
+      def excluded_files
+        [
+          'test', 'Gemfile*', 'Rakefile', '.gitignore', '.bundle', '.git', '.deps.zip',
+          '.kybuscode.zip', 'kybusbot.yaml', 'vendor', '.ruby-version'
+        ]
+      end
     end
   end
 end

data/lib/kybus/aws/lambda.rb

@@ -1,158 +1,115 @@
 # frozen_string_literal: true
 
 require 'digest'
-
-def calculate_md5(file_path)
-  md5 = Digest::MD5.new
-  File.open(file_path, 'rb') do |file|
-    buffer = String.new
-    md5.update(buffer) while file.read(4096, buffer)
-  end
-  md5.hexdigest
-end
+require_relative 'layer_manager'
+require_relative 'lambda_trigger'
 
 module Kybus
   module AWS
     class Lambda < Resource
-      attr_reader :url, :name
+      attr_reader :name
 
       def initialize(configs, name)
         super(configs)
         @name = name
+        @layers = configs['layers']
+        @triggers = configs['triggers']
+        @layer_manager = LayerManager.new(lambda_client, function_name)
+        @trigger_manager = LambdaTrigger.new(lambda_client, function_name, @triggers)
       end
 
       def lambda_client
         @lambda_client ||= Aws::Lambda::Client.new(region: @region)
       end
 
+      def url
+        @trigger_manager.url
+      end
+
       def function_name
         @name
       end
 
       def deploy_lambda!
-        layer_arn_deps = create_or_update_layer('.deps.zip', "#{function_name}-deps")
+        layer_arns = @layers.map { |layer| handle_layer(layer) }
 
-        function_exists = begin
-          lambda_client.get_function(function_name:)
-        rescue StandardError
-          false
-        end
-
-        if function_exists
-          update_lambda!(layer_arn_deps)
+        if lambda_function_exists?
+          update_lambda!(layer_arns)
         else
-          create_lambda!(layer_arn_deps)
+          create_lambda!(layer_arns)
         end
       end
 
-      def create_layer(zip_file, layer_name, zipfile_hash)
-        response = lambda_client.publish_layer_version({
-                                                         layer_name:,
-                                                         content: {
-                                                           zip_file: File.read(zip_file)
-                                                         },
-                                                         description: zipfile_hash,
-                                                         compatible_runtimes: ["ruby3.3"]
-                                                       })
-        puts "Layer '#{layer_name}' created: #{response.layer_version_arn}"
-        response.layer_version_arn
-      end
+      def handle_layer(layer)
+        puts "Processing layer:\n#{layer.to_yaml}"
+        case layer['type']
+        when 'codezip'
+          raise 'Checksum file required for codezip layer' unless layer['checksumfile']
 
-      def create_or_update_layer(zip_file, layer_name)
-        layer_exists = begin
-          response = lambda_client.list_layer_versions({
-            layer_name: layer_name,
-            max_items: 1
-          })
-
-          response.layer_versions.first
-        end
+          @layer_manager.create_or_update_layer(layer['zipfile'], layer['name'], layer['checksumfile'])
+        when 'existing'
+          layer_arn = @layer_manager.get_layer_arn(layer['name'])
+          raise "Layer #{layer['name']} not found" unless layer_arn
 
-        zipfile_hash = calculate_md5('Gemfile.lock')
-        
-        if !layer_exists || layer_exists.description != zipfile_hash
-          create_layer(zip_file, layer_name, zipfile_hash)
+          layer_arn
         else
-          puts "Layer unmodified: #{layer_name}"
-          layer_exists.layer_version_arn
+          raise "Unknown layer type: #{layer['type']}"
         end
       end
 
-      def update_lambda!(layer_arn_deps)
-        with_retries(Aws::Lambda::Errors::ResourceConflictException) do
-          lambda_client.update_function_configuration({
-                                                        function_name:,
-                                                        layers: [layer_arn_deps],
-                                                        timeout: @config['timeout'] || 3,
-                                                        environment: {
-                                                          variables: {
-                                                            'SECRET_TOKEN' => @config['secret_token']
-                                                          }
-                                                        }
-                                                      })
-        end
+      def lambda_function_exists?
+        lambda_client.get_function(function_name:)
+        true
+      rescue Aws::Lambda::Errors::ResourceNotFoundException
+        false
+      end
 
-        with_retries(Aws::Lambda::Errors::ResourceConflictException) do
-          lambda_client.update_function_code(function_name:, zip_file: File.read('.kybuscode.zip'))
-        end
+      def update_lambda!(layer_arns)
+        update_function_configuration(layer_arns)
+        update_function_code
         puts "Lambda function '#{function_name}' updated."
       end
 
-      def create_lambda!(layer_arn_deps)
+      def update_function_configuration(layer_arns)
         with_retries(Aws::Lambda::Errors::ResourceConflictException) do
-          lambda_client.create_function({
-                                          function_name:,
-                                          runtime: 'ruby3.3',
-                                          role: "arn:aws:iam::#{account_id}:role/#{function_name}",
-                                          handler: 'handler.lambda_handler',
-                                          layers: [layer_arn_deps],
-                                          code: {
-                                            zip_file: File.read('.kybuscode.zip')
-                                          },
-                                          timeout: @config['timeout'] || 3,
-                                          environment: {
-                                            variables: {
-                                              'SECRET_TOKEN' => @config['secret_token']
-                                            }
-                                          }
-                                        })
-          puts "Lambda function '#{function_name}' created."
+          lambda_client.update_function_configuration(
+            function_name:,
+            layers: layer_arns,
+            timeout: @config['timeout'] || 3,
+            environment: { variables: { 'SECRET_TOKEN' => @config['secret_token'] } }
+          )
         end
       end
 
-      def make_public!
+      def update_function_code
         with_retries(Aws::Lambda::Errors::ResourceConflictException) do
-          response = lambda_client.create_function_url_config({
-                                                                function_name:,
-                                                                auth_type: 'NONE'
-                                                              })
-          puts "Function URL created: #{response.function_url}"
-          @url = response.function_url
-        rescue Aws::Lambda::Errors::ResourceConflictException
-          response = lambda_client.get_function_url_config({
-                                                             function_name:
-                                                           })
-          puts "Function URL exists: #{response.function_url}"
-          @url = response.function_url
+          lambda_client.update_function_code(
+            function_name:,
+            zip_file: File.read('.kybuscode.zip')
+          )
         end
+      end
 
-        begin
-          response = lambda_client.add_permission({
-                                                    function_name:,
-                                                    statement_id: 'AllowPublicInvoke',
-                                                    action: 'lambda:InvokeFunctionUrl',
-                                                    principal: '*',
-                                                    function_url_auth_type: 'NONE'
-                                                  })
-          puts "Permission added successfully: #{response}"
-        rescue Aws::Lambda::Errors::ServiceError => e
-          puts "Error adding permission: #{e.message}"
+      def create_lambda!(layer_arns)
+        with_retries(Aws::Lambda::Errors::ResourceConflictException) do
+          puts "Creating function #{function_name} with role: #{"arn:aws:iam::#{account_id}:role/#{function_name}"}"
+          lambda_client.create_function(
+            function_name:,
+            runtime: 'ruby3.3',
+            role: "arn:aws:iam::#{account_id}:role/#{function_name}",
+            handler: @config['handler'] || 'handler.lambda_handler',
+            layers: layer_arns,
+            code: { zip_file: File.read('.kybuscode.zip') },
+            timeout: @config['timeout'] || 3,
+            environment: { variables: { 'SECRET_TOKEN' => @config['secret_token'] } }
+          )
+          puts "Lambda function '#{function_name}' created."
         end
       end
 
       def create_or_update!
         deploy_lambda!
-        make_public!
+        @trigger_manager.add_triggers
       end
 
       def destroy!

data/lib/kybus/aws/lambda_trigger.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Kybus
+  module AWS
+    class LambdaTrigger
+      attr_reader :url
+
+      def initialize(lambda_client, function_name, triggers)
+        @lambda_client = lambda_client
+        @function_name = function_name
+        @triggers = triggers
+      end
+
+      def add_triggers
+        @triggers.each do |trigger|
+          case trigger['type']
+          when 'url'
+            create_function_url(trigger['public'])
+          when 'sqs'
+            add_sqs_trigger(trigger['queue_arn'])
+          else
+            raise "Unknown trigger type: #{trigger['type']}"
+          end
+        end
+      end
+
+      private
+
+      def create_function_url(is_public)
+        @url = begin
+          @lambda_client.create_function_url_config(function_name: @function_name, auth_type: 'NONE')
+        rescue Aws::Lambda::Errors::ResourceConflictException
+          @lambda_client.get_function_url_config(function_name: @function_name)
+        end.function_url
+        puts "Function URL created: #{@url}"
+
+        return unless is_public
+
+        add_public_permission
+      end
+
+      def add_public_permission
+        response = @lambda_client.add_permission(
+          function_name: @function_name,
+          statement_id: 'AllowPublicInvoke',
+          action: 'lambda:InvokeFunctionUrl',
+          principal: '*',
+          function_url_auth_type: 'NONE'
+        )
+        puts "Permission added successfully: #{response}"
+      rescue Aws::Lambda::Errors::ServiceError => e
+        puts "Error adding permission: #{e.message}"
+      end
+
+      def add_sqs_trigger(queue_arn)
+        raise("Invalid ARN for queue: #{queue_arn}") if queue_arn.nil?
+
+        puts "Adding trigger for lambda: #{@function_name} with sqs #{queue_arn}"
+        @lambda_client.create_event_source_mapping({
+                                                     event_source_arn: queue_arn,
+                                                     function_name: @function_name,
+                                                     enabled: true,
+                                                     batch_size: 1
+                                                   })
+        puts "SQS trigger added to Lambda function '#{@function_name}' for queue '#{queue_arn}'."
+      rescue Aws::Lambda::Errors::ResourceConflictException
+        puts 'SQS Trigger already exists'
+      end
+    end
+  end
+end

data/lib/kybus/aws/layer_manager.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+def calculate_md5(file_path)
+  md5 = Digest::MD5.new
+  File.open(file_path, 'rb') do |file|
+    buffer = String.new
+    md5.update(buffer) while file.read(4096, buffer)
+  end
+  md5.hexdigest
+end
+
+module Kybus
+  module AWS
+    class LayerManager
+      def initialize(lambda_client, function_name)
+        @lambda_client = lambda_client
+        @function_name = function_name
+      end
+
+      def create_or_update_layer(zip_file, layer_name, checksum_file)
+        layer_exists = layer_version_exists?(layer_name)
+        zipfile_hash = calculate_md5(checksum_file)
+
+        if !layer_exists || layer_exists.description != zipfile_hash
+          create_layer(zip_file, layer_name, zipfile_hash)
+        else
+          puts "Layer unmodified: #{layer_name}"
+          layer_exists.layer_version_arn
+        end
+      end
+
+      def create_layer(zip_file, layer_name, zipfile_hash)
+        response = @lambda_client.publish_layer_version(
+          layer_name:,
+          content: { zip_file: File.read(zip_file) },
+          description: zipfile_hash,
+          compatible_runtimes: ['ruby3.3']
+        )
+        puts "Layer '#{layer_name}' created: #{response.layer_version_arn}"
+        response.layer_version_arn
+      end
+
+      def layer_version_exists?(layer_name)
+        response = @lambda_client.list_layer_versions(layer_name:, max_items: 1)
+        response.layer_versions.first
+      rescue Aws::Lambda::Errors::ResourceNotFoundException
+        nil
+      end
+
+      def get_layer_arn(layer_name)
+        layer_version = layer_version_exists?(layer_name)
+        layer_version&.layer_version_arn
+      end
+    end
+  end
+end

data/lib/kybus/aws/queue.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Kybus
+  module AWS
+    class Queue < Resource
+      attr_reader :queue_url, :name
+
+      def initialize(configs, name)
+        require 'aws-sdk-sqs'
+        super(configs)
+        @name = name
+      end
+
+      def sqs_client
+        @sqs_client ||= Aws::SQS::Client.new(region: @region)
+      end
+
+      def create_or_update!
+        create_queue!
+        make_write_policy.create_or_update!
+        make_processor_policy.create_or_update!
+      end
+
+      def create_queue!
+        response = sqs_client.create_queue({
+                                             queue_name: name,
+                                             attributes: {
+                                               'DelaySeconds' => '0',
+                                               'MessageRetentionPeriod' => '86400'
+                                             }
+                                           })
+        @queue_url = response.queue_url
+        puts "Queue '#{name}' created with URL #{@queue_url}"
+        @queue_url
+      rescue Aws::SQS::Errors::QueueNameExists
+        @queue_url = sqs_client.get_queue_url(name:).queue_url
+        puts "Queue '#{name}' already exists with URL #{@queue_url}"
+        @queue_url
+      end
+
+      def destroy!
+        sqs_client.delete_queue(queue_url: @queue_url)
+        puts "Queue '#{name}' deleted."
+      rescue Aws::SQS::Errors::NonExistentQueue
+        puts "Queue '#{name}' not found."
+      end
+
+      def arn
+        "arn:aws:sqs:#{region}:#{account_id}:#{name}"
+      end
+
+      def queue_write_policy_document
+        {
+          Version: '2012-10-17',
+          Statement: [
+            {
+              Effect: 'Allow',
+              Action: [
+                'sqs:Get*',
+                'sqs:SendMessage'
+              ],
+              Resource: [
+                arn
+              ]
+            }
+          ]
+        }
+      end
+
+      def queue_processor_policy_document
+        {
+          Version: '2012-10-17',
+          Statement: [{
+            Effect: 'Allow',
+            Action: [
+              'sqs:ReceiveMessage',
+              'sqs:SendMessage',
+              'sqs:Get*',
+              'sqs:DeleteMEssage',
+              'sqs:ChangeMessageVisibility'
+            ],
+            Resource: [arn]
+          }]
+        }
+      end
+
+      def make_write_policy
+        policy_name = "#{name}_queue_publisher"
+        @make_write_policy ||= Policy.new(@config, policy_name, queue_write_policy_document)
+      end
+
+      def make_processor_policy
+        policy_name = "#{name}_queue_processor"
+        @make_processor_policy ||= Policy.new(@config, policy_name, queue_processor_policy_document)
+      end
+    end
+  end
+end

data/lib/kybus/aws/resource.rb

@@ -3,6 +3,8 @@
 module Kybus
   module AWS
     class Resource
+      attr_reader :region
+
       def initialize(config)
         @config = config
         @region = @config['region'] || 'us-east-1'

data/lib/kybus/aws/zip_creator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'zip'
+require 'fileutils'
+module Kybus
+  module AWS
+    class ZipCreator
+      def initialize(zip_name, directory, exclude_files: [], extra_files: {}, zip_root: '')
+        @zip_name = zip_name
+        @directory = directory
+        @exclude_files = exclude_files
+        @extra_files = extra_files
+        @zip_root = zip_root
+      end
+
+      def create_zip
+        FileUtils.rm(@zip_name, force: true)
+        entries = fetch_entries
+
+        Zip::File.open(@zip_name, Zip::File::CREATE) do |zipfile|
+          add_extra_files(zipfile)
+          add_entries(zipfile, entries)
+        end
+      end
+
+      private
+
+      def fetch_entries
+        Dir.entries(@directory) - %w[. ..]
+      end
+
+      def add_extra_files(zipfile)
+        @extra_files.each do |entry, contents|
+          zipfile.get_output_stream(entry) { |f| f.puts(contents) }
+        end
+      end
+
+      def add_entries(zipfile, entries)
+        entries.each do |entry|
+          entry_path = File.join(@directory, entry)
+          next if should_exclude?(entry, entry_path)
+
+          puts "Adding #{entry} to #{@zip_name}"
+
+          if File.directory?(entry_path)
+            add_directory(zipfile, entry_path)
+          else
+            add_file(zipfile, entry_path)
+          end
+        end
+      end
+
+      def should_exclude?(entry, entry_path)
+        @exclude_files.any? { |pattern| File.fnmatch(pattern, entry) || File.fnmatch(pattern, entry_path) }
+      end
+
+      def add_directory(zipfile, entry_path)
+        zipfile.mkdir("#{@zip_root}#{entry_path.sub(@directory, '')}")
+        Dir[File.join(entry_path, '**', '**')].each do |file|
+          next if should_exclude?(file, file)
+
+          zipfile.add("#{@zip_root}#{file.sub(@directory, '')}", file)
+        end
+      end
+
+      def add_file(zipfile, entry_path)
+        zipfile.add("#{@zip_root}#{entry_path.sub(@directory, '')}", entry_path)
+      end
+    end
+  end
+end

data/lib/kybus/aws.rb

@@ -4,5 +4,6 @@ require_relative 'aws/resource'
 require_relative 'aws/code_packager'
 require_relative 'aws/lambda'
 require_relative 'aws/log_group'
+require_relative 'aws/queue'
 require_relative 'aws/policy'
 require_relative 'aws/role'

data/lib/kybus/cli/bot/deployer.rb

@@ -15,6 +15,7 @@ require 'zip'
 require 'kybus/aws'
 require_relative 'deployers/telegram_configurator'
 require_relative 'deployers/aws_bot_deployer'
+require_relative 'deployers/aws_bot_job_runner_deployer'
 
 module Kybus
   class CLI < Thor
@@ -27,8 +28,9 @@ module Kybus
       def initialize(options)
         @params = options
         load_kybusdeploy_file!
-        @telegram = ::Kybus::CLI::BotDeployerTelegramConfigurator.new(@url, config_with_options)
+        @telegram = ::Kybus::CLI::BotDeployerTelegramConfigurator.new(nil, config_with_options)
         @lambda = ::Kybus::CLI::AWSBotDeployer.new(config_with_options)
+        @job_executor = ::Kybus::CLI::AWSBotJobRunnerDeployer.new(config_with_options)
       end
 
       def run_migrations!
@@ -52,6 +54,7 @@ module Kybus
 
       def deploy_lambda!
         @lambda.create_or_update!
+        @job_executor.create_or_update!
         @telegram.url = @lambda.url
       end