barbeque 0.7.0 → 1.0.0

data/lib/barbeque/message/base.rb

@@ -7,7 +7,7 @@ module Barbeque
       attr_reader :type           # [String] "JobExecution", "JobRetry", etc
 
       # @param [Aws::SQS::Types::Message] raw_message
-      # @param [Hash] parse result of `raw_message.body`
+      # @param message_body [Hash] parse result of `raw_message.body`
       def initialize(raw_message, message_body)
         assign_body(message_body)
         @id             = raw_message.message_id

data/lib/barbeque/message_handler/job_execution.rb

@@ -1,91 +1,44 @@
-require 'barbeque/docker_image'
 require 'barbeque/execution_log'
-require 'barbeque/runner'
-require 'barbeque/slack_client'
+require 'barbeque/executor'
+require 'barbeque/slack_notifier'
 
 module Barbeque
   module MessageHandler
     class JobExecution
       # @param [Barbeque::Message::JobExecution] message
-      # @param [Barbeque::JobQueue] job_queue
-      def initialize(message:, job_queue:)
+      # @param [Barbeque::MessageQueue] message_queue
+      def initialize(message:, message_queue:)
         @message = message
-        @job_queue = job_queue
+        @message_queue = message_queue
       end
 
       def run
         begin
-          job_execution = Barbeque::JobExecution.create(message_id: @message.id, job_definition: job_definition, job_queue: @job_queue)
+          job_execution = Barbeque::JobExecution.create(message_id: @message.id, job_definition: job_definition, job_queue: @message_queue.job_queue)
         rescue ActiveRecord::RecordNotUnique => e
           raise DuplicatedExecution.new(e.message)
         end
-        job_execution.update!(status: :running)
+        Barbeque::ExecutionLog.save_message(job_execution, @message)
+        @message_queue.delete_message(@message)
 
         begin
-          stdout, stderr, status = run_command
+          Executor.create.start_execution(job_execution, job_envs)
         rescue Exception => e
           job_execution.update!(status: :error, finished_at: Time.now)
-          log_result(job_execution, '', '')
-          notify_slack(job_execution)
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_execution, '', "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}")
+          Barbeque::SlackNotifier.notify_job_execution(job_execution)
           raise e
         end
-        job_execution.update!(status: status.success? ? :success : :failed, finished_at: Time.now)
-        notify_slack(job_execution)
-
-        log_result(job_execution, stdout, stderr)
       end
 
       private
 
-      def log_result(execution, stdout, stderr)
-        Barbeque::ExecutionLog.save_message(execution, @message)  # TODO: Should be saved earlier
-        Barbeque::ExecutionLog.save_stdout_and_stderr(execution, stdout, stderr)
-      end
-
-      def notify_slack(job_execution)
-        return if job_execution.slack_notification.nil?
-
-        client = Barbeque::SlackClient.new(job_execution.slack_notification.channel)
-        if job_execution.success?
-          if job_execution.slack_notification.notify_success
-            client.notify_success("*[SUCCESS]* Succeeded to execute #{job_execution_link(job_execution)}")
-          end
-        elsif job_execution.failed?
-          client.notify_failure(
-            "*[FAILURE]* Failed to execute #{job_execution_link(job_execution)}" \
-            " #{job_execution.slack_notification.failure_notification_text}"
-          )
-        else
-          client.notify_failure(
-            "*[ERROR]* Failed to execute #{job_execution_link(job_execution)}" \
-            " #{job_execution.slack_notification.failure_notification_text}"
-          )
-        end
-      end
-
-      def job_execution_link(job_execution)
-        "<#{job_execution_url(job_execution)}|#{job_execution.job_definition.job} ##{job_execution.id}>"
-      end
-
-      def job_execution_url(job_execution)
-        Barbeque::Engine.routes.url_helpers.job_execution_url(job_execution, host: ENV['BARBEQUE_HOST'])
-      end
-
-      # @return [String] stdout
-      # @return [String] stderr
-      # @return [Process::Status] status
-      def run_command
-        image  = DockerImage.new(job_definition.app.docker_image)
-        runner = Runner.create(docker_image: image)
-        runner.run(job_definition.command, job_envs)
-      end
-
       def job_envs
         {
           'BARBEQUE_JOB'         => @message.job,
           'BARBEQUE_MESSAGE'     => @message.body.to_json,
           'BARBEQUE_MESSAGE_ID'  => @message.id,
-          'BARBEQUE_QUEUE_NAME'  => @job_queue.name,
+          'BARBEQUE_QUEUE_NAME'  => @message_queue.job_queue.name,
           'BARBEQUE_RETRY_COUNT' => '0',
         }
       end

data/lib/barbeque/message_handler/job_retry.rb

@@ -1,7 +1,6 @@
-require 'barbeque/docker_image'
 require 'barbeque/execution_log'
-require 'barbeque/runner'
-require 'barbeque/slack_client'
+require 'barbeque/executor'
+require 'barbeque/slack_notifier'
 
 module Barbeque
   module MessageHandler
@@ -9,10 +8,10 @@ module Barbeque
 
     class JobRetry
       # @param [Barbeque::Message::JobExecution] message
-      # @param [Barbeque::JobQueue] job_queue
-      def initialize(message:, job_queue:)
+      # @param [Barbeque::MessageQueue] message_queue
+      def initialize(message:, message_queue:)
         @message = message
-        @job_queue = job_queue
+        @message_queue = message_queue
       end
 
       def run
@@ -21,71 +20,25 @@ module Barbeque
         rescue ActiveRecord::RecordNotUnique => e
           raise DuplicatedExecution.new(e.message)
         end
-        job_execution.update!(status: :retried)
-        job_retry.update!(status: :running)
+        @message_queue.delete_message(@message)
 
         begin
-          stdout, stderr, result = run_command
+          Executor.create.start_retry(job_retry, job_envs)
         rescue Exception => e
           job_retry.update!(status: :error, finished_at: Time.now)
           job_execution.update!(status: :error)
-          log_result(job_retry, '', '')
-          notify_slack(job_retry)
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_retry, '', "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}")
+          Barbeque::SlackNotifier.notify_job_retry(job_retry)
           raise e
         end
-        status = result.success? ? :success : :failed
-        job_retry.update!(status: status, finished_at: Time.now)
-        job_execution.update!(status: status)
-        notify_slack(job_retry)
-
-        log_result(job_retry, stdout, stderr)
       end
 
       private
 
-      def log_result(job_retry, stdout, stderr)
-        Barbeque::ExecutionLog.save_stdout_and_stderr(job_retry, stdout, stderr)
-      end
-
-      # @param [Barbeque::JobRetry] job_retry
-      def notify_slack(job_retry)
-        return if job_retry.slack_notification.nil?
-
-        client = Barbeque::SlackClient.new(job_retry.slack_notification.channel)
-        if job_retry.success?
-          if job_retry.slack_notification.notify_success
-            client.notify_success("*[SUCCESS]* Succeeded to retry #{job_retry_link(job_retry)}")
-          end
-        elsif job_retry.failed?
-          client.notify_failure(
-            "*[FAILURE]* Failed to retry #{job_retry_link(job_retry)}" \
-            " #{job_execution.slack_notification.failure_notification_text}"
-          )
-        else
-          client.notify_failure(
-            "*[ERROR]* Failed to retry #{job_retry_link(job_retry)}" \
-            " #{job_execution.slack_notification.failure_notification_text}"
-          )
-        end
-      end
-
-      def job_retry_link(job_retry)
-        url = Barbeque::Engine.routes.url_helpers.job_execution_job_retry_url(
-          job_retry.job_execution, job_retry, host: ENV['BARBEQUE_HOST']
-        )
-        "<#{url}|#{job_retry.job_definition.job}'s retry ##{job_retry.id}>"
-      end
-
       def job_execution
         @job_execution ||= Barbeque::JobExecution.find_by!(message_id: @message.retry_message_id)
       end
 
-      def run_command
-        image  = DockerImage.new(job_execution.app.docker_image)
-        runner = Runner.create(docker_image: image)
-        runner.run(job_execution.job_definition.command, job_envs)
-      end
-
       def job_envs
         if job_execution.execution_log.nil?
           raise MessageNotFound.new('failed to fetch retried message')
@@ -95,7 +48,7 @@ module Barbeque
           'BARBEQUE_JOB'         => job_execution.job_definition.job,
           'BARBEQUE_MESSAGE'     => job_execution.execution_log['message'],
           'BARBEQUE_MESSAGE_ID'  => @message.retry_message_id,
-          'BARBEQUE_QUEUE_NAME'  => @job_queue.name,
+          'BARBEQUE_QUEUE_NAME'  => @message_queue.job_queue.name,
           'BARBEQUE_RETRY_COUNT' => job_execution.job_retries.count.to_s,
         }
       end

data/lib/barbeque/message_handler/notification.rb

@@ -1,19 +1,15 @@
-require 'barbeque/docker_image'
-require 'barbeque/execution_log'
-require 'barbeque/runner'
-require 'barbeque/slack_client'
 require 'barbeque/message_handler/job_execution'
 
 module Barbeque
   module MessageHandler
     class Notification < JobExecution
       # @param [Barbeque::Message::Notification] message
-      # @param [Barbeque::JobQueue] job_queue
-      def initialize(message:, job_queue:)
+      # @param [Barbeque::MessageQueue] message_queue
+      def initialize(message:, message_queue:)
         @message = message
-        @job_queue = job_queue
+        @message_queue = message_queue
 
-        subscription = SNSSubscription.find_by!(topic_arn: @message.topic_arn, job_queue_id: @job_queue.id)
+        subscription = SNSSubscription.find_by!(topic_arn: @message.topic_arn, job_queue_id: @message_queue.job_queue.id)
         @message.set_params_from_subscription(subscription)
       end
     end

data/lib/barbeque/message_queue.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk'
+require 'barbeque/config'
 require 'barbeque/message'
 
 module Barbeque
@@ -11,20 +12,29 @@ module Barbeque
       @stop      = false
     end
 
-    # Receive a message and delete them all from SQS queue immediately.
-    # TODO: Stop removing them immediately to implement retry.
+    # Receive a message from SQS queue.
     # @return [Barbeque::Message::Base]
     def dequeue
-      while valid_messages.empty?
+      loop do
         return nil if @stop
-        messages = receive_messages
-        messages = reject_invalid_messages(messages)
-        valid_messages.concat(messages)
+        message = receive_message
+        if message
+          if message.valid?
+            return message
+          else
+            delete_message(message)
+          end
+        end
       end
+    end
 
-      valid_messages.shift.tap do |message|
-        delete_message(message)
-      end
+    # Remove a message from SQS queue.
+    # @param [Barbeque::Message::Base] message
+    def delete_message(message)
+      client.delete_message(
+        queue_url: @job_queue.queue_url,
+        receipt_handle: message.receipt_handle,
+      )
     end
 
     def stop!
@@ -33,30 +43,15 @@ module Barbeque
 
     private
 
-    def receive_messages
+    def receive_message
       result = client.receive_message(
         queue_url: @job_queue.queue_url,
-        wait_time_seconds: Barbeque::JobQueue::SQS_RECEIVE_MESSAGE_WAIT_TIME,
-      )
-      result.messages.map { |m| Barbeque::Message.parse(m) }
-    end
-
-    def reject_invalid_messages(messages)
-      messages, invalid_messages = messages.partition(&:valid?)
-      invalid_messages.each { |m| delete_message(m) }
-      messages
-    end
-
-    # Remove a message from SQS queue.
-    def delete_message(message)
-      client.delete_message(
-        queue_url:      @job_queue.queue_url,
-        receipt_handle: message.receipt_handle,
+        wait_time_seconds: Barbeque.config.sqs_receive_message_wait_time,
+        max_number_of_messages: 1,
       )
-    end
-
-    def valid_messages
-      @valid_messages ||= []
+      if result.messages[0]
+        Barbeque::Message.parse(result.messages[0])
+      end
     end
 
     def client

data/lib/barbeque/retry_poller.rb

@@ -0,0 +1,34 @@
+module Barbeque
+  class RetryPoller
+    def initialize
+      @stop_requested = false
+    end
+
+    def run
+      Barbeque::JobRetry.running.find_in_batches do |job_retries|
+        job_retries.shuffle.each do |job_retry|
+          if @stop_requested
+            return
+          end
+          job_retry.with_lock do
+            if job_retry.running?
+              poll(job_retry)
+            end
+          end
+        end
+      end
+      sleep 1
+    end
+
+    def stop
+      @stop_requested = true
+    end
+
+    private
+
+    def poll(job_retry)
+      executor = Executor.create
+      executor.poll_retry(job_retry)
+    end
+  end
+end

data/lib/barbeque/runner.rb

@@ -1,13 +1,57 @@
 require 'barbeque/config'
-require 'barbeque/runner/docker'
-require 'barbeque/runner/hako'
+require 'barbeque/exception_handler'
+require 'barbeque/execution_log'
+require 'barbeque/message_handler'
+require 'barbeque/message_queue'
 
 module Barbeque
-  module Runner
-    # @param [Barbeque::DockerImage] docker_image
-    def self.create(docker_image:)
-      klass = const_get(Barbeque.config.runner, false)
-      klass.new(Barbeque.config.runner_options.merge(docker_image: docker_image))
+  # Part of barbeque-worker.
+  # Runner dequeues a message from {MessageQueue} (Amazon SQS) and dispatches
+  # it to message handler.
+
+  class Runner
+    DEFAULT_QUEUE = 'default'
+
+    def initialize(queue_name: ENV['BARBEQUE_QUEUE'] || DEFAULT_QUEUE)
+      @queue_name = queue_name
+    end
+
+    def run
+      keep_maximum_concurrent_executions
+
+      message = message_queue.dequeue
+      return unless message
+
+      handler = MessageHandler.const_get(message.type, false)
+      handler.new(message: message, message_queue: message_queue).run
+    end
+
+    def stop
+      message_queue.stop!
+    end
+
+    private
+
+    def message_queue
+      @message_queue ||= MessageQueue.new(@queue_name)
+    end
+
+    def keep_maximum_concurrent_executions
+      max_num = Barbeque.config.maximum_concurrent_executions
+      unless max_num
+        # nil means unlimited
+        return
+      end
+
+      loop do
+        current_num = Barbeque::JobExecution.where(status: [:running, :retried]).count
+        if current_num < max_num
+          return
+        end
+        interval = Barbeque.config.runner_wait_seconds
+        Rails.logger.info("#{current_num} executions are running but maximum_concurrent_executions is configured to #{max_num}. Waiting #{interval} seconds...")
+        sleep(interval)
+      end
     end
   end
 end

data/lib/barbeque/slack_notifier.rb

@@ -0,0 +1,73 @@
+require 'barbeque/slack_client'
+
+module Barbeque
+  module SlackNotifier
+    class << self
+      # @param [Barbeque::JobExecution] job_execution
+      def notify_job_execution(job_execution)
+        return if job_execution.slack_notification.nil?
+
+        client = Barbeque::SlackClient.new(job_execution.slack_notification.channel)
+        if job_execution.success?
+          if job_execution.slack_notification.notify_success
+            client.notify_success("*[SUCCESS]* Succeeded to execute #{job_execution_link(job_execution)}")
+          end
+        elsif job_execution.failed?
+          client.notify_failure(
+            "*[FAILURE]* Failed to execute #{job_execution_link(job_execution)}" \
+            " #{job_execution.slack_notification.failure_notification_text}"
+          )
+        else
+          client.notify_failure(
+            "*[ERROR]* Failed to execute #{job_execution_link(job_execution)}" \
+            " #{job_execution.slack_notification.failure_notification_text}"
+          )
+        end
+      end
+
+      # @param [Barbeque::JobRetry] job_retry
+      def notify_job_retry(job_retry)
+        return if job_retry.slack_notification.nil?
+
+        client = Barbeque::SlackClient.new(job_retry.slack_notification.channel)
+        if job_retry.success?
+          if job_retry.slack_notification.notify_success
+            client.notify_success("*[SUCCESS]* Succeeded to retry #{job_retry_link(job_retry)}")
+          end
+        elsif job_retry.failed?
+          client.notify_failure(
+            "*[FAILURE]* Failed to retry #{job_retry_link(job_retry)}" \
+            " #{job_retry.slack_notification.failure_notification_text}"
+          )
+        else
+          client.notify_failure(
+            "*[ERROR]* Failed to retry #{job_retry_link(job_retry)}" \
+            " #{job_retry.slack_notification.failure_notification_text}"
+          )
+        end
+      end
+
+      private
+
+      def barbeque_host
+        ENV['BARBEQUE_HOST']
+      end
+
+      def job_execution_link(job_execution)
+        "<#{job_execution_url(job_execution)}|#{job_execution.job_definition.job} ##{job_execution.id}>"
+      end
+
+      def job_execution_url(job_execution)
+        Barbeque::Engine.routes.url_helpers.job_execution_url(job_execution, host: barbeque_host)
+      end
+
+      def job_retry_link(job_retry)
+        "<#{job_retry_url(job_retry)}|#{job_retry.job_definition.job}'s retry ##{job_retry.id}>"
+      end
+
+      def job_retry_url(job_retry)
+        Barbeque::Engine.routes.url_helpers.job_execution_job_retry_url(job_retry.job_execution, job_retry, host: barbeque_host)
+      end
+    end
+  end
+end