barbeque 0.7.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e23cea0a69664e6f2351d1651daaf0b42355a71f
-  data.tar.gz: 4c76a4e57cbb43baa302841f7eb565a1f3602fb6
+  metadata.gz: fdd657c4fecd71f71469707c7583541cc8d3dab7
+  data.tar.gz: f6911be9efb1970c036697bcc8e5a46edbdb605d
 SHA512:
-  metadata.gz: 1021988ef2880d1bb8009d5c2fb397803a0e9a02577ff66cd961638f415dd877012340bb86564b6855efa1c65d44751b1bffc16aea1bce98697465f3c4a1b940
-  data.tar.gz: 0e48ddda04e6ebb25e5cff2bcc7c21f440848b2a5588286eb9e89289d70d5b6863bdb00ebe9c7d6dc70424bba37bbc143795e7cc055fcc089518a57a3375f8c7
+  metadata.gz: 4d89b39fd628a67c7144613989d77c0b4fc3407c307e9409991263ad704b2e667d73981519ed59a543d24ba21c94e390830965c9e6eb3f823127acbf3d3996c2
+  data.tar.gz: 5697f62a1d5053fd2ab82ac60cabc4a6ad70eea87317fd6d800e251ab242e7a55edfc86299eec598c3b6d4c0111829f3f7387e21f7ddc726ce96d7e22d3e12d7

data/README.md

@@ -25,7 +25,7 @@ In Barbeque worker, they are done on Docker container.
 ## Why Barbeque?
 
 - You can achieve job-level auto scaling using tools like [Amazon ECS](https://aws.amazon.com/ecs/) [EC2 Auto Scaling group](https://aws.amazon.com/autoscaling/)
-  - For Amazon ECS, Barbeque has Hako runner
+  - For Amazon ECS, Barbeque has Hako executor
 - You don't have to manage infrastructure for each application like Resque or Sidekiq
 
 For details, see [Scalable Job Queue System Built with Docker // Speaker Deck](https://speakerdeck.com/k0kubun/scalable-job-queue-system-built-with-docker).
@@ -45,6 +45,15 @@ You also need to prepare MySQL, Amazon SQS and Amazon S3.
 $ rake barbeque:worker BARBEQUE_QUEUE=default
 ```
 
+The rake task launches four worker processes.
+
+- Two runners
+  - receives message from SQS queue, starts job execution and stores its identifier to the database
+- One execution poller
+  - gets execution status and reflect it to the database
+- One retry poller
+  - gets retried execution status and reflect it to the database
+
 ## Usage
 
 Web API documentation is available at [doc/toc.md](./doc/toc.md).
@@ -53,5 +62,21 @@ Web API documentation is available at [doc/toc.md](./doc/toc.md).
 
 [barbeque\_client.gem](https://github.com/cookpad/barbeque_client) has API client and ActiveJob integration.
 
+## Executor
+Barbeque executor can be customized in config/barbeque.yml. Executor is responsible for starting executions and getting status of executions.
+
+Barbeque has currently two executors.
+
+### Docker (default)
+Barbeque::Executor::Docker starts execution by `docker run --detach` and gets status by `docker inspect`.
+
+### Hako
+Barbeque::Executor::Hako starts execution by `hako oneshot --no-wait` and gets status from S3 task notification.
+
+#### Requirement
+You must configure CloudWatch Events for putting S3 task notification.
+See Hako's documentation for detail.
+https://github.com/eagletmt/hako/blob/master/docs/ecs-task-notification.md
+
 ## License
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/app/controllers/barbeque/job_queues_controller.rb

@@ -1,3 +1,5 @@
+require 'barbeque/config'
+
 class Barbeque::JobQueuesController < Barbeque::ApplicationController
   def index
     @job_queues = Barbeque::JobQueue.all
@@ -49,10 +51,7 @@ class Barbeque::JobQueuesController < Barbeque::ApplicationController
     Aws::SQS::Client.new.create_queue(
       queue_name: job_queue.sqs_queue_name,
       attributes: {
-        # All SQS queues' "ReceiveMessageWaitTimeSeconds" are configured to be 20s (maximum).
-        # This should be as large as possible to reduce API-calling cost by long polling.
-        # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html#API_CreateQueue_RequestParameters
-        'ReceiveMessageWaitTimeSeconds' => Barbeque::JobQueue::SQS_RECEIVE_MESSAGE_WAIT_TIME.to_s,
+        'ReceiveMessageWaitTimeSeconds' => Barbeque.config.sqs_receive_message_wait_time.to_s,
       },
     )
   end

data/app/models/barbeque/docker_container.rb

@@ -0,0 +1,2 @@
+class Barbeque::DockerContainer < Barbeque::ApplicationRecord
+end

data/app/models/barbeque/ecs_hako_task.rb

@@ -0,0 +1,2 @@
+class Barbeque::EcsHakoTask < Barbeque::ApplicationRecord
+end

data/app/models/barbeque/job_queue.rb

@@ -4,11 +4,6 @@ class Barbeque::JobQueue < Barbeque::ApplicationRecord
 
   has_many :sns_subscriptions, class_name: 'SNSSubscription', dependent: :destroy
 
-  # All SQS queues' "ReceiveMessageWaitTimeSeconds" are configured to be 20s (maximum).
-  # This should be as large as possible to reduce API-calling cost by long polling.
-  # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html#API_CreateQueue_RequestParameters
-  SQS_RECEIVE_MESSAGE_WAIT_TIME = 20
-
   # SQS queue allows [a-zA-Z0-9_-]+ as queue name. Its maximum length is 80.
   validates :name, presence: true, uniqueness: true, format: /\A[a-zA-Z0-9_-]+\z/,
     length: { maximum: SQS_NAME_MAX_LENGTH - SQS_NAME_PREFIX.length }

data/app/services/barbeque/message_enqueuing_service.rb

@@ -10,7 +10,7 @@ class Barbeque::MessageEnqueuingService
   # @param [String] application
   # @param [String] job
   # @param [Object] message
-  # @param optional [String] queue
+  # @param [String] queue
   def initialize(application:, job:, message:, queue: nil)
     @application = application
     @job         = job

data/app/services/barbeque/sns_subscription_service.rb

@@ -67,7 +67,7 @@ class Barbeque::SNSSubscriptionService
     )
   end
 
-  # @paaram [String] queue_arn
+  # @param [String] queue_arn
   # @param [Array<String>] topic_arns
   # @return [String] JSON formatted policy
   def generate_policy(queue_arn:, topic_arns:)

data/db/migrate/20170711085157_create_barbeque_docker_containers.rb

@@ -0,0 +1,12 @@
+class CreateBarbequeDockerContainers < ActiveRecord::Migration[5.0]
+  def change
+    create_table :barbeque_docker_containers, options: 'ENGINE=InnoDB ROW_FORMAT=dynamic DEFAULT CHARSET=utf8mb4' do |t|
+      t.string :message_id, null: false
+      t.string :container_id, null: false
+
+      t.timestamps
+
+      t.index ['message_id'], unique: true
+    end
+  end
+end

data/db/migrate/20170712075449_create_barbeque_ecs_hako_tasks.rb

@@ -0,0 +1,13 @@
+class CreateBarbequeEcsHakoTasks < ActiveRecord::Migration[5.0]
+  def change
+    create_table :barbeque_ecs_hako_tasks, options: 'ENGINE=InnoDB ROW_FORMAT=dynamic DEFAULT CHARSET=utf8mb4' do |t|
+      t.string :message_id, null: false
+      t.string :cluster, null: false
+      t.string :task_arn, null: false
+
+      t.timestamps
+
+      t.index ['message_id'], unique: true
+    end
+  end
+end

data/db/migrate/20170724025542_add_index_to_job_execution_status.rb

@@ -0,0 +1,6 @@
+class AddIndexToJobExecutionStatus < ActiveRecord::Migration[5.0]
+  def change
+    add_index :barbeque_job_executions, :status
+    add_index :barbeque_job_retries, :status
+  end
+end

data/lib/barbeque/config.rb

@@ -3,7 +3,7 @@ require 'yaml'
 
 module Barbeque
   class Config
-    attr_accessor :exception_handler, :runner, :runner_options
+    attr_accessor :exception_handler, :executor, :executor_options, :sqs_receive_message_wait_time, :maximum_concurrent_executions, :runner_wait_seconds
 
     def initialize(options = {})
       options.each do |key, value|
@@ -13,15 +13,20 @@ module Barbeque
           raise KeyError.new("Unexpected option '#{key}' was specified.")
         end
       end
-      runner_options.symbolize_keys!
+      executor_options.symbolize_keys!
     end
   end
 
   module ConfigBuilder
     DEFAULT_CONFIG = {
       'exception_handler' => 'RailsLogger',
-      'runner'            => 'Docker',
-      'runner_options'    => {},
+      'executor' => 'Docker',
+      'executor_options' => {},
+      # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html#API_CreateQueue_RequestParameters
+      'sqs_receive_message_wait_time' => 10,
+      # nil means unlimited
+      'maximum_concurrent_executions' => nil,
+      'runner_wait_seconds' => 10,
     }
 
     def config

data/lib/barbeque/execution_log.rb

@@ -43,16 +43,7 @@ module Barbeque
           'stderr' => stderr,
         }
       else
-        # Try to load legacy format
-        begin
-          s3_object = ExecutionLog.s3_client.get_object(
-            bucket: s3_bucket_name,
-            key: legacy_s3_key_for(execution),
-          )
-          JSON.parse(s3_object.body.read)
-        rescue Aws::S3::Errors::NoSuchKey
-          nil
-        end
+        nil
       end
     end
 
@@ -68,11 +59,6 @@ module Barbeque
       "#{execution.app.name}/#{execution.job_definition.job}/#{execution.message_id}/#{filename}"
     end
 
-    # @param [Barbeque::JobExecution,Barbeque::JobRetry] execution
-    def legacy_s3_key_for(execution)
-      "#{execution.app.name}/#{execution.job_definition.job}/#{execution.message_id}"
-    end
-
     # @param [Barbeque::JobExecution,Barbeque::JobRetry] execution
     # @param [String] filename
     # @return [String]

data/lib/barbeque/execution_poller.rb

@@ -0,0 +1,34 @@
+module Barbeque
+  class ExecutionPoller
+    def initialize
+      @stop_requested = false
+    end
+
+    def run
+      Barbeque::JobExecution.running.find_in_batches do |job_executions|
+        job_executions.shuffle.each do |job_execution|
+          if @stop_requested
+            return
+          end
+          job_execution.with_lock do
+            if job_execution.running?
+              poll(job_execution)
+            end
+          end
+        end
+      end
+      sleep 1
+    end
+
+    def stop
+      @stop_requested = true
+    end
+
+    private
+
+    def poll(job_execution)
+      executor = Executor.create
+      executor.poll_execution(job_execution)
+    end
+  end
+end

data/lib/barbeque/executor.rb

@@ -0,0 +1,29 @@
+require 'barbeque/config'
+require 'barbeque/executor/docker'
+require 'barbeque/executor/hako'
+
+module Barbeque
+  # Executor is responsible for starting executions and getting status of
+  # executions.
+  # Each executor must implement these methods.
+  # - #initialize(options)
+  #   - Create a executor with executor_options specified in config/barbeque.yml.
+  # - #start_execution(job_execution, envs)
+  #   - Start execution with environment variables. An executor must update the
+  #     execution status from pending.
+  # - #poll_execution(job_execution)
+  #   - Get the execution status and update the job_execution columns.
+  # - #start_retry(job_retry, envs)
+  #   - Start retry with environment variables. An executor must update the
+  #     retry status from pending and the corresponding execution status.
+  # - #poll_retry(job_retry)
+  #   - Get the execution status and update the job_retry and job_execution
+  #     columns.
+
+  module Executor
+    def self.create
+      klass = const_get(Barbeque.config.executor, false)
+      klass.new(Barbeque.config.executor_options)
+    end
+  end
+end

data/lib/barbeque/executor/docker.rb

@@ -0,0 +1,131 @@
+require 'barbeque/docker_image'
+require 'barbeque/slack_notifier'
+require 'open3'
+
+module Barbeque
+  module Executor
+    class Docker
+      class DockerCommandError < StandardError
+      end
+
+      def initialize(_options)
+      end
+
+      # @param [Barbeque::JobExecution] job_execution
+      # @param [Hash] envs
+      def start_execution(job_execution, envs)
+        docker_image = DockerImage.new(job_execution.job_definition.app.docker_image)
+        cmd = build_docker_run_command(docker_image, job_execution.job_definition.command, envs)
+        stdout, stderr, status = Open3.capture3(*cmd)
+        if status.success?
+          job_execution.update!(status: :running)
+          Barbeque::DockerContainer.create!(message_id: job_execution.message_id, container_id: stdout.chomp)
+        else
+          job_execution.update!(status: :failed, finished_at: Time.zone.now)
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_execution, stdout, stderr)
+          Barbeque::SlackNotifier.notify_job_execution(job_execution)
+        end
+      end
+
+      # @param [Barbeque::JobRetry] job_retry
+      # @param [Hash] envs
+      def start_retry(job_retry, envs)
+        job_execution = job_retry.job_execution
+        docker_image = DockerImage.new(job_execution.job_definition.app.docker_image)
+        cmd = build_docker_run_command(docker_image, job_execution.job_definition.command, envs)
+        stdout, stderr, status = Open3.capture3(*cmd)
+        if status.success?
+          Barbeque::DockerContainer.create!(message_id: job_retry.message_id, container_id: stdout.chomp)
+          Barbeque::ApplicationRecord.transaction do
+            job_execution.update!(status: :retried)
+            job_retry.update!(status: :running)
+          end
+        else
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_retry, stdout, stderr)
+          Barbeque::ApplicationRecord.transaction do
+            job_retry.update!(status: :failed, finished_at: Time.zone.now)
+            job_execution.update!(status: :failed)
+          end
+          Barbeque::SlackNotifier.notify_job_retry(job_retry)
+        end
+      end
+
+      # @param [Barbeque::JobExecution] job_execution
+      def poll_execution(job_execution)
+        container = Barbeque::DockerContainer.find_by!(message_id: job_execution.message_id)
+        info = inspect_container(container.container_id)
+        if info['State'] && info['State']['Status'] != 'running'
+          finished_at = Time.zone.parse(info['State']['FinishedAt'])
+          exit_code = info['State']['ExitCode']
+          job_execution.update!(status: exit_code == 0 ? :success : :failed, finished_at: finished_at)
+
+          stdout, stderr = get_logs(container.container_id)
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_execution, stdout, stderr)
+          Barbeque::SlackNotifier.notify_job_execution(job_execution)
+        end
+      end
+
+      # @param [Barbeque::JobRetry] job_retry
+      def poll_retry(job_retry)
+        container = Barbeque::DockerContainer.find_by!(message_id: job_retry.message_id)
+        job_execution = job_retry.job_execution
+        info = inspect_container(container.container_id)
+        if info['State'] && info['State']['Status'] != 'running'
+          finished_at = Time.zone.parse(info['State']['FinishedAt'])
+          exit_code = info['State']['ExitCode']
+          status = exit_code == 0 ? :success : :failed
+          Barbeque::ApplicationRecord.transaction do
+            job_retry.update!(status: status, finished_at: finished_at)
+            job_execution.update!(status: status)
+          end
+
+          stdout, stderr = get_logs(container.container_id)
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_retry, stdout, stderr)
+          Barbeque::SlackNotifier.notify_job_retry(job_retry)
+        end
+      end
+
+      private
+
+      # @param [Barbeque::DockerImage] docker_image
+      # @param [Array<String>] command
+      # @param [Hash] envs
+      def build_docker_run_command(docker_image, command, envs)
+        ['docker', 'run', '--detach', *env_options(envs), docker_image.to_s, *command]
+      end
+
+      def env_options(envs)
+        envs.flat_map do |key, value|
+          ['--env', "#{key}=#{value}"]
+        end
+      end
+
+      # @param [String] container_id
+      # @return [Hash] container info
+      def inspect_container(container_id)
+        stdout, stderr, status = Open3.capture3('docker', 'inspect', container_id)
+        if status.success?
+          begin
+            JSON.parse(stdout)[0]
+          rescue JSON::ParserError => e
+            raise DockerCommandError.new("Unable to parse JSON: #{e.class}: #{e.message}: #{stdout}")
+          end
+        else
+          raise DockerCommandError.new("Unable to inspect Docker container #{container.container_id}: STDOUT: #{stdout}; STDERR: #{stderr}")
+        end
+      end
+
+      # @param [String] container_id
+      # @return [String] stdout
+      # @return [String] stderr
+      def get_logs(container_id)
+        stdout, stderr, status = Open3.capture3('docker', 'logs', container_id)
+        if status.success?
+          [stdout, stderr]
+        else
+          raise DockerCommandError.new("Unable to get Docker container logs #{container.container_id}: STDOUT: #{stdout}; STDERR: #{stderr}")
+        end
+      end
+    end
+  end
+end

data/lib/barbeque/executor/hako.rb

@@ -0,0 +1,159 @@
+require 'barbeque/docker_image'
+require 'barbeque/slack_notifier'
+require 'open3'
+require 'uri'
+
+module Barbeque
+  module Executor
+    class Hako
+      class HakoCommandError < StandardError
+      end
+
+      # @param [String] hako_dir
+      # @param [Hash] hako_env
+      # @param [String] yaml_dir
+      def initialize(hako_dir:, hako_env: {}, yaml_dir:, oneshot_notification_prefix:)
+        @hako_dir = hako_dir
+        @hako_env = hako_env
+        @yaml_dir = yaml_dir
+        uri = URI.parse(oneshot_notification_prefix)
+        @s3_bucket = uri.host
+        @s3_prefix = uri.path.sub(%r{\A/}, '')
+        @s3_region = URI.decode_www_form(uri.query || '').to_h['region']
+      end
+
+      # @param [Barbeque::JobExecution] job_execution
+      # @param [Hash] envs
+      def start_execution(job_execution, envs)
+        docker_image = DockerImage.new(job_execution.job_definition.app.docker_image)
+        cmd = build_hako_oneshot_command(docker_image, job_execution.job_definition.command, envs)
+        stdout, stderr, status = Bundler.with_clean_env { Open3.capture3(@hako_env, *cmd, chdir: @hako_dir) }
+        if status.success?
+          job_execution.update!(status: :running)
+          cluster, task_arn = extract_task_info(stdout)
+          Barbeque::EcsHakoTask.create!(message_id: job_execution.message_id, cluster: cluster, task_arn: task_arn)
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_execution, stdout, stderr)
+        else
+          job_execution.update!(status: :failed, finished_at: Time.zone.now)
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_execution, stdout, stderr)
+          Barbeque::SlackNotifier.notify_job_execution(job_execution)
+        end
+      end
+
+      # @param [Barbeque::JobRetry] job_retry
+      # @param [Hash] envs
+      def start_retry(job_retry, envs)
+        job_execution = job_retry.job_execution
+        docker_image = DockerImage.new(job_execution.job_definition.app.docker_image)
+        cmd = build_hako_oneshot_command(docker_image, job_execution.job_definition.command, envs)
+        stdout, stderr, status = Bundler.with_clean_env { Open3.capture3(@hako_env, *cmd, chdir: @hako_dir) }
+        if status.success?
+          cluster, task_arn = extract_task_info(stdout)
+          Barbeque::EcsHakoTask.create!(message_id: job_retry.message_id, cluster: cluster, task_arn: task_arn)
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_retry, stdout, stderr)
+          Barbeque::ApplicationRecord.transaction do
+            job_execution.update!(status: :retried)
+            job_retry.update!(status: :running)
+          end
+        else
+          Barbeque::ExecutionLog.save_stdout_and_stderr(job_retry, stdout, stderr)
+          Barbeque::ApplicationRecord.transaction do
+            job_retry.update!(status: :failed, finished_at: Time.zone.now)
+            job_execution.update!(status: :failed)
+          end
+          Barbeque::SlackNotifier.notify_job_retry(job_retry)
+        end
+      end
+
+      # @param [Barbeque::JobExecution] job_execution
+      def poll_execution(job_execution)
+        hako_task = Barbeque::EcsHakoTask.find_by!(message_id: job_execution.message_id)
+        result = get_stopped_result(hako_task)
+        if result
+          detail = result.fetch('detail')
+          task = Aws::Json::Parser.new(Aws::ECS::Client.api.operation('describe_tasks').output.shape.member(:tasks).shape.member).parse(JSON.dump(detail))
+          status = :failed
+          task.containers.each do |container|
+            if container.name == 'app'
+              status = container.exit_code == 0 ? :success : :failed
+            end
+          end
+          job_execution.update!(status: status, finished_at: task.stopped_at)
+          Barbeque::SlackNotifier.notify_job_execution(job_execution)
+        end
+      end
+
+      # @param [Barbeque::JobRetry] job_execution
+      def poll_retry(job_retry)
+        hako_task = Barbeque::EcsHakoTask.find_by!(message_id: job_retry.message_id)
+        job_execution = job_retry.job_execution
+        result = get_stopped_result(hako_task)
+        if result
+          detail = result.fetch('detail')
+          task = Aws::Json::Parser.new(Aws::ECS::Client.api.operation('describe_tasks').output.shape.member(:tasks).shape.member).parse(JSON.dump(detail))
+          status = :failed
+          task.containers.each do |container|
+            if container.name == 'app'
+              status = container.exit_code == 0 ? :success : :failed
+            end
+          end
+          Barbeque::ApplicationRecord.transaction do
+            job_retry.update!(status: status, finished_at: task.stopped_at)
+            job_execution.update!(status: status)
+          end
+          Barbeque::SlackNotifier.notify_job_retry(job_retry)
+        end
+      end
+
+      private
+
+      def build_hako_oneshot_command(docker_image, command, envs)
+        [
+          'bundle', 'exec', 'hako', 'oneshot', '--no-wait', '--tag', docker_image.tag,
+          *env_options(envs), File.join(@yaml_dir, "#{docker_image.repository}.yml"), '--', *command,
+        ]
+      end
+
+      def env_options(envs)
+        envs.map do |key, value|
+          "--env=#{key}=#{value}"
+        end
+      end
+
+      def s3_key_for_stopped_result(hako_task)
+        "#{@s3_prefix}/#{hako_task.task_arn}/stopped.json"
+      end
+
+      def s3_client
+        @s3_client ||= Aws::S3::Client.new(region: @s3_region, http_read_timeout: 5)
+      end
+
+      def get_stopped_result(hako_task)
+        object = s3_client.get_object(bucket: @s3_bucket, key: s3_key_for_stopped_result(hako_task))
+        JSON.parse(object.body.read)
+      rescue Aws::S3::Errors::NoSuchKey
+        nil
+      end
+
+      def extract_task_info(stdout)
+        last_line = stdout.lines.last
+        if last_line
+          begin
+            task_info = JSON.parse(last_line)
+            cluster = task_info['cluster']
+            task_arn = task_info['task_arn']
+            if cluster && task_arn
+              [cluster, task_arn]
+            else
+              raise HakoCommandError.new("Unable find cluster and task_arn in JSON: #{stdout}")
+            end
+          rescue JSON::ParserError => e
+            raise HakoCommandError.new("Unable parse the last line as JSON: #{stdout}")
+          end
+        else
+          raise HakoCommandError.new('stdout is empty')
+        end
+      end
+    end
+  end
+end