barbeque 0.0.1 → 0.0.10

data/app/views/layouts/barbeque/application.html.haml

@@ -0,0 +1,24 @@
+!!!
+%html
+  %head
+    %meta{ content: 'text/html; charset=UTF-8', 'http-equiv': 'Content-Type' }
+    %meta{ charset: 'UTF-8' }
+    %title Barbeque
+    %meta{ content: 'width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no', name: 'viewport' }
+    -# FIXME: Don't rely on others' CDN
+    %link{ href: 'https://maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css', rel: 'stylesheet', type: 'text/css' }
+    %link{ href: 'https://code.ionicframework.com/ionicons/2.0.1/css/ionicons.min.css', rel: 'stylesheet', type: 'text/css' }
+    = stylesheet_link_tag    'barbeque/application', media: 'all'
+    = javascript_include_tag 'barbeque/application'
+    = csrf_meta_tags
+
+  %body.skin-blue.sidebar-mini{ class: "#{controller_path.parameterize(separator: '_')}_controller #{action_name}_action" }
+    .wrapper
+      = render partial: 'layouts/barbeque/header'
+      = render partial: 'layouts/barbeque/sidebar'
+      .content-wrapper
+        - if content_for?(:header)
+          %section.content-header
+            = content_for(:header)
+        %section.content
+          = yield

data/app/views/layouts/barbeque/apps.html.haml

@@ -0,0 +1,6 @@
+- content_for(:header) do
+  %h1
+    .fa.fa-users
+    Applications
+
+= render template: 'layouts/barbeque/application'

data/app/views/layouts/barbeque/job_definitions.html.haml

@@ -0,0 +1,6 @@
+- content_for(:header) do
+  %h1
+    .fa.fa-tasks
+    Job Definitions
+
+= render template: 'layouts/barbeque/application'

data/app/views/layouts/barbeque/job_executions.html.haml

@@ -0,0 +1,5 @@
+- content_for(:header) do
+  %h1
+    Job Executions
+
+= render template: 'layouts/barbeque/application'

data/app/views/layouts/barbeque/job_queues.html.haml

@@ -0,0 +1,6 @@
+- content_for(:header) do
+  %h1
+    .fa.fa-cubes
+    Queues
+
+= render template: 'layouts/barbeque/application'

data/config/initializers/garage.rb

@@ -0,0 +1,4 @@
+require 'garage'
+
+Garage.configure {}
+Garage.configuration.strategy = Garage::Strategy::NoAuthentication

data/config/routes.rb

@@ -1,2 +1,34 @@
 Barbeque::Engine.routes.draw do
+  root to: 'apps#index'
+
+  resources :apps, except: :index
+
+  resources :job_definitions do
+    get :stats
+    get :execution_stats
+  end
+
+  resources :job_executions, only: :show do
+    post :retry
+
+    resources :job_retries, only: :show
+  end
+
+  resources :job_queues
+
+  scope :v1, module: 'api', as: :v1 do
+    resources :apps, only: [], param: :name, constraints: { name: /[\w-]+/ } do
+      resource :revision_lock, only: [:create, :destroy]
+    end
+
+    resources :job_executions, only: :show, param: :message_id,
+      constraints: { message_id: /[a-f\d]{8}-([a-f\d]{4}-){3}[a-f\d]{12}/ } do
+      resources :job_retries, only: [:create], path: 'retries'
+    end
+  end
+
+  scope :v2, module: 'api', as: :v2 do
+    resources :job_executions, only: :create, param: :message_id,
+      constraints: { message_id: /[a-f\d]{8}-([a-f\d]{4}-){3}[a-f\d]{12}/ }
+  end
 end

data/db/migrate/20160829023237_prefix_barbeque_to_tables.rb

@@ -0,0 +1,10 @@
+class PrefixBarbequeToTables < ActiveRecord::Migration[5.0]
+  def change
+    rename_table :apps, :barbeque_apps
+    rename_table :job_definitions, :barbeque_job_definitions
+    rename_table :job_executions, :barbeque_job_executions
+    rename_table :job_queues, :barbeque_job_queues
+    rename_table :job_retries, :barbeque_job_retries
+    rename_table :slack_notifications, :barbeque_slack_notifications
+  end
+end

data/lib/barbeque.rb

@@ -1,5 +1,3 @@
-require "barbeque/engine"
-
-module Barbeque
-  # Your code goes here...
-end
+require 'barbeque/docker_image'
+require 'barbeque/engine'
+require 'barbeque/execution_log'

data/lib/barbeque/configuration.rb

@@ -0,0 +1,19 @@
+require 'erb'
+require 'yaml'
+require 'hashie'
+
+module Barbeque
+  module Configuration
+    def config
+      @config ||= build_config
+    end
+
+    def build_config
+      filename = Rails.root.join('config', 'barbeque.yml').to_s
+      hash = YAML.load(ERB.new(File.read(filename)).result)
+      Hashie::Mash.new(hash[Rails.env])
+    end
+  end
+
+  extend Configuration
+end

data/lib/barbeque/docker_image.rb

@@ -0,0 +1,20 @@
+module Barbeque
+  class DockerImage
+    DEFAULT_TAG = 'latest'
+
+    def initialize(str)
+      # See: https://github.com/docker/docker/blob/v1.10.2/image/spec/v1.md
+      result = str.match(%r{((?<registry>[^/]+)?/)?(?<repository>[\w./-]+)(:(?<tag>[\w.-]+))?\z})
+      @repository = result[:repository]
+      @tag        = result[:tag] || DEFAULT_TAG
+      @registry   = result[:registry] || ENV['BARBEQUE_DOCKER_REGISTRY']
+    end
+
+    attr_reader :registry, :repository
+    attr_accessor :tag
+
+    def to_s
+      [registry, "#{repository}:#{tag}"].compact.join('/')
+    end
+  end
+end

data/lib/barbeque/engine.rb

@@ -1,5 +1,16 @@
 module Barbeque
   class Engine < ::Rails::Engine
     isolate_namespace Barbeque
+
+    config.before_configuration do
+      # Listing gems which are mountable engine or have railtie.
+      require 'adminlte2-rails'
+      require 'coffee-rails'
+      require 'hamlit'
+      require 'jquery-rails'
+      require 'kaminari'
+      require 'sass-rails'
+      require 'weak_parameters'
+    end
   end
 end

data/lib/barbeque/execution_log.rb

@@ -0,0 +1,51 @@
+require 'aws-sdk'
+require 'active_support'
+require 'active_support/core_ext'
+
+module Barbeque
+  class ExecutionLog
+    DEFAULT_S3_BUCKET_NAME = 'barbeque'
+
+    class << self
+      delegate :save, :load, to: :new
+    end
+
+    # @param [Barbeque::JobExecution,Barbeque::JobRetry] execution
+    # @param [Hash] log
+    def save(execution:, log:)
+      s3.put_object(
+        bucket: s3_bucket_name,
+        key:    s3_key_for(execution: execution),
+        body:   log.to_json,
+      )
+    end
+
+    # @param [Barbeque::JobExecution,Barbeque::JobRetry] execution
+    # @return [Hash] log
+    def load(execution:)
+      return {} if execution.pending?
+
+      s3_object = s3.get_object(
+        bucket: s3_bucket_name,
+        key:    s3_key_for(execution: execution),
+      )
+      JSON.load(s3_object.body.read)
+    end
+
+    private
+
+    def s3_bucket_name
+      @s3_bucket_name ||= ENV['BARBEQUE_S3_BUCKET_NAME'] || DEFAULT_S3_BUCKET_NAME
+    end
+
+    # @param [Barbeque::JobExecution,Barbeque::JobRetry] execution
+    # @param [String] message_id
+    def s3_key_for(execution:)
+      File.join(execution.app.name, execution.job_definition.job, execution.message_id)
+    end
+
+    def s3
+      @s3 ||= Aws::S3::Client.new
+    end
+  end
+end

data/lib/barbeque/message.rb

@@ -0,0 +1,28 @@
+require 'barbeque/message/base'
+require 'barbeque/message/invalid_message'
+require 'barbeque/message/job_execution'
+require 'barbeque/message/job_retry'
+
+module Barbeque
+  module Message
+    class << self
+      # @param [Aws::SQS::Types::Message] raw_message
+      # @return [Barbeque::Message::Base]
+      def parse(raw_message)
+        body = JSON.parse(raw_message.body)
+        klass = find_class(body['Type'])
+        klass.new(raw_message, body)
+      rescue JSON::ParserError
+        InvalidMessage.new(raw_message, {})
+      end
+
+      private
+
+      def find_class(type)
+        Message.const_get(type, false)
+      rescue NameError
+        InvalidMessage
+      end
+    end
+  end
+end

data/lib/barbeque/message/base.rb

@@ -0,0 +1,29 @@
+module Barbeque
+  module Message
+    # A model wrapping Aws::SQS::Types::Message.
+    class Base
+      attr_reader :id             # [String] Barbeque::JobExecution is associated via `message_id`
+      attr_reader :receipt_handle # [String] Used to ack a message
+      attr_reader :type           # [String] "JobExecution", "JobRetry", etc
+
+      # @param [Aws::SQS::Types::Message] raw_message
+      # @param [Hash] parse result of `raw_message.body`
+      def initialize(raw_message, message_body)
+        assign_body(message_body)
+        @id             = raw_message.message_id
+        @receipt_handle = raw_message.receipt_handle
+      end
+
+      # To distinguish with `Barbeque::Message::InvalidMessage`
+      def valid?
+        true
+      end
+
+      private
+
+      def assign_body(message_body)
+        @type = message_body['Type']
+      end
+    end
+  end
+end

data/lib/barbeque/message/invalid_message.rb

@@ -0,0 +1,11 @@
+require 'barbeque/message/base'
+
+module Barbeque
+  module Message
+    class InvalidMessage < Base
+      def valid?
+        false
+      end
+    end
+  end
+end

data/lib/barbeque/message/job_execution.rb

@@ -0,0 +1,20 @@
+require 'barbeque/message/base'
+
+module Barbeque
+  module Message
+    class JobExecution < Base
+      attr_reader :body        # [Object] free-format JSON
+      attr_reader :application # [String] To specify `job_definitions.application_id`
+      attr_reader :job         # [String] To specify `job_definitions.name`
+
+      private
+
+      def assign_body(message_body)
+        super
+        @application = message_body['Application']
+        @job  = message_body['Job']
+        @body = message_body['Message']
+      end
+    end
+  end
+end

data/lib/barbeque/message/job_retry.rb

@@ -0,0 +1,16 @@
+require 'barbeque/message/base'
+
+module Barbeque
+  module Message
+    class JobRetry < Base
+      attr_reader :retry_message_id # [String] JobExection's message_id
+
+      private
+
+      def assign_body(message_body)
+        super
+        @retry_message_id = message_body['RetryMessageId']
+      end
+    end
+  end
+end

data/lib/barbeque/message_handler.rb

@@ -0,0 +1,8 @@
+module Barbeque
+  module MessageHandler
+    class DuplicatedExecution < StandardError; end
+  end
+end
+
+require 'barbeque/message_handler/job_execution'
+require 'barbeque/message_handler/job_retry'

data/lib/barbeque/message_handler/job_execution.rb

@@ -0,0 +1,86 @@
+require 'barbeque/docker_image'
+require 'barbeque/execution_log'
+require 'barbeque/runner'
+require 'barbeque/slack_client'
+
+module Barbeque
+  module MessageHandler
+    class JobExecution
+      # @param [Barbeque::Message::JobExecution] message
+      # @param [Barbeque::JobQueue] job_queue
+      def initialize(message:, job_queue:)
+        @message = message
+        @job_queue = job_queue
+      end
+
+      def run
+        job_execution = Barbeque::JobExecution.find_or_initialize_by(message_id: @message.id)
+        raise DuplicatedExecution if job_execution.persisted?
+        job_execution.update!(job_definition: job_definition, job_queue_id: @job_queue.id)
+
+        stdout, stderr, status = run_command
+        job_execution.update!(status: status.success? ? :success : :failed, finished_at: Time.now)
+        notify_slack(job_execution, status)
+
+        log_result(job_execution, stdout, stderr)
+      end
+
+      private
+
+      def log_result(execution, stdout, stderr)
+        log = { message: @message.body.to_json, stdout: stdout, stderr: stderr }
+        Barbeque::ExecutionLog.save(execution: execution, log: log)
+      end
+
+      def notify_slack(job_execution, status)
+        return if job_execution.slack_notification.nil?
+
+        client = Barbeque::SlackClient.new(job_execution.slack_notification.channel)
+        if status.success?
+          if job_execution.slack_notification.notify_success
+            client.notify_success("*[SUCCESS]* Succeeded to execute #{job_execution_link(job_execution)}")
+          end
+        else
+          client.notify_failure(
+            "*[FAILURE]* Failed to execute #{job_execution_link(job_execution)}" \
+            " #{job_execution.slack_notification.failure_notification_text}"
+          )
+        end
+      end
+
+      def job_execution_link(job_execution)
+        "<#{job_execution_url(job_execution)}|#{job_execution.job_definition.job} ##{job_execution.id}>"
+      end
+
+      def job_execution_url(job_execution)
+        Barbeque::Engine.routes.url_helpers.job_execution_url(job_execution, host: ENV['BARBEQUE_HOST'])
+      end
+
+      # @return [String] stdout
+      # @return [String] stderr
+      # @return [Process::Status] status
+      def run_command
+        image  = DockerImage.new(job_definition.app.docker_image)
+        runner = Runner.create(docker_image: image)
+        runner.run(job_definition.command, job_envs)
+      end
+
+      def job_envs
+        {
+          'BARBEQUE_JOB'         => @message.job,
+          'BARBEQUE_MESSAGE'     => @message.body.to_json,
+          'BARBEQUE_MESSAGE_ID'  => @message.id,
+          'BARBEQUE_QUEUE_NAME'  => @job_queue.name,
+          'BARBEQUE_RETRY_COUNT' => '0',
+        }
+      end
+
+      def job_definition
+        @job_definition ||= Barbeque::JobDefinition.joins(:app).find_by!(
+          job: @message.job,
+          barbeque_apps: { name: @message.application },
+        )
+      end
+    end
+  end
+end

data/lib/barbeque/message_handler/job_retry.rb

@@ -0,0 +1,83 @@
+require 'barbeque/docker_image'
+require 'barbeque/execution_log'
+require 'barbeque/runner'
+require 'barbeque/slack_client'
+
+module Barbeque
+  module MessageHandler
+    class JobRetry
+      # @param [Barbeque::Message::JobExecution] message
+      # @param [Barbeque::JobQueue] job_queue
+      def initialize(message:, job_queue:)
+        @message = message
+        @job_queue = job_queue
+      end
+
+      def run
+        job_retry = Barbeque::JobRetry.find_or_initialize_by(message_id: @message.id)
+        job_retry.update!(job_execution: job_execution)
+        job_execution.update!(status: 'retried')
+
+        stdout, stderr, result = run_command
+        status = result.success? ? :success : :failed
+        job_retry.update!(status: status, finished_at: Time.now)
+        job_execution.update!(status: status)
+        notify_slack(job_retry, result)
+
+        log_result(job_retry, stdout, stderr)
+      end
+
+      private
+
+      def log_result(job_retry, stdout, stderr)
+        log = { stdout: stdout, stderr: stderr }
+        Barbeque::ExecutionLog.save(execution: job_retry, log: log)
+      end
+
+      # @param [Barbeque::JobRetry] job_retry
+      # @param [Process::Status] result
+      def notify_slack(job_retry, result)
+        return if job_retry.slack_notification.nil?
+
+        client = Barbeque::SlackClient.new(job_retry.slack_notification.channel)
+        if result.success?
+          if job_retry.slack_notification.notify_success
+            client.notify_success("*[SUCCESS]* Succeeded to retry #{job_retry_link(job_retry)}")
+          end
+        else
+          client.notify_failure(
+            "*[FAILURE]* Failed to retry #{job_retry_link(job_retry)}" \
+            " #{job_execution.slack_notification.failure_notification_text}"
+          )
+        end
+      end
+
+      def job_retry_link(job_retry)
+        url = Barbeque::Engine.routes.url_helpers.job_execution_job_retry_url(
+          job_retry.job_execution, job_retry, host: ENV['BARBEQUE_HOST']
+        )
+        "<#{url}|#{job_retry.job_definition.job}'s retry ##{job_retry.id}>"
+      end
+
+      def job_execution
+        @job_execution ||= Barbeque::JobExecution.find_by!(message_id: @message.retry_message_id)
+      end
+
+      def run_command
+        image  = DockerImage.new(job_execution.app.docker_image)
+        runner = Runner.create(docker_image: image)
+        runner.run(job_execution.job_definition.command, job_envs)
+      end
+
+      def job_envs
+        {
+          'BARBEQUE_JOB'         => job_execution.job_definition.job,
+          'BARBEQUE_MESSAGE'     => job_execution.execution_log['message'],
+          'BARBEQUE_MESSAGE_ID'  => @message.retry_message_id,
+          'BARBEQUE_QUEUE_NAME'  => @job_queue.name,
+          'BARBEQUE_RETRY_COUNT' => job_execution.job_retries.count.to_s,
+        }
+      end
+    end
+  end
+end