barbeque 0.0.1 → 0.0.10

data/lib/barbeque/message_queue.rb

@@ -0,0 +1,66 @@
+require 'aws-sdk'
+require 'barbeque/message'
+
+module Barbeque
+  class MessageQueue
+    attr_reader :job_queue
+
+    def initialize(queue_name)
+      @job_queue = Barbeque::JobQueue.find_by!(name: queue_name)
+      @messages  = []
+      @stop      = false
+    end
+
+    # Receive a message and delete them all from SQS queue immediately.
+    # TODO: Stop removing them immediately to implement retry.
+    # @return [Barbeque::Message::Base]
+    def dequeue
+      while valid_messages.empty?
+        return nil if @stop
+        messages = receive_messages
+        messages = reject_invalid_messages(messages)
+        valid_messages.concat(messages)
+      end
+
+      valid_messages.shift.tap do |message|
+        delete_message(message)
+      end
+    end
+
+    def stop!
+      @stop = true
+    end
+
+    private
+
+    def receive_messages
+      result = client.receive_message(
+        queue_url: @job_queue.queue_url,
+        wait_time_seconds: Barbeque::JobQueue::SQS_RECEIVE_MESSAGE_WAIT_TIME,
+      )
+      result.messages.map { |m| Barbeque::Message.parse(m) }
+    end
+
+    def reject_invalid_messages(messages)
+      messages, invalid_messages = messages.partition(&:valid?)
+      invalid_messages.each { |m| delete_message(m) }
+      messages
+    end
+
+    # Remove a message from SQS queue.
+    def delete_message(message)
+      client.delete_message(
+        queue_url:      @job_queue.queue_url,
+        receipt_handle: message.receipt_handle,
+      )
+    end
+
+    def valid_messages
+      @valid_messages ||= []
+    end
+
+    def client
+      @client ||= Aws::SQS::Client.new
+    end
+  end
+end

data/lib/barbeque/runner.rb

@@ -0,0 +1,12 @@
+require 'barbeque/configuration'
+require 'barbeque/runner/docker'
+
+module Barbeque
+  module Runner
+    # @param [Barbeque::DockerImage] docker_image
+    def self.create(docker_image:)
+      klass = const_get(Barbeque.config.runner, false)
+      klass.new(docker_image: docker_image)
+    end
+  end
+end

data/lib/barbeque/runner/docker.rb

@@ -0,0 +1,34 @@
+require 'open3'
+
+module Barbeque
+  module Runner
+    class Docker
+      # @param [Barbeque::DockerImage] docker_image
+      def initialize(docker_image:)
+        @docker_image = docker_image.to_s
+      end
+
+      # @param [Array<String>] command
+      # @param [Hash] envs
+      # @return [String] stdout
+      # @return [String] stderr
+      # @return [Process::Status] status
+      def run(command, envs)
+        cmd = build_docker_run_command(command, envs)
+        Bundler.with_clean_env { Open3.capture3(*cmd) }
+      end
+
+      private
+
+      def build_docker_run_command(command, envs)
+        ['docker', 'run', *env_options(envs), @docker_image, *command]
+      end
+
+      def env_options(envs)
+        envs.flat_map do |key, value|
+          ['--env', "#{key}=#{value}"]
+        end
+      end
+    end
+  end
+end

data/lib/barbeque/slack_client.rb

@@ -0,0 +1,48 @@
+require 'net/http'
+require 'json'
+require 'uri'
+
+module Barbeque
+  class SlackClient
+    def initialize(channel)
+      @channel = channel
+    end
+
+    def notify_success(message)
+      post_slack(
+        attachments: [{
+          text: message,
+          color: 'good',
+          mrkdwn_in: ['text'],
+        }],
+      )
+    end
+
+    def notify_failure(message)
+      post_slack(
+        attachments: [{
+          text: message,
+          color: 'danger',
+          mrkdwn_in: ['text'],
+        }],
+      )
+    end
+
+    private
+
+    def post_slack(payload)
+      Net::HTTP.post_form(
+        endpoint_uri,
+        payload: default_payload.merge(payload).to_json,
+      )
+    end
+
+    def default_payload
+      { link_names: 1, channel: @channel }
+    end
+
+    def endpoint_uri
+      @endpoint_uri ||= URI.parse(ENV['SLACK_WEBHOOK_URL'])
+    end
+  end
+end

data/lib/barbeque/version.rb

@@ -1,3 +1,3 @@
 module Barbeque
-  VERSION = '0.0.1'
+  VERSION = '0.0.10'
 end

data/lib/barbeque/worker.rb

@@ -0,0 +1,58 @@
+require 'barbeque/message_handler'
+require 'barbeque/message_queue'
+require 'serverengine'
+
+module Barbeque
+  module Worker
+    class UnexpectedMessageType < StandardError; end
+
+    DEFAULT_QUEUE = 'default'
+
+    def self.run
+      options = {
+        worker_type: 'process',
+        workers:     (ENV['BARBEQUE_WORKER_NUM'] || 4).to_i,
+        daemonize:   ENV['DAEMONIZE_BARBEQUE'] == '1',
+        log:         Rails.env.production? ? Rails.root.join("log/barbeque_worker.log").to_s : $stdout,
+        log_level:   Rails.env.production? ? :info : :debug,
+        pid_path:    Rails.root.join('tmp/pids/barbeque_worker.pid').to_s,
+        supervisor:  Rails.env.production?,
+      }
+
+      worker = ServerEngine.create(nil, Barbeque::Worker, options)
+      worker.run
+    end
+
+    def initialize
+      @queue_name = ENV['BARBEQUE_QUEUE'] || DEFAULT_QUEUE
+    end
+
+    def run
+      until @stop
+        execute_job
+      end
+    end
+
+    def stop
+      @stop = true
+      message_queue.stop!
+    end
+
+    def execute_job
+      message = message_queue.dequeue
+      return unless message
+
+      handler = MessageHandler.const_get(message.type, false)
+      handler.new(message: message, job_queue: message_queue.job_queue).run
+    rescue => e
+      # Use Raven.capture_exception
+      Rails.logger.fatal("[ERROR] #{e.inspect}\n#{e.backtrace.join("\n")}")
+    end
+
+    private
+
+    def message_queue
+      @message_queue ||= MessageQueue.new(@queue_name)
+    end
+  end
+end

data/lib/tasks/barbeque_tasks.rake

@@ -1,4 +1,9 @@
-# desc "Explaining what the task does"
-# task :barbeque do
-#   # Task goes here
-# end
+namespace :barbeque do
+  desc 'Start worker to execute jobs'
+  task worker: :environment do
+    ENV['BARBEQUE_QUEUE'] ||= 'default'
+
+    require 'barbeque/worker'
+    Barbeque::Worker.run
+  end
+end

metadata

@@ -1,15 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: barbeque
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.10
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-29 00:00:00.000000000 Z
+date: 2016-09-01 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: adminlte2-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coffee-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hamlit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jquery-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: kaminari
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -24,6 +122,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: sass-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: serverengine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: the_garage
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +164,62 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: uglifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: weak_parameters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: factory_girl_rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
@@ -52,7 +234,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: barbeque engine
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Job queue interface to run job with Docker
 email:
 - takashi-kokubun@cookpad.com
 executables: []
@@ -64,23 +260,65 @@ files:
 - Rakefile
 - app/assets/config/barbeque_manifest.js
 - app/assets/javascripts/barbeque/application.js
-- app/assets/stylesheets/barbeque/application.css
+- app/assets/javascripts/barbeque/job_definitions.coffee
+- app/assets/stylesheets/barbeque/application.scss
+- app/assets/stylesheets/barbeque/common.scss
+- app/assets/stylesheets/barbeque/job_definitions.scss
+- app/controllers/barbeque/api/application_controller.rb
+- app/controllers/barbeque/api/job_executions_controller.rb
+- app/controllers/barbeque/api/job_retries_controller.rb
+- app/controllers/barbeque/api/revision_locks_controller.rb
 - app/controllers/barbeque/application_controller.rb
+- app/controllers/barbeque/apps_controller.rb
+- app/controllers/barbeque/job_definitions_controller.rb
+- app/controllers/barbeque/job_executions_controller.rb
+- app/controllers/barbeque/job_queues_controller.rb
+- app/controllers/barbeque/job_retries_controller.rb
 - app/helpers/barbeque/application_helper.rb
+- app/helpers/barbeque/job_definitions_helper.rb
+- app/helpers/barbeque/job_executions_helper.rb
 - app/jobs/barbeque/application_job.rb
 - app/mailers/barbeque/application_mailer.rb
-- app/models/api/application_resource.rb
-- app/models/api/job_execution_resource.rb
-- app/models/api/job_retry_resource.rb
-- app/models/app.rb
-- app/models/application_record.rb
+- app/models/barbeque/api/application_resource.rb
+- app/models/barbeque/api/job_execution_resource.rb
+- app/models/barbeque/api/job_retry_resource.rb
+- app/models/barbeque/api/revision_lock_resource.rb
+- app/models/barbeque/app.rb
 - app/models/barbeque/application_record.rb
-- app/models/job_definition.rb
-- app/models/job_execution.rb
-- app/models/job_queue.rb
-- app/models/job_retry.rb
-- app/models/slack_notification.rb
-- app/views/layouts/barbeque/application.html.erb
+- app/models/barbeque/job_definition.rb
+- app/models/barbeque/job_execution.rb
+- app/models/barbeque/job_queue.rb
+- app/models/barbeque/job_retry.rb
+- app/models/barbeque/slack_notification.rb
+- app/services/barbeque/message_enqueuing_service.rb
+- app/services/barbeque/message_retrying_service.rb
+- app/views/barbeque/apps/_form.html.haml
+- app/views/barbeque/apps/edit.html.haml
+- app/views/barbeque/apps/index.html.haml
+- app/views/barbeque/apps/new.html.haml
+- app/views/barbeque/apps/show.html.haml
+- app/views/barbeque/job_definitions/_form.html.haml
+- app/views/barbeque/job_definitions/_slack_notification_field.html.haml
+- app/views/barbeque/job_definitions/edit.html.haml
+- app/views/barbeque/job_definitions/index.html.haml
+- app/views/barbeque/job_definitions/new.html.haml
+- app/views/barbeque/job_definitions/show.html.haml
+- app/views/barbeque/job_definitions/stats.html.haml
+- app/views/barbeque/job_executions/show.html.haml
+- app/views/barbeque/job_queues/_form.html.haml
+- app/views/barbeque/job_queues/edit.html.haml
+- app/views/barbeque/job_queues/index.html.haml
+- app/views/barbeque/job_queues/new.html.haml
+- app/views/barbeque/job_queues/show.html.haml
+- app/views/barbeque/job_retries/show.html.haml
+- app/views/layouts/barbeque/_header.html.haml
+- app/views/layouts/barbeque/_sidebar.html.haml
+- app/views/layouts/barbeque/application.html.haml
+- app/views/layouts/barbeque/apps.html.haml
+- app/views/layouts/barbeque/job_definitions.html.haml
+- app/views/layouts/barbeque/job_executions.html.haml
+- app/views/layouts/barbeque/job_queues.html.haml
+- config/initializers/garage.rb
 - config/routes.rb
 - db/migrate/20160217020910_create_job_queues.rb
 - db/migrate/20160219010912_create_job_executions.rb
@@ -91,11 +329,28 @@ files:
 - db/migrate/20160415043427_create_slack_notifications.rb
 - db/migrate/20160509041452_add_job_queue_id_to_job_executions.rb
 - db/migrate/20160516041710_create_job_retries.rb
+- db/migrate/20160829023237_prefix_barbeque_to_tables.rb
 - lib/barbeque.rb
+- lib/barbeque/configuration.rb
+- lib/barbeque/docker_image.rb
 - lib/barbeque/engine.rb
+- lib/barbeque/execution_log.rb
+- lib/barbeque/message.rb
+- lib/barbeque/message/base.rb
+- lib/barbeque/message/invalid_message.rb
+- lib/barbeque/message/job_execution.rb
+- lib/barbeque/message/job_retry.rb
+- lib/barbeque/message_handler.rb
+- lib/barbeque/message_handler/job_execution.rb
+- lib/barbeque/message_handler/job_retry.rb
+- lib/barbeque/message_queue.rb
+- lib/barbeque/runner.rb
+- lib/barbeque/runner/docker.rb
+- lib/barbeque/slack_client.rb
 - lib/barbeque/version.rb
+- lib/barbeque/worker.rb
 - lib/tasks/barbeque_tasks.rake
-homepage: 
+homepage: https://github.com/cookpad/barbeque
 licenses:
 - MIT
 metadata: {}
@@ -118,6 +373,5 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: barbeque engine
+summary: Job queue interface to run job with Docker
 test_files: []
-has_rdoc: