barbeque 0.0.1 → 0.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -3
  3. data/Rakefile +2 -6
  4. data/app/assets/javascripts/barbeque/application.js +5 -0
  5. data/app/assets/javascripts/barbeque/job_definitions.coffee +10 -0
  6. data/app/assets/stylesheets/barbeque/application.scss +7 -0
  7. data/app/assets/stylesheets/barbeque/common.scss +22 -0
  8. data/app/assets/stylesheets/barbeque/job_definitions.scss +21 -0
  9. data/app/controllers/barbeque/api/application_controller.rb +22 -0
  10. data/app/controllers/barbeque/api/job_executions_controller.rb +35 -0
  11. data/app/controllers/barbeque/api/job_retries_controller.rb +28 -0
  12. data/app/controllers/barbeque/api/revision_locks_controller.rb +34 -0
  13. data/app/controllers/barbeque/apps_controller.rb +43 -0
  14. data/app/controllers/barbeque/job_definitions_controller.rb +86 -0
  15. data/app/controllers/barbeque/job_executions_controller.rb +19 -0
  16. data/app/controllers/barbeque/job_queues_controller.rb +59 -0
  17. data/app/controllers/barbeque/job_retries_controller.rb +10 -0
  18. data/app/helpers/barbeque/job_definitions_helper.rb +14 -0
  19. data/app/helpers/barbeque/job_executions_helper.rb +18 -0
  20. data/app/models/{api → barbeque/api}/application_resource.rb +3 -1
  21. data/app/models/{api → barbeque/api}/job_execution_resource.rb +1 -1
  22. data/app/models/{api → barbeque/api}/job_retry_resource.rb +1 -1
  23. data/app/models/barbeque/api/revision_lock_resource.rb +7 -0
  24. data/app/models/{app.rb → barbeque/app.rb} +1 -1
  25. data/app/models/barbeque/job_definition.rb +26 -0
  26. data/app/models/{job_execution.rb → barbeque/job_execution.rb} +7 -4
  27. data/app/models/{job_queue.rb → barbeque/job_queue.rb} +1 -1
  28. data/app/models/{job_retry.rb → barbeque/job_retry.rb} +7 -2
  29. data/app/models/{slack_notification.rb → barbeque/slack_notification.rb} +1 -1
  30. data/app/services/barbeque/message_enqueuing_service.rb +41 -0
  31. data/app/services/barbeque/message_retrying_service.rb +32 -0
  32. data/app/views/barbeque/apps/_form.html.haml +34 -0
  33. data/app/views/barbeque/apps/edit.html.haml +3 -0
  34. data/app/views/barbeque/apps/index.html.haml +24 -0
  35. data/app/views/barbeque/apps/new.html.haml +3 -0
  36. data/app/views/barbeque/apps/show.html.haml +47 -0
  37. data/app/views/barbeque/job_definitions/_form.html.haml +45 -0
  38. data/app/views/barbeque/job_definitions/_slack_notification_field.html.haml +35 -0
  39. data/app/views/barbeque/job_definitions/edit.html.haml +3 -0
  40. data/app/views/barbeque/job_definitions/index.html.haml +24 -0
  41. data/app/views/barbeque/job_definitions/new.html.haml +3 -0
  42. data/app/views/barbeque/job_definitions/show.html.haml +90 -0
  43. data/app/views/barbeque/job_definitions/stats.html.haml +52 -0
  44. data/app/views/barbeque/job_executions/show.html.haml +92 -0
  45. data/app/views/barbeque/job_queues/_form.html.haml +29 -0
  46. data/app/views/barbeque/job_queues/edit.html.haml +3 -0
  47. data/app/views/barbeque/job_queues/index.html.haml +22 -0
  48. data/app/views/barbeque/job_queues/new.html.haml +3 -0
  49. data/app/views/barbeque/job_queues/show.html.haml +22 -0
  50. data/app/views/barbeque/job_retries/show.html.haml +59 -0
  51. data/app/views/layouts/barbeque/_header.html.haml +10 -0
  52. data/app/views/layouts/barbeque/_sidebar.html.haml +16 -0
  53. data/app/views/layouts/barbeque/application.html.haml +24 -0
  54. data/app/views/layouts/barbeque/apps.html.haml +6 -0
  55. data/app/views/layouts/barbeque/job_definitions.html.haml +6 -0
  56. data/app/views/layouts/barbeque/job_executions.html.haml +5 -0
  57. data/app/views/layouts/barbeque/job_queues.html.haml +6 -0
  58. data/config/initializers/garage.rb +4 -0
  59. data/config/routes.rb +32 -0
  60. data/db/migrate/20160829023237_prefix_barbeque_to_tables.rb +10 -0
  61. data/lib/barbeque.rb +3 -5
  62. data/lib/barbeque/configuration.rb +19 -0
  63. data/lib/barbeque/docker_image.rb +20 -0
  64. data/lib/barbeque/engine.rb +11 -0
  65. data/lib/barbeque/execution_log.rb +51 -0
  66. data/lib/barbeque/message.rb +28 -0
  67. data/lib/barbeque/message/base.rb +29 -0
  68. data/lib/barbeque/message/invalid_message.rb +11 -0
  69. data/lib/barbeque/message/job_execution.rb +20 -0
  70. data/lib/barbeque/message/job_retry.rb +16 -0
  71. data/lib/barbeque/message_handler.rb +8 -0
  72. data/lib/barbeque/message_handler/job_execution.rb +86 -0
  73. data/lib/barbeque/message_handler/job_retry.rb +83 -0
  74. data/lib/barbeque/message_queue.rb +66 -0
  75. data/lib/barbeque/runner.rb +12 -0
  76. data/lib/barbeque/runner/docker.rb +34 -0
  77. data/lib/barbeque/slack_client.rb +48 -0
  78. data/lib/barbeque/version.rb +1 -1
  79. data/lib/barbeque/worker.rb +58 -0
  80. data/lib/tasks/barbeque_tasks.rake +9 -4
  81. metadata +272 -18
  82. data/app/assets/stylesheets/barbeque/application.css +0 -15
  83. data/app/models/application_record.rb +0 -3
  84. data/app/models/job_definition.rb +0 -14
  85. data/app/views/layouts/barbeque/application.html.erb +0 -14
@@ -0,0 +1,10 @@
1
+ class Barbeque::JobRetriesController < Barbeque::ApplicationController
2
+ def show
3
+ @job_execution = Barbeque::JobExecution.find(params[:job_execution_id])
4
+ @message = @job_execution.execution_log['message']
5
+
6
+ @job_retry = Barbeque::JobRetry.find(params[:id])
7
+ @stdout = @job_retry.execution_log['stdout']
8
+ @stderr = @job_retry.execution_log['stderr']
9
+ end
10
+ end
@@ -0,0 +1,14 @@
1
+ module Barbeque::JobDefinitionsHelper
2
+ def distance_of_time(from, to)
3
+ return '' if from.nil? || to.nil?
4
+
5
+ secs = (to - from).to_i
6
+ mins = secs / 60
7
+ hours = mins / 60
8
+ days = hours / 24
9
+
10
+ text = "%02d:%02d:%02d" % [hours % 24, mins % 60, secs % 60]
11
+ text.prepend("#{days}days ") if days > 0
12
+ text
13
+ end
14
+ end
@@ -0,0 +1,18 @@
1
+ module Barbeque::JobExecutionsHelper
2
+ def status_label(status)
3
+ color =
4
+ case status
5
+ when 'success'
6
+ 'success'
7
+ when 'failed'
8
+ 'danger'
9
+ when 'retried'
10
+ 'warning'
11
+ when 'pending'
12
+ 'info'
13
+ else
14
+ 'default'
15
+ end
16
+ content_tag(:span, status.upcase, class: "label label-#{color}")
17
+ end
18
+ end
@@ -1,4 +1,6 @@
1
- class Api::ApplicationResource
1
+ require 'garage'
2
+
3
+ class Barbeque::Api::ApplicationResource
2
4
  include Garage::Representer
3
5
  include Garage::Authorizable
4
6
 
@@ -1,4 +1,4 @@
1
- class Api::JobExecutionResource < Api::ApplicationResource
1
+ class Barbeque::Api::JobExecutionResource < Barbeque::Api::ApplicationResource
2
2
  property :message_id
3
3
 
4
4
  property :status
@@ -1,4 +1,4 @@
1
- class Api::JobRetryResource < Api::ApplicationResource
1
+ class Barbeque::Api::JobRetryResource < Barbeque::Api::ApplicationResource
2
2
  property :message_id
3
3
 
4
4
  property :status
@@ -0,0 +1,7 @@
1
+ class Barbeque::Api::RevisionLockResource < Barbeque::Api::ApplicationResource
2
+ property :revision
3
+
4
+ def revision
5
+ Barbeque::DockerImage.new(@model.docker_image).tag
6
+ end
7
+ end
@@ -1,4 +1,4 @@
1
- class App < ApplicationRecord
1
+ class Barbeque::App < Barbeque::ApplicationRecord
2
2
  validates :name, presence: true, uniqueness: true
3
3
  validates :docker_image, presence: true
4
4
 
@@ -0,0 +1,26 @@
1
+ class Barbeque::JobDefinition < Barbeque::ApplicationRecord
2
+ belongs_to :app
3
+ has_many :job_executions, dependent: :destroy
4
+ has_one :slack_notification, dependent: :destroy
5
+
6
+ validates :job, uniqueness: { scope: :app_id }
7
+
8
+ attr_readonly :app_id
9
+ attr_readonly :job
10
+
11
+ serialize :command, Array
12
+
13
+ accepts_nested_attributes_for :slack_notification, allow_destroy: true
14
+
15
+ DATE_HOUR_SQL = 'date_format(created_at, "%Y-%m-%d %H:00:00")'
16
+
17
+ def execution_stats(from, to)
18
+ job_executions.where(created_at: from .. to).group(DATE_HOUR_SQL).order(DATE_HOUR_SQL).pluck("#{DATE_HOUR_SQL}, count(1), avg(timestampdiff(second, created_at, finished_at))").map do |date_hour, count, avg_time|
19
+ {
20
+ date_hour: Time.zone.parse("#{date_hour} UTC"),
21
+ count: count,
22
+ avg_time: avg_time,
23
+ }
24
+ end
25
+ end
26
+ end
@@ -1,6 +1,4 @@
1
- require 'job_executor/storage'
2
-
3
- class JobExecution < ApplicationRecord
1
+ class Barbeque::JobExecution < Barbeque::ApplicationRecord
4
2
  belongs_to :job_definition
5
3
  belongs_to :job_queue
6
4
  has_one :slack_notification, through: :job_definition
@@ -16,7 +14,12 @@ class JobExecution < ApplicationRecord
16
14
 
17
15
  paginates_per 15
18
16
 
17
+ # @return [Hash] - A hash created by `JobExecutor::Job#log_result`
18
+ def execution_log
19
+ @execution_log ||= Barbeque::ExecutionLog.load(execution: self)
20
+ end
21
+
19
22
  def to_resource
20
- Api::JobExecutionResource.new(self)
23
+ Barbeque::Api::JobExecutionResource.new(self)
21
24
  end
22
25
  end
@@ -1,4 +1,4 @@
1
- class JobQueue < ApplicationRecord
1
+ class Barbeque::JobQueue < Barbeque::ApplicationRecord
2
2
  SQS_NAME_PREFIX = ENV['BARBEQUE_SQS_NAME_PREFIX'] || 'Barbeque-'
3
3
  SQS_NAME_MAX_LENGTH = 80
4
4
 
@@ -1,4 +1,4 @@
1
- class JobRetry < ApplicationRecord
1
+ class Barbeque::JobRetry < Barbeque::ApplicationRecord
2
2
  belongs_to :job_execution
3
3
  has_one :job_definition, through: :job_execution
4
4
  has_one :app, through: :job_definition
@@ -11,7 +11,12 @@ class JobRetry < ApplicationRecord
11
11
  retried: 3,
12
12
  }
13
13
 
14
+ # @return [Hash] - A hash created by `JobExecutor::Retry#log_result`
15
+ def execution_log
16
+ @execution_log ||= Barbeque::ExecutionLog.load(execution: self)
17
+ end
18
+
14
19
  def to_resource
15
- Api::JobRetryResource.new(self)
20
+ Barbeque::Api::JobRetryResource.new(self)
16
21
  end
17
22
  end
@@ -1,4 +1,4 @@
1
- class SlackNotification < ApplicationRecord
1
+ class Barbeque::SlackNotification < Barbeque::ApplicationRecord
2
2
  belongs_to :job_definition, optional: true
3
3
 
4
4
  validates :channel, presence: true
@@ -0,0 +1,41 @@
1
+ require 'aws-sdk'
2
+
3
+ class Barbeque::MessageEnqueuingService
4
+ DEFAULT_QUEUE = ENV['BARBEQUE_DEFAULT_QUEUE'] || 'default'
5
+
6
+ # @param [String] application
7
+ # @param [String] job
8
+ # @param [Object] message
9
+ # @param optional [String] queue
10
+ def initialize(application:, job:, message:, queue: nil)
11
+ @application = application
12
+ @job = job
13
+ @queue = queue || DEFAULT_QUEUE
14
+ @message = message
15
+ end
16
+
17
+ # @return [String] message_id
18
+ def run
19
+ queue = Barbeque::JobQueue.find_by!(name: @queue)
20
+ response = client.send_message(
21
+ queue_url: queue.queue_url,
22
+ message_body: build_message.to_json,
23
+ )
24
+ response.message_id
25
+ end
26
+
27
+ private
28
+
29
+ def build_message
30
+ {
31
+ 'Type' => 'JobExecution',
32
+ 'Application' => @application,
33
+ 'Job' => @job,
34
+ 'Message' => @message,
35
+ }
36
+ end
37
+
38
+ def client
39
+ @client ||= Aws::SQS::Client.new
40
+ end
41
+ end
@@ -0,0 +1,32 @@
1
+ require 'aws-sdk'
2
+
3
+ class Barbeque::MessageRetryingService
4
+ DEFAULT_DELAY_SECONDS = 0
5
+
6
+ def initialize(message_id:, delay_seconds: nil)
7
+ @message_id = message_id
8
+ @delay_seconds = delay_seconds || DEFAULT_DELAY_SECONDS
9
+ end
10
+
11
+ def run
12
+ execution = Barbeque::JobExecution.find_by!(message_id: @message_id)
13
+ client.send_message(
14
+ queue_url: execution.job_queue.queue_url,
15
+ message_body: build_message.to_json,
16
+ delay_seconds: @delay_seconds,
17
+ )
18
+ end
19
+
20
+ private
21
+
22
+ def build_message
23
+ {
24
+ 'Type' => 'JobRetry',
25
+ 'RetryMessageId' => @message_id,
26
+ }
27
+ end
28
+
29
+ def client
30
+ @client ||= Aws::SQS::Client.new
31
+ end
32
+ end
@@ -0,0 +1,34 @@
1
+ .box.box-primary
2
+ .box-header
3
+ %h3.box-title.with_padding
4
+ #{action_name.capitalize} Application
5
+
6
+ .box-body
7
+ = form_for @app do |f|
8
+ - if @app.errors.any?
9
+ %strong #{pluralize(@app.errors.count, 'error')} prohibited this application from being saved:
10
+ %ul
11
+ - @app.errors.full_messages.each do |msg|
12
+ %li= msg
13
+
14
+ .row.form-group
15
+ .col-md-4
16
+ = f.label :name
17
+ - if @app.persisted?
18
+ -# Name can't be changed after it's created.
19
+ .app_name= @app.name
20
+ - else
21
+ = f.text_field :name, class: 'form-control'
22
+
23
+ .row.form-group
24
+ .col-md-4
25
+ = f.label :docker_image
26
+ = f.text_field :docker_image, class: 'form-control'
27
+
28
+ .row.form-group
29
+ .col-md-8
30
+ = f.label :description
31
+ = f.text_area :description, class: 'form-control', rows: 10
32
+
33
+ .form-group
34
+ = f.submit 'Save', class: 'btn btn-primary'
@@ -0,0 +1,3 @@
1
+ = render 'form'
2
+
3
+ = link_to 'Back', root_path
@@ -0,0 +1,24 @@
1
+ .box.box-primary
2
+ .box-header
3
+ %h3.box-title.with_padding
4
+ All Applications
5
+ = link_to new_app_path, class: 'btn btn-primary pull-right' do
6
+ New Application
7
+
8
+ .box-body
9
+ %table.table.table-bordered
10
+ %thead
11
+ %tr
12
+ %th Name
13
+ %th Docker image
14
+ %th Description
15
+ %th
16
+
17
+ %tbody
18
+ - @apps.each do |app|
19
+ %tr
20
+ %td= app.name
21
+ %td= app.docker_image
22
+ %td= app.description
23
+ %td
24
+ = link_to 'View Details', app, class: 'btn btn-default btn-sm'
@@ -0,0 +1,3 @@
1
+ = render 'form'
2
+
3
+ = link_to 'Back', root_path
@@ -0,0 +1,47 @@
1
+ .box.box-primary
2
+ .box-header
3
+ %h3.box-title.with_padding
4
+ Application Details
5
+
6
+ .box-body
7
+ %p#notice= notice
8
+
9
+ %table.table.table-bordered
10
+ %tbody
11
+ %tr
12
+ %th Name
13
+ %td= @app.name
14
+ %tr
15
+ %th Docker image
16
+ %td= @app.docker_image
17
+ %tr
18
+ %th Description
19
+ %td= @app.description
20
+
21
+ = link_to 'Edit', edit_app_path(@app), class: 'btn btn-primary'
22
+ = link_to 'Destroy', app_path(@app),
23
+ class: 'btn', method: :delete, data: { confirm: 'Are you sure to delete application?' }
24
+
25
+ .box
26
+ .box-header
27
+ %h3.box-title.with_padding Job Definitions
28
+ = link_to 'New Job Definition', new_job_definition_path(job_definition: { app_id: @app.id }), class: 'btn btn-primary pull-right'
29
+ .box-body
30
+ %table.table.table-bordered
31
+ %thead
32
+ %tr
33
+ %th Application
34
+ %th Job
35
+ %th Description
36
+ %th
37
+
38
+ %tbody
39
+ - @app.job_definitions.each do |job_definition|
40
+ %tr
41
+ %td= job_definition.app.name
42
+ %td= job_definition.job
43
+ %td= job_definition.description
44
+ %td
45
+ = link_to 'View Details', job_definition, class: 'btn btn-default btn-sm'
46
+
47
+ = link_to 'Back', root_path
@@ -0,0 +1,45 @@
1
+ .box.box-primary
2
+ .box-header
3
+ %h3.box-title.with_padding
4
+ #{action_name.capitalize} Job Definition
5
+
6
+ .box-body
7
+ = form_for @job_definition do |f|
8
+ - if @job_definition.errors.any?
9
+ %strong #{pluralize(@job_definition.errors.count, 'error')} prohibited this job_definition from being saved:
10
+ %ul
11
+ - @job_definition.errors.full_messages.each do |msg|
12
+ %li= msg
13
+
14
+ .row.form-group
15
+ .col-md-4
16
+ = f.label :app_id
17
+ - if @job_definition.persisted?
18
+ .job_definition_app_name= @job_definition.app.name
19
+ - else
20
+ = f.collection_select :app_id, Barbeque::App.pluck(:id, :name), :first, :second,
21
+ { prompt: true }, class: 'form-control'
22
+
23
+ .row.form-group
24
+ .col-md-4
25
+ = f.label :job
26
+ - if @job_definition.persisted?
27
+ .job_definition_job= @job_definition.job
28
+ - else
29
+ = f.text_field :job, class: 'form-control', placeholder: 'SomeAsyncJob'
30
+
31
+ .row.form-group
32
+ .col-md-8
33
+ = f.label :command
34
+ = f.text_field :command, value: Shellwords.join(@job_definition.command),
35
+ class: 'form-control', placeholder: 'bundle exec rake ...'
36
+
37
+ .row.form-group
38
+ .col-md-8
39
+ = f.label :description
40
+ = f.text_area :description, class: 'form-control', rows: 10
41
+
42
+ = render 'slack_notification_field', job_definition: @job_definition
43
+
44
+ .form-group
45
+ = f.submit 'Save', class: 'btn btn-primary'
@@ -0,0 +1,35 @@
1
+ = fields_for job_definition do |job_definition_f|
2
+ = job_definition_f.fields_for :slack_notification do |f|
3
+ = f.hidden_field :id
4
+
5
+ .row
6
+ .col-md-8
7
+ %label Slack Notification
8
+
9
+ - use_slack_notification = f.object.persisted?
10
+ .row.form-group
11
+ .col-md-8.use_slack_notification_wrapper
12
+ = f.label :_destroy_true do
13
+ = f.radio_button :_destroy, true, class: 'use_slack_notification', checked: !use_slack_notification
14
+ No
15
+ = f.label :_destroy_false do
16
+ = f.radio_button :_destroy, false, class: 'use_slack_notification', checked: use_slack_notification
17
+ Yes
18
+
19
+ .slack_notification_field{ class: ('active' if use_slack_notification) }
20
+ .row.form-group
21
+ .col-md-8
22
+ = f.text_field :channel, placeholder: '#slack-channel', class: 'form-control'
23
+
24
+ .row.form-group
25
+ .col-md-8
26
+ = f.check_box :notify_success
27
+ = f.label :notify_success, 'Notify success event to Slack'
28
+
29
+ .row
30
+ .col-md-8
31
+ Failure notification text
32
+ .row.form-group
33
+ .col-md-8
34
+ = f.text_field :failure_notification_text,
35
+ placeholder: '@YourName', class: 'form-control'