barbeque 0.7.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,7 @@ module Barbeque
7
7
  attr_reader :type # [String] "JobExecution", "JobRetry", etc
8
8
 
9
9
  # @param [Aws::SQS::Types::Message] raw_message
10
- # @param [Hash] parse result of `raw_message.body`
10
+ # @param message_body [Hash] parse result of `raw_message.body`
11
11
  def initialize(raw_message, message_body)
12
12
  assign_body(message_body)
13
13
  @id = raw_message.message_id
@@ -1,91 +1,44 @@
1
- require 'barbeque/docker_image'
2
1
  require 'barbeque/execution_log'
3
- require 'barbeque/runner'
4
- require 'barbeque/slack_client'
2
+ require 'barbeque/executor'
3
+ require 'barbeque/slack_notifier'
5
4
 
6
5
  module Barbeque
7
6
  module MessageHandler
8
7
  class JobExecution
9
8
  # @param [Barbeque::Message::JobExecution] message
10
- # @param [Barbeque::JobQueue] job_queue
11
- def initialize(message:, job_queue:)
9
+ # @param [Barbeque::MessageQueue] message_queue
10
+ def initialize(message:, message_queue:)
12
11
  @message = message
13
- @job_queue = job_queue
12
+ @message_queue = message_queue
14
13
  end
15
14
 
16
15
  def run
17
16
  begin
18
- job_execution = Barbeque::JobExecution.create(message_id: @message.id, job_definition: job_definition, job_queue: @job_queue)
17
+ job_execution = Barbeque::JobExecution.create(message_id: @message.id, job_definition: job_definition, job_queue: @message_queue.job_queue)
19
18
  rescue ActiveRecord::RecordNotUnique => e
20
19
  raise DuplicatedExecution.new(e.message)
21
20
  end
22
- job_execution.update!(status: :running)
21
+ Barbeque::ExecutionLog.save_message(job_execution, @message)
22
+ @message_queue.delete_message(@message)
23
23
 
24
24
  begin
25
- stdout, stderr, status = run_command
25
+ Executor.create.start_execution(job_execution, job_envs)
26
26
  rescue Exception => e
27
27
  job_execution.update!(status: :error, finished_at: Time.now)
28
- log_result(job_execution, '', '')
29
- notify_slack(job_execution)
28
+ Barbeque::ExecutionLog.save_stdout_and_stderr(job_execution, '', "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}")
29
+ Barbeque::SlackNotifier.notify_job_execution(job_execution)
30
30
  raise e
31
31
  end
32
- job_execution.update!(status: status.success? ? :success : :failed, finished_at: Time.now)
33
- notify_slack(job_execution)
34
-
35
- log_result(job_execution, stdout, stderr)
36
32
  end
37
33
 
38
34
  private
39
35
 
40
- def log_result(execution, stdout, stderr)
41
- Barbeque::ExecutionLog.save_message(execution, @message) # TODO: Should be saved earlier
42
- Barbeque::ExecutionLog.save_stdout_and_stderr(execution, stdout, stderr)
43
- end
44
-
45
- def notify_slack(job_execution)
46
- return if job_execution.slack_notification.nil?
47
-
48
- client = Barbeque::SlackClient.new(job_execution.slack_notification.channel)
49
- if job_execution.success?
50
- if job_execution.slack_notification.notify_success
51
- client.notify_success("*[SUCCESS]* Succeeded to execute #{job_execution_link(job_execution)}")
52
- end
53
- elsif job_execution.failed?
54
- client.notify_failure(
55
- "*[FAILURE]* Failed to execute #{job_execution_link(job_execution)}" \
56
- " #{job_execution.slack_notification.failure_notification_text}"
57
- )
58
- else
59
- client.notify_failure(
60
- "*[ERROR]* Failed to execute #{job_execution_link(job_execution)}" \
61
- " #{job_execution.slack_notification.failure_notification_text}"
62
- )
63
- end
64
- end
65
-
66
- def job_execution_link(job_execution)
67
- "<#{job_execution_url(job_execution)}|#{job_execution.job_definition.job} ##{job_execution.id}>"
68
- end
69
-
70
- def job_execution_url(job_execution)
71
- Barbeque::Engine.routes.url_helpers.job_execution_url(job_execution, host: ENV['BARBEQUE_HOST'])
72
- end
73
-
74
- # @return [String] stdout
75
- # @return [String] stderr
76
- # @return [Process::Status] status
77
- def run_command
78
- image = DockerImage.new(job_definition.app.docker_image)
79
- runner = Runner.create(docker_image: image)
80
- runner.run(job_definition.command, job_envs)
81
- end
82
-
83
36
  def job_envs
84
37
  {
85
38
  'BARBEQUE_JOB' => @message.job,
86
39
  'BARBEQUE_MESSAGE' => @message.body.to_json,
87
40
  'BARBEQUE_MESSAGE_ID' => @message.id,
88
- 'BARBEQUE_QUEUE_NAME' => @job_queue.name,
41
+ 'BARBEQUE_QUEUE_NAME' => @message_queue.job_queue.name,
89
42
  'BARBEQUE_RETRY_COUNT' => '0',
90
43
  }
91
44
  end
@@ -1,7 +1,6 @@
1
- require 'barbeque/docker_image'
2
1
  require 'barbeque/execution_log'
3
- require 'barbeque/runner'
4
- require 'barbeque/slack_client'
2
+ require 'barbeque/executor'
3
+ require 'barbeque/slack_notifier'
5
4
 
6
5
  module Barbeque
7
6
  module MessageHandler
@@ -9,10 +8,10 @@ module Barbeque
9
8
 
10
9
  class JobRetry
11
10
  # @param [Barbeque::Message::JobExecution] message
12
- # @param [Barbeque::JobQueue] job_queue
13
- def initialize(message:, job_queue:)
11
+ # @param [Barbeque::MessageQueue] message_queue
12
+ def initialize(message:, message_queue:)
14
13
  @message = message
15
- @job_queue = job_queue
14
+ @message_queue = message_queue
16
15
  end
17
16
 
18
17
  def run
@@ -21,71 +20,25 @@ module Barbeque
21
20
  rescue ActiveRecord::RecordNotUnique => e
22
21
  raise DuplicatedExecution.new(e.message)
23
22
  end
24
- job_execution.update!(status: :retried)
25
- job_retry.update!(status: :running)
23
+ @message_queue.delete_message(@message)
26
24
 
27
25
  begin
28
- stdout, stderr, result = run_command
26
+ Executor.create.start_retry(job_retry, job_envs)
29
27
  rescue Exception => e
30
28
  job_retry.update!(status: :error, finished_at: Time.now)
31
29
  job_execution.update!(status: :error)
32
- log_result(job_retry, '', '')
33
- notify_slack(job_retry)
30
+ Barbeque::ExecutionLog.save_stdout_and_stderr(job_retry, '', "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}")
31
+ Barbeque::SlackNotifier.notify_job_retry(job_retry)
34
32
  raise e
35
33
  end
36
- status = result.success? ? :success : :failed
37
- job_retry.update!(status: status, finished_at: Time.now)
38
- job_execution.update!(status: status)
39
- notify_slack(job_retry)
40
-
41
- log_result(job_retry, stdout, stderr)
42
34
  end
43
35
 
44
36
  private
45
37
 
46
- def log_result(job_retry, stdout, stderr)
47
- Barbeque::ExecutionLog.save_stdout_and_stderr(job_retry, stdout, stderr)
48
- end
49
-
50
- # @param [Barbeque::JobRetry] job_retry
51
- def notify_slack(job_retry)
52
- return if job_retry.slack_notification.nil?
53
-
54
- client = Barbeque::SlackClient.new(job_retry.slack_notification.channel)
55
- if job_retry.success?
56
- if job_retry.slack_notification.notify_success
57
- client.notify_success("*[SUCCESS]* Succeeded to retry #{job_retry_link(job_retry)}")
58
- end
59
- elsif job_retry.failed?
60
- client.notify_failure(
61
- "*[FAILURE]* Failed to retry #{job_retry_link(job_retry)}" \
62
- " #{job_execution.slack_notification.failure_notification_text}"
63
- )
64
- else
65
- client.notify_failure(
66
- "*[ERROR]* Failed to retry #{job_retry_link(job_retry)}" \
67
- " #{job_execution.slack_notification.failure_notification_text}"
68
- )
69
- end
70
- end
71
-
72
- def job_retry_link(job_retry)
73
- url = Barbeque::Engine.routes.url_helpers.job_execution_job_retry_url(
74
- job_retry.job_execution, job_retry, host: ENV['BARBEQUE_HOST']
75
- )
76
- "<#{url}|#{job_retry.job_definition.job}'s retry ##{job_retry.id}>"
77
- end
78
-
79
38
  def job_execution
80
39
  @job_execution ||= Barbeque::JobExecution.find_by!(message_id: @message.retry_message_id)
81
40
  end
82
41
 
83
- def run_command
84
- image = DockerImage.new(job_execution.app.docker_image)
85
- runner = Runner.create(docker_image: image)
86
- runner.run(job_execution.job_definition.command, job_envs)
87
- end
88
-
89
42
  def job_envs
90
43
  if job_execution.execution_log.nil?
91
44
  raise MessageNotFound.new('failed to fetch retried message')
@@ -95,7 +48,7 @@ module Barbeque
95
48
  'BARBEQUE_JOB' => job_execution.job_definition.job,
96
49
  'BARBEQUE_MESSAGE' => job_execution.execution_log['message'],
97
50
  'BARBEQUE_MESSAGE_ID' => @message.retry_message_id,
98
- 'BARBEQUE_QUEUE_NAME' => @job_queue.name,
51
+ 'BARBEQUE_QUEUE_NAME' => @message_queue.job_queue.name,
99
52
  'BARBEQUE_RETRY_COUNT' => job_execution.job_retries.count.to_s,
100
53
  }
101
54
  end
@@ -1,19 +1,15 @@
1
- require 'barbeque/docker_image'
2
- require 'barbeque/execution_log'
3
- require 'barbeque/runner'
4
- require 'barbeque/slack_client'
5
1
  require 'barbeque/message_handler/job_execution'
6
2
 
7
3
  module Barbeque
8
4
  module MessageHandler
9
5
  class Notification < JobExecution
10
6
  # @param [Barbeque::Message::Notification] message
11
- # @param [Barbeque::JobQueue] job_queue
12
- def initialize(message:, job_queue:)
7
+ # @param [Barbeque::MessageQueue] message_queue
8
+ def initialize(message:, message_queue:)
13
9
  @message = message
14
- @job_queue = job_queue
10
+ @message_queue = message_queue
15
11
 
16
- subscription = SNSSubscription.find_by!(topic_arn: @message.topic_arn, job_queue_id: @job_queue.id)
12
+ subscription = SNSSubscription.find_by!(topic_arn: @message.topic_arn, job_queue_id: @message_queue.job_queue.id)
17
13
  @message.set_params_from_subscription(subscription)
18
14
  end
19
15
  end
@@ -1,4 +1,5 @@
1
1
  require 'aws-sdk'
2
+ require 'barbeque/config'
2
3
  require 'barbeque/message'
3
4
 
4
5
  module Barbeque
@@ -11,20 +12,29 @@ module Barbeque
11
12
  @stop = false
12
13
  end
13
14
 
14
- # Receive a message and delete them all from SQS queue immediately.
15
- # TODO: Stop removing them immediately to implement retry.
15
+ # Receive a message from SQS queue.
16
16
  # @return [Barbeque::Message::Base]
17
17
  def dequeue
18
- while valid_messages.empty?
18
+ loop do
19
19
  return nil if @stop
20
- messages = receive_messages
21
- messages = reject_invalid_messages(messages)
22
- valid_messages.concat(messages)
20
+ message = receive_message
21
+ if message
22
+ if message.valid?
23
+ return message
24
+ else
25
+ delete_message(message)
26
+ end
27
+ end
23
28
  end
29
+ end
24
30
 
25
- valid_messages.shift.tap do |message|
26
- delete_message(message)
27
- end
31
+ # Remove a message from SQS queue.
32
+ # @param [Barbeque::Message::Base] message
33
+ def delete_message(message)
34
+ client.delete_message(
35
+ queue_url: @job_queue.queue_url,
36
+ receipt_handle: message.receipt_handle,
37
+ )
28
38
  end
29
39
 
30
40
  def stop!
@@ -33,30 +43,15 @@ module Barbeque
33
43
 
34
44
  private
35
45
 
36
- def receive_messages
46
+ def receive_message
37
47
  result = client.receive_message(
38
48
  queue_url: @job_queue.queue_url,
39
- wait_time_seconds: Barbeque::JobQueue::SQS_RECEIVE_MESSAGE_WAIT_TIME,
40
- )
41
- result.messages.map { |m| Barbeque::Message.parse(m) }
42
- end
43
-
44
- def reject_invalid_messages(messages)
45
- messages, invalid_messages = messages.partition(&:valid?)
46
- invalid_messages.each { |m| delete_message(m) }
47
- messages
48
- end
49
-
50
- # Remove a message from SQS queue.
51
- def delete_message(message)
52
- client.delete_message(
53
- queue_url: @job_queue.queue_url,
54
- receipt_handle: message.receipt_handle,
49
+ wait_time_seconds: Barbeque.config.sqs_receive_message_wait_time,
50
+ max_number_of_messages: 1,
55
51
  )
56
- end
57
-
58
- def valid_messages
59
- @valid_messages ||= []
52
+ if result.messages[0]
53
+ Barbeque::Message.parse(result.messages[0])
54
+ end
60
55
  end
61
56
 
62
57
  def client
@@ -0,0 +1,34 @@
1
+ module Barbeque
2
+ class RetryPoller
3
+ def initialize
4
+ @stop_requested = false
5
+ end
6
+
7
+ def run
8
+ Barbeque::JobRetry.running.find_in_batches do |job_retries|
9
+ job_retries.shuffle.each do |job_retry|
10
+ if @stop_requested
11
+ return
12
+ end
13
+ job_retry.with_lock do
14
+ if job_retry.running?
15
+ poll(job_retry)
16
+ end
17
+ end
18
+ end
19
+ end
20
+ sleep 1
21
+ end
22
+
23
+ def stop
24
+ @stop_requested = true
25
+ end
26
+
27
+ private
28
+
29
+ def poll(job_retry)
30
+ executor = Executor.create
31
+ executor.poll_retry(job_retry)
32
+ end
33
+ end
34
+ end
@@ -1,13 +1,57 @@
1
1
  require 'barbeque/config'
2
- require 'barbeque/runner/docker'
3
- require 'barbeque/runner/hako'
2
+ require 'barbeque/exception_handler'
3
+ require 'barbeque/execution_log'
4
+ require 'barbeque/message_handler'
5
+ require 'barbeque/message_queue'
4
6
 
5
7
  module Barbeque
6
- module Runner
7
- # @param [Barbeque::DockerImage] docker_image
8
- def self.create(docker_image:)
9
- klass = const_get(Barbeque.config.runner, false)
10
- klass.new(Barbeque.config.runner_options.merge(docker_image: docker_image))
8
+ # Part of barbeque-worker.
9
+ # Runner dequeues a message from {MessageQueue} (Amazon SQS) and dispatches
10
+ # it to message handler.
11
+
12
+ class Runner
13
+ DEFAULT_QUEUE = 'default'
14
+
15
+ def initialize(queue_name: ENV['BARBEQUE_QUEUE'] || DEFAULT_QUEUE)
16
+ @queue_name = queue_name
17
+ end
18
+
19
+ def run
20
+ keep_maximum_concurrent_executions
21
+
22
+ message = message_queue.dequeue
23
+ return unless message
24
+
25
+ handler = MessageHandler.const_get(message.type, false)
26
+ handler.new(message: message, message_queue: message_queue).run
27
+ end
28
+
29
+ def stop
30
+ message_queue.stop!
31
+ end
32
+
33
+ private
34
+
35
+ def message_queue
36
+ @message_queue ||= MessageQueue.new(@queue_name)
37
+ end
38
+
39
+ def keep_maximum_concurrent_executions
40
+ max_num = Barbeque.config.maximum_concurrent_executions
41
+ unless max_num
42
+ # nil means unlimited
43
+ return
44
+ end
45
+
46
+ loop do
47
+ current_num = Barbeque::JobExecution.where(status: [:running, :retried]).count
48
+ if current_num < max_num
49
+ return
50
+ end
51
+ interval = Barbeque.config.runner_wait_seconds
52
+ Rails.logger.info("#{current_num} executions are running but maximum_concurrent_executions is configured to #{max_num}. Waiting #{interval} seconds...")
53
+ sleep(interval)
54
+ end
11
55
  end
12
56
  end
13
57
  end
@@ -0,0 +1,73 @@
1
+ require 'barbeque/slack_client'
2
+
3
+ module Barbeque
4
+ module SlackNotifier
5
+ class << self
6
+ # @param [Barbeque::JobExecution] job_execution
7
+ def notify_job_execution(job_execution)
8
+ return if job_execution.slack_notification.nil?
9
+
10
+ client = Barbeque::SlackClient.new(job_execution.slack_notification.channel)
11
+ if job_execution.success?
12
+ if job_execution.slack_notification.notify_success
13
+ client.notify_success("*[SUCCESS]* Succeeded to execute #{job_execution_link(job_execution)}")
14
+ end
15
+ elsif job_execution.failed?
16
+ client.notify_failure(
17
+ "*[FAILURE]* Failed to execute #{job_execution_link(job_execution)}" \
18
+ " #{job_execution.slack_notification.failure_notification_text}"
19
+ )
20
+ else
21
+ client.notify_failure(
22
+ "*[ERROR]* Failed to execute #{job_execution_link(job_execution)}" \
23
+ " #{job_execution.slack_notification.failure_notification_text}"
24
+ )
25
+ end
26
+ end
27
+
28
+ # @param [Barbeque::JobRetry] job_retry
29
+ def notify_job_retry(job_retry)
30
+ return if job_retry.slack_notification.nil?
31
+
32
+ client = Barbeque::SlackClient.new(job_retry.slack_notification.channel)
33
+ if job_retry.success?
34
+ if job_retry.slack_notification.notify_success
35
+ client.notify_success("*[SUCCESS]* Succeeded to retry #{job_retry_link(job_retry)}")
36
+ end
37
+ elsif job_retry.failed?
38
+ client.notify_failure(
39
+ "*[FAILURE]* Failed to retry #{job_retry_link(job_retry)}" \
40
+ " #{job_retry.slack_notification.failure_notification_text}"
41
+ )
42
+ else
43
+ client.notify_failure(
44
+ "*[ERROR]* Failed to retry #{job_retry_link(job_retry)}" \
45
+ " #{job_retry.slack_notification.failure_notification_text}"
46
+ )
47
+ end
48
+ end
49
+
50
+ private
51
+
52
+ def barbeque_host
53
+ ENV['BARBEQUE_HOST']
54
+ end
55
+
56
+ def job_execution_link(job_execution)
57
+ "<#{job_execution_url(job_execution)}|#{job_execution.job_definition.job} ##{job_execution.id}>"
58
+ end
59
+
60
+ def job_execution_url(job_execution)
61
+ Barbeque::Engine.routes.url_helpers.job_execution_url(job_execution, host: barbeque_host)
62
+ end
63
+
64
+ def job_retry_link(job_retry)
65
+ "<#{job_retry_url(job_retry)}|#{job_retry.job_definition.job}'s retry ##{job_retry.id}>"
66
+ end
67
+
68
+ def job_retry_url(job_retry)
69
+ Barbeque::Engine.routes.url_helpers.job_execution_job_retry_url(job_retry.job_execution, job_retry, host: barbeque_host)
70
+ end
71
+ end
72
+ end
73
+ end