barbeque 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7811c426dbfc4b9b7514f444e45845ce6f894a67
4
- data.tar.gz: 32fb25ca095959abc65c0e751d8edcfd36efee9d
3
+ metadata.gz: 234d0202835f94f4ecb493933e9805c1302148a2
4
+ data.tar.gz: cac682927dfc758791fcebf8e30a4d37daf98d76
5
5
  SHA512:
6
- metadata.gz: ab8d6fb436a2dedbd01be1e094199ffb580bcfe5091bb457dd8260223421ed7afd26d02699b0be7aae18ca998a9283bffa54d53a0f1593074e459d728b65b119
7
- data.tar.gz: 8fb3fedcce8573d92bb6019583d229db768c8ed24e6c107874ea2a02f86fb11d2ad29e68d46013b3b379e191c7d861619b6b389917b557dea8481ad8d3a9df8b
6
+ metadata.gz: 76e7a70a6bffa609ce74dbb90783a7835da3876c34e75c19f711bad301d6b517b01e7bc7c2b6a614ec71c6297921cf36c49b01ecc8d5a072a2606cdb29ca99d0
7
+ data.tar.gz: 5ab873c9a336c285af66b758931628eaa604e05266454960c4a2974a6ddaa9ab0bcd9e87866c3b2cc8d47bafd0a913ab56fe06675949770667976eae0fcd67f5
@@ -0,0 +1,139 @@
1
+ class Barbeque::SnsSubscriptionsController < Barbeque::ApplicationController
2
+ def index
3
+ @sns_subscriptions = Barbeque::SNSSubscription.all
4
+ end
5
+
6
+ def show
7
+ @sns_subscription = Barbeque::SNSSubscription.find(params[:id])
8
+ end
9
+
10
+ def new
11
+ @sns_topic_arns = fetch_sns_topic_arns
12
+ @sns_subscription = Barbeque::SNSSubscription.new
13
+ end
14
+
15
+ def edit
16
+ @sns_subscription = Barbeque::SNSSubscription.find(params[:id])
17
+ end
18
+
19
+ def create
20
+ @sns_subscription = Barbeque::SNSSubscription.new(params.require(:sns_subscription).permit(:topic_arn, :job_queue_id, :job_definition_id))
21
+
22
+ if @sns_subscription.save
23
+ update_sqs_policy!
24
+ subscribe_topic!
25
+ redirect_to @sns_subscription, notice: 'SNS subscription was successfully created.'
26
+ else
27
+ render :new
28
+ end
29
+ end
30
+
31
+ def update
32
+ @sns_subscription = Barbeque::SNSSubscription.find(params[:id])
33
+ if @sns_subscription.update(params.require(:sns_subscription).permit(:job_definition_id))
34
+ update_sqs_policy!
35
+ redirect_to @sns_subscription, notice: 'SNS subscription was successfully updated.'
36
+ else
37
+ render :edit
38
+ end
39
+ end
40
+
41
+ def destroy
42
+ @sns_subscription = Barbeque::SNSSubscription.find(params[:id])
43
+ @sns_subscription.destroy
44
+ update_sqs_policy!
45
+ unsubscribe_topic!
46
+ redirect_to sns_subscriptions_path, notice: 'SNS subscription was successfully destroyed.'
47
+ end
48
+
49
+ private
50
+
51
+ def fetch_sns_topic_arns
52
+ sns_client.list_topics.topics.map(&:topic_arn)
53
+ end
54
+
55
+ def update_sqs_policy!
56
+ attrs = sqs_client.get_queue_attributes(
57
+ queue_url: @sns_subscription.job_queue.queue_url,
58
+ attribute_names: ['QueueArn'],
59
+ )
60
+ queue_arn = attrs.attributes['QueueArn']
61
+ topic_arns = @sns_subscription.job_queue.sns_subscriptions.map(&:topic_arn)
62
+
63
+ if topic_arns.present?
64
+ policy = generate_policy(queue_arn: queue_arn, topic_arns: topic_arns)
65
+ else
66
+ policy = '' # Be blank when there're no subscriptions.
67
+ end
68
+
69
+ sqs_client.set_queue_attributes(
70
+ queue_url: @sns_subscription.job_queue.queue_url,
71
+ attributes: { 'Policy' => policy },
72
+ )
73
+ end
74
+
75
+ # @paaram [String] queue_arn
76
+ # @param [Array<String>] topic_arns
77
+ # @return [String] JSON formatted policy
78
+ def generate_policy(queue_arn:, topic_arns:)
79
+ {
80
+ 'Version' => '2012-10-17',
81
+ 'Statement' => [
82
+ 'Effect' => 'Allow',
83
+ 'Principal' => '*',
84
+ 'Action' => 'sqs:SendMessage',
85
+ 'Resource' => queue_arn,
86
+ 'Condition' => {
87
+ 'ArnEquals' => {
88
+ 'aws:SourceArn' => topic_arns,
89
+ }
90
+ }
91
+ ]
92
+ }.to_json
93
+ end
94
+
95
+ def subscribe_topic!
96
+ sns_client.subscribe(
97
+ topic_arn: @sns_subscription.topic_arn,
98
+ protocol: 'sqs',
99
+ endpoint: queue_arn
100
+ )
101
+ end
102
+
103
+ def unsubscribe_topic!
104
+ sqs_attrs = sqs_client.get_queue_attributes(
105
+ queue_url: @sns_subscription.job_queue.queue_url,
106
+ attribute_names: ['QueueArn'],
107
+ )
108
+ queue_arn = sqs_attrs.attributes['QueueArn']
109
+
110
+ subscriptions = sns_client.list_subscriptions_by_topic(
111
+ topic_arn: @sns_subscription.topic_arn,
112
+ )
113
+ subscription_arn = subscriptions.subscriptions.find {|subscription| subscription.endpoint == queue_arn }.try!(:subscription_arn)
114
+
115
+ if subscription_arn
116
+ sns_client.unsubscribe(
117
+ subscription_arn: subscription_arn,
118
+ )
119
+ end
120
+ end
121
+
122
+ def sqs_client
123
+ @sqs_client ||= Aws::SQS::Client.new
124
+ end
125
+
126
+ def sns_client
127
+ @sns_client ||= Aws::SNS::Client.new
128
+ end
129
+
130
+ def queue_arn
131
+ return @queue_arn if @queue_arn
132
+
133
+ sqs_attrs = sqs_client.get_queue_attributes(
134
+ queue_url: @sns_subscription.job_queue.queue_url,
135
+ attribute_names: ['QueueArn'],
136
+ )
137
+ @queue_arn = sqs_attrs.attributes['QueueArn']
138
+ end
139
+ end
@@ -2,6 +2,8 @@ class Barbeque::JobQueue < Barbeque::ApplicationRecord
2
2
  SQS_NAME_PREFIX = ENV['BARBEQUE_SQS_NAME_PREFIX'] || 'Barbeque-'
3
3
  SQS_NAME_MAX_LENGTH = 80
4
4
 
5
+ has_many :sns_subscriptions, class_name: 'SNSSubscription', dependent: :destroy
6
+
5
7
  # All SQS queues' "ReceiveMessageWaitTimeSeconds" are configured to be 20s (maximum).
6
8
  # This should be as large as possible to reduce API-calling cost by long polling.
7
9
  # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html#API_CreateQueue_RequestParameters
@@ -0,0 +1,11 @@
1
+ module Barbeque
2
+ class SNSSubscription < ApplicationRecord
3
+ belongs_to :job_queue
4
+ belongs_to :job_definition
5
+ has_one :app, through: :job_definition
6
+
7
+ validates :topic_arn,
8
+ uniqueness: { scope: :job_queue, message: 'should be set with only one queue' },
9
+ presence: true
10
+ end
11
+ end
@@ -0,0 +1,39 @@
1
+ .box.box-primary
2
+ .box-header
3
+ %h3.box-title.with_padding
4
+ #{action_name.capitalize} SNS Subscription
5
+
6
+ .box-body
7
+ = form_for @sns_subscription do |f|
8
+ - if @sns_subscription.errors.any?
9
+ %strong #{pluralize(@sns_subscription.errors.count, 'error')} prohibited this SNS subscription from being saved:
10
+ %ul
11
+ - @sns_subscription.errors.full_messages.each do |msg|
12
+ %li= msg
13
+
14
+ .row.form-group
15
+ .col-md-8
16
+ = f.label :topic_arn
17
+ - if @sns_subscription.persisted?
18
+ .sns_subscription_topic_arn= @sns_subscription.topic_arn
19
+ - else
20
+ = f.collection_select :topic_arn, @sns_topic_arns.map {|t| [t] }, :first, :first,
21
+ { prompt: true }, class: 'form-control'
22
+
23
+ .row.form-group
24
+ .col-md-4
25
+ = f.label :job_queue_id
26
+ - if @sns_subscription.persisted?
27
+ .sns_subscription_job_queue_id= link_to @sns_subscription.job_queue.name, @sns_subscription.job_queue
28
+ - else
29
+ = f.collection_select :job_queue_id, Barbeque::JobQueue.pluck(:id, :name), :first, :second,
30
+ { prompt: true }, class: 'form-control'
31
+
32
+ .row.form-group
33
+ .col-md-4
34
+ = f.label :job_definition_id
35
+ = f.collection_select :job_definition_id, Barbeque::JobDefinition.pluck(:id, :job), :first, :second,
36
+ { prompt: true }, class: 'form-control'
37
+
38
+ .form-group
39
+ = f.submit 'Save', class: 'btn btn-primary'
@@ -0,0 +1,3 @@
1
+ = render 'form'
2
+
3
+ = link_to 'Back', sns_subscription_path(@sns_subscription)
@@ -0,0 +1,24 @@
1
+ .box.box-primary
2
+ .box-header
3
+ %h3.box-title.with_padding
4
+ All SNS Subscriptions
5
+ = link_to new_sns_subscription_path, class: 'btn btn-primary pull-right' do
6
+ New SNS Subscriptions
7
+
8
+ .box-body
9
+ %table.table.table-bordered
10
+ %thead
11
+ %tr
12
+ %th topic_arn
13
+ %th Job Queue
14
+ %th Job Definition
15
+ %th
16
+
17
+ %tbody
18
+ - @sns_subscriptions.each do |sns_subscription|
19
+ %tr
20
+ %td= sns_subscription.topic_arn
21
+ %td= sns_subscription.job_queue.name
22
+ %td= sns_subscription.job_definition.job
23
+ %td
24
+ = link_to 'View Details', sns_subscription, class: 'btn btn-default btn-sm'
@@ -0,0 +1,3 @@
1
+ = render 'form'
2
+
3
+ = link_to 'Back', sns_subscriptions_path
@@ -0,0 +1,28 @@
1
+ .row
2
+ .col-sm-7
3
+ .box.box-primary
4
+ .box-header
5
+ %h3.box-title.with_padding
6
+ SNS Subscription Details
7
+
8
+ .box-body
9
+ %p#notice= notice
10
+
11
+ %table.table.table-bordered
12
+ %tbody
13
+ %tr
14
+ %th topic_arn
15
+ %td= @sns_subscription.topic_arn
16
+ %tr
17
+ %th Job Queue
18
+ %td= link_to @sns_subscription.job_queue.name, @sns_subscription.job_queue
19
+ %tr
20
+ %th Job Defenition
21
+ %td= link_to @sns_subscription.job_definition.job, @sns_subscription.job_definition
22
+ %tr
23
+
24
+ = link_to 'Edit', edit_sns_subscription_path(@sns_subscription), class: 'btn btn-primary'
25
+ = link_to 'Destroy', sns_subscription_path(@sns_subscription),
26
+ class: 'btn', method: :delete, data: { confirm: 'Are you sure to delete SNS subscription?' }
27
+
28
+ = link_to 'Back', sns_subscriptions_path
@@ -14,6 +14,10 @@
14
14
  = link_to job_queues_path do
15
15
  %i.fa.fa-cubes
16
16
  %span Queues
17
+ %li{ class: ('active' if controller_path == 'barbeque/sns_subscriptions') }
18
+ = link_to sns_subscriptions_path do
19
+ %i.fa.fa-calendar-plus-o
20
+ %span SNS Subscriptions
17
21
  %li{ class: ('active' if controller_path == 'barbeque/monitors') }
18
22
  = link_to monitors_path do
19
23
  %i.fa.fa-tachometer
@@ -6,7 +6,7 @@
6
6
  %title Barbeque
7
7
  %meta{ content: 'width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no', name: 'viewport' }
8
8
  -# FIXME: Don't rely on others' CDN
9
- %link{ href: 'https://maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css', rel: 'stylesheet', type: 'text/css' }
9
+ %link{ href: 'https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css', rel: 'stylesheet', type: 'text/css' }
10
10
  %link{ href: 'https://code.ionicframework.com/ionicons/2.0.1/css/ionicons.min.css', rel: 'stylesheet', type: 'text/css' }
11
11
  = stylesheet_link_tag 'barbeque/application', media: 'all'
12
12
  = javascript_include_tag 'barbeque/application'
data/config/routes.rb CHANGED
@@ -16,6 +16,8 @@ Barbeque::Engine.routes.draw do
16
16
 
17
17
  resources :job_queues
18
18
 
19
+ resources :sns_subscriptions
20
+
19
21
  resources :monitors, only: %i[index]
20
22
 
21
23
  scope :v1, module: 'api', as: :v1 do
@@ -0,0 +1,11 @@
1
+ class CreateBarbequeSnsSubscriptions < ActiveRecord::Migration[5.0]
2
+ def change
3
+ create_table :barbeque_sns_subscriptions do |t|
4
+ t.string :topic_arn, null: false
5
+ t.integer :job_queue_id, null: false
6
+ t.integer :job_definition_id, null: false
7
+
8
+ t.timestamps
9
+ end
10
+ end
11
+ end
@@ -3,6 +3,7 @@ require 'barbeque/message/base'
3
3
  require 'barbeque/message/invalid_message'
4
4
  require 'barbeque/message/job_execution'
5
5
  require 'barbeque/message/job_retry'
6
+ require 'barbeque/message/notification'
6
7
 
7
8
  module Barbeque
8
9
  module Message
@@ -4,8 +4,8 @@ module Barbeque
4
4
  module Message
5
5
  class JobExecution < Base
6
6
  attr_reader :body # [Object] free-format JSON
7
- attr_reader :application # [String] To specify `job_definitions.application_id`
8
- attr_reader :job # [String] To specify `job_definitions.name`
7
+ attr_reader :application # [String] To specify `job_definitions.app.name`
8
+ attr_reader :job # [String] To specify `job_definitions.job`
9
9
 
10
10
  private
11
11
 
@@ -0,0 +1,28 @@
1
+ require 'barbeque/message/base'
2
+
3
+ module Barbeque
4
+ module Message
5
+ class Notification < Base
6
+ attr_reader :body # [Object] free-format JSON
7
+ attr_reader :topic_arn # [String] To specify `subscription.topic_arn`
8
+ attr_reader :application # [String] To specify `job_definitions.app.name`
9
+ attr_reader :job # [String] To specify `job_definitions.job`
10
+
11
+ # @param [Barneque::SNSSubscription] subscription
12
+ # @return [Barbeque::Message::Notification]
13
+ def set_params_from_subscription(subscription)
14
+ @application = subscription.app.name
15
+ @job = subscription.job_definition.job
16
+ self
17
+ end
18
+
19
+ private
20
+
21
+ def assign_body(message_body)
22
+ super
23
+ @topic_arn = message_body['TopicArn']
24
+ @body = message_body['Message']
25
+ end
26
+ end
27
+ end
28
+ end
@@ -6,3 +6,4 @@ end
6
6
 
7
7
  require 'barbeque/message_handler/job_execution'
8
8
  require 'barbeque/message_handler/job_retry'
9
+ require 'barbeque/message_handler/notification'
@@ -0,0 +1,21 @@
1
+ require 'barbeque/docker_image'
2
+ require 'barbeque/execution_log'
3
+ require 'barbeque/runner'
4
+ require 'barbeque/slack_client'
5
+ require 'barbeque/message_handler/job_execution'
6
+
7
+ module Barbeque
8
+ module MessageHandler
9
+ class Notification < JobExecution
10
+ # @param [Barbeque::Message::Notification] message
11
+ # @param [Barbeque::JobQueue] job_queue
12
+ def initialize(message:, job_queue:)
13
+ @message = message
14
+ @job_queue = job_queue
15
+
16
+ subscription = SNSSubscription.find_by!(topic_arn: @message.topic_arn, job_queue_id: @job_queue.id)
17
+ @message.set_params_from_subscription(subscription)
18
+ end
19
+ end
20
+ end
21
+ end
@@ -1,3 +1,3 @@
1
1
  module Barbeque
2
- VERSION = '0.3.0'
2
+ VERSION = '0.4.0'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: barbeque
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takashi Kokubun
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-04-17 00:00:00.000000000 Z
11
+ date: 2017-04-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: adminlte2-rails
@@ -289,6 +289,7 @@ files:
289
289
  - app/controllers/barbeque/job_queues_controller.rb
290
290
  - app/controllers/barbeque/job_retries_controller.rb
291
291
  - app/controllers/barbeque/monitors_controller.rb
292
+ - app/controllers/barbeque/sns_subscriptions_controller.rb
292
293
  - app/helpers/barbeque/application_helper.rb
293
294
  - app/helpers/barbeque/job_definitions_helper.rb
294
295
  - app/helpers/barbeque/job_executions_helper.rb
@@ -305,6 +306,7 @@ files:
305
306
  - app/models/barbeque/job_queue.rb
306
307
  - app/models/barbeque/job_retry.rb
307
308
  - app/models/barbeque/slack_notification.rb
309
+ - app/models/barbeque/sns_subscription.rb
308
310
  - app/services/barbeque/message_enqueuing_service.rb
309
311
  - app/services/barbeque/message_retrying_service.rb
310
312
  - app/views/barbeque/apps/_form.html.haml
@@ -327,6 +329,11 @@ files:
327
329
  - app/views/barbeque/job_queues/show.html.haml
328
330
  - app/views/barbeque/job_retries/show.html.haml
329
331
  - app/views/barbeque/monitors/index.html.haml
332
+ - app/views/barbeque/sns_subscriptions/_form.html.haml
333
+ - app/views/barbeque/sns_subscriptions/edit.html.haml
334
+ - app/views/barbeque/sns_subscriptions/index.html.haml
335
+ - app/views/barbeque/sns_subscriptions/new.html.haml
336
+ - app/views/barbeque/sns_subscriptions/show.html.haml
330
337
  - app/views/layouts/barbeque/_header.html.haml
331
338
  - app/views/layouts/barbeque/_sidebar.html.haml
332
339
  - app/views/layouts/barbeque/application.html.haml
@@ -347,6 +354,7 @@ files:
347
354
  - db/migrate/20160509041452_add_job_queue_id_to_job_executions.rb
348
355
  - db/migrate/20160516041710_create_job_retries.rb
349
356
  - db/migrate/20160829023237_prefix_barbeque_to_tables.rb
357
+ - db/migrate/20170420030157_create_barbeque_sns_subscriptions.rb
350
358
  - lib/barbeque.rb
351
359
  - lib/barbeque/config.rb
352
360
  - lib/barbeque/docker_image.rb
@@ -358,9 +366,11 @@ files:
358
366
  - lib/barbeque/message/invalid_message.rb
359
367
  - lib/barbeque/message/job_execution.rb
360
368
  - lib/barbeque/message/job_retry.rb
369
+ - lib/barbeque/message/notification.rb
361
370
  - lib/barbeque/message_handler.rb
362
371
  - lib/barbeque/message_handler/job_execution.rb
363
372
  - lib/barbeque/message_handler/job_retry.rb
373
+ - lib/barbeque/message_handler/notification.rb
364
374
  - lib/barbeque/message_queue.rb
365
375
  - lib/barbeque/runner.rb
366
376
  - lib/barbeque/runner/docker.rb
@@ -390,7 +400,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
390
400
  version: '0'
391
401
  requirements: []
392
402
  rubyforge_project:
393
- rubygems_version: 2.6.10
403
+ rubygems_version: 2.6.11
394
404
  signing_key:
395
405
  specification_version: 4
396
406
  summary: Job queue system to run job with Docker