barbeque 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7811c426dbfc4b9b7514f444e45845ce6f894a67
-  data.tar.gz: 32fb25ca095959abc65c0e751d8edcfd36efee9d
+  metadata.gz: 234d0202835f94f4ecb493933e9805c1302148a2
+  data.tar.gz: cac682927dfc758791fcebf8e30a4d37daf98d76
 SHA512:
-  metadata.gz: ab8d6fb436a2dedbd01be1e094199ffb580bcfe5091bb457dd8260223421ed7afd26d02699b0be7aae18ca998a9283bffa54d53a0f1593074e459d728b65b119
-  data.tar.gz: 8fb3fedcce8573d92bb6019583d229db768c8ed24e6c107874ea2a02f86fb11d2ad29e68d46013b3b379e191c7d861619b6b389917b557dea8481ad8d3a9df8b
+  metadata.gz: 76e7a70a6bffa609ce74dbb90783a7835da3876c34e75c19f711bad301d6b517b01e7bc7c2b6a614ec71c6297921cf36c49b01ecc8d5a072a2606cdb29ca99d0
+  data.tar.gz: 5ab873c9a336c285af66b758931628eaa604e05266454960c4a2974a6ddaa9ab0bcd9e87866c3b2cc8d47bafd0a913ab56fe06675949770667976eae0fcd67f5

data/app/controllers/barbeque/sns_subscriptions_controller.rb

@@ -0,0 +1,139 @@
+class Barbeque::SnsSubscriptionsController < Barbeque::ApplicationController
+  def index
+    @sns_subscriptions = Barbeque::SNSSubscription.all
+  end
+
+  def show
+    @sns_subscription = Barbeque::SNSSubscription.find(params[:id])
+  end
+
+  def new
+    @sns_topic_arns = fetch_sns_topic_arns
+    @sns_subscription = Barbeque::SNSSubscription.new
+  end
+
+  def edit
+    @sns_subscription = Barbeque::SNSSubscription.find(params[:id])
+  end
+
+  def create
+    @sns_subscription = Barbeque::SNSSubscription.new(params.require(:sns_subscription).permit(:topic_arn, :job_queue_id, :job_definition_id))
+
+    if @sns_subscription.save
+      update_sqs_policy!
+      subscribe_topic!
+      redirect_to @sns_subscription, notice: 'SNS subscription was successfully created.'
+    else
+      render :new
+    end
+  end
+
+  def update
+    @sns_subscription = Barbeque::SNSSubscription.find(params[:id])
+    if @sns_subscription.update(params.require(:sns_subscription).permit(:job_definition_id))
+      update_sqs_policy!
+      redirect_to @sns_subscription, notice: 'SNS subscription was successfully updated.'
+    else
+      render :edit
+    end
+  end
+
+  def destroy
+    @sns_subscription = Barbeque::SNSSubscription.find(params[:id])
+    @sns_subscription.destroy
+    update_sqs_policy!
+    unsubscribe_topic!
+    redirect_to sns_subscriptions_path, notice: 'SNS subscription was successfully destroyed.'
+  end
+  
+  private
+
+  def fetch_sns_topic_arns
+    sns_client.list_topics.topics.map(&:topic_arn)
+  end
+
+  def update_sqs_policy!
+    attrs = sqs_client.get_queue_attributes(
+      queue_url: @sns_subscription.job_queue.queue_url,
+      attribute_names: ['QueueArn'],
+    )
+    queue_arn = attrs.attributes['QueueArn']
+    topic_arns = @sns_subscription.job_queue.sns_subscriptions.map(&:topic_arn)
+
+    if topic_arns.present?
+      policy = generate_policy(queue_arn: queue_arn, topic_arns: topic_arns)
+    else
+      policy = '' # Be blank when there're no subscriptions.
+    end
+
+    sqs_client.set_queue_attributes(
+      queue_url: @sns_subscription.job_queue.queue_url,
+      attributes: { 'Policy' => policy },
+    )
+  end
+
+  # @paaram [String] queue_arn
+  # @param [Array<String>] topic_arns
+  # @return [String] JSON formatted policy
+  def generate_policy(queue_arn:, topic_arns:)
+    {
+      'Version' => '2012-10-17',
+      'Statement' => [
+        'Effect' => 'Allow',
+        'Principal' => '*',
+        'Action' => 'sqs:SendMessage',
+        'Resource' => queue_arn,
+        'Condition' => {
+          'ArnEquals' => {
+            'aws:SourceArn' => topic_arns,
+          }
+        }
+      ]
+    }.to_json
+  end
+
+  def subscribe_topic!
+    sns_client.subscribe(
+      topic_arn: @sns_subscription.topic_arn,
+      protocol: 'sqs',
+      endpoint: queue_arn
+    )
+  end
+
+  def unsubscribe_topic!
+    sqs_attrs = sqs_client.get_queue_attributes(
+      queue_url: @sns_subscription.job_queue.queue_url,
+      attribute_names: ['QueueArn'],
+    )
+    queue_arn = sqs_attrs.attributes['QueueArn']
+
+    subscriptions = sns_client.list_subscriptions_by_topic(
+      topic_arn: @sns_subscription.topic_arn,
+    )
+    subscription_arn = subscriptions.subscriptions.find {|subscription| subscription.endpoint == queue_arn }.try!(:subscription_arn)
+
+    if subscription_arn
+      sns_client.unsubscribe(
+        subscription_arn: subscription_arn,
+      )
+    end
+  end
+
+  def sqs_client
+    @sqs_client ||= Aws::SQS::Client.new
+  end
+
+  def sns_client
+    @sns_client ||= Aws::SNS::Client.new
+  end
+
+  def queue_arn
+    return @queue_arn if @queue_arn
+
+    sqs_attrs = sqs_client.get_queue_attributes(
+      queue_url: @sns_subscription.job_queue.queue_url,
+      attribute_names: ['QueueArn'],
+    )
+    @queue_arn = sqs_attrs.attributes['QueueArn']
+  end
+end

data/app/models/barbeque/job_queue.rb

@@ -2,6 +2,8 @@ class Barbeque::JobQueue < Barbeque::ApplicationRecord
   SQS_NAME_PREFIX = ENV['BARBEQUE_SQS_NAME_PREFIX'] || 'Barbeque-'
   SQS_NAME_MAX_LENGTH = 80
 
+  has_many :sns_subscriptions, class_name: 'SNSSubscription', dependent: :destroy
+
   # All SQS queues' "ReceiveMessageWaitTimeSeconds" are configured to be 20s (maximum).
   # This should be as large as possible to reduce API-calling cost by long polling.
   # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html#API_CreateQueue_RequestParameters

data/app/models/barbeque/sns_subscription.rb

@@ -0,0 +1,11 @@
+module Barbeque
+  class SNSSubscription < ApplicationRecord
+    belongs_to :job_queue
+    belongs_to :job_definition
+    has_one :app, through: :job_definition
+
+    validates :topic_arn,
+      uniqueness: { scope: :job_queue, message: 'should be set with only one queue' },
+      presence: true
+  end
+end

data/app/views/barbeque/sns_subscriptions/_form.html.haml

@@ -0,0 +1,39 @@
+.box.box-primary
+  .box-header
+    %h3.box-title.with_padding
+      #{action_name.capitalize} SNS Subscription
+
+  .box-body
+    = form_for @sns_subscription do |f|
+      - if @sns_subscription.errors.any?
+        %strong #{pluralize(@sns_subscription.errors.count, 'error')} prohibited this SNS subscription from being saved:
+        %ul
+          - @sns_subscription.errors.full_messages.each do |msg|
+            %li= msg
+
+      .row.form-group
+        .col-md-8
+          = f.label :topic_arn
+          - if @sns_subscription.persisted?
+            .sns_subscription_topic_arn= @sns_subscription.topic_arn
+          - else
+            = f.collection_select :topic_arn, @sns_topic_arns.map {|t| [t] }, :first, :first,
+              { prompt: true }, class: 'form-control'
+
+      .row.form-group
+        .col-md-4
+          = f.label :job_queue_id
+          - if @sns_subscription.persisted?
+            .sns_subscription_job_queue_id= link_to @sns_subscription.job_queue.name, @sns_subscription.job_queue
+          - else
+            = f.collection_select :job_queue_id, Barbeque::JobQueue.pluck(:id, :name), :first, :second,
+              { prompt: true }, class: 'form-control'
+
+      .row.form-group
+        .col-md-4
+          = f.label :job_definition_id
+          = f.collection_select :job_definition_id, Barbeque::JobDefinition.pluck(:id, :job), :first, :second,
+            { prompt: true }, class: 'form-control'
+
+      .form-group
+        = f.submit 'Save', class: 'btn btn-primary'

data/app/views/barbeque/sns_subscriptions/edit.html.haml

@@ -0,0 +1,3 @@
+= render 'form'
+
+= link_to 'Back', sns_subscription_path(@sns_subscription)

data/app/views/barbeque/sns_subscriptions/index.html.haml

@@ -0,0 +1,24 @@
+.box.box-primary
+  .box-header
+    %h3.box-title.with_padding
+      All SNS Subscriptions
+    = link_to new_sns_subscription_path, class: 'btn btn-primary pull-right' do
+      New SNS Subscriptions
+
+  .box-body
+    %table.table.table-bordered
+      %thead
+        %tr
+          %th topic_arn
+          %th Job Queue
+          %th Job Definition
+          %th
+
+      %tbody
+        - @sns_subscriptions.each do |sns_subscription|
+          %tr
+            %td= sns_subscription.topic_arn
+            %td= sns_subscription.job_queue.name
+            %td= sns_subscription.job_definition.job
+            %td
+              = link_to 'View Details', sns_subscription, class: 'btn btn-default btn-sm'

data/app/views/barbeque/sns_subscriptions/new.html.haml

@@ -0,0 +1,3 @@
+= render 'form'
+
+= link_to 'Back', sns_subscriptions_path

data/app/views/barbeque/sns_subscriptions/show.html.haml

@@ -0,0 +1,28 @@
+.row
+  .col-sm-7
+    .box.box-primary
+      .box-header
+        %h3.box-title.with_padding
+          SNS Subscription Details
+
+      .box-body
+        %p#notice= notice
+
+        %table.table.table-bordered
+          %tbody
+            %tr
+              %th topic_arn
+              %td= @sns_subscription.topic_arn
+            %tr
+              %th Job Queue
+              %td= link_to @sns_subscription.job_queue.name, @sns_subscription.job_queue
+            %tr
+              %th Job Defenition
+              %td= link_to @sns_subscription.job_definition.job, @sns_subscription.job_definition
+            %tr
+
+        = link_to 'Edit', edit_sns_subscription_path(@sns_subscription), class: 'btn btn-primary'
+        = link_to 'Destroy', sns_subscription_path(@sns_subscription),
+          class: 'btn', method: :delete, data: { confirm: 'Are you sure to delete SNS subscription?' }
+
+= link_to 'Back', sns_subscriptions_path

data/app/views/layouts/barbeque/_sidebar.html.haml

@@ -14,6 +14,10 @@
         = link_to job_queues_path do
           %i.fa.fa-cubes
           %span Queues
+      %li{ class: ('active' if controller_path == 'barbeque/sns_subscriptions') }
+        = link_to sns_subscriptions_path do
+          %i.fa.fa-calendar-plus-o
+          %span SNS Subscriptions
       %li{ class: ('active' if controller_path == 'barbeque/monitors') }
         = link_to monitors_path do
           %i.fa.fa-tachometer

data/app/views/layouts/barbeque/application.html.haml

@@ -6,7 +6,7 @@
     %title Barbeque
     %meta{ content: 'width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no', name: 'viewport' }
     -# FIXME: Don't rely on others' CDN
-    %link{ href: 'https://maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css', rel: 'stylesheet', type: 'text/css' }
+    %link{ href: 'https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css', rel: 'stylesheet', type: 'text/css' }
     %link{ href: 'https://code.ionicframework.com/ionicons/2.0.1/css/ionicons.min.css', rel: 'stylesheet', type: 'text/css' }
     = stylesheet_link_tag    'barbeque/application', media: 'all'
     = javascript_include_tag 'barbeque/application'

data/config/routes.rb

@@ -16,6 +16,8 @@ Barbeque::Engine.routes.draw do
 
   resources :job_queues
 
+  resources :sns_subscriptions
+
   resources :monitors, only: %i[index]
 
   scope :v1, module: 'api', as: :v1 do

data/db/migrate/20170420030157_create_barbeque_sns_subscriptions.rb

@@ -0,0 +1,11 @@
+class CreateBarbequeSnsSubscriptions < ActiveRecord::Migration[5.0]
+  def change
+    create_table :barbeque_sns_subscriptions do |t|
+      t.string :topic_arn, null: false
+      t.integer :job_queue_id, null: false
+      t.integer :job_definition_id, null: false
+
+      t.timestamps
+    end
+  end
+end

data/lib/barbeque/message.rb

@@ -3,6 +3,7 @@ require 'barbeque/message/base'
 require 'barbeque/message/invalid_message'
 require 'barbeque/message/job_execution'
 require 'barbeque/message/job_retry'
+require 'barbeque/message/notification'
 
 module Barbeque
   module Message

data/lib/barbeque/message/job_execution.rb

@@ -4,8 +4,8 @@ module Barbeque
   module Message
     class JobExecution < Base
       attr_reader :body        # [Object] free-format JSON
-      attr_reader :application # [String] To specify `job_definitions.application_id`
-      attr_reader :job         # [String] To specify `job_definitions.name`
+      attr_reader :application # [String] To specify `job_definitions.app.name`
+      attr_reader :job         # [String] To specify `job_definitions.job`
 
       private
 

data/lib/barbeque/message/notification.rb

@@ -0,0 +1,28 @@
+require 'barbeque/message/base'
+
+module Barbeque
+  module Message
+    class Notification < Base
+      attr_reader :body        # [Object] free-format JSON
+      attr_reader :topic_arn   # [String] To specify `subscription.topic_arn`
+      attr_reader :application # [String] To specify `job_definitions.app.name`
+      attr_reader :job         # [String] To specify `job_definitions.job`
+
+      # @param [Barneque::SNSSubscription] subscription
+      # @return [Barbeque::Message::Notification]
+      def set_params_from_subscription(subscription)
+        @application = subscription.app.name
+        @job = subscription.job_definition.job
+        self
+      end
+
+      private
+
+      def assign_body(message_body)
+        super
+        @topic_arn = message_body['TopicArn']
+        @body = message_body['Message']
+      end
+    end
+  end
+end

data/lib/barbeque/message_handler.rb

@@ -6,3 +6,4 @@ end
 
 require 'barbeque/message_handler/job_execution'
 require 'barbeque/message_handler/job_retry'
+require 'barbeque/message_handler/notification'

data/lib/barbeque/message_handler/notification.rb

@@ -0,0 +1,21 @@
+require 'barbeque/docker_image'
+require 'barbeque/execution_log'
+require 'barbeque/runner'
+require 'barbeque/slack_client'
+require 'barbeque/message_handler/job_execution'
+
+module Barbeque
+  module MessageHandler
+    class Notification < JobExecution
+      # @param [Barbeque::Message::Notification] message
+      # @param [Barbeque::JobQueue] job_queue
+      def initialize(message:, job_queue:)
+        @message = message
+        @job_queue = job_queue
+
+        subscription = SNSSubscription.find_by!(topic_arn: @message.topic_arn, job_queue_id: @job_queue.id)
+        @message.set_params_from_subscription(subscription)
+      end
+    end
+  end
+end

data/lib/barbeque/version.rb

@@ -1,3 +1,3 @@
 module Barbeque
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: barbeque
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-17 00:00:00.000000000 Z
+date: 2017-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: adminlte2-rails
@@ -289,6 +289,7 @@ files:
 - app/controllers/barbeque/job_queues_controller.rb
 - app/controllers/barbeque/job_retries_controller.rb
 - app/controllers/barbeque/monitors_controller.rb
+- app/controllers/barbeque/sns_subscriptions_controller.rb
 - app/helpers/barbeque/application_helper.rb
 - app/helpers/barbeque/job_definitions_helper.rb
 - app/helpers/barbeque/job_executions_helper.rb
@@ -305,6 +306,7 @@ files:
 - app/models/barbeque/job_queue.rb
 - app/models/barbeque/job_retry.rb
 - app/models/barbeque/slack_notification.rb
+- app/models/barbeque/sns_subscription.rb
 - app/services/barbeque/message_enqueuing_service.rb
 - app/services/barbeque/message_retrying_service.rb
 - app/views/barbeque/apps/_form.html.haml
@@ -327,6 +329,11 @@ files:
 - app/views/barbeque/job_queues/show.html.haml
 - app/views/barbeque/job_retries/show.html.haml
 - app/views/barbeque/monitors/index.html.haml
+- app/views/barbeque/sns_subscriptions/_form.html.haml
+- app/views/barbeque/sns_subscriptions/edit.html.haml
+- app/views/barbeque/sns_subscriptions/index.html.haml
+- app/views/barbeque/sns_subscriptions/new.html.haml
+- app/views/barbeque/sns_subscriptions/show.html.haml
 - app/views/layouts/barbeque/_header.html.haml
 - app/views/layouts/barbeque/_sidebar.html.haml
 - app/views/layouts/barbeque/application.html.haml
@@ -347,6 +354,7 @@ files:
 - db/migrate/20160509041452_add_job_queue_id_to_job_executions.rb
 - db/migrate/20160516041710_create_job_retries.rb
 - db/migrate/20160829023237_prefix_barbeque_to_tables.rb
+- db/migrate/20170420030157_create_barbeque_sns_subscriptions.rb
 - lib/barbeque.rb
 - lib/barbeque/config.rb
 - lib/barbeque/docker_image.rb
@@ -358,9 +366,11 @@ files:
 - lib/barbeque/message/invalid_message.rb
 - lib/barbeque/message/job_execution.rb
 - lib/barbeque/message/job_retry.rb
+- lib/barbeque/message/notification.rb
 - lib/barbeque/message_handler.rb
 - lib/barbeque/message_handler/job_execution.rb
 - lib/barbeque/message_handler/job_retry.rb
+- lib/barbeque/message_handler/notification.rb
 - lib/barbeque/message_queue.rb
 - lib/barbeque/runner.rb
 - lib/barbeque/runner/docker.rb
@@ -390,7 +400,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Job queue system to run job with Docker