capistrano_multiconfig_parallel 0.0.1

data/lib/capistrano_multiconfig_parallel/base.rb

@@ -0,0 +1,92 @@
+require 'rubygems'
+require 'bundler'
+require 'bundler/setup'
+require 'configurations'
+require_relative './configuration'
+
+# base module that has the statis methods that this gem is using
+module CapistranoMulticonfigParallel
+  include CapistranoMulticonfigParallel::Configuration
+
+  ENV_KEY_JOB_ID = 'multi_cap_job_id'
+  MULTI_KEY = 'multi'
+  SINGLE_KEY = 'single'
+
+  CUSTOM_COMMANDS = {
+    CapistranoMulticonfigParallel::MULTI_KEY => {
+      menu: 'show_menu',
+      stages: 'deploy_multi_stages'
+    },
+    CapistranoMulticonfigParallel::SINGLE_KEY => {
+      stages: 'deploy_multi_stages'
+    }
+  }
+
+  class << self
+    attr_accessor :show_task_progress, :interactive_menu, :execute_in_sequence, :logger, :show_task_progress_tree
+
+    def root
+      File.expand_path(File.dirname(__dir__))
+    end
+
+    def verify_app_dependencies(stages)
+      applications = stages.map { |stage| stage.split(':').reverse[1] }
+      wrong = CapistranoMulticonfigParallel.configuration.application_dependencies.find do |hash|
+        !applications.include?(hash[:app]) || (hash[:dependencies].present? && hash[:dependencies].find { |val| !applications.include?(val) })
+      end
+      raise ArgumentError, "invalid configuration for #{wrong.inspect}" if wrong.present?
+    end
+
+    def log_directory
+      File.join(CapistranoMulticonfigParallel.detect_root.to_s, 'log')
+    end
+
+    def main_log_file
+      File.join(log_directory, 'multi_cap.log')
+    end
+
+    def websokect_log_file
+      File.join(log_directory, 'multi_cap_webscoket.log')
+    end
+
+    def enable_logging
+      return unless CapistranoMulticonfigParallel::CelluloidManager.debug_enabled
+      FileUtils.mkdir_p(log_directory)
+      log_file = File.open(main_log_file, 'w')
+      log_file.sync = true
+      self.logger = ::Logger.new(main_log_file)
+      Celluloid.logger = logger
+    end
+
+    def log_message(message)
+      return unless logger.present?
+      error_message = message.respond_to?(:message) ? message.message : message.inspect
+      err_backtrace = message.respond_to?(:backtrace) ? message.backtrace.join("\n\n") : ''
+      if err_backtrace.present?
+        logger.debug(class: message.class,
+                     message: error_message,
+                     backtrace: err_backtrace)
+      else
+        logger.debug(message)
+      end
+    end
+
+    def detect_root
+      if ENV['MULTI_CAP_ROOT']
+        Pathname.new(ENV['MULTI_CAP_ROOT'])
+      elsif defined?(::Rails)
+        ::Rails.root
+      else
+        try_detect_capfile
+      end
+    end
+
+    def try_detect_capfile
+      root = Pathname.new(FileUtils.pwd)
+      root = root.parent unless root.directory?
+      root = root.parent until File.exist?(root.join('Capfile')) || root.root?
+      raise "Can't detect Rails application root" if root.root?
+      root
+    end
+  end
+end

data/lib/capistrano_multiconfig_parallel/celluloid/celluloid_manager.rb

@@ -0,0 +1,178 @@
+require_relative './celluloid_worker'
+require_relative './terminal_table'
+module CapistranoMulticonfigParallel
+  # rubocop:disable ClassLength
+  class CelluloidManager
+    include Celluloid
+    include Celluloid::Notifications
+    include Celluloid::Logger
+
+    cattr_accessor :debug_enabled
+
+    attr_accessor :jobs, :job_to_worker, :worker_to_job, :actor_system, :job_to_condition, :mutex, :registration_complete
+
+    attr_reader :worker_supervisor, :workers
+    trap_exit :worker_died
+
+    def initialize(job_manager)
+      # start SupervisionGroup
+      @worker_supervisor = Celluloid::SupervisionGroup.run!
+      @job_manager = job_manager
+      @registration_complete = false
+      # Get a handle on the SupervisionGroup::Member
+      @actor_system = Celluloid.boot
+      @mutex = Mutex.new
+      # http://rubydoc.info/gems/celluloid/Celluloid/SupervisionGroup/Member
+      @workers = @worker_supervisor.pool(CapistranoMulticonfigParallel::CelluloidWorker, as: :workers, size: 10)
+      # Get a handle on the PoolManager
+      # http://rubydoc.info/gems/celluloid/Celluloid/PoolManager
+      # @workers = workers_pool.actor
+      @conditions = []
+      @jobs = {}
+      @job_to_worker = {}
+      @worker_to_job = {}
+      @job_to_condition = {}
+
+      @worker_supervisor.supervise_as(:terminal_server, CapistranoMulticonfigParallel::TerminalTable, Actor.current)
+      @worker_supervisor.supervise_as(:web_server, CelluloidPubsub::WebServer, self.class.websocket_config.merge(enable_debug: self.class.debug_websocket?))
+    end
+
+    def self.debug_enabled?
+      debug_enabled
+    end
+
+    def self.debug_websocket?
+      websocket_config['enable_debug'].to_s == 'true'
+    end
+    
+    def self.websocket_config
+      config = CapistranoMulticonfigParallel.configuration.websocket_server
+      config.present? && config.is_a?(Hash) ? config.stringify_keys : {}
+    end
+    
+    def generate_job_id(job)
+      primary_key = @jobs.size + 1
+      job['id'] = primary_key
+      @jobs[primary_key] = job
+      @jobs[primary_key]
+      job['id']
+    end
+
+    # call to send an actor
+    # a job
+    def delegate(job)
+      job = job.stringify_keys
+      job['id'] = generate_job_id(job) if job['worker_action'] != 'worker_died'
+      @jobs[job['id']] = job
+      job['env_options'][CapistranoMulticonfigParallel::ENV_KEY_JOB_ID] = job['id']
+      # debug(@jobs)
+      # start work and send it to the background
+      @workers.async.work(job, Actor.current)
+    end
+
+    # call back from actor once it has received it's job
+    # actor should do this asap
+    def register_worker_for_job(job, worker)
+      job = job.stringify_keys
+      if job['id'].blank?
+        debug("job id not found. delegating again the job #{job.inspect}") if self.class.debug_enabled?
+        delegate(job)
+      else
+        worker.job_id = job['id'] if worker.job_id.blank?
+        @job_to_worker[job['id']] = worker
+        @worker_to_job[worker.mailbox.address] = job
+        debug("worker #{worker.job_id} registed into manager") if self.class.debug_enabled?
+        Actor.current.link worker
+        if @job_manager.jobs.size == @job_to_worker.size
+          @registration_complete = true
+        end
+      end
+    end
+
+    def process_jobs(&block)
+      @job_to_worker.pmap do |_job_id, worker|
+        worker.async.start_task
+      end
+      if block_given?
+        block.call
+      else
+        wait_task_confirmations
+      end
+      results2 = []
+      @job_to_condition.pmap do |_job_id, hash|
+        results2 << hash[:last_condition].wait
+      end
+      @job_manager.condition.signal(results2) if results2.size == @jobs.size
+    end
+
+    def wait_task_confirmations
+      return unless CapistranoMulticonfigParallel.configuration.task_confirmation_active
+      CapistranoMulticonfigParallel.configuration.task_confirmations.each_with_index do |task, index|
+        results = []
+        @jobs.pmap do |job_id, _job|
+          results << @job_to_condition[job_id][:first_condition][index].wait
+        end
+        if results.size == @jobs.size
+          confirm_task_approval(results, task)
+        end
+      end
+    end
+
+    def confirm_task_approval(results, task)
+      return unless results.present?
+      set :apps_symlink_confirmation, ask_confirm("Do you want  to continue the deployment and execute #{task}?", 'Y')
+      until fetch(:apps_symlink_confirmation).present?
+        sleep(0.1) # keep current thread alive
+      end
+      return if fetch(:apps_symlink_confirmation).blank? || fetch(:apps_symlink_confirmation).downcase != 'y'
+      @jobs.pmap do |job_id, job|
+        worker = get_worker_for_job(job_id)
+        worker.publish_rake_event('approved' => 'yes',
+                                  'action' => 'invoke',
+                                  'job_id' => job['id'],
+                                  'task' => task
+        )
+      end
+    end
+
+    def get_worker_for_job(job)
+      if job.present?
+        if job.is_a?(Hash)
+          job = job.stringify_keys
+          @job_to_worker[job['id']]
+        else
+          @job_to_worker[job.to_i]
+        end
+      else
+        return nil
+      end
+    end
+
+    # lookup status of job by asking actor running it
+    def get_job_status(job)
+      status = nil
+      if job.present?
+        if job.is_a?(Hash)
+          job = job.stringify_keys
+          actor = @registered_jobs[job['id']]
+          status = actor.status
+        else
+          actor = @registered_jobs[job.to_i]
+          status = actor.status
+        end
+      end
+      status
+    end
+
+    def worker_died(worker, reason)
+      debug("worker with mailbox #{worker.mailbox.inspect} died  for reason:  #{reason}") if self.class.debug_enabled?
+      job = @worker_to_job[worker.mailbox.address]
+      @worker_to_job.delete(worker.mailbox.address)
+      debug "restarting #{job} on new worker" if self.class.debug_enabled?
+      return if job.blank? || job['worker_action'] == 'worker_died'
+      return unless job['worker_action'] == 'deploy'
+      job = job.merge(:action => 'deploy:rollback', 'worker_action' => 'worker_died')
+      delegate(job)
+    end
+  end
+end

data/lib/capistrano_multiconfig_parallel/celluloid/celluloid_worker.rb

@@ -0,0 +1,238 @@
+require_relative './child_process'
+require_relative './state_machine'
+module CapistranoMulticonfigParallel
+  # rubocop:disable ClassLength
+  # worker that will spawn a child process in order to execute a capistrano job and monitor that process
+  #
+  # @!attribute job
+  #   @return [Hash] options used for executing capistrano task
+  #   @option options [String] :id The id of the job ( will ge automatically generated by CapistranoMulticonfigParallel::CelluloidManager when delegating job)
+  #   @option options [String] :app The application name that will be deployed
+  #   @option options [String] :env The stage used for that application
+  #   @option options [String] :action The action that this action will be doing (deploy, or other task)
+  #   @option options [Hash] :env_options  options that are available  in the environment variable ENV when this task is going to be executed
+  #   @option options [Array] :task_arguments arguments to the task
+  #
+  # @!attribute manager
+  #   @return [CapistranoMulticonfigParallel::CelluloidManager] the instance of the manager that delegated the job to this worker
+  #
+  class CelluloidWorker
+    include Celluloid
+    include Celluloid::Notifications
+    include Celluloid::Logger
+    class TaskFailed < StandardError; end
+
+    attr_accessor :job, :manager, :job_id, :app_name, :env_name, :action_name, :env_options, :machine, :client, :task_argv, :execute_deploy, :executed_dry_run,
+                  :rake_tasks, :current_task_number, # tracking tasks
+                  :successfull_subscription, :subscription_channel, :publisher_channel, # for subscriptions and publishing events
+                  :task_confirmations, :manager_condition, :last_manager_condition # for task conifirmations from manager
+
+    def work(job, manager)
+      @job = job
+      @manager = manager
+
+      process_job(job) if job.present?
+      debug("worker #{@job_id} received #{job.inspect}") if debug_enabled?
+      @subscription_channel = "worker_#{@job_id}"
+      @machine = CapistranoMulticonfigParallel::StateMachine.new(job, Actor.current)
+      setup_worker_condition
+      manager.register_worker_for_job(job, Actor.current)
+    end
+
+    def debug_enabled?
+      @manager.class.debug_enabled?
+    end
+    
+
+    def start_task
+      debug("exec worker #{@job_id} starts task with #{@job.inspect}") if debug_enabled?
+      @task_confirmations = CapistranoMulticonfigParallel.configuration.task_confirmations
+      @client = CelluloidPubsub::Client.connect(actor: Actor.current, enable_debug:  @manager.class.debug_websocket?) do |ws|
+        ws.subscribe(@subscription_channel)
+      end
+    end
+
+    def publish_rake_event(data)
+      @client.publish(rake_actor_id(data), data)
+    end
+
+    def rake_actor_id(data)
+      data['action'].present? && data['action'] == 'count' ? "rake_worker_#{@job_id}_count" : "rake_worker_#{@job_id}"
+    end
+
+    def on_message(message)
+      debug("worker #{@job_id} received:  #{message.inspect}") if debug_enabled?
+      if @client.succesfull_subscription?(message)
+        @successfull_subscription = true
+        execute_after_succesfull_subscription
+      else
+        handle_subscription(message)
+      end
+    end
+
+    def execute_after_succesfull_subscription
+      setup_task_arguments
+      if (@action_name == 'deploy' || @action_name == 'deploy:rollback') && CapistranoMulticonfigParallel.show_task_progress
+        @executed_dry_run = true
+        @rake_tasks = []
+        @task_argv << '--dry-run'
+        @task_argv << 'count_rake=true'
+        @child_process = CapistranoMulticonfigParallel::ChildProcess.new
+        Actor.current.link @child_process
+        debug("worker #{@job_id} executes: bundle exec multi_cap #{@task_argv.join(' ')}") if debug_enabled?
+        @child_process.async.work("bundle exec multi_cap #{@task_argv.join(' ')}", actor: Actor.current, dry_run: true)
+      else
+        async.execute_deploy
+      end
+    end
+
+    def rake_tasks
+      @rake_tasks ||= []
+    end
+
+    def execute_deploy
+      @execute_deploy = true
+      debug("invocation chain #{@job_id} is : #{@rake_tasks.inspect}") if debug_enabled? && CapistranoMulticonfigParallel.show_task_progress
+      if !defined?(@child_process) || @child_process.nil?
+        @child_process = CapistranoMulticonfigParallel::ChildProcess.new
+        Actor.current.link @child_process
+      else
+        @client.unsubscribe("rake_worker_#{@job_id}_count")
+        @child_process.exit_status = nil
+      end
+      setup_task_arguments
+      debug("worker #{@job_id} executes: bundle exec multi_cap #{@task_argv.join(' ')}") if debug_enabled?
+      @child_process.async.work("bundle exec multi_cap #{@task_argv.join(' ')}", actor: Actor.current, silent: true)
+    end
+
+    def on_close(code, reason)
+      debug("worker #{@job_id} websocket connection closed: #{code.inspect}, #{reason.inspect}") if debug_enabled?
+    end
+
+    def handle_subscription(message)
+      if message_is_about_a_task?(message)
+        save_tasks_to_be_executed(message)
+        update_machine_state(message['task']) # if message['action'] == 'invoke'
+        debug("worker #{@job_id} state is #{@machine.state}") if debug_enabled?
+        task_approval(message)
+      else
+        debug("worker #{@job_id} could not handle  #{message}") if debug_enabled?
+      end
+    end
+
+    def message_is_about_a_task?(message)
+      message.present? && message.is_a?(Hash) && message['action'].present? && message['job_id'].present? && message['task'].present?
+    end
+
+    def task_approval(message)
+      if @task_confirmations.include?(message['task']) && message['action'] == 'invoke'
+        @manager_condition[message['task']].call(message['task'])
+      else
+        publish_rake_event(message.merge('approved' => 'yes'))
+      end
+    end
+
+    def save_tasks_to_be_executed(message)
+      return unless message['action'] == 'count'
+      debug("worler #{@job_id} current invocation chain : #{@rake_tasks.inspect}") if debug_enabled?
+      @rake_tasks = [] if @rake_tasks.blank?
+      @rake_tasks << message['task'] if @rake_tasks.last != message['task']
+    end
+
+    def update_machine_state(name)
+      debug("worker #{@job_id} triest to transition from #{@machine.state} to  #{name}") if debug_enabled?
+      @machine.transitions.on(name.to_s, @machine.state => name.to_s)
+      @machine.go_to_transition(name.to_s)
+      raise(CapistranoMulticonfigParallel::CelluloidWorker::TaskFailed, "task #{@action} failed ") if name == 'deploy:failed' # force worker to rollback
+    end
+
+    def setup_command_line(*options)
+      @task_argv = []
+      options.each do |option|
+        @task_argv << option
+      end
+      @task_argv
+    end
+
+    def setup_task_arguments
+      #   stage = "#{@app_name}:#{@env_name} #{@action_name}"
+      stage = @app_name.present? ? "#{@app_name}:#{@env_name}" : "#{@env_name}"
+      array_options = ["#{stage}"]
+      array_options << "#{@action_name}[#{@task_arguments.join(',')}]"
+      @env_options.each do |key, value|
+        array_options << "#{key}=#{value}" if value.present?
+      end
+      array_options << '--trace' if debug_enabled?
+      setup_command_line(*array_options)
+    end
+
+    def send_msg(channel, message = nil)
+      publish channel, message.present? && message.is_a?(Hash) ? { job_id: @job_id }.merge(message) : { job_id: @job_id, time: Time.now }
+    end
+
+    def process_job(job)
+      @job_id = job['id']
+      @app_name = job['app']
+      @env_name = job['env']
+      @action_name = job['action']
+      @env_options = {}
+      job['env_options'].each do |key, value|
+        @env_options[key] = value if value.present?
+      end
+      @task_arguments = job['task_arguments']
+    end
+
+    def need_confirmation_for_tasks?
+      executes_deploy? && CapistranoMulticonfigParallel.configuration.task_confirmation_active
+    end
+
+    def executes_deploy?
+      (@action_name == 'deploy' || @action_name == 'deploy:rollback')
+    end
+
+    def setup_worker_condition
+      job_termination_condition = Celluloid::Condition.new
+      job_confirmation_conditions = []
+      CapistranoMulticonfigParallel.configuration.task_confirmations.each do |_task|
+        if need_confirmation_for_tasks?
+          job_confirmation_conditions << Celluloid::Condition.new
+        else
+          job_confirmation_conditions << proc { |sum| sum }
+        end
+      end
+      @manager.job_to_condition[@job_id] = { first_condition: job_confirmation_conditions, last_condition: job_termination_condition }
+      construct_blocks_for_conditions(job_confirmation_conditions, job_termination_condition)
+    end
+
+    def construct_blocks_for_conditions(job_confirmation_conditions, job_termination_condition)
+      hash_conditions = {}
+      CapistranoMulticonfigParallel.configuration.task_confirmations.each_with_index do |task, index|
+        blk = lambda do |sum|
+          need_confirmation_for_tasks? ? job_confirmation_conditions[index].signal(sum) : job_confirmation_conditions[index].call(sum)
+        end
+        hash_conditions[task] = blk
+      end
+      blk_termination = lambda do |sum|
+        job_termination_condition.signal(sum)
+      end
+      @manager_condition = hash_conditions
+      @last_manager_condition = blk_termination
+    end
+
+    def crashed?
+      @action_name == 'deploy:rollback'
+    end
+
+    def notify_finished(exit_status)
+      return unless @execute_deploy
+      if exit_status.exitstatus != 0
+        debug("worker #{job_id} tries to terminate")
+        terminate
+      else
+        update_machine_state('FINISHED')
+        debug("worker #{job_id} notifies manager has finished")
+        @last_manager_condition.call('yes')
+      end
+    end
+  end
+end

data/lib/capistrano_multiconfig_parallel/celluloid/child_process.rb

@@ -0,0 +1,104 @@
+module CapistranoMulticonfigParallel
+  # class that is used to execute the capistrano tasks and it is invoked by the celluloid worker
+  class ChildProcess
+    include Celluloid
+    include Celluloid::Logger
+
+    attr_accessor :actor, :pid, :exit_status, :process, :filename
+
+    def finalize
+      EM.stop
+      @timer.cancel
+      super
+      true
+    end
+
+    alias_method :terminate, :finalize
+
+    def work(cmd, options = {})
+      @options = options
+      @actor = @options.fetch(:actor, nil)
+      EM.run do
+        EM.next_tick do
+          set_worker_log
+          start_async_deploy(cmd, options)
+        end
+        @timer = EM::PeriodicTimer.new(0.1) do
+          check_exit_status
+        end
+      end
+      EM.error_handler do|e|
+        puts "Error during event loop for worker #{@actor.job_id}: #{e.inspect}" if @actor.debug_enabled?
+        puts e.backtrace if @actor.debug_enabled?
+        EM.stop
+      end
+    end
+
+    def set_worker_log
+      FileUtils.mkdir_p(CapistranoMulticonfigParallel.log_directory)
+      @filename = File.join(CapistranoMulticonfigParallel.log_directory, "worker_#{@actor.job_id}.log")
+      FileUtils.rm_rf(@filename) if @options[:dry_run] || @actor.executed_dry_run != true
+    end
+
+    def check_exit_status
+      return unless @exit_status.present?
+      @timer.cancel
+      EM.stop
+      if @options[:dry_run]
+        debug("worker #{@actor.job_id} starts execute deploy") if @actor.debug_enabled?
+        @actor.async.execute_deploy
+      else
+        debug("worker #{@actor.job_id} startsnotify finished") if @actor.debug_enabled?
+        @actor.notify_finished(@exit_status)
+      end
+    end
+
+    def start_async_deploy(cmd, options)
+      RightScale::RightPopen.popen3_async(
+        cmd,
+        target: self,
+        environment: options[:environment].present? ? options[:environment] : nil,
+        pid_handler: :on_pid,
+        stdout_handler: :on_read_stdout,
+        stderr_handler: :on_read_stderr,
+        watch_handler: :watch_handler,
+        async_exception_handler: :async_exception_handler,
+        exit_handler: :on_exit)
+    end
+
+    def on_pid(pid)
+      @pid ||= pid
+    end
+
+    def on_input_stdin(data)
+      io_callback('stdin', data)
+    end
+
+    def on_read_stdout(data)
+      io_callback('stdout', data)
+    end
+
+    def on_read_stderr(data)
+      io_callback('stderr', data)
+    end
+
+    def on_exit(status)
+      debug "Child process for worker #{@actor.job_id} on_exit  disconnected due to error #{status.inspect}" if @actor.debug_enabled?
+      @exit_status = status
+    end
+
+    def async_exception_handler(*data)
+      debug "Child process for worker #{@actor.job_id} async_exception_handler  disconnected due to error #{data.inspect}" if @actor.debug_enabled?
+      io_callback('stderr', data)
+      @exit_status = 1
+    end
+
+    def watch_handler(process)
+      @process ||= process
+    end
+
+    def io_callback(_io, data)
+      File.open(@filename, 'a') { |f| f << data }
+    end
+  end
+end

data/lib/capistrano_multiconfig_parallel/celluloid/rake_worker.rb

@@ -0,0 +1,83 @@
+module CapistranoMulticonfigParallel
+  # class that handles the rake task and waits for approval from the celluloid worker
+  class RakeWorker
+    include Celluloid
+    include Celluloid::Logger
+
+    attr_accessor :env, :client, :job_id, :action, :task, :task_approved, :successfull_subscription, :subscription_channel, :publisher_channel
+
+    def work(env, task, options = {})
+      @options = options.stringify_keys
+      @env = env
+      @job_id = find_job_id
+      @subscription_channel = @options['rake_actor_id']
+      @publisher_channel = "worker_#{find_job_id}"
+      @action = @options['rake_actor_id'].include?('_count') ? 'count' : 'invoke'
+      @task = task
+      @task_approved = false
+      @successfull_subscription = false
+      @client = CelluloidPubsub::Client.connect(actor: Actor.current, enable_debug: CapistranoMulticonfigParallel::CelluloidManager.debug_websocket?) do |ws|
+        ws.subscribe(@subscription_channel)
+      end if !defined?(@client) || @client.nil?
+    end
+
+    def debug_enabled?
+      @client.debug_enabled?
+    end
+
+    def task_name
+      @task.name
+    end
+
+    def find_job_id
+      @env[CapistranoMulticonfigParallel::ENV_KEY_JOB_ID]
+    end
+
+    def task_data
+      {
+        action: @action,
+        task: task_name,
+        job_id: find_job_id
+      }
+    end
+
+    def publish_new_work(env, task)
+      work(env, task, rake_actor_id: @options['rake_actor_id'])
+      publish_to_worker(task_data)
+    end
+
+    def publish_to_worker(data)
+      @client.publish(@publisher_channel, data)
+    end
+
+    def on_message(message)
+      debug("Rake worker #{find_job_id} received after parse #{message}") if debug_enabled?
+      if @client.succesfull_subscription?(message)
+        publish_subscription_successfull
+      elsif message.present? && message['client_action'].blank?
+        task_approval(message)
+      else
+        warn "unknown action: #{message.inspect}" if debug_enabled?
+      end
+    end
+
+    def publish_subscription_successfull
+      debug("Rake worker #{find_job_id} received  parse #{message}") if debug_enabled?
+      publish_to_worker(task_data)
+      @successfull_subscription = true
+    end
+
+    def task_approval(message)
+      if @job_id.to_i == message['job_id'].to_i && message['task'] == task_name && message['approved'] == 'yes'
+        @task_approved = true
+      else
+        warn "unknown invocation #{message.inspect}" if debug_enabled?
+      end
+    end
+
+    def on_close(code, reason)
+      debug("websocket connection closed: #{code.inspect}, #{reason.inspect}") if debug_enabled?
+      terminate
+    end
+  end
+end