robot-controller 1.0.2 → 2.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b58cc1ed1c3e12f06ef0e412441871239e9f20f
-  data.tar.gz: 09218f41c6e86bbb3249ddcd8b6ea4025a578f5e
+  metadata.gz: f2f8958d56c26a4a2139a77a4c17a496ea96fe47
+  data.tar.gz: 03a8f5d1f0390f46772fc9e18d80fa8223e964b7
 SHA512:
-  metadata.gz: 0b60e86072316bd452ca4ec7954b9febae5a7c8bbe6edf7a00147185b1a66e35be955becb949afac32e917f4803cabf8c1ada649a7ae1cf4cdbcc328fbdbc077
-  data.tar.gz: d17f2760ceda033d37dc3c975aec84c1e2dabf8c0f7edce7fe35c74cbd9d96537efa9bbbf82ef84d5cc2394bcf601a7dea222fd73d05b95871d988e012cd9d5e
+  metadata.gz: bd0432d25bf71cab0e6bfa18a5dd3e67210d9d27772b0f86e4f334c34d114d2aff590184192293d93656ae130db9a2b8b7b2e6d1b8f6ffdba97bb84f049d9284
+  data.tar.gz: 5da783211f9b8c77138d1cdb0d0bfbac1c29818d4f2caeda54981032f4701baca9197286d91eec276e7a7395ee94095b442b29cbfcc1d931d2721a72c21a4fda

data/.gitignore

@@ -37,5 +37,3 @@ build/
 .rvmrc
 .ruby-gemset
 .ruby-version
-
-config/environments

data/.rubocop.yml

@@ -0,0 +1 @@
+inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -0,0 +1,48 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2015-06-23 11:24:30 -0700 using RuboCop version 0.31.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Metrics/AbcSize:
+  Max: 37
+
+# Offense count: 1
+Metrics/CyclomaticComplexity:
+  Max: 9
+
+# Offense count: 17
+# Configuration parameters: AllowURI, URISchemes.
+Metrics/LineLength:
+  Max: 130
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 22
+
+# Offense count: 1
+Metrics/PerceivedComplexity:
+  Max: 9
+
+# Offense count: 1
+# Configuration parameters: Exclude.
+Style/FileName:
+  Enabled: false
+
+# Offense count: 1
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/FormatString:
+  Enabled: false
+
+# Offense count: 5
+# Cop supports --auto-correct.
+Style/NumericLiterals:
+  MinDigits: 6
+
+# Offense count: 1
+# Configuration parameters: Methods.
+Style/SingleLineBlockParams:
+  Enabled: false

data/README.md

@@ -30,12 +30,14 @@ controller, then add:
 
     Usage: controller ( boot | quit )
            controller ( start | status | stop | restart | log ) [worker]
+           controller verify [--verbose]
            controller [--help]
 
     Example:
       % controller boot    # start bluepilld and jobs
       % controller status  # check on status of jobs
-      % controller log 1_dor_accessionWF_descriptive-metadata # view log for worker
+      % controller verify  # verify robots are running as configured
+      % controller log 1_dor_accessionWF_descriptive-metadata # view log for worker 
       % controller stop    # stop jobs
       % controller quit    # stop bluepilld
   
@@ -48,3 +50,44 @@ controller, then add:
 
 * `v1.0.0`: Initial version
 * `v1.0.1`: Add 'rake' as dependency
+
+### `verify` command
+
+You can run the `verify` command with an optional `--verbose` to print out
+details about whether the robots processes are running as configured.
+
+When no errors are detected, the output looks like so:
+
+    % bundle exec controller verify
+    OK
+
+    % bundle exec controller verify --verbose
+    OK robot1 is up
+    OK robot2 is up
+    OK robot3 is not enabled
+    OK robot4 is not enabled
+
+If `robot2` were down and `robot3` were up, the output would look like so:
+
+    % bundle exec controller verify
+    ERROR robot2 is down (0 out of 3 processes running)
+    ERROR robot3 is not enabled but 1 process is running
+
+    % bundle exec controller verify --verbose
+    OK robot1 is up
+    ERROR robot2 is down (0 out of 3 processes running)
+    ERROR robot3 is not enabled but 1 process is running
+    OK robot4 is not enabled
+
+The various states are determined as follows:
+
+- If the robot is enabled:
+  - `OK`: all N processes are running
+  - `ERROR`: not all N processes are running
+- If the robot is NOT enabled:
+  - `OK`: no processes are running
+  - `ERROR`: 1 or more processes are running
+- If the robot is unknown by the suite:
+  - `ERROR`: always
+
+NOTE: The queues on which the robots are running are NOT verified.

data/Rakefile

@@ -7,7 +7,7 @@ Dir.glob('lib/tasks/*.rake').each { |r| import r }
 
 require 'rspec/core/rake_task'
 
-desc "Run specs"
+desc 'Run specs'
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => [ :yard, :spec ]
+task default: [:yard, :spec]

data/VERSION

@@ -1 +1 @@
-1.0.2
+2.0.beta1

data/bin/controller

@@ -1,15 +1,19 @@
 #!/usr/bin/env ruby
+require 'yaml'
+require 'robot-controller'
 
 if ARGV.size == 0
   puts '
 Usage: controller ( boot | quit )
        controller ( start | status | stop | restart | log ) [worker]
+       controller verify [--verbose]
        controller [--help]
 
 Example:
   % controller boot    # start bluepilld and jobs
   % controller status  # check on status of jobs
   % controller log dor_accessionWF_descriptive-metadata # view log for worker
+  % controller verify  # verify robots are running as configured
   % controller stop    # stop jobs
   % controller quit    # stop bluepilld
 
@@ -18,37 +22,85 @@ Environment:
   BLUEPILL_LOGFILE - output log (default: log/bluepill.log)
   ROBOT_ENVIRONMENT - (default: development)
 '
-  exit -1
+  exit(-1)
 end
 
 ENV['ROBOT_ENVIRONMENT'] ||= 'development'
 ENV['BLUEPILL_BASE_DIR'] ||= File.expand_path('run/bluepill')
-ENV['BLUEPILL_LOGFILE']  ||= File.expand_path('log/bluepill.log')
+ENV['BLUEPILL_LOGFILE'] ||= File.expand_path('log/bluepill.log')
 
-unless File.directory?('config') &&
-       File.directory?(File.dirname(ENV['BLUEPILL_BASE_DIR'])) &&
-       File.directory?(File.dirname(ENV['BLUEPILL_LOGFILE']))
-  raise "Run from root directory"
-end
+fail 'bluepill requires config directory' unless File.directory?('config')
+fail 'bluepill requires run directory' unless File.directory?(File.dirname(ENV['BLUEPILL_BASE_DIR']))
+fail 'bluepill requires log directory' unless File.directory?(File.dirname(ENV['BLUEPILL_LOGFILE']))
 
 cmd = 'bluepill'
 cmd << ' --no-privileged'
 cmd << " --base-dir #{ENV['BLUEPILL_BASE_DIR']}"
 cmd << " --logfile #{ENV['BLUEPILL_LOGFILE']}"
 
-if ARGV[0] == 'boot'
+case ARGV[0].downcase
+when 'boot'
   fn = 'config/bluepill.rb' # allow override
-  unless File.file?(fn)
-    require 'robot-controller'
-    fn = RobotController.bluepill_config
-  end
+  fn = RobotController.bluepill_config unless File.file?(fn)
   if File.file?(fn)
-    puts "Loading #{fn}"
+    # puts "Loading #{fn}"
     exec "#{cmd} load #{fn}"
     # NOTREACHED
   end
   puts "ERROR: Cannot find bluepill configuration file for #{ENV['ROBOT_ENVIRONMENT']}"
-  exit -1
+  exit(-1)
+when 'verify'
+  verbose = (ARGV[1] == '--verbose')
+
+  # load list of all possible robots
+  robots = YAML.load_file('config/robots.yml')
+  fail ArgumentError unless robots.is_a? Array
+
+  # determine how many processes should be running for each robot
+  running = {}
+  robots.each do |robot|
+    running[robot] = 0
+  end
+  RobotController::Parser.load("robots_#{ENV['ROBOT_ENVIRONMENT']}.yml").each do |h|
+    running[h[:robot]] = h[:n]
+  end
+
+  # verify that all robots running are known to the config/robots.yml
+  running.each_key do |robot|
+    if !robots.include?(robot)
+      puts "ERROR: '#{robot}' robot is unknown to the suite. Check config/robots.yml" 
+      running.delete(robot)
+    end
+  end
+
+  # verify suite
+  verify = RobotController::Verify.new(running)
+  begin
+    statuses = verify.verify
+  rescue => e
+    puts "ERROR: Cannot run verification for any robots. #{e.class}: #{e.message}" 
+    exit(-1)
+  end
+
+  # print output for each status
+  ok = true
+  puts 'ERROR: No robots to verify?' if statuses.size == 0
+  statuses.each_pair do |robot, status|
+    case status[:state]
+    when :up
+      puts "OK: #{robot} is up (#{status[:running]} running)" if verbose
+    when :not_enabled
+      puts "OK: #{robot} is not enabled (#{status[:running]} running)" if verbose
+    when :down
+      ok = false
+      puts "ERROR: #{robot} is down (#{status[:running]} of #{running[robot]} running)"
+    else
+      ok = false
+      puts "ERROR: #{robot} cannot be verified (state=#{status[:state]})"
+    end
+  end
+  puts 'OK' if ok && !verbose && statuses.size > 0
+  exit(0)
 else
   exec "#{cmd} #{ARGV.join(' ')}"
   # NOTREACHED

data/config/environments/robots_development.yml

@@ -0,0 +1,3 @@
+'*':
+  - dor_sampleWF_sample-robot
+  - dor_sampleWF_another-sample-robot

data/config/robots.yml

@@ -0,0 +1,3 @@
+- dor_sampleWF_sample-robot
+- dor_sampleWF_another-sample-robot
+- dor_sampleWF_yet-another-sample-robot

data/example/lib/tasks/environment.rake

@@ -1,4 +1,4 @@
-desc "Load environment from boot file"
+desc 'Load environment from boot file'
 task :environment do
   # needs to load the boot file
   require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'config', 'boot'))

data/lib/robot-controller.rb

@@ -1,5 +1,4 @@
 # Monitors and controls running workflow robots off of priority queues and within a cluster.
-
 module RobotController
   # e.g., `1.2.3`
   VERSION = File.read(File.join(File.dirname(__FILE__), '..', 'VERSION')).strip
@@ -8,4 +7,6 @@ module RobotController
     File.join(File.dirname(__FILE__), 'robot-controller', 'bluepill.rb')
   end
 
-end
+  autoload :Verify, 'robot-controller/verify'
+  autoload :Parser, 'robot-controller/robots'
+end

data/lib/robot-controller/bluepill.rb

@@ -1,8 +1,10 @@
+require 'robot-controller'
+
+# current directory
 WORKDIR = Dir.pwd
 
-robot_environment = ENV['ROBOT_ENVIRONMENT'] || 'development'
-require 'robot-controller/robots'
-ROBOTS = RobotConfigParser.new.load("robots_#{robot_environment}.yml")
+# setup robots configuration
+ROBOTS = RobotController::Parser.new.load("robots_#{ENV['ROBOT_ENVIRONMENT'] || 'development'}.yml")
 #
 # Expect ROBOTS = [
 #  {:robot => 'x', :queues => ['a', 'b'], :n => 1}
@@ -12,12 +14,12 @@ ROBOTS = RobotConfigParser.new.load("robots_#{robot_environment}.yml")
 
 # set application name to parent directory name
 Bluepill.application File.basename(File.dirname(File.dirname(WORKDIR))),
-  :log_file => "#{WORKDIR}/log/bluepill.log" do |app|
+                     log_file: "#{WORKDIR}/log/bluepill.log" do |app|
   app.working_dir = WORKDIR
-  ROBOTS.each_index do |i|    
+  ROBOTS.each_index do |i|
     ROBOTS[i][:n].to_i.times do |j|
       # prefix process name with index number to prevent duplicate process names
-      prefix = sprintf("robot%02d_%02d", i+1, j+1)
+      prefix = sprintf('robot%02d_%02d', i + 1, j + 1)
       app.process("#{prefix}_#{ROBOTS[i][:robot]}") do |process|
         puts "Creating robot #{process.name}"
 
@@ -26,19 +28,21 @@ Bluepill.application File.basename(File.dirname(File.dirname(WORKDIR))),
 
         # use environment for these resque variables
         process.environment = {
+          'TERM_CHILD' => '1', # TERM, KILL, USR1 sent to worker process if running
+          'RESQUE_TERM_TIMEOUT' => '10.0', # seconds to wait before sending KILL after TERM
           'QUEUES' => queues,
           'ROBOT_ENVIRONMENT' => robot_environment,
           'INTERVAL' => '5'
         }
         process.environment['VERBOSE'] = 'yes' if ENV['ROBOT_VERBOSE'] == 'yes'
         process.environment['VVERBOSE'] = 'yes' if ENV['ROBOT_VVERBOSE'] == 'yes'
-        
+
         # process configuration
         process.group = robot_environment
         process.stdout = process.stderr = "#{WORKDIR}/log/#{ROBOTS[i][:robot]}.log"
 
         # spawn worker processes using robot-controller
-        process.start_command = "rake environment resque:work"
+        process.start_command = 'rake environment resque:work'
 
         # we use bluepill to daemonize the resque workers rather than using
         # resque's BACKGROUND flag

data/lib/robot-controller/robots.rb

@@ -1,96 +1,93 @@
 require 'yaml'
 
-class RobotConfigParser
-  ROBOT_INSTANCE_MAX = 16
+#
+module RobotController
+  #
+  class Parser
+    ROBOT_INSTANCE_MAX = 16
 
-  # parse_instances(1) == 1
-  # parse_instances(16) == 16
-  # parse_instances(0) == 1
-  # parse_instances(99) => RuntimeError
-  def parse_instances(n)
-    if n > ROBOT_INSTANCE_MAX
-      raise RuntimeError, "TooManyInstances: #{n} > #{ROBOT_INSTANCE_MAX}"
-    end
-    n = 1 if n < 1
-    n
-  end
+    class << self
+      # main entry point
+      def load(robots_fn, dir = 'config/environments', host = nil)
+        # Validate parameters
+        robots_fn = File.join(dir, robots_fn) if dir
+        fail "FileNotFound: #{robots_fn}" unless File.file?(robots_fn)
 
-  # parse_lanes('') == ['default']
-  # parse_lanes(' ') == ['default']
-  # parse_lanes(' , ') == ['default']
-  # parse_lanes(' , ,') == ['default']
-  # parse_lanes('*') == ['*']
-  # parse_lanes('1') == ['1']
-  # parse_lanes('A') == ['A']
-  # parse_lanes('A , B') == ['A', 'B']
-  # parse_lanes('A,B,C') == ['A','B','C']
-  # parse_lanes('A-C,E') == ['A-C', 'E']
-  def parse_lanes(lanes_spec)
-    return ['default'] if lanes_spec.split(/,/).collect {|l| l.strip}.join('') == ''
-    lanes_spec.split(/,/).collect {|l| l.strip }.uniq
-  end
+        # read the YAML file
+        # puts "Loading #{robots_fn}"
+        robots =  YAML.load_file(robots_fn)
+        # puts robots
 
-  # build_queues('z','A') => ['z_A']
-  # build_queues('z','A,C') => ['z_A', 'z_C']
-  def build_queues(robot, lanes)
-    queues = []
-    parse_lanes(lanes).each do |i|
-      queues << [robot, i].join('_')
-    end
-    queues
-  end
+        # determine current host
+        host = `hostname -s`.strip unless host
+        # puts host
 
-  # main entry point
-  def load(robots_fn, dir = 'config/environments', host = nil)
-    # Validate parameters
-    robots_fn = File.join(dir, robots_fn) if dir
-    unless File.file?(robots_fn)
-      raise RuntimeError, "FileNotFound: #{robots_fn}"
-    end
-    
-    # read the YAML file
-    puts "Loading #{robots_fn}"
-    robots =  YAML.load_file(robots_fn)
-    # puts robots
-  
-    # determine current host
-    host = `hostname -s`.strip unless host
-    # puts host
+        # host = 'sul-robots1-dev' # XXX
+        fail "HostMismatch: #{host} not defined in #{robots_fn}" unless robots.include?(host) || robots.include?('*')
+        host = '*' unless robots.include?(host)
 
-    # host = 'sul-robots1-dev' # XXX
-    unless robots.include?(host)
-      raise RuntimeError, "HostMismatch: #{host} not defined in #{robots_fn}"
-    end
+        parse_yaml(robots[host])
+      end
 
-    # parse YAML lines for host where i is robot[:lane[:instances]]
-    r = []
-    robots[host].each do |i|  
-      robot = i.split(/:/).collect {|j| j.strip}
-      robot.each do |j|
-        if j.strip == ''
-          raise RuntimeError, "SyntaxError: #{i}"
-        end
+      # parse_instances(1) == 1
+      # parse_instances(16) == 16
+      # parse_instances(0) == 1
+      # parse_instances(99) => RuntimeError
+      def parse_instances(n)
+        fail "TooManyInstances: #{n} > #{ROBOT_INSTANCE_MAX}" if n > ROBOT_INSTANCE_MAX
+        n = 1 if n < 1
+        n
       end
-    
-      # add defaults
-      if robot.size == 1
-        robot << 'default'
+
+      # parse_lanes('') == ['default']
+      # parse_lanes(' ') == ['default']
+      # parse_lanes(' , ') == ['default']
+      # parse_lanes(' , ,') == ['default']
+      # parse_lanes('*') == ['*']
+      # parse_lanes('1') == ['1']
+      # parse_lanes('A') == ['A']
+      # parse_lanes('A , B') == ['A', 'B']
+      # parse_lanes('A,B,C') == ['A','B','C']
+      # parse_lanes('A-C,E') == ['A-C', 'E']
+      def parse_lanes(lanes_spec)
+        return ['default'] if lanes_spec.split(/,/).collect(&:strip).join('') == ''
+        lanes_spec.split(/,/).collect(&:strip).uniq
       end
-      if robot.size == 2
-        robot << '1'
+
+      # build_queues('z','A') => ['z_A']
+      # build_queues('z','A,C') => ['z_A', 'z_C']
+      def build_queues(robot, lanes)
+        queues = []
+        parse_lanes(lanes).each do |i|
+          queues << [robot, i].join('_')
+        end
+        queues
       end
-    
-      # build queues for robot instances
-      unless robot.size == 3
-        raise RuntimeError, "SyntaxError: #{i}"
+
+      def parse_yaml(robots)
+        # parse YAML lines for host where i is robot[:lane[:instances]]
+        r = []
+        robots.each do |i|
+          robot = i.split(/:/).collect(&:strip)
+          robot.each do |j|
+            fail "SyntaxError: #{i}" if j.strip == ''
+          end
+
+          # add defaults
+          robot << 'default' if robot.size == 1
+          robot << '1' if robot.size == 2
+
+          # build queues for robot instances
+          fail "SyntaxError: #{i}" unless robot.size == 3
+          robot[2] = parse_instances(robot[2].to_i)
+          # puts robot.join(' : ')
+          queues = build_queues(robot[0], robot[1])
+          # puts queues
+
+          r << { robot: robot[0], queues: queues, n: robot[2] }
+        end
+        r
       end
-      robot[2] = parse_instances(robot[2].to_i)
-      # puts robot.join(' : ')
-      queues = build_queues(robot[0], robot[1])
-      # puts queues
-  
-      r << {:robot => robot[0], :queues => queues, :n => robot[2] }
     end
-    r
   end
 end

data/lib/robot-controller/tasks.rb

@@ -1 +1 @@
-require 'resque/tasks'
+require 'resque/tasks'

data/lib/robot-controller/verify.rb

@@ -0,0 +1,209 @@
+# verification class
+module RobotController
+  # Usage:
+  #   RobotController::Verify.new('robot1' => 1, 'robot2' => 2, 'robot3' => 0)
+  #   => {
+  #     'robot1': { state: :up, running: 1 },
+  #     'robot2': { state: :down, running: 0 },
+  #     'robot3': { state: :not_enabled, running: 0 }
+  #   }
+  # ----
+  #
+  # When no errors are detected, the output looks like so:
+  #   % bundle exec controller verify
+  #   OK
+  #
+  #   % bundle exec controller verify --verbose
+  #   OK robot1 is up
+  #   OK robot2 is up
+  #   OK robot3 is not enabled
+  #   OK robot4 is not enabled
+  #
+  # If robot2 were down and robot3 were up, the output would look like so:
+  #
+  #   % bundle exec controller verify
+  #   ERROR robot2 is down (0 out of 3 processes running)
+  #   ERROR robot3 is not enabled but 1 process is running
+  #
+  #   % bundle exec controller verify --verbose
+  #   OK robot1 is up
+  #   ERROR robot2 is down (0 out of 3 processes running)
+  #   ERROR robot3 is not enabled but 1 process is running
+  #   OK robot4 is not enabled
+  #
+  # The various states are determined as follows:
+  #
+  #   If the robot is enabled:
+  #     OK: all N processes are running
+  #     ERROR: not all N processes are running
+  #   If the robot is NOT enabled:
+  #     OK: no processes are running
+  #     ERROR: 1 or more processes are running
+  #   If the robot is unknown by the suite:
+  #     ERROR: always
+  class Verify
+    attr_reader :robots
+
+    # @param [Hash] nprocesses expected number of processes for all robots
+    def initialize(nprocesses)
+      fail ArgumentError if nprocesses.nil? || !nprocesses.is_a?(Hash)
+      fail ArgumentError, 'Empty argument' if nprocesses.size == 0
+      @running = nprocesses
+      @robots = @running.each_key.to_a
+      @status = nil
+    end
+
+    # @param [Boolean] reload forces a reload of status information
+    # @return [Hash<Hash>] status of all robots
+    # {
+    #   'robot1' : {
+    #     state: :up,
+    #     running: 2
+    #   },
+    #   'robot2' : {
+    #     state: :down,
+    #     running: 0
+    #   },
+    #   'robot3' : {
+    #     state: :not_enabled,
+    #     running: 0
+    #   }
+    # }
+    def verify(reload = true)
+      @status = nil if reload
+      r = {}
+      robots.each do |robot|
+        r[robot] = robot_status(robot)
+      end
+      r
+    end
+
+    # @param [String] robot name
+    # @return [Integer] number of running processes expected otherwise nil
+    def running(robot)
+      @running[robot]
+    end
+
+    protected
+
+    # @param [String] robot name
+    # @return [Hash] { state: :up | :down | :not_enabled, running: n }
+    def robot_status(robot)
+      if status[robot].nil?
+        fail "ERROR: No status information for #{robot}"
+      else
+        status[robot]
+      end
+    end
+
+    #
+    # @return [Hash] status
+    # {
+    #   'robot1': {
+    #     state: :up,
+    #     running: 1
+    #   },
+    #   'robot2': ...,
+    #   'robot3': ...
+    # }
+    def status
+      if @status.nil?
+        # run controller_status to get all robot states
+        states = self.class.parse_status_output(controller_status)
+        fail 'No output from controller status' unless states.size > 0
+
+        # convert states into status metrics for all robots with state
+        @status = {}
+        robots.each do |robot|
+          matches = states.select { |state| state[:robot] == robot }
+          @status[robot] = self.class.consolidate_states_into_status(matches)
+        end
+
+        # cross-check against all robots
+        robots.each do |robot|
+          if @status[robot].nil?
+            @status[robot] = {
+              state: (running(robot) == 0 ? :not_enabled : :unknown),
+              running: 0
+            }
+          elsif @status[robot][:running] != running(robot)
+            @status[robot][:state] = :down
+          end
+        end
+      end
+      @status
+    end
+
+    # Runs 'bundle exec controller status' and returns/yields output
+    # @yield [Array[String]] output
+    def controller_status
+      IO.popen('bundle exec controller status 2>&1').readlines.map(&:strip)
+    end
+
+    # -- Class methods --
+    class << self
+      #
+      # @param [Array<String>] output from bundle exec controller status
+      # @return [Array<Hash>] status
+      # [
+      #  {
+      #  robot: 'robot123',
+      #  nth: 1
+      #  pid: 123
+      #  state: :down | :up
+      #  },
+      #  robot: 'robot456',
+      #  nth: 1
+      #  pid: 456
+      #  state: :down | :up
+      #  }
+      # ]
+      def parse_status_output(output)
+        output.inject([]) { |a, e| a << parse_status_line(e) }.compact
+      end
+
+      #
+      # @param [String] line as bluepill outputs them...
+      # robot01_01_dor_gisAssemblyWF_assign-placenames(pid:29481): up
+      # robot02_01_dor_gisAssemblyWF_author-data(pid:29697): down
+      # robot03_01_dor_gisAssemblyWF_author-metadata(pid:29512): unmonitored
+      #
+      # @return [Hash] status {
+      #  robot: 'robot123',
+      #  nth: 1
+      #  pid: 123
+      #  state: :down | :up
+      # }
+      def parse_status_line(line)
+        if line =~ /^robot\d\d_(\d\d)_(.+)\(pid:(\d+)\):\s+(.+)$/
+          return {
+            nth:   Regexp.last_match[1].to_i,
+            robot: Regexp.last_match[2].to_s,
+            pid:   Regexp.last_match[3].to_i,
+            state: (Regexp.last_match[4].to_s == 'up') ? :up : :down
+          }
+        end
+        nil
+      end
+
+      # reduces individuals states into a single status
+      def consolidate_states_into_status(statuses)
+        if statuses.is_a?(Array) && statuses.size > 0
+          # XXX: assumes all statuses are for the same robot
+          running = 0
+          state = :up
+          statuses.each do |status|
+            running += 1 if status[:state] == :up
+            state = :down unless status[:state] == :up
+          end
+          {
+            state: state,
+            running: running
+          }
+        else
+          fail 'No information from bundle exec controller status'
+        end
+      end
+    end
+  end
+end

data/lib/tasks/doc.rake

@@ -1,5 +1,5 @@
-desc "Generate RDoc"
-task :doc => ['doc:generate']
+desc 'Generate RDoc'
+task doc: ['doc:generate']
 
 namespace :doc do
   project_root = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
@@ -10,27 +10,26 @@ namespace :doc do
     require 'yard/rake/yardoc_task'
 
     YARD::Rake::YardocTask.new(:generate) do |yt|
-      yt.files   =  Dir.glob(File.join(project_root, 'lib', '*.rb')) + 
-                    Dir.glob(File.join(project_root, 'lib', '**', '*.rb')) +
-                   [ File.join(project_root, 'README.rdoc') ]
-                   
+      yt.files   =  Dir.glob(File.join(project_root, 'lib', '*.rb')) +
+        Dir.glob(File.join(project_root, 'lib', '**', '*.rb')) +
+        [File.join(project_root, 'README.rdoc')]
+
       yt.options = ['--output-dir', doc_destination, '--readme', 'README.md']
     end
   rescue LoadError
-    desc "Generate YARD Documentation"
+    desc 'Generate YARD Documentation'
     task :generate do
-      abort "Please install the YARD gem to generate rdoc."
+      abort 'Please install the YARD gem to generate rdoc.'
     end
   end
 
-  desc "Remove generated documenation"
+  desc 'Remove generated documenation'
   task :clean do
-    rm_r doc_destination if File.exists?(doc_destination)
+    rm_r doc_destination if File.exist?(doc_destination)
   end
-
 end
 
-desc "Build Yard documentation"
+desc 'Build Yard documentation'
 task :yard do
   YARD::Rake::YardocTask.new do |t|
     t.files = ['lib/**/*.rb', 'bin/**/*.rb']

data/robot-controller.gemspec

@@ -1,36 +1,35 @@
 # -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib/', __FILE__)
-$:.unshift lib unless $:.include?(lib)
+$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib)
 
 require 'robot-controller'
 
 Gem::Specification.new do |s|
-  s.name        = "robot-controller"
+  s.name        = 'robot-controller'
   s.version     = RobotController::VERSION
   s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Darren Hardy"]
-  s.email       = ["drh@stanford.edu"]
-  s.homepage    = "http://github.com/sul-dlss/robot-controller"
-  s.summary     = "Monitors and controls running workflow robots off of priority queues and within a cluster"
+  s.authors     = ['Darren Hardy']
+  s.email       = ['drh@stanford.edu']
+  s.homepage    = 'http://github.com/sul-dlss/robot-controller'
+  s.summary     = 'Monitors and controls running workflow robots off of priority queues and within a cluster'
   s.has_rdoc    = true
   s.licenses    = ['ALv2', 'Stanford University']
-  
+
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- spec/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_paths = ['lib']
- 
-  s.required_rubygems_version = ">= 1.3.6"
-  
-  s.add_dependency 'bluepill', '~> 0.0.67'
+
+  s.required_rubygems_version = '>= 1.3.6'
+
+  s.add_dependency 'bluepill', '0.0.68' # pin bluepill to prevent nil status output regression
   s.add_dependency 'resque', '~> 1.25.2'
   s.add_dependency 'rake', '~> 10.3.2'
-  
+
   s.add_development_dependency 'awesome_print'
   s.add_development_dependency 'pry'
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'redcarpet' # provides Markdown
   s.add_development_dependency 'version_bumper'
   s.add_development_dependency 'yard'
- 
 end

data/spec/spec_helper.rb

@@ -10,7 +10,7 @@ SimpleCov.start
 require 'bundler/setup'
 Bundler.require(:default, :development)
 
-RSpec.configure do |config|  
+RSpec.configure do |_config|
 end
 
-Rails = Object.new unless defined? Rails
+Rails = Object.new unless defined? Rails

data/spec/unit/robots_spec.rb

@@ -1,87 +1,84 @@
 require 'robot-controller/robots'
 
-describe RobotConfigParser do
-  
-  context "simple" do
-    subject {
-      RobotConfigParser.new.load('standard.yml', 'spec/fixtures', 'host1')
-    }
+describe RobotController::Parser do
+  context 'simple' do
+    subject do
+      RobotController::Parser.load('standard.yml', 'spec/fixtures', 'host1')
+    end
 
-    it "pass1" do
-     expect(subject).to eq [
-        {:robot=>"X", :queues=>["X_default"], :n=>1},
-        {:robot=>"Y", :queues=>["Y_B"], :n=>1},
-        {:robot=>"Z", :queues=>["Z_C"], :n=>3}
+    it 'pass1' do
+      expect(subject).to eq [
+        { robot: 'X', queues: ['X_default'], n: 1 },
+        { robot: 'Y', queues: ['Y_B'], n: 1 },
+        { robot: 'Z', queues: ['Z_C'], n: 3 }
       ]
-    end    
+    end
   end
-  
-  context "expanded" do
-    subject {
-      RobotConfigParser.new.load('standard.yml', 'spec/fixtures', 'host2')
-    }
 
-    it "pass2" do
-     expect(subject).to eq [
-        {:robot=>"A", :queues=>["A_*"], :n=>1},
-        {:robot=>"B", :queues=>["B_X", "B_Y"], :n=>1},
-        {:robot=>"C", :queues=>["C_X", "C_Y", "C_Z"], :n=>5},
-        {:robot=>"D", :queues=>["D_default"], :n=>1},
+  context 'expanded' do
+    subject do
+      RobotController::Parser.load('standard.yml', 'spec/fixtures', 'host2')
+    end
+
+    it 'pass2' do
+      expect(subject).to eq [
+        { robot: 'A', queues: ['A_*'], n: 1 },
+        { robot: 'B', queues: %w(B_X B_Y), n: 1 },
+        { robot: 'C', queues: %w(C_X C_Y C_Z), n: 5 },
+        { robot: 'D', queues: ['D_default'], n: 1 }
       ]
-    end    
+    end
   end
-  
-  context "parse_instances" do
-    subject {
-      RobotConfigParser.new
-    }
-    it "valid inputs" do
+
+  context 'parse_instances' do
+    subject do
+      RobotController::Parser
+    end
+    it 'valid inputs' do
       expect(subject.parse_instances(0)).to eq 1
       expect(subject.parse_instances(1)).to eq 1
       expect(subject.parse_instances(16)).to eq 16
     end
-    
-    it "invalid inputs" do
-      expect {
+
+    it 'invalid inputs' do
+      expect do
         subject.parse_instances(17)
-      }.to raise_error RuntimeError
+      end.to raise_error RuntimeError
     end
   end
-  
-  context "parse_lanes" do
-    subject {
-      RobotConfigParser.new
-    }
-    
-    it "valid inputs" do
+
+  context 'parse_lanes' do
+    subject do
+      RobotController::Parser
+    end
+
+    it 'valid inputs' do
       expect(subject.parse_lanes('*')).to eq ['*']
       expect(subject.parse_lanes('')).to eq ['default']
       expect(subject.parse_lanes('default')).to eq ['default']
       expect(subject.parse_lanes('A')).to eq ['A']
-      expect(subject.parse_lanes('A,B')).to eq ['A', 'B']
+      expect(subject.parse_lanes('A,B')).to eq %w(A B)
     end
-    
-    it "tricky inputs" do
+
+    it 'tricky inputs' do
       expect(subject.parse_lanes(' ')).to eq ['default']
       expect(subject.parse_lanes(' , ')).to eq ['default']
       expect(subject.parse_lanes(' ,,')).to eq ['default']
-      expect(subject.parse_lanes('A , B')).to eq ['A','B']
+      expect(subject.parse_lanes('A , B')).to eq %w(A B)
       expect(subject.parse_lanes('A-B')).to eq ['A-B']
-      expect(subject.parse_lanes('A,B,A')).to eq ['A','B']
+      expect(subject.parse_lanes('A,B,A')).to eq %w(A B)
     end
-
   end
-  
-  context "build_queues" do
-    subject {
-      RobotConfigParser.new
-    }
-    
-    it "valid inputs" do
-      expect(subject.build_queues('z','*')).to eq ['z_*']
-      expect(subject.build_queues('z','default')).to eq ['z_default']
-      expect(subject.build_queues('z','A,B,C')).to eq ['z_A','z_B','z_C']
+
+  context 'build_queues' do
+    subject do
+      RobotController::Parser
+    end
+
+    it 'valid inputs' do
+      expect(subject.build_queues('z', '*')).to eq ['z_*']
+      expect(subject.build_queues('z', 'default')).to eq ['z_default']
+      expect(subject.build_queues('z', 'A,B,C')).to eq %w(z_A z_B z_C)
     end
-    
   end
-end
+end

data/spec/unit/verify_spec.rb

@@ -0,0 +1,144 @@
+require 'robot-controller'
+
+describe RobotController::Verify do
+  context 'initialization' do
+    subject do
+      RobotController::Verify.new('robot1' => 1, 'robot2' => 2, 'robot3' => 0)
+    end
+
+    it 'has correct robots' do
+      expect(subject.robots).to eq %w(robot1 robot2 robot3)
+    end
+
+    it 'has correct enabled' do
+      expect(subject.running('robot1')).to eq 1
+      expect(subject.running('robot2')).to eq 2
+      expect(subject.running('robot3')).to eq 0
+    end
+
+    it 'handles empty parameters' do
+      expect { RobotController::Verify.new }.to raise_error(ArgumentError)
+    end
+  end
+
+  context 'parse_status methods' do
+    subject do
+      RobotController::Verify.new('dor_gisAssemblyWF_assign-placenames' => 1)
+    end
+    it 'parse single line' do
+      expect(subject.class.parse_status_line('robot01_01_dor_gisAssemblyWF_assign-placenames(pid:29481): up')).to eq(
+        nth: 1,
+        pid: 29481,
+        robot: 'dor_gisAssemblyWF_assign-placenames',
+        state: :up
+      )
+      expect(subject.class.parse_status_line('robot01_02_dor_gisAssemblyWF_assign-placenames(pid:29482): starting')).to eq(
+        nth: 2,
+        pid: 29482,
+        robot: 'dor_gisAssemblyWF_assign-placenames',
+        state: :down
+      )
+    end
+
+    it 'parses a single line with error' do
+      expect(subject.class.parse_status_line('garbage')).to be_nil
+    end
+
+    it 'parse all lines' do
+      expect(subject.class.parse_status_output([
+        'robot01_01_dor_gisAssemblyWF_assign-placenames(pid:29481): starting',
+        'robot01_02_dor_gisAssemblyWF_assign-placenames(pid:29482): unmonitored',
+        'robot01_03_dor_gisAssemblyWF_assign-placenames(pid:29483): up'])).to eq(
+          [
+            {
+              nth: 1,
+              pid: 29481,
+              robot: 'dor_gisAssemblyWF_assign-placenames',
+              state: :down
+            }, {
+              nth: 2,
+              pid: 29482,
+              robot: 'dor_gisAssemblyWF_assign-placenames',
+              state: :down
+            }, {
+              nth: 3,
+              pid: 29483,
+              robot: 'dor_gisAssemblyWF_assign-placenames',
+              state: :up
+            }
+          ]
+        )
+    end
+  end
+
+  context 'verify method with single process' do
+    subject do
+      RobotController::Verify.new('dor_gisAssemblyWF_assign-placenames' => 1)
+    end
+
+    it 'runs controller status for up' do
+      allow(subject).to receive(:controller_status).and_return([
+        'robot01_01_dor_gisAssemblyWF_assign-placenames(pid:29483): up'
+      ])
+      expect(subject.verify).to eq(
+        'dor_gisAssemblyWF_assign-placenames' => { state: :up, running: 1 }
+      )
+    end
+
+    it 'runs controller status for down' do
+      allow(subject).to receive(:controller_status).and_return([
+        'robot01_01_dor_gisAssemblyWF_assign-placenames(pid:29481): down'
+      ])
+      expect(subject.verify).to eq(
+        'dor_gisAssemblyWF_assign-placenames' => { state: :down, running: 0 }
+      )
+    end
+
+    it 'runs controller status even when broken' do
+      allow(subject).to receive(:controller_status).and_return([])
+      expect { subject.verify }.to raise_error(RuntimeError)
+    end
+  end
+
+  context 'verify method with multiple processes' do
+    subject do
+      RobotController::Verify.new('dor_gisAssemblyWF_assign-placenames' => 3)
+    end
+
+    it 'runs controller status for up' do
+      allow(subject).to receive(:controller_status).and_return([
+        'robot01_01_dor_gisAssemblyWF_assign-placenames(pid:29483): up',
+        'robot01_02_dor_gisAssemblyWF_assign-placenames(pid:29484): up',
+        'robot02_01_dor_gisAssemblyWF_foobar(pid:29485): up',
+        'robot01_02_dor_gisAssemblyWF_assign-placenames(pid:29486): up'
+      ])
+      expect(subject.verify).to eq(
+        'dor_gisAssemblyWF_assign-placenames' => { state: :up, running: 3 }
+      )
+    end
+
+    it 'runs controller status for down' do
+      allow(subject).to receive(:controller_status).and_return([
+        'robot01_01_dor_gisAssemblyWF_assign-placenames(pid:29481): starting',
+        'robot01_02_dor_gisAssemblyWF_assign-placenames(pid:29482): unmonitored',
+        'robot01_03_dor_gisAssemblyWF_assign-placenames(pid:29483): up',
+        'robot02_01_dor_gisAssemblyWF_foobar(pid:29484): up',
+        'robot02_02_dor_gisAssemblyWF_foobar(pid:29485): down'
+      ])
+      expect(subject.verify).to eq(
+        'dor_gisAssemblyWF_assign-placenames' => { state: :down, running: 1 }
+      )
+    end
+
+    it 'runs controller status for running mismatch' do
+      allow(subject).to receive(:controller_status).and_return([
+        'robot01_01_dor_gisAssemblyWF_assign-placenames(pid:29483): up',
+        'robot01_02_dor_gisAssemblyWF_assign-placenames(pid:29484): up',
+        'robot02_01_dor_gisAssemblyWF_foobar(pid:29486): up'
+      ])
+      expect(subject.verify).to eq(
+        'dor_gisAssemblyWF_assign-placenames' => { state: :down, running: 2 }
+      )
+    end
+  end
+end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: robot-controller
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.0.beta1
 platform: ruby
 authors:
 - Darren Hardy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-25 00:00:00.000000000 Z
+date: 2015-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bluepill
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.67
+        version: 0.0.68
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.67
+        version: 0.0.68
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
@@ -145,23 +145,29 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rubocop.yml
+- .rubocop_todo.yml
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - VERSION
 - bin/controller
+- config/environments/robots_development.yml
+- config/robots.yml
 - example/config/environments/robots_development.yml
 - example/lib/tasks/environment.rake
 - lib/robot-controller.rb
 - lib/robot-controller/bluepill.rb
 - lib/robot-controller/robots.rb
 - lib/robot-controller/tasks.rb
+- lib/robot-controller/verify.rb
 - lib/tasks/doc.rake
 - robot-controller.gemspec
 - spec/fixtures/standard.yml
 - spec/spec_helper.rb
 - spec/unit/robots_spec.rb
+- spec/unit/verify_spec.rb
 homepage: http://github.com/sul-dlss/robot-controller
 licenses:
 - ALv2
@@ -183,7 +189,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Monitors and controls running workflow robots off of priority queues and
@@ -192,4 +198,5 @@ test_files:
 - spec/fixtures/standard.yml
 - spec/spec_helper.rb
 - spec/unit/robots_spec.rb
+- spec/unit/verify_spec.rb
 has_rdoc: true