robot-controller 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0acb2a8aa3413747076dac05fc5c889357c00fd4
-  data.tar.gz: 8692def8629151ccb3217d591909a3342870d945
+  metadata.gz: c01c2e8362bea0af35c415127c35e2b6fd5a643d
+  data.tar.gz: 3bea75dfaa22001504e2a7db019ae406e4b81c00
 SHA512:
-  metadata.gz: ad5c6d906a1fc3cb465d4f42992502bcc54a485d1de556ed004c8776d970e60c1441e4c96c35dbc1d9492d399276f72bcd7fdf1cb2c3113c64c752fae8383329
-  data.tar.gz: a5e75b22049082d804928398d17e86f877ac82c07aa18556709bd8e7f8452aba8f5d00c72bbaccbc8a9c388a70979e0192f055cc2471eae7e15b9f58efbe28cc
+  metadata.gz: e4c6de1781095edee97a52db140f4e6b9e7ca538ccaf66982ef0ce7f50467e2a998ac8c37334bc89fabd58dfe0420a7eb66604b51e593effd6a6870655f6bcf4
+  data.tar.gz: cc1b11f2655f960195a56ce760d19ebc39babd02e797002b19dce8ad3c4336213f2bb7a8ce7ad8b736ece255a259b2cf47db832fd1fcc2ffe4b8751dcc12160e

data/README.md

@@ -16,18 +16,31 @@ In your `Rakefile`, add the following (if you don't want to include the environm
     
 Create the following configuration files based on the examples in `example/config`:
 
-    config/boot.rb
-    config/environments/development.rb
-    config/environments/bluepill_development.rb
-    config/environments/workflows_development.rb
+    config/environments/robots_development.yml
+    
+Then to use the controller to boot the robots:
+
+    % bundle exec controller boot
+    
+If you want to *override* the bluepill configuration but still use the 
+controller, then add:
+
+    config/bluepill.rb
 
 ### Usage
 
-    Usage: controller [ boot | quit ]
-           controller [ start | status | stop | restart | log ] [worker]
+    Usage: controller ( boot | quit )
+           controller ( start | status | stop | restart | log ) [worker]
+           controller [--help]
 
     Example:
-     % controller boot    # start bluepilld and jobs
-     % controller status  # check on status of jobs
-     % controller stop    # stop jobs
-     % controller quit    # stop bluepilld
+      % controller boot    # start bluepilld and jobs
+      % controller status  # check on status of jobs
+      % controller log dor_accessionWF_descriptive-metadata # view log for worker
+      % controller stop    # stop jobs
+      % controller quit    # stop bluepilld
+  
+    Environment:
+      BLUEPILL_BASEDIR - where bluepill stores its state (default: run/bluepill)
+      BLUEPILL_LOGFILE - output log (default: log/bluepill.log)
+      ROBOT_ENVIRONMENT - (default: development)

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.3.0

data/bin/controller

@@ -17,11 +17,6 @@ Environment:
   BLUEPILL_BASEDIR - where bluepill stores its state (default: run/bluepill)
   BLUEPILL_LOGFILE - output log (default: log/bluepill.log)
   ROBOT_ENVIRONMENT - (default: development)
-  
-Configuration files (in search order)
-  config/environments/bluepill_development.rb
-  config/environments/bluepill.rb
-  config/bluepill.rb
 '
   exit -1
 end
@@ -36,13 +31,15 @@ cmd << " --base-dir #{ENV['BLUEPILL_BASE_DIR']}"
 cmd << " --logfile #{ENV['BLUEPILL_LOGFILE']}"
 
 if ARGV[0] == 'boot'
-  ["config/environments/bluepill_#{ENV['ROBOT_ENVIRONMENT']}.rb",
-   "config/environments/bluepill.rb",
-   "config/bluepill.rb"].each do |fn|
-    if File.file?(fn)
-      system "#{cmd} load #{fn}"
-      exit 0
-    end
+  fn = 'config/bluepill.rb' # allow override
+  unless File.file?(fn)
+    require 'robot-controller'
+    fn = RobotController.bluepill_config
+  end
+  if File.file?(fn)
+    puts "Loading #{fn}"
+    system "#{cmd} load #{fn}"
+    exit 0
   end
   puts "ERROR: Cannot find bluepill configuration file for #{ENV['ROBOT_ENVIRONMENT']}"
   exit -1

data/example/config/boot.rb

@@ -33,9 +33,3 @@ Resque.redis = REDIS_URL
 require 'active_support/core_ext' # camelcase
 require 'robot-controller'
 
-
-
-
-
-
-  

data/example/config/environments/robots_development.yml

@@ -0,0 +1,86 @@
+#
+#  Robot allocation strategy
+#
+#  Format:
+#
+#    host:
+#      - robot[:lane[:instances]]
+#
+#  where
+#    1. robot is a single robot identifier (fully-qualified with
+#       REPO_SUITE_ROBOT, e.g., "dor_accessionWF_technical-metadata").
+#    2. lane is a single integer (4), a range (3-5), or a list (2,4,8), 
+#       or an asterix (*).
+#    3. instances is a single integer.
+#
+#  Both lane and instances are optional. Lane defaults to *, and 
+#  instances defaults to 1.
+#
+#  When a robot is allocated to multiple lanes, it reads them in
+#  PRIORITY ORDER. That is, if a robot is listening to lanes 1, 2, and 3,
+#  it works on lane 1 until empty, then lane 2 until empty, and then
+#  lane 3 until empty. In the meantime, if a job comes in on a faster
+#  lane, it works on that after finishing it's current job (i.e., after
+#  working on a job in lane 3, if a job comes in on lane 1 in the intermin,
+#  the robot will work on the lane 1 job next before returning to lane 3).
+#
+#  In general, lanes 1-5 are reserved for priority queues, and lanes 6-N
+#  are dedicated lanes. 
+#
+#  Note that the syntax is YAML, so the lists must not contain spaces or
+#  needs to be quoted.
+#  
+#    RIGHT
+#      - dor_accessionWF_technical-metadata:10:5
+#      - 'dor_accessionWF_technical-metadata : 10 : 5'
+#
+#    WRONG
+#      - dor_accessionWF_technical-metadata : 10 : 5
+#      - dor_accessionWF_technical-metadata: 10: 5
+#
+
+#
+# Robot 1 (8 CPU) hosts shelving and publish only
+#
+sul-robots1-dev:
+  - dor_accessionWF_shelve:*:3                # 3 robots for all lanes
+  - dor_accessionWF_shelve:6:3                # 3 robots for lane 6
+  - dor_accessionWF_shelve:7:3                # 3 robots for lane 7
+  - dor_accessionWF_shelve:8:3                # 3 robots for lane 8
+  - dor_accessionWF_shelve:9:3                # 3 robots for lane 9
+  - dor_accessionWF_shelve:10:3               # 3 robots for lane 10
+  - dor_accessionWF_publish:1-5:3             # 3 robots for lanes 1 through 5
+  - dor_accessionWF_publish:6:3               # 3 robots for lane 6
+  - dor_accessionWF_publish:7:3               # 3 robots for lane 7
+  - dor_accessionWF_publish:8:3               # 3 robots for lane 8
+  - dor_accessionWF_publish:9:3               # 3 robots for lane 9
+  - dor_accessionWF_publish:10:3              # 3 robots for lane 10
+  - dor_accessionWF_publish:6,9               # 1 robot for lane 6 and 9
+
+#
+# Robot 2 (16 CPU) hosts technical metadata creation only
+#
+sul-robots2-dev:
+  - dor_accessionWF_technical-metadata:1:5    # 5 robots for lane 1
+  - dor_accessionWF_technical-metadata:2:5    # 5 robots for lane 2
+  - dor_accessionWF_technical-metadata:3-5:5  # 5 robots for lanes 3, 4, 5
+  - dor_accessionWF_technical-metadata:6:2    # 2 robots for lane 6
+  - dor_accessionWF_technical-metadata:7:2    # 2 robots for lane 7
+  - dor_accessionWF_technical-metadata:8:2    # 2 robots for lane 8
+  - dor_accessionWF_technical-metadata:9:2    # 2 robots for lane 9
+  - dor_accessionWF_technical-metadata:10:2   # 2 robots for lane 10
+
+#
+# Robot 3 (4 CPU) hosts helper robots for all accessioning workflows
+#
+sul-robots3-dev:
+  - dor_accessionWF_descriptive-metadata      # 1 robot for all lanes
+  - dor_accessionWF_rights-metadata           # 1 robot for all lanes
+  - dor_accessionWF_content-metadata          # 1 robot for all lanes
+  - dor_accessionWF_technical-metadata        # 1 robot for all lanes
+  - dor_accessionWF_remediate-object          # 1 robot for all lanes
+  - dor_accessionWF_shelve:*:5                # 5 robots for all lanes
+  - dor_accessionWF_publish:*:10              # 10 robots for all lanes
+  - dor_accessionWF_provenance-metadata       # 1 robot for all lanes
+  - dor_accessionWF_sdr-ingest-transfer       # 1 robot for all lanes
+  - dor_accessionWF_end-accession             # 1 robot for all lanes

data/lib/robot-controller.rb

@@ -2,4 +2,9 @@
 module RobotController
   # e.g., `1.2.3`
   VERSION = File.read(File.join(File.dirname(__FILE__), '..', 'VERSION')).strip
+  
+  def self.bluepill_config
+    File.join(File.dirname(__FILE__), 'robot-controller', 'bluepill.rb')
+  end
+  
 end

data/lib/robot-controller/bluepill.rb

@@ -0,0 +1,79 @@
+WORKDIR = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+robot_environment = ENV['ROBOT_ENVIRONMENT'] || 'development'
+require 'robot-controller/robots'
+#
+# Expect ROBOTS = [
+#  {:robot => 'x', :queues => ['a', 'b'], :n => 1}
+#  {:robot => 'z', :queues => ['b'], :n => 3}
+# ]
+#
+Bluepill.application File.basename(File.dirname(File.dirname(WORKDIR))),
+  :log_file => "#{WORKDIR}/log/bluepill.log" do |app|
+  app.working_dir = WORKDIR
+  ROBOTS.each_index do |i|    
+    # prefix process name with index number to prevent duplicate process names
+    prefix = sprintf("robot%02d", i+1)
+    app.process("#{prefix}_#{ROBOTS[i][:robot]}") do |process|
+      puts "Creating robot #{process.name}"
+
+      # queue order is *VERY* important
+      queues = ROBOTS[i][:queues].join(',')
+
+      # use environment for these resque variables
+      process.environment = {
+        'QUEUES' => queues,
+        'ROBOT_ENVIRONMENT' => robot_environment
+      }
+      process.environment['VERBOSE'] = 'yes' if robot_environment != 'production'
+
+      # process configuration
+      process.group = robot_environment
+      process.stdout = process.stderr = "#{WORKDIR}/log/#{ROBOTS[i][:robot]}.log"
+
+      # spawn worker processes using robot-controller
+      process.environment['COUNT'] = ROBOTS[i][:n]
+      process.start_command = "rake workers" 
+      
+      # we use bluepill to daemonize the resque workers rather than using
+      # resque's BACKGROUND flag
+      process.daemonize = true
+      
+      # bluepill manages pid files
+      # process.pid_file = "#{WORKDIR}/run/#{process.name}.pid"
+
+      # graceful stops
+      process.stop_grace_time = 360.seconds # must be greater than stop_signals total
+      process.stop_signals = [
+        :quit, 300.seconds, # waits for jobs, then exits gracefully
+        :term, 10.seconds,  # kills jobs and exits
+        :kill               # no mercy
+      ]
+
+      # process monitoring
+
+      # backoff if process is flapping between states
+      # process.checks :flapping,
+      #                :times => 2, :within => 30.seconds,
+      #                :retry_in => 7.seconds
+
+      # restart if process runs for longer than 15 mins of CPU time
+      # process.checks :running_time,
+      #                :every => 5.minutes, :below => 15.minutes
+
+      # restart if CPU usage > 75% for 3 times, check every 10 seconds
+      # process.checks :cpu_usage,
+      #                :every => 10.seconds,
+      #                :below => 75, :times => 3,
+      #                :include_children => true
+      #
+      # restart the process or any of its children
+      # if MEM usage > 100MB for 3 times, check every 10 seconds
+      # process.checks :mem_usage,
+      #                :every => 10.seconds,
+      #                :below => 100.megabytes, :times => 3,
+      #                :include_children => true
+
+      # NOTE: there is an implicit process.keepalive
+    end
+  end
+end

data/lib/robot-controller/robots.rb

@@ -0,0 +1,126 @@
+require 'yaml'
+
+class RobotConfigParser
+  ROBOT_INSTANCE_MAX = 16
+  LANE_INSTANCE_MAX = 99  # sprintf("%02d") maximum
+
+  # parse_instances(1) == 1
+  # parse_instances(16) == 16
+  # parse_instances(0) == 1
+  # parse_instances(99) => RuntimeError
+  def parse_instances(n)
+    if n > ROBOT_INSTANCE_MAX
+      raise RuntimeError, "TooManyInstances: #{n} > #{ROBOT_INSTANCE_MAX}"
+    end
+    n = 1 if n < 1
+    n
+  end
+
+  # parse_lanes('*') == ['*']
+  # parse_lanes('0') == [0]
+  # parse_lanes('1') == [1]
+  # parse_lanes('1-5') == [1,2,3,4,5]
+  # parse_lanes('1,2,3') == [1,2,3]
+  # parse_lanes('1-5,8') == [1,2,3,4,5,8]
+  # parse_lanes('-1') == [0, 1]
+  # parse_lanes('100') == RuntimeException
+  def parse_lanes(lanes_spec)
+    lanes = []
+  
+    # parse each comma-seperated specification
+    lanes_spec.split(/,/).each do |i|
+      # this is a range element
+      if i =~ /-/
+        x = i.split(/-/)
+        Range.new(x[0].to_i, x[1].to_i).each do |j|
+          lanes << j
+        end
+      # a wildcard
+      elsif i == '*'
+        lanes << '*'
+      # simple integer
+      else
+        lanes << i.to_i
+      end
+    end
+  
+    # verify that lanes are all within 1 .. LANE_INSTANCE_MAX
+    lanes.each do |j|
+      if j.is_a?(Integer)
+        if j > LANE_INSTANCE_MAX
+          raise RuntimeError, "SyntaxError: Lane #{j} > #{LANE_INSTANCE_MAX}"
+        elsif j < 0
+          raise RuntimeError, "SyntaxError: Lane #{j} < 0"
+        end
+      end
+    end
+    lanes
+  end
+
+  # build_queues('a','1') => ['a_01']
+  # build_queues('a','1,3') => ['a_01', 'a_03']
+  # build_queues('a','1-3') => ['a_01', 'a_02', 'a_03']
+  def build_queues(robot, lanes)
+    queues = []
+    parse_lanes(lanes).each do |i|
+      queues << [robot, i == '*' ? '*' : sprintf("%02d", i)].join('_')
+    end
+    queues
+  end
+
+  # main entry point
+  def load(env)
+    # read the YAML file
+    robots_fn = File.join('config', 'environments', "robots_#{env}.yml")
+    unless File.file?(robots_fn)
+      raise RuntimeError, "FileNotFound: #{robots_fn}"
+    end
+    
+    puts "Loading #{robots_fn}"
+    robots =  YAML.load_file(robots_fn)
+    # puts robots
+  
+    # determine current host
+    host = `hostname -s`.strip
+    # puts host
+
+    # host = 'sul-robots1-dev' # XXX
+    unless robots.include?(host)
+      raise RuntimeError, "HostMismatch: #{host} not defined in #{robots_fn}"
+    end
+
+    # parse YAML lines for host where i is robot[:lane[:instances]]
+    r = []
+    robots[host].each do |i|  
+      robot = i.split(/:/)
+      robot.each do |j|
+        if j.strip == ''
+          raise RuntimeError, "SyntaxError: #{i}"
+        end
+      end
+    
+      # add defaults
+      if robot.size == 1
+        robot << '*'
+      end
+      if robot.size == 2
+        robot << '1'
+      end
+    
+      # build queues for robot instances
+      unless robot.size == 3
+        raise RuntimeError, "SyntaxError: #{i}"
+      end
+      robot[2] = parse_instances(robot[2].to_i)
+      # puts robot.join(' : ')
+      queues = build_queues(robot[0], robot[1])
+      # puts queues
+  
+      r << {:robot => robot[0], :queues => queues, :n => robot[2] }
+    end
+    r
+  end
+end
+
+ROBOTS = RobotConfigParser.new.load(ENV['ROBOT_ENVIRONMENT'] || 'development')
+# puts ROBOTS

data/robot-controller.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
  
   s.required_rubygems_version = ">= 1.3.6"
   
-  s.add_dependency 'bluepill', '~> 0.0.66'
+  s.add_dependency 'bluepill', '~> 0.0.67'
   
   s.add_development_dependency 'awesome_print'
   s.add_development_dependency 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: robot-controller
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Darren Hardy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-15 00:00:00.000000000 Z
+date: 2014-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bluepill
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.66
+        version: 0.0.67
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.66
+        version: 0.0.67
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
@@ -124,10 +124,11 @@ files:
 - VERSION
 - bin/controller
 - example/config/boot.rb
-- example/config/environments/bluepill_development.rb
-- example/config/environments/workflows_development.rb
+- example/config/environments/robots_development.yml
 - example/lib/tasks/environment.rake
 - lib/robot-controller.rb
+- lib/robot-controller/bluepill.rb
+- lib/robot-controller/robots.rb
 - lib/robot-controller/tasks.rb
 - lib/tasks/doc.rake
 - robot-controller.gemspec

data/example/config/environments/bluepill_development.rb

@@ -1,98 +0,0 @@
-WORKDIR=File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
-robot_environment = ENV['ROBOT_ENVIRONMENT'] || 'development'
-workflows = File.expand_path(File.join(WORKDIR, 'config', 'environments', "workflows_#{robot_environment}.rb"))
-puts "Loading #{workflows}"
-require workflows
-
-Bluepill.application 'robot-controller', 
-  :log_file => "#{WORKDIR}/log/bluepill.log" do |app|
-  app.working_dir = WORKDIR
-  WORKFLOW_STEPS.each do |qualified_wf|
-    wf = qualified_wf.gsub(/:/, '_')
-    app.process(wf) do |process|
-      # use configuration for number of workers -- default is 1
-      n = WORKFLOW_N[qualified_wf] ? WORKFLOW_N[qualified_wf].to_i : 1
-      puts "Creating #{n} worker#{n>1?'s':' '} for #{qualified_wf}"
-      
-      # queue order is *VERY* important
-      #
-      # XXX: make this configurable based on wf
-      # WORKFLOW_PRIORITIES[wf] is the name of a second worker that reads the given queues
-      #
-      # see RobotMaster::Queue#queue_name for naming convention
-      # @example
-      #     queue_name('dor:assemblyWF:jp2-create')
-      #     => 'dor_assemblyWF_jp2-create_default'
-      #     queue_name('dor:assemblyWF:jp2-create', 100)
-      #     => 'dor_assemblyWF_jp2-create_high'
-      #
-      queues = []
-      %w{critical high default low}.each do |p| 
-        queues << "#{wf}_#{p}"
-      end
-      queues = queues.join(',')
-      # puts "Using queues #{queues}"
-      
-      # use environment for these resque variables
-      process.environment = {
-        'QUEUES' => "#{queues}",
-        'VERBOSE' => 'yes',
-        'ROBOT_ENVIRONMENT' => robot_environment
-      }
-
-      # process configuration
-      process.group = robot_environment
-      process.stdout = process.stderr = "#{WORKDIR}/log/#{wf}.log"
-
-      # let bluepill manage pid files
-      # process.pid_file = "#{WORKDIR}/run/#{wf}.pid"
-      
-      # spawn n worker processes
-      if n > 1
-        process.start_command = "env COUNT=#{n} rake workers" # not resque:workers
-      else # 1 worker
-        process.start_command = "rake environment resque:work"
-      end
-      # puts "Using #{process.start_command}"
-      # puts "Using #{process.environment}"
-      
-      # we use bluepill to daemonize the resque workers rather than using 
-      # resque's BACKGROUND flag
-      process.daemonize = true
-
-      # graceful stops
-      process.stop_grace_time = 60.seconds # must be greater than stop_signals total
-      process.stop_signals = [
-        :quit, 45.seconds, # waits for jobs, then exits gracefully
-        :term, 10.seconds, # kills jobs and exits
-        :kill              # no mercy
-      ]
-      
-      # process monitoring
-      
-      # backoff if process is flapping between states
-      # process.checks :flapping, 
-      #                :times => 2, :within => 30.seconds, 
-      #                :retry_in => 7.seconds
-      
-      # restart if process runs for longer than 15 mins of CPU time
-      # process.checks :running_time, 
-      #                :every => 5.minutes, :below => 15.minutes
-      
-      # restart if CPU usage > 75% for 3 times, check every 10 seconds
-      # process.checks :cpu_usage, 
-      #                :every => 10.seconds, 
-      #                :below => 75, :times => 3,
-      #                :include_children => true
-      # 
-      # restart the process or any of its children 
-      # if MEM usage > 100MB for 3 times, check every 10 seconds
-      # process.checks :mem_usage, 
-      #                :every => 10.seconds, 
-      #                :below => 100.megabytes, :times => 3, 
-      #                :include_children => true
-                     
-      # NOTE: there is an implicit process.keepalive
-    end
-  end
-end

data/example/config/environments/workflows_development.rb

@@ -1,25 +0,0 @@
-# will spawn worker(s) for each of the given workflows (fully qualified as "repo:wf:robot")
-WORKFLOW_STEPS = %w{
-  dor:accessionWF:start-accession
-  dor:accessionWF:descriptive-metadata
-  dor:accessionWF:rights-metadata
-  dor:accessionWF:content-metadata
-  dor:accessionWF:technical-metadata
-  dor:accessionWF:remediate-object
-  dor:accessionWF:shelve
-  dor:accessionWF:publish
-  dor:accessionWF:provenance-metadata
-  dor:accessionWF:sdr-ingest-transfer
-  dor:accessionWF:sdr-ingest-received
-  dor:accessionWF:end-accession
-  dor:assemblyWF:start-assembly
-  dor:assemblyWF:jp2-create
-  dor:assemblyWF:checksum-compute
-  dor:assemblyWF:exif-collect
-  dor:assemblyWF:accessioning-initiate
-}
-
-# number of workers for the given workflows
-WORKFLOW_N = Hash[*%w{
-  dor:assemblyWF:checksum-compute     3
-}]