robot-controller 0.2.0

data/.gitignore

@@ -0,0 +1,37 @@
+Gemfile.lock
+log
+run
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2014 by The Board of Trustees of the Leland Stanford
+Junior University.  All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/README.md

@@ -0,0 +1,33 @@
+robot-controller
+================
+
+Monitors and controls running workflow robots off of priority queues and within a cluster.
+
+## Configuration
+
+In your `Gemfile`, add:
+
+    gem 'robot-controller'
+
+In your `Rakefile`, add the following (if you don't want to include the environment unconditionally):
+
+    require 'resque/tasks'
+    require 'robot-controller/tasks'
+    
+Create the following configuration files based on the examples in `example/config`:
+
+    config/boot.rb
+    config/environments/development.rb
+    config/environments/bluepill_development.rb
+    config/environments/workflows_development.rb
+
+### Usage
+
+    Usage: controller [ boot | quit ]
+           controller [ start | status | stop | restart | log ] [worker]
+
+    Example:
+     % controller boot    # start bluepilld and jobs
+     % controller status  # check on status of jobs
+     % controller stop    # stop jobs
+     % controller quit    # stop bluepilld

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'rake'
+require 'version_bumper'
+require 'robot-controller/tasks'
+
+Dir.glob('lib/tasks/*.rake').each { |r| import r }
+
+task :default => [ :yard ]

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/bin/controller

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+if ARGV.size == 0
+  puts '
+Usage: controller [ boot | quit ]
+       controller [ start | status | stop | restart | log ] [worker]
+
+Example:
+  % controller boot    # start bluepilld and jobs
+  % controller status  # check on status of jobs
+  % controller log dor_accessionWF_descriptive-metadata # view log for worker
+  % controller stop    # stop jobs
+  % controller quit    # stop bluepilld
+'
+  exit -1
+end
+
+ENV['ROBOT_ENVIRONMENT'] ||= 'development'
+
+cmd = "bluepill --no-privileged"
+cmd << " --base-dir #{ENV['BLUEPILL_BASE_DIR'] || File.expand_path('run/bluepill')}"
+cmd << " --logfile #{ENV['BLUEPILL_LOGFILE'] || File.expand_path('log/bluepill.log')}"
+
+if ARGV[0] == 'boot' 
+  system "#{cmd} load config/environments/bluepill_#{ENV['ROBOT_ENVIRONMENT']}.rb"
+else
+  system "#{cmd} #{ARGV.join(' ')}"
+end

data/example/config/boot.rb

@@ -0,0 +1,44 @@
+$:.unshift File.expand_path(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'rubygems'
+require 'bundler/setup'
+require 'logger'
+
+# Load the environment file based on Environment.  Default to development
+ENV['ROBOT_ENVIRONMENT'] ||= 'development'
+require File.expand_path(File.join(File.dirname(__FILE__), 'environments', ENV['ROBOT_ENVIRONMENT']))
+
+ENV['ROBOT_LOG'] ||= 'stdout'
+ENV['ROBOT_LOG_LEVEL'] ||= 'info'
+ROBOT_LOG = Logger.new(ENV['ROBOT_LOG'].downcase == 'stdout' ? STDOUT : ENV['ROBOT_LOG'])
+ROBOT_LOG.level = Logger::SEV_LABEL.index(ENV['ROBOT_LOG_LEVEL'].upcase) || Logger::INFO
+
+# if running under debugging and using stdout, then run unbuffered
+STDOUT.sync = true if ENV['ROBOT_LOG_LEVEL'].downcase == 'debug' and ENV['ROBOT_LOG'].downcase == 'stdout'
+
+# @see http://rubydoc.info/gems/redis/3.0.7/file/README.md
+# @see https://github.com/resque/resque
+#
+# Set the redis connection. Takes any of:
+#   String - a redis url string (e.g., 'redis://host:port')
+#   String - 'hostname:port[:db][/namespace]'
+#   Redis - a redis connection that will be namespaced :resque
+#   Redis::Namespace - a namespaced redis connection that will be used as-is
+#   Redis::Distributed - a distributed redis connection that will be used as-is
+#   Hash - a redis connection hash (e.g. {:host => 'localhost', :port => 6379, :db => 0})
+require 'resque'
+REDIS_URL ||= "localhost:6379/resque:#{ENV['ROBOT_ENVIRONMENT']}"
+Resque.redis = REDIS_URL
+
+require 'active_support/core_ext' # camelcase
+require 'druid-tools'
+require 'robot-controller'
+require 'robots'
+
+
+
+
+
+
+
+  

data/example/config/environments/bluepill_development.rb

@@ -0,0 +1,98 @@
+WORKDIR=File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+robot_environment = ENV['ROBOT_ENVIRONMENT'] || 'development'
+workflows = File.expand_path(File.join(WORKDIR, 'config', 'environments', "workflows_#{robot_environment}.rb"))
+puts "Loading #{workflows}"
+require workflows
+
+Bluepill.application 'robot-controller', 
+  :log_file => "#{WORKDIR}/log/bluepill.log" do |app|
+  app.working_dir = WORKDIR
+  WORKFLOW_STEPS.each do |qualified_wf|
+    wf = qualified_wf.gsub(/:/, '_')
+    app.process(wf) do |process|
+      # use configuration for number of workers -- default is 1
+      n = WORKFLOW_N[qualified_wf] ? WORKFLOW_N[qualified_wf].to_i : 1
+      puts "Creating #{n} worker#{n>1?'s':' '} for #{qualified_wf}"
+      
+      # queue order is *VERY* important
+      #
+      # XXX: make this configurable based on wf
+      # WORKFLOW_PRIORITIES[wf] is the name of a second worker that reads the given queues
+      #
+      # see RobotMaster::Queue#queue_name for naming convention
+      # @example
+      #     queue_name('dor:assemblyWF:jp2-create')
+      #     => 'dor_assemblyWF_jp2-create_default'
+      #     queue_name('dor:assemblyWF:jp2-create', 100)
+      #     => 'dor_assemblyWF_jp2-create_high'
+      #
+      queues = []
+      %w{critical high default low}.each do |p| 
+        queues << "#{wf}_#{p}"
+      end
+      queues = queues.join(',')
+      # puts "Using queues #{queues}"
+      
+      # use environment for these resque variables
+      process.environment = {
+        'QUEUES' => "#{queues}",
+        'VERBOSE' => 'yes',
+        'ROBOT_ENVIRONMENT' => robot_environment
+      }
+
+      # process configuration
+      process.group = robot_environment
+      process.stdout = process.stderr = "#{WORKDIR}/log/#{wf}.log"
+
+      # let bluepill manage pid files
+      # process.pid_file = "#{WORKDIR}/run/#{wf}.pid"
+      
+      # spawn n worker processes
+      if n > 1
+        process.start_command = "env COUNT=#{n} rake workers" # not resque:workers
+      else # 1 worker
+        process.start_command = "rake environment resque:work"
+      end
+      # puts "Using #{process.start_command}"
+      # puts "Using #{process.environment}"
+      
+      # we use bluepill to daemonize the resque workers rather than using 
+      # resque's BACKGROUND flag
+      process.daemonize = true
+
+      # graceful stops
+      process.stop_grace_time = 60.seconds # must be greater than stop_signals total
+      process.stop_signals = [
+        :quit, 45.seconds, # waits for jobs, then exits gracefully
+        :term, 10.seconds, # kills jobs and exits
+        :kill              # no mercy
+      ]
+      
+      # process monitoring
+      
+      # backoff if process is flapping between states
+      # process.checks :flapping, 
+      #                :times => 2, :within => 30.seconds, 
+      #                :retry_in => 7.seconds
+      
+      # restart if process runs for longer than 15 mins of CPU time
+      # process.checks :running_time, 
+      #                :every => 5.minutes, :below => 15.minutes
+      
+      # restart if CPU usage > 75% for 3 times, check every 10 seconds
+      # process.checks :cpu_usage, 
+      #                :every => 10.seconds, 
+      #                :below => 75, :times => 3,
+      #                :include_children => true
+      # 
+      # restart the process or any of its children 
+      # if MEM usage > 100MB for 3 times, check every 10 seconds
+      # process.checks :mem_usage, 
+      #                :every => 10.seconds, 
+      #                :below => 100.megabytes, :times => 3, 
+      #                :include_children => true
+                     
+      # NOTE: there is an implicit process.keepalive
+    end
+  end
+end

data/example/config/environments/workflows_development.rb

@@ -0,0 +1,31 @@
+# will spawn worker(s) for each of the given workflows (fully qualified as "repo:wf:robot")
+WORKFLOW_STEPS = %w{
+  dor:accessionWF:start-accession
+  dor:accessionWF:descriptive-metadata
+  dor:accessionWF:rights-metadata
+  dor:accessionWF:content-metadata
+  dor:accessionWF:technical-metadata
+  dor:accessionWF:remediate-object
+  dor:accessionWF:shelve
+  dor:accessionWF:publish
+  dor:accessionWF:provenance-metadata
+  dor:accessionWF:sdr-ingest-transfer
+  dor:accessionWF:sdr-ingest-received
+  dor:accessionWF:end-accession
+  dor:assemblyWF:start-assembly
+  dor:assemblyWF:jp2-create
+  dor:assemblyWF:checksum-compute
+  dor:assemblyWF:exif-collect
+  dor:assemblyWF:accessioning-initiate
+}
+
+# number of workers for the given workflows
+WORKFLOW_N = Hash[*%w{
+  dor:assemblyWF:checksum-compute     3
+}]
+
+# starts up 2 workers -- one for this priority and another for all
+# XXX: not implemented
+WORKFLOW_PRIORITIES = Hash[*%w{
+  dor:assemblyWF:checksum-compute     critical,high
+}]

data/example/lib/tasks/environment.rake

@@ -0,0 +1,5 @@
+desc "Load environment from boot file"
+task :environment do
+  # needs to load the boot file
+  require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'config', 'boot'))
+end

data/lib/robot-controller.rb

@@ -0,0 +1,5 @@
+# Monitors and controls running workflow robots off of priority queues and within a cluster.
+module RobotController
+  # e.g., `1.2.3`
+  VERSION = File.read(File.join(File.dirname(__FILE__), '..', 'VERSION')).strip
+end

data/lib/robot-controller/tasks.rb

@@ -0,0 +1,10 @@
+desc "Start multiple Resque workers using environment"
+task :workers => [ :environment ] do
+  threads = []
+  (ENV['COUNT'] || '1').to_i.times do
+    threads << Thread.new do
+      system "rake environment resque:work" # XXX is better way to do this?
+    end
+  end
+  threads.each { |thread| thread.join }
+end

data/lib/tasks/doc.rake

@@ -0,0 +1,38 @@
+desc "Generate RDoc"
+task :doc => ['doc:generate']
+
+namespace :doc do
+  project_root = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+  doc_destination = File.join(project_root, 'rdoc')
+
+  begin
+    require 'yard'
+    require 'yard/rake/yardoc_task'
+
+    YARD::Rake::YardocTask.new(:generate) do |yt|
+      yt.files   =  Dir.glob(File.join(project_root, 'lib', '*.rb')) + 
+                    Dir.glob(File.join(project_root, 'lib', '**', '*.rb')) +
+                   [ File.join(project_root, 'README.rdoc') ]
+                   
+      yt.options = ['--output-dir', doc_destination, '--readme', 'README.md']
+    end
+  rescue LoadError
+    desc "Generate YARD Documentation"
+    task :generate do
+      abort "Please install the YARD gem to generate rdoc."
+    end
+  end
+
+  desc "Remove generated documenation"
+  task :clean do
+    rm_r doc_destination if File.exists?(doc_destination)
+  end
+
+end
+
+desc "Build Yard documentation"
+task :yard do
+  YARD::Rake::YardocTask.new do |t|
+    t.files = ['lib/**/*.rb', 'bin/**/*.rb']
+  end
+end

data/robot-controller.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'robot-controller'
+
+Gem::Specification.new do |s|
+  s.name        = "robot-controller"
+  s.version     = RobotController::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Darren Hardy"]
+  s.email       = ["drh@stanford.edu"]
+  s.homepage    = "http://github.com/sul-dlss/robot-controller"
+  s.summary     = "Monitors and controls running workflow robots off of priority queues and within a cluster"
+  s.has_rdoc    = true
+  s.licenses    = ['ALv2', 'Stanford University']
+  
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+ 
+  s.required_rubygems_version = ">= 1.3.6"
+  
+  s.add_dependency 'bluepill', '~> 0.0.66'
+  
+  s.add_development_dependency 'awesome_print'
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'redcarpet' # provides Markdown
+  s.add_development_dependency 'version_bumper'
+  s.add_development_dependency 'yard'
+ 
+end

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification
+name: robot-controller
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- Darren Hardy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bluepill
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.66
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.66
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: version_bumper
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- drh@stanford.edu
+executables:
+- controller
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- bin/controller
+- example/config/boot.rb
+- example/config/environments/bluepill_development.rb
+- example/config/environments/workflows_development.rb
+- example/lib/tasks/environment.rake
+- lib/robot-controller.rb
+- lib/robot-controller/tasks.rb
+- lib/tasks/doc.rake
+- robot-controller.gemspec
+homepage: http://github.com/sul-dlss/robot-controller
+licenses:
+- ALv2
+- Stanford University
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Monitors and controls running workflow robots off of priority queues and
+  within a cluster
+test_files: []
+has_rdoc: true