robot-controller 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,37 @@
1
+ Gemfile.lock
2
+ log
3
+ run
4
+ *.gem
5
+ *.rbc
6
+ /.config
7
+ /coverage/
8
+ /InstalledFiles
9
+ /pkg/
10
+ /spec/reports/
11
+ /test/tmp/
12
+ /test/version_tmp/
13
+ /tmp/
14
+
15
+ ## Specific to RubyMotion:
16
+ .dat*
17
+ .repl_history
18
+ build/
19
+
20
+ ## Documentation cache and generated files:
21
+ /.yardoc/
22
+ /_yardoc/
23
+ /doc/
24
+ /rdoc/
25
+
26
+ ## Environment normalisation:
27
+ /.bundle/
28
+ /lib/bundler/man/
29
+
30
+ # for a library or gem, you might want to ignore these files since the code is
31
+ # intended to run in multiple environments; otherwise, check them in:
32
+ # Gemfile.lock
33
+ # .ruby-version
34
+ # .ruby-gemset
35
+
36
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
37
+ .rvmrc
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,14 @@
1
+ Copyright (c) 2014 by The Board of Trustees of the Leland Stanford
2
+ Junior University. All rights reserved.
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License"); you
5
+ may not use this file except in compliance with the License. You
6
+ may obtain a copy of the License at
7
+
8
+ http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13
+ implied. See the License for the specific language governing
14
+ permissions and limitations under the License.
@@ -0,0 +1,33 @@
1
+ robot-controller
2
+ ================
3
+
4
+ Monitors and controls running workflow robots off of priority queues and within a cluster.
5
+
6
+ ## Configuration
7
+
8
+ In your `Gemfile`, add:
9
+
10
+ gem 'robot-controller'
11
+
12
+ In your `Rakefile`, add the following (if you don't want to include the environment unconditionally):
13
+
14
+ require 'resque/tasks'
15
+ require 'robot-controller/tasks'
16
+
17
+ Create the following configuration files based on the examples in `example/config`:
18
+
19
+ config/boot.rb
20
+ config/environments/development.rb
21
+ config/environments/bluepill_development.rb
22
+ config/environments/workflows_development.rb
23
+
24
+ ### Usage
25
+
26
+ Usage: controller [ boot | quit ]
27
+ controller [ start | status | stop | restart | log ] [worker]
28
+
29
+ Example:
30
+ % controller boot # start bluepilld and jobs
31
+ % controller status # check on status of jobs
32
+ % controller stop # stop jobs
33
+ % controller quit # stop bluepilld
@@ -0,0 +1,8 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+ require 'version_bumper'
4
+ require 'robot-controller/tasks'
5
+
6
+ Dir.glob('lib/tasks/*.rake').each { |r| import r }
7
+
8
+ task :default => [ :yard ]
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.2.0
@@ -0,0 +1,28 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ if ARGV.size == 0
4
+ puts '
5
+ Usage: controller [ boot | quit ]
6
+ controller [ start | status | stop | restart | log ] [worker]
7
+
8
+ Example:
9
+ % controller boot # start bluepilld and jobs
10
+ % controller status # check on status of jobs
11
+ % controller log dor_accessionWF_descriptive-metadata # view log for worker
12
+ % controller stop # stop jobs
13
+ % controller quit # stop bluepilld
14
+ '
15
+ exit -1
16
+ end
17
+
18
+ ENV['ROBOT_ENVIRONMENT'] ||= 'development'
19
+
20
+ cmd = "bluepill --no-privileged"
21
+ cmd << " --base-dir #{ENV['BLUEPILL_BASE_DIR'] || File.expand_path('run/bluepill')}"
22
+ cmd << " --logfile #{ENV['BLUEPILL_LOGFILE'] || File.expand_path('log/bluepill.log')}"
23
+
24
+ if ARGV[0] == 'boot'
25
+ system "#{cmd} load config/environments/bluepill_#{ENV['ROBOT_ENVIRONMENT']}.rb"
26
+ else
27
+ system "#{cmd} #{ARGV.join(' ')}"
28
+ end
@@ -0,0 +1,44 @@
1
+ $:.unshift File.expand_path(File.join(File.dirname(__FILE__), "..", "lib"))
2
+
3
+ require 'rubygems'
4
+ require 'bundler/setup'
5
+ require 'logger'
6
+
7
+ # Load the environment file based on Environment. Default to development
8
+ ENV['ROBOT_ENVIRONMENT'] ||= 'development'
9
+ require File.expand_path(File.join(File.dirname(__FILE__), 'environments', ENV['ROBOT_ENVIRONMENT']))
10
+
11
+ ENV['ROBOT_LOG'] ||= 'stdout'
12
+ ENV['ROBOT_LOG_LEVEL'] ||= 'info'
13
+ ROBOT_LOG = Logger.new(ENV['ROBOT_LOG'].downcase == 'stdout' ? STDOUT : ENV['ROBOT_LOG'])
14
+ ROBOT_LOG.level = Logger::SEV_LABEL.index(ENV['ROBOT_LOG_LEVEL'].upcase) || Logger::INFO
15
+
16
+ # if running under debugging and using stdout, then run unbuffered
17
+ STDOUT.sync = true if ENV['ROBOT_LOG_LEVEL'].downcase == 'debug' and ENV['ROBOT_LOG'].downcase == 'stdout'
18
+
19
+ # @see http://rubydoc.info/gems/redis/3.0.7/file/README.md
20
+ # @see https://github.com/resque/resque
21
+ #
22
+ # Set the redis connection. Takes any of:
23
+ # String - a redis url string (e.g., 'redis://host:port')
24
+ # String - 'hostname:port[:db][/namespace]'
25
+ # Redis - a redis connection that will be namespaced :resque
26
+ # Redis::Namespace - a namespaced redis connection that will be used as-is
27
+ # Redis::Distributed - a distributed redis connection that will be used as-is
28
+ # Hash - a redis connection hash (e.g. {:host => 'localhost', :port => 6379, :db => 0})
29
+ require 'resque'
30
+ REDIS_URL ||= "localhost:6379/resque:#{ENV['ROBOT_ENVIRONMENT']}"
31
+ Resque.redis = REDIS_URL
32
+
33
+ require 'active_support/core_ext' # camelcase
34
+ require 'druid-tools'
35
+ require 'robot-controller'
36
+ require 'robots'
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
@@ -0,0 +1,98 @@
1
+ WORKDIR=File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
2
+ robot_environment = ENV['ROBOT_ENVIRONMENT'] || 'development'
3
+ workflows = File.expand_path(File.join(WORKDIR, 'config', 'environments', "workflows_#{robot_environment}.rb"))
4
+ puts "Loading #{workflows}"
5
+ require workflows
6
+
7
+ Bluepill.application 'robot-controller',
8
+ :log_file => "#{WORKDIR}/log/bluepill.log" do |app|
9
+ app.working_dir = WORKDIR
10
+ WORKFLOW_STEPS.each do |qualified_wf|
11
+ wf = qualified_wf.gsub(/:/, '_')
12
+ app.process(wf) do |process|
13
+ # use configuration for number of workers -- default is 1
14
+ n = WORKFLOW_N[qualified_wf] ? WORKFLOW_N[qualified_wf].to_i : 1
15
+ puts "Creating #{n} worker#{n>1?'s':' '} for #{qualified_wf}"
16
+
17
+ # queue order is *VERY* important
18
+ #
19
+ # XXX: make this configurable based on wf
20
+ # WORKFLOW_PRIORITIES[wf] is the name of a second worker that reads the given queues
21
+ #
22
+ # see RobotMaster::Queue#queue_name for naming convention
23
+ # @example
24
+ # queue_name('dor:assemblyWF:jp2-create')
25
+ # => 'dor_assemblyWF_jp2-create_default'
26
+ # queue_name('dor:assemblyWF:jp2-create', 100)
27
+ # => 'dor_assemblyWF_jp2-create_high'
28
+ #
29
+ queues = []
30
+ %w{critical high default low}.each do |p|
31
+ queues << "#{wf}_#{p}"
32
+ end
33
+ queues = queues.join(',')
34
+ # puts "Using queues #{queues}"
35
+
36
+ # use environment for these resque variables
37
+ process.environment = {
38
+ 'QUEUES' => "#{queues}",
39
+ 'VERBOSE' => 'yes',
40
+ 'ROBOT_ENVIRONMENT' => robot_environment
41
+ }
42
+
43
+ # process configuration
44
+ process.group = robot_environment
45
+ process.stdout = process.stderr = "#{WORKDIR}/log/#{wf}.log"
46
+
47
+ # let bluepill manage pid files
48
+ # process.pid_file = "#{WORKDIR}/run/#{wf}.pid"
49
+
50
+ # spawn n worker processes
51
+ if n > 1
52
+ process.start_command = "env COUNT=#{n} rake workers" # not resque:workers
53
+ else # 1 worker
54
+ process.start_command = "rake environment resque:work"
55
+ end
56
+ # puts "Using #{process.start_command}"
57
+ # puts "Using #{process.environment}"
58
+
59
+ # we use bluepill to daemonize the resque workers rather than using
60
+ # resque's BACKGROUND flag
61
+ process.daemonize = true
62
+
63
+ # graceful stops
64
+ process.stop_grace_time = 60.seconds # must be greater than stop_signals total
65
+ process.stop_signals = [
66
+ :quit, 45.seconds, # waits for jobs, then exits gracefully
67
+ :term, 10.seconds, # kills jobs and exits
68
+ :kill # no mercy
69
+ ]
70
+
71
+ # process monitoring
72
+
73
+ # backoff if process is flapping between states
74
+ # process.checks :flapping,
75
+ # :times => 2, :within => 30.seconds,
76
+ # :retry_in => 7.seconds
77
+
78
+ # restart if process runs for longer than 15 mins of CPU time
79
+ # process.checks :running_time,
80
+ # :every => 5.minutes, :below => 15.minutes
81
+
82
+ # restart if CPU usage > 75% for 3 times, check every 10 seconds
83
+ # process.checks :cpu_usage,
84
+ # :every => 10.seconds,
85
+ # :below => 75, :times => 3,
86
+ # :include_children => true
87
+ #
88
+ # restart the process or any of its children
89
+ # if MEM usage > 100MB for 3 times, check every 10 seconds
90
+ # process.checks :mem_usage,
91
+ # :every => 10.seconds,
92
+ # :below => 100.megabytes, :times => 3,
93
+ # :include_children => true
94
+
95
+ # NOTE: there is an implicit process.keepalive
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,31 @@
1
+ # will spawn worker(s) for each of the given workflows (fully qualified as "repo:wf:robot")
2
+ WORKFLOW_STEPS = %w{
3
+ dor:accessionWF:start-accession
4
+ dor:accessionWF:descriptive-metadata
5
+ dor:accessionWF:rights-metadata
6
+ dor:accessionWF:content-metadata
7
+ dor:accessionWF:technical-metadata
8
+ dor:accessionWF:remediate-object
9
+ dor:accessionWF:shelve
10
+ dor:accessionWF:publish
11
+ dor:accessionWF:provenance-metadata
12
+ dor:accessionWF:sdr-ingest-transfer
13
+ dor:accessionWF:sdr-ingest-received
14
+ dor:accessionWF:end-accession
15
+ dor:assemblyWF:start-assembly
16
+ dor:assemblyWF:jp2-create
17
+ dor:assemblyWF:checksum-compute
18
+ dor:assemblyWF:exif-collect
19
+ dor:assemblyWF:accessioning-initiate
20
+ }
21
+
22
+ # number of workers for the given workflows
23
+ WORKFLOW_N = Hash[*%w{
24
+ dor:assemblyWF:checksum-compute 3
25
+ }]
26
+
27
+ # starts up 2 workers -- one for this priority and another for all
28
+ # XXX: not implemented
29
+ WORKFLOW_PRIORITIES = Hash[*%w{
30
+ dor:assemblyWF:checksum-compute critical,high
31
+ }]
@@ -0,0 +1,5 @@
1
+ desc "Load environment from boot file"
2
+ task :environment do
3
+ # needs to load the boot file
4
+ require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'config', 'boot'))
5
+ end
@@ -0,0 +1,5 @@
1
+ # Monitors and controls running workflow robots off of priority queues and within a cluster.
2
+ module RobotController
3
+ # e.g., `1.2.3`
4
+ VERSION = File.read(File.join(File.dirname(__FILE__), '..', 'VERSION')).strip
5
+ end
@@ -0,0 +1,10 @@
1
+ desc "Start multiple Resque workers using environment"
2
+ task :workers => [ :environment ] do
3
+ threads = []
4
+ (ENV['COUNT'] || '1').to_i.times do
5
+ threads << Thread.new do
6
+ system "rake environment resque:work" # XXX is better way to do this?
7
+ end
8
+ end
9
+ threads.each { |thread| thread.join }
10
+ end
@@ -0,0 +1,38 @@
1
+ desc "Generate RDoc"
2
+ task :doc => ['doc:generate']
3
+
4
+ namespace :doc do
5
+ project_root = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
6
+ doc_destination = File.join(project_root, 'rdoc')
7
+
8
+ begin
9
+ require 'yard'
10
+ require 'yard/rake/yardoc_task'
11
+
12
+ YARD::Rake::YardocTask.new(:generate) do |yt|
13
+ yt.files = Dir.glob(File.join(project_root, 'lib', '*.rb')) +
14
+ Dir.glob(File.join(project_root, 'lib', '**', '*.rb')) +
15
+ [ File.join(project_root, 'README.rdoc') ]
16
+
17
+ yt.options = ['--output-dir', doc_destination, '--readme', 'README.md']
18
+ end
19
+ rescue LoadError
20
+ desc "Generate YARD Documentation"
21
+ task :generate do
22
+ abort "Please install the YARD gem to generate rdoc."
23
+ end
24
+ end
25
+
26
+ desc "Remove generated documenation"
27
+ task :clean do
28
+ rm_r doc_destination if File.exists?(doc_destination)
29
+ end
30
+
31
+ end
32
+
33
+ desc "Build Yard documentation"
34
+ task :yard do
35
+ YARD::Rake::YardocTask.new do |t|
36
+ t.files = ['lib/**/*.rb', 'bin/**/*.rb']
37
+ end
38
+ end
@@ -0,0 +1,34 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib/', __FILE__)
3
+ $:.unshift lib unless $:.include?(lib)
4
+
5
+ require 'robot-controller'
6
+
7
+ Gem::Specification.new do |s|
8
+ s.name = "robot-controller"
9
+ s.version = RobotController::VERSION
10
+ s.platform = Gem::Platform::RUBY
11
+ s.authors = ["Darren Hardy"]
12
+ s.email = ["drh@stanford.edu"]
13
+ s.homepage = "http://github.com/sul-dlss/robot-controller"
14
+ s.summary = "Monitors and controls running workflow robots off of priority queues and within a cluster"
15
+ s.has_rdoc = true
16
+ s.licenses = ['ALv2', 'Stanford University']
17
+
18
+ s.files = `git ls-files`.split("\n")
19
+ s.test_files = `git ls-files -- spec/*`.split("\n")
20
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
21
+ s.require_paths = ['lib']
22
+
23
+ s.required_rubygems_version = ">= 1.3.6"
24
+
25
+ s.add_dependency 'bluepill', '~> 0.0.66'
26
+
27
+ s.add_development_dependency 'awesome_print'
28
+ s.add_development_dependency 'pry'
29
+ s.add_development_dependency 'rake'
30
+ s.add_development_dependency 'redcarpet' # provides Markdown
31
+ s.add_development_dependency 'version_bumper'
32
+ s.add_development_dependency 'yard'
33
+
34
+ end
metadata ADDED
@@ -0,0 +1,177 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: robot-controller
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Darren Hardy
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2014-04-04 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bluepill
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: 0.0.66
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: 0.0.66
30
+ - !ruby/object:Gem::Dependency
31
+ name: awesome_print
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: pry
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: rake
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ - !ruby/object:Gem::Dependency
79
+ name: redcarpet
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ type: :development
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ - !ruby/object:Gem::Dependency
95
+ name: version_bumper
96
+ requirement: !ruby/object:Gem::Requirement
97
+ none: false
98
+ requirements:
99
+ - - ! '>='
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ type: :development
103
+ prerelease: false
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ none: false
106
+ requirements:
107
+ - - ! '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ - !ruby/object:Gem::Dependency
111
+ name: yard
112
+ requirement: !ruby/object:Gem::Requirement
113
+ none: false
114
+ requirements:
115
+ - - ! '>='
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ none: false
122
+ requirements:
123
+ - - ! '>='
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ description:
127
+ email:
128
+ - drh@stanford.edu
129
+ executables:
130
+ - controller
131
+ extensions: []
132
+ extra_rdoc_files: []
133
+ files:
134
+ - .gitignore
135
+ - Gemfile
136
+ - LICENSE
137
+ - README.md
138
+ - Rakefile
139
+ - VERSION
140
+ - bin/controller
141
+ - example/config/boot.rb
142
+ - example/config/environments/bluepill_development.rb
143
+ - example/config/environments/workflows_development.rb
144
+ - example/lib/tasks/environment.rake
145
+ - lib/robot-controller.rb
146
+ - lib/robot-controller/tasks.rb
147
+ - lib/tasks/doc.rake
148
+ - robot-controller.gemspec
149
+ homepage: http://github.com/sul-dlss/robot-controller
150
+ licenses:
151
+ - ALv2
152
+ - Stanford University
153
+ post_install_message:
154
+ rdoc_options: []
155
+ require_paths:
156
+ - lib
157
+ required_ruby_version: !ruby/object:Gem::Requirement
158
+ none: false
159
+ requirements:
160
+ - - ! '>='
161
+ - !ruby/object:Gem::Version
162
+ version: '0'
163
+ required_rubygems_version: !ruby/object:Gem::Requirement
164
+ none: false
165
+ requirements:
166
+ - - ! '>='
167
+ - !ruby/object:Gem::Version
168
+ version: 1.3.6
169
+ requirements: []
170
+ rubyforge_project:
171
+ rubygems_version: 1.8.25
172
+ signing_key:
173
+ specification_version: 3
174
+ summary: Monitors and controls running workflow robots off of priority queues and
175
+ within a cluster
176
+ test_files: []
177
+ has_rdoc: true