ripe 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.travis.yml +17 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +41 -0
- data/Rakefile +11 -0
- data/bin/ripe +62 -0
- data/lib/ripe/block.rb +41 -0
- data/lib/ripe/controller.rb +51 -0
- data/lib/ripe/liquid_block.rb +17 -0
- data/lib/ripe/multi_block.rb +35 -0
- data/lib/ripe/parallel_block.rb +13 -0
- data/lib/ripe/serial_block.rb +37 -0
- data/lib/ripe/task.rb +17 -0
- data/lib/ripe/task_migration.rb +18 -0
- data/lib/ripe/version.rb +3 -0
- data/lib/ripe/worker.rb +174 -0
- data/lib/ripe/worker_migration.rb +26 -0
- data/lib/ripe/working_block.rb +41 -0
- data/lib/ripe.rb +6 -0
- data/ripe.gemspec +35 -0
- data/share/moab.sh +14 -0
- data/spec/ripe_spec.rb +7 -0
- data/spec/spec_helper.rb +1 -0
- metadata +239 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f0ae395af344b3bc8349fbd1e028a80be84bbd77
|
4
|
+
data.tar.gz: 067f49928266b15f63c2cd021a162f14678be86d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6778dfe639d44a022296592ad693da8f7bcdc30dca8506fb26d0ab75533bc5fbc17fba738cc0c13c765191695883abc7b501b1a2aca1b4a9350e6aa2db8dd834
|
7
|
+
data.tar.gz: 1efa8def1b83a718604001fd7cc875f6ec28919813485a5006b4db340dfd8dc9697d08c799b06b17733c17908f542b5fbdbf6dcbf8e4cd14b39ac1a51b0af9b0
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013-2015 Nicolas De Jay
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# Ripe
|
2
|
+
[](https://travis-ci.org/ndejay/ripe)
|
3
|
+
|
4
|
+
ripe is an abstraction layer between the MOAB/Torque stack and your pipeline.
|
5
|
+
|
6
|
+
With ripe, you can easily collate tasks into workflows which can then be
|
7
|
+
applied to samples without the headache of manually dealing with the queuing
|
8
|
+
system.
|
9
|
+
|
10
|
+
Notes:
|
11
|
+
|
12
|
+
- MOAB is a scheduler -- it takes care of priorities and allocations.
|
13
|
+
- Torque is a resource manager -- it launches the jobs on the nodes.
|
14
|
+
|
15
|
+
## Installation
|
16
|
+
|
17
|
+
Add this line to your application's Gemfile:
|
18
|
+
|
19
|
+
```ruby
|
20
|
+
gem 'ripe', :git => 'git://github.com/ndejay/ripe.git'
|
21
|
+
```
|
22
|
+
|
23
|
+
And then execute:
|
24
|
+
|
25
|
+
$ bundle
|
26
|
+
|
27
|
+
Or install it yourself as:
|
28
|
+
|
29
|
+
$ gem install ripe
|
30
|
+
|
31
|
+
## Usage
|
32
|
+
|
33
|
+
TODO: Write usage instructions here
|
34
|
+
|
35
|
+
## Contributing
|
36
|
+
|
37
|
+
1. Fork it ( https://github.com/ndejay/ripe/fork )
|
38
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
39
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
40
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
41
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
require 'rspec/core/rake_task'
|
2
|
+
require 'bundler/gem_tasks'
|
3
|
+
|
4
|
+
# Default directory to look in is `/specs`
|
5
|
+
# Run with `rake spec`
|
6
|
+
|
7
|
+
RSpec::Core::RakeTask.new(:spec) do |task|
|
8
|
+
# task.rspec_opts = ['--color', '--format', 'nested']
|
9
|
+
end
|
10
|
+
|
11
|
+
task :default => :spec
|
data/bin/ripe
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'ripe'
|
4
|
+
require 'thor'
|
5
|
+
|
6
|
+
include Ripe
|
7
|
+
|
8
|
+
# Look for workflows and blocks in ./.ripe wherever invoked, then in
|
9
|
+
# directories specified in the $RIPELIB environment variable.
|
10
|
+
$RIPELIBS = "#{ENV['PWD']}/.ripe:#{ENV['RIPELIB']}".split(/:/)
|
11
|
+
|
12
|
+
def Task(handle, vars = {})
|
13
|
+
WorkingBlock.new($RIPELIBS.map{ |path| "#{path}/tasks/#{handle}.sh" }.select{ |path| File.exists? path }.first, vars)
|
14
|
+
end
|
15
|
+
|
16
|
+
class CLI < Thor
|
17
|
+
desc 'console', 'Enter ripe console'
|
18
|
+
def console
|
19
|
+
require 'irb'
|
20
|
+
require 'irbtools'
|
21
|
+
|
22
|
+
ARGV.clear # This is necessary because otherwise all arguments
|
23
|
+
# get sent to IRB
|
24
|
+
|
25
|
+
$ripe = Controller.new
|
26
|
+
$ripe.attach
|
27
|
+
|
28
|
+
IRB.start "ripe #{Ripe::VERSION}"
|
29
|
+
end
|
30
|
+
|
31
|
+
desc 'prepare SAMPLES', 'Prepare jobs from template workflow'
|
32
|
+
option :workflow, :aliases => '-w', :type => :string, :required => true,
|
33
|
+
:desc => 'Workflow to be applied'
|
34
|
+
option :options, :aliases => '-o', :type => :string, :required => false,
|
35
|
+
:desc => 'Options', :default => ''
|
36
|
+
def prepare(*samples)
|
37
|
+
return if (samples.length == 0)
|
38
|
+
|
39
|
+
additional_vars = options[:options].split(/,/).map do |pair|
|
40
|
+
key, value = pair.split(/=/)
|
41
|
+
{ key.to_sym => value }
|
42
|
+
end
|
43
|
+
additional_vars = additional_vars.inject(&:merge) || {}
|
44
|
+
|
45
|
+
# Expect $callback to be a lambda function that takes one argument (sample)
|
46
|
+
# and returns a Block, and $vars to be a dictionary mapping arguments to
|
47
|
+
# values (i.e. resource allocation)
|
48
|
+
workflows = $RIPELIBS.map{ |path| "#{path}/workflows/#{options[:workflow]}.rb" }
|
49
|
+
require_relative workflows.select{ |path| File.exists? path }.first
|
50
|
+
|
51
|
+
$ripe = Controller.new
|
52
|
+
$ripe.attach_or_create # Create .ripe if it doesn't exist
|
53
|
+
$ripe.prepare(samples, $callback, $vars.merge(additional_vars))
|
54
|
+
end
|
55
|
+
|
56
|
+
desc 'version', 'Retrieve ripe version'
|
57
|
+
def version
|
58
|
+
puts "ripe version #{Ripe::VERSION}"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
CLI.start(ARGV)
|
data/lib/ripe/block.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require_relative 'parallel_block'
|
2
|
+
require_relative 'serial_block'
|
3
|
+
|
4
|
+
module Ripe
|
5
|
+
class Block
|
6
|
+
attr_reader :id, :blocks
|
7
|
+
attr_accessor :vars
|
8
|
+
|
9
|
+
def initialize(id, blocks = [], vars = {})
|
10
|
+
@id, @blocks, @vars = id, blocks, vars
|
11
|
+
end
|
12
|
+
|
13
|
+
def prune(protect, depend)
|
14
|
+
self
|
15
|
+
end
|
16
|
+
|
17
|
+
# Syntactic sugar of the form: Block1 | Block2 | Block3
|
18
|
+
def |(block)
|
19
|
+
ParallelBlock.new(self, block)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Syntactic sugar of the form: Block1 + Block2 + Block3
|
23
|
+
def +(block)
|
24
|
+
SerialBlock.new(self, block)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
class NilClass
|
30
|
+
# Syntactic sugar of the form: nil | Block1
|
31
|
+
def |(block)
|
32
|
+
raise NoMethodError unless Block > block.class
|
33
|
+
block
|
34
|
+
end
|
35
|
+
|
36
|
+
# Syntactic sugar of the form: nil + Block1
|
37
|
+
def +(block)
|
38
|
+
raise NoMethodError unless Block > block.class
|
39
|
+
block
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'fileutils'
|
3
|
+
require_relative 'block'
|
4
|
+
require_relative 'worker'
|
5
|
+
require_relative 'worker_migration'
|
6
|
+
require_relative 'working_block'
|
7
|
+
require_relative 'liquid_block'
|
8
|
+
require_relative 'task'
|
9
|
+
require_relative 'task_migration'
|
10
|
+
|
11
|
+
module Ripe
|
12
|
+
class Controller
|
13
|
+
def initialize
|
14
|
+
@repository_path = '.ripe'
|
15
|
+
@has_repository = Dir.exists? @repository_path
|
16
|
+
end
|
17
|
+
|
18
|
+
def attach
|
19
|
+
ActiveRecord::Base.establish_connection({
|
20
|
+
adapter: 'sqlite3',
|
21
|
+
database: "#{@repository_path}/meta.db"
|
22
|
+
})
|
23
|
+
end
|
24
|
+
|
25
|
+
def attach_or_create
|
26
|
+
@has_repository ? attach : create
|
27
|
+
end
|
28
|
+
|
29
|
+
def create
|
30
|
+
FileUtils.mkdir_p(@repository_path)
|
31
|
+
@has_repository = true
|
32
|
+
|
33
|
+
begin
|
34
|
+
attach
|
35
|
+
WorkerMigration.up
|
36
|
+
TaskMigration.up
|
37
|
+
rescue
|
38
|
+
destroy
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def destroy
|
43
|
+
FileUtils.rm("#{@repository_path}/meta.db") if File.exists? "#{@repository_path}/meta.db"
|
44
|
+
FileUtils.rm("#{@repository_path}/workers") if Dir.exists? "#{@repository_path}/workers"
|
45
|
+
end
|
46
|
+
|
47
|
+
def prepare(samples, callback, vars = {})
|
48
|
+
Worker.prepare(samples, callback, vars)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'liquid'
|
2
|
+
require_relative 'working_block'
|
3
|
+
|
4
|
+
module Ripe
|
5
|
+
class LiquidBlock < WorkingBlock
|
6
|
+
def initialize(filename, vars = {})
|
7
|
+
super(filename, vars)
|
8
|
+
end
|
9
|
+
|
10
|
+
def command
|
11
|
+
vars = @vars.inject({}) { |memo, (k, v)| memo[k.to_s] = v; memo }
|
12
|
+
|
13
|
+
template = Liquid::Template.parse(File.new(@filename).read)
|
14
|
+
template.render(vars)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require_relative 'block'
|
2
|
+
|
3
|
+
module Ripe
|
4
|
+
# Forward declaration to prevent cyclic dependencies
|
5
|
+
class Block; end
|
6
|
+
|
7
|
+
class MultiBlock < Block
|
8
|
+
def initialize(id, *blocks)
|
9
|
+
# Ignore nil objects
|
10
|
+
super(id, blocks.compact, {})
|
11
|
+
end
|
12
|
+
|
13
|
+
def prune(protect, depend)
|
14
|
+
return self if protect
|
15
|
+
|
16
|
+
@blocks = @blocks.map { |block| block.prune(protect, depend) }.compact
|
17
|
+
case @blocks.length
|
18
|
+
when 0
|
19
|
+
nil
|
20
|
+
when 1
|
21
|
+
@blocks.first
|
22
|
+
else
|
23
|
+
self
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def topology
|
28
|
+
[@id] + @blocks.map(&:topology)
|
29
|
+
end
|
30
|
+
|
31
|
+
def targets_exist?
|
32
|
+
@blocks.map(&:targets_exist?).inject(:&)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require_relative 'multi_block'
|
2
|
+
|
3
|
+
module Ripe
|
4
|
+
class SerialBlock < MultiBlock
|
5
|
+
def initialize(*blocks)
|
6
|
+
super(:+, *blocks)
|
7
|
+
end
|
8
|
+
|
9
|
+
def command
|
10
|
+
@blocks.map { |block| "(\n%s\n)" % block.command }.join(' ; ')
|
11
|
+
end
|
12
|
+
|
13
|
+
alias :super_prune :prune
|
14
|
+
|
15
|
+
def prune(protect, depend)
|
16
|
+
return super_prune(protect, depend) if !depend
|
17
|
+
return self if protect
|
18
|
+
|
19
|
+
@blocks = @blocks.map do |block|
|
20
|
+
new_protect = !block.targets_exist?
|
21
|
+
new_block = block.prune(protect, depend)
|
22
|
+
protect = new_protect
|
23
|
+
new_block
|
24
|
+
end
|
25
|
+
@blocks = @blocks.compact
|
26
|
+
|
27
|
+
case @blocks.length
|
28
|
+
when 0
|
29
|
+
nil
|
30
|
+
when 1
|
31
|
+
@blocks.first
|
32
|
+
else
|
33
|
+
self
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/ripe/task.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'fileutils'
|
3
|
+
require_relative 'worker'
|
4
|
+
|
5
|
+
module Ripe
|
6
|
+
class Task < ActiveRecord::Base
|
7
|
+
belongs_to :worker
|
8
|
+
|
9
|
+
def dir
|
10
|
+
"#{self.worker.dir}"
|
11
|
+
end
|
12
|
+
|
13
|
+
def log
|
14
|
+
"#{self.dir}/#{self.id}.log"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require_relative 'worker'
|
3
|
+
|
4
|
+
module Ripe
|
5
|
+
class TaskMigration < ActiveRecord::Migration
|
6
|
+
def self.up
|
7
|
+
create_table :tasks do |t|
|
8
|
+
t.belongs_to :worker
|
9
|
+
t.string :sample
|
10
|
+
t.string :block
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.down
|
15
|
+
drop_table :tasks
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
data/lib/ripe/version.rb
ADDED
data/lib/ripe/worker.rb
ADDED
@@ -0,0 +1,174 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'fileutils'
|
3
|
+
require_relative 'task'
|
4
|
+
|
5
|
+
module Ripe
|
6
|
+
class Worker < ActiveRecord::Base
|
7
|
+
has_many :tasks, dependent: :destroy
|
8
|
+
|
9
|
+
def dir
|
10
|
+
".ripe/workers/#{self.id}"
|
11
|
+
end
|
12
|
+
|
13
|
+
def sh
|
14
|
+
"#{self.dir}/job.sh"
|
15
|
+
end
|
16
|
+
|
17
|
+
def stdout
|
18
|
+
"#{self.dir}/job.stdout"
|
19
|
+
end
|
20
|
+
|
21
|
+
def stderr
|
22
|
+
"#{self.dir}/job.stderr"
|
23
|
+
end
|
24
|
+
|
25
|
+
after_create do
|
26
|
+
FileUtils.mkdir_p dir if !Dir.exists? dir
|
27
|
+
end
|
28
|
+
|
29
|
+
before_destroy do
|
30
|
+
FileUtils.rm_r dir if Dir.exists? dir
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.prepare(samples, callback, vars = {})
|
34
|
+
vars = {
|
35
|
+
wd: Dir.pwd,
|
36
|
+
mode: :patch,
|
37
|
+
group_num: 1,
|
38
|
+
}.merge(vars)
|
39
|
+
|
40
|
+
return if ![:patch, :force, :depend].include? vars[:mode].to_sym
|
41
|
+
|
42
|
+
samples = samples.map do |sample|
|
43
|
+
block = callback.call(sample, vars).prune(vars[:mode].to_sym == :force,
|
44
|
+
vars[:mode].to_sym == :depend)
|
45
|
+
if block != nil
|
46
|
+
puts "Preparing sample #{sample}"
|
47
|
+
[sample, block]
|
48
|
+
else
|
49
|
+
puts "Nothing to do for sample #{sample}"
|
50
|
+
nil
|
51
|
+
end
|
52
|
+
end
|
53
|
+
samples = samples.compact
|
54
|
+
|
55
|
+
samples.each_slice(vars[:group_num].to_i).map do |worker_samples|
|
56
|
+
worker = Worker.create(handle: vars[:handle])
|
57
|
+
|
58
|
+
blocks = worker_samples.map do |sample, block|
|
59
|
+
# Preorder traversal of blocks -- assign incremental numbers starting from
|
60
|
+
# 1 to each node as it is being traversed.
|
61
|
+
post_var_assign = lambda do |subblock|
|
62
|
+
if subblock.blocks.length == 0
|
63
|
+
task = worker.tasks.create({
|
64
|
+
sample: sample,
|
65
|
+
block: subblock.id,
|
66
|
+
})
|
67
|
+
subblock.vars.merge!(log: task.log)
|
68
|
+
else
|
69
|
+
subblock.blocks.each(&post_var_assign)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
post_var_assign.call(block)
|
74
|
+
block
|
75
|
+
end
|
76
|
+
|
77
|
+
vars = vars.merge({
|
78
|
+
name: worker.id,
|
79
|
+
stdout: worker.stdout,
|
80
|
+
stderr: worker.stderr,
|
81
|
+
command: SerialBlock.new(*blocks).command,
|
82
|
+
})
|
83
|
+
|
84
|
+
file = File.new(worker.sh, 'w')
|
85
|
+
file.puts LiquidBlock.new("#{PATH}/share/moab.sh", vars).command
|
86
|
+
file.close
|
87
|
+
|
88
|
+
worker.update({
|
89
|
+
status: :prepared,
|
90
|
+
ppn: vars[:ppn],
|
91
|
+
queue: vars[:queue],
|
92
|
+
walltime: vars[:walltime],
|
93
|
+
})
|
94
|
+
worker
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def self.sync
|
99
|
+
lists = {idle: '-i', blocked: '-b', active: '-r'}
|
100
|
+
lists = lists.map do |status, op|
|
101
|
+
showq = `showq -u $(whoami) #{op} | grep $(whoami)`.split("\n")
|
102
|
+
showq.map do |job|
|
103
|
+
{
|
104
|
+
moab_id: job[/^([0-9]+) /, 1],
|
105
|
+
time: job[/ ([0-9]{1,2}(\:[0-9]{2})+) /, 1],
|
106
|
+
status: status,
|
107
|
+
}
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
# Update status
|
112
|
+
lists = lists.inject(&:+).each do |job|
|
113
|
+
moab_id = job[:moab_id]
|
114
|
+
time = job[:time]
|
115
|
+
status = job[:status]
|
116
|
+
worker = Worker.find_by(moab_id: moab_id)
|
117
|
+
|
118
|
+
if worker
|
119
|
+
worker.update(time: time)
|
120
|
+
unless ['cancelled', status].include? worker.status
|
121
|
+
checkjob = `checkjob #{moab_id}`
|
122
|
+
worker.update({
|
123
|
+
host: checkjob[/Allocated Nodes:\n\[(.*):[0-9]+\]\n/, 1],
|
124
|
+
status: status, # Queued jobs that appear become either idle, blocked or active
|
125
|
+
})
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
# Mark workers that were previously in active, blocked or idle as completed
|
131
|
+
# if they cannot be found anymore.
|
132
|
+
jobs = lists.map { |job| job[:moab_id] }
|
133
|
+
Worker.where('status in (:statuses)',
|
134
|
+
:statuses => ['active', 'idle', 'blocked']).each do |worker|
|
135
|
+
if jobs.include? worker.moab_id
|
136
|
+
jobs.delete(worker.moab_id) # Remove from list
|
137
|
+
elsif (worker.status != 'cancelled')
|
138
|
+
if File.exists? worker.stdout
|
139
|
+
stdout = File.new(worker.stdout).readlines.join
|
140
|
+
else
|
141
|
+
stdout = ""
|
142
|
+
end
|
143
|
+
worker.update({
|
144
|
+
cpu_used: stdout[/Resources:[ \t]*cput=([0-9]{1,2}(\:[0-9]{2})+),/, 1],
|
145
|
+
exit_code: stdout[/Exit code:[ \t]*(.*)$/, 1],
|
146
|
+
host: stdout[/Nodes:[ \t]*(.*)$/, 1],
|
147
|
+
memory_used: stdout[/Resources:.*,mem=([0-9]*[a-zA-Z]*),/, 1],
|
148
|
+
time: stdout[/Resources:.*,walltime=([0-9]{1,2}(\:[0-9]{2})+)$/, 1],
|
149
|
+
status: :completed,
|
150
|
+
})
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def start!
|
156
|
+
raise "Worker #{id} could not be started: not prepared" unless self.status == 'prepared'
|
157
|
+
start
|
158
|
+
end
|
159
|
+
|
160
|
+
def start
|
161
|
+
update(status: :queueing, moab_id: `qsub '#{self.sh}'`.strip.split(/\./).first) # Send to queue first
|
162
|
+
end
|
163
|
+
|
164
|
+
def cancel!
|
165
|
+
raise "Worker #{id} could not be cancelled: not started" unless ['queueing', 'idle', 'blocked', 'active'].include? self.status
|
166
|
+
cancel
|
167
|
+
end
|
168
|
+
|
169
|
+
def cancel
|
170
|
+
`canceljob #{self.moab_id}`
|
171
|
+
update(status: :cancelled)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require_relative 'task'
|
3
|
+
|
4
|
+
module Ripe
|
5
|
+
class WorkerMigration < ActiveRecord::Migration
|
6
|
+
def self.up
|
7
|
+
create_table :workers do |t|
|
8
|
+
t.string :cpu_used
|
9
|
+
t.string :exit_code
|
10
|
+
t.string :handle
|
11
|
+
t.string :host
|
12
|
+
t.string :moab_id
|
13
|
+
t.string :memory_used
|
14
|
+
t.integer :ppn
|
15
|
+
t.string :queue
|
16
|
+
t.string :time
|
17
|
+
t.string :status, default: :unprepared
|
18
|
+
t.string :walltime
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.down
|
23
|
+
drop_table :workers
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require_relative 'block.rb'
|
2
|
+
|
3
|
+
module Ripe
|
4
|
+
class WorkingBlock < Block
|
5
|
+
def initialize(filename, vars = {})
|
6
|
+
@filename = filename
|
7
|
+
super(File.basename(@filename), [], vars)
|
8
|
+
end
|
9
|
+
|
10
|
+
def topology
|
11
|
+
[@id]
|
12
|
+
end
|
13
|
+
|
14
|
+
def command
|
15
|
+
declarations = vars.map do |key, value|
|
16
|
+
lh = key.upcase
|
17
|
+
rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" :
|
18
|
+
"\"#{value}\""
|
19
|
+
"#{lh}=#{rh}"
|
20
|
+
end
|
21
|
+
|
22
|
+
"\n# <#{id}>" +
|
23
|
+
("\n" * 2) + declarations.join("\n") +
|
24
|
+
("\n" * 2) + "exec 1>\"$LOG\" 2>&1" +
|
25
|
+
("\n" * 2) + File.new(@filename).read + "\necho \"##.DONE.##\"" +
|
26
|
+
("\n" * 2) + "# </#{id}>\n"
|
27
|
+
end
|
28
|
+
|
29
|
+
def prune(protect, depend)
|
30
|
+
targets_exist? && !protect ? nil : self
|
31
|
+
end
|
32
|
+
|
33
|
+
def targets_exist?
|
34
|
+
statuses = @vars.select { |key, _| !key[/^output_/].nil? }.values.flatten
|
35
|
+
targets_exist = statuses.map { |target| File.exists? target }.inject(:&)
|
36
|
+
|
37
|
+
# If there are no targets at all, then assume that all targets exist
|
38
|
+
targets_exist == nil ? true : targets_exist
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/ripe.rb
ADDED
data/ripe.gemspec
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ripe/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "ripe"
|
8
|
+
spec.version = Ripe::VERSION
|
9
|
+
spec.authors = ["Nicolas De Jay"]
|
10
|
+
spec.email = ["ndj@pinkfilter.org"]
|
11
|
+
spec.summary = %q{Abstraction layer between the MOAB/Torque stack and your pipeline.}
|
12
|
+
# spec.description = %q{Write a longer description. Optional.}
|
13
|
+
spec.homepage = "https://github.com/ndejay/ripe"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
|
24
|
+
spec.add_development_dependency "activerecord"
|
25
|
+
spec.add_development_dependency "fileutils"
|
26
|
+
spec.add_development_dependency "liquid"
|
27
|
+
spec.add_development_dependency "sqlite3"
|
28
|
+
spec.add_development_dependency "hirb"
|
29
|
+
spec.add_development_dependency "irbtools"
|
30
|
+
spec.add_development_dependency "thor"
|
31
|
+
spec.add_development_dependency "wirb"
|
32
|
+
|
33
|
+
spec.add_development_dependency "rspec"
|
34
|
+
spec.add_development_dependency "rspec-nc"
|
35
|
+
end
|
data/share/moab.sh
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
#PBS -N {{ name }}
|
4
|
+
#PBS -A {{ project_name }}
|
5
|
+
#PBS -q {{ queue }}
|
6
|
+
#PBS -l nodes={{ node_count }}:ppn={{ ppn }}
|
7
|
+
#PBS -l walltime={{ walltime }}
|
8
|
+
#PBS -o {{ stdout }}
|
9
|
+
#PBS -e {{ stderr }}
|
10
|
+
#PBS -V
|
11
|
+
|
12
|
+
cd "{{ wd }}"
|
13
|
+
|
14
|
+
{{ command }}
|
data/spec/ripe_spec.rb
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'ripe'
|
metadata
ADDED
@@ -0,0 +1,239 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ripe
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Nicolas De Jay
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-03-19 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.7'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: activerecord
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fileutils
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: liquid
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: sqlite3
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: hirb
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: irbtools
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: thor
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: wirb
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rspec
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: rspec-nc
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">="
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0'
|
181
|
+
description:
|
182
|
+
email:
|
183
|
+
- ndj@pinkfilter.org
|
184
|
+
executables:
|
185
|
+
- ripe
|
186
|
+
extensions: []
|
187
|
+
extra_rdoc_files: []
|
188
|
+
files:
|
189
|
+
- ".gitignore"
|
190
|
+
- ".travis.yml"
|
191
|
+
- Gemfile
|
192
|
+
- LICENSE.txt
|
193
|
+
- README.md
|
194
|
+
- Rakefile
|
195
|
+
- bin/ripe
|
196
|
+
- lib/ripe.rb
|
197
|
+
- lib/ripe/block.rb
|
198
|
+
- lib/ripe/controller.rb
|
199
|
+
- lib/ripe/liquid_block.rb
|
200
|
+
- lib/ripe/multi_block.rb
|
201
|
+
- lib/ripe/parallel_block.rb
|
202
|
+
- lib/ripe/serial_block.rb
|
203
|
+
- lib/ripe/task.rb
|
204
|
+
- lib/ripe/task_migration.rb
|
205
|
+
- lib/ripe/version.rb
|
206
|
+
- lib/ripe/worker.rb
|
207
|
+
- lib/ripe/worker_migration.rb
|
208
|
+
- lib/ripe/working_block.rb
|
209
|
+
- ripe.gemspec
|
210
|
+
- share/moab.sh
|
211
|
+
- spec/ripe_spec.rb
|
212
|
+
- spec/spec_helper.rb
|
213
|
+
homepage: https://github.com/ndejay/ripe
|
214
|
+
licenses:
|
215
|
+
- MIT
|
216
|
+
metadata: {}
|
217
|
+
post_install_message:
|
218
|
+
rdoc_options: []
|
219
|
+
require_paths:
|
220
|
+
- lib
|
221
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
222
|
+
requirements:
|
223
|
+
- - ">="
|
224
|
+
- !ruby/object:Gem::Version
|
225
|
+
version: '0'
|
226
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
227
|
+
requirements:
|
228
|
+
- - ">="
|
229
|
+
- !ruby/object:Gem::Version
|
230
|
+
version: '0'
|
231
|
+
requirements: []
|
232
|
+
rubyforge_project:
|
233
|
+
rubygems_version: 2.2.2
|
234
|
+
signing_key:
|
235
|
+
specification_version: 4
|
236
|
+
summary: Abstraction layer between the MOAB/Torque stack and your pipeline.
|
237
|
+
test_files:
|
238
|
+
- spec/ripe_spec.rb
|
239
|
+
- spec/spec_helper.rb
|