ripe 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +0 -5
- data/Guardfile +2 -4
- data/README.md +1 -0
- data/bin/ripe +1 -59
- data/lib/ripe.rb +8 -1
- data/lib/ripe/blocks.rb +13 -0
- data/lib/ripe/blocks/block.rb +142 -0
- data/lib/ripe/blocks/liquid_block.rb +48 -0
- data/lib/ripe/blocks/multi_block.rb +71 -0
- data/lib/ripe/blocks/parallel_block.rb +29 -0
- data/lib/ripe/blocks/serial_block.rb +61 -0
- data/lib/ripe/blocks/working_block.rb +101 -0
- data/lib/ripe/cli.rb +121 -0
- data/lib/ripe/cli/helper.rb +31 -0
- data/lib/ripe/db.rb +7 -0
- data/lib/ripe/db/task.rb +42 -0
- data/lib/ripe/db/task_migration.rb +33 -0
- data/lib/ripe/db/worker.rb +64 -0
- data/lib/ripe/db/worker_migration.rb +41 -0
- data/lib/ripe/dsl.rb +2 -4
- data/lib/ripe/dsl/task_dsl.rb +4 -2
- data/lib/ripe/dsl/workflow_dsl.rb +5 -0
- data/lib/ripe/library.rb +34 -45
- data/lib/ripe/repo.rb +24 -23
- data/lib/ripe/version.rb +1 -1
- data/lib/ripe/worker_controller.rb +72 -144
- data/lib/ripe/worker_controller/preparer.rb +172 -0
- data/lib/ripe/worker_controller/syncer.rb +118 -0
- data/spec/cli_spec.rb +14 -0
- data/spec/library_spec.rb +18 -18
- data/spec/spec_helper.rb +2 -0
- data/spec/testpack.rb +16 -5
- data/spec/testpack/.ripe/meta.db +0 -0
- data/spec/testpack/.ripe/tasks/bar.sh +3 -0
- data/spec/testpack/{ripe → .ripe}/tasks/foo.sh +0 -0
- data/spec/testpack/.ripe/workers/1/1.sh +16 -0
- data/spec/testpack/.ripe/workers/1/2.sh +16 -0
- data/spec/testpack/.ripe/workers/1/job.sh +54 -0
- data/spec/testpack/.ripe/workers/2/3.sh +16 -0
- data/spec/testpack/.ripe/workers/2/4.sh +16 -0
- data/spec/testpack/.ripe/workers/2/job.sh +54 -0
- data/spec/testpack/.ripe/workers/3/5.sh +16 -0
- data/spec/testpack/.ripe/workers/3/6.sh +16 -0
- data/spec/testpack/.ripe/workers/3/job.sh +54 -0
- data/spec/testpack/.ripe/workflows/foobar.rb +23 -0
- data/spec/testpack/{case/Sample1 → Sample1}/bar_output.txt +0 -0
- data/spec/testpack/{case/Sample1 → Sample1}/foo_input.txt +0 -0
- data/spec/testpack/{case/Sample1 → Sample1}/foo_output.txt +0 -0
- data/spec/testpack/{case/Sample2 → Sample2}/bar_output.txt +0 -0
- data/spec/testpack/{case/Sample2 → Sample2}/foo_input.txt +0 -0
- data/spec/testpack/{case/Sample2 → Sample2}/foo_output.txt +0 -0
- data/spec/testpack/{case/Sample3 → Sample3}/bar_output.txt +0 -0
- data/spec/testpack/{case/Sample3 → Sample3}/foo_input.txt +0 -0
- data/spec/testpack/{case/Sample3 → Sample3}/foo_output.txt +0 -0
- data/spec/worker_controller_spec.rb +143 -0
- metadata +66 -40
- data/lib/ripe/block.rb +0 -41
- data/lib/ripe/liquid_block.rb +0 -17
- data/lib/ripe/multi_block.rb +0 -35
- data/lib/ripe/parallel_block.rb +0 -13
- data/lib/ripe/serial_block.rb +0 -37
- data/lib/ripe/task.rb +0 -21
- data/lib/ripe/task_migration.rb +0 -18
- data/lib/ripe/worker.rb +0 -44
- data/lib/ripe/worker_migration.rb +0 -26
- data/lib/ripe/working_block.rb +0 -41
- data/spec/block_spec.rb +0 -7
- data/spec/ripe_spec.rb +0 -7
- data/spec/testpack/ripe/tasks/bar.sh +0 -3
- data/spec/testpack/ripe/workflows/foobar.rb +0 -23
data/lib/ripe/block.rb
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
require_relative 'parallel_block'
|
|
2
|
-
require_relative 'serial_block'
|
|
3
|
-
|
|
4
|
-
module Ripe
|
|
5
|
-
class Block
|
|
6
|
-
attr_reader :id, :blocks
|
|
7
|
-
attr_accessor :vars
|
|
8
|
-
|
|
9
|
-
def initialize(id, blocks = [], vars = {})
|
|
10
|
-
@id, @blocks, @vars = id, blocks, vars
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
def prune(protect, depend)
|
|
14
|
-
self
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
# Syntactic sugar of the form: Block1 | Block2 | Block3
|
|
18
|
-
def |(block)
|
|
19
|
-
ParallelBlock.new(self, block)
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
# Syntactic sugar of the form: Block1 + Block2 + Block3
|
|
23
|
-
def +(block)
|
|
24
|
-
SerialBlock.new(self, block)
|
|
25
|
-
end
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
class NilClass
|
|
30
|
-
# Syntactic sugar of the form: nil | Block1
|
|
31
|
-
def |(block)
|
|
32
|
-
raise NoMethodError unless Block > block.class
|
|
33
|
-
block
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
# Syntactic sugar of the form: nil + Block1
|
|
37
|
-
def +(block)
|
|
38
|
-
raise NoMethodError unless Block > block.class
|
|
39
|
-
block
|
|
40
|
-
end
|
|
41
|
-
end
|
data/lib/ripe/liquid_block.rb
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
require 'liquid'
|
|
2
|
-
require_relative 'working_block'
|
|
3
|
-
|
|
4
|
-
module Ripe
|
|
5
|
-
class LiquidBlock < WorkingBlock
|
|
6
|
-
def initialize(filename, vars = {})
|
|
7
|
-
super(filename, vars)
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def command
|
|
11
|
-
vars = @vars.inject({}) { |memo, (k, v)| memo[k.to_s] = v; memo }
|
|
12
|
-
|
|
13
|
-
template = Liquid::Template.parse(File.new(@filename).read)
|
|
14
|
-
template.render(vars)
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
end
|
data/lib/ripe/multi_block.rb
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
require_relative 'block'
|
|
2
|
-
|
|
3
|
-
module Ripe
|
|
4
|
-
# Forward declaration to prevent cyclic dependencies
|
|
5
|
-
class Block; end
|
|
6
|
-
|
|
7
|
-
class MultiBlock < Block
|
|
8
|
-
def initialize(id, *blocks)
|
|
9
|
-
# Ignore nil objects
|
|
10
|
-
super(id, blocks.compact, {})
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
def prune(protect, depend)
|
|
14
|
-
return self if protect
|
|
15
|
-
|
|
16
|
-
@blocks = @blocks.map { |block| block.prune(protect, depend) }.compact
|
|
17
|
-
case @blocks.length
|
|
18
|
-
when 0
|
|
19
|
-
nil
|
|
20
|
-
when 1
|
|
21
|
-
@blocks.first
|
|
22
|
-
else
|
|
23
|
-
self
|
|
24
|
-
end
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def topology
|
|
28
|
-
[@id] + @blocks.map(&:topology)
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
def targets_exist?
|
|
32
|
-
@blocks.map(&:targets_exist?).inject(:&)
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
end
|
data/lib/ripe/parallel_block.rb
DELETED
data/lib/ripe/serial_block.rb
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
require_relative 'multi_block'
|
|
2
|
-
|
|
3
|
-
module Ripe
|
|
4
|
-
class SerialBlock < MultiBlock
|
|
5
|
-
def initialize(*blocks)
|
|
6
|
-
super(:+, *blocks)
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
def command
|
|
10
|
-
@blocks.map { |block| "(\n%s\n)" % block.command }.join(' ; ')
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
alias :super_prune :prune
|
|
14
|
-
|
|
15
|
-
def prune(protect, depend)
|
|
16
|
-
return super_prune(protect, depend) if !depend
|
|
17
|
-
return self if protect
|
|
18
|
-
|
|
19
|
-
@blocks = @blocks.map do |block|
|
|
20
|
-
new_protect = !block.targets_exist?
|
|
21
|
-
new_block = block.prune(protect, depend)
|
|
22
|
-
protect = new_protect
|
|
23
|
-
new_block
|
|
24
|
-
end
|
|
25
|
-
@blocks = @blocks.compact
|
|
26
|
-
|
|
27
|
-
case @blocks.length
|
|
28
|
-
when 0
|
|
29
|
-
nil
|
|
30
|
-
when 1
|
|
31
|
-
@blocks.first
|
|
32
|
-
else
|
|
33
|
-
self
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
end
|
data/lib/ripe/task.rb
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
require 'active_record'
|
|
2
|
-
require 'fileutils'
|
|
3
|
-
require_relative 'worker'
|
|
4
|
-
|
|
5
|
-
module Ripe
|
|
6
|
-
class Task < ActiveRecord::Base
|
|
7
|
-
belongs_to :worker
|
|
8
|
-
|
|
9
|
-
def dir
|
|
10
|
-
"#{self.worker.dir}"
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
def log
|
|
14
|
-
"#{self.dir}/#{self.id}.log"
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def sh
|
|
18
|
-
"#{self.dir}/#{self.id}.sh"
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
end
|
data/lib/ripe/task_migration.rb
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
require 'active_record'
|
|
2
|
-
require_relative 'worker'
|
|
3
|
-
|
|
4
|
-
module Ripe
|
|
5
|
-
class TaskMigration < ActiveRecord::Migration
|
|
6
|
-
def self.up
|
|
7
|
-
create_table :tasks do |t|
|
|
8
|
-
t.belongs_to :worker
|
|
9
|
-
t.string :sample
|
|
10
|
-
t.string :block
|
|
11
|
-
end
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def self.down
|
|
15
|
-
drop_table :tasks
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
end
|
data/lib/ripe/worker.rb
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
require 'active_record'
|
|
2
|
-
require 'fileutils'
|
|
3
|
-
require_relative 'task'
|
|
4
|
-
require_relative 'worker_controller'
|
|
5
|
-
|
|
6
|
-
module Ripe
|
|
7
|
-
class Worker < ActiveRecord::Base
|
|
8
|
-
has_many :tasks, dependent: :destroy
|
|
9
|
-
|
|
10
|
-
def dir
|
|
11
|
-
"#{Repo::REPOSITORY_PATH}/workers/#{self.id}"
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def sh
|
|
15
|
-
"#{self.dir}/job.sh"
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def stdout
|
|
19
|
-
"#{self.dir}/job.stdout"
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def stderr
|
|
23
|
-
"#{self.dir}/job.stderr"
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
after_create do
|
|
27
|
-
FileUtils.mkdir_p dir if !Dir.exists? dir
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
before_destroy do
|
|
31
|
-
FileUtils.rm_r dir if Dir.exists? dir
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def start
|
|
35
|
-
raise "Worker #{id} could not be started: not prepared" unless self.status == 'prepared'
|
|
36
|
-
WorkerController.instance.start(self)
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
def cancel
|
|
40
|
-
raise "Worker #{id} could not be cancelled: not started" unless ['queueing', 'idle', 'blocked', 'active'].include? self.status
|
|
41
|
-
WorkerController.instance.cancel(self)
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
|
-
end
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
require 'active_record'
|
|
2
|
-
require_relative 'task'
|
|
3
|
-
|
|
4
|
-
module Ripe
|
|
5
|
-
class WorkerMigration < ActiveRecord::Migration
|
|
6
|
-
def self.up
|
|
7
|
-
create_table :workers do |t|
|
|
8
|
-
t.string :cpu_used
|
|
9
|
-
t.string :exit_code
|
|
10
|
-
t.string :handle
|
|
11
|
-
t.string :host
|
|
12
|
-
t.string :moab_id
|
|
13
|
-
t.string :memory_used
|
|
14
|
-
t.integer :ppn
|
|
15
|
-
t.string :queue
|
|
16
|
-
t.string :time
|
|
17
|
-
t.string :status, default: :unprepared
|
|
18
|
-
t.string :walltime
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def self.down
|
|
23
|
-
drop_table :workers
|
|
24
|
-
end
|
|
25
|
-
end
|
|
26
|
-
end
|
data/lib/ripe/working_block.rb
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
require_relative 'block.rb'
|
|
2
|
-
|
|
3
|
-
module Ripe
|
|
4
|
-
class WorkingBlock < Block
|
|
5
|
-
def initialize(filename, vars = {})
|
|
6
|
-
@filename = filename
|
|
7
|
-
super(File.basename(@filename), [], vars)
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def topology
|
|
11
|
-
[@id]
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def command
|
|
15
|
-
declarations = vars.map do |key, value|
|
|
16
|
-
lh = key.upcase
|
|
17
|
-
rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" :
|
|
18
|
-
"\"#{value}\""
|
|
19
|
-
"#{lh}=#{rh}"
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
"\n# <#{id}>" +
|
|
23
|
-
("\n" * 2) + declarations.join("\n") +
|
|
24
|
-
("\n" * 2) + "exec 1>\"$LOG\" 2>&1" +
|
|
25
|
-
("\n" * 2) + File.new(@filename).read + "\necho \"##.DONE.##\"" +
|
|
26
|
-
("\n" * 2) + "# </#{id}>\n"
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def prune(protect, depend)
|
|
30
|
-
targets_exist? && !protect ? nil : self
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def targets_exist?
|
|
34
|
-
statuses = @vars.select { |key, _| !key[/^output_/].nil? }.values.flatten
|
|
35
|
-
targets_exist = statuses.map { |target| File.exists? target }.inject(:&)
|
|
36
|
-
|
|
37
|
-
# If there are no targets at all, then assume that all targets exist
|
|
38
|
-
targets_exist == nil ? true : targets_exist
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
end
|
data/spec/block_spec.rb
DELETED
data/spec/ripe_spec.rb
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
workflow 'foobar' do
|
|
2
|
-
param :node_count, 1
|
|
3
|
-
param :ppn, 8
|
|
4
|
-
param :project_name, 'abc-012-ab'
|
|
5
|
-
param :queue, 'queue'
|
|
6
|
-
param :walltime, '12:00:00'
|
|
7
|
-
|
|
8
|
-
describe do |sample, params|
|
|
9
|
-
foo = task 'foo' do
|
|
10
|
-
param :input_foo, "#{sample}/input_foo"
|
|
11
|
-
param :foo_message, 'FOO'
|
|
12
|
-
param :output_foo, "#{sample}/output_foo"
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
bar = task 'bar' do
|
|
16
|
-
param :input_bar, "#{sample}/input_bar"
|
|
17
|
-
param :bar_message, 'BAR'
|
|
18
|
-
param :output_bar, "#{sample}/output_bar"
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
foo + bar
|
|
22
|
-
end
|
|
23
|
-
end
|