ripe 0.2.2 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/.travis.yml +3 -0
- data/CHANGELOG.md +13 -0
- data/lib/ripe/blocks.rb +2 -0
- data/lib/ripe/blocks/bash_block.rb +107 -0
- data/lib/ripe/blocks/erb_block.rb +89 -0
- data/lib/ripe/blocks/liquid_block.rb +23 -2
- data/lib/ripe/blocks/working_block.rb +15 -40
- data/lib/ripe/cli.rb +7 -2
- data/lib/ripe/cli/helper.rb +21 -0
- data/lib/ripe/db/worker.rb +12 -0
- data/lib/ripe/db/worker_migration.rb +2 -0
- data/lib/ripe/dsl/task_dsl.rb +17 -4
- data/lib/ripe/library.rb +4 -7
- data/lib/ripe/repo.rb +3 -0
- data/lib/ripe/version.rb +1 -1
- data/lib/ripe/worker_controller.rb +7 -1
- data/lib/ripe/worker_controller/preparer.rb +9 -6
- data/lib/ripe/worker_controller/syncer.rb +1 -1
- data/spec/cli_spec.rb +16 -0
- data/spec/library_spec.rb +9 -9
- data/spec/testpack.rb +3 -1
- data/spec/testpack/.ripe/tasks/foo.sh.erb +3 -0
- data/spec/testpack/.ripe/workers/1/1.sh +1 -0
- data/spec/testpack/.ripe/workers/1/2.sh +1 -0
- data/spec/testpack/.ripe/workers/1/3.sh +12 -0
- data/spec/testpack/.ripe/workers/2/4.sh +8 -7
- data/spec/testpack/.ripe/workers/2/5.sh +17 -0
- data/spec/testpack/.ripe/workers/2/6.sh +12 -0
- data/spec/testpack/.ripe/workers/3/{5.sh → 7.sh} +1 -0
- data/spec/testpack/.ripe/workers/3/{6.sh → 8.sh} +1 -0
- data/spec/testpack/.ripe/workers/3/9.sh +12 -0
- data/spec/testpack/.ripe/workflows/foobar.rb +7 -1
- data/spec/testpack/Sample1/foo_erb_output.txt +1 -0
- data/spec/testpack/Sample2/foo_erb_output.txt +1 -0
- data/spec/testpack/Sample3/foo_erb_output.txt +1 -0
- data/spec/worker_controller_spec.rb +6 -6
- metadata +25 -10
- data/spec/testpack/.ripe/workers/2/3.sh +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c591b62a4a100240062bdebd04a0a8f3bec68f6a
|
4
|
+
data.tar.gz: 7c976775445a90aac08c9b24d4bcfa660be4ac7f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 12e2e041891e65c853a7fbeb2b992bb8c47c7c6d947d49e686fcf62984ff5b2ca70b51d9a7c0c1c1fc9b612a8abd859a0d8bfae9c019e3d5bcf3addb81204324
|
7
|
+
data.tar.gz: 2a2747b8c3f31a32e329f368422664c7c2d7e0e1224e8d4bf6664e4865d9c8b07c23ef14a66d77170cc503fcd42de38b08d0154a461da821cb6b044b82845d57
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.4.0
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,16 @@
|
|
1
|
+
HEAD
|
2
|
+
----
|
3
|
+
|
4
|
+
v0.3.0
|
5
|
+
------
|
6
|
+
|
7
|
+
- [ndejay/ripe#13](http://github.com/ndejay/ripe/issues/13): Change the permissions of the database to the same as the user's umask.
|
8
|
+
- [ndejay/ripe#10](http://github.com/ndejay/ripe/issues/10): Support config file for workflows.
|
9
|
+
- [ndejay/ripe#8](http://github.com/ndejay/ripe/issues/8): Support ERB task templates.
|
10
|
+
- [ndejay/ripe#6](http://github.com/ndejay/ripe/issues/6): Track project_name and user in the ripe metadata db.
|
11
|
+
- Track status for workers started locally.
|
12
|
+
- Automatically create accessors for `DB::Worker#status`.
|
13
|
+
|
1
14
|
v0.2.2
|
2
15
|
------
|
3
16
|
|
data/lib/ripe/blocks.rb
CHANGED
@@ -10,4 +10,6 @@ require_relative 'blocks/block'
|
|
10
10
|
require_relative 'blocks/parallel_block'
|
11
11
|
require_relative 'blocks/serial_block'
|
12
12
|
require_relative 'blocks/working_block'
|
13
|
+
require_relative 'blocks/bash_block'
|
14
|
+
require_relative 'blocks/erb_block'
|
13
15
|
require_relative 'blocks/liquid_block'
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'liquid'
|
2
|
+
|
3
|
+
module Ripe
|
4
|
+
|
5
|
+
module Blocks
|
6
|
+
|
7
|
+
##
|
8
|
+
# This class represents a working block that should be processed using the
|
9
|
+
# Bash adaptor block. In +ripe+ <= 0.2.1, this templating system was the
|
10
|
+
# default behaviour of {WorkingBlock}.
|
11
|
+
#
|
12
|
+
# Keys with string values in the format of:
|
13
|
+
#
|
14
|
+
# vars["some_key"] = "value"
|
15
|
+
#
|
16
|
+
# are converted as follows:
|
17
|
+
#
|
18
|
+
# SOME_KEY="value"
|
19
|
+
#
|
20
|
+
# Keys with array values in the format of:
|
21
|
+
#
|
22
|
+
# vars["some_key"] = ["one", "two"]
|
23
|
+
#
|
24
|
+
# are converted as follows:
|
25
|
+
#
|
26
|
+
# SOME_KEY=("one" "two")
|
27
|
+
#
|
28
|
+
# @see Ripe::Blocks::WorkingBlock
|
29
|
+
|
30
|
+
class BashBlock < WorkingBlock
|
31
|
+
|
32
|
+
##
|
33
|
+
# Create a new, empty {BashBlock}.
|
34
|
+
#
|
35
|
+
# @param filename [String] filename of the template file
|
36
|
+
# @param vars [Hash<Symbol, String>] key-value pairs
|
37
|
+
|
38
|
+
def initialize(filename, vars = {})
|
39
|
+
super(filename, vars)
|
40
|
+
end
|
41
|
+
|
42
|
+
##
|
43
|
+
# Return working block +parameters+ as a sequence of bash variable
|
44
|
+
# assignments.
|
45
|
+
#
|
46
|
+
# @return [String] sequence of bash variable assignments
|
47
|
+
|
48
|
+
def declarations
|
49
|
+
vars.map do |key, value|
|
50
|
+
lh = key.upcase
|
51
|
+
rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" : "\"#{value}\""
|
52
|
+
"#{lh}=#{rh}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
##
|
57
|
+
# (see Block#command)
|
58
|
+
#
|
59
|
+
# The resulting string contains the result of the application of
|
60
|
+
# parameters to the +task+ from which the {BashBlock} was defined.
|
61
|
+
#
|
62
|
+
# @see Ripe::DB::Task
|
63
|
+
# @see Ripe::DSL::TaskDSL
|
64
|
+
|
65
|
+
def command
|
66
|
+
<<-EOF.gsub(/^[ ]+/, '')
|
67
|
+
|
68
|
+
# <#{id}>
|
69
|
+
|
70
|
+
#{declarations.join("\n")}
|
71
|
+
|
72
|
+
exec 1>"$LOG" 2>&1
|
73
|
+
|
74
|
+
#{File.new(@filename).read}
|
75
|
+
echo "##.DONE.##"
|
76
|
+
|
77
|
+
# </#{id}>
|
78
|
+
EOF
|
79
|
+
end
|
80
|
+
|
81
|
+
##
|
82
|
+
# Return string handle for referring to this type of `WorkingBlock`.
|
83
|
+
#
|
84
|
+
# @see Ripe::DSL::TaskDSL
|
85
|
+
#
|
86
|
+
# @return [String]
|
87
|
+
|
88
|
+
def self.id
|
89
|
+
'bash'
|
90
|
+
end
|
91
|
+
|
92
|
+
##
|
93
|
+
# Return expected file extension type for this type of `WorkingBlock`.
|
94
|
+
#
|
95
|
+
# @see Ripe::DSL::TaskDSL
|
96
|
+
#
|
97
|
+
# @return [String]
|
98
|
+
|
99
|
+
def self.extension
|
100
|
+
'sh'
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
|
105
|
+
end
|
106
|
+
|
107
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'erb'
|
2
|
+
|
3
|
+
module Ripe
|
4
|
+
|
5
|
+
module Blocks
|
6
|
+
|
7
|
+
##
|
8
|
+
# This class represents a working block that should be processed using the
|
9
|
+
# ERB templating system.
|
10
|
+
#
|
11
|
+
# Keys defined as:
|
12
|
+
#
|
13
|
+
# vars["some_key"] = "value"
|
14
|
+
#
|
15
|
+
# can be substituted in the ERB template using:
|
16
|
+
#
|
17
|
+
# <%= vars.some_key %>
|
18
|
+
#
|
19
|
+
# @see Ripe::Blocks::ERBBlock
|
20
|
+
|
21
|
+
class ERBBlock < WorkingBlock
|
22
|
+
#
|
23
|
+
##
|
24
|
+
# Create a new, empty {LiquidBlock}.
|
25
|
+
#
|
26
|
+
# @param filename [String] filename of the template file
|
27
|
+
# @param vars [Hash<Symbol, String>] key-value pairs
|
28
|
+
|
29
|
+
def initialize(filename, vars = {})
|
30
|
+
super(filename, vars)
|
31
|
+
end
|
32
|
+
|
33
|
+
##
|
34
|
+
# (see Block#command)
|
35
|
+
#
|
36
|
+
# The resulting string contains the render result of the liquid template
|
37
|
+
# based on the parameters specified in +vars+.
|
38
|
+
|
39
|
+
def command
|
40
|
+
vars = @vars
|
41
|
+
vars.define_singleton_method(:get_binding) { binding } # Expose private method
|
42
|
+
vars.define_singleton_method(:method_missing) do |name|
|
43
|
+
return self[name] if key? name
|
44
|
+
self.each { |k,v| return v if k.to_s.to_sym == name }
|
45
|
+
# super.method_missing name
|
46
|
+
end
|
47
|
+
|
48
|
+
template = <<-EOF.gsub(/^[ ]+/, '')
|
49
|
+
|
50
|
+
# <#{id}>
|
51
|
+
|
52
|
+
exec 1>"<%= vars.log %>" 2>&1
|
53
|
+
|
54
|
+
#{File.new(@filename).read}
|
55
|
+
echo "##.DONE.##"
|
56
|
+
|
57
|
+
# </#{id}>
|
58
|
+
EOF
|
59
|
+
|
60
|
+
ERB.new(template).result(vars.get_binding)
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# Return string handle for referring to this type of `WorkingBlock`.
|
65
|
+
#
|
66
|
+
# @see Ripe::DSL::TaskDSL
|
67
|
+
#
|
68
|
+
# @return [String]
|
69
|
+
|
70
|
+
def self.id
|
71
|
+
'erb'
|
72
|
+
end
|
73
|
+
|
74
|
+
##
|
75
|
+
# Return expected file extension type for this type of `WorkingBlock`.
|
76
|
+
#
|
77
|
+
# @see Ripe::DSL::TaskDSL
|
78
|
+
#
|
79
|
+
# @return [String]
|
80
|
+
|
81
|
+
def self.extension
|
82
|
+
'sh.erb'
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
@@ -6,8 +6,7 @@ module Ripe
|
|
6
6
|
|
7
7
|
##
|
8
8
|
# This class represents a working block that should be processed using the
|
9
|
-
# Liquid templating engine
|
10
|
-
# {WorkingBlock}.
|
9
|
+
# Liquid templating engine.
|
11
10
|
#
|
12
11
|
# @see Ripe::Blocks::WorkingBlock
|
13
12
|
|
@@ -43,6 +42,28 @@ module Ripe
|
|
43
42
|
template.render(declarations)
|
44
43
|
end
|
45
44
|
|
45
|
+
##
|
46
|
+
# Return string handle for referring to this type of `WorkingBlock`.
|
47
|
+
#
|
48
|
+
# @see Ripe::DSL::TaskDSL
|
49
|
+
#
|
50
|
+
# @return [String]
|
51
|
+
|
52
|
+
def self.id
|
53
|
+
'liquid'
|
54
|
+
end
|
55
|
+
|
56
|
+
##
|
57
|
+
# Return expected file extension type for this type of `WorkingBlock`.
|
58
|
+
#
|
59
|
+
# @see Ripe::DSL::TaskDSL
|
60
|
+
#
|
61
|
+
# @return [String]
|
62
|
+
|
63
|
+
def self.extension
|
64
|
+
'sh.liquid'
|
65
|
+
end
|
66
|
+
|
46
67
|
end
|
47
68
|
|
48
69
|
end
|
@@ -4,10 +4,15 @@ module Ripe
|
|
4
4
|
|
5
5
|
##
|
6
6
|
# This class represents a {Ripe::CLI::TaskCLI} that has been parametrized.
|
7
|
-
# In the block arborescence, {WorkingBlock}s are always leaf nodes
|
7
|
+
# In the block arborescence, {WorkingBlock}s are always leaf nodes and therefore
|
8
|
+
# contain scripts to be executed.
|
9
|
+
#
|
10
|
+
# In order version of ripe (<= 0.2.1), {WorkingBlock} and {BashBlock} were
|
11
|
+
# considered the same entity.
|
8
12
|
#
|
9
13
|
# @see Ripe::CLI::TaskCLI
|
10
14
|
# @see Ripe::WorkerController::Preparer#prepare
|
15
|
+
# @see Ripe::Blocks::BashBlock
|
11
16
|
|
12
17
|
class WorkingBlock < Block
|
13
18
|
|
@@ -22,45 +27,6 @@ module Ripe
|
|
22
27
|
super(File.basename(@filename), [], vars)
|
23
28
|
end
|
24
29
|
|
25
|
-
##
|
26
|
-
# Return working block +parameters+ as a sequence of bash variable
|
27
|
-
# assignments.
|
28
|
-
#
|
29
|
-
# @return [String] sequence of bash variable assignments
|
30
|
-
|
31
|
-
def declarations
|
32
|
-
vars.map do |key, value|
|
33
|
-
lh = key.upcase
|
34
|
-
rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" : "\"#{value}\""
|
35
|
-
"#{lh}=#{rh}"
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
##
|
40
|
-
# (see Block#command)
|
41
|
-
#
|
42
|
-
# The resulting string contains the result of the application of
|
43
|
-
# parameters to the +task+ from which the {WorkingBlock} was defined.
|
44
|
-
#
|
45
|
-
# @see Ripe::DB::Task
|
46
|
-
# @see Ripe::DSL::TaskDSL
|
47
|
-
|
48
|
-
def command
|
49
|
-
<<-EOF.gsub(/^[ ]+/, '')
|
50
|
-
|
51
|
-
# <#{id}>
|
52
|
-
|
53
|
-
#{declarations.join("\n")}
|
54
|
-
|
55
|
-
exec 1>"$LOG" 2>&1
|
56
|
-
|
57
|
-
#{File.new(@filename).read}
|
58
|
-
echo "##.DONE.##"
|
59
|
-
|
60
|
-
# </#{id}>
|
61
|
-
EOF
|
62
|
-
end
|
63
|
-
|
64
30
|
##
|
65
31
|
# (see Block#prune)
|
66
32
|
#
|
@@ -96,6 +62,15 @@ module Ripe
|
|
96
62
|
[@id]
|
97
63
|
end
|
98
64
|
|
65
|
+
##
|
66
|
+
# Collect a list of classes derived from `WorkingBlock`.
|
67
|
+
#
|
68
|
+
# @return Array<Class>
|
69
|
+
|
70
|
+
def self.subclasses
|
71
|
+
ObjectSpace.each_object(Class).select { |klass| klass < self }
|
72
|
+
end
|
73
|
+
|
99
74
|
end
|
100
75
|
|
101
76
|
end
|
data/lib/ripe/cli.rb
CHANGED
@@ -71,6 +71,8 @@ module Ripe
|
|
71
71
|
desc 'prepare SAMPLES', 'Prepare jobs from template workflow'
|
72
72
|
option :workflow, :aliases => '-w', :type => :string, :required => true,
|
73
73
|
:desc => 'Workflow to be applied'
|
74
|
+
option :config, :aliases => '-c', :type => :string, :required => false,
|
75
|
+
:desc => 'Config file for workflows.'
|
74
76
|
option :options, :aliases => '-o', :type => :string, :required => false,
|
75
77
|
:desc => 'Options', :default => ''
|
76
78
|
option :start, :aliases => '-s', :type => :boolean, :required => false,
|
@@ -92,8 +94,11 @@ module Ripe
|
|
92
94
|
|
93
95
|
abort 'No samples specified.' if samples.length == 0
|
94
96
|
|
95
|
-
|
96
|
-
|
97
|
+
config = options[:config] ? Helper.parse_config(options[:config]) : {}
|
98
|
+
workflow_options = config[options[:workflow].to_sym] ||= {}
|
99
|
+
workflow_options.merge!(Helper.parse_cli_opts(options[:options]))
|
100
|
+
|
101
|
+
workers = repo.controller.prepare(options[:workflow], samples, workflow_options)
|
97
102
|
|
98
103
|
repo.controller.start(workers) if options[:start]
|
99
104
|
end
|
data/lib/ripe/cli/helper.rb
CHANGED
@@ -24,6 +24,27 @@ module Ripe
|
|
24
24
|
params.inject(&:merge) || {}
|
25
25
|
end
|
26
26
|
|
27
|
+
##
|
28
|
+
# Read and parse a json configuration file into a symbolized hash with
|
29
|
+
# the content of that file.
|
30
|
+
#
|
31
|
+
# @param filename [String] name of a configuration file in json format
|
32
|
+
# @return [Hash] a symbolized hash of the content of the json file, or
|
33
|
+
# nothing if the file was not found or ill defined.
|
34
|
+
|
35
|
+
def self.parse_config(filename)
|
36
|
+
begin
|
37
|
+
file = File.read(filename)
|
38
|
+
begin
|
39
|
+
JSON.parse(file, :symbolize_names => true)
|
40
|
+
rescue
|
41
|
+
abort "Configuration file found but ill defined: #{filename}"
|
42
|
+
end
|
43
|
+
rescue
|
44
|
+
abort "Configuration file specified but not found: #{filename}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
27
48
|
end
|
28
49
|
|
29
50
|
end
|
data/lib/ripe/db/worker.rb
CHANGED
@@ -57,6 +57,18 @@ module Ripe
|
|
57
57
|
FileUtils.rm_r dir if Dir.exists? dir
|
58
58
|
end
|
59
59
|
|
60
|
+
# Automatically create accessors for `#status`.
|
61
|
+
#
|
62
|
+
# worker.status == :prepared
|
63
|
+
#
|
64
|
+
# becomes
|
65
|
+
#
|
66
|
+
# worker.prepared?
|
67
|
+
|
68
|
+
[:unprepared, :prepared, :queueing, :idle,
|
69
|
+
:blocked, :active, :active_local, :cancelled, :completed].map do |s|
|
70
|
+
define_method("#{s}?") { status.to_s == s.to_s }
|
71
|
+
end
|
60
72
|
end
|
61
73
|
|
62
74
|
end
|
data/lib/ripe/dsl/task_dsl.rb
CHANGED
@@ -22,6 +22,10 @@ module Ripe
|
|
22
22
|
# t.param :param2, 'val2'
|
23
23
|
# end
|
24
24
|
#
|
25
|
+
# foo = task 'foo', task: 'bash' do |t|
|
26
|
+
# t.param :param1, 'val1'
|
27
|
+
# end
|
28
|
+
#
|
25
29
|
# It internally uses +Ripe::DSL::TaskDSL+ to provide the DSL.
|
26
30
|
#
|
27
31
|
# @see Ripe::DSL::TaskDSL
|
@@ -32,12 +36,21 @@ module Ripe
|
|
32
36
|
# @param block [Proc] executes block in the context of +TaskDSL+
|
33
37
|
# @return [WorkingBlock, nil]
|
34
38
|
|
35
|
-
def task(handle, &block)
|
36
|
-
|
37
|
-
|
39
|
+
def task(handle, vars = {type: 'bash'}, &block)
|
40
|
+
# Obtain list of `WorkingBlock` types.
|
41
|
+
working_block = Blocks::WorkingBlock.subclasses.select { |k| k.id == vars[:type] }.first
|
42
|
+
|
43
|
+
filename = "#{handle}.#{working_block.extension}"
|
44
|
+
full_filename = Library.find(:task, filename)
|
45
|
+
|
46
|
+
if full_filename == nil
|
47
|
+
abort "Could not find task `#{filename}`."
|
48
|
+
# else
|
49
|
+
# puts "Found task `#{handle}` @ `#{full_filename}`."
|
50
|
+
end
|
38
51
|
|
39
52
|
params = TaskDSL.new(handle, &block).params
|
40
|
-
|
53
|
+
working_block.new(full_filename, params)
|
41
54
|
end
|
42
55
|
|
43
56
|
##
|
data/lib/ripe/library.rb
CHANGED
@@ -27,17 +27,14 @@ module Ripe
|
|
27
27
|
#
|
28
28
|
# @param comp [Symbol] Type of component: either +:workflow+ or
|
29
29
|
# +:task+.
|
30
|
-
# @param
|
30
|
+
# @param filename [String] Filename of component
|
31
31
|
# @return [String, nil] Full path of the component if found, and +nil+
|
32
32
|
# otherwise.
|
33
33
|
|
34
|
-
def find(comp,
|
35
|
-
ext = { task: 'sh',
|
36
|
-
workflow: 'rb' }
|
37
|
-
|
34
|
+
def find(comp, filename)
|
38
35
|
search = paths.map do |path|
|
39
|
-
|
40
|
-
(File.exists?
|
36
|
+
file = "#{path}/#{comp}s/#{filename}"
|
37
|
+
(File.exists? file) ? file : nil
|
41
38
|
end
|
42
39
|
|
43
40
|
search.compact.first
|
data/lib/ripe/repo.rb
CHANGED
data/lib/ripe/version.rb
CHANGED
@@ -48,7 +48,13 @@ module Ripe
|
|
48
48
|
|
49
49
|
def local(workers)
|
50
50
|
distribute workers do |worker|
|
51
|
+
worker.update(status: :active_local)
|
52
|
+
|
51
53
|
`bash #{worker.sh}`
|
54
|
+
exit_code = $?.to_i
|
55
|
+
|
56
|
+
worker.update(status: :completed,
|
57
|
+
exit_code: exit_code)
|
52
58
|
end
|
53
59
|
end
|
54
60
|
|
@@ -60,7 +66,7 @@ module Ripe
|
|
60
66
|
|
61
67
|
def start(workers)
|
62
68
|
distribute workers do |worker|
|
63
|
-
if worker.
|
69
|
+
if worker.prepared?
|
64
70
|
worker.update(status: :queueing,
|
65
71
|
moab_id: `qsub '#{worker.sh}'`.strip.split(/\./).first)
|
66
72
|
else
|
@@ -56,7 +56,7 @@ module Ripe
|
|
56
56
|
# and default params
|
57
57
|
|
58
58
|
def load_workflow(workflow, params)
|
59
|
-
filename = Library.find(:workflow, workflow)
|
59
|
+
filename = Library.find(:workflow, "#{workflow}.rb")
|
60
60
|
abort "Could not find workflow #{workflow}." if filename == nil
|
61
61
|
require_relative filename
|
62
62
|
|
@@ -131,10 +131,12 @@ module Ripe
|
|
131
131
|
File.open(worker.sh, 'w') { |f| f.write(worker_block.command) }
|
132
132
|
|
133
133
|
worker.update({
|
134
|
-
status:
|
135
|
-
ppn:
|
136
|
-
queue:
|
137
|
-
walltime:
|
134
|
+
status: :prepared,
|
135
|
+
ppn: params[:ppn],
|
136
|
+
queue: params[:queue],
|
137
|
+
walltime: params[:walltime],
|
138
|
+
user: `whoami`.chomp,
|
139
|
+
project_name: params[:project_name],
|
138
140
|
})
|
139
141
|
|
140
142
|
worker
|
@@ -163,8 +165,9 @@ module Ripe
|
|
163
165
|
block: subblock.id,
|
164
166
|
})
|
165
167
|
|
166
|
-
File.open(task.sh, 'w') { |f| f.write(subblock.command) }
|
167
168
|
subblock.vars.merge!(log: task.log)
|
169
|
+
File.open(task.sh, 'w') { |f| f.write(subblock.command) }
|
170
|
+
subblock.vars
|
168
171
|
else
|
169
172
|
subblock.blocks.each(&post_var_assign)
|
170
173
|
end
|
data/spec/cli_spec.rb
CHANGED
@@ -11,4 +11,20 @@ describe CLI do
|
|
11
11
|
expect(test_hash_opts).to eql ref_hash_opts
|
12
12
|
end
|
13
13
|
end
|
14
|
+
|
15
|
+
describe '::Helper#parse_config' do
|
16
|
+
it 'parses json file info symbolized hash' do
|
17
|
+
file = Tempfile.new("json")
|
18
|
+
begin
|
19
|
+
file.write("{\"a\": 1, \"b\": 2, \"c\": \"bob\"}")
|
20
|
+
file.rewind
|
21
|
+
test_hash_config = CLI::Helper.parse_config(file.path)
|
22
|
+
ref_hash_config = {:a=>1, :b=>2, :c=>"bob"}
|
23
|
+
expect(test_hash_config).to eql ref_hash_config
|
24
|
+
ensure
|
25
|
+
file.close
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
end
|
14
30
|
end
|
data/spec/library_spec.rb
CHANGED
@@ -12,9 +12,9 @@ describe Library do
|
|
12
12
|
end
|
13
13
|
|
14
14
|
it 'cannot resolve components of the test library' do
|
15
|
-
expect(@library.find(:task, 'foo')).to eql nil
|
16
|
-
expect(@library.find(:task, 'bar')).to eql nil
|
17
|
-
expect(@library.find(:workflow, 'foobar')).to eql nil
|
15
|
+
expect(@library.find(:task, 'foo.sh')).to eql nil
|
16
|
+
expect(@library.find(:task, 'bar.sh')).to eql nil
|
17
|
+
expect(@library.find(:workflow, 'foobar.rb')).to eql nil
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
@@ -37,17 +37,17 @@ describe Library do
|
|
37
37
|
end
|
38
38
|
|
39
39
|
it 'resolves task components of the test library' do
|
40
|
-
expect(@library.find(:task, 'foo')).to eql @test.tasks['foo']
|
41
|
-
expect(@library.find(:task, 'bar')).to eql @test.tasks['bar']
|
40
|
+
expect(@library.find(:task, 'foo.sh')).to eql @test.tasks['foo']
|
41
|
+
expect(@library.find(:task, 'bar.sh')).to eql @test.tasks['bar']
|
42
42
|
end
|
43
43
|
|
44
44
|
it 'resolves workflows components of the test library' do
|
45
|
-
expect(@library.find(:workflow, 'foobar')).to eql @test.workflows['foobar']
|
45
|
+
expect(@library.find(:workflow, 'foobar.rb')).to eql @test.workflows['foobar']
|
46
46
|
end
|
47
47
|
|
48
|
-
it 'cannot resolve non-existing
|
49
|
-
expect(@library.find(:task, 'other')).to eql nil
|
50
|
-
expect(@library.find(:workflow, 'other')).to eql nil
|
48
|
+
it 'cannot resolve non-existing components' do
|
49
|
+
expect(@library.find(:task, 'other.sh')).to eql nil
|
50
|
+
expect(@library.find(:workflow, 'other.rb')).to eql nil
|
51
51
|
end
|
52
52
|
end
|
53
53
|
end
|
data/spec/testpack.rb
CHANGED
@@ -8,6 +8,7 @@ module Ripe
|
|
8
8
|
@tasks = {
|
9
9
|
'foo' => "#{@lib_path}/tasks/foo.sh",
|
10
10
|
'bar' => "#{@lib_path}/tasks/bar.sh",
|
11
|
+
'foo_erb' => "#{@lib_path}/tasks/foo_erb.sh"
|
11
12
|
}
|
12
13
|
@workflows = {
|
13
14
|
'foobar' => "#{@lib_path}/workflows/foobar.rb",
|
@@ -20,7 +21,8 @@ module Ripe
|
|
20
21
|
@steps = [
|
21
22
|
'foo_input.txt',
|
22
23
|
'foo_output.txt',
|
23
|
-
'bar_output.txt'
|
24
|
+
'bar_output.txt',
|
25
|
+
'foo_erb_output.txt'
|
24
26
|
]
|
25
27
|
end
|
26
28
|
end
|
@@ -1,16 +1,17 @@
|
|
1
1
|
|
2
|
-
# <
|
2
|
+
# <foo.sh>
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
4
|
+
INPUT_FOO="Sample2/foo_input.txt"
|
5
|
+
FOO_MESSAGE="For You"
|
6
|
+
OUTPUT_FOO="Sample2/foo_output.txt"
|
7
|
+
LOG=".ripe/workers/2/4.log"
|
7
8
|
|
8
9
|
exec 1>"$LOG" 2>&1
|
9
10
|
|
10
|
-
#
|
11
|
+
# Foo is certainly one of the most important prerequisites to Bar.
|
11
12
|
|
12
|
-
echo "$(
|
13
|
+
echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
|
13
14
|
|
14
15
|
echo "##.DONE.##"
|
15
16
|
|
16
|
-
# </
|
17
|
+
# </foo.sh>
|
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
# <bar.sh>
|
3
|
+
|
4
|
+
INPUT_BAR="Sample2/foo_input.txt"
|
5
|
+
BAR_MESSAGE="Bar"
|
6
|
+
OUTPUT_BAR="Sample2/bar_output.txt"
|
7
|
+
LOG=".ripe/workers/2/5.log"
|
8
|
+
|
9
|
+
exec 1>"$LOG" 2>&1
|
10
|
+
|
11
|
+
# Bar is the most important consequence of Foo.
|
12
|
+
|
13
|
+
echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
|
14
|
+
|
15
|
+
echo "##.DONE.##"
|
16
|
+
|
17
|
+
# </bar.sh>
|
@@ -18,6 +18,12 @@ workflow 'foobar' do
|
|
18
18
|
param :output_bar, "#{sample}/bar_output.txt"
|
19
19
|
end
|
20
20
|
|
21
|
-
foo
|
21
|
+
foo_erb = task 'foo', type: 'erb' do
|
22
|
+
param :input_foo, "#{sample}/foo_input.txt"
|
23
|
+
param :foo_message, 'For You'
|
24
|
+
param :output_foo, "#{sample}/foo_erb_output.txt"
|
25
|
+
end
|
26
|
+
|
27
|
+
foo + (bar | foo_erb)
|
22
28
|
end
|
23
29
|
end
|
@@ -0,0 +1 @@
|
|
1
|
+
One Foo For You
|
@@ -0,0 +1 @@
|
|
1
|
+
Two Foo For You
|
@@ -0,0 +1 @@
|
|
1
|
+
Three Foo For You
|
@@ -6,7 +6,7 @@ require 'digest'
|
|
6
6
|
require 'fileutils'
|
7
7
|
|
8
8
|
def signature(file)
|
9
|
-
Digest::MD5.hexdigest(File.read(file))
|
9
|
+
Digest::MD5.hexdigest(File.read(file).gsub(/LOG=.*\n/, '').gsub(/exec 1>".*\n/, '') )
|
10
10
|
end
|
11
11
|
|
12
12
|
describe WorkerController do
|
@@ -64,8 +64,8 @@ describe WorkerController do
|
|
64
64
|
ref_tasks = DB::Worker.find(1).tasks
|
65
65
|
test_tasks = DB::Worker.find(4).tasks
|
66
66
|
|
67
|
-
expect(ref_tasks.length).to eql
|
68
|
-
expect(test_tasks.length).to eql
|
67
|
+
expect(ref_tasks.length).to eql 3
|
68
|
+
expect(test_tasks.length).to eql 3
|
69
69
|
|
70
70
|
ref_tasks.zip(test_tasks).map do |ref, test|
|
71
71
|
ref_hash = signature(ref.sh)
|
@@ -86,7 +86,7 @@ describe WorkerController do
|
|
86
86
|
ref_tasks = DB::Worker.find(1).tasks
|
87
87
|
test_tasks = DB::Worker.find(5).tasks
|
88
88
|
|
89
|
-
expect(ref_tasks.length).to eql
|
89
|
+
expect(ref_tasks.length).to eql 3
|
90
90
|
expect(test_tasks.length).to eql 1
|
91
91
|
|
92
92
|
ref_hash = signature(ref_tasks.first.sh)
|
@@ -107,8 +107,8 @@ describe WorkerController do
|
|
107
107
|
ref_tasks = DB::Worker.find(2).tasks
|
108
108
|
test_tasks = DB::Worker.find(6).tasks
|
109
109
|
|
110
|
-
expect(ref_tasks.length).to eql
|
111
|
-
expect(test_tasks.length).to eql
|
110
|
+
expect(ref_tasks.length).to eql 3
|
111
|
+
expect(test_tasks.length).to eql 3
|
112
112
|
|
113
113
|
ref_tasks.zip(test_tasks).map do |ref, test|
|
114
114
|
ref_hash = signature(ref.sh)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ripe
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicolas De Jay
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -226,7 +226,9 @@ files:
|
|
226
226
|
- bin/ripe
|
227
227
|
- lib/ripe.rb
|
228
228
|
- lib/ripe/blocks.rb
|
229
|
+
- lib/ripe/blocks/bash_block.rb
|
229
230
|
- lib/ripe/blocks/block.rb
|
231
|
+
- lib/ripe/blocks/erb_block.rb
|
230
232
|
- lib/ripe/blocks/liquid_block.rb
|
231
233
|
- lib/ripe/blocks/multi_block.rb
|
232
234
|
- lib/ripe/blocks/parallel_block.rb
|
@@ -257,23 +259,30 @@ files:
|
|
257
259
|
- spec/testpack/.ripe/meta.db
|
258
260
|
- spec/testpack/.ripe/tasks/bar.sh
|
259
261
|
- spec/testpack/.ripe/tasks/foo.sh
|
262
|
+
- spec/testpack/.ripe/tasks/foo.sh.erb
|
260
263
|
- spec/testpack/.ripe/workers/1/1.sh
|
261
264
|
- spec/testpack/.ripe/workers/1/2.sh
|
265
|
+
- spec/testpack/.ripe/workers/1/3.sh
|
262
266
|
- spec/testpack/.ripe/workers/1/job.sh
|
263
|
-
- spec/testpack/.ripe/workers/2/3.sh
|
264
267
|
- spec/testpack/.ripe/workers/2/4.sh
|
268
|
+
- spec/testpack/.ripe/workers/2/5.sh
|
269
|
+
- spec/testpack/.ripe/workers/2/6.sh
|
265
270
|
- spec/testpack/.ripe/workers/2/job.sh
|
266
|
-
- spec/testpack/.ripe/workers/3/
|
267
|
-
- spec/testpack/.ripe/workers/3/
|
271
|
+
- spec/testpack/.ripe/workers/3/7.sh
|
272
|
+
- spec/testpack/.ripe/workers/3/8.sh
|
273
|
+
- spec/testpack/.ripe/workers/3/9.sh
|
268
274
|
- spec/testpack/.ripe/workers/3/job.sh
|
269
275
|
- spec/testpack/.ripe/workflows/foobar.rb
|
270
276
|
- spec/testpack/Sample1/bar_output.txt
|
277
|
+
- spec/testpack/Sample1/foo_erb_output.txt
|
271
278
|
- spec/testpack/Sample1/foo_input.txt
|
272
279
|
- spec/testpack/Sample1/foo_output.txt
|
273
280
|
- spec/testpack/Sample2/bar_output.txt
|
281
|
+
- spec/testpack/Sample2/foo_erb_output.txt
|
274
282
|
- spec/testpack/Sample2/foo_input.txt
|
275
283
|
- spec/testpack/Sample2/foo_output.txt
|
276
284
|
- spec/testpack/Sample3/bar_output.txt
|
285
|
+
- spec/testpack/Sample3/foo_erb_output.txt
|
277
286
|
- spec/testpack/Sample3/foo_input.txt
|
278
287
|
- spec/testpack/Sample3/foo_output.txt
|
279
288
|
- spec/worker_controller_spec.rb
|
@@ -297,7 +306,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
297
306
|
version: '0'
|
298
307
|
requirements: []
|
299
308
|
rubyforge_project:
|
300
|
-
rubygems_version: 2.
|
309
|
+
rubygems_version: 2.5.1
|
301
310
|
signing_key:
|
302
311
|
specification_version: 4
|
303
312
|
summary: Abstraction layer between the MOAB/Torque stack and your pipeline.
|
@@ -309,24 +318,30 @@ test_files:
|
|
309
318
|
- spec/testpack/.ripe/meta.db
|
310
319
|
- spec/testpack/.ripe/tasks/bar.sh
|
311
320
|
- spec/testpack/.ripe/tasks/foo.sh
|
321
|
+
- spec/testpack/.ripe/tasks/foo.sh.erb
|
312
322
|
- spec/testpack/.ripe/workers/1/1.sh
|
313
323
|
- spec/testpack/.ripe/workers/1/2.sh
|
324
|
+
- spec/testpack/.ripe/workers/1/3.sh
|
314
325
|
- spec/testpack/.ripe/workers/1/job.sh
|
315
|
-
- spec/testpack/.ripe/workers/2/3.sh
|
316
326
|
- spec/testpack/.ripe/workers/2/4.sh
|
327
|
+
- spec/testpack/.ripe/workers/2/5.sh
|
328
|
+
- spec/testpack/.ripe/workers/2/6.sh
|
317
329
|
- spec/testpack/.ripe/workers/2/job.sh
|
318
|
-
- spec/testpack/.ripe/workers/3/
|
319
|
-
- spec/testpack/.ripe/workers/3/
|
330
|
+
- spec/testpack/.ripe/workers/3/7.sh
|
331
|
+
- spec/testpack/.ripe/workers/3/8.sh
|
332
|
+
- spec/testpack/.ripe/workers/3/9.sh
|
320
333
|
- spec/testpack/.ripe/workers/3/job.sh
|
321
334
|
- spec/testpack/.ripe/workflows/foobar.rb
|
322
335
|
- spec/testpack/Sample1/bar_output.txt
|
336
|
+
- spec/testpack/Sample1/foo_erb_output.txt
|
323
337
|
- spec/testpack/Sample1/foo_input.txt
|
324
338
|
- spec/testpack/Sample1/foo_output.txt
|
325
339
|
- spec/testpack/Sample2/bar_output.txt
|
340
|
+
- spec/testpack/Sample2/foo_erb_output.txt
|
326
341
|
- spec/testpack/Sample2/foo_input.txt
|
327
342
|
- spec/testpack/Sample2/foo_output.txt
|
328
343
|
- spec/testpack/Sample3/bar_output.txt
|
344
|
+
- spec/testpack/Sample3/foo_erb_output.txt
|
329
345
|
- spec/testpack/Sample3/foo_input.txt
|
330
346
|
- spec/testpack/Sample3/foo_output.txt
|
331
347
|
- spec/worker_controller_spec.rb
|
332
|
-
has_rdoc:
|
@@ -1,16 +0,0 @@
|
|
1
|
-
|
2
|
-
# <foo.sh>
|
3
|
-
|
4
|
-
INPUT_FOO="Sample2/foo_input.txt"
|
5
|
-
FOO_MESSAGE="For You"
|
6
|
-
OUTPUT_FOO="Sample2/foo_output.txt"
|
7
|
-
|
8
|
-
exec 1>"$LOG" 2>&1
|
9
|
-
|
10
|
-
# Foo is certainly one of the most important prerequisites to Bar.
|
11
|
-
|
12
|
-
echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
|
13
|
-
|
14
|
-
echo "##.DONE.##"
|
15
|
-
|
16
|
-
# </foo.sh>
|