ripe 0.2.2 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.ruby-version +1 -1
  3. data/.travis.yml +3 -0
  4. data/CHANGELOG.md +13 -0
  5. data/lib/ripe/blocks.rb +2 -0
  6. data/lib/ripe/blocks/bash_block.rb +107 -0
  7. data/lib/ripe/blocks/erb_block.rb +89 -0
  8. data/lib/ripe/blocks/liquid_block.rb +23 -2
  9. data/lib/ripe/blocks/working_block.rb +15 -40
  10. data/lib/ripe/cli.rb +7 -2
  11. data/lib/ripe/cli/helper.rb +21 -0
  12. data/lib/ripe/db/worker.rb +12 -0
  13. data/lib/ripe/db/worker_migration.rb +2 -0
  14. data/lib/ripe/dsl/task_dsl.rb +17 -4
  15. data/lib/ripe/library.rb +4 -7
  16. data/lib/ripe/repo.rb +3 -0
  17. data/lib/ripe/version.rb +1 -1
  18. data/lib/ripe/worker_controller.rb +7 -1
  19. data/lib/ripe/worker_controller/preparer.rb +9 -6
  20. data/lib/ripe/worker_controller/syncer.rb +1 -1
  21. data/spec/cli_spec.rb +16 -0
  22. data/spec/library_spec.rb +9 -9
  23. data/spec/testpack.rb +3 -1
  24. data/spec/testpack/.ripe/tasks/foo.sh.erb +3 -0
  25. data/spec/testpack/.ripe/workers/1/1.sh +1 -0
  26. data/spec/testpack/.ripe/workers/1/2.sh +1 -0
  27. data/spec/testpack/.ripe/workers/1/3.sh +12 -0
  28. data/spec/testpack/.ripe/workers/2/4.sh +8 -7
  29. data/spec/testpack/.ripe/workers/2/5.sh +17 -0
  30. data/spec/testpack/.ripe/workers/2/6.sh +12 -0
  31. data/spec/testpack/.ripe/workers/3/{5.sh → 7.sh} +1 -0
  32. data/spec/testpack/.ripe/workers/3/{6.sh → 8.sh} +1 -0
  33. data/spec/testpack/.ripe/workers/3/9.sh +12 -0
  34. data/spec/testpack/.ripe/workflows/foobar.rb +7 -1
  35. data/spec/testpack/Sample1/foo_erb_output.txt +1 -0
  36. data/spec/testpack/Sample2/foo_erb_output.txt +1 -0
  37. data/spec/testpack/Sample3/foo_erb_output.txt +1 -0
  38. data/spec/worker_controller_spec.rb +6 -6
  39. metadata +25 -10
  40. data/spec/testpack/.ripe/workers/2/3.sh +0 -16
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 13032eb238736b94168961b94cfd79896ad89ffd
4
- data.tar.gz: 6951a4e3da93749f18e44618f7ddc617edafad44
3
+ metadata.gz: c591b62a4a100240062bdebd04a0a8f3bec68f6a
4
+ data.tar.gz: 7c976775445a90aac08c9b24d4bcfa660be4ac7f
5
5
  SHA512:
6
- metadata.gz: b3538313e05b7189027942f553b8886993ff361b49b99b61c8ca91af91baf319e95f51d511bb32556501ca3b7caa5d499d7acda98bf4e7318265596c2ece29c8
7
- data.tar.gz: 733a2265845c4c8049756d9e06f23337a034714a97d19d232db3c7052610eb322d23f5b79a55273ba95f9f40e95ba9fd5217ce50f9b4cd2552c498d1fa73696d
6
+ metadata.gz: 12e2e041891e65c853a7fbeb2b992bb8c47c7c6d947d49e686fcf62984ff5b2ca70b51d9a7c0c1c1fc9b612a8abd859a0d8bfae9c019e3d5bcf3addb81204324
7
+ data.tar.gz: 2a2747b8c3f31a32e329f368422664c7c2d7e0e1224e8d4bf6664e4865d9c8b07c23ef14a66d77170cc503fcd42de38b08d0154a461da821cb6b044b82845d57
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- ruby-2.1.5
1
+ 2.4.0
data/.travis.yml CHANGED
@@ -1,4 +1,7 @@
1
1
  language: ruby
2
+ rvm:
3
+ - ruby-2.4.0
4
+ - ruby-head
2
5
  cache: bundler
3
6
  script: bundle exec rake spec
4
7
  notifications:
data/CHANGELOG.md CHANGED
@@ -1,3 +1,16 @@
1
+ HEAD
2
+ ----
3
+
4
+ v0.3.0
5
+ ------
6
+
7
+ - [ndejay/ripe#13](http://github.com/ndejay/ripe/issues/13): Change the permissions of the database to the same as the user's umask.
8
+ - [ndejay/ripe#10](http://github.com/ndejay/ripe/issues/10): Support config file for workflows.
9
+ - [ndejay/ripe#8](http://github.com/ndejay/ripe/issues/8): Support ERB task templates.
10
+ - [ndejay/ripe#6](http://github.com/ndejay/ripe/issues/6): Track project_name and user in the ripe metadata db.
11
+ - Track status for workers started locally.
12
+ - Automatically create accessors for `DB::Worker#status`.
13
+
1
14
  v0.2.2
2
15
  ------
3
16
 
data/lib/ripe/blocks.rb CHANGED
@@ -10,4 +10,6 @@ require_relative 'blocks/block'
10
10
  require_relative 'blocks/parallel_block'
11
11
  require_relative 'blocks/serial_block'
12
12
  require_relative 'blocks/working_block'
13
+ require_relative 'blocks/bash_block'
14
+ require_relative 'blocks/erb_block'
13
15
  require_relative 'blocks/liquid_block'
@@ -0,0 +1,107 @@
1
+ require 'liquid'
2
+
3
+ module Ripe
4
+
5
+ module Blocks
6
+
7
+ ##
8
+ # This class represents a working block that should be processed using the
9
+ # Bash adaptor block. In +ripe+ <= 0.2.1, this templating system was the
10
+ # default behaviour of {WorkingBlock}.
11
+ #
12
+ # Keys with string values in the format of:
13
+ #
14
+ # vars["some_key"] = "value"
15
+ #
16
+ # are converted as follows:
17
+ #
18
+ # SOME_KEY="value"
19
+ #
20
+ # Keys with array values in the format of:
21
+ #
22
+ # vars["some_key"] = ["one", "two"]
23
+ #
24
+ # are converted as follows:
25
+ #
26
+ # SOME_KEY=("one" "two")
27
+ #
28
+ # @see Ripe::Blocks::WorkingBlock
29
+
30
+ class BashBlock < WorkingBlock
31
+
32
+ ##
33
+ # Create a new, empty {BashBlock}.
34
+ #
35
+ # @param filename [String] filename of the template file
36
+ # @param vars [Hash<Symbol, String>] key-value pairs
37
+
38
+ def initialize(filename, vars = {})
39
+ super(filename, vars)
40
+ end
41
+
42
+ ##
43
+ # Return working block +parameters+ as a sequence of bash variable
44
+ # assignments.
45
+ #
46
+ # @return [String] sequence of bash variable assignments
47
+
48
+ def declarations
49
+ vars.map do |key, value|
50
+ lh = key.upcase
51
+ rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" : "\"#{value}\""
52
+ "#{lh}=#{rh}"
53
+ end
54
+ end
55
+
56
+ ##
57
+ # (see Block#command)
58
+ #
59
+ # The resulting string contains the result of the application of
60
+ # parameters to the +task+ from which the {BashBlock} was defined.
61
+ #
62
+ # @see Ripe::DB::Task
63
+ # @see Ripe::DSL::TaskDSL
64
+
65
+ def command
66
+ <<-EOF.gsub(/^[ ]+/, '')
67
+
68
+ # <#{id}>
69
+
70
+ #{declarations.join("\n")}
71
+
72
+ exec 1>"$LOG" 2>&1
73
+
74
+ #{File.new(@filename).read}
75
+ echo "##.DONE.##"
76
+
77
+ # </#{id}>
78
+ EOF
79
+ end
80
+
81
+ ##
82
+ # Return string handle for referring to this type of `WorkingBlock`.
83
+ #
84
+ # @see Ripe::DSL::TaskDSL
85
+ #
86
+ # @return [String]
87
+
88
+ def self.id
89
+ 'bash'
90
+ end
91
+
92
+ ##
93
+ # Return expected file extension type for this type of `WorkingBlock`.
94
+ #
95
+ # @see Ripe::DSL::TaskDSL
96
+ #
97
+ # @return [String]
98
+
99
+ def self.extension
100
+ 'sh'
101
+ end
102
+
103
+ end
104
+
105
+ end
106
+
107
+ end
@@ -0,0 +1,89 @@
1
+ require 'erb'
2
+
3
+ module Ripe
4
+
5
+ module Blocks
6
+
7
+ ##
8
+ # This class represents a working block that should be processed using the
9
+ # ERB templating system.
10
+ #
11
+ # Keys defined as:
12
+ #
13
+ # vars["some_key"] = "value"
14
+ #
15
+ # can be substituted in the ERB template using:
16
+ #
17
+ # <%= vars.some_key %>
18
+ #
19
+ # @see Ripe::Blocks::ERBBlock
20
+
21
+ class ERBBlock < WorkingBlock
22
+ #
23
+ ##
24
+ # Create a new, empty {LiquidBlock}.
25
+ #
26
+ # @param filename [String] filename of the template file
27
+ # @param vars [Hash<Symbol, String>] key-value pairs
28
+
29
+ def initialize(filename, vars = {})
30
+ super(filename, vars)
31
+ end
32
+
33
+ ##
34
+ # (see Block#command)
35
+ #
36
+ # The resulting string contains the render result of the liquid template
37
+ # based on the parameters specified in +vars+.
38
+
39
+ def command
40
+ vars = @vars
41
+ vars.define_singleton_method(:get_binding) { binding } # Expose private method
42
+ vars.define_singleton_method(:method_missing) do |name|
43
+ return self[name] if key? name
44
+ self.each { |k,v| return v if k.to_s.to_sym == name }
45
+ # super.method_missing name
46
+ end
47
+
48
+ template = <<-EOF.gsub(/^[ ]+/, '')
49
+
50
+ # <#{id}>
51
+
52
+ exec 1>"<%= vars.log %>" 2>&1
53
+
54
+ #{File.new(@filename).read}
55
+ echo "##.DONE.##"
56
+
57
+ # </#{id}>
58
+ EOF
59
+
60
+ ERB.new(template).result(vars.get_binding)
61
+ end
62
+
63
+ ##
64
+ # Return string handle for referring to this type of `WorkingBlock`.
65
+ #
66
+ # @see Ripe::DSL::TaskDSL
67
+ #
68
+ # @return [String]
69
+
70
+ def self.id
71
+ 'erb'
72
+ end
73
+
74
+ ##
75
+ # Return expected file extension type for this type of `WorkingBlock`.
76
+ #
77
+ # @see Ripe::DSL::TaskDSL
78
+ #
79
+ # @return [String]
80
+
81
+ def self.extension
82
+ 'sh.erb'
83
+ end
84
+
85
+ end
86
+
87
+ end
88
+
89
+ end
@@ -6,8 +6,7 @@ module Ripe
6
6
 
7
7
  ##
8
8
  # This class represents a working block that should be processed using the
9
- # Liquid templating engine, rather than the simple +bash+ engine defined in
10
- # {WorkingBlock}.
9
+ # Liquid templating engine.
11
10
  #
12
11
  # @see Ripe::Blocks::WorkingBlock
13
12
 
@@ -43,6 +42,28 @@ module Ripe
43
42
  template.render(declarations)
44
43
  end
45
44
 
45
+ ##
46
+ # Return string handle for referring to this type of `WorkingBlock`.
47
+ #
48
+ # @see Ripe::DSL::TaskDSL
49
+ #
50
+ # @return [String]
51
+
52
+ def self.id
53
+ 'liquid'
54
+ end
55
+
56
+ ##
57
+ # Return expected file extension type for this type of `WorkingBlock`.
58
+ #
59
+ # @see Ripe::DSL::TaskDSL
60
+ #
61
+ # @return [String]
62
+
63
+ def self.extension
64
+ 'sh.liquid'
65
+ end
66
+
46
67
  end
47
68
 
48
69
  end
@@ -4,10 +4,15 @@ module Ripe
4
4
 
5
5
  ##
6
6
  # This class represents a {Ripe::CLI::TaskCLI} that has been parametrized.
7
- # In the block arborescence, {WorkingBlock}s are always leaf nodes.
7
+ # In the block arborescence, {WorkingBlock}s are always leaf nodes and therefore
8
+ # contain scripts to be executed.
9
+ #
10
+ # In order version of ripe (<= 0.2.1), {WorkingBlock} and {BashBlock} were
11
+ # considered the same entity.
8
12
  #
9
13
  # @see Ripe::CLI::TaskCLI
10
14
  # @see Ripe::WorkerController::Preparer#prepare
15
+ # @see Ripe::Blocks::BashBlock
11
16
 
12
17
  class WorkingBlock < Block
13
18
 
@@ -22,45 +27,6 @@ module Ripe
22
27
  super(File.basename(@filename), [], vars)
23
28
  end
24
29
 
25
- ##
26
- # Return working block +parameters+ as a sequence of bash variable
27
- # assignments.
28
- #
29
- # @return [String] sequence of bash variable assignments
30
-
31
- def declarations
32
- vars.map do |key, value|
33
- lh = key.upcase
34
- rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" : "\"#{value}\""
35
- "#{lh}=#{rh}"
36
- end
37
- end
38
-
39
- ##
40
- # (see Block#command)
41
- #
42
- # The resulting string contains the result of the application of
43
- # parameters to the +task+ from which the {WorkingBlock} was defined.
44
- #
45
- # @see Ripe::DB::Task
46
- # @see Ripe::DSL::TaskDSL
47
-
48
- def command
49
- <<-EOF.gsub(/^[ ]+/, '')
50
-
51
- # <#{id}>
52
-
53
- #{declarations.join("\n")}
54
-
55
- exec 1>"$LOG" 2>&1
56
-
57
- #{File.new(@filename).read}
58
- echo "##.DONE.##"
59
-
60
- # </#{id}>
61
- EOF
62
- end
63
-
64
30
  ##
65
31
  # (see Block#prune)
66
32
  #
@@ -96,6 +62,15 @@ module Ripe
96
62
  [@id]
97
63
  end
98
64
 
65
+ ##
66
+ # Collect a list of classes derived from `WorkingBlock`.
67
+ #
68
+ # @return Array<Class>
69
+
70
+ def self.subclasses
71
+ ObjectSpace.each_object(Class).select { |klass| klass < self }
72
+ end
73
+
99
74
  end
100
75
 
101
76
  end
data/lib/ripe/cli.rb CHANGED
@@ -71,6 +71,8 @@ module Ripe
71
71
  desc 'prepare SAMPLES', 'Prepare jobs from template workflow'
72
72
  option :workflow, :aliases => '-w', :type => :string, :required => true,
73
73
  :desc => 'Workflow to be applied'
74
+ option :config, :aliases => '-c', :type => :string, :required => false,
75
+ :desc => 'Config file for workflows.'
74
76
  option :options, :aliases => '-o', :type => :string, :required => false,
75
77
  :desc => 'Options', :default => ''
76
78
  option :start, :aliases => '-s', :type => :boolean, :required => false,
@@ -92,8 +94,11 @@ module Ripe
92
94
 
93
95
  abort 'No samples specified.' if samples.length == 0
94
96
 
95
- workers = repo.controller.prepare(options[:workflow], samples,
96
- Helper.parse_cli_opts(options[:options]))
97
+ config = options[:config] ? Helper.parse_config(options[:config]) : {}
98
+ workflow_options = config[options[:workflow].to_sym] ||= {}
99
+ workflow_options.merge!(Helper.parse_cli_opts(options[:options]))
100
+
101
+ workers = repo.controller.prepare(options[:workflow], samples, workflow_options)
97
102
 
98
103
  repo.controller.start(workers) if options[:start]
99
104
  end
@@ -24,6 +24,27 @@ module Ripe
24
24
  params.inject(&:merge) || {}
25
25
  end
26
26
 
27
+ ##
28
+ # Read and parse a json configuration file into a symbolized hash with
29
+ # the content of that file.
30
+ #
31
+ # @param filename [String] name of a configuration file in json format
32
+ # @return [Hash] a symbolized hash of the content of the json file, or
33
+ # nothing if the file was not found or ill defined.
34
+
35
+ def self.parse_config(filename)
36
+ begin
37
+ file = File.read(filename)
38
+ begin
39
+ JSON.parse(file, :symbolize_names => true)
40
+ rescue
41
+ abort "Configuration file found but ill defined: #{filename}"
42
+ end
43
+ rescue
44
+ abort "Configuration file specified but not found: #{filename}"
45
+ end
46
+ end
47
+
27
48
  end
28
49
 
29
50
  end
@@ -57,6 +57,18 @@ module Ripe
57
57
  FileUtils.rm_r dir if Dir.exists? dir
58
58
  end
59
59
 
60
+ # Automatically create accessors for `#status`.
61
+ #
62
+ # worker.status == :prepared
63
+ #
64
+ # becomes
65
+ #
66
+ # worker.prepared?
67
+
68
+ [:unprepared, :prepared, :queueing, :idle,
69
+ :blocked, :active, :active_local, :cancelled, :completed].map do |s|
70
+ define_method("#{s}?") { status.to_s == s.to_s }
71
+ end
60
72
  end
61
73
 
62
74
  end
@@ -24,6 +24,8 @@ module Ripe
24
24
  t.string :time
25
25
  t.string :status, default: :unprepared
26
26
  t.string :walltime
27
+ t.string :user
28
+ t.string :project_name
27
29
  end
28
30
  end
29
31
 
@@ -22,6 +22,10 @@ module Ripe
22
22
  # t.param :param2, 'val2'
23
23
  # end
24
24
  #
25
+ # foo = task 'foo', task: 'bash' do |t|
26
+ # t.param :param1, 'val1'
27
+ # end
28
+ #
25
29
  # It internally uses +Ripe::DSL::TaskDSL+ to provide the DSL.
26
30
  #
27
31
  # @see Ripe::DSL::TaskDSL
@@ -32,12 +36,21 @@ module Ripe
32
36
  # @param block [Proc] executes block in the context of +TaskDSL+
33
37
  # @return [WorkingBlock, nil]
34
38
 
35
- def task(handle, &block)
36
- filename = Library.find(:task, handle)
37
- abort "Could not find task #{handle}." if filename == nil
39
+ def task(handle, vars = {type: 'bash'}, &block)
40
+ # Obtain list of `WorkingBlock` types.
41
+ working_block = Blocks::WorkingBlock.subclasses.select { |k| k.id == vars[:type] }.first
42
+
43
+ filename = "#{handle}.#{working_block.extension}"
44
+ full_filename = Library.find(:task, filename)
45
+
46
+ if full_filename == nil
47
+ abort "Could not find task `#{filename}`."
48
+ # else
49
+ # puts "Found task `#{handle}` @ `#{full_filename}`."
50
+ end
38
51
 
39
52
  params = TaskDSL.new(handle, &block).params
40
- Blocks::WorkingBlock.new(filename, params)
53
+ working_block.new(full_filename, params)
41
54
  end
42
55
 
43
56
  ##
data/lib/ripe/library.rb CHANGED
@@ -27,17 +27,14 @@ module Ripe
27
27
  #
28
28
  # @param comp [Symbol] Type of component: either +:workflow+ or
29
29
  # +:task+.
30
- # @param handle [String] Name of component
30
+ # @param filename [String] Filename of component
31
31
  # @return [String, nil] Full path of the component if found, and +nil+
32
32
  # otherwise.
33
33
 
34
- def find(comp, handle)
35
- ext = { task: 'sh',
36
- workflow: 'rb' }
37
-
34
+ def find(comp, filename)
38
35
  search = paths.map do |path|
39
- filename = "#{path}/#{comp}s/#{handle}.#{ext[comp]}"
40
- (File.exists? filename) ? filename : nil
36
+ file = "#{path}/#{comp}s/#{filename}"
37
+ (File.exists? file) ? file : nil
41
38
  end
42
39
 
43
40
  search.compact.first
data/lib/ripe/repo.rb CHANGED
@@ -74,6 +74,9 @@ module Ripe
74
74
  # Create the tables
75
75
  DB::WorkerMigration.up
76
76
  DB::TaskMigration.up
77
+
78
+ # Set the database's permissions to the user's umask
79
+ FileUtils.chmod(0666 - File.umask(), DATABASE_PATH)
77
80
  rescue
78
81
  destroy
79
82
  end
data/lib/ripe/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Ripe
2
- VERSION = "0.2.2"
2
+ VERSION = "0.3.0"
3
3
  end
@@ -48,7 +48,13 @@ module Ripe
48
48
 
49
49
  def local(workers)
50
50
  distribute workers do |worker|
51
+ worker.update(status: :active_local)
52
+
51
53
  `bash #{worker.sh}`
54
+ exit_code = $?.to_i
55
+
56
+ worker.update(status: :completed,
57
+ exit_code: exit_code)
52
58
  end
53
59
  end
54
60
 
@@ -60,7 +66,7 @@ module Ripe
60
66
 
61
67
  def start(workers)
62
68
  distribute workers do |worker|
63
- if worker.status == 'prepared'
69
+ if worker.prepared?
64
70
  worker.update(status: :queueing,
65
71
  moab_id: `qsub '#{worker.sh}'`.strip.split(/\./).first)
66
72
  else
@@ -56,7 +56,7 @@ module Ripe
56
56
  # and default params
57
57
 
58
58
  def load_workflow(workflow, params)
59
- filename = Library.find(:workflow, workflow)
59
+ filename = Library.find(:workflow, "#{workflow}.rb")
60
60
  abort "Could not find workflow #{workflow}." if filename == nil
61
61
  require_relative filename
62
62
 
@@ -131,10 +131,12 @@ module Ripe
131
131
  File.open(worker.sh, 'w') { |f| f.write(worker_block.command) }
132
132
 
133
133
  worker.update({
134
- status: :prepared,
135
- ppn: params[:ppn],
136
- queue: params[:queue],
137
- walltime: params[:walltime],
134
+ status: :prepared,
135
+ ppn: params[:ppn],
136
+ queue: params[:queue],
137
+ walltime: params[:walltime],
138
+ user: `whoami`.chomp,
139
+ project_name: params[:project_name],
138
140
  })
139
141
 
140
142
  worker
@@ -163,8 +165,9 @@ module Ripe
163
165
  block: subblock.id,
164
166
  })
165
167
 
166
- File.open(task.sh, 'w') { |f| f.write(subblock.command) }
167
168
  subblock.vars.merge!(log: task.log)
169
+ File.open(task.sh, 'w') { |f| f.write(subblock.command) }
170
+ subblock.vars
168
171
  else
169
172
  subblock.blocks.each(&post_var_assign)
170
173
  end
@@ -90,7 +90,7 @@ module Ripe
90
90
 
91
91
  @completed_workers = running_workers.select do |worker|
92
92
  !running_job_ids.include?(worker.moab_id) &&
93
- worker.status != 'cancelled'
93
+ !worker.cancelled? && worker.user == `whoami`.chomp
94
94
  end
95
95
  end
96
96
 
data/spec/cli_spec.rb CHANGED
@@ -11,4 +11,20 @@ describe CLI do
11
11
  expect(test_hash_opts).to eql ref_hash_opts
12
12
  end
13
13
  end
14
+
15
+ describe '::Helper#parse_config' do
16
+ it 'parses json file info symbolized hash' do
17
+ file = Tempfile.new("json")
18
+ begin
19
+ file.write("{\"a\": 1, \"b\": 2, \"c\": \"bob\"}")
20
+ file.rewind
21
+ test_hash_config = CLI::Helper.parse_config(file.path)
22
+ ref_hash_config = {:a=>1, :b=>2, :c=>"bob"}
23
+ expect(test_hash_config).to eql ref_hash_config
24
+ ensure
25
+ file.close
26
+ end
27
+
28
+ end
29
+ end
14
30
  end
data/spec/library_spec.rb CHANGED
@@ -12,9 +12,9 @@ describe Library do
12
12
  end
13
13
 
14
14
  it 'cannot resolve components of the test library' do
15
- expect(@library.find(:task, 'foo')).to eql nil
16
- expect(@library.find(:task, 'bar')).to eql nil
17
- expect(@library.find(:workflow, 'foobar')).to eql nil
15
+ expect(@library.find(:task, 'foo.sh')).to eql nil
16
+ expect(@library.find(:task, 'bar.sh')).to eql nil
17
+ expect(@library.find(:workflow, 'foobar.rb')).to eql nil
18
18
  end
19
19
  end
20
20
 
@@ -37,17 +37,17 @@ describe Library do
37
37
  end
38
38
 
39
39
  it 'resolves task components of the test library' do
40
- expect(@library.find(:task, 'foo')).to eql @test.tasks['foo']
41
- expect(@library.find(:task, 'bar')).to eql @test.tasks['bar']
40
+ expect(@library.find(:task, 'foo.sh')).to eql @test.tasks['foo']
41
+ expect(@library.find(:task, 'bar.sh')).to eql @test.tasks['bar']
42
42
  end
43
43
 
44
44
  it 'resolves workflows components of the test library' do
45
- expect(@library.find(:workflow, 'foobar')).to eql @test.workflows['foobar']
45
+ expect(@library.find(:workflow, 'foobar.rb')).to eql @test.workflows['foobar']
46
46
  end
47
47
 
48
- it 'cannot resolve non-existing componenets' do
49
- expect(@library.find(:task, 'other')).to eql nil
50
- expect(@library.find(:workflow, 'other')).to eql nil
48
+ it 'cannot resolve non-existing components' do
49
+ expect(@library.find(:task, 'other.sh')).to eql nil
50
+ expect(@library.find(:workflow, 'other.rb')).to eql nil
51
51
  end
52
52
  end
53
53
  end
data/spec/testpack.rb CHANGED
@@ -8,6 +8,7 @@ module Ripe
8
8
  @tasks = {
9
9
  'foo' => "#{@lib_path}/tasks/foo.sh",
10
10
  'bar' => "#{@lib_path}/tasks/bar.sh",
11
+ 'foo_erb' => "#{@lib_path}/tasks/foo_erb.sh"
11
12
  }
12
13
  @workflows = {
13
14
  'foobar' => "#{@lib_path}/workflows/foobar.rb",
@@ -20,7 +21,8 @@ module Ripe
20
21
  @steps = [
21
22
  'foo_input.txt',
22
23
  'foo_output.txt',
23
- 'bar_output.txt'
24
+ 'bar_output.txt',
25
+ 'foo_erb_output.txt'
24
26
  ]
25
27
  end
26
28
  end
@@ -0,0 +1,3 @@
1
+ # Foo is certainly one of the most important prerequisites to Bar.
2
+
3
+ echo "$(cat "<%= vars.input_foo %>") <%= vars.foo_message %>" > "<%= vars.output_foo %>"
@@ -4,6 +4,7 @@
4
4
  INPUT_FOO="Sample1/foo_input.txt"
5
5
  FOO_MESSAGE="For You"
6
6
  OUTPUT_FOO="Sample1/foo_output.txt"
7
+ LOG=".ripe/workers/1/1.log"
7
8
 
8
9
  exec 1>"$LOG" 2>&1
9
10
 
@@ -4,6 +4,7 @@
4
4
  INPUT_BAR="Sample1/foo_input.txt"
5
5
  BAR_MESSAGE="Bar"
6
6
  OUTPUT_BAR="Sample1/bar_output.txt"
7
+ LOG=".ripe/workers/1/2.log"
7
8
 
8
9
  exec 1>"$LOG" 2>&1
9
10
 
@@ -0,0 +1,12 @@
1
+
2
+ # <foo.sh.erb>
3
+
4
+ exec 1>".ripe/workers/1/3.log" 2>&1
5
+
6
+ # Foo is certainly one of the most important prerequisites to Bar.
7
+
8
+ echo "$(cat "Sample1/foo_input.txt") For You" > "Sample1/foo_erb_output.txt"
9
+
10
+ echo "##.DONE.##"
11
+
12
+ # </foo.sh.erb>
@@ -1,16 +1,17 @@
1
1
 
2
- # <bar.sh>
2
+ # <foo.sh>
3
3
 
4
- INPUT_BAR="Sample2/foo_input.txt"
5
- BAR_MESSAGE="Bar"
6
- OUTPUT_BAR="Sample2/bar_output.txt"
4
+ INPUT_FOO="Sample2/foo_input.txt"
5
+ FOO_MESSAGE="For You"
6
+ OUTPUT_FOO="Sample2/foo_output.txt"
7
+ LOG=".ripe/workers/2/4.log"
7
8
 
8
9
  exec 1>"$LOG" 2>&1
9
10
 
10
- # Bar is the most important consequence of Foo.
11
+ # Foo is certainly one of the most important prerequisites to Bar.
11
12
 
12
- echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
13
+ echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
13
14
 
14
15
  echo "##.DONE.##"
15
16
 
16
- # </bar.sh>
17
+ # </foo.sh>
@@ -0,0 +1,17 @@
1
+
2
+ # <bar.sh>
3
+
4
+ INPUT_BAR="Sample2/foo_input.txt"
5
+ BAR_MESSAGE="Bar"
6
+ OUTPUT_BAR="Sample2/bar_output.txt"
7
+ LOG=".ripe/workers/2/5.log"
8
+
9
+ exec 1>"$LOG" 2>&1
10
+
11
+ # Bar is the most important consequence of Foo.
12
+
13
+ echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
14
+
15
+ echo "##.DONE.##"
16
+
17
+ # </bar.sh>
@@ -0,0 +1,12 @@
1
+
2
+ # <foo.sh.erb>
3
+
4
+ exec 1>".ripe/workers/2/6.log" 2>&1
5
+
6
+ # Foo is certainly one of the most important prerequisites to Bar.
7
+
8
+ echo "$(cat "Sample2/foo_input.txt") For You" > "Sample2/foo_erb_output.txt"
9
+
10
+ echo "##.DONE.##"
11
+
12
+ # </foo.sh.erb>
@@ -4,6 +4,7 @@
4
4
  INPUT_FOO="Sample3/foo_input.txt"
5
5
  FOO_MESSAGE="For You"
6
6
  OUTPUT_FOO="Sample3/foo_output.txt"
7
+ LOG=".ripe/workers/3/7.log"
7
8
 
8
9
  exec 1>"$LOG" 2>&1
9
10
 
@@ -4,6 +4,7 @@
4
4
  INPUT_BAR="Sample3/foo_input.txt"
5
5
  BAR_MESSAGE="Bar"
6
6
  OUTPUT_BAR="Sample3/bar_output.txt"
7
+ LOG=".ripe/workers/3/8.log"
7
8
 
8
9
  exec 1>"$LOG" 2>&1
9
10
 
@@ -0,0 +1,12 @@
1
+
2
+ # <foo.sh.erb>
3
+
4
+ exec 1>".ripe/workers/3/9.log" 2>&1
5
+
6
+ # Foo is certainly one of the most important prerequisites to Bar.
7
+
8
+ echo "$(cat "Sample3/foo_input.txt") For You" > "Sample3/foo_erb_output.txt"
9
+
10
+ echo "##.DONE.##"
11
+
12
+ # </foo.sh.erb>
@@ -18,6 +18,12 @@ workflow 'foobar' do
18
18
  param :output_bar, "#{sample}/bar_output.txt"
19
19
  end
20
20
 
21
- foo + bar
21
+ foo_erb = task 'foo', type: 'erb' do
22
+ param :input_foo, "#{sample}/foo_input.txt"
23
+ param :foo_message, 'For You'
24
+ param :output_foo, "#{sample}/foo_erb_output.txt"
25
+ end
26
+
27
+ foo + (bar | foo_erb)
22
28
  end
23
29
  end
@@ -0,0 +1 @@
1
+ One Foo For You
@@ -0,0 +1 @@
1
+ Two Foo For You
@@ -0,0 +1 @@
1
+ Three Foo For You
@@ -6,7 +6,7 @@ require 'digest'
6
6
  require 'fileutils'
7
7
 
8
8
  def signature(file)
9
- Digest::MD5.hexdigest(File.read(file))
9
+ Digest::MD5.hexdigest(File.read(file).gsub(/LOG=.*\n/, '').gsub(/exec 1>".*\n/, '') )
10
10
  end
11
11
 
12
12
  describe WorkerController do
@@ -64,8 +64,8 @@ describe WorkerController do
64
64
  ref_tasks = DB::Worker.find(1).tasks
65
65
  test_tasks = DB::Worker.find(4).tasks
66
66
 
67
- expect(ref_tasks.length).to eql 2
68
- expect(test_tasks.length).to eql 2
67
+ expect(ref_tasks.length).to eql 3
68
+ expect(test_tasks.length).to eql 3
69
69
 
70
70
  ref_tasks.zip(test_tasks).map do |ref, test|
71
71
  ref_hash = signature(ref.sh)
@@ -86,7 +86,7 @@ describe WorkerController do
86
86
  ref_tasks = DB::Worker.find(1).tasks
87
87
  test_tasks = DB::Worker.find(5).tasks
88
88
 
89
- expect(ref_tasks.length).to eql 2
89
+ expect(ref_tasks.length).to eql 3
90
90
  expect(test_tasks.length).to eql 1
91
91
 
92
92
  ref_hash = signature(ref_tasks.first.sh)
@@ -107,8 +107,8 @@ describe WorkerController do
107
107
  ref_tasks = DB::Worker.find(2).tasks
108
108
  test_tasks = DB::Worker.find(6).tasks
109
109
 
110
- expect(ref_tasks.length).to eql 2
111
- expect(test_tasks.length).to eql 2
110
+ expect(ref_tasks.length).to eql 3
111
+ expect(test_tasks.length).to eql 3
112
112
 
113
113
  ref_tasks.zip(test_tasks).map do |ref, test|
114
114
  ref_hash = signature(ref.sh)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ripe
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nicolas De Jay
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-07-14 00:00:00.000000000 Z
11
+ date: 2017-01-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -226,7 +226,9 @@ files:
226
226
  - bin/ripe
227
227
  - lib/ripe.rb
228
228
  - lib/ripe/blocks.rb
229
+ - lib/ripe/blocks/bash_block.rb
229
230
  - lib/ripe/blocks/block.rb
231
+ - lib/ripe/blocks/erb_block.rb
230
232
  - lib/ripe/blocks/liquid_block.rb
231
233
  - lib/ripe/blocks/multi_block.rb
232
234
  - lib/ripe/blocks/parallel_block.rb
@@ -257,23 +259,30 @@ files:
257
259
  - spec/testpack/.ripe/meta.db
258
260
  - spec/testpack/.ripe/tasks/bar.sh
259
261
  - spec/testpack/.ripe/tasks/foo.sh
262
+ - spec/testpack/.ripe/tasks/foo.sh.erb
260
263
  - spec/testpack/.ripe/workers/1/1.sh
261
264
  - spec/testpack/.ripe/workers/1/2.sh
265
+ - spec/testpack/.ripe/workers/1/3.sh
262
266
  - spec/testpack/.ripe/workers/1/job.sh
263
- - spec/testpack/.ripe/workers/2/3.sh
264
267
  - spec/testpack/.ripe/workers/2/4.sh
268
+ - spec/testpack/.ripe/workers/2/5.sh
269
+ - spec/testpack/.ripe/workers/2/6.sh
265
270
  - spec/testpack/.ripe/workers/2/job.sh
266
- - spec/testpack/.ripe/workers/3/5.sh
267
- - spec/testpack/.ripe/workers/3/6.sh
271
+ - spec/testpack/.ripe/workers/3/7.sh
272
+ - spec/testpack/.ripe/workers/3/8.sh
273
+ - spec/testpack/.ripe/workers/3/9.sh
268
274
  - spec/testpack/.ripe/workers/3/job.sh
269
275
  - spec/testpack/.ripe/workflows/foobar.rb
270
276
  - spec/testpack/Sample1/bar_output.txt
277
+ - spec/testpack/Sample1/foo_erb_output.txt
271
278
  - spec/testpack/Sample1/foo_input.txt
272
279
  - spec/testpack/Sample1/foo_output.txt
273
280
  - spec/testpack/Sample2/bar_output.txt
281
+ - spec/testpack/Sample2/foo_erb_output.txt
274
282
  - spec/testpack/Sample2/foo_input.txt
275
283
  - spec/testpack/Sample2/foo_output.txt
276
284
  - spec/testpack/Sample3/bar_output.txt
285
+ - spec/testpack/Sample3/foo_erb_output.txt
277
286
  - spec/testpack/Sample3/foo_input.txt
278
287
  - spec/testpack/Sample3/foo_output.txt
279
288
  - spec/worker_controller_spec.rb
@@ -297,7 +306,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
297
306
  version: '0'
298
307
  requirements: []
299
308
  rubyforge_project:
300
- rubygems_version: 2.2.2
309
+ rubygems_version: 2.5.1
301
310
  signing_key:
302
311
  specification_version: 4
303
312
  summary: Abstraction layer between the MOAB/Torque stack and your pipeline.
@@ -309,24 +318,30 @@ test_files:
309
318
  - spec/testpack/.ripe/meta.db
310
319
  - spec/testpack/.ripe/tasks/bar.sh
311
320
  - spec/testpack/.ripe/tasks/foo.sh
321
+ - spec/testpack/.ripe/tasks/foo.sh.erb
312
322
  - spec/testpack/.ripe/workers/1/1.sh
313
323
  - spec/testpack/.ripe/workers/1/2.sh
324
+ - spec/testpack/.ripe/workers/1/3.sh
314
325
  - spec/testpack/.ripe/workers/1/job.sh
315
- - spec/testpack/.ripe/workers/2/3.sh
316
326
  - spec/testpack/.ripe/workers/2/4.sh
327
+ - spec/testpack/.ripe/workers/2/5.sh
328
+ - spec/testpack/.ripe/workers/2/6.sh
317
329
  - spec/testpack/.ripe/workers/2/job.sh
318
- - spec/testpack/.ripe/workers/3/5.sh
319
- - spec/testpack/.ripe/workers/3/6.sh
330
+ - spec/testpack/.ripe/workers/3/7.sh
331
+ - spec/testpack/.ripe/workers/3/8.sh
332
+ - spec/testpack/.ripe/workers/3/9.sh
320
333
  - spec/testpack/.ripe/workers/3/job.sh
321
334
  - spec/testpack/.ripe/workflows/foobar.rb
322
335
  - spec/testpack/Sample1/bar_output.txt
336
+ - spec/testpack/Sample1/foo_erb_output.txt
323
337
  - spec/testpack/Sample1/foo_input.txt
324
338
  - spec/testpack/Sample1/foo_output.txt
325
339
  - spec/testpack/Sample2/bar_output.txt
340
+ - spec/testpack/Sample2/foo_erb_output.txt
326
341
  - spec/testpack/Sample2/foo_input.txt
327
342
  - spec/testpack/Sample2/foo_output.txt
328
343
  - spec/testpack/Sample3/bar_output.txt
344
+ - spec/testpack/Sample3/foo_erb_output.txt
329
345
  - spec/testpack/Sample3/foo_input.txt
330
346
  - spec/testpack/Sample3/foo_output.txt
331
347
  - spec/worker_controller_spec.rb
332
- has_rdoc:
@@ -1,16 +0,0 @@
1
-
2
- # <foo.sh>
3
-
4
- INPUT_FOO="Sample2/foo_input.txt"
5
- FOO_MESSAGE="For You"
6
- OUTPUT_FOO="Sample2/foo_output.txt"
7
-
8
- exec 1>"$LOG" 2>&1
9
-
10
- # Foo is certainly one of the most important prerequisites to Bar.
11
-
12
- echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
13
-
14
- echo "##.DONE.##"
15
-
16
- # </foo.sh>