ripe 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +0 -5
  3. data/Guardfile +2 -4
  4. data/README.md +1 -0
  5. data/bin/ripe +1 -59
  6. data/lib/ripe.rb +8 -1
  7. data/lib/ripe/blocks.rb +13 -0
  8. data/lib/ripe/blocks/block.rb +142 -0
  9. data/lib/ripe/blocks/liquid_block.rb +48 -0
  10. data/lib/ripe/blocks/multi_block.rb +71 -0
  11. data/lib/ripe/blocks/parallel_block.rb +29 -0
  12. data/lib/ripe/blocks/serial_block.rb +61 -0
  13. data/lib/ripe/blocks/working_block.rb +101 -0
  14. data/lib/ripe/cli.rb +121 -0
  15. data/lib/ripe/cli/helper.rb +31 -0
  16. data/lib/ripe/db.rb +7 -0
  17. data/lib/ripe/db/task.rb +42 -0
  18. data/lib/ripe/db/task_migration.rb +33 -0
  19. data/lib/ripe/db/worker.rb +64 -0
  20. data/lib/ripe/db/worker_migration.rb +41 -0
  21. data/lib/ripe/dsl.rb +2 -4
  22. data/lib/ripe/dsl/task_dsl.rb +4 -2
  23. data/lib/ripe/dsl/workflow_dsl.rb +5 -0
  24. data/lib/ripe/library.rb +34 -45
  25. data/lib/ripe/repo.rb +24 -23
  26. data/lib/ripe/version.rb +1 -1
  27. data/lib/ripe/worker_controller.rb +72 -144
  28. data/lib/ripe/worker_controller/preparer.rb +172 -0
  29. data/lib/ripe/worker_controller/syncer.rb +118 -0
  30. data/spec/cli_spec.rb +14 -0
  31. data/spec/library_spec.rb +18 -18
  32. data/spec/spec_helper.rb +2 -0
  33. data/spec/testpack.rb +16 -5
  34. data/spec/testpack/.ripe/meta.db +0 -0
  35. data/spec/testpack/.ripe/tasks/bar.sh +3 -0
  36. data/spec/testpack/{ripe → .ripe}/tasks/foo.sh +0 -0
  37. data/spec/testpack/.ripe/workers/1/1.sh +16 -0
  38. data/spec/testpack/.ripe/workers/1/2.sh +16 -0
  39. data/spec/testpack/.ripe/workers/1/job.sh +54 -0
  40. data/spec/testpack/.ripe/workers/2/3.sh +16 -0
  41. data/spec/testpack/.ripe/workers/2/4.sh +16 -0
  42. data/spec/testpack/.ripe/workers/2/job.sh +54 -0
  43. data/spec/testpack/.ripe/workers/3/5.sh +16 -0
  44. data/spec/testpack/.ripe/workers/3/6.sh +16 -0
  45. data/spec/testpack/.ripe/workers/3/job.sh +54 -0
  46. data/spec/testpack/.ripe/workflows/foobar.rb +23 -0
  47. data/spec/testpack/{case/Sample1 → Sample1}/bar_output.txt +0 -0
  48. data/spec/testpack/{case/Sample1 → Sample1}/foo_input.txt +0 -0
  49. data/spec/testpack/{case/Sample1 → Sample1}/foo_output.txt +0 -0
  50. data/spec/testpack/{case/Sample2 → Sample2}/bar_output.txt +0 -0
  51. data/spec/testpack/{case/Sample2 → Sample2}/foo_input.txt +0 -0
  52. data/spec/testpack/{case/Sample2 → Sample2}/foo_output.txt +0 -0
  53. data/spec/testpack/{case/Sample3 → Sample3}/bar_output.txt +0 -0
  54. data/spec/testpack/{case/Sample3 → Sample3}/foo_input.txt +0 -0
  55. data/spec/testpack/{case/Sample3 → Sample3}/foo_output.txt +0 -0
  56. data/spec/worker_controller_spec.rb +143 -0
  57. metadata +66 -40
  58. data/lib/ripe/block.rb +0 -41
  59. data/lib/ripe/liquid_block.rb +0 -17
  60. data/lib/ripe/multi_block.rb +0 -35
  61. data/lib/ripe/parallel_block.rb +0 -13
  62. data/lib/ripe/serial_block.rb +0 -37
  63. data/lib/ripe/task.rb +0 -21
  64. data/lib/ripe/task_migration.rb +0 -18
  65. data/lib/ripe/worker.rb +0 -44
  66. data/lib/ripe/worker_migration.rb +0 -26
  67. data/lib/ripe/working_block.rb +0 -41
  68. data/spec/block_spec.rb +0 -7
  69. data/spec/ripe_spec.rb +0 -7
  70. data/spec/testpack/ripe/tasks/bar.sh +0 -3
  71. data/spec/testpack/ripe/workflows/foobar.rb +0 -23
data/lib/ripe/block.rb DELETED
@@ -1,41 +0,0 @@
1
- require_relative 'parallel_block'
2
- require_relative 'serial_block'
3
-
4
- module Ripe
5
- class Block
6
- attr_reader :id, :blocks
7
- attr_accessor :vars
8
-
9
- def initialize(id, blocks = [], vars = {})
10
- @id, @blocks, @vars = id, blocks, vars
11
- end
12
-
13
- def prune(protect, depend)
14
- self
15
- end
16
-
17
- # Syntactic sugar of the form: Block1 | Block2 | Block3
18
- def |(block)
19
- ParallelBlock.new(self, block)
20
- end
21
-
22
- # Syntactic sugar of the form: Block1 + Block2 + Block3
23
- def +(block)
24
- SerialBlock.new(self, block)
25
- end
26
- end
27
- end
28
-
29
- class NilClass
30
- # Syntactic sugar of the form: nil | Block1
31
- def |(block)
32
- raise NoMethodError unless Block > block.class
33
- block
34
- end
35
-
36
- # Syntactic sugar of the form: nil + Block1
37
- def +(block)
38
- raise NoMethodError unless Block > block.class
39
- block
40
- end
41
- end
@@ -1,17 +0,0 @@
1
- require 'liquid'
2
- require_relative 'working_block'
3
-
4
- module Ripe
5
- class LiquidBlock < WorkingBlock
6
- def initialize(filename, vars = {})
7
- super(filename, vars)
8
- end
9
-
10
- def command
11
- vars = @vars.inject({}) { |memo, (k, v)| memo[k.to_s] = v; memo }
12
-
13
- template = Liquid::Template.parse(File.new(@filename).read)
14
- template.render(vars)
15
- end
16
- end
17
- end
@@ -1,35 +0,0 @@
1
- require_relative 'block'
2
-
3
- module Ripe
4
- # Forward declaration to prevent cyclic dependencies
5
- class Block; end
6
-
7
- class MultiBlock < Block
8
- def initialize(id, *blocks)
9
- # Ignore nil objects
10
- super(id, blocks.compact, {})
11
- end
12
-
13
- def prune(protect, depend)
14
- return self if protect
15
-
16
- @blocks = @blocks.map { |block| block.prune(protect, depend) }.compact
17
- case @blocks.length
18
- when 0
19
- nil
20
- when 1
21
- @blocks.first
22
- else
23
- self
24
- end
25
- end
26
-
27
- def topology
28
- [@id] + @blocks.map(&:topology)
29
- end
30
-
31
- def targets_exist?
32
- @blocks.map(&:targets_exist?).inject(:&)
33
- end
34
- end
35
- end
@@ -1,13 +0,0 @@
1
- require_relative 'multi_block'
2
-
3
- module Ripe
4
- class ParallelBlock < MultiBlock
5
- def initialize(*blocks)
6
- super(:|, *blocks)
7
- end
8
-
9
- def command
10
- @blocks.map { |block| "(\n%s\n) & " % block.command }.join('') + 'wait'
11
- end
12
- end
13
- end
@@ -1,37 +0,0 @@
1
- require_relative 'multi_block'
2
-
3
- module Ripe
4
- class SerialBlock < MultiBlock
5
- def initialize(*blocks)
6
- super(:+, *blocks)
7
- end
8
-
9
- def command
10
- @blocks.map { |block| "(\n%s\n)" % block.command }.join(' ; ')
11
- end
12
-
13
- alias :super_prune :prune
14
-
15
- def prune(protect, depend)
16
- return super_prune(protect, depend) if !depend
17
- return self if protect
18
-
19
- @blocks = @blocks.map do |block|
20
- new_protect = !block.targets_exist?
21
- new_block = block.prune(protect, depend)
22
- protect = new_protect
23
- new_block
24
- end
25
- @blocks = @blocks.compact
26
-
27
- case @blocks.length
28
- when 0
29
- nil
30
- when 1
31
- @blocks.first
32
- else
33
- self
34
- end
35
- end
36
- end
37
- end
data/lib/ripe/task.rb DELETED
@@ -1,21 +0,0 @@
1
- require 'active_record'
2
- require 'fileutils'
3
- require_relative 'worker'
4
-
5
- module Ripe
6
- class Task < ActiveRecord::Base
7
- belongs_to :worker
8
-
9
- def dir
10
- "#{self.worker.dir}"
11
- end
12
-
13
- def log
14
- "#{self.dir}/#{self.id}.log"
15
- end
16
-
17
- def sh
18
- "#{self.dir}/#{self.id}.sh"
19
- end
20
- end
21
- end
@@ -1,18 +0,0 @@
1
- require 'active_record'
2
- require_relative 'worker'
3
-
4
- module Ripe
5
- class TaskMigration < ActiveRecord::Migration
6
- def self.up
7
- create_table :tasks do |t|
8
- t.belongs_to :worker
9
- t.string :sample
10
- t.string :block
11
- end
12
- end
13
-
14
- def self.down
15
- drop_table :tasks
16
- end
17
- end
18
- end
data/lib/ripe/worker.rb DELETED
@@ -1,44 +0,0 @@
1
- require 'active_record'
2
- require 'fileutils'
3
- require_relative 'task'
4
- require_relative 'worker_controller'
5
-
6
- module Ripe
7
- class Worker < ActiveRecord::Base
8
- has_many :tasks, dependent: :destroy
9
-
10
- def dir
11
- "#{Repo::REPOSITORY_PATH}/workers/#{self.id}"
12
- end
13
-
14
- def sh
15
- "#{self.dir}/job.sh"
16
- end
17
-
18
- def stdout
19
- "#{self.dir}/job.stdout"
20
- end
21
-
22
- def stderr
23
- "#{self.dir}/job.stderr"
24
- end
25
-
26
- after_create do
27
- FileUtils.mkdir_p dir if !Dir.exists? dir
28
- end
29
-
30
- before_destroy do
31
- FileUtils.rm_r dir if Dir.exists? dir
32
- end
33
-
34
- def start
35
- raise "Worker #{id} could not be started: not prepared" unless self.status == 'prepared'
36
- WorkerController.instance.start(self)
37
- end
38
-
39
- def cancel
40
- raise "Worker #{id} could not be cancelled: not started" unless ['queueing', 'idle', 'blocked', 'active'].include? self.status
41
- WorkerController.instance.cancel(self)
42
- end
43
- end
44
- end
@@ -1,26 +0,0 @@
1
- require 'active_record'
2
- require_relative 'task'
3
-
4
- module Ripe
5
- class WorkerMigration < ActiveRecord::Migration
6
- def self.up
7
- create_table :workers do |t|
8
- t.string :cpu_used
9
- t.string :exit_code
10
- t.string :handle
11
- t.string :host
12
- t.string :moab_id
13
- t.string :memory_used
14
- t.integer :ppn
15
- t.string :queue
16
- t.string :time
17
- t.string :status, default: :unprepared
18
- t.string :walltime
19
- end
20
- end
21
-
22
- def self.down
23
- drop_table :workers
24
- end
25
- end
26
- end
@@ -1,41 +0,0 @@
1
- require_relative 'block.rb'
2
-
3
- module Ripe
4
- class WorkingBlock < Block
5
- def initialize(filename, vars = {})
6
- @filename = filename
7
- super(File.basename(@filename), [], vars)
8
- end
9
-
10
- def topology
11
- [@id]
12
- end
13
-
14
- def command
15
- declarations = vars.map do |key, value|
16
- lh = key.upcase
17
- rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" :
18
- "\"#{value}\""
19
- "#{lh}=#{rh}"
20
- end
21
-
22
- "\n# <#{id}>" +
23
- ("\n" * 2) + declarations.join("\n") +
24
- ("\n" * 2) + "exec 1>\"$LOG\" 2>&1" +
25
- ("\n" * 2) + File.new(@filename).read + "\necho \"##.DONE.##\"" +
26
- ("\n" * 2) + "# </#{id}>\n"
27
- end
28
-
29
- def prune(protect, depend)
30
- targets_exist? && !protect ? nil : self
31
- end
32
-
33
- def targets_exist?
34
- statuses = @vars.select { |key, _| !key[/^output_/].nil? }.values.flatten
35
- targets_exist = statuses.map { |target| File.exists? target }.inject(:&)
36
-
37
- # If there are no targets at all, then assume that all targets exist
38
- targets_exist == nil ? true : targets_exist
39
- end
40
- end
41
- end
data/spec/block_spec.rb DELETED
@@ -1,7 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Ripe::Block do
4
- it 'does stuff' do
5
- true
6
- end
7
- end
data/spec/ripe_spec.rb DELETED
@@ -1,7 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Ripe do
4
- it 'does stuff' do
5
- true
6
- end
7
- end
@@ -1,3 +0,0 @@
1
- # Bar is the most important consequence of Foo.
2
-
3
- echo "$(cut -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
@@ -1,23 +0,0 @@
1
- workflow 'foobar' do
2
- param :node_count, 1
3
- param :ppn, 8
4
- param :project_name, 'abc-012-ab'
5
- param :queue, 'queue'
6
- param :walltime, '12:00:00'
7
-
8
- describe do |sample, params|
9
- foo = task 'foo' do
10
- param :input_foo, "#{sample}/input_foo"
11
- param :foo_message, 'FOO'
12
- param :output_foo, "#{sample}/output_foo"
13
- end
14
-
15
- bar = task 'bar' do
16
- param :input_bar, "#{sample}/input_bar"
17
- param :bar_message, 'BAR'
18
- param :output_bar, "#{sample}/output_bar"
19
- end
20
-
21
- foo + bar
22
- end
23
- end