ripe 0.2.0 → 0.2.1

data/spec/testpack/.ripe/workers/1/job.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+#PBS -N 1
+#PBS -A abc-012-ab
+#PBS -q debug
+#PBS -l nodes=1:ppn=1
+#PBS -l walltime=5:00
+#PBS -o .ripe/workers/1/job.stdout
+#PBS -e .ripe/workers/1/job.stderr
+#PBS -V
+
+cd "/Users/nicolas.dejay/Dropbox/Code/ripe/spec/testpack"
+
+(
+(
+
+# <foo.sh>
+
+INPUT_FOO="Sample1/foo_input.txt"
+FOO_MESSAGE="For You"
+OUTPUT_FOO="Sample1/foo_output.txt"
+LOG=".ripe/workers/1/1.log"
+
+exec 1>"$LOG" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
+
+echo "##.DONE.##"
+
+# </foo.sh>
+
+) ; (
+
+# <bar.sh>
+
+INPUT_BAR="Sample1/foo_input.txt"
+BAR_MESSAGE="Bar"
+OUTPUT_BAR="Sample1/bar_output.txt"
+LOG=".ripe/workers/1/2.log"
+
+exec 1>"$LOG" 2>&1
+
+# Bar is the most important consequence of Foo.
+
+echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
+
+echo "##.DONE.##"
+
+# </bar.sh>
+
+)
+)

data/spec/testpack/.ripe/workers/2/3.sh

@@ -0,0 +1,16 @@
+
+# <foo.sh>
+
+INPUT_FOO="Sample2/foo_input.txt"
+FOO_MESSAGE="For You"
+OUTPUT_FOO="Sample2/foo_output.txt"
+
+exec 1>"$LOG" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
+
+echo "##.DONE.##"
+
+# </foo.sh>

data/spec/testpack/.ripe/workers/2/4.sh

@@ -0,0 +1,16 @@
+
+# <bar.sh>
+
+INPUT_BAR="Sample2/foo_input.txt"
+BAR_MESSAGE="Bar"
+OUTPUT_BAR="Sample2/bar_output.txt"
+
+exec 1>"$LOG" 2>&1
+
+# Bar is the most important consequence of Foo.
+
+echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
+
+echo "##.DONE.##"
+
+# </bar.sh>

data/spec/testpack/.ripe/workers/2/job.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+#PBS -N 2
+#PBS -A abc-012-ab
+#PBS -q debug
+#PBS -l nodes=1:ppn=1
+#PBS -l walltime=5:00
+#PBS -o .ripe/workers/2/job.stdout
+#PBS -e .ripe/workers/2/job.stderr
+#PBS -V
+
+cd "/Users/nicolas.dejay/Dropbox/Code/ripe/spec/testpack"
+
+(
+(
+
+# <foo.sh>
+
+INPUT_FOO="Sample2/foo_input.txt"
+FOO_MESSAGE="For You"
+OUTPUT_FOO="Sample2/foo_output.txt"
+LOG=".ripe/workers/2/3.log"
+
+exec 1>"$LOG" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
+
+echo "##.DONE.##"
+
+# </foo.sh>
+
+) ; (
+
+# <bar.sh>
+
+INPUT_BAR="Sample2/foo_input.txt"
+BAR_MESSAGE="Bar"
+OUTPUT_BAR="Sample2/bar_output.txt"
+LOG=".ripe/workers/2/4.log"
+
+exec 1>"$LOG" 2>&1
+
+# Bar is the most important consequence of Foo.
+
+echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
+
+echo "##.DONE.##"
+
+# </bar.sh>
+
+)
+)

data/spec/testpack/.ripe/workers/3/5.sh

@@ -0,0 +1,16 @@
+
+# <foo.sh>
+
+INPUT_FOO="Sample3/foo_input.txt"
+FOO_MESSAGE="For You"
+OUTPUT_FOO="Sample3/foo_output.txt"
+
+exec 1>"$LOG" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
+
+echo "##.DONE.##"
+
+# </foo.sh>

data/spec/testpack/.ripe/workers/3/6.sh

@@ -0,0 +1,16 @@
+
+# <bar.sh>
+
+INPUT_BAR="Sample3/foo_input.txt"
+BAR_MESSAGE="Bar"
+OUTPUT_BAR="Sample3/bar_output.txt"
+
+exec 1>"$LOG" 2>&1
+
+# Bar is the most important consequence of Foo.
+
+echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
+
+echo "##.DONE.##"
+
+# </bar.sh>

data/spec/testpack/.ripe/workers/3/job.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+#PBS -N 3
+#PBS -A abc-012-ab
+#PBS -q debug
+#PBS -l nodes=1:ppn=1
+#PBS -l walltime=5:00
+#PBS -o .ripe/workers/3/job.stdout
+#PBS -e .ripe/workers/3/job.stderr
+#PBS -V
+
+cd "/Users/nicolas.dejay/Dropbox/Code/ripe/spec/testpack"
+
+(
+(
+
+# <foo.sh>
+
+INPUT_FOO="Sample3/foo_input.txt"
+FOO_MESSAGE="For You"
+OUTPUT_FOO="Sample3/foo_output.txt"
+LOG=".ripe/workers/3/5.log"
+
+exec 1>"$LOG" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
+
+echo "##.DONE.##"
+
+# </foo.sh>
+
+) ; (
+
+# <bar.sh>
+
+INPUT_BAR="Sample3/foo_input.txt"
+BAR_MESSAGE="Bar"
+OUTPUT_BAR="Sample3/bar_output.txt"
+LOG=".ripe/workers/3/6.log"
+
+exec 1>"$LOG" 2>&1
+
+# Bar is the most important consequence of Foo.
+
+echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
+
+echo "##.DONE.##"
+
+# </bar.sh>
+
+)
+)

data/spec/testpack/.ripe/workflows/foobar.rb

@@ -0,0 +1,23 @@
+workflow 'foobar' do
+  param :node_count,    1
+  param :ppn,           1
+  param :project_name,  'abc-012-ab'
+  param :queue,         'debug'
+  param :walltime,      '5:00'
+
+  describe do |sample, params|
+    foo = task 'foo' do
+      param :input_foo,   "#{sample}/foo_input.txt"
+      param :foo_message, 'For You'
+      param :output_foo,  "#{sample}/foo_output.txt"
+    end
+
+    bar = task 'bar' do
+      param :input_bar,   "#{sample}/foo_input.txt"
+      param :bar_message, 'Bar'
+      param :output_bar,  "#{sample}/bar_output.txt"
+    end
+
+    foo + bar
+  end
+end

data/spec/worker_controller_spec.rb

@@ -0,0 +1,143 @@
+require 'spec_helper'
+
+include Ripe::DSL # required by dirty hack +WorkerController#prepare+
+
+require 'digest'
+require 'fileutils'
+
+def signature(file)
+  Digest::MD5.hexdigest(File.read(file))
+end
+
+describe WorkerController do
+  context 'when RIPELIB contains the test library' do
+    before :all do
+      @test = TestPack.new
+
+      @oldwd  = Dir.pwd
+      @tmpdir = Dir.mktmpdir 'ripe'
+      Dir.chdir(@tmpdir)
+
+      ENV['RIPELIB'] = @test.lib_path
+      @repo = Repo.new
+      @repo.attach_or_create
+      @library = Library
+      @controller = @repo.controller
+
+      @test.samples.each do |sample|
+        FileUtils.mkdir_p(sample)
+        @test.steps.each do |step|
+          source = "#{@test.path}/#{sample}/#{step}"
+          dest = "#{sample}/#{step}"
+          FileUtils.cp(source, dest)
+        end
+      end
+    end
+
+    after :all do
+      FileUtils.rm_r("#{@tmpdir}")
+      Dir.chdir(@oldwd)
+    end
+
+    describe '#prepare' do
+      it 'prepares workers' do
+        workers = @controller.prepare 'foobar', @test.samples, pwd: @test.path, mode: :force
+        # Prepares workers 1-2-3
+        expect(DB::Worker.all.length).to eql 3
+        # Returns the workers from the call to prepare
+        expect(workers).to eql DB::Worker.all.to_a
+      end
+
+      it 'prepares workers with accurate task scripts' do
+        DB::Task.all.each do |task|
+          test_hash = signature(task.sh)
+          ref_hash = signature("#{@test.path}/#{task.sh}")
+          expect(test_hash).to eql ref_hash
+        end
+      end
+
+      it 'properly prepares workers in force mode' do
+        sample = @test.samples[0]
+
+        @controller.prepare 'foobar', [sample], pwd: @test.path, mode: :force
+
+        ref_tasks = DB::Worker.find(1).tasks
+        test_tasks = DB::Worker.find(4).tasks
+
+        expect(ref_tasks.length).to eql 2
+        expect(test_tasks.length).to eql 2
+
+        ref_tasks.zip(test_tasks).map do |ref, test|
+          ref_hash = signature(ref.sh)
+          test_hash = signature(test.sh)
+          expect(test_hash).to eql ref_hash
+        end
+      end
+
+      it 'properly prepares workers in patch mode' do
+        sample = @test.samples[0]
+        step = @test.steps[1]
+
+        # Delete the first output
+        FileUtils.rm_r("#{sample}/#{step}")
+
+        @controller.prepare 'foobar', [sample], pwd: @test.path, mode: :patch
+
+        ref_tasks = DB::Worker.find(1).tasks
+        test_tasks = DB::Worker.find(5).tasks
+
+        expect(ref_tasks.length).to eql 2
+        expect(test_tasks.length).to eql 1
+
+        ref_hash = signature(ref_tasks.first.sh)
+        test_hash = signature(test_tasks.first.sh)
+
+        expect(test_hash).to eql ref_hash
+      end
+
+      it 'properly prepares workers in depend mode' do
+        sample = @test.samples[1]
+        step = @test.steps[1]
+
+        # Delete the first output
+        FileUtils.rm_r("#{sample}/#{step}")
+
+        @controller.prepare 'foobar', [sample], pwd: @test.path, mode: :depend
+
+        ref_tasks = DB::Worker.find(2).tasks
+        test_tasks = DB::Worker.find(6).tasks
+
+        expect(ref_tasks.length).to eql 2
+        expect(test_tasks.length).to eql 2
+
+        ref_tasks.zip(test_tasks).map do |ref, test|
+          ref_hash = signature(ref.sh)
+          test_hash = signature(test.sh)
+          expect(test_hash).to eql ref_hash
+        end
+      end
+
+      describe '#local' do
+        it 'runs worker jobs locally' do
+          worker = DB::Worker.find(1)
+          @controller.local worker
+          @test.steps.map do |step|
+            test_hash = signature("#{@test.samples[0]}/#{step}")
+            ref_hash = signature("#{@test.path}/#{@test.samples[0]}/#{step}")
+            expect(test_hash).to eql ref_hash
+          end
+        end
+      end
+
+      describe '#sync' do
+        it 'does not affected unprepared workers' do
+          before_statuses = DB::Worker.where(status: :prepared).map(&:id)
+          @controller.sync
+          after_statuses = DB::Worker.where(status: :prepared).map(&:id)
+
+          expect(after_statuses).to eql before_statuses
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ripe
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Nicolas De Jay
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-15 00:00:00.000000000 Z
+date: 2015-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -224,42 +224,58 @@ files:
 - Rakefile
 - bin/ripe
 - lib/ripe.rb
-- lib/ripe/block.rb
+- lib/ripe/blocks.rb
+- lib/ripe/blocks/block.rb
+- lib/ripe/blocks/liquid_block.rb
+- lib/ripe/blocks/multi_block.rb
+- lib/ripe/blocks/parallel_block.rb
+- lib/ripe/blocks/serial_block.rb
+- lib/ripe/blocks/working_block.rb
+- lib/ripe/cli.rb
+- lib/ripe/cli/helper.rb
+- lib/ripe/db.rb
+- lib/ripe/db/task.rb
+- lib/ripe/db/task_migration.rb
+- lib/ripe/db/worker.rb
+- lib/ripe/db/worker_migration.rb
 - lib/ripe/dsl.rb
 - lib/ripe/dsl/task_dsl.rb
 - lib/ripe/dsl/workflow_dsl.rb
 - lib/ripe/library.rb
-- lib/ripe/liquid_block.rb
-- lib/ripe/multi_block.rb
-- lib/ripe/parallel_block.rb
 - lib/ripe/repo.rb
-- lib/ripe/serial_block.rb
-- lib/ripe/task.rb
-- lib/ripe/task_migration.rb
 - lib/ripe/version.rb
-- lib/ripe/worker.rb
 - lib/ripe/worker_controller.rb
-- lib/ripe/worker_migration.rb
-- lib/ripe/working_block.rb
+- lib/ripe/worker_controller/preparer.rb
+- lib/ripe/worker_controller/syncer.rb
 - ripe.gemspec
 - share/moab.sh
-- spec/block_spec.rb
+- spec/cli_spec.rb
 - spec/library_spec.rb
-- spec/ripe_spec.rb
 - spec/spec_helper.rb
 - spec/testpack.rb
-- spec/testpack/case/Sample1/bar_output.txt
-- spec/testpack/case/Sample1/foo_input.txt
-- spec/testpack/case/Sample1/foo_output.txt
-- spec/testpack/case/Sample2/bar_output.txt
-- spec/testpack/case/Sample2/foo_input.txt
-- spec/testpack/case/Sample2/foo_output.txt
-- spec/testpack/case/Sample3/bar_output.txt
-- spec/testpack/case/Sample3/foo_input.txt
-- spec/testpack/case/Sample3/foo_output.txt
-- spec/testpack/ripe/tasks/bar.sh
-- spec/testpack/ripe/tasks/foo.sh
-- spec/testpack/ripe/workflows/foobar.rb
+- spec/testpack/.ripe/meta.db
+- spec/testpack/.ripe/tasks/bar.sh
+- spec/testpack/.ripe/tasks/foo.sh
+- spec/testpack/.ripe/workers/1/1.sh
+- spec/testpack/.ripe/workers/1/2.sh
+- spec/testpack/.ripe/workers/1/job.sh
+- spec/testpack/.ripe/workers/2/3.sh
+- spec/testpack/.ripe/workers/2/4.sh
+- spec/testpack/.ripe/workers/2/job.sh
+- spec/testpack/.ripe/workers/3/5.sh
+- spec/testpack/.ripe/workers/3/6.sh
+- spec/testpack/.ripe/workers/3/job.sh
+- spec/testpack/.ripe/workflows/foobar.rb
+- spec/testpack/Sample1/bar_output.txt
+- spec/testpack/Sample1/foo_input.txt
+- spec/testpack/Sample1/foo_output.txt
+- spec/testpack/Sample2/bar_output.txt
+- spec/testpack/Sample2/foo_input.txt
+- spec/testpack/Sample2/foo_output.txt
+- spec/testpack/Sample3/bar_output.txt
+- spec/testpack/Sample3/foo_input.txt
+- spec/testpack/Sample3/foo_output.txt
+- spec/worker_controller_spec.rb
 homepage: https://github.com/ndejay/ripe
 licenses:
 - MIT
@@ -285,21 +301,31 @@ signing_key:
 specification_version: 4
 summary: Abstraction layer between the MOAB/Torque stack and your pipeline.
 test_files:
-- spec/block_spec.rb
+- spec/cli_spec.rb
 - spec/library_spec.rb
-- spec/ripe_spec.rb
 - spec/spec_helper.rb
 - spec/testpack.rb
-- spec/testpack/case/Sample1/bar_output.txt
-- spec/testpack/case/Sample1/foo_input.txt
-- spec/testpack/case/Sample1/foo_output.txt
-- spec/testpack/case/Sample2/bar_output.txt
-- spec/testpack/case/Sample2/foo_input.txt
-- spec/testpack/case/Sample2/foo_output.txt
-- spec/testpack/case/Sample3/bar_output.txt
-- spec/testpack/case/Sample3/foo_input.txt
-- spec/testpack/case/Sample3/foo_output.txt
-- spec/testpack/ripe/tasks/bar.sh
-- spec/testpack/ripe/tasks/foo.sh
-- spec/testpack/ripe/workflows/foobar.rb
+- spec/testpack/.ripe/meta.db
+- spec/testpack/.ripe/tasks/bar.sh
+- spec/testpack/.ripe/tasks/foo.sh
+- spec/testpack/.ripe/workers/1/1.sh
+- spec/testpack/.ripe/workers/1/2.sh
+- spec/testpack/.ripe/workers/1/job.sh
+- spec/testpack/.ripe/workers/2/3.sh
+- spec/testpack/.ripe/workers/2/4.sh
+- spec/testpack/.ripe/workers/2/job.sh
+- spec/testpack/.ripe/workers/3/5.sh
+- spec/testpack/.ripe/workers/3/6.sh
+- spec/testpack/.ripe/workers/3/job.sh
+- spec/testpack/.ripe/workflows/foobar.rb
+- spec/testpack/Sample1/bar_output.txt
+- spec/testpack/Sample1/foo_input.txt
+- spec/testpack/Sample1/foo_output.txt
+- spec/testpack/Sample2/bar_output.txt
+- spec/testpack/Sample2/foo_input.txt
+- spec/testpack/Sample2/foo_output.txt
+- spec/testpack/Sample3/bar_output.txt
+- spec/testpack/Sample3/foo_input.txt
+- spec/testpack/Sample3/foo_output.txt
+- spec/worker_controller_spec.rb
 has_rdoc: