ripe 0.2.0 → 0.2.1

data/lib/ripe/worker_controller/preparer.rb

@@ -0,0 +1,172 @@
+module Ripe
+
+  class WorkerController
+
+    ##
+    # This class controls worker preparation from a given workflow, list of
+    # samples and parameters.  It applies the workflow to each of the specified
+    # samples.
+    #
+    # @attr workers [Array<Worker>] workers prepared in current batch
+    #
+    # @see Ripe::DSL::WorkflowDSL#describe
+    # @see Ripe::WorkerController#prepare
+
+    class Preparer
+
+      attr_accessor :workers
+
+      ##
+      # Prepare workers by applying the workflow callback and its parameters to
+      # each sample.
+      #
+      # @param workflow [String] the name of a workflow to apply on the sample
+      #   list
+      # @param samples [Array<String>] list of samples to apply the callback to
+      # @param params [Hash<Symbol, String>] a list of worker-wide parameters
+
+      def initialize(workflow, samples, params = {})
+        # Extract callback and params from input
+        callback, params = load_workflow(workflow, params)
+
+        if ![:patch, :force, :depend].include?(params[:mode].to_sym)
+          abort "Invalid mode #{params[:mode]}."
+        end
+
+        # Apply the workflow to each sample
+        sample_blocks = prepare_sample_blocks(samples, callback, params)
+
+        # Split samples into groups of +:group_num+ samples and produce a
+        # worker from each of these groups.
+        @workers = sample_blocks.each_slice(params[:group_num].to_i).map do |worker_blocks|
+          prepare_worker(worker_blocks, params)
+        end
+      end
+
+      ##
+      # Load a workflow and return its +callback+ and +params+ components.
+      #
+      # @param workflow [String] the name of a workflow
+      # @param params [Hash<Symbol, String>] a list of worker-wide parameters
+      # @return [Proc, Hash<Symbol, String>] a list containing the workflow callback
+      #   and default params
+
+      def load_workflow(workflow, params)
+        filename = Library.find(:workflow, workflow)
+        abort "Could not find workflow #{workflow}." if filename == nil
+        require_relative filename
+
+        # Imports +$workflow+ from the workflow component.  This is a dirty
+        # hack to help make the +DSL::WorkflowDSL+ more convenient for the
+        # end user.
+
+        params = {
+          wd:        Dir.pwd,
+          mode:      :patch,
+          group_num: 1,
+        }.merge($workflow.params.merge(params))
+
+        [$workflow.callback, params]
+      end
+
+      ##
+      # Apply the workflow (callback) to each sample, producing a single root
+      # block per sample.
+      #
+      # @param samples [Array<String>] a list of samples
+      # @param callback [Proc] workflow callback to be applied to each sample
+      # @param params [Hash] a list of worker-wide parameters
+      # @return [Hash] a +{sample => block}+ hash
+
+      def prepare_sample_blocks(samples, callback, params)
+        sample_blocks = samples.map do |sample|
+          block = callback.call(sample, params).prune(params[:mode].to_sym == :force,
+                                                      params[:mode].to_sym == :depend)
+          if block != nil
+            puts "Preparing sample #{sample}"
+            {sample => block}
+          else
+            puts "Nothing to do for sample #{sample}"
+            nil
+          end
+        end
+
+        # Produce a {sample => block} hash
+        sample_blocks.compact.inject(&:merge)
+      end
+
+      ##
+      # Prepare a worker from a group of sample blocks.
+      #
+      # @param worker_sample_blocks [Hash] a list containing as many elements
+      #   as there are samples in the group, with each element containing
+      #   +[String, Blocks::Block]+
+      # @param params [Hash] worker-level parameter list
+      # @return [DB::Worker] worker
+
+      def prepare_worker(worker_sample_blocks, params)
+        worker = DB::Worker.create(handle: params[:handle])
+        worker_blocks = prepare_worker_blocks(worker_sample_blocks, worker)
+
+        # Combine all grouped sample blocks into a single worker block
+
+        params = params.merge({
+          name:    worker.id,
+          stdout:  worker.stdout,
+          stderr:  worker.stderr,
+          command: Blocks::SerialBlock.new(*worker_blocks).command,
+        })
+
+        worker_block = Blocks::LiquidBlock.new("#{PATH}/share/moab.sh", params)
+        File.open(worker.sh, 'w') { |f| f.write(worker_block.command) }
+
+        worker.update({
+          status:   :prepared,
+          ppn:      params[:ppn],
+          queue:    params[:queue],
+          walltime: params[:walltime],
+        })
+
+        worker
+      end
+
+      ##
+      # Organize worker blocks into tasks and prepare them.
+      #
+      # @param worker_sample_blocks [Array<Hash<String, Blocks::Block>>] a list
+      # containing as many elements as there are samples in the group
+      # @param worker [DB::Worker] worker
+      # @return [Array<Blocks::Block>] a list of all the prepared blocks for a
+      #   worker
+
+      def prepare_worker_blocks(worker_sample_blocks, worker)
+        worker_sample_blocks.map do |sample, block|
+          # Preorder traversal of blocks -- assign incremental numbers starting from
+          # 1 to each node as it is being traversed, as well as producing the job
+          # file for each task.
+          post_var_assign = lambda do |subblock|
+            if subblock.blocks.length == 0
+              # This section is only called when the subblock is actually a working
+              # block (a leaf in the block arborescence).
+              task = worker.tasks.create({
+                sample: sample,
+                block:  subblock.id,
+              })
+
+              File.open(task.sh, 'w') { |f| f.write(subblock.command) }
+              subblock.vars.merge!(log: task.log)
+            else
+              subblock.blocks.each(&post_var_assign)
+            end
+          end
+
+          post_var_assign.call(block)
+          block
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/ripe/worker_controller/syncer.rb

@@ -0,0 +1,118 @@
+module Ripe
+
+  class WorkerController
+
+    ##
+    # This class controls worker syncing with the compute cluster queue.
+    #
+    # @attr_reader running_jobs [Array<Hash<Symbol, String>>] a list of running
+    #   jobs as well as certain parameters (+moab_id+, +time+ and +status).
+    # @attr_reader completed_jobs [Array<DB::Worker>] a list of completed
+    #   workers
+    # @attr_reader workers [Array<DB::Worker>] list of workers that have been
+    #   updated (or completed)
+    #
+    # @see Ripe::WorkerController#sync
+
+    class Syncer
+
+      attr_reader :running_jobs, :completed_jobs, :workers
+
+      ##
+      # Synchronize the status of jobs with the internal list of workers.
+
+      def initialize
+        @workers = []
+
+        fetch_running_jobs
+        update_running_workers
+        fetch_completed_jobs
+        update_completed_workers
+      end
+
+      ##
+      # Fetch status for all running jobs.
+      #
+      # @return [void]
+
+      def fetch_running_jobs
+        lists = {idle: '-i', blocked: '-b', active:  '-r'}
+        lists = lists.map do |status, op|
+          showq = `showq -u $(whoami) #{op} | grep $(whoami)`.split("\n")
+          showq.map do |job|
+            {
+              moab_id:   job[/^([0-9]+) /, 1],
+              time:      job[/  ([0-9]{1,2}(\:[0-9]{2})+)  /, 1],
+              status:    status,
+            }
+          end
+        end
+        @running_jobs = lists.inject(&:+)
+      end
+
+      ##
+      # Update the status of running workers from the running jobs.
+      #
+      # @return [void]
+
+      def update_running_workers
+        @workers += @running_jobs.map do |job|
+          worker = DB::Worker.find_by(moab_id: job[:moab_id])
+          if worker
+            worker.update(time: job[:time])
+            unless ['cancelled', job[:status]].include?(worker.status)
+              checkjob = `checkjob #{job[:moab_id]}`
+              worker.update({
+                host:      checkjob[/Allocated Nodes:\n\[(.*):[0-9]+\]\n/, 1],
+                # Queued jobs that appear become either idle, blocked or active
+                status:    job[:status],
+              })
+            end
+          end
+          worker
+        end
+      end
+
+      ##
+      # Fetch a list of completed workers from the running jobs: these are jobs
+      # that were previously marked as active, blocked or idle that can no
+      # be found on the compute cluster queue.
+      #
+      # @return [void]
+
+      def fetch_completed_jobs
+        running_job_ids = @running_jobs.map { |job| job[:moab_id] }
+
+        running_workers = DB::Worker.where('status in (:statuses)',
+                                           :statuses => ['active', 'idle', 'blocked'])
+
+        @completed_workers = running_workers.select do |worker|
+          !running_job_ids.include?(worker.moab_id) &&
+            worker.status != 'cancelled'
+        end
+      end
+
+      ##
+      # Update the status of completed workers from the running jobs.
+      #
+      # @return [void]
+
+      def update_completed_workers
+        @workers += @completed_workers.map do |worker|
+          stdout = (File.exists?(worker.stdout)) ? File.new(worker.stdout).readlines.join : ""
+          worker.update({
+            cpu_used:    stdout[/Resources:[ \t]*cput=([0-9]{1,2}(\:[0-9]{2})+),/, 1],
+            exit_code:   stdout[/Exit code:[ \t]*(.*)$/, 1],
+            host:        stdout[/Nodes:[ \t]*(.*)$/, 1],
+            memory_used: stdout[/Resources:.*,mem=([0-9]*[a-zA-Z]*),/, 1],
+            time:        stdout[/Resources:.*,walltime=([0-9]{1,2}(\:[0-9]{2})+)$/, 1],
+            status:      :completed,
+          })
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/cli_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe CLI do
+  describe '::Helper#parse_cli_opts' do
+    it 'parses string options into hash options' do 
+      string_opts = 'a=1,b=2,c=3'
+      test_hash_opts = CLI::Helper.parse_cli_opts(string_opts)
+
+      ref_hash_opts = {a: '1', b: '2', c: '3'}
+
+      expect(test_hash_opts).to eql ref_hash_opts
+    end
+  end
+end

data/spec/library_spec.rb

@@ -1,28 +1,28 @@
 require 'spec_helper'
 
-describe Ripe::Library do
+describe Library do
   context 'when RIPELIB env is empty' do
-    before(:each) do
+    before :each do
       ENV['RIPELIB'] = ''
-      @library = Ripe::Library.new
+      @library = Library
     end
 
     it 'looks in the working directory' do
-      expect(@library.paths).to eql ["#{Dir.pwd}/#{Ripe::Repo::REPOSITORY_PATH}"]
+      expect(@library.paths).to eql ["#{Dir.pwd}/#{Repo::REPOSITORY_PATH}"]
     end
 
     it 'cannot resolve components of the test library' do
-      expect(@library.find_task('foo')).to eql nil
-      expect(@library.find_task('bar')).to eql nil
-      expect(@library.find_workflow('foobar')).to eql nil
+      expect(@library.find(:task, 'foo')).to eql nil
+      expect(@library.find(:task, 'bar')).to eql nil
+      expect(@library.find(:workflow, 'foobar')).to eql nil
     end
   end
 
   context 'when RIPELIB contains the test library' do
-    before(:each) do
-      @test = Ripe::TestPack.new
-      ENV['RIPELIB'] = @test.path
-      @library = Ripe::Library.new
+    before :each do
+      @test = TestPack.new
+      ENV['RIPELIB'] = @test.lib_path
+      @library = Library
     end
 
     it 'looks in two directories' do
@@ -32,22 +32,22 @@ describe Ripe::Library do
     it 'looks in the working directory first' do
       # It looks in the working directory, and then in the directory
       # specified in RIPELIB.
-      expect(@library.paths[0]).to eql "#{Dir.pwd}/#{Ripe::Repo::REPOSITORY_PATH}"
-      expect(@library.paths[1]).to eql @test.path
+      expect(@library.paths[0]).to eql "#{Dir.pwd}/#{Repo::REPOSITORY_PATH}"
+      expect(@library.paths[1]).to eql @test.lib_path
     end
 
     it 'resolves task components of the test library' do
-      expect(@library.find_task('foo')).to eql @test.tasks['foo']
-      expect(@library.find_task('bar')).to eql @test.tasks['bar']
+      expect(@library.find(:task, 'foo')).to eql @test.tasks['foo']
+      expect(@library.find(:task, 'bar')).to eql @test.tasks['bar']
     end
 
     it 'resolves workflows components of the test library' do
-      expect(@library.find_workflow('foobar')).to eql @test.workflows['foobar']
+      expect(@library.find(:workflow, 'foobar')).to eql @test.workflows['foobar']
     end
 
     it 'cannot resolve non-existing componenets' do
-      expect(@library.find_task('other')).to eql nil
-      expect(@library.find_workflow('other')).to eql nil
+      expect(@library.find(:task, 'other')).to eql nil
+      expect(@library.find(:workflow, 'other')).to eql nil
     end
   end
 end

data/spec/spec_helper.rb

@@ -5,3 +5,5 @@ end
 
 require_relative '../lib/ripe'
 require_relative 'testpack'
+
+include Ripe

data/spec/testpack.rb

@@ -1,16 +1,27 @@
 module Ripe
   class TestPack
-    attr_reader :path, :tasks, :workflows
+    attr_reader :path, :lib_path, :tasks, :workflows, :samples, :steps
 
     def initialize
-      @path = "#{Ripe::PATH}/spec/testpack/ripe"
+      @path = "#{PATH}/spec/testpack"
+      @lib_path = "#{@path}/#{Repo::REPOSITORY_PATH}"
       @tasks = {
-        'foo' => "#{@path}/tasks/foo.sh",
-        'bar' => "#{@path}/tasks/bar.sh",
+        'foo' => "#{@lib_path}/tasks/foo.sh",
+        'bar' => "#{@lib_path}/tasks/bar.sh",
       }
       @workflows = {
-        'foobar' => "#{@path}/workflows/foobar.rb",
+        'foobar' => "#{@lib_path}/workflows/foobar.rb",
       }
+      @samples = [
+        'Sample1',
+        'Sample2',
+        'Sample3'
+      ]
+      @steps = [
+        'foo_input.txt',
+        'foo_output.txt',
+        'bar_output.txt'
+      ]
     end
   end
 end

data/spec/testpack/.ripe/tasks/bar.sh

@@ -0,0 +1,3 @@
+# Bar is the most important consequence of Foo.
+
+echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"

data/spec/testpack/.ripe/workers/1/1.sh

@@ -0,0 +1,16 @@
+
+# <foo.sh>
+
+INPUT_FOO="Sample1/foo_input.txt"
+FOO_MESSAGE="For You"
+OUTPUT_FOO="Sample1/foo_output.txt"
+
+exec 1>"$LOG" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
+
+echo "##.DONE.##"
+
+# </foo.sh>

data/spec/testpack/.ripe/workers/1/2.sh

@@ -0,0 +1,16 @@
+
+# <bar.sh>
+
+INPUT_BAR="Sample1/foo_input.txt"
+BAR_MESSAGE="Bar"
+OUTPUT_BAR="Sample1/bar_output.txt"
+
+exec 1>"$LOG" 2>&1
+
+# Bar is the most important consequence of Foo.
+
+echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
+
+echo "##.DONE.##"
+
+# </bar.sh>