ripe 0.2.0 → 0.2.1

data/lib/ripe/dsl.rb

@@ -1,4 +1,2 @@
-require_relative 'repo'
-require_relative 'working_block'
-require_relative 'dsl/task_dsl.rb'
-require_relative 'dsl/workflow_dsl.rb'
+require_relative 'dsl/task_dsl'
+require_relative 'dsl/workflow_dsl'

data/lib/ripe/dsl/task_dsl.rb

@@ -33,16 +33,18 @@ module Ripe
     # @return [WorkingBlock, nil]
 
     def task(handle, &block)
-      filename = Repo.new.library.find_task(handle)
+      filename = Library.find(:task, handle)
       abort "Could not find task #{handle}." if filename == nil
 
       params = TaskDSL.new(handle, &block).params
-      WorkingBlock.new(filename, params)
+      Blocks::WorkingBlock.new(filename, params)
     end
 
     ##
     # This class provides a DSL for defining a task.  It should only be called
     # by #task.
+    #
+    # @attr_reader params [Hash<Symbol, String>] list of parameters
 
     class TaskDSL
 

data/lib/ripe/dsl/workflow_dsl.rb

@@ -37,6 +37,11 @@ module Ripe
     ##
     # This class provides a DSL for defining a workflow.  It should only be
     # called by #workflow.
+    #
+    # @attr_reader handle [String] the name of the workflow
+    # @attr_reader params [Hash<Symbol, String>] list of parameters
+    # @attr_reader callback [Proc] the block describing what to do apply to
+    #   each sample
 
     class WorkflowDSL
 

data/lib/ripe/library.rb

@@ -1,59 +1,48 @@
 module Ripe
 
   ##
-  # This class represents a library containing all the components accessible
-  # to ripe (tasks and workflows) based on what is contained in the +RIPELIB+
-  # environment variable.
+  # This singleton class represents a library containing all the components
+  # accessible to ripe (tasks and workflows) based on what is contained in the
+  # +RIPELIB+ environment variable.
 
-  class Library
+  module Library
 
-    attr_reader :paths
+    class << self
 
-    ##
-    # Create a new library spanning all paths in the +RIPELIB+ environment
-    # variable.
+      ##
+      # Provide a list of search paths
+      #
+      # @return [List] Return the list of search paths
 
-    def initialize
-      # Prepends the working directory to the list of paths so that the
-      # working directory is always looked in first.
-
-      @paths = "#{Dir.pwd}/#{Repo::REPOSITORY_PATH}:#{ENV['RIPELIB']}".split(/:/)
-    end
-
-    ##
-    # Search throughout the library for a task component by the name of
-    # +handle+.  When there is more than one match, give precendence to the
-    # component whose path is declared first.
-    #
-    # @param handle [String] Task to search for
-    # @return [String, nil] Return the full path of the component if found,
-    #   and +nil+ otherwise.
-
-    def find_task(handle)
-      search = @paths.map do |path|
-        filename = "#{path}/tasks/#{handle}.sh"
-        (File.exists? filename) ? filename : nil
+      def paths
+        # Prepend the working directory to the list of paths so that the
+        # working directory is always looked in first.
+        "#{Dir.pwd}/#{Repo::REPOSITORY_PATH}:#{ENV['RIPELIB']}".split(/:/)
       end
 
-      search.compact.first
-    end
-
-    ##
-    # Search throughout the library for a workflow component by the name of
-    # +handle+.  When there is more than one match, give precendence to
-    # component whose path is declared first.
-    #
-    # @param handle [String] Workflow to search for
-    # @return [String, nil] Return the full path of the component if found,
-    #   and +nil+ otherwise.
-
-    def find_workflow(handle)
-      search = @paths.map do |path|
-        filename = "#{path}/workflows/#{handle}.rb"
-        (File.exists? filename) ? filename : nil
+      ##
+      # Search throughout the library for a task or workflow component by the
+      # name of +handle+.  When there is more than one match, give precendence to
+      # component whose path is declared first.
+      #
+      # @param comp [Symbol] Type of component: either +:workflow+ or
+      #   +:task+.
+      # @param handle [String] Name of component
+      # @return [String, nil] Full path of the component if found, and +nil+
+      #   otherwise.
+ 
+      def find(comp, handle)
+        ext = { task:     'sh',
+                workflow: 'rb' }
+
+        search = paths.map do |path|
+          filename = "#{path}/#{comp}s/#{handle}.#{ext[comp]}"
+          (File.exists? filename) ? filename : nil
+        end
+
+        search.compact.first
       end
 
-      search.compact.first
     end
 
   end

data/lib/ripe/repo.rb

@@ -1,14 +1,5 @@
 require 'active_record'
 require 'fileutils'
-require_relative 'block'
-require_relative 'worker'
-require_relative 'worker_controller'
-require_relative 'worker_migration'
-require_relative 'working_block'
-require_relative 'library'
-require_relative 'liquid_block'
-require_relative 'task'
-require_relative 'task_migration'
 
 module Ripe
 
@@ -17,11 +8,12 @@ module Ripe
   # +git+ repository and is the starting point of the package.  It
   # instantiates:
   #
-  # * a library containing information as to where to retrieve ripe
-  #   components such as tasks and workflows;
   # * a database that contains all worker metadata; and
-  # * a controller that communicates the database with the compute cluster
-  #   interface.
+  # * a controller that communicates with both the database and the compute
+  #   cluster interface.
+  #
+  # @attr_reader controller [WorkerController] a controller that communicates
+  #   with both the database and the computer cluster interface.
   #
   # @see Ripe::WorkerController
   # @see Ripe::Library
@@ -32,19 +24,27 @@ module Ripe
     DATABASE_PATH   = "#{REPOSITORY_PATH}/meta.db"
     WORKERS_PATH    = "#{REPOSITORY_PATH}/workers"
 
-    attr_reader :library, :controller
+    attr_reader :controller
 
     ##
-    # Initialize a repository
+    # Initialize a repository.
 
     def initialize
-      @has_repository = File.exists? REPOSITORY_PATH
-      @library        = Library.new
-      @controller     = WorkerController.instance
+      @has_repository = File.exists? DATABASE_PATH
+      @controller     = WorkerController.new
+    end
+
+    ##
+    # Return whether the ripe repository exists.
+    #
+    # @return [Boolean] whether the repository exists
+
+    def has_repository?
+      @has_repository
     end
 
     ##
-    # Attach to an existing database
+    # Attach to an existing database.
 
     def attach
       ActiveRecord::Base.establish_connection({
@@ -54,14 +54,15 @@ module Ripe
     end
 
     ##
-    # Attach to an existing database, and creates one if none is found.
+    # Attach to an existing database, and creates one if a database cannot be
+    # found.
 
     def attach_or_create
       @has_repository ? attach : create
     end
 
     ##
-    # Create a database
+    # Create a database.
 
     def create
       FileUtils.mkdir_p(REPOSITORY_PATH)
@@ -71,8 +72,8 @@ module Ripe
         attach
 
         # Create the tables
-        WorkerMigration.up
-        TaskMigration.up
+        DB::WorkerMigration.up
+        DB::TaskMigration.up
       rescue
         destroy
       end

data/lib/ripe/version.rb

@@ -1,3 +1,3 @@
 module Ripe
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/ripe/worker_controller.rb

@@ -1,5 +1,5 @@
-require 'singleton'
-require_relative 'worker'
+require_relative 'worker_controller/preparer'
+require_relative 'worker_controller/syncer'
 
 module Ripe
 
@@ -10,177 +10,105 @@ module Ripe
 
   class WorkerController
 
-    include Singleton
-
     ##
-    # Prepares workers by applying the workflow callback and its parameters to
+    # Prepare workers by applying the workflow callback and its parameters to
     # each sample.
     #
     # @see Ripe::DSL::WorkflowDSL#describe
+    # @see Ripe::WorkerController::Preparer
     #
-    # @param samples [List] list of samples to apply the callback to
-    # @param callback [Proc] a callback function that takes as arguments the name
-    #   of sample and a hash of parameters provided by the workflow and by the
-    #   command line.
-    # @param vars [Hash] a list of worker-wide parameters
-
-    def prepare(samples, callback, vars = {})
-      vars = {
-        wd:        Dir.pwd,
-        mode:      :patch,
-        group_num: 1,
-      }.merge(vars)
-
-      return if ![:patch, :force, :depend].include? vars[:mode].to_sym
-
-      samples = samples.map do |sample|
-        block = callback.call(sample, vars).prune(vars[:mode].to_sym == :force,
-                                                  vars[:mode].to_sym == :depend)
-        if block != nil
-          puts "Preparing sample #{sample}"
-          [sample, block]
-        else
-          puts "Nothing to do for sample #{sample}"
-          nil
-        end
-      end
-      samples = samples.compact
-
-      samples.each_slice(vars[:group_num].to_i).map do |worker_samples|
-        worker = Worker.create(handle: vars[:handle])
-
-        blocks = worker_samples.map do |sample, block|
-          # Preorder traversal of blocks -- assign incremental numbers starting from
-          # 1 to each node as it is being traversed, as well as producing the job
-          # file for each task.
-          post_var_assign = lambda do |subblock|
-            if subblock.blocks.length == 0
-              # This section is only called when the subblock is actually a working
-              # block (a leaf in the block arborescence).
-              task = worker.tasks.create({
-                sample: sample,
-                block:  subblock.id,
-              })
-
-              file = File.new(task.sh, 'w')
-              file.puts subblock.command
-              file.close
-
-              subblock.vars.merge!(log: task.log)
-            else
-              subblock.blocks.each(&post_var_assign)
-            end
-          end
-
-          post_var_assign.call(block)
-          block
-        end
+    # @param (see Preparer#initialize)
+    # @return [Array<Worker>] workers prepared in current batch
 
-        vars = vars.merge({
-          name:    worker.id,
-          stdout:  worker.stdout,
-          stderr:  worker.stderr,
-          command: SerialBlock.new(*blocks).command,
-        })
-
-        file = File.new(worker.sh, 'w')
-        file.puts LiquidBlock.new("#{PATH}/share/moab.sh", vars).command
-        file.close
-
-        worker.update({
-          status:   :prepared,
-          ppn:      vars[:ppn],
-          queue:    vars[:queue],
-          walltime: vars[:walltime],
-        })
-        worker
-      end
+    def prepare(workflow, samples, params = {})
+      Preparer.new(workflow, samples, params).workers
     end
 
     ##
-    # Submit a job to the compute cluster system
+    # Apply a block to a list of workers.
     #
-    # @param worker [Worker] the worker to submit
-
-    def start(worker)
-      worker.update(status: :queueing,
-                    moab_id: `qsub '#{worker.sh}'`.strip.split(/\./).first)
+    # @param workers [Array<DB::Worker>, DB::Worker] a list of workers or a
+    #   single worker
+    # @return [Array<DB::Worker>] the list of workers given in arguments,
+    #   with modified states
+
+    def distribute(workers, &block)
+      workers = [workers] if workers.is_a? DB::Worker
+      workers.map { |w| block.call(w) }
     end
 
     ##
-    # Cancel a job in the compute cluster system
+    # Run worker job code into bash locally.
     #
-    # @param worker [Worker] the worker to cancel
+    # @param (see #distribute)
+    # @return (see #distribute)
 
-    def cancel(worker)
-      `canceljob #{worker.moab_id}`
-      worker.update(status: :cancelled)
+    def local(workers)
+      distribute workers do |worker|
+        `bash #{worker.sh}`
+      end
     end
 
     ##
-    # Synchronize the status of jobs with the internal list of workers.
-
-    def sync
-      lists = {idle: '-i', blocked: '-b', active:  '-r'}
-      lists = lists.map do |status, op|
-        showq = `showq -u $(whoami) #{op} | grep $(whoami)`.split("\n")
-        showq.map do |job|
-          {
-            moab_id:   job[/^([0-9]+) /, 1],
-            time:      job[/  ([0-9]{1,2}(\:[0-9]{2})+)  /, 1],
-            status:    status,
-          }
+    # Submit worker jobs to the compute cluster system.
+    #
+    # @param (see #distribute)
+    # @return (see #distribute)
+
+    def start(workers)
+      distribute workers do |worker|
+        if worker.status == 'prepared'
+          worker.update(status: :queueing,
+                        moab_id: `qsub '#{worker.sh}'`.strip.split(/\./).first)
+        else
+          puts "Worker #{worker.id} could not be started: not prepared"
         end
       end
+    end
 
-      # Update status
-      lists = lists.inject(&:+).each do |job|
-        moab_id   = job[:moab_id]
-        time      = job[:time]
-        status    = job[:status]
-        worker    = Worker.find_by(moab_id: moab_id)
-
-        if worker
-          worker.update(time: time)
-          unless ['cancelled', status].include? worker.status
-            checkjob = `checkjob #{moab_id}`
-            worker.update({
-              host:      checkjob[/Allocated Nodes:\n\[(.*):[0-9]+\]\n/, 1],
-              status:    status, # Queued jobs that appear become either idle, blocked or active
-            })
-          end
+    ##
+    # Cancel worker jobs in the compute cluster system.
+    #
+    # @param (see #distribute)
+    # @return (see #distribute)
+
+    def cancel(workers)
+      distribute workers do |worker|
+        if ['queueing', 'idle', 'blocked', 'active'].include? worker.status
+          `canceljob #{worker.moab_id}`
+          worker.update(status: :cancelled)
+        else
+          puts "Worker #{worker.id} could not be cancelled: not started"
         end
       end
+    end
 
-      # Mark workers that were previously in active, blocked or idle as completed
-      # if they cannot be found anymore.
-      jobs = lists.map { |job| job[:moab_id] }
-      Worker.where('status in (:statuses)',
-                   :statuses => ['active', 'idle', 'blocked']).each do |worker|
-        if jobs.include? worker.moab_id
-          jobs.delete(worker.moab_id) # Remove from list
-        elsif (worker.status != 'cancelled')
-          if File.exists? worker.stdout
-            stdout = File.new(worker.stdout).readlines.join
-          else
-            stdout = ""
-          end
-          worker.update({
-            cpu_used:    stdout[/Resources:[ \t]*cput=([0-9]{1,2}(\:[0-9]{2})+),/, 1],
-            exit_code:   stdout[/Exit code:[ \t]*(.*)$/, 1],
-            host:        stdout[/Nodes:[ \t]*(.*)$/, 1],
-            memory_used: stdout[/Resources:.*,mem=([0-9]*[a-zA-Z]*),/, 1],
-            time:        stdout[/Resources:.*,walltime=([0-9]{1,2}(\:[0-9]{2})+)$/, 1],
-            status:      :completed,
-          })
-        end
-      end
+    ##
+    # Synchronize the status of jobs with the internal list of workers.
+    #
+    # @see Ripe::WorkerController::Syncer
+    #
+    # @return [Array<DB::Worker>] the list of updated workers
+
+    def sync
+      Syncer.new.workers
     end
 
+    ##
+    # List the n most recent workers.
+    #
+    # @param n [Integer] the number of most recent workers
+    # @return [Array<DB::Worker>] the list of +n+ most recent workers
+
     def list(n = 20)
-      Worker.last(n)
+      DB::Worker.last(n)
     end
 
+    ##
+    # Launch the an interactive text editor from the console.
+    #
+    # @return [void]
+
     def edit(*args)
       system("$EDITOR #{args.join(' ')}")
     end