rbbt-util 5.1.0 → 5.2.0

data/lib/rbbt/workflow/definition.rb

@@ -0,0 +1,71 @@
+require 'rbbt-util'
+require 'rbbt/workflow/annotate'
+
+module Workflow
+  include AnnotatedModule
+
+
+  AnnotatedModule.add_consummable_annotation(self,
+                                             :result_description => "",
+                                             :result_type        => nil,
+                                             :dependencies       => [])
+  def helper(name, &block)
+    helpers[name] = block
+  end
+
+  def desc(description)
+    @description = description
+  end
+
+  def returns(description)
+    @result_description = description
+  end
+
+  def dep(*dependency_list, &block)
+    dependency_list << block if block_given?
+    dependencies.concat dependency_list
+  end
+
+  def task(name, &block)
+    if Hash === name
+      type = name.first.last
+      name = name.first.first
+    else
+      result_type = consume_result_type || :marshal
+    end
+
+    name = name.to_sym
+
+    block = self.method(name) unless block_given?
+
+    task_info = {
+      :name => name,
+      :inputs             => consume_inputs,
+      :description        => consume_description,
+      :input_types        => consume_input_types,
+      :result_type        => (Array === type ? type.to_sym : type),
+      :input_defaults     => consume_input_defaults,
+      :input_descriptions => consume_input_descriptions,
+      :input_options      => consume_input_options
+    }
+    
+    task = Task.setup(task_info, &block)
+
+    last_task = task
+
+    tasks[name] = task
+    task_dependencies[name] = consume_dependencies
+  end
+  
+  def export_exec(*names)
+    exec_exports.concat names
+  end
+
+  def export_asynchronous(*names)
+    asynchronous_exports.concat names
+  end
+
+  def export_synchronous(*names)
+    synchronous_exports.concat names
+  end
+end

data/lib/rbbt/workflow/soap.rb

@@ -15,7 +15,7 @@ class WorkflowSOAP < SimpleWS
   end
 
   def initialize(workflow, *args)
-    super(workflow.to_s,*args)
+    super(workflow.to_s, *args)
     @workflow = workflow
     @workflow.synchronous_exports.each do |name| synchronous name end
     @workflow.asynchronous_exports.each do |name| asynchronous name end
@@ -38,6 +38,12 @@ class WorkflowSOAP < SimpleWS
       job(jobid).info.to_yaml
     end
 
+    desc "Job management: Load job result as string "
+    param_desc :jobid => "Job identifier", :return => "String containing the result of the job"
+    serve :load_string, %w(jobid), :jobid => :string, :return => :string do |jobid|
+      Open.read(job(jobid).path)
+    end
+
     desc "Job management: Abort the job"
     param_desc :jobid => "Job identifier"
     serve :abort, %w(jobid), :jobid => :string, :return => false do |jobid|
@@ -56,15 +62,19 @@ class WorkflowSOAP < SimpleWS
       job(jobid).status.to_sym == :error
     end
 
-    desc "Job management: Load job result as string "
-    param_desc :jobid => "Job identifier", :return => "String containing the result of the job"
-    serve :load_string, %w(jobid), :jobid => :string, :return => :string do |jobid|
-      job(jobid).load.to_s
+    desc "Job management: Check if the job has finished with error. The last message is the error message"
+    param_desc :jobid => "Job identifier", :return => "True if the job has status 'error'"
+    serve :clean, %w(jobid), :jobid => :string, :return => nil do |jobid|
+      job(jobid).clean
+      nil
     end
+
+
   end
 
   def synchronous(*tasknames)
     tasknames.each do |name|
+      name = name.to_sym
       task = @workflow.tasks[name]
       desc @workflow.task_description[name] if @workflow.task_description.include? name
 

data/lib/rbbt/workflow/step.rb

@@ -10,7 +10,7 @@ class Step
   class Aborted < Exception; end
 
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil)
-    @path = path
+    @path = Path.setup(path)
     @task = task
     @bindings = bindings
     @dependencies = case
@@ -84,15 +84,17 @@ class Step
   end
 
   def run(no_load = false)
-    result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq, :no_load => no_load do
+    result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path }.uniq, :no_load => no_load do
       if Step === Step.log_relay_step and not self == Step.log_relay_step
         relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
       end
 
       FileUtils.rm info_file if File.exists? info_file
 
-      set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
-      @dependencies.each{|dependency| 
+      set_info :pid, Process.pid
+
+      set_info :dependencies, dependencies.collect{|dep| [dep.task.name, dep.name]}
+      dependencies.each{|dependency| 
         begin
           dependency.relay_log self
           dependency.run true
@@ -106,7 +108,7 @@ class Step
         end
       }
       
-      log(:started, "Starting task #{task.name || ""}")
+      log(:started, "Starting task #{task.name || ""} [#{Process.pid}]")
 
       set_info :started, Time.now
       
@@ -116,6 +118,20 @@ class Step
               exec
             rescue Step::Aborted
               log(:error, "Aborted")
+
+              children_pids = info[:children_pids]
+              if children_pids and children_pids.any?
+                Log.medium("Killing children: #{ children_pids * ", " }")
+                children_pids.each do |pid|
+                  Log.medium("Killing child #{ pid }")
+                  begin
+                    Process.kill "INT", pid
+                  rescue Exception
+                    Log.medium("Exception killing child #{ pid }: #{$!.message}")
+                  end
+                end
+              end
+
               raise $!
             rescue Exception
               backtrace = $!.backtrace
@@ -148,11 +164,25 @@ class Step
       FileUtils.mkdir_p File.dirname(path) unless File.exists? File.dirname(path)
       begin
         run
-      rescue
+        children_pids = info[:children_pids]
+        if children_pids
+          children_pids.each do |pid|
+            if Misc.pid_exists? pid
+              begin
+                Process.waitpid pid
+              rescue Errno::ECHILD
+                Log.error "Waiting on #{ pid } failed: #{$!.message}"
+              end
+            end
+          end
+        end
+      rescue Exception
+        Log.debug("Exception caught on forked process: #{$!.message}")
         exit -1
       end
+      set_info :pid, nil
+      exit 0
     end
-    set_info :pid, @pid
     Process.detach(@pid)
     self
   end
@@ -164,12 +194,30 @@ class Step
       false
     else
       Log.medium "Aborting #{path}: #{ @pid }"
-      Process.kill("INT", @pid)
+      begin
+        Process.kill("INT", @pid)
+        Process.waitpid @pid
+      rescue Exception
+        Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
+      end
       log(:aborted, "Job aborted by user")
       true
     end
   end
 
+  def child(&block)
+    child_pid = Process.fork &block
+    children_pids = info[:children_pids]
+    if children_pids.nil?
+      children_pids = [child_pid]
+    else
+      children_pids << child_pid
+    end
+    #Process.detach(child_pid)
+    set_info :children_pids, children_pids
+    child_pid
+  end
+
   def load
     raise "Can not load: Step is waiting for proces #{@pid} to finish" if not done?
     result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path} do
@@ -182,6 +230,7 @@ class Step
     if File.exists?(path) or File.exists?(info_file)
       begin
         FileUtils.rm info_file if File.exists? info_file
+        FileUtils.rm info_file + '.lock' if File.exists? info_file + '.lock'
         FileUtils.rm path if File.exists? path
         FileUtils.rm path + '.lock' if File.exists? path + '.lock'
         FileUtils.rm_rf files_dir if File.exists? files_dir

data/lib/rbbt/workflow/usage.rb

@@ -0,0 +1,72 @@
+
+module Task
+  def doc(deps = nil)
+
+    puts "## #{ name }:"
+    puts "\n" << description if description and not description.empty?
+    puts 
+
+    inputs.each do |name|
+      short = name.to_s.chars.first
+
+      description = input_descriptions[name]
+      default = input_defaults[name]
+      type = input_types[name]
+
+      puts "  * -#{short}, --#{name}=<#{ type }>#{default ? " (default: #{default})" : ""}:"
+      puts "    " << description if description and not description.empty?
+      puts
+    end
+
+    if deps and deps.any?
+      puts
+      puts "From dependencies:"
+      puts
+      deps.each do |dep|
+        puts "  #{dep.name}:"
+        puts
+        dep.inputs.each do |name|
+          short = name.to_s.chars.first
+
+          description = dep.input_descriptions[name]
+          default = dep.input_defaults[name]
+          type = dep.input_types[name]
+
+          puts "  * -#{short}, --#{name}=<#{ type }>#{default ? " (default: #{default})" : ""}:"
+          puts "    " << description if description and not description.empty?
+          puts
+        end
+      end
+    end
+  end
+end
+
+module Workflow
+  def doc(task = nil)
+
+    if task.nil?
+      puts self.to_s 
+      puts "=" * self.to_s.length
+      puts
+
+      puts "## TASKS"
+      puts
+      tasks.each do |name,task|
+        puts "  * #{ name }:"
+        puts "    " << task.description if task.description and not task.description.empty?
+        puts
+      end
+    else
+
+      if Task === task
+        task_name = task.name
+      else
+        task_name = task
+        task = self.tasks[task_name]
+      end
+      dependencies = self.rec_dependencies(task_name).collect{|dep_name| self.tasks[dep_name.to_sym]}
+
+      task.doc(dependencies)
+    end
+  end
+end

data/share/lib/R/util.R

@@ -22,6 +22,10 @@ rbbt.ruby <- function(code, load = TRUE, flat = FALSE, type = 'tsv', ...){
           data = scan(file, ...)
           return(data);
       }
+
+      if(type == 'string'){
+          return(file);
+      }
       
       return(NULL);
   }else{
@@ -268,4 +272,12 @@ rbbt.plot.matrix <- function(x, ...){
     layout(1);
 }
 
+rbbt.log <- function(m){
+    cat(paste(m,"\n"), file = stderr())
+}
+
+rbbt.ddd <- function(o){
+    cat(toString(o), file = stderr())
+    cat("\n", file = stderr())
+}
 

data/share/rbbt_commands/conf/web_user/add

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+require 'rbbt'
+require 'highline/import'
+
+if Rbbt.etc.web_users.exists?
+  web_users = Rbbt.etc.web_users.yaml
+else
+  web_users = {}
+end
+
+user = ARGV.shift
+
+raise "No user provided" if user.nil?
+
+password = ask("Enter your password:  ") { |q| q.echo = "x" }
+
+raise "No password provided" if password.nil?
+
+password_check = ask("Re-enter your password:  ") { |q| q.echo = "x" }
+
+raise "Passwords don't match" if password != password_check
+
+web_users[user] = password
+
+Rbbt.etc.web_users.write(web_users.to_yaml)

data/share/rbbt_commands/conf/web_user/list

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'rbbt'
+
+if Rbbt.etc.web_users.exists?
+    puts Rbbt.etc.web_users.yaml.keys
+else
+    "Path #{Rbbt.etc.web_users} not found"
+end

data/share/rbbt_commands/conf/web_user/remove

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'rbbt'
+require 'highline/import'
+
+if Rbbt.etc.web_users.exists?
+  web_users = Rbbt.etc.web_users.yaml
+else
+  web_users = {}
+end
+
+user = ARGV.shift
+
+raise "No user provided" if user.nil?
+
+web_users.delete user
+
+Rbbt.etc.web_users.write(web_users.to_yaml)

data/share/rbbt_commands/workflow/remote/add

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'rbbt'
+
+url, workflow = ARGV
+url = File.join(url, workflow) unless url =~ /\/#{workflow}$/
+config_file = Rbbt.etc.remote_workflows
+remote_workflows = config_file.exists? ? config_file.yaml : {}
+remote_workflows[workflow] = url
+Open.write(config_file.find(:user), remote_workflows.to_yaml)
+

data/share/rbbt_commands/workflow/remote/list

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'rbbt'
+
+config_file = Rbbt.etc.remote_workflows
+config_file.yaml.sort_by{|k,v| k}.each do |workflow, server|
+  puts [workflow, server] * "\t"
+end
+

data/share/rbbt_commands/workflow/remote/remove

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'rbbt'
+
+workflow = ARGV.shift
+config_file = Rbbt.etc.remote_workflows
+remote_workflows = config_file.exists? ? config_file.yaml : {}
+remote_workflows.delete workflow
+Open.write(config_file.find(:user), remote_workflows.to_yaml)

data/share/rbbt_commands/workflow/task

@@ -0,0 +1,181 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+
+def usage(workflow = nil, task = nil)
+  if workflow.nil?
+    puts "No workflow specified"
+    exit -1 
+  end
+
+  if task.nil?
+    workflow.doc
+  else
+    workflow.doc(task)
+  end
+  exit 0
+end
+
+def SOPT_options(workflow, task)
+  sopt_options = []
+  workflow.rec_inputs(task.name).each do |name|
+    short = name.to_s.chars.first
+    boolean = workflow.rec_input_types(task.name)[name].to_sym == :boolean
+    
+    sopt_options << "-#{short}--#{name}#{boolean ? '' : '*'}"
+  end
+
+  sopt_options * ":"
+end
+
+def fix_options(workflow, task, job_options)
+  option_types = workflow.rec_input_types(task.name)
+
+  job_options_cleaned = {}
+
+  job_options.each do |name, value|
+    value = case option_types[name].to_sym
+            when :float
+              value.to_f
+            when :integer
+              value.to_i
+            when :string, :text
+              case
+              when value == '-'
+                STDIN.read
+              when (String === value and File.exists?(value))
+                Open.read(value)
+              else
+                value
+              end
+            when :array
+              if Array === value
+                value
+              else
+                case
+                when value == '-'
+                  STDIN.read
+                when (String === value and File.exists?(value))
+                  Open.read(value)
+                else
+                  value
+                end.split(/[,|\s]/)
+              end
+            when :tsv
+              if TSV === value
+                value
+              else
+                begin
+                  if value == '-'
+                    TSV.open(STDIN).to_s :sort
+                  else
+                    TSV.new(value).to_s :sort
+                  end
+                rescue
+                  value
+                end
+              end
+            else
+              value
+            end
+    job_options_cleaned[name] = value
+  end
+
+  job_options_cleaned
+end
+
+options = SOPT.get "-t--task*:--profile:-l--log*:-h--help:-n--name*:-cl--clean:-rcl-recursive_clean:-pn--printname"
+
+workflow = ARGV.shift
+usage if workflow.nil?
+
+task     = ARGV.shift
+
+
+# Set log, fork, clean, recursive_clean and help
+Log.severity = options[:log].to_i if options.include? :log
+help = !!options.delete(:help)
+do_fork = !!options.delete(:fork)
+clean = !!options.delete(:clean)
+recursive_clean = !!options.delete(:recursive_clean)
+
+# Get workflow
+
+if Rbbt.etc.remote_workflows.exists?
+  remote_workflows = Rbbt.etc.remote_workflows.yaml
+else
+  remote_workflows = {}
+end
+
+if remote_workflows.include? workflow
+  require 'rbbt/rest/client'
+  workflow = WorkflowRESTClient.new remote_workflows[workflow], workflow
+else
+  Workflow.require_workflow workflow
+  workflow = Workflow.workflows.last
+end
+
+# Set task
+namespace = nil, nil
+
+case 
+when task.nil?
+  usage workflow
+when (task =~ /\./)
+  namespace, task = options.delete(:task).split('.')
+  namespace = Misc.string2const(namespace)
+else
+  task_name = task.to_sym
+  task = workflow.tasks[task_name]
+  raise "Task not found: #{ task_name }" if task.nil?
+end
+
+usage workflow, task if help
+
+name = options.delete(:name) || "Default"
+
+# get job args
+sopt_option_string = SOPT_options(workflow, task)
+job_options = SOPT.get sopt_option_string
+job_options = fix_options(workflow,task, job_options)
+
+#- get job
+job = workflow.job(task.name, name, job_options)
+
+# clean job
+job.clean if clean
+job.recursive_clean if recursive_clean
+
+# run
+if do_fork
+  job.fork
+  while not job.done?
+    Log.debug "#{job.step}: #{job.messages.last}"
+    sleep 2
+  end
+  raise job.messages.last if job.error?
+  res = job.load
+else
+  res = job.run
+end
+
+if options.delete(:printname)
+  puts job.name
+  exit
+else
+  Log.low "Job name: #{job.name}" 
+end
+
+case
+when Array === res
+  puts res * "\n"
+when TSV === res
+  puts res
+when Hash === res
+  puts res.to_yaml
+else
+  puts res
+end
+