rbbt-util 5.2.4 → 5.3.0

data/lib/rbbt/util/semaphore.rb

@@ -0,0 +1,71 @@
+require 'inline'
+
+module RbbtSemaphore
+  inline(:C) do |builder|
+    builder.prefix <<-EOF
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <semaphore.h>
+#include <time.h>
+#include <assert.h>
+#include <errno.h>
+#include <signal.h>
+#include <fcntl.h>
+    EOF
+
+    builder.c_singleton <<-EOF
+void create_semaphore(char* name, int value){
+  sem_open(name, O_CREAT, S_IRWXU, value);
+}
+    EOF
+    builder.c_singleton <<-EOF
+void delete_semaphore(char* name){
+  sem_unlink(name);
+}
+    EOF
+
+    builder.c_singleton <<-EOF
+void wait_semaphore(char* name){
+  sem_t* sem;
+  sem = sem_open(name, O_EXCL);
+  sem_wait(sem);
+  sem_close(sem);
+}
+    EOF
+
+    builder.c_singleton <<-EOF
+void post_semaphore(char* name){
+  sem_t* sem;
+  sem = sem_open(name, O_EXCL);
+  sem_post(sem);
+  sem_close(sem);
+}
+    EOF
+  end
+
+  def self.with_semaphore(size, file = nil)
+    file = Misc.digest(rand.to_s) if file.nil?
+    file.gsub!('/', '_')
+    begin
+      RbbtSemaphore.create_semaphore(file, size)
+      yield file
+    ensure
+      RbbtSemaphore.delete_semaphore(file)
+    end
+  end
+
+  def self.fork_each_on_semaphore(elems, size, file = nil)
+    with_semaphore(size, file) do |file|
+      pids = elems.collect do |elem| 
+        Process.fork do 
+          RbbtSemaphore.wait_semaphore(file)
+          yield elem
+          RbbtSemaphore.post_semaphore(file)
+        end
+      end
+      pids.each do |pid| Process.waitpid pid end
+    end
+  end
+end
+

data/lib/rbbt/workflow.rb

@@ -58,8 +58,9 @@ module Workflow
     else
       case
         # Points to workflow file
-      when ((File.exists?(wf_name) and not File.directory?(wf_name)) or File.exists?(wf_name + '.rb')) 
+      when ((File.exists?(wf_name) and not File.directory?(wf_name)) or File.exists?(wf_name + '.rb') or File.exists?(wf_name))
         $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
+        wf_name = "./" << wf_name unless wf_name[0] == "/"
         require wf_name
         Log.medium "Workflow loaded from file: #{ wf_name }"
         return true
@@ -112,19 +113,17 @@ module Workflow
     begin
       require_local_workflow(wf_name) 
     rescue Exception
-      Log.debug $!.message 
-      Log.debug $!.backtrace.first
       raise "Workflow not found: #{ wf_name }" if wf_name == Misc.snake_case(wf_name)
-      Log.debug "Trying with humanized: '#{Misc.snake_case wf_name}'"
       begin
         require_local_workflow(Misc.snake_case(wf_name))
       rescue Exception
-        Log.debug $!.message
-        raise "Workflow not found: #{ wf_name }"
+        Log.error("Workflow not found: #{ wf_name }")
+        raise $!
       end
     end
   end
 
+  attr_accessor :description
   attr_accessor :libdir, :workdir 
   attr_accessor :helpers, :tasks
   attr_accessor :task_dependencies, :task_description, :last_task 
@@ -134,7 +133,8 @@ module Workflow
 
   def workdir
     @workdir ||= if defined? Rbbt
-                   Rbbt.var.jobs[self].find
+                   text = Module === self ? self.to_s : "Misc"
+                   Rbbt.var.jobs[text].find
                  else
                    Path.setup('var/jobs')
                  end
@@ -144,6 +144,17 @@ module Workflow
     @libdir = Path.caller_lib_dir if @libdir.nil?
     @libdir 
   end
+  
+  def workflow_description
+    @workflow_description ||= begin
+                       file = @libdir['workflow.md']
+                       if file.exists?
+                         file.read
+                       else
+                         ""
+                       end
+                     end
+  end
 
   def helpers
     @helpers ||= {}
@@ -232,4 +243,20 @@ module Workflow
       Misc.path_relative_to(task_dir, f).sub(".info",'')
     }
   end
+
+  def local_persist_setup
+    class << self
+      include LocalPersist
+    end
+    self.local_persist_dir = Rbbt.var.cache.persistence.find :lib
+  end
+
+  def local_workdir_setup
+    self.workdir = Rbbt.var.jobs.find :lib
+  end
+
+  def make_local
+    local_persist_setup
+    local_workdir_setup
+  end
 end

data/lib/rbbt/workflow/accessor.rb

@@ -2,6 +2,8 @@ require 'rbbt/util/open'
 require 'yaml'
 
 class Step
+   
+  INFO_SERIALIAZER = Marshal
 
   def name
     @path.sub(/.*\/#{Regexp.quote task.name.to_s}\/(.*)/, '\1')
@@ -22,22 +24,24 @@ class Step
   end
 
   def info
-    return {} if not File.exists? info_file
+    return {} if not Open.exists? info_file
     begin
-      File.open(info_file) do |file|
-        YAML.load(file) || {}
+      Open.open(info_file) do |file|
+        INFO_SERIALIAZER.load(file) || {}
       end
     rescue Exception
-      Log.debug "Error loading yaml: " + info_file
+      Log.debug "Error loading info file: " + info_file
       raise $!
     end
   end
 
   def set_info(key, value)
-    Misc.lock(info_file) do
+    return nil if @exec
+    value = Annotated.purge value 
+    Open.lock(info_file) do
       i = info
       i[key] = value
-      Open.write(info_file, i.to_yaml)
+      Open.write(info_file, INFO_SERIALIAZER.dump(i))
       value
     end
   end
@@ -71,7 +75,7 @@ class Step
   end
 
   def started?
-    File.exists? info_file
+    Open.exists? info_file
   end
 
   def done?
@@ -79,7 +83,7 @@ class Step
   end
 
   def running?
-    return nil if not File.exists? info_file
+    return nil if not Open.exists? info_file
     return nil if info[:pid].nil?
     return Misc.pid_exists? info[:pid]
   end

data/lib/rbbt/workflow/step.rb

@@ -1,11 +1,13 @@
 require 'rbbt/persist'
 require 'rbbt/persist/tsv'
 require 'rbbt/util/log'
+require 'rbbt/util/semaphore'
 require 'rbbt/workflow/accessor'
 
 class Step
   attr_accessor :path, :task, :inputs, :dependencies, :bindings
   attr_accessor :pid
+  attr_accessor :exec
 
   class Aborted < Exception; end
 
@@ -62,6 +64,7 @@ class Step
   end
 
   def exec
+    @exec = true if @exec.nil?
     result = @task.exec_in((bindings ? bindings : self), *@inputs)
     prepare_result result, @task.result_description
   end
@@ -83,11 +86,12 @@ class Step
 
   def run(no_load = false)
     result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path }.uniq, :no_load => no_load do
+      @exec = false
       if Step === Step.log_relay_step and not self == Step.log_relay_step
         relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
       end
 
-      FileUtils.rm info_file if File.exists? info_file
+      Open.rm info_file if Open.exists? info_file
 
       set_info :pid, Process.pid
 
@@ -95,6 +99,7 @@ class Step
       dependencies.each{|dependency| 
         begin
           dependency.relay_log self
+          dependency.clean if not dependency.done? and dependency.error?
           dependency.run true
         rescue Exception
           backtrace = $!.backtrace
@@ -158,37 +163,43 @@ class Step
     end
   end
 
-  def fork
+  def fork(semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil?
     @pid = Process.fork do
       trap(:INT) { raise Step::Aborted.new "INT signal recieved" }
-      FileUtils.mkdir_p File.dirname(path) unless File.exists? File.dirname(path)
       begin
-        run
-      rescue Exception
-        Log.debug("Exception caught on forked process: #{$!.message}")
-        exit -1
-      end
+        RbbtSemaphore.wait_semaphore(semaphore) if semaphore
+        FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
+        begin
+          run(true)
+        rescue Exception
+          Log.debug("Exception caught on forked process: #{$!.message}")
+          exit -1
+        end
 
-      begin
-        children_pids = info[:children_pids]
-        if children_pids
-          children_pids.each do |pid|
-            if Misc.pid_exists? pid
-              begin
-                Process.waitpid pid
-              rescue Errno::ECHILD
-                Log.error "Waiting on #{ pid } failed: #{$!.message}"
+        begin
+          children_pids = info[:children_pids]
+          if children_pids
+            children_pids.each do |pid|
+              if Misc.pid_exists? pid
+                begin
+                  Process.waitpid pid
+                rescue Errno::ECHILD
+                  Log.error "Waiting on #{ pid } failed: #{$!.message}"
+                end
               end
             end
+            set_info :children_done, Time.now
           end
+        rescue Exception
+          Log.debug("Exception waiting for children: #{$!.message}")
+          exit -1
         end
-      rescue Exception
-        Log.debug("Exception waiting for children: #{$!.message}")
-        exit -1
+        set_info :pid, nil
+        exit 0
+      ensure
+        RbbtSemaphore.post_semaphore(semaphore) if semaphore
       end
-      set_info :pid, nil
-      exit 0
     end
     Process.detach(@pid)
     self
@@ -234,13 +245,13 @@ class Step
   end
 
   def clean
-    if File.exists?(path) or File.exists?(info_file)
+    if Open.exists?(path) or Open.exists?(info_file)
       begin
-        FileUtils.rm info_file if File.exists? info_file
-        FileUtils.rm info_file + '.lock' if File.exists? info_file + '.lock'
-        FileUtils.rm path if File.exists? path
-        FileUtils.rm path + '.lock' if File.exists? path + '.lock'
-        FileUtils.rm_rf files_dir if File.exists? files_dir
+        Open.rm info_file if Open.exists? info_file
+        Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'
+        Open.rm path if Open.exists? path
+        Open.rm path + '.lock' if Open.exists? path + '.lock'
+        Open.rm_rf files_dir if Open.exists? files_dir
       end
     end
     self
@@ -250,6 +261,11 @@ class Step
     @dependencies.collect{|step| step.rec_dependencies}.flatten.concat  @dependencies
   end
 
+  def recursive_clean
+    rec_dependencies.each{|step| step.clean }
+    clean
+  end
+
   def step(name)
     rec_dependencies.select{|step| step.task.name.to_sym == name.to_sym}.first
   end

data/lib/rbbt/workflow/usage.rb

@@ -15,6 +15,7 @@ module Task
         puts "  #{dep.name}:"
         puts
         puts SOPT.input_doc(dep.inputs, dep.input_types, dep.input_descriptions, dep.input_defaults)
+        puts
       end
     end
   end
@@ -27,6 +28,8 @@ module Workflow
       puts self.to_s 
       puts "=" * self.to_s.length
       puts
+      puts "\n" << workflow_description if workflow_description and not workflow_description.empty?
+      puts
 
       puts "## TASKS"
       puts

data/share/lib/R/util.R

@@ -167,7 +167,38 @@ rbbt.acc <- function(data, new){
 rbbt.png_plot <- function(filename, width, height, p, ...){
     png(filename=filename, width=width, height=height, ...);
     eval(parse(text=p));
+}
+
+rbbt.heatmap <- function(filename, width, height, data, take_log=FALSE, ...){
+    require(gplots, quietly = TRUE, warn.conflicts = FALSE)
+    library(pls, quietly = TRUE, warn.conflicts = FALSE)
+    opar = par()
+    png(filename=filename, width=width, height=height);
+
+    #par(cex.lab=0.5, cex=0.5, ...)
+
+    data = as.matrix(data)
+    data[is.nan(data)] = NA
+
+    #data = data[rowSums(!is.na(data))!=0, colSums(!is.na(data))!=0]
+    data = data[rowSums(is.na(data))==0, ]
+
+    if (take_log){
+        for (study in colnames(data)){
+            skip = sum(data[, study] <= 0) != 0
+            if (!skip){
+                data[, study] = log(data[, study])
+            }
+        }
+        data = data[, colSums(is.na(data))==0]
+    }
+
+    data = stdize(data)
+
+    heatmap.2(data, margins = c(20,5), scale='column')
+
     dev.off();
+    par(opar)
 }
 
 rbbt.init <- function(data, new){

data/share/rbbt_commands/app/start

@@ -3,7 +3,8 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 
-options = SOPT.get "-e--environment*:-p--port*:-s--server*"
+options = SOPT.get "-e--environment*:-p--port*:-s--server*:-f--finder"
+options[:Port] ||= options[:port]
 
 app = ARGV.shift
 
@@ -12,8 +13,8 @@ app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : R
 app_dir = app_dir[app]
 
 server = options[:server] || 'thin'
-
 Misc.in_dir(app_dir) do
-  `env RBBT_LOG=#{Log.severity} #{options.include?(:environment)? "env RACK_ENV=#{options[:environment]}" : ""} \
-  #{server} start -p #{options[:port]} #{ARGV.collect{|a| "'#{a}'"} * " "}`
+  require 'rack'
+  ENV["RBBT_FINDER"] = true if options.include?(:finder)
+  Rack::Server.start(options.merge(:config => 'config.ru'))
 end

data/share/rbbt_commands/study/task

@@ -0,0 +1,222 @@
+#!/usr/bin/env ruby
+
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/entity/study'
+require 'rbbt/entity/study/genotypes'
+require 'rbbt/entity/study/cnv'
+require 'rbbt/entity/study/expression'
+
+def usage(workflow = nil, task = nil)
+  puts SOPT.doc
+  puts "## WORKFLOW"
+  puts
+  if workflow.nil?
+    puts "No workflow specified"
+    exit -1 
+  end
+
+  if task.nil?
+    workflow.load_tasks if workflow.respond_to? :load_tasks
+    workflow.doc
+  else
+    puts workflow.to_s
+    puts "=" * workflow.to_s.length
+    puts
+    puts workflow.workflow_description
+    puts
+    workflow.doc(task)
+  end
+
+  exit 0
+end
+
+def SOPT_options(workflow, task)
+  sopt_options = []
+  workflow.rec_inputs(task.name).each do |name|
+    short = name.to_s.chars.first
+    boolean = workflow.rec_input_types(task.name)[name].to_sym == :boolean
+    
+    sopt_options << "-#{short}--#{name}#{boolean ? '' : '*'}"
+  end
+
+  sopt_options * ":"
+end
+
+def fix_options(workflow, task, job_options)
+  option_types = workflow.rec_input_types(task.name)
+
+  job_options_cleaned = {}
+
+  job_options.each do |name, value|
+    value = case option_types[name].to_sym
+            when :float
+              value.to_f
+            when :integer
+              value.to_i
+            when :string, :text
+              case
+              when value == '-'
+                STDIN.read
+              when (String === value and File.exists?(value) and not File.directory?(value))
+                Open.read(value)
+              else
+                value
+              end
+            when :array
+              if Array === value
+                value
+              else
+                str = case
+                when value == '-'
+                  STDIN.read
+                when (String === value and File.exists?(value))
+                  Open.read(value)
+                else
+                  value
+                end
+                
+                if $array_separator
+                  str.split(/#{$array_separator}/)
+                else
+                  str.split(/[,|\s]/)
+                end
+              end
+            when :tsv
+              case value
+              when TSV
+                value
+              when '-'
+                TSV.open(STDIN)
+              else
+                TSV.open(value)
+              end
+            else
+              value
+            end
+    job_options_cleaned[name] = value
+  end
+
+  job_options_cleaned
+end
+
+options = SOPT.get <<EOF
+-h--help Show this help:
+-as--array_separator* Change the character that separates elements of Arrays, ',', '|', or '\\n' by default:
+-cl--clean Clean the last step of the job so that it gets recomputed:
+-rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely:
+-n--name* Job name to use. The name 'Default' is used by default:
+-pn--printname Print the name of the job and exit without starting it:
+-rw--require_workflow* Workflows to require, separated by commas
+EOF
+
+(options[:require_workflow] || "").split(',').each do |workflow|
+  Workflow.require_workflow workflow
+end
+
+study = ARGV.shift.dup
+if Open.exists? study
+  dir = study
+  study = Study.setup(File.basename(study))
+  study.dir = Path.setup(dir, Study)
+else
+  Study.setup(study)
+end
+workflow = study.workflow
+
+usage if workflow.nil?
+
+task     = ARGV.shift
+
+# Set log, fork, clean, recursive_clean and help
+help = !!options.delete(:help)
+do_fork = !!options.delete(:fork)
+do_exec = !!options.delete(:exec)
+clean = !!options.delete(:clean)
+recursive_clean = !!options.delete(:recursive_clean)
+$array_separator = options.delete(:array_separator)
+
+# Set task
+namespace = nil, nil
+
+case 
+when task.nil?
+  usage workflow
+when (task =~ /\./)
+  namespace, task = options.delete(:task).split('.')
+  namespace = Misc.string2const(namespace)
+else
+  task_name = task.to_sym
+  task = workflow.tasks[task_name]
+  raise "Task not found: #{ task_name }" if task.nil?
+end
+
+usage workflow, task if help
+
+name = options.delete(:name) || "Default"
+
+# get job args
+sopt_option_string = SOPT_options(workflow, task)
+job_options = SOPT.get sopt_option_string
+job_options = fix_options(workflow, task, job_options)
+
+#- get job
+
+job = study.job(task.name, job_options)
+
+# clean job
+if clean and job.done? != false
+  job.clean 
+  sleep 1
+  job = study.job(task.name, job_options)
+end
+
+if recursive_clean and job.done?
+  job.recursive_clean 
+  sleep 1
+  job = study.job(task.name, job_options)
+end
+
+# run
+if do_exec
+  res = job.exec
+  case
+  when Array === res
+    puts res * "\n"
+  when TSV === res
+    puts res
+  when Hash === res
+    puts res.to_yaml
+  else
+    puts res
+  end
+  exit 0
+end
+
+if do_fork
+  job.fork
+  while not job.done?
+    Log.debug "#{job.step}: #{job.messages.last}"
+    sleep 2
+  end
+  raise job.messages.last if job.error?
+else
+  res = job.run(true)
+end
+
+if options.delete(:printname)
+  puts job.name
+  exit 0
+else
+  Log.low "Job name: #{job.name}" 
+end
+
+if Step === res
+  puts Open.read(res.path) if File.exists? res.path
+else
+  puts res
+end
+
+exit 0