rbbt-util 3.2.1 → 4.0.0

data/lib/rbbt/workflow/annotate.rb

@@ -0,0 +1,60 @@
+module AnnotatedModule
+  def self.extended(base)
+    if not base.respond_to? :inputs
+      class << base
+        attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults
+
+        def description
+          i = @description; @description = ""; i
+        end
+
+        def inputs
+          i = @inputs; @inputs = []; i
+        end
+
+        def input_types
+          i = @input_types; @input_types = {}; i
+        end
+
+        def input_descriptions
+          i = @input_descriptions; @input_descriptions = {}; i
+        end
+
+        def input_defaults
+          i = @input_defaults; @input_defaults = {}; i
+        end
+
+        def description
+          i = @description; @description = ""; i
+        end
+      end
+
+      base.description = ""
+      base.inputs = []
+      base.input_types = {}
+      base.input_descriptions = {}
+      base.input_defaults = {}
+
+    end
+  end
+
+  def desc(description)
+    @description = description
+  end
+
+  def dep(*dependencies, &block)
+    dependencies << block if block_given?
+    @dependencies.concat dependencies
+  end
+
+  def input(name, type = nil, desc = nil, default = nil)
+    name = name.to_sym
+    type = type.to_sym
+    @inputs << name
+    @input_types[name] = type unless type.nil?
+    @input_descriptions[name] = desc unless desc.nil?
+    @input_defaults[name] = default unless default.nil?
+  end
+end
+
+

data/lib/rbbt/workflow/soap.rb

@@ -0,0 +1,100 @@
+require 'base64'
+require 'simplews'
+
+class WorkflowSOAP < SimpleWS
+  attr_accessor :workflow
+  def job(jobid)
+    workdir = @workflow.workdir
+    if workdir.respond_to? :find
+      workdir_find = workdir.find 
+    else
+      workdir_find = workdir
+    end
+ 
+    @workflow.load(File.join(workdir_find, jobid))
+  end
+
+  def initialize(workflow, *args)
+    super(workflow.to_s,*args)
+    @workflow = workflow
+    @workflow.synchronous_exports.each do |name| synchronous name end
+    @workflow.asynchronous_exports.each do |name| asynchronous name end
+
+    desc "Job management: Check the status of a job"
+    param_desc :jobid => "Job identifier", :return => "Status code. Special status codes are: 'done' and 'error'"
+    serve :status, [:jobid], :jobid => :string, :return => :string do |jobid|
+      (job(jobid).status || :queued).to_s
+    end
+
+    desc "Job management: Return an array with the messages issued by the job"
+    param_desc :jobid => "Job identifier", :return => "Array with message strings"
+    serve :messages, ['jobid'], :job => :string, :return => :array do |jobid|
+      job(jobid).messages || []
+    end
+
+    desc "Job management: Return a YAML string containing arbitrary information set up by the job"
+    param_desc :jobid => "Job identifier", :return => "Hash with arbitrary values in YAML format"
+    serve :info, ['jobid'], :jobid => :string, :return => :string do |jobid|
+      job(jobid).info.to_yaml
+    end
+
+    desc "Job management: Abort the job"
+    param_desc :jobid => "Job identifier"
+    serve :abort, %w(jobid), :jobid => :string, :return => false do |jobid|
+      job(jobid).abort
+    end
+
+    desc "Job management: Check if the job is done. Could have finished successfully, with error, or have been aborted"
+    param_desc :jobid => "Job identifier", :return => "True if the job has status 'done', 'error' or 'aborted'"
+    serve :done, %w(jobid), :jobid => :string, :return => :boolean do |jobid|
+      [:done, :error, :aborted].include?((job(jobid).status || :queued).to_sym)
+    end
+
+    desc "Job management: Check if the job has finished with error. The last message is the error message"
+    param_desc :jobid => "Job identifier", :return => "True if the job has status 'error'"
+    serve :error, %w(jobid), :jobid => :string, :return => :boolean do |jobid|
+      job(jobid).status.to_sym == :error
+    end
+
+    desc "Job management: Load job result as string "
+    param_desc :jobid => "Job identifier", :return => "String containing the result of the job"
+    serve :load_string, %w(jobid), :jobid => :string, :return => :string do |jobid|
+      job(jobid).load.to_s
+    end
+  end
+
+  def synchronous(*tasknames)
+    tasknames.each do |name|
+      task = @workflow.tasks[name]
+      desc @workflow.task_description[name] if @workflow.task_description.include? name
+
+      rec_inputs = @workflow.rec_inputs name
+      rec_input_descriptions= @workflow.rec_input_descriptions name
+      rec_input_types= @workflow.rec_input_types name
+
+      param_desc rec_input_descriptions
+      serve name, rec_inputs, rec_input_types, &task
+    end
+  end
+
+  def asynchronous(*tasknames)
+    tasknames.each do |name|
+      task = @workflow.tasks[name]
+      desc @workflow.task_description[name] if @workflow.task_description.include? name
+
+      rec_inputs = @workflow.rec_inputs name
+      rec_input_descriptions= @workflow.rec_input_descriptions name
+      rec_input_types= @workflow.rec_input_types name
+
+      param_desc rec_input_descriptions.merge(:suggested_name => "Suggested Name", :return => "Job identifier")
+      serve name, [:suggested_name] + rec_inputs, rec_input_types.merge(:suggested_name => :string, :return => :string) do |jobname, *inputs|
+        inputs = Hash[*@workflow.rec_inputs(name).zip(inputs).flatten]
+    
+        step = @workflow.job(name, jobname, inputs)
+        step.fork
+        @workflow.id_for step.path
+      end
+    end
+  end
+end
+

data/lib/rbbt/workflow/step.rb

@@ -0,0 +1,102 @@
+require 'rbbt/persist'
+require 'rbbt/persist/tsv'
+require 'rbbt/util/log'
+require 'rbbt/workflow/accessor'
+
+class Step
+  attr_accessor :path, :task, :inputs, :dependencies
+  attr_accessor :pid
+
+  class Aborted < Exception; end
+
+  def initialize(path, task = nil, inputs = nil, dependencies = nil)
+    @path = path
+    @task = task
+    @dependencies = case
+                    when dependencies.nil? 
+                      []
+                    when Array === dependencies
+                      dependencies
+                    else
+                      [dependencies]
+                    end
+    @inputs = inputs || []
+  end
+
+  def exec
+    @task.exec_in self, *@inputs
+  end
+
+  def join
+    if @pid.nil?
+      while not done? do
+        Log.debug "Waiting: #{info[:step]}"
+        sleep 5
+      end
+    else
+      Process.waitpid @pid
+      @pid = nil
+    end
+    self
+  end
+
+  def run
+    Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq do
+      log task.name, "Starting task: #{ name }"
+      set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
+      @dependencies.each{|dependency| dependency.run}
+      set_info :status, :start
+      set_info :inputs, Misc.zip2hash(task.inputs, @inputs) unless task.inputs.nil?
+      res = exec
+      set_info :status, :done
+      res
+    end
+  end
+
+  def fork
+    raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil?
+    @pid = Process.fork do
+      begin
+        trap(:INT) { raise Step::Aborted.new "INT signal recieved" }
+        run
+      rescue Exception
+        log(:error, "#{$!.class}: #{$!.message}")
+      end
+    end
+    set_info :pid, @pid
+    self
+  end
+
+  def abort
+    if @pid.nil?
+      false
+    else
+      Process.kill("INT", @pid)
+      true
+    end
+  end
+
+  def load
+    raise "Can not load: Step is waiting for proces #{@pid} to finish" if not done?
+    Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path} do
+      exec
+    end
+  end
+
+  def clean
+    if File.exists?(path) or File.exists?(info_file)
+      begin
+        FileUtils.rm info_file
+        FileUtils.rm path 
+        FileUtils.rm_rf files_dir
+      end
+    end
+  end
+
+  def rec_dependencies
+    @dependencies.collect{|step| step.rec_dependencies}.flatten.concat  @dependencies
+  end
+  def step(name)
+    rec_dependencies.select{|step| step.task.name.to_sym == name.to_sym}.first
+  end
+end

data/lib/rbbt/workflow/task.rb

@@ -0,0 +1,76 @@
+require 'rbbt/util/misc'
+require 'rbbt/persist'
+
+module Task
+  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :description, :name
+
+  def self.setup(options = {}, &block)
+    block.extend Task
+    options = IndiferentHash.setup options
+    block.singleton_methods.
+      select{|method| method.to_s[-1] != "="[0]}.each{|method|
+      if block.respond_to?(method + "=") and options.include? method.to_sym
+        block.send(method + '=', options[method.to_sym]) 
+      end
+    }
+    block
+  end
+
+  def parse_description
+    if description =~ /\n\n/
+      short_description, rest = description.match(/(.*?)\n\n(.*)/).values_at 1, 2
+    else
+      short_description = description
+      rest = nil
+    end
+
+    if rest.nil?
+      long_description = ""
+    end
+  end
+
+  def param_options
+  end
+
+  def take_input_values(input_values)
+    return [] if @inputs.nil?
+    values = []
+    @inputs.each do |input|
+      value = input_values[input] ||
+        IndiferentHash.setup(@input_defaults || {})[input]
+      values << value
+    end
+    values
+  end
+
+  def exec(*args)
+    case
+    when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
+      self.call *take_input_values(IndiferentHash.setup(args.first))
+    else
+      self.call *args
+    end
+  end
+
+  def exec_in(object, *args)
+    case
+    when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
+      object.instance_exec *IndiferentHash.setup(args.first).values_at(*inputs), &self
+    else
+      object.instance_exec *args, &self 
+    end
+  end
+
+  def persist_exec(filename, *args)
+    Persist.persist "Task", @persistence_type, :file => filename do
+      exec *args
+    end
+  end
+
+  def persist_exec_in(filename, *args)
+    Persist.persist "Task", @persistence_type, :file => filename do
+      exec_in *args
+    end
+  end
+
+end

data/test/rbbt/resource/test_path.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '../../../test_helper')
+require 'rbbt/resource/path'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+class TestTSV < Test::Unit::TestCase
+
+  def test_prev
+    path = Path.setup "/tmp"
+    assert_equal "/tmp/bar/foo", path.foo("bar")
+  end
+end

data/test/rbbt/test_annotations.rb

@@ -0,0 +1,106 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+require 'rbbt/annotations'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+module AnnotatedString
+  extend Annotation
+  self.annotation :annotation_str
+
+  def add_annot
+    self + annotation_str
+  end
+end
+
+module AnnotatedString2
+  extend Annotation
+  include AnnotatedString
+  self.annotation :annotation_str2
+end
+
+class TestAnnotations < Test::Unit::TestCase
+
+  def test_annotated_string
+    assert_equal %w(annotation_str), AnnotatedString.annotations.collect{|a| a.to_s}
+  end
+
+  def test_string
+    str = "string"
+    annotation_str = "Annotation String"
+    AnnotatedString.setup(str, annotation_str)
+    assert_equal [AnnotatedString], str.annotation_types
+    assert_equal annotation_str, str.annotation_str
+  end
+
+  def test_array
+    ary = ["string"]
+    annotation_str = "Annotation String"
+    AnnotatedArray.setup_chain(ary)
+    AnnotatedString.setup(ary, annotation_str)
+    assert_equal [AnnotatedString], ary.annotation_types
+    assert_equal annotation_str, ary.annotation_str
+    assert_equal annotation_str, ary[0].annotation_str
+  end
+
+  def test_info
+    ary = ["string"]
+    annotation_str = "Annotation String"
+    AnnotatedString.setup(ary, annotation_str)
+    assert_equal({:annotation_str => annotation_str, :annotation_types => [AnnotatedString]}, ary.info)
+  end
+
+  def test_load
+    str = "string"
+    annotation_str = "Annotation String"
+    info = {:annotation_str => annotation_str, :annotation_types => [AnnotatedString]}
+
+    Annotated.load(str, info)
+    assert_equal annotation_str, str.annotation_str
+  end
+
+  def test_tsv
+    str1 = "string1"
+    annotation_str1 = "Annotation String 1"
+    str2 = "string2"
+    annotation_str2 = "Annotation String 2"
+    AnnotatedString.setup(str1, annotation_str1)
+    AnnotatedString.setup(str2, annotation_str2)
+    assert_equal str1, Annotated.tsv([str1, str2], :all)[str1.id]["literal"] 
+    assert_equal annotation_str1, Annotated.tsv([str1, str2], :annotation_str, :JSON)[str1.id]["annotation_str"] 
+  end
+
+  def test_literal
+    str = "string"
+    annotation_str = "Annotation String"
+    AnnotatedString.setup(str, annotation_str)
+    assert_equal ["string"], str.tsv_values("literal")
+  end
+
+  def test_load_tsv
+    str1 = "string1"
+    annotation_str1 = "Annotation String 1"
+    str2 = "string2"
+    annotation_str2 = "Annotation String 2"
+    AnnotatedString.setup(str1, annotation_str1)
+    AnnotatedString.setup(str2, annotation_str2)
+    assert_equal annotation_str1, Annotated.load_tsv(Annotated.tsv([str1, str2], :all)).sort.first.annotation_str
+    assert_equal str1, Annotated.load_tsv(Annotated.tsv([str1, str2], :literal, :JSON)).sort.first
+  end
+
+  def test_inheritance
+    str = "string1"
+    annotation_str1 = "Annotation String 1"
+    annotation_str2 = "Annotation String 2"
+    assert_equal [AnnotatedString], AnnotatedString2.inheritance
+    AnnotatedString2.setup(str, annotation_str1, annotation_str2)
+    assert_equal annotation_str1, str.annotation_str
+    assert_equal annotation_str2, str.annotation_str2
+  end
+
+  def test_annotation_methods
+    str = "string"
+    annotation_str = "Annotation String"
+    AnnotatedString.setup(str, annotation_str)
+    assert_equal str + annotation_str, str.add_annot
+  end
+end