rbbt-util 5.44.1 → 6.0.3

data/lib/rbbt/workflow/step.rb

@@ -1,362 +1,2 @@
-require 'rbbt/persist'
-require 'rbbt/persist/tsv'
-require 'rbbt/util/log'
-require 'rbbt/util/semaphore'
-require 'rbbt/workflow/step/accessor'
-require 'rbbt/workflow/step/produce'
-require 'rbbt/workflow/step/status'
-require 'rbbt/workflow/step/info'
-require 'rbbt/workflow/step/save_load_inputs'
-
-class Step
-  RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
-
-  class << self
-    attr_accessor :lock_dir
-    
-    def lock_dir
-      @lock_dir ||= begin
-                      dir = Rbbt.tmp.step_info_locks.find
-                      FileUtils.mkdir_p dir unless Open.exists? dir
-                      dir
-                    end
-    end
-  end
-
-  def clean_name
-    @clean_name ||= begin
-                      info[:clean_name] || path.sub(/_[a-z0-9]{32}/, '')
-                    end
-  end
-
-
-  def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil, &block)
-    path = Path.setup(Misc.sanitize_filename(path)) if String === path
-    path = path.call if Proc === path
-
-    task = block if block_given?
-
-    @path = path
-    @task = task
-    @bindings = bindings
-    @dependencies = case
-                    when dependencies.nil? 
-                      []
-                    when Array === dependencies
-                      dependencies
-                    else
-                      [dependencies]
-                    end
-    @mutex = Mutex.new
-    @info_mutex = Mutex.new
-    @inputs = inputs 
-    NamedArray.setup @inputs, task.inputs.collect{|s| s.to_s} if task and task.respond_to? :inputs and task.inputs
-  end
-
-  def workflow
-    @workflow ||= begin
-                    wf = info[:workflow] 
-                    wf = nil if wf == ""
-                    wf ||= @task.workflow if @task && @task.respond_to?(:workflow)
-                    wf ||= path.split("/")[-3]
-                  end
-  end
-
-
-
-  def inputs
-    return @inputs if NamedArray === @inputs
-
-    load_inputs_from_info if @inputs.nil? 
-
-    task_inputs = task.respond_to?(:inputs) ? task.inputs : nil
-    NamedArray.setup(@inputs, task_inputs) unless NamedArray === @inputs
-
-    @inputs || []
-  end
-
-  def copy_files_dir
-    if File.symlink?(self.files_dir)
-      begin
-        realpath = Open.realpath(self.files_dir)
-        Open.rm self.files_dir
-        Open.cp realpath, self.files_dir
-      rescue
-        Log.warn "Copy files_dir for #{self.workflow_short_path}: " + $!.message
-      end
-    end
-  end
-
-  def recursive_inputs(connected = false)
-    if NamedArray === inputs
-      i = {}
-      inputs.zip(inputs.fields).each do |v,f|
-        i[f] = v
-      end
-    else
-      i = {}
-    end
-    rec_dependencies(connected).each do |dep|
-      next unless NamedArray === dep.inputs
-
-      dep.inputs.zip(dep.inputs.fields).each do |v,f|
-        if i.include?(f) && i[f] != v
-          next
-          #Log.debug "Variable '#{f}' reused with different values: #{[Misc.fingerprint(i[f]), Misc.fingerprint(v)] * " <-> "}"
-        else 
-          i[f] = v
-        end
-      end
-
-      dep.archived_inputs.each do |k,v|
-        i[k] = v unless i.include? k
-      end
-    end
-
-    self.archived_inputs.each do |k,v|
-      i[k] = v unless i.include? k
-    end
-
-    #dependencies.each do |dep|
-    #  di = dep.recursive_inputs
-    #  next unless NamedArray === di
-    #  di.fields.zip(di).each do |k,v|
-    #    i[k] = v unless i.include? k
-    #  end
-    #end
-    
-    v = i.values
-    NamedArray.setup v, i.keys 
-    v
-  end
-
-
-  def path
-    if Proc === @path
-      @path = Path.setup(Misc.sanitize_filename(@path.call))
-    else
-      @path
-    end
-  end
-
-  class << self
-    attr_accessor :log_relay_step
-  end
-
-  
-  # May be deprecated: This is no loger used
-  def relay_log(step)
-    return self if self.task_name.nil?
-
-    if not self.respond_to? :original_log
-      class << self
-        attr_accessor :relay_step
-        alias original_log log 
-        def log(status, message = nil)
-          self.status = status
-          message Log.uncolor message
-          relay_step.log([self.task_name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
-        end
-      end
-    end
-    @relay_step = step
-
-    self
-  end
-
-  def prepare_result(value, description = nil, entity_info = nil)
-    res = case 
-          when IO === value
-            begin
-              res = case result_type
-                    when :array
-                      array = []
-                      while line = value.gets
-                        array << line.chomp
-                      end
-                      array
-                    when :tsv
-                      begin
-                        TSV.open(value)
-                      rescue IOError
-                        TSV.setup({})
-                      end
-                    else
-                      value.read
-                    end
-              value.join if value.respond_to? :join
-              res
-            rescue Exception
-              value.abort if value.respond_to? :abort
-              self.abort
-              raise $!
-            end
-          when (not defined? Entity or description.nil? or not Entity.formats.include? description)
-            value
-          when (Annotated === value and info.empty?)
-            value
-          when Annotated === value
-            annotations = value.annotations
-            entity_info ||= begin 
-                              entity_info = info.dup
-                              entity_info.merge! info[:inputs] if info[:inputs]
-                              entity_info
-                            end
-            entity_info.each do |k,v|
-              value.send("#{h}=", v) if annotations.include? k
-            end
-
-            value
-    else
-      entity_info ||= begin 
-                        entity_info = info.dup
-                        entity_info.merge! info[:inputs] if info[:inputs]
-                        entity_info
-                      end
-      Entity.formats[description].setup(value, entity_info.merge(:format => description))
-    end
-
-    if Annotated === res
-      dep_hash = nil
-      res.annotations.each do |a|
-        a = a.to_s
-        varname = "@" + a
-        next unless res.instance_variable_get(varname).nil? 
-
-        dep_hash ||= begin
-                       h = {}
-                       rec_dependencies.each{|dep| h[dep.task_name.to_s] ||= dep }
-                       h
-                     end
-        dep = dep_hash[a]
-        next if dep.nil?
-        res.send(a.to_s+"=", dep.load)
-      end 
-    end
-
-    res
-  end
-
-
-  def child(&block)
-    child_pid = Process.fork &block
-    children_pids = info[:children_pids]
-    if children_pids.nil?
-      children_pids = [child_pid]
-    else
-      children_pids << child_pid
-    end
-    set_info :children_pids, children_pids
-    child_pid
-  end
-
-  def cmd(*args)
-    all_args = *args
-
-    all_args << {} unless Hash === all_args.last
-
-    level = all_args.last[:log] || 0
-    level = 0 if TrueClass === level
-    level = 10 if FalseClass === level
-    level = level.to_i
-
-    all_args.last[:log] = true
-    all_args.last[:pipe] = true
-
-    io = CMD.cmd(*all_args)
-    child_pid = io.pids.first
-
-    children_pids = info[:children_pids]
-    if children_pids.nil?
-      children_pids = [child_pid]
-    else
-      children_pids << child_pid
-    end
-    set_info :children_pids, children_pids
-
-    while c = io.getc
-      STDERR << c if Log.severity <= level
-      if c == "\n"
-        if pid
-          Log.logn "STDOUT [#{pid}]: ", level
-        else
-          Log.logn "STDOUT: ", level
-        end
-      end
-    end 
-
-    io.join
-
-    nil
-  end
-
-
-  def load
-    res = begin
-            @result = nil if IO === @result && @result.closed?
-            if @result && @path != @result &&  ! StreamArray === @result
-              res = @result
-            else
-              join if not done?
-              res = @path.exists? ? Persist.load_file(@path, result_type) : run
-            end
-
-            if result_description
-              entity_info = info.dup
-              # Also load entity_info in rbbt6
-              entity_info.merge! info[:inputs] if Hash === info[:inputs]
-              res = prepare_result res, result_description, entity_info 
-            end
-
-            res
-          rescue IOError
-            if @result
-              @result = nil
-              retry
-            end
-            raise $!
-          ensure
-            @result = nil if IO === @result
-          end
-
-    res
-  end
-
-  def update
-    if dirty?
-      dependencies.collect{|d| d.update } if dependencies
-      clean
-    end
-  end
-
-  def writable?
-    Open.writable?(self.path) && Open.writable?(self.info_file)
-  end
-
-  def recursive_clean
-    dependencies.each do |step| 
-      step.recursive_clean 
-    end if dependencies
-    clean if Open.exists?(self.info_file)
-    self
-  end
-
-  def step(name)
-    @steps ||= {}
-    @steps[name] ||= begin
-                       name = name.to_sym
-                       deps = rec_dependencies.select{|step| 
-                         step.task_name && step.task_name.to_sym == name
-                       }
-                       raise "Dependency step not found: #{ name }" if deps.empty?
-                       if (deps & self.dependencies).any?
-                         (deps & self.dependencies).last
-                       else
-                         deps.last
-                       end
-                     end
-  end
-end
-
-require 'rbbt/workflow/step/run'
-require 'rbbt/workflow/step/accessor'
+require_relative '../refactor'
+Rbbt.require_instead 'scout/workflow/step'

data/lib/rbbt/workflow/task.rb

@@ -1,118 +1,118 @@
-require 'rbbt/util/misc'
-require 'rbbt/persist'
-
-module Task
-  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow, :resumable
-
-  def self.setup(options = {}, &block)
-    block.extend Task
-    options = IndiferentHash.setup options
-    block.singleton_methods.
-      select{|method| method.to_s[-1] != "="[0]}.each{|method|
-      if block.respond_to?(method.to_s + "=") and options.include? method.to_sym
-        block.send(method.to_s + '=', options[method.to_sym]) 
-      end
-    }
-    block
-  end
-
-  def parse_description
-    if description =~ /\n\n/
-      short_description, rest = description.match(/(.*?)\n\n(.*)/).values_at 1, 2
-    else
-      short_description = description
-      rest = nil
-    end
-
-    if rest.nil?
-      long_description = ""
-    end
-  end
-
-  def take_input_values(input_values)
-    return [] if @inputs.nil?
-    values = []
-    defaults = IndiferentHash.setup(@input_defaults || {})
-    @inputs.each do |input|
-      value = input_values[input]
-      value = defaults[input] if value.nil?
-      values << value
-    end
-    values
-  end
-
-  def exec(*args)
-    case
-    when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
-      self.call *take_input_values(IndiferentHash.setup(args.first))
-    else
-      self.call *args
-    end
-  end
-
-  def exec_in(object, *args)
-    case
-    when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
-      object.instance_exec *IndiferentHash.setup(args.first).values_at(*inputs), &self
-    else
-      object.instance_exec *args, &self 
-    end
-  end
-
-  def persist_exec(filename, *args)
-    Persist.persist "Task", @persistence_type, :file => filename do
-      exec *args
-    end
-  end
-
-  def persist_exec_in(filename, *args)
-    Persist.persist "Task", @persistence_type, :file => filename do
-      exec_in *args
-    end
-  end
-
-  def self.dep_inputs(deps, workflow = nil)
-    seen = []
-    task_inputs = {}
-    deps.each do |dep|
-      if Symbol === dep
-        wf, task = [workflow, workflow.tasks[dep.to_sym]]
-      elsif Array === dep and dep.first
-        wf, task_name, options = dep
-        options, task_name = task_name, nil if Hash === task_name
-        options, wf = wf, nil if Hash === wf
-        task_name, wf = wf, workflow if task_name.nil? and Symbol === wf or String === wf
-        next if task_name.nil?
-        task = wf.tasks[task_name.to_sym]
-      else
-        next
-      end
-
-      task_name ||= task.name
-
-      maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
-      raise "Dependency task not found: #{dep}" if task.nil?
-      next if seen.include? [wf, task_name, maps]
-
-      task.workflow = wf if wf
-
-      seen << [wf, task_name, maps]
-      new_inputs = task.inputs - maps
-      next unless new_inputs.any?
-      if task_inputs[task].nil?
-        task_inputs[task] = new_inputs
-      else
-        task_inputs[task] = (task_inputs[task] + new_inputs).uniq
-      end
-    end
-    task_inputs
-  end
-
-  def dep_inputs(deps, workflow = nil)
-    return {} if deps.empty?
-    task_inputs = Task.dep_inputs deps, workflow
-    task_inputs.each do |task, inputs|
-      inputs.replace (inputs - self.inputs)
-    end
-  end
-end
+#require 'rbbt/util/misc'
+#require 'rbbt/persist'
+#
+#module Task
+#  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow, :resumable
+#
+#  def self.setup(options = {}, &block)
+#    block.extend Task
+#    options = IndiferentHash.setup options
+#    block.singleton_methods.
+#      select{|method| method.to_s[-1] != "="[0]}.each{|method|
+#      if block.respond_to?(method.to_s + "=") and options.include? method.to_sym
+#        block.send(method.to_s + '=', options[method.to_sym]) 
+#      end
+#    }
+#    block
+#  end
+#
+#  def parse_description
+#    if description =~ /\n\n/
+#      short_description, rest = description.match(/(.*?)\n\n(.*)/).values_at 1, 2
+#    else
+#      short_description = description
+#      rest = nil
+#    end
+#
+#    if rest.nil?
+#      long_description = ""
+#    end
+#  end
+#
+#  def take_input_values(input_values)
+#    return [] if @inputs.nil?
+#    values = []
+#    defaults = IndiferentHash.setup(@input_defaults || {})
+#    @inputs.each do |input|
+#      value = input_values[input]
+#      value = defaults[input] if value.nil?
+#      values << value
+#    end
+#    values
+#  end
+#
+#  def exec(*args)
+#    case
+#    when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
+#      self.call *take_input_values(IndiferentHash.setup(args.first))
+#    else
+#      self.call *args
+#    end
+#  end
+#
+#  def exec_in(object, *args)
+#    case
+#    when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
+#      object.instance_exec *IndiferentHash.setup(args.first).values_at(*inputs), &self
+#    else
+#      object.instance_exec *args, &self 
+#    end
+#  end
+#
+#  def persist_exec(filename, *args)
+#    Persist.persist "Task", @persistence_type, :file => filename do
+#      exec *args
+#    end
+#  end
+#
+#  def persist_exec_in(filename, *args)
+#    Persist.persist "Task", @persistence_type, :file => filename do
+#      exec_in *args
+#    end
+#  end
+#
+#  def self.dep_inputs(deps, workflow = nil)
+#    seen = []
+#    task_inputs = {}
+#    deps.each do |dep|
+#      if Symbol === dep
+#        wf, task = [workflow, workflow.tasks[dep.to_sym]]
+#      elsif Array === dep and dep.first
+#        wf, task_name, options = dep
+#        options, task_name = task_name, nil if Hash === task_name
+#        options, wf = wf, nil if Hash === wf
+#        task_name, wf = wf, workflow if task_name.nil? and Symbol === wf or String === wf
+#        next if task_name.nil?
+#        task = wf.tasks[task_name.to_sym]
+#      else
+#        next
+#      end
+#
+#      task_name ||= task.name
+#
+#      maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
+#      raise "Dependency task not found: #{dep}" if task.nil?
+#      next if seen.include? [wf, task_name, maps]
+#
+#      task.workflow = wf if wf
+#
+#      seen << [wf, task_name, maps]
+#      new_inputs = task.inputs - maps
+#      next unless new_inputs.any?
+#      if task_inputs[task].nil?
+#        task_inputs[task] = new_inputs
+#      else
+#        task_inputs[task] = (task_inputs[task] + new_inputs).uniq
+#      end
+#    end
+#    task_inputs
+#  end
+#
+#  def dep_inputs(deps, workflow = nil)
+#    return {} if deps.empty?
+#    task_inputs = Task.dep_inputs deps, workflow
+#    task_inputs.each do |task, inputs|
+#      inputs.replace (inputs - self.inputs)
+#    end
+#  end
+#end