rbbt-util 5.26.77 → 5.26.78

data/lib/rbbt/workflow/definition.rb

@@ -80,6 +80,8 @@ module Workflow
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist? dep.files_dir
         Open.ln_h dep.path, self.tmp_path
       else
+        Open.rm_rf self.files_dir 
+        Open.link dep.files_dir, self.files_dir
         Open.link dep.path, self.path
       end
       nil

data/lib/rbbt/workflow/integration/nextflow.rb

@@ -0,0 +1,37 @@
+module Workflow
+  def nextflow_file(file, name = nil)
+    file = file + '.nf' unless File.exists?(file) || ! File.exists?(file + '.nf')
+    file = File.expand_path(file)
+    name ||= File.basename(file).sub(/\.nf$/,'')
+    params = Open.read(file).scan(/params\.\w+/).collect{|p| p.split(".").last}.uniq
+
+    params.each do |param|
+      input param, :string
+    end
+    task name => :text do 
+      work = file('work')
+      output = file('output')
+      profile = config :profile, :nextflow
+      Misc.in_dir output do
+        if profile
+          cmd("nextflow run -work-dir #{work} -name #{clean_name} -ansi-log false  -profile #{profile} #{file}", inputs.to_hash.merge('add_option_dashes' => true))
+        else
+          cmd("nextflow run -work-dir #{work} -name #{clean_name} -ansi-log false #{file}", inputs.to_hash.merge('add_option_dashes' => true))
+        end
+      end
+    end
+  end
+
+  def nextflow_dir(path)
+    main = File.join(path, 'main.nf')
+    nextflow_file main, File.basename(path)
+  end
+
+  def nextflow(path)
+    if File.directory?(path)
+      nextflow_dir path
+    else
+      nextflow_file path
+    end
+  end
+end

data/lib/rbbt/workflow/remote/client.rb

@@ -0,0 +1,69 @@
+require 'json'
+require 'rbbt/workflow'
+require 'rbbt/workflow/step'
+require 'rbbt/util/misc'
+
+require 'rbbt/workflow/remote/remote_step'
+
+require 'rbbt/workflow/remote/rest/get'
+require 'rbbt/workflow/remote/rest/adaptor'
+
+require 'rbbt/workflow/remote/ssh/get'
+require 'rbbt/workflow/remote/ssh/adaptor'
+
+class WorkflowRemoteClient 
+  include Workflow
+
+  attr_accessor :url, :name, :exec_exports, :synchronous_exports, :asynchronous_exports, :stream_exports
+
+  def initialize(url, name)
+    Log.debug{ "Loading remote workflow #{ name }: #{ url }" }
+    @url, @name = url, name
+
+    rest = url.include?('ssh://') ? false : true
+
+    if rest
+      self.extend WorkflowRESTClient
+    else
+      self.extend WorkflowSSHClient
+    end
+    init_remote_tasks
+  end
+  
+  def to_s
+    name
+  end
+
+  def job(task, name, inputs)
+    task_info = task_info(task)
+    fixed_inputs = {}
+    input_types = IndiferentHash.setup(task_info[:input_types])
+
+    inputs.each do |k,v| 
+      k = k.to_sym
+      if TSV === v
+        fixed_inputs[k] = v.to_s
+      else
+        next if input_types[k].nil?
+        case input_types[k].to_sym
+        when :tsv, :array, :file, :text
+          fixed_inputs[k] = (String === v and Open.exists?(v)) ? Open.open(v) : v
+        else
+          fixed_inputs[k] = v
+        end
+      end
+    end
+
+    stream_input = @can_stream ? task_info(task)[:input_options].select{|k,o| o[:stream] }.collect{|k,o| k }.first : nil
+    RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task), @stream_exports.include?(task), stream_input)
+  end
+
+  def load_id(id)
+    task, name = id.split("/")
+    step = RemoteStep.new url, task, nil
+    step.name = name
+    step.result_type = task_info(task)[:result_type]
+    step.result_description = task_info(task)[:result_description]
+    step
+  end
+end

data/lib/rbbt/workflow/remote/remote_step.rb

@@ -0,0 +1,308 @@
+class WorkflowRemoteClient
+  class RemoteStep < Step
+
+    attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec, :is_stream, :stream_input
+
+    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, is_stream = false, stream_input = nil)
+      @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec, @is_stream, @stream_input = base_url, task, base_name, inputs, result_type, result_description, is_exec, is_stream, stream_input
+      @base_url = "http://" << @base_url unless @base_url =~ /^[a-z]+:\/\//
+      @mutex = Mutex.new
+      rest = base_url.include?('ssh:') ? false : true
+
+      if rest
+        @adaptor = WorkflowRESTClient
+      else
+        @adaptor = WorkflowSSHClient
+      end
+
+      self.extend @adaptor
+    end
+
+    def clean_name
+      @base_name
+    end
+
+    def run(no_load = false)
+      no_load = @is_stream ? :stream : true if no_load
+
+      @mutex.synchronize do
+        @result ||= begin
+                      if @is_exec
+                        exec(no_load)
+                      elsif no_load == :stream
+                        _run_job(:stream)
+                      elsif no_load
+                        init_job 
+                        nil
+                      else
+                        init_job 
+                        join
+                        self.load
+                      end
+                    ensure
+                      @started = true
+                    end
+      end
+
+      return @result if no_load == :stream
+      no_load ? Misc.add_GET_param(path, "_format", "raw") : @result
+    end
+
+
+    def self.get_streams(inputs, stream_input = nil)
+      new_inputs = {}
+      inputs.each do |k,v|
+        stream = stream_input.to_s == k.to_s
+        if Step === v 
+          unless (v.done? or v.streaming?) # or RestClient::Step === v)
+            v.run(true) and v.grace 
+          end
+
+          begin
+            if stream
+              new_inputs[k] = TSV.get_stream(v)
+            else
+              new_inputs[k] = v.load
+            end
+          rescue Exception
+            raise $!
+          end
+        else
+          new_inputs[k] = v
+        end
+      end
+      new_inputs
+    end
+
+    def get_streams
+      return if @inputs_done
+      @inputs = RemoteStep.get_streams @inputs, @stream_input
+      @inputs_done = true
+      @inputs
+    end
+
+    def dup_inputs
+      return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
+      Log.low "Dupping inputs for #{path}"
+      dupped_inputs = {}
+      @inputs.collect do |k,input|
+        dupped_inputs[k] = Step.dup_stream input
+      end
+      @inputs = dupped_inputs
+      @dupped = true
+    end
+
+    def name
+      return nil if @is_exec
+      return @path if @url.nil?
+      (Array === @url ? @url.first : @url).split("/").last
+    end
+
+    def name=(name)
+      @url = [base_url,task, name] * "/"
+    end
+
+    def task_name
+      return task if task
+      init_job
+      (Array === @url ? @url.first : @url).split("/")[-2]
+    end
+
+    def nopid?
+      false
+    end
+
+    def info(check_lock=false)
+      @done = @info && @info[:status] && (@info[:status].to_sym == :done || @info[:status].to_sym == :error)
+
+      if !@done && (@last_info_time.nil? || (Time.now - @last_info_time) > 0.5)
+        update = true 
+      else
+        update = false
+      end
+
+      @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => update) do
+        @last_info_time = Time.now
+        init_job unless @url
+        info = @adaptor.get_json(File.join(@url, 'info'))
+        info = @adaptor.fix_hash(info)
+        info[:status] = info[:status].to_sym if String === info[:status]
+        info
+      end
+      @info
+    end
+
+    def status
+      return :done if @done
+      return nil unless url or started?
+      #return :streaming if @streaming 
+      begin
+        status = info[:status]
+        @done = true if status and status.to_sym == :done
+        status
+      rescue
+        nil
+      ensure
+        @info = nil
+      end
+    end
+
+    def started?
+      @result != nil || @started || @streaming
+    end
+
+    def done?
+      init_job unless @url
+      @done || status.to_s == 'done' || status.to_s == 'noinfo'
+    end
+
+    def files
+      @adaptor.get_json(File.join(url, 'files'))
+    end
+
+    def file(file)
+      @adaptor.get_raw(File.join(url, 'file', file))
+    end
+
+    def get_stream
+      case @result
+      when IO 
+        @result
+      when String
+        StringIO.new @result
+      else
+        nil
+      end
+    end
+
+    def grace
+      produce unless @started
+      sleep 0.1 unless started?
+      sleep 0.5 unless started?
+      sleep 1 unless started?
+      while not (done? or started?)
+        sleep 1 
+      end
+    end
+
+    #{{{ MANAGEMENT
+
+
+    def path
+      if @url
+        Misc.add_GET_param(@url, "_format", "raw")
+      else
+        [base_url, task, @base_name + '-' +  Misc.fingerprint(inputs)] * "/"
+      end
+    end
+
+    def fork(noload=false, semaphore=nil)
+      init_job(:asynchronous)
+    end
+
+    def running?
+      ! %w(done error aborted noinfo).include? status.to_s
+    end
+
+    def exec(noload = false)
+      @result ||= begin
+                    if noload == :stream
+                      _run_job(:exec)
+                    else
+                      exec_job 
+                    end
+                  ensure
+                    @started = true
+                  end
+    end
+
+    def join
+      init_job unless @url
+      Log.debug{ "Joining RestClient: #{path}" }
+      if IO === @result
+        res = @result
+        @result = nil
+        Misc.consume_stream(res, true) 
+      end
+
+      if not (self.done? || self.aborted? || self.error?)
+        self.info 
+        return self if self.done? || self.aborted? || self.error?
+        sleep 0.2 unless self.done? || self.aborted? || self.error?
+        sleep 1 unless self.done? || self.aborted? || self.error?
+        while not (self.done? || self.aborted? || self.error?)
+          sleep 3
+        end
+      end
+
+      self
+    end
+
+    def load_res(res, result_type = nil)
+      stream = true if res.respond_to? :read
+      join unless stream
+      result_type ||= self.result_type
+      case result_type
+      when :string
+        stream ? res.read : res
+      when :boolean
+        (stream ? res.read : res) == 'true'
+      when :tsv
+        if stream
+          TSV.open(res, :monitor => true)
+        else
+          TSV.open(StringIO.new(res))
+        end
+      when :annotations
+        if stream
+          Annotated.load_tsv(TSV.open(res))
+        else
+          Annotated.load_tsv(TSV.open(StringIO.new(res)))
+        end
+      when :array
+        (stream ? res.read : res).split("\n")
+        res.split("\n")
+      else
+        json_text = if IO === res
+                      res.read
+                    else
+                      res
+                    end
+        begin
+          JSON.parse json_text
+        rescue
+          case
+          when json_text =~ /^\d+$/
+            json_text.to_i
+          when json_text =~ /^\d+\.\d/
+            json_text.to_f
+          else
+            raise $!
+          end
+        end
+      end
+    end
+
+    def _restart
+      @done = nil
+      @name = nil
+      @started = nil
+      @aborted = nil
+      new_inputs = {}
+      inputs.each do |k,i| 
+        if File === i 
+          new_inputs[k] = File.open(i.path)
+        else
+          new_inputs[k] = i
+        end
+      end
+      @inputs = new_inputs
+      @info = nil
+    end
+
+    def input_checks
+      []
+    end
+
+  end
+end

data/lib/rbbt/workflow/remote/rest/adaptor.rb

@@ -0,0 +1,158 @@
+require 'rest-client'
+
+module WorkflowRESTClient
+  def self.__prepare_inputs_for_restclient(inputs)
+    inputs.each do |k,v|
+      if v.respond_to? :path and not v.respond_to? :original_filename
+        class << v
+          def original_filename
+            File.expand_path(path)
+          end
+        end
+      end
+
+      if Array === v and v.empty?
+        inputs[k] = "EMPTY_ARRAY"
+      end
+    end
+  end
+
+  def workflow_description
+    WorkflowRESTClient.get_raw(File.join(url, 'description'))
+  end
+
+  def documentation
+    @documention ||= IndiferentHash.setup(WorkflowRESTClient.get_json(File.join(url, "documentation"),{}))
+  end
+
+  def task_info(task)
+    @task_info ||= {}
+    @task_info[task]
+    
+    if @task_info[task].nil?
+      task_info = WorkflowRESTClient.get_json(File.join(url, task.to_s, 'info'))
+      task_info = WorkflowRESTClient.fix_hash(task_info)
+
+      task_info[:result_type] = task_info[:result_type].to_sym
+      task_info[:export] = task_info[:export].to_sym
+      task_info[:input_types] = WorkflowRESTClient.fix_hash(task_info[:input_types], true)
+      task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
+
+      @task_info[task] = task_info
+    end
+    @task_info[task]
+  end
+
+  def exported_tasks
+    (@asynchronous_exports  + @synchronous_exports + @exec_exports).compact.flatten
+  end
+
+  def tasks
+    @tasks ||= Hash.new do |hash,task_name| 
+      info = task_info(task_name)
+      task = Task.setup info do |*args|
+        raise "This is a remote task" 
+      end
+      task.name = task_name.to_sym
+      hash[task_name] = task
+    end
+  end
+
+  def load_tasks
+    exported_tasks.each{|name| tasks[name]}
+    nil
+  end
+
+  def task_dependencies
+    @task_dependencies ||= Hash.new do |hash,task| 
+      hash[task] = if exported_tasks.include? task
+        WorkflowRESTClient.get_json(File.join(url, task.to_s, 'dependencies'))
+      else
+        []
+      end
+    end
+  end
+
+  def init_remote_tasks
+    task_exports = WorkflowRESTClient.get_json(url)
+    @asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
+    @synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
+    @exec_exports = task_exports["exec"].collect{|task| task.to_sym }
+    @stream_exports = task_exports["stream"].collect{|task| task.to_sym }
+    @can_stream = task_exports["can_stream"]
+  end
+
+  def self.execute_job(base_url, task, task_params, cache_type)
+    self.capture_exception do
+      task_url = URI.encode(File.join(base_url, task.to_s))
+
+      sout, sin = Misc.pipe
+
+      post_thread = Thread.new(Thread.current) do |parent|
+        bl = lambda do |rok|
+          if Net::HTTPOK === rok
+            _url = rok["RBBT-STREAMING-JOB-URL"]
+            @url = File.join(task_url, File.basename(_url)) if _url
+            rok.read_body do |c,_a, _b|
+              sin.write c
+            end
+            sin.close
+          else
+            err = StringIO.new
+            rok.read_body do |c,_a, _b|
+              err.write c
+            end
+            text = begin
+                     reader = Zlib::GzipReader.new(err)
+                     reader.read
+                   rescue
+                     err.rewind
+                     err.read
+                   end
+            ne = @adaptor.parse_exception text
+            case ne
+            when String
+              parent.raise e.class, ne
+            when Exception
+              parent.raise ne
+            else
+              parent.raise "Error in RestClient: " << rok.message
+            end
+          end
+        end
+
+        task_params.each do |k,v|
+          task_params[k] = v.read if IO === v
+        end
+
+        Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
+        RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+      end
+
+      # It seems like now response body are now decoded by Net::HTTP after 2.1
+      # https://github.com/rest-client/rest-client/blob/cf3e5a115bcdb8f3344aeac0e45b44d67fac1a42/history.md
+      decode = Gem.loaded_specs["rest-client"].version < Gem::Version.create('2.1')
+      if decode
+        reader = Zlib::GzipReader.new(sout)
+        res_io = Misc.open_pipe do |sin|
+          while c = reader.read(Misc::BLOCK_SIZE)
+            sin.write c
+          end
+          sin.close
+          @done = true
+        end
+        ConcurrentStream.setup(res_io, :threads => [post_thread]) do
+          @done = true
+          @streaming = false
+        end
+      else
+        ConcurrentStream.setup(sout, :threads => [post_thread]) do
+          @done = true
+          @streaming = false
+        end
+      end
+
+    end
+  end
+
+end