rbbt-util 5.21.97 → 5.21.98

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 10ef32da69366f0d5db85823d95e5e2461817656
-  data.tar.gz: 7db675c42bfb34aae38611247d22cc5c3c6e4572
+  metadata.gz: 12d10e9ae9d12ed354fe3d8df24a14ea030bb1eb
+  data.tar.gz: 13416b83ec00f7dae0fc97479746cd636cf53a5c
 SHA512:
-  metadata.gz: db31f57eaa344bbbcac12c5a85dad2e46b555f6d19545a55854017c6dc8147d8d321d84f0a2578c29104b1d70b6de322ff5d49dd44c01452fb36fb2ebb59aa0b
-  data.tar.gz: 36f1dbc9bdf5242b02ce9944ce5ae0b93b7b51df4a1a2c68a6370498f7571ca2a9523b5867ce6a637367d26c431d4738a0159de17285f34310a9b0a29d62107a
+  metadata.gz: d86681d7392cd8a0dae501a10a792e91ca93a026adb918d386d5a5b83916525f66947d233bbe5302cf65c2c02ae2b19582ce52872946b98265c5d8752f02ea2f
+  data.tar.gz: a93dc769c50f8d08e74d8088caa7c953aee25a78b47576f0015fd8b73f113d736a234534e1faf791eee6e938686db4dcd03a9f1ac44e6643ce5aaca8b8da0a56

data/lib/rbbt/resource.rb

@@ -24,13 +24,19 @@ module Resource
     base.subdir = ''
     base.resources = {}
     base.rake_dirs = {}
+    base.search_paths = Path::SEARCH_PATHS.dup
     base.remote_server = Resource.remote_servers[base.to_s]
     base
   end
 
   attr_accessor :pkgdir, :subdir, :resources, :rake_dirs, :remote_server, :search_paths
 
-  def root()
+  def set_libdir(value = nil)
+    _libdir = value || Path.caller_lib_dir
+    search_paths.merge!(:lib => File.join(_libdir, '{TOPLEVEL}', '{SUBPATH}'))
+  end
+
+  def root
     Path.setup @subdir || "", @pkgdir, self, @search_paths
   end
 

data/lib/rbbt/rest/client/step.rb

@@ -361,6 +361,7 @@ class WorkflowRESTClient
     end
 
     def clean
+      init_job
       begin
         params = {:_update => :clean}
         WorkflowRESTClient.clean_url(url, params) if @url

data/lib/rbbt/util/misc/inspect.rb

@@ -234,6 +234,16 @@ module Misc
             "{"<< obj.collect{|k,v| obj2str(k) + '=>' << obj2str(v)}*"," << "}"
           when Symbol 
             obj.to_s
+          when (defined? Path and Path)
+            if obj.exists?
+              if obj.directory?
+                "directory: #{obj.glob("**/*")}"
+              else
+                "file md5: #{digest(obj.read)}"
+              end
+            else
+              obj + " (file missing)"
+            end
           when String
             if obj.length > HASH2MD5_MAX_STRING_LENGTH
               sample_large_obj(obj, HASH2MD5_MAX_STRING_LENGTH)

data/lib/rbbt/util/misc/multipart_payload.rb

@@ -1,6 +1,15 @@
 require 'net/http'
 require 'rbbt-util'
 
+class Net::HTTPGenericRequest
+  alias send_request_with_body_stream_orig send_request_with_body_stream
+  def send_request_with_body_stream(*args)
+    Thread.new do
+      send_request_with_body_stream_orig(*args)
+    end
+  end
+end
+
 module RbbtMutiplartPayload
   BOUNDARY = "Rbbt_Param_Stream"
   EOL = "\r\n"
@@ -97,6 +106,7 @@ module RbbtMutiplartPayload
 
     req = Net::HTTP::Post.new(uri.path)
     if stream_input
+      Log.low "Streaming input #{stream_input.to_s}"
       req.content_type = "multipart/form-data; boundary=" + RbbtMutiplartPayload::BOUNDARY + '; stream=' + stream_input.to_s
       req.body_stream = post_data_stream
     else
@@ -110,6 +120,7 @@ module RbbtMutiplartPayload
     timeout = 60 * timeout_minutes
     Misc.open_pipe do |sin|
       Net::HTTP.start(uri.hostname, uri.port, :read_timeout => timeout) do |http|
+        Log.low "Starting connection for streaming job"
         http.request(req) do |res|
           if Net::HTTPSuccess === res
             url_path = res["RBBT-STREAMING-JOB-URL"]

data/lib/rbbt/util/tar.rb

@@ -93,22 +93,6 @@ module Misc
   # directory
   def self.untar(io, destination)
     return _untar_cmd(io, destination)
-
-    #Gem::Package::TarReader.new io do |tar|
-    #  tar.each do |tarfile|
-    #    destination_file = File.join destination, tarfile.full_name
-
-    #    if tarfile.directory?
-    #      FileUtils.mkdir_p destination_file
-    #    else
-    #      destination_directory = File.dirname(destination_file)
-    #      FileUtils.mkdir_p destination_directory unless File.directory?(destination_directory)
-    #      File.open destination_file, "wb" do |f|
-    #        f.print tarfile.read
-    #      end
-    #    end
-    #  end
-    #end
   end
 end
 

data/lib/rbbt/workflow/accessor.rb

@@ -730,11 +730,12 @@ module Workflow
     _inputs
   end
 
-  def real_dependencies(task, jobname, inputs, dependencies)
+  def real_dependencies(task, orig_jobname, inputs, dependencies)
     real_dependencies = []
     path_deps = {}
     dependencies.each do |dependency|
       _inputs = IndiferentHash.setup(inputs.dup)
+      jobname = orig_jobname
       jobname = _inputs[:jobname] if _inputs.include? :jobname
 
       real_dep = case dependency
@@ -803,26 +804,24 @@ module Workflow
 
   TAG = :hash
   def step_path(taskname, jobname, inputs, dependencies, extension = nil)
-    #Proc.new{
-      raise "Jobname makes an invalid path: #{ jobname }" if jobname =~ /\.\./
-      if inputs.length > 0 or dependencies.any?
-        tagged_jobname = case TAG
-                         when :hash
-                           hash_str = Misc.obj2digest({:inputs => inputs, :dependencies => dependencies})
-                           jobname + '_' << hash_str
-                         else
-                           jobname
-                         end
-      else
-        tagged_jobname = jobname
-      end
+    raise "Jobname makes an invalid path: #{ jobname }" if jobname =~ /\.\./
+    if inputs.length > 0 or dependencies.any?
+      tagged_jobname = case TAG
+                       when :hash
+                         hash_str = Misc.obj2digest({:inputs => inputs, :dependencies => dependencies})
+                         jobname + '_' << hash_str
+                       else
+                         jobname
+                       end
+    else
+      tagged_jobname = jobname
+    end
 
-      if extension and not extension.empty?
-        tagged_jobname = tagged_jobname + ('.' << extension.to_s)
-      end
+    if extension and not extension.empty?
+      tagged_jobname = tagged_jobname + ('.' << extension.to_s)
+    end
 
-      workdir[taskname][tagged_jobname].find
-    #}
+    workdir[taskname][tagged_jobname].find
   end
 
   def id_for(path)

data/lib/rbbt/workflow/step.rb

@@ -72,7 +72,7 @@ class Step
 
     load_inputs_from_info if @inputs.nil? 
 
-    NamedArray.setup(@inputs, task.inputs) if task && task.inputs && ! NamedArray === @inputs
+    NamedArray.setup(@inputs, task.inputs) if task && task.inputs && !(NamedArray === @inputs)
 
     @inputs || []
   end

data/lib/rbbt/workflow/step/run.rb

@@ -145,6 +145,53 @@ class Step
             raise $!
           end
 
+          new_inputs = []
+          @inputs.each_with_index do |input,i|
+            name = @task.inputs[i]
+            type = @task.input_types[name]
+
+            if type == :directory
+              directory_inputs = file('directory_inputs')
+              input_source = directory_inputs['.source'][name].find
+              input_dir = directory_inputs[name].find
+
+              case input
+              when Path
+                if input.directory?
+                  new_inputs << input
+                else
+                  input.open do |io|
+                    begin
+                      Misc.untar(io, input_source)
+                    rescue
+                      raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
+                    end
+                  end
+                  tar_1 = input_source.glob("*")
+                  raise ParameterException, "When using tar.gz files for directories, the directory must be the single first level entry" if tar_1.length != 1
+                FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
+                  new_inputs << input_dir
+                end
+              when File, IO, Tempfile
+                begin
+                  Misc.untar(Open.gunzip(input), input_source)
+                rescue
+                  raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
+                end
+                tar_1 = input_source.glob("*")
+                raise ParameterException, "When using tar.gz files for directories, the directory must be the single first level entry" if tar_1.length != 1
+                FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
+                new_inputs << input_dir
+              else
+                raise ParameterException, "Format of directory input '#{name}' not understood: #{Misc.fingerprint input}"
+              end
+            else
+              new_inputs << input
+            end
+          end if @inputs
+          
+          @inputs = new_inputs if @inputs
+
           if not task.inputs.nil?
             info_inputs = @inputs.collect do |i| 
               if Path === i 
@@ -200,7 +247,8 @@ class Step
                 end
               end
               begin
-                if status != :done
+                status = self.status
+                if status != :done and status != :error and status != :aborted
                   Misc.insist do
                     set_info :done, (done_time = Time.now)
                     set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)

data/share/Rlib/util.R

@@ -654,7 +654,15 @@ rbbt.install.github <- function(pkg, ...){
 
 rbbt.require <- function(pkg, ...){
     list.of.packages <- c(pkg)
-    new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])]
+
+    clean.packages <- c()
+    for (pkg in list.of.packages){ 
+        parts = strsplit(pkg,'/')[[1]]
+        clean.packages <- c(clean.packages, parts[length(parts)])
+    }
+
+    new.packages <- list.of.packages[!(clean.packages %in% installed.packages()[,"Package"])]
+
     for (pkg in new.packages){ 
         if (!rbbt.install.github(pkg, ...)){
             if (!rbbt.install.CRAN(pkg, ...)){
@@ -664,6 +672,7 @@ rbbt.require <- function(pkg, ...){
             }
         }
     }
-    library(list.of.packages, character.only=T)
+
+    library(clean.packages, character.only=T)
 }
 

data/share/rbbt_commands/app/start

@@ -62,7 +62,8 @@ Misc.in_dir(app_dir) do
   ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
 
   if options[:stream]
-    raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma/
+    #raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma/
+    raise "No streaming available for any server other than unicorn" unless options[:server] =~ /^unicorn/
     ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
   end
 
@@ -77,8 +78,8 @@ Misc.in_dir(app_dir) do
   end
 
   case server
-  when 'unicorn'
-    system ENV, "unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:port] || "2887"}"
+  when 'passenger'
+    system ENV, "env RBBT_LOG=0 passenger start -R '#{config_ru_file}' -p #{options[:port] || "2887"}"
   when 'puma_alt'
     system ENV, "puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload"
   else

data/share/rbbt_commands/workflow/info

@@ -126,8 +126,8 @@ if all
 end
 
 if recursive
-  puts Log.color(:magenta, "Recursive inputs") << ": "
   inputs = step.info[:inputs]
+  inputs = {}
   deps = step.info[:dependencies].collect{|v| get_step v.last }
 
   while deps.any? do
@@ -137,19 +137,22 @@ if recursive
     deps.concat (dep.info[:dependencies] || []).collect{|v| get_step v.last }
   end
 
-  inputs.each do |input,value|
-    case value
-    when nil
-      puts Misc.format_definition_list_item("  " << input.to_s, 'nil', 80, 20, :blue)
-    when Array
-      puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), 80, 20, :blue).gsub("\n\n","\n")
-    when TrueClass, FalseClass
-      puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, 80, 20, :blue)
-    else
-      lines = value.to_s.split("\n").collect{|l| l.length >= 60 ? l[0..45] + " ..." : l }
-      text = lines[0..5].compact * "\n\n"
-      text << "\n\n...\n\n" if lines.length > 6
-      puts Misc.format_definition_list_item("  " << input.to_s, text, 80, 20, :blue).gsub("\n\n","\n")
+  if inputs.any?
+    puts Log.color(:magenta, "Recursive inputs") << ": "
+    inputs.each do |input,value|
+      case value
+      when nil
+        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', 80, 20, :blue)
+      when Array
+        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), 80, 20, :blue).gsub("\n\n","\n")
+      when TrueClass, FalseClass
+        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, 80, 20, :blue)
+      else
+        lines = value.to_s.split("\n").collect{|l| l.length >= 60 ? l[0..45] + " ..." : l }
+        text = lines[0..5].compact * "\n\n"
+        text << "\n\n...\n\n" if lines.length > 6
+        puts Misc.format_definition_list_item("  " << input.to_s, text, 80, 20, :blue).gsub("\n\n","\n")
+      end
     end
   end
 end

data/share/rbbt_commands/workflow/task

@@ -108,6 +108,8 @@ def fix_options(workflow, task, job_options)
             when :array
               if input_options[name] and input_options[name][:stream] and String === value and Misc.is_filename? value
                 get_value_stream(value)
+              elsif input_options[name] and input_options[name][:stream] and value == "-"
+                STDIN
               else
                 if Array === value
                   value
@@ -146,6 +148,8 @@ def fix_options(workflow, task, job_options)
                   TSV.open(StringIO.new(value), :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 end
               end
+            when :directory
+              Path.setup(File.expand_path(value))
             else
               value
             end
@@ -321,7 +325,7 @@ begin
 
   if do_exec or (job.respond_to?(:is_exec) and job.is_exec)
     res = job.exec(:stream)
-    res = JSON.parse(res.read) if (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep === job)
+    #res = JSON.parse(res.read) if (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep === job)
 
     case
     when res.respond_to?(:gets)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.97
+  version: 5.21.98
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-21 00:00:00.000000000 Z
+date: 2017-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake