rbbt-util 5.27.8 → 5.27.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d3d7b4d19322b271a94d3e2fe046158857bbedf9adf79d0408b4c50d2ba9254
-  data.tar.gz: 78d6fcffe3d5efd08ec42e0493f3f7c731d31001b726ade88802460e5958354b
+  metadata.gz: 99b1ad9e8d9403c7f77e22b16497456324c340b08122b765033530f713e35c3c
+  data.tar.gz: 2d33590e395b9ca715e1c16d509d0a62a1ebeab2315b700ff4377a08803c89af
 SHA512:
-  metadata.gz: cc8ba1db9052fb92dc87bd037b8745f232ff20110b7e3d0da4d2e3c9f0ed90b4bdf18f68887c8a1ad1eb44f1828e9fbd614770259e73deda196047cf4f783d06
-  data.tar.gz: 3623a36d9282152a71710a71767fb829ce1258f29161451ffc1b0045c0eee152267dd62c8ca8af0e2e3de7f4ef86d9242279c3d34bbd7a3fdfa18841d857ac8a
+  metadata.gz: '097ce6c8c16c17441f5a8748ba4ab33afecf6d22d4d54c7ce53475cd642407ae5013346ba482f7fa9375176ab6790febfe030e9f7959ba7b928b5d3762382237'
+  data.tar.gz: 2fb616d7d66fc2eb979c9911921faecf163f51dd3323ac127265bcaefd0e8a4e3aab9ef41cc9bdf8bd351d44d4ef2cc388c692d5dc9a2d1b811155f21e3f0889

data/lib/rbbt/entity.rb

@@ -261,7 +261,7 @@ module Entity
 
         define_method method_name do |*args|
           id = self.id
-          persist_name = orig_method_name.to_s << ":" << (Array === id ? Misc.obj2digest(id) : id)
+          persist_name = orig_method_name.to_s + ":" << (Array === id ? Misc.obj2digest(id) : id)
 
           persist_options = options
           persist_options = persist_options.merge(:other => {:args => args}) if args and args.any?

data/lib/rbbt/fix_width_table.rb

@@ -28,7 +28,7 @@ class FixWidthTable
     else
       Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
       if in_memory
-        @file        = Open.open(@filename, :mode => 'r:ASCII-ASCII'){|f| StringIO.new f.read}
+        @file        = Open.open(@filename, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
       else
         @file        = File.open(@filename, 'r:ASCII-8BIT')
       end

data/lib/rbbt/persist.rb

@@ -365,12 +365,12 @@ module Persist
     type ||= :marshal
 
     persist_options ||= {}
-    if type == :memory and persist_options[:file] and persist_options[:persist] 
+    if type == :memory && persist_options[:file] && persist_options[:persist] 
       repo = persist_options[:repo] || Persist::MEMORY
       if persist_options[:persist] == :update || persist_options[:update]
         repo.delete persist_options[:file]
       end
-      return repo[persist_options[:file]] ||= yield 
+      return repo[persist_options[:file]] ||= yield
     end
 
     if FalseClass === persist_options[:persist]

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -727,6 +727,8 @@ module TSV
         Log.exception $!
         abort_stream(into, $!)
         raise $!
+      ensure
+        Log::ProgressBar.remove_bar(bar) if bar
       end
 
       into

data/lib/rbbt/util/misc.rb

@@ -2,6 +2,7 @@ require 'lockfile'
 require 'digest/md5'
 require 'cgi'
 require 'zlib'
+require 'etc'
 require 'rubygems/package'
 
 require 'rbbt/util/tar'

data/lib/rbbt/util/misc/development.rb

@@ -292,14 +292,21 @@ def self.add_libdir(dir=nil)
     end
   end
 
-  def self.bootstrap(elems, num = :current, options = {}, &block)
+  def self.bootstrap(elems, num = nil, options = {}, &block)
     IndiferentHash.setup options
+
+    num = Rbbt::Config.get :cpus, :default_bootstrap_cpus, :bootstrap_cpus if num == :current || num == nil if defined?(Rbbt::Config)
     num = :current if num.nil?
     cpus = case num
            when :current
-            n = 10
-            n = elems.length / 2 if n > elems.length/2
-            n
+             n = Etc.nprocessors
+             n = elems.length / 2 if n > elems.length/2
+
+             if $BOOTSTRAPPED_CURRENT && $BOOTSTRAPPED_CURRENT + n > Etc.nprocessors
+               1
+             else
+               n
+             end
            when String
              num.to_i
            when Integer
@@ -318,7 +325,12 @@ def self.add_libdir(dir=nil)
     respawn = options[:respawn] and options[:cpus] and options[:cpus].to_i > 1
 
     index = (0..elems.length-1).to_a.collect{|v| v.to_s }
+
     TSV.traverse index, options do |pos|
+      if num == :current
+        $BOOTSTRAPPED_CURRENT ||= n 
+        $BOOTSTRAPPED_CURRENT += 0 
+      end
       elem = elems[pos.to_i]
       elems.annotate elem if elems.respond_to? :annotate
       res = begin

data/lib/rbbt/util/misc/inspect.rb

@@ -401,7 +401,7 @@ module Misc
 
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
-    m = text.match(/(?:version.*?|#{cmd}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
     return nil if m.nil?
     m[1]
   end

data/lib/rbbt/util/open.rb

@@ -94,7 +94,8 @@ module Open
              end
 
     begin
-      wget_options = options.merge( '-O' => '-')
+      wget_options = options.dup
+      wget_options = wget_options.merge( '-O' => '-') unless options.include?('--output-document')
       wget_options[:pipe] = pipe unless pipe.nil?
       wget_options[:stderr] = stderr unless stderr.nil?
 
@@ -824,4 +825,8 @@ module Open
   def self.broken_link?(path)
     File.symlink?(path) && ! File.exists?(File.readlink(path))
   end
+
+  def self.download(url, path)
+    Open.wget(url, "--output-document" => path)
+  end
 end

data/lib/rbbt/util/tmpfile.rb

@@ -17,14 +17,14 @@ module TmpFile
 
   # Creates a random file name, with the given suffix and a random number
   # up to +max+
-  def self.random_name(s = "tmp-", max = 10000000)
+  def self.random_name(s = "tmp-", max = 1_000_000_000)
     n = rand(max)
     s + n.to_s
   end
 
   # Creates a random filename in the temporary directory
-  def self.tmp_file(s = "tmp-", max=10000000, dir = TMPDIR)
-    File.expand_path(File.join(dir, random_name(s,max)))
+  def self.tmp_file(s = "tmp-", max=1_000_000_000, dir = TMPDIR)
+    File.expand_path(File.join(dir, random_name(s, max)))
   end
 
   def self.with_file(content = nil, erase = true, options = {})
@@ -33,7 +33,7 @@ module TmpFile
 
     prefix = options[:prefix] || "tmp-"
     tmpdir = options[:tmpdir] || TMPDIR
-    max = options[:max] || 10000000
+    max = options[:max] || 1_000_000_000
     tmpfile = tmp_file prefix, max, tmpdir
     if options[:extension]
       tmpfile += ".#{options[:extension]}"

data/lib/rbbt/workflow.rb

@@ -10,9 +10,6 @@ require 'rbbt/workflow/util/provenance'
 
 module Workflow
 
-  STEP_CACHE = {}
-  LOAD_STEP_CACHE = {}
-
   class TaskNotFoundException < Exception 
     def initialize(workflow, task = nil)
       if task
@@ -264,11 +261,11 @@ module Workflow
   end
 
   def step_cache
-    @step_cache ||= Workflow::STEP_CACHE
+    Thread.current[:step_cache] ||= {}
   end
 
   def self.load_step_cache
-    @load_step_cache ||= Workflow::LOAD_STEP_CACHE
+    Thread.current[:load_step_cache] ||= {}
   end
 
 

data/lib/rbbt/workflow/accessor.rb

@@ -418,7 +418,7 @@ module Workflow
                        when :hash
                          clean_inputs = Annotated.purge(inputs)
                          clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
-                         deps_str = dependencies.collect{|d| Step === d ? d.short_path : d }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << d.short_path : d }
                          key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
                          key_str = Misc.obj2str(key_obj)
                          hash_str = Misc.digest(key_str)

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -137,14 +137,16 @@ class RemoteWorkflow
 
         post_thread = Thread.new(Thread.current) do |parent|
           bl = lambda do |rok|
-            if Net::HTTPOK === rok 
+            case rok
+            when Net::HTTPOK
               _url = rok["RBBT-STREAMING-JOB-URL"]
               @url = File.join(task_url, File.basename(_url)) if _url
               rok.read_body do |c,_a, _b|
                 sin.write c
               end
               sin.close
-            elsif Net::HTTPSeeOther === rok
+            when Net::HTTPRedirection, Net::HTTPAccepted
+              Thread.current.report_on_exception = false
               raise TryThis.new(rok)
             else
               err = StringIO.new
@@ -178,7 +180,8 @@ class RemoteWorkflow
           begin
             RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
           rescue TryThis
-            RestClient::Request.execute(:method => :get, :url => $!.payload.header[:location], :block_response => bl)
+            url = $!.payload["location"]
+            RestClient::Request.execute(:method => :get, :url => url, :block_response => bl)
           end
         end
 

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -26,8 +26,8 @@ class RemoteStep < Step
 
   def cache_file
     begin
-      digest = Misc.obj2digest([base_url, task, base_name, inputs])
-      Rbbt.var.cache.REST[[task, clean_name, digest].compact * "."].find
+      digest = Misc.obj2digest([base_url, task.to_s, base_name, inputs])
+      Rbbt.var.cache.REST[task.to_s][[clean_name, digest].compact * "."].find
     rescue
       Log.exception $!
       raise $!
@@ -143,7 +143,11 @@ class RemoteStep < Step
     @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => update) do
       @last_info_time = Time.now
       init_job unless @url
-      info = @adaptor.get_json(File.join(@url, 'info'))
+      info = begin
+               @adaptor.get_json(File.join(@url, 'info'))
+             rescue
+               {:status => :noinfo}
+             end
       info = RemoteWorkflow.fix_hash(info)
       info[:status] = info[:status].to_sym if String === info[:status]
       info
@@ -243,6 +247,7 @@ class RemoteStep < Step
   def join
     return true if cache_files.any?
     init_job unless @url
+    produce unless @started
     Log.debug{ "Joining RemoteStep: #{path}" }
 
     if IO === @result
@@ -258,7 +263,6 @@ class RemoteStep < Step
       sleep 1 unless self.done? || self.aborted? || self.error?
       while not (self.done? || self.aborted? || self.error?)
         sleep 3
-        iif [self.done?, self.status, self.info]
       end
     end
 
@@ -311,6 +315,10 @@ class RemoteStep < Step
     end
   end
 
+  def short_path
+    init_job unless @url
+    [@task, @name] * "/"
+  end
 
   def input_checks
     []

data/share/rbbt_commands/app/start

@@ -15,6 +15,7 @@ $ rbbt app start [options]
 -Ho--Host* Host name
 -B--Bind* Bind IP
 -p--port* TCP port
+-w--workers* Number of workers for cluster mode (puma)
 -s--server* Server type:  thin, webrick, unicorn, etc
 -f--finder Start server with finder functionality
 -fs--file_server Activate file serving for resources
@@ -84,6 +85,12 @@ Misc.in_dir(app_dir) do
     end
   end
 
+  fixed_options = {}
+  options.each do |k,v| fixed_options[k.to_sym] = v end
+  options = fixed_options
+
+  options[:workers] = options[:workers].to_i if options[:workers]
+
   case server
   when 'passenger'
     system ENV, "env RBBT_LOG=0 passenger start -R '#{config_ru_file}' -p #{options[:Port] || "2887"}"
@@ -91,6 +98,7 @@ Misc.in_dir(app_dir) do
     system ENV, "puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload"
   else
     options[:config] = config_ru_file
+    options[:threads] = "8:8"
     Rack::Server.start(options)
   end
 end

data/share/rbbt_commands/system/clean

@@ -97,7 +97,7 @@ TSV.traverse jobs do |file,i|
   else
     info = begin
              Open.open(i[:info_file]) do |f|
-               Step::INFO_SERIALIAZER.load(f)
+               Step::INFO_SERIALIZER.load(f)
              end
            rescue
              {:status => :noinfo}
@@ -134,7 +134,7 @@ TSV.traverse jobs do |file,i|
     end
   end
 
-  if (force and status !~ /done/) or 
+  if (force && status !~ /done/) or 
     status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync)$/ or
     (status == "noinfo" and not done) or
     status == "" 

data/share/rbbt_commands/tsv/uncollapse

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+options = SOPT.setup <<EOF
+Read a TSV file and dump it
+
+$ rbbt tsv read [options] <filename.tsv|->
+
+Use - to read from STDIN
+
+-k--key_field* Key field
+-f--fields* Fields
+-t--type* Type
+-m--merge* Merge from multiple rows
+-h--help Print this help
+
+EOF
+rbbt_usage and exit 0 if options[:help]
+
+file = ARGV.shift
+
+file = STDIN if file == '-'
+
+tsv = TSV.open file, :merge => true, :type => :double
+
+puts tsv.to_unmerged_expanded_s

data/share/rbbt_commands/workflow/info

@@ -153,7 +153,7 @@ if recursive
     dep = deps.shift
     inputs = {} if inputs.nil?
     inputs = inputs.merge(dep.info[:inputs] || {})
-    deps = deps.concat dep.dependencies
+    deps = deps.concat dep.dependencies if dep.dependencies
   end
 
   inputs = inputs.merge step.archived_inputs

data/share/rbbt_commands/workflow/server

@@ -123,15 +123,11 @@ TmpFile.with_file do |app_dir|
     else
       options[:config] = config_ru_file
 
-			clean_options = {}
-			options.each do |k,v|
-        begin
-          clean_options[k.to_sym] = v
-        rescue
-        end
-      end
+      fixed_options = {}
+      options.each do |k,v| fixed_options[k.to_sym] = v end
+      options = fixed_options
 
-      Rack::Server.start(clean_options)
+      Rack::Server.start(options)
     end
   end
 end

data/share/rbbt_commands/workflow/task

@@ -337,6 +337,8 @@ if clean_task
       dep.clean 
       dep.set_info :status, :cleaned
     end
+
+    job.clean if job.task_name.to_s == clean_task.to_s
   end
 end
 

data/test/rbbt/util/test_misc.rb

@@ -690,5 +690,30 @@ Written by Mike Haertel and others, see
     EOF
     assert_equal "3.1", Misc.scan_version_text(txt, "grep")
   end
+
+  def test_bootstrap_current_only_once
+
+     
+    max = Etc.nprocessors * 10
+    TmpFile.with_file do |f|
+      RbbtSemaphore.with_semaphore(1) do |sem|
+        Misc.bootstrap (1..max).to_a do
+          Open.open(f, :mode => 'a') do |sin|
+            sin.puts Process.pid.to_s
+          end
+          Misc.bootstrap (1..10).to_a do
+            RbbtSemaphore.synchronize(sem) do
+              Open.open(f, :mode => 'a') do |sin|
+                sin.puts [Process.ppid.to_s, Process.pid.to_s] * ":"
+              end
+            end
+          end
+        end
+      end
+      assert_equal max * 10 + max, Open.read(f).split("\n").length
+      assert_equal Etc.nprocessors * 2, Open.read(f).split("\n").uniq.length
+    end
+
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.27.8
+  version: 5.27.13
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-01 00:00:00.000000000 Z
+date: 2020-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -403,6 +403,7 @@ files:
 - share/rbbt_commands/tsv/sort
 - share/rbbt_commands/tsv/subset
 - share/rbbt_commands/tsv/transpose
+- share/rbbt_commands/tsv/uncollapse
 - share/rbbt_commands/tsv/unzip
 - share/rbbt_commands/tsv/values
 - share/rbbt_commands/tsv/write_excel