rbbt-util 5.26.116 → 5.26.117

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e3981775c0531bd71ddb841401d78b7e84f83c44579347b8da42bb71f9c3a67
-  data.tar.gz: 791d1a58540888c2a9a338d706f1188cc684a6b2bc50dce8fc4c38169de898f0
+  metadata.gz: e59ee0bcb0a764c4bc10206585d0a26c7a59f881cca10dcaf3fbf693b0ed9208
+  data.tar.gz: 929459e6de52b9ba2f13b7fa55c2aac38a78ff13c6990f63143c68e2b35a6af8
 SHA512:
-  metadata.gz: 44efa5d106893dee737ef79c17c0a655ff1fa63e924045d433fcc769f73fe5470bef22de23d50561559c97361716329429dbb26b8dce366c5cc875fa1dcc67fa
-  data.tar.gz: 7350980369fd646a5e9992d4683a15c7c944627f4aa5f5753b3691c21783fac6bcf76bf9021cc13192a92a0afeea0b06a7259917563e8fdb7df10411c0279d66
+  metadata.gz: c3d7ed4da1ec8ac8b63e1a91bccbb87b60a1d6c60fe3807f4c9f85f8c1dc82a1fd206382175184278dac40bce536a732e3da38383c0828e446b10cfcf647ac3d
+  data.tar.gz: fbed1b9e52b78a3f56eda4e5a0c0b39c9f21f8ff8faa66725ebaf2c11d27e3c71d4d630ce5acd7616dd86697d82114859fd46e5b45a256cad6364e800c045934

data/etc/app.d/semaphores.rb

@@ -1,3 +1,5 @@
+# This is used by rbbt-rest when issuing a new job
+
 $rest_cache_semaphore = "/REST_SEMAPHORE"
 parallel_rest_jobs = Rbbt::Config.get('parallel_rest_jobs', :parallel_rest_jobs, :rest_jobs, :default => 2)
 begin

data/lib/rbbt/entity.rb

@@ -5,6 +5,13 @@ module Entity
   
   UNPERSISTED_PREFIX = "entity_unpersisted_property_"
 
+  class DontPersist < Exception
+    attr_accessor :payload
+    def self.initialize(payload)
+      @payload = payload
+    end
+  end
+
   class << self
     attr_accessor :formats, :entity_property_cache
   end
@@ -203,8 +210,12 @@ module Entity
           persist_options = options
           persist_options = persist_options.merge(:other => {:args => args}) if args and args.any?
 
-          Persist.persist(persist_name, type, persist_options.merge(:persist => true)) do
-            self.send(orig_name, *args)
+          begin
+            Persist.persist(persist_name, type, persist_options.merge(:persist => true)) do
+              self.send(orig_name, *args)
+            end
+          rescue DontPersist
+            $!.payload
           end
         end
       end

data/lib/rbbt/hpc.rb

@@ -295,6 +295,8 @@ EOF
 
         target = File.expand_path(sync)
         coda +=<<-EOF
+
+# Sync data to target location
 mkdir -p "$(dirname '#{target}')"
 rsync -avztAXHP --copy-unsafe-links "#{source}/" "#{target}/" &>> #{fsync} 
 sync_es="$?" 
@@ -310,6 +312,9 @@ fi
           if singularity
             coda +=<<-EOF
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -v /dev/shm/sem.*.{in,out,process} /dev/shm/sem.Session-PID.*.sem 2> /dev/null >> #{fsync}
+
+
+# Clean container directory
 if [ $exit_status == '0' -a $sync_es == '0' ]; then 
     singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean -f &>> #{fsync}
     singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv .rbbt/var/jobs &>> #{fsync}

data/lib/rbbt/persist.rb

@@ -41,12 +41,19 @@ module Persist
     return false if not Open.exists? path
     return false if TrueClass === persist_options[:update]
 
+    expiration = persist_options[:expiration]
+    if expiration
+      seconds = Misc.timespan(expiration)
+      patht = Open.mtime(path)
+      return false if Time.now > patht + seconds
+    end
+  
     check = persist_options[:check]
     return true if check.nil?
 
     missing = check.reject{|file| Open.exists?(file) }
     return false if missing.any?
-  
+
     return true unless ENV["RBBT_UPDATE"]
 
     if Array === check
@@ -497,7 +504,7 @@ module LocalPersist
   attr_accessor :local_persist_dir
 
   def local_persist_dir
-    @local_persist_dir ||= Rbbt.var.cache.persistence if defined? Rbbt
+    @local_persist_dir ||= Rbbt.var.cache.persistence.find(:lib) if defined? Rbbt
     @local_persist_dir
   end
 
@@ -505,13 +512,19 @@ module LocalPersist
     @local_persist_dir = value
   end
 
-  def local_persist(name, type = nil, options= {}, persist_options = nil, &block)
+  def self.local_persist(name, type = nil, options= {}, persist_options = nil, &block)
     persist_options ||= {}
-    persist_options = {:dir => local_persist_dir}.merge persist_options
+    persist_options = {:dir => Rbbt.var.cache.persistence.find(:lib)}.merge persist_options
     persist_options[:other] = options
     Persist.persist(name, type, persist_options, &block)
   end
 
+  def local_persist(name, type = nil, options= {}, persist_options = nil, &block)
+    persist_options ||= {}
+    persist_options = {:dir => local_persist_dir}.merge persist_options
+    self.local_persist(name, type, options, persist_options, &block)
+  end
+
   def local_persist_tsv(source, name, opt = {}, options= {}, &block)
     Persist.persist_tsv(source, name, opt, options.merge({:dir => local_persist_dir, :persist => true}), &block)
   end

data/lib/rbbt/resource.rb

@@ -86,29 +86,36 @@ module Resource
       lock_filename = nil # it seems like this was locked already.
 
       Misc.lock lock_filename do
-        Net::HTTP.get_response URI(url) do |response|
-          case response
-          when Net::HTTPSuccess, Net::HTTPOK
-            Misc.sensiblewrite(final_path) do |file|
-              response.read_body do |chunk|
-                file.write chunk
+        uri = URI(url)
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        request = Net::HTTP::Get.new(uri.request_uri)
+        timeout = 60 * 10
+        Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
+          http.request request do |response|
+            case response
+            when Net::HTTPSuccess, Net::HTTPOK
+              Misc.sensiblewrite(final_path) do |file|
+                response.read_body do |chunk|
+                  file.write chunk
+                end
               end
-            end
-          when Net::HTTPRedirection, Net::HTTPFound
-            location = response['location']
-            Log.debug("Feching directory from: #{location}. Into: #{final_path}")
-            FileUtils.mkdir_p final_path unless File.exist? final_path
-            TmpFile.with_file do |tmp_dir|
-              Misc.in_dir tmp_dir do
-                CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+            when Net::HTTPRedirection, Net::HTTPFound
+              location = response['location']
+              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
+              FileUtils.mkdir_p final_path unless File.exist? final_path
+              TmpFile.with_file do |tmp_dir|
+                Misc.in_dir tmp_dir do
+                  CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+                end
+                FileUtils.mv tmp_dir, final_path
               end
-              FileUtils.mv tmp_dir, final_path
+            when Net::HTTPInternalServerError
+              @server_missing_resource_cache << url
+              raise "Resource Not Found"
+            else
+              raise "Response not understood: #{response.inspect}"
             end
-          when Net::HTTPInternalServerError
-            @server_missing_resource_cache << url
-            raise "Resource Not Found"
-          else
-            raise "Response not understood: #{response.inspect}"
           end
         end
       end

data/lib/rbbt/resource/util.rb

@@ -4,6 +4,7 @@ module Path
     file = caller.reject{|l| 
       l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
       l =~ /rbbt\/resource\/path\.rb/ or
+      l =~ /rbbt\/persist.rb/ or
       l =~ /rbbt\/util\/misc\.rb/ or
       l =~ /progress-monitor\.rb/ 
     }.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?

data/lib/rbbt/tsv/stream.rb

@@ -67,7 +67,7 @@ module TSV
         end
 
         first_line = parser.first_line
-        first_line = nil if first_line.empty?
+        first_line = nil if first_line == ""
 
         lines         << first_line
         key_fields    << parser.key_field

data/lib/rbbt/workflow.rb

@@ -46,8 +46,8 @@ module Workflow
   end
 
   def self.require_remote_workflow(wf_name, url)
-    require 'rbbt/rest/client'
-    eval "Object::#{wf_name} = WorkflowRESTClient.new '#{ url }', '#{wf_name}'"
+    require 'rbbt/workflow/remote/client'
+    eval "Object::#{wf_name} = WorkflowRemoteClient.new '#{ url }', '#{wf_name}'"
   end
 
   def self.require_remote_workflow(wf_name, url)
@@ -682,14 +682,15 @@ module Workflow
   end
 
   def self.process_remote_tasks(remote_tasks)
-    require 'rbbt/rest/client'
+    require 'rbbt/workflow/remote/client'
     remote_tasks.each do |workflow, info|
       wf = Workflow.require_workflow workflow
       wf.remote_tasks ||= {}
       IndiferentHash.setup wf.remote_tasks
       info.each do |remote, tasks|
-        remote_wf = WorkflowRESTClient.new remote, workflow
+        remote_wf = WorkflowRemoteClient.new remote, workflow
         tasks.each do |task|
+          Log.debug "Add remote task #{task} in #{wf} using #{remote_wf.url}"
           wf.remote_tasks[task.to_sym] = remote_wf
         end
       end

data/lib/rbbt/workflow/remote/remote_step.rb

@@ -191,8 +191,10 @@ class WorkflowRemoteClient
     def path
       if @url
         Misc.add_GET_param(@url, "_format", "raw")
-      else
+      elsif @base_name
         [base_url, task, @base_name + '-' +  Misc.fingerprint(inputs)] * "/"
+      else
+        nil
       end
     end
 

data/share/rbbt_commands/system/clean

@@ -34,6 +34,8 @@ force = options.delete :force
 dirty = options.delete :dirty
 time = options.delete :older
 
+ENV["RBBT_UPDATE"] = 'true' if dirty
+
 time = Misc.timespan time, 'd' if time
 
 puts Log.color(:magenta, "# System clean")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.116
+  version: 5.26.117
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-16 00:00:00.000000000 Z
+date: 2020-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake