rbbt-util 5.32.30 → 5.33.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e173c4373fc6358b63964247a0d81bab2bd632a2e6bce1b93ff9ac1bd6aa53ca
-  data.tar.gz: 26c3352b53615f1e1c057727ac6f129dedbb9fabb09742513b2368b0b2bd7327
+  metadata.gz: 1a6b198e57fe73c0e36e7aea6164bafd1a8a97d10b3a992f17d0ba6dc531a3c7
+  data.tar.gz: 5aa8b8753fc88d5de6fa8f6db31b9b71a57ee5dcc86380041582a4985984f749
 SHA512:
-  metadata.gz: f8d84e234a76a38352786f869aaf5fee4f4b159a402b9241707c18f7cbe595d630a645e75bad7ae512652d664b98c61a13c24beb972e1c55db2b300a8c82744b
-  data.tar.gz: 0fdcfe8522907197eeacfffa0551dbde81db0fc859f0f85eb4281e87df44c9a0b90026423dfad79fd6c9cd8429be655eadc94e94c85cd79d0b971134ea6427ee
+  metadata.gz: a61b6e6671803d73b7993b2de9c6cf16d4f463ddd04f9378b54c77a00eb7292b4e817eea14cce81704fe3f2640d38ede202a3928d293ab1fe7e0e67fa87a35b0
+  data.tar.gz: d41b1477aab34bfc08610602a3e333a5701edb58d9fa10ecf24436659c4e996bb000ceccb20bbcba1f153ec4e888264ae47ab28689b9e2879601da0b7822e831

data/lib/rbbt/persist/tsv/adapter.rb

@@ -111,6 +111,31 @@ module Persist
       end
     end
 
+    def with_read(&block)
+      if read? || write?
+        return yield
+      else
+        read_and_close &block
+      end
+    end
+
+    def with_write(&block)
+      if write?
+        return yield
+      else
+        if self.read?
+          self.write_and_read do
+            return yield
+          end
+        else
+          self.write_and_close do
+            return yield
+          end
+        end
+      end
+    end
+
+
     def read_and_close
       if read? || write?
         begin

data/lib/rbbt/persist.rb

@@ -423,6 +423,8 @@ module Persist
           end
         end
 
+        repo.read
+
         case
         when (keys.length == 1 and keys.first == subkey + 'NIL')
           nil
@@ -430,19 +432,19 @@ module Persist
           []
         when (keys.length == 1 and keys.first =~ /:SINGLE$/)
           key = keys.first
-          values = repo.read_and_close do
+          values = repo.with_read do
             repo[key]
           end
           Annotated.load_tsv_values(key, values, "literal", "annotation_types", "JSON")
         when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
-          repo.read_and_close do
+          repo.with_read do
             keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
               Annotated.load_tsv_values(key, v, "literal", "annotation_types", "JSON")
             }
           end
         when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
-          repo.read_and_close do
+          repo.with_read do
 
             res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
@@ -457,7 +459,7 @@ module Persist
         else
           entities = yield
 
-          repo.write_and_close do 
+          repo.write_and_read do 
             case
             when entities.nil?
               repo[subkey + "NIL"] = nil

data/lib/rbbt/resource/path.rb

@@ -462,4 +462,8 @@ module Path
       Path.setup relative_file , @pkgdir, @resource
     end
   end
+
+  def clean_annotations
+    "" << self.to_s
+  end
 end

data/lib/rbbt/tsv/util.rb

@@ -297,6 +297,8 @@ module TSV
     when :double
       if field.nil?
         through do |k,v| new[k] = v.first end
+      elsif field == :all
+        through do |k,v| new[k] = v.flatten.compact end
       else
         pos = identify_field field
         through do |k,v| new[k] = v[pos] end
@@ -313,7 +315,16 @@ module TSV
       end
     end
     self.annotate(new)
-    new.fields = new.fields[0..0] if new.fields
+    if new.fields
+      case field
+      when nil
+        new.fields = new.fields[0..0]
+      when :all
+        new.fields = [new.fields * "+"]
+      else
+        new.fields = [field]
+      end
+    end
     new.type = :flat
     new
   end

data/lib/rbbt/util/concurrency/processes.rb

@@ -205,7 +205,8 @@ class RbbtProcessQueue
     end
 
     @queue.close_read
-    Log.info "Cpu process (#{num_processes}) started with master: #{@master_pid}"
+    Log.info "Cpu process #{@master_pid} with #{num_processes} workers."
+    Log.low "Signal #{@master_pid} USR1/USR2 (#10/#12) to increase/decrease workers."
   end
 
   def init(&block)

data/lib/rbbt/workflow/step/accessor.rb

@@ -44,7 +44,7 @@ class Step
   end
 
   def self.files_dir(path)
-    path.nil? ? nil : path + '.files'
+    path.nil? ? nil : Path.setup(path + '.files')
   end
 
   def self.info_file(path)
@@ -154,8 +154,12 @@ class Step
     value = Annotated.purge value if defined? Annotated
     Open.lock(info_file, :lock => info_lock) do
       i = info(false).dup
+      value = Annotated.purge(value)
+
       i[key] = value 
+
       dump = Step.serialize_info(i)
+
       @info_cache = IndiferentHash.setup(i)
       Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now

data/lib/rbbt/workflow/step/info.rb

@@ -101,7 +101,7 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      if dep_info
+      if Hash === dep_info
         dep_info[:inputs].each do |k,v|
           all_inputs[k] = v unless all_inputs.include?(k)
         end if dep_info[:inputs]

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -26,7 +26,11 @@ module Workflow
 
         type = :path if file.split(".").last == 'as_path'
 
+        type = :nofile if file.split(".").last == 'nofile'
+
         case type
+        when :nofile
+          inputs[input.to_sym]  = Open.realpath(file)
         when :path
           inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
         when :io

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -65,7 +65,10 @@ module Workflow
 
       IndiferentHash.setup(resources)
 
-      default_resources = rules["default_resources"] || rules["defaults"]["resources"]
+      default_resources = rules["default_resources"] 
+      default_resources ||= rules["defaults"]["resources"] if rules["defaults"]
+      default_resources ||= {}
+
       default_resources.each{|k,v| resources[k] ||= v } if default_resources
 
       resources
@@ -98,6 +101,10 @@ module Workflow
       candidates
     end
 
+    def self.process(*args)
+      self.new.process(*args)
+    end
+
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
     def initialize(timer = 5, available_resources = {})
@@ -176,7 +183,9 @@ module Workflow
       end
     end
 
-    def process(rules, jobs)
+    def process(rules, jobs = nil)
+      jobs, rules = rules, {} if jobs.nil?
+      jobs = [jobs] if Step === jobs
       begin
 
         workload = Orchestrator.workload(jobs)

data/share/rbbt_commands/workflow/task

@@ -191,6 +191,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
 -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
 --fork Run job asyncronously and monitor progress. It monitors detached processes as well
+--orchestrate Run the job through the orchestrator
 --detach Run job asyncronously and detach process
 --exec Run job with no persistence
 -O--output* Save job result into file
@@ -438,6 +439,9 @@ begin
     end
 
     job.fork
+  elsif options[:orchestrate]
+    require 'rbbt/workflow/util/orchestrator'
+    Workflow::Orchestrator.process job
   else
     job.run(:stream)
     res = job

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.30
+  version: 5.33.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-14 00:00:00.000000000 Z
+date: 2022-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake