scout-gear 10.7.13 → 10.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da828c9bca6f9c8fd81012609f680393fb3ee4a64858927c49bb6c67e7919365
-  data.tar.gz: 804033931a4f77c5f5ee592c139fa00bbe0ae71b1622f945b01c7fb979e4bafe
+  metadata.gz: a16e6fadbdab77f5dd56efff644adb5b79c7765642d87cee21f297eb4e783bee
+  data.tar.gz: 2d4e225ec569e3fc8f569835b865effc6b3428e23b6feba3d4f9e8d092cb8a13
 SHA512:
-  metadata.gz: 97403960a81eaca702c6c0eb62efacc8810cb0fa2005492b708834917e9665cdb3b9d19b68a8bdfefe9a26070f87d8ed35052642f7a7d2565750be6b0a677e4e
-  data.tar.gz: 4fbc6798b0775464f4b10084b74064cefc5accac797a64d09c84485eecb92b8dc09367f8a25589274ceb0fefffd769c9c0ed9e6ed7f303f45426c4115033882b
+  metadata.gz: 3103bc1c224105e7a5d5ac74181a62c6457d6b38cb4f59db2615be3962cef24777f25918977d738c02903b0b831606991c1015b2a7fa77fb03def9fce1b29cc7
+  data.tar.gz: a09e7bf564fc5e4711bee14df56b7e8cd108b52cd9f02318d255964b58a9bc7585829a411629733c9cdd78060ac8d9b932491924d8227c03540fe93f55d04ccf

data/VERSION

@@ -1 +1 @@
-10.7.13
+10.8.1

data/bin/scout

@@ -66,7 +66,7 @@ require 'scout/simple_opt'
 require 'scout/log'
 
 options = SOPT.setup <<EOF
-Ruby bioinformatics toolkit
+Scout
 
 $ #{$0} <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value'
 
@@ -75,7 +75,7 @@ $ #{$0} <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-valu
 --dev*                  #{Log.color :yellow, "Find development libraries in the directory specified"}
 --nocolor               #{Log.color :yellow, "Disable colored output"}
 --nobar                 #{Log.color :yellow, "Disable progress report"}
---locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
+--locate_file           #{Log.color :yellow, "Report the location of the script but do not execute it"}
 -ck--config_keys*       #{Log.color :yellow, "Override some config keys"}
 EOF
 

data/lib/scout/association/index.rb

@@ -69,7 +69,7 @@ module Association
         end
       end
 
-      tsv = transformer.tsv **kwargs.merge(data: data, fields: fields)
+      tsv = transformer.tsv **kwargs.merge(data: data, fields: fields).except(:identifiers)
     end
     index.extend Index
     index.parse_key_field

data/lib/scout/association.rb

@@ -106,8 +106,19 @@ module Association
 
     transformer.traverse key_field: original_source_header, fields: all_fields.values_at(*field_pos) do |k,v|
       v = v.dup if TSV === obj
-      k = source_index[k] if source_index
-      v[0] = Array === v[0] ? target_index.values_at(*v[0]) : target_index[v[0]] if target_index
+      if source_index
+        k = source_index[k]
+        next if k.nil? or k.empty?
+      end
+      if target_index
+        if Array === v[0]
+          v[0] = target_index.values_at(*v[0])
+          v = v.reject{|l| l[0].nil? || l[0].empty?}
+        else
+          v[0] = target_index[v[0]]
+          next if v[0].nil? or v[0].empty?
+        end
+      end
       [k, v]
     end
 

data/lib/scout/knowledge_base/entity.rb

@@ -78,6 +78,13 @@ class KnowledgeBase
       else
         identifier_files = database_identifier_files(name)
       end
+      if registered_identifiers = registered_options(name)[:identifiers]
+        if Array === registered_identifiers
+          identifier_files.concat registered_identifiers
+        else
+          identifier_files.push registered_identifiers
+        end
+      end
       identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
       identifier_files.uniq!
       identifier_files.collect!{|f| (Path === f) ? f : Path.setup(f.dup) }
@@ -95,6 +102,13 @@ class KnowledgeBase
       else
         identifier_files = database_identifier_files(name)
       end
+      if registered_identifiers = registered_options(name)[:identifiers]
+        if Array === registered_identifiers
+          identifier_files.concat registered_identifiers
+        else
+          identifier_files.push registered_identifiers
+        end
+      end
       identifier_files.concat Entity.identifier_files(target(name)) if defined? Entity
       identifier_files.uniq!
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if self.namespace

data/lib/scout/knowledge_base/traverse.rb

@@ -106,7 +106,7 @@ class KnowledgeBase
         new_paths = _fp(rest, clean_matches, new_assignments)
         next unless new_paths
         paths[match] = new_paths
-      end
+      end if matches
 
       return false if paths.empty?
 

data/lib/scout/persist/engine/tokyocabinet.rb

@@ -108,43 +108,43 @@ if continue
       end
     end
 
-  def write_and_close
-    begin
-      write
-      yield
-    ensure
-      close
+    def write_and_close
+      begin
+        write
+        yield
+      ensure
+        close
+      end
     end
-  end
 
-  def self.importtsv(database, stream)
-    begin
-      bin = case database
-            when TokyoCabinet::HDB
-              'tchmgr'
-            when TokyoCabinet::BDB
-              'tcbmgr'
-            else
-              raise "Database not HDB or BDB: #{Log.fingerprint database}"
-            end
-      
-      database.close
-      CMD.cmd("#{bin} version", :log => false)
-      FileUtils.mkdir_p File.dirname(database.persistence_path)
-      CMD.cmd("#{bin} importtsv '#{database.persistence_path}'", :in => stream, :log => false, :dont_close_in => true)
-    rescue
-      Log.debug("tchmgr importtsv failed for: #{database.persistence_path}")
+    def self.importtsv(database, stream)
+      begin
+        bin = case database
+              when TokyoCabinet::HDB
+                'tchmgr'
+              when TokyoCabinet::BDB
+                'tcbmgr'
+              else
+                raise "Database not HDB or BDB: #{Log.fingerprint database}"
+              end
+
+        database.close
+        CMD.cmd("#{bin} version", :log => false)
+        FileUtils.mkdir_p File.dirname(database.persistence_path)
+        CMD.cmd("#{bin} importtsv '#{database.persistence_path}'", :in => stream, :log => false, :dont_close_in => true)
+      rescue
+        Log.debug("tchmgr importtsv failed for: #{database.persistence_path}")
+      end
     end
-  end
 
-  class << self
-    alias load_stream importtsv
-  end
+    class << self
+      alias load_stream importtsv
+    end
 
-  def importtsv(stream)
-    ScoutCabinet.load_stream(self, stream)
-  end
+    def importtsv(stream)
+      ScoutCabinet.load_stream(self, stream)
+    end
 
-  alias load_stream importtsv
+    alias load_stream importtsv
   end
 end

data/lib/scout/persist/tsv/adapter/base.rb

@@ -37,6 +37,7 @@ module TSVAdapter
     else
       begin
         TSV.setup(base, base.load_annotation_hash)
+        base.filename = base.persistence_path if base.filename.nil?
       rescue
         TSV.setup(base)
         base.save_annotation_hash

data/lib/scout/tsv/change_id/translate.rb

@@ -56,6 +56,7 @@ module TSV
 
     files.each do |file|
       #next if Path === file && ! Open.exist?(file)
+      Path.setup file if String === file and not Path === file
       begin
         file = file.produce if Path === file
         raise "Could no produce file" if FalseClass === file

data/lib/scout/tsv/index.rb

@@ -47,7 +47,7 @@ module TSV
              when :all
                "Index[#{target}]"
              else
-               "Index[#{Log.fingerprint(fields)}->#{target}]"
+               "Index[#{Array === fields ? fields * "," : fields}->#{target}]"
              end
 
     prefix += select_prefix_str(kwargs[:select])

data/lib/scout/tsv/traverse.rb

@@ -43,10 +43,12 @@ module TSV
     Log.debug log_message
     bar = log_message if TrueClass === bar
 
+    invert = select.delete :invert if Hash === select
     type_swap_tag = [type.to_s, @type.to_s] * "_"
     Log::ProgressBar.with_obj_bar(self, bar) do |bar|
       with_unnamed unnamed do
         each do |key,values|
+          next unless TSV.select key, values, select, invert: invert if select
           bar.tick if bar
           values = [values] if @type == :single
           if positions.nil?

data/lib/scout/tsv/util/reorder.rb

@@ -1,11 +1,11 @@
 require 'matrix'
 
 module TSV
-  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, **kwargs) 
-    res = self.annotate({})
+  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, data: nil, unnamed: true, **kwargs) 
+    res = data || self.annotate({})
     res.type = kwargs[:type] if kwargs.include?(:type)
     kwargs[:one2one] = one2one
-    key_field_name, field_names = with_unnamed do
+    key_field_name, field_names = with_unnamed unnamed do
       traverse key_field, fields, **kwargs do |k,v|
         if res.type == :double && merge && res.include?(k)
           current = res[k]
@@ -23,11 +23,11 @@ module TSV
             res[k] = merged
           end
         elsif res.type == :flat
-          res[k] ||= []
           if merge == :concat
+            res[k] ||= []
             res[k].concat v
           else
-            res[k] += v
+            res[k] = res[k].nil? ? v : res[k] + v
           end
         else
           res[k] = v

data/lib/scout/tsv.rb

@@ -18,7 +18,6 @@ module TSV
   extend Annotation
   annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer, :entity_options
 
-
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
     key, fields_str = field_options.split("~")

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -251,7 +251,13 @@ module Workflow
           workload = new_workload
           sleep timer
         end
-        all_jobs.each{|s| s.join }
+        all_jobs.each{|s| 
+          begin
+            s.join
+          rescue
+            Log.warn "Job #{s.short_path} ended with exception #{$!.class.to_s}: #{$!.message}"
+          end
+        }
       rescue TryAgain
         retry
       end
@@ -275,9 +281,12 @@ module Workflow
     produce_list
   end
 
-  def self.produce(jobs, produce_cpus: Etc.nprocessors, produce_timer: 5)
+  def self.produce(jobs, produce_cpus: Etc.nprocessors, produce_timer: 1)
     jobs = [jobs] unless Array === jobs
     orchestrator = Orchestrator.new produce_timer.to_i, cpus: produce_cpus.to_i
-    orchestrator.process({}, jobs)
+    begin
+      orchestrator.process({}, jobs)
+    rescue Orchestrator::NoWork
+    end
   end
 end

data/lib/scout/workflow/entity.rb

@@ -2,11 +2,21 @@ require 'scout/entity'
 require 'scout/workflow'
 
 module EntityWorkflow
+  attr_accessor :entity_name
+
+  def entity_name=(name)
+    @entity_name = name
+    helper name do
+      entity
+    end
+  end
 
   def self.extended(base)
     base.extend Workflow
     base.extend Entity
 
+    base.entity_name ||= 'entity'
+
     base.instance_variable_set(:@annotation_inputs, IndiferentHash.setup({})) unless base.instance_variables.include?(:@annotation_inputs)
     class << base
       def annotation_input(name, type=nil, desc=nil, default=nil, options = {})
@@ -20,7 +30,7 @@ module EntityWorkflow
       base.setup(clean_name.dup, inputs.to_hash)
     end
 
-    base.helper :entity_list do 
+    base.helper :entity_list do
       list = inputs.last
       list = list.load if Step === list
       base.setup(list, inputs.to_hash)
@@ -46,12 +56,13 @@ module EntityWorkflow
         input annotation
       end
     end
+
     case property_type
     when :single, :single2array
-      input :entity, :string, "#{self.to_s} identifier", nil, jobname: true
+      input entity_name, :string, "#{self.to_s} identifier", nil, jobname: true
       task(task_name => result_type, &block)
     when :both
-      input :entity, :string, "#{self.to_s} identifier", nil, jobname: true
+      input entity_name, :string, "#{self.to_s} identifier", nil, jobname: true
       input :list, :array, "#{self.to_s} identifier list"
       task(task_name => result_type, &block)
     else
@@ -59,7 +70,8 @@ module EntityWorkflow
       task(task_name => result_type, &block)
     end
 
-    property task_name => property_type do |*args|
+    property_name = task_name.to_s.sub(/^(#{entity_name}_list|#{entity_name}|list)_/, '')
+    property property_name => property_type do |*args|
       job = job(task_name, *args)
       Array === job ? job.collect(&:run) : job.run
     end

data/lib/scout/workflow/step.rb

@@ -34,6 +34,15 @@ class Step
     @mutex.synchronize(&block)
   end
 
+  def provided_inputs
+    @provided_inputs ||= begin
+                  if info_file && Open.exists?(info_file)
+                    info[:provided_inputs]
+                  else
+                    {}
+                  end
+                end
+  end
   def inputs
     @inputs ||= begin
                   if info_file && Open.exists?(info_file)
@@ -174,6 +183,7 @@ class Step
         reset_info :status => :setup, :issued => Time.now,
           :pid => Process.pid, :pid_hostname => Misc.hostname, 
           :task_name => task_name, :workflow => workflow.to_s,
+          :provided_inputs => Annotation.purge(provided_inputs),
           :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
           :dependencies => (dependencies || []) .collect{|d| d.path }
 

data/lib/scout/workflow/task.rb

@@ -83,11 +83,12 @@ module Task
 
       id = DEFAULT_NAME if id.nil?
 
+      sanitized_id = Path.sanitize_filename(id, 150)
       if non_default_inputs.any? && !(non_default_inputs == [jobname_input] && provided_inputs[jobname_input] == id)
         hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
-        name = [id, hash] * "_"
+        name = [sanitized_id, hash] * "_"
       else
-        name = id
+        name = sanitized_id
       end
 
       extension = self.extension

data/lib/scout/workflow/usage.rb

@@ -139,14 +139,16 @@ module Workflow
     dep_tree = {}
     task = self.tasks[task_name]
     raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
-    task.deps.each do |workflow, task, options|
-      next if seen.include? dep
-      seen << [workflow, task, options.merge(seen_options)]
-      next if task.nil?
 
-      key = [workflow, task]
+    task.deps.each do |workflow, dep_task, options|
+      dep = [workflow, dep_task, options.merge(seen_options)]
+      #next if seen.include? dep
+      seen << dep
+      next if dep_task.nil?
 
-      dep_tree[key] = workflow.dep_tree(task, seen, options.merge(seen_options))
+      dep_key = [workflow, dep_task]
+
+      dep_tree[dep_key] = workflow.dep_tree(dep_task, seen, options.merge(seen_options))
     end if task.deps
 
     @dep_tree[key] = dep_tree if save
@@ -163,6 +165,7 @@ module Workflow
   def _prov_tasks(tree)
     tasks = [] 
     heap = tree.values
+    heap = [tree]
     while heap.any?
       t = heap.pop
       t.each do |k,v|
@@ -188,10 +191,8 @@ module Workflow
       first = last.nil?
       last = _prov_tasks(workflow.dep_tree(task_name))
 
-      break if child
-
       if child
-        description << "->" << task_name.to_s
+        #·description << "->" << task_name.to_s
       elsif first
         description << "" << task_name.to_s
       else
@@ -204,7 +205,6 @@ module Workflow
   end
 
   def prov_tree(tree, offset = 0, seen = [])
-
     return "" if tree.empty?
 
     lines = []

data/lib/scout/workflow.rb

@@ -133,10 +133,11 @@ module Workflow
         begin
           workflow_name, *subworkflows = complete_workflow_name.split("::")
           workflow_file = workflow_name
-          workflow_file = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow_file)
-          workflow_file = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file)
-          workflow_file = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file)
-          if Open.exists?(workflow_file)
+          workflow_file = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow_file) && ! Open.directory?(workflow_file)
+          workflow_file = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file) && ! Open.directory?(workflow_file)
+          workflow_file = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file) && ! Open.directory?(workflow_file)
+
+          if Open.exists?(workflow_file) && ! Open.directory?(workflow_file)
             self.main = nil
             require_workflow_file(workflow_file)
           elsif autoinstall

data/lib/scout-gear.rb

@@ -1,7 +1,7 @@
 require 'scout-essentials'
 require_relative 'scout/tsv'
 
-Path.path_maps[:scout_gear] = File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
+Path.path_maps[:scout_gear_lib] = File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
 
 Persist.cache_dir     = Scout.var.cache.persistence
 TmpFile.tmpdir        = Scout.tmp.find :user

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.7.13 ruby lib
+# stub: scout-gear 10.8.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.7.13".freeze
+  s.version = "10.8.1".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "1980-01-02"
+  s.date = "2025-06-12"
   s.description = "Scout gear: workflow, TSVs, persistence, entities, associations, and knowledge_bases.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -250,7 +250,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.7".freeze
+  s.rubygems_version = "3.6.6".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/find

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'scout-gear'
-require 'scout/offsite'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
@@ -80,5 +79,5 @@ if where.nil? || where == 'all' || path.path_maps.include?(where.to_sym)
     puts location
   end
 else
-  puts SSHLine.command(where, $0, ARGV, options.merge("where" => :all))
+  raise ParameterException, "Where '#{where}' not identified. Try scout-camp if looking for a remote file"
 end

data/scout_commands/workflow/process

@@ -40,6 +40,8 @@ queue_dir = Scout.var.queue
 
 class TrayAgain < Exception; end
 
+options.keys_to_sym!
+
 begin
   if ARGV.empty?
     files = queue_dir.glob_all("*/*/*")
@@ -67,10 +69,9 @@ begin
   end
 
   jobs = files.collect{|file| Workflow.queue_job(file) }
+  
+
   begin
-    options.keys.each do |key|
-      options[key.to_sym] = options.delete(key)
-    end
     Workflow.produce(jobs, **options)
   rescue Workflow::Orchestrator::NoWork
   end

data/scout_commands/workflow/task

@@ -115,7 +115,8 @@ else
   when "serial"
     job.run(true) 
   when "local"
-    orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
+    refresh = Scout::Config.get :refresh, :deploy, :local, :orchestrator, default: 3
+    orchestrator = Workflow::Orchestrator.new refresh.to_f, "cpus" => Misc.processors
     orchestrator.process({}, job)
   when "slurm"
     require 'rbbt-scout'

data/test/scout/tsv/test_traverse.rb

@@ -374,5 +374,47 @@ row3    a
       assert Annotation::AnnotatedObject === data["row1"]
     end
   end
+
+  def test_tsv_traverse_select
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    all_values = []
+    tsv.traverse "ValueA", :all, select: {ValueA: "A"} do |k,v|
+      all_values.concat(v)
+    end
+    refute all_values.flatten.include? "row1"
+    refute all_values.flatten.include? "a"
+    refute all_values.flatten.include? "aaa"
+
+    assert_include all_values.flatten, "row2"
+
+    tsv.traverse "ValueA", :all, select: {ValueA: "a", invert: true} do |k,v|
+      all_values.concat(v)
+    end
+    refute all_values.flatten.include? "row1"
+    refute all_values.flatten.include? "a"
+    refute all_values.flatten.include? "aaa"
+
+    assert_include all_values.flatten, "row2"
+
+    all_values = []
+    tsv.traverse "Id", :all do |k,v|
+      all_values.concat(v)
+    end
+    assert_include all_values.flatten, "row1"
+    assert_include all_values.flatten, "a"
+    assert_include all_values.flatten, "aaa"
+  end
+
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.7.13
+  version: 10.8.1
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2025-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -334,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.6
 specification_version: 4
 summary: basic gear for scouts
 test_files: []