scout-gear 10.7.13 → 10.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da828c9bca6f9c8fd81012609f680393fb3ee4a64858927c49bb6c67e7919365
-  data.tar.gz: 804033931a4f77c5f5ee592c139fa00bbe0ae71b1622f945b01c7fb979e4bafe
+  metadata.gz: 7a3b309720cd9c3116460e682e1da0810a8601a406de1a86bb2e63c2dc1b2e7e
+  data.tar.gz: 03ee884261ff3393b2d9e52c1361260bd2c55d3038c244251dc75b357891eb93
 SHA512:
-  metadata.gz: 97403960a81eaca702c6c0eb62efacc8810cb0fa2005492b708834917e9665cdb3b9d19b68a8bdfefe9a26070f87d8ed35052642f7a7d2565750be6b0a677e4e
-  data.tar.gz: 4fbc6798b0775464f4b10084b74064cefc5accac797a64d09c84485eecb92b8dc09367f8a25589274ceb0fefffd769c9c0ed9e6ed7f303f45426c4115033882b
+  metadata.gz: 66b6d4780304a2f50f6a7aa384c86df78909851d1c9176d71ebaa67c7bb43b334b10b092fb4d636d1dbeada466db19d17af12d28f181de27ec0fd54450a1367c
+  data.tar.gz: 4efa2ebbc04db80cae4592887ab7f825cdc4783ffd381c1073c5d1e762f034b9c0c0a6e581948bfca96195653d88500c07525e922d7229c81306893a13e41809

data/VERSION

@@ -1 +1 @@
-10.7.13
+10.8.0

data/lib/scout/association/index.rb

@@ -69,7 +69,7 @@ module Association
         end
       end
 
-      tsv = transformer.tsv **kwargs.merge(data: data, fields: fields)
+      tsv = transformer.tsv **kwargs.merge(data: data, fields: fields).except(:identifiers)
     end
     index.extend Index
     index.parse_key_field

data/lib/scout/association.rb

@@ -106,8 +106,19 @@ module Association
 
     transformer.traverse key_field: original_source_header, fields: all_fields.values_at(*field_pos) do |k,v|
       v = v.dup if TSV === obj
-      k = source_index[k] if source_index
-      v[0] = Array === v[0] ? target_index.values_at(*v[0]) : target_index[v[0]] if target_index
+      if source_index
+        k = source_index[k]
+        next if k.nil? or k.empty?
+      end
+      if target_index
+        if Array === v[0]
+          v[0] = target_index.values_at(*v[0])
+          v = v.reject{|l| l[0].nil? || l[0].empty?}
+        else
+          v[0] = target_index[v[0]]
+          next if v[0].nil? or v[0].empty?
+        end
+      end
       [k, v]
     end
 

data/lib/scout/knowledge_base/entity.rb

@@ -78,6 +78,13 @@ class KnowledgeBase
       else
         identifier_files = database_identifier_files(name)
       end
+      if registered_identifiers = registered_options(name)[:identifiers]
+        if Array === registered_identifiers
+          identifier_files.concat registered_identifiers
+        else
+          identifier_files.push registered_identifiers
+        end
+      end
       identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
       identifier_files.uniq!
       identifier_files.collect!{|f| (Path === f) ? f : Path.setup(f.dup) }
@@ -95,6 +102,13 @@ class KnowledgeBase
       else
         identifier_files = database_identifier_files(name)
       end
+      if registered_identifiers = registered_options(name)[:identifiers]
+        if Array === registered_identifiers
+          identifier_files.concat registered_identifiers
+        else
+          identifier_files.push registered_identifiers
+        end
+      end
       identifier_files.concat Entity.identifier_files(target(name)) if defined? Entity
       identifier_files.uniq!
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if self.namespace

data/lib/scout/persist/engine/tokyocabinet.rb

@@ -108,43 +108,43 @@ if continue
       end
     end
 
-  def write_and_close
-    begin
-      write
-      yield
-    ensure
-      close
+    def write_and_close
+      begin
+        write
+        yield
+      ensure
+        close
+      end
     end
-  end
 
-  def self.importtsv(database, stream)
-    begin
-      bin = case database
-            when TokyoCabinet::HDB
-              'tchmgr'
-            when TokyoCabinet::BDB
-              'tcbmgr'
-            else
-              raise "Database not HDB or BDB: #{Log.fingerprint database}"
-            end
-      
-      database.close
-      CMD.cmd("#{bin} version", :log => false)
-      FileUtils.mkdir_p File.dirname(database.persistence_path)
-      CMD.cmd("#{bin} importtsv '#{database.persistence_path}'", :in => stream, :log => false, :dont_close_in => true)
-    rescue
-      Log.debug("tchmgr importtsv failed for: #{database.persistence_path}")
+    def self.importtsv(database, stream)
+      begin
+        bin = case database
+              when TokyoCabinet::HDB
+                'tchmgr'
+              when TokyoCabinet::BDB
+                'tcbmgr'
+              else
+                raise "Database not HDB or BDB: #{Log.fingerprint database}"
+              end
+
+        database.close
+        CMD.cmd("#{bin} version", :log => false)
+        FileUtils.mkdir_p File.dirname(database.persistence_path)
+        CMD.cmd("#{bin} importtsv '#{database.persistence_path}'", :in => stream, :log => false, :dont_close_in => true)
+      rescue
+        Log.debug("tchmgr importtsv failed for: #{database.persistence_path}")
+      end
     end
-  end
 
-  class << self
-    alias load_stream importtsv
-  end
+    class << self
+      alias load_stream importtsv
+    end
 
-  def importtsv(stream)
-    ScoutCabinet.load_stream(self, stream)
-  end
+    def importtsv(stream)
+      ScoutCabinet.load_stream(self, stream)
+    end
 
-  alias load_stream importtsv
+    alias load_stream importtsv
   end
 end

data/lib/scout/persist/tsv/adapter/base.rb

@@ -37,6 +37,7 @@ module TSVAdapter
     else
       begin
         TSV.setup(base, base.load_annotation_hash)
+        base.filename = base.persistence_path if base.filename.nil?
       rescue
         TSV.setup(base)
         base.save_annotation_hash

data/lib/scout/tsv/change_id/translate.rb

@@ -56,6 +56,7 @@ module TSV
 
     files.each do |file|
       #next if Path === file && ! Open.exist?(file)
+      Path.setup file if String === file and not Path === file
       begin
         file = file.produce if Path === file
         raise "Could no produce file" if FalseClass === file

data/lib/scout/tsv/index.rb

@@ -47,7 +47,7 @@ module TSV
              when :all
                "Index[#{target}]"
              else
-               "Index[#{Log.fingerprint(fields)}->#{target}]"
+               "Index[#{Array === fields ? fields * "," : fields}->#{target}]"
              end
 
     prefix += select_prefix_str(kwargs[:select])

data/lib/scout/tsv/traverse.rb

@@ -43,10 +43,12 @@ module TSV
     Log.debug log_message
     bar = log_message if TrueClass === bar
 
+    invert = select.delete :invert if Hash === select
     type_swap_tag = [type.to_s, @type.to_s] * "_"
     Log::ProgressBar.with_obj_bar(self, bar) do |bar|
       with_unnamed unnamed do
         each do |key,values|
+          next unless TSV.select key, values, select, invert: invert if select
           bar.tick if bar
           values = [values] if @type == :single
           if positions.nil?

data/lib/scout/tsv/util/reorder.rb

@@ -1,11 +1,11 @@
 require 'matrix'
 
 module TSV
-  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, **kwargs) 
-    res = self.annotate({})
+  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, data: nil, unnamed: true, **kwargs) 
+    res = data || self.annotate({})
     res.type = kwargs[:type] if kwargs.include?(:type)
     kwargs[:one2one] = one2one
-    key_field_name, field_names = with_unnamed do
+    key_field_name, field_names = with_unnamed unnamed do
       traverse key_field, fields, **kwargs do |k,v|
         if res.type == :double && merge && res.include?(k)
           current = res[k]
@@ -23,11 +23,11 @@ module TSV
             res[k] = merged
           end
         elsif res.type == :flat
-          res[k] ||= []
           if merge == :concat
+            res[k] ||= []
             res[k].concat v
           else
-            res[k] += v
+            res[k] = res[k].nil? ? v : res[k] + v
           end
         else
           res[k] = v

data/lib/scout/tsv.rb

@@ -18,7 +18,6 @@ module TSV
   extend Annotation
   annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer, :entity_options
 
-
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
     key, fields_str = field_options.split("~")

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -251,7 +251,13 @@ module Workflow
           workload = new_workload
           sleep timer
         end
-        all_jobs.each{|s| s.join }
+        all_jobs.each{|s| 
+          begin
+            s.join
+          rescue
+            Log.warn "Job #{s.short_path} ended with exception #{$!.class.to_s}: #{$!.message}"
+          end
+        }
       rescue TryAgain
         retry
       end
@@ -275,9 +281,12 @@ module Workflow
     produce_list
   end
 
-  def self.produce(jobs, produce_cpus: Etc.nprocessors, produce_timer: 5)
+  def self.produce(jobs, produce_cpus: Etc.nprocessors, produce_timer: 1)
     jobs = [jobs] unless Array === jobs
     orchestrator = Orchestrator.new produce_timer.to_i, cpus: produce_cpus.to_i
-    orchestrator.process({}, jobs)
+    begin
+      orchestrator.process({}, jobs)
+    rescue Orchestrator::NoWork
+    end
   end
 end

data/lib/scout/workflow/step.rb

@@ -34,6 +34,15 @@ class Step
     @mutex.synchronize(&block)
   end
 
+  def provided_inputs
+    @provided_inputs ||= begin
+                  if info_file && Open.exists?(info_file)
+                    info[:provided_inputs]
+                  else
+                    {}
+                  end
+                end
+  end
   def inputs
     @inputs ||= begin
                   if info_file && Open.exists?(info_file)
@@ -174,6 +183,7 @@ class Step
         reset_info :status => :setup, :issued => Time.now,
           :pid => Process.pid, :pid_hostname => Misc.hostname, 
           :task_name => task_name, :workflow => workflow.to_s,
+          :provided_inputs => Annotation.purge(provided_inputs),
           :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
           :dependencies => (dependencies || []) .collect{|d| d.path }
 

data/lib/scout/workflow.rb

@@ -133,10 +133,11 @@ module Workflow
         begin
           workflow_name, *subworkflows = complete_workflow_name.split("::")
           workflow_file = workflow_name
-          workflow_file = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow_file)
-          workflow_file = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file)
-          workflow_file = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file)
-          if Open.exists?(workflow_file)
+          workflow_file = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow_file) && ! Open.directory?(workflow_file)
+          workflow_file = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file) && ! Open.directory?(workflow_file)
+          workflow_file = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file) && ! Open.directory?(workflow_file)
+
+          if Open.exists?(workflow_file) && ! Open.directory?(workflow_file)
             self.main = nil
             require_workflow_file(workflow_file)
           elsif autoinstall

data/lib/scout-gear.rb

@@ -1,7 +1,7 @@
 require 'scout-essentials'
 require_relative 'scout/tsv'
 
-Path.path_maps[:scout_gear] = File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
+Path.path_maps[:scout_gear_lib] = File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
 
 Persist.cache_dir     = Scout.var.cache.persistence
 TmpFile.tmpdir        = Scout.tmp.find :user

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.7.13 ruby lib
+# stub: scout-gear 10.8.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.7.13".freeze
+  s.version = "10.8.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "1980-01-02"
+  s.date = "2025-06-05"
   s.description = "Scout gear: workflow, TSVs, persistence, entities, associations, and knowledge_bases.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -250,7 +250,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.7".freeze
+  s.rubygems_version = "3.6.6".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/find

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'scout-gear'
-require 'scout/offsite'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
@@ -80,5 +79,5 @@ if where.nil? || where == 'all' || path.path_maps.include?(where.to_sym)
     puts location
   end
 else
-  puts SSHLine.command(where, $0, ARGV, options.merge("where" => :all))
+  raise ParameterException, "Where '#{where}' not identified. Try scout-camp if looking for a remote file"
 end

data/scout_commands/workflow/process

@@ -40,6 +40,8 @@ queue_dir = Scout.var.queue
 
 class TrayAgain < Exception; end
 
+options.keys_to_sym!
+
 begin
   if ARGV.empty?
     files = queue_dir.glob_all("*/*/*")
@@ -67,10 +69,9 @@ begin
   end
 
   jobs = files.collect{|file| Workflow.queue_job(file) }
+  
+
   begin
-    options.keys.each do |key|
-      options[key.to_sym] = options.delete(key)
-    end
     Workflow.produce(jobs, **options)
   rescue Workflow::Orchestrator::NoWork
   end

data/test/scout/tsv/test_traverse.rb

@@ -374,5 +374,47 @@ row3    a
       assert Annotation::AnnotatedObject === data["row1"]
     end
   end
+
+  def test_tsv_traverse_select
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    all_values = []
+    tsv.traverse "ValueA", :all, select: {ValueA: "A"} do |k,v|
+      all_values.concat(v)
+    end
+    refute all_values.flatten.include? "row1"
+    refute all_values.flatten.include? "a"
+    refute all_values.flatten.include? "aaa"
+
+    assert_include all_values.flatten, "row2"
+
+    tsv.traverse "ValueA", :all, select: {ValueA: "a", invert: true} do |k,v|
+      all_values.concat(v)
+    end
+    refute all_values.flatten.include? "row1"
+    refute all_values.flatten.include? "a"
+    refute all_values.flatten.include? "aaa"
+
+    assert_include all_values.flatten, "row2"
+
+    all_values = []
+    tsv.traverse "Id", :all do |k,v|
+      all_values.concat(v)
+    end
+    assert_include all_values.flatten, "row1"
+    assert_include all_values.flatten, "a"
+    assert_include all_values.flatten, "aaa"
+  end
+
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.7.13
+  version: 10.8.0
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2025-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -334,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.6
 specification_version: 4
 summary: basic gear for scouts
 test_files: []