scout-gear 10.6.1 → 10.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 217809c3c2f63740c3208ed81c6850a14cae2c6dc0687d185384b13e7747d07e
-  data.tar.gz: d174ef3d473a5b3f25b520d9fb088b4717527f53c5c322727da4d89f935a3e61
+  metadata.gz: ac64d599e0df97b8ba5abd7aef4855ed1181ab9d9a3ba39196255b6649f56a0c
+  data.tar.gz: 9f1cf16da9432beb46eeb486de5bab1c57e460b09f2126556126eb356c6a6fd6
 SHA512:
-  metadata.gz: e5387627839d6370801c22d6fa3b483db9ad3ea04242b7e999c8827c2999ced34c25369643a3c7aba4736afa2abf0caf390ed68ba2aa5dd1156b3ef7f063a9ec
-  data.tar.gz: 8b5f90d48f7712e7b6b692f03b9eb7fab9b22540c7f8ffa2daae0158f13be362369ed018a23d13f435b24ebce6bc761a9634386ddf84cec100f0ab0447095157
+  metadata.gz: 27015e493797ff3e80631a49e0d9bf2ccc115997e86d449b217dd55adc6dd0abd7701019c19467e9c48aa6bc7ff3c372238fcd58200069533686e2729870938f
+  data.tar.gz: 4930f226926fdc18b06952f037f8e60b45b49116be115409cbcee36bf66202f0bb909691f20fddc6711babbb2ec9b604a5f3e551742ab52cb3f96748665b13a6

data/VERSION

@@ -1 +1 @@
-10.6.1
+10.7.0

data/lib/scout/entity.rb

@@ -3,6 +3,7 @@ require_relative 'entity/format'
 require_relative 'entity/property'
 require_relative 'entity/object'
 require_relative 'entity/identifiers'
+
 module Entity
   def self.extended(base)
     base.extend Annotation

data/lib/scout/tsv/annotation/repo.rb

@@ -1,17 +1,21 @@
 module Persist
+  REPO_CACHE = {}
   def self.annotation_repo_persist(repo, name, &block)
 
     if String === repo
-      repo = repo.find if Path === repo
-      repo = Persist.open_tokyocabinet(repo, false, :list, :BDB)
-      repo_fields = ["literal", "annotation_types", "JSON"]
-      TSV.setup(repo, :fields => repo_fields, :key_field => "Annotation ID")
-      repo.save_annotation_hash
+      repo = REPO_CACHE[repo] ||= begin
+                                    repo = repo.find if Path === repo
+                                    repo = Persist.open_tokyocabinet(repo, false, :list, :BDB)
+                                    repo_fields = ["literal", "annotation_types", "JSON"]
+                                    TSV.setup(repo, :fields => repo_fields, :key_field => "Annotation ID")
+                                    repo.save_annotation_hash
+                                    repo
+                                  end
       repo.close
-    else
-      repo_fields = repo.fields
     end
 
+    repo_fields = repo.fields
+
     subkey = name + ":"
 
     keys = repo.read_and_close do

data/lib/scout/tsv/attach.rb

@@ -154,7 +154,7 @@ module TSV
 
             other_values.zip(overlaps).each do |v,overlap|
               if type == :list
-                current_values[overlap] = v if current_values[overlap].nil? || String === current_values[overlap] && current_values[overlap].empty?
+                current_values[overlap] = v if current_values[overlap].nil? || (String === current_values[overlap] && current_values[overlap].empty?)
               elsif type == :flat
                 next if v.nil?
                 v = [v] unless Array === v
@@ -206,7 +206,7 @@ module TSV
               if overlap == :key
                 other_key = Array === v ? v : v.first
               elsif type == :list
-                new_values[overlap] = v if v[overlap].nil? || String === v[overlap] && v[overlap].empty?
+                new_values[overlap] = v if new_values[overlap].nil? || (String === new_values[overlap] && new_values[overlap].empty?)
               else
                 v = [v] unless Array === v
                 new_values[overlap].concat v

data/lib/scout/tsv/transformer.rb

@@ -80,6 +80,7 @@ module TSV
     def each(*args, **kwargs, &block)
       kwargs[:into] = @dumper
       kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
+      @dumper.namespace ||= @namespace
       @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
       Open.traverse(@parser, *args, **kwargs) do |k,v|
         NamedArray.setup(v, @parser.fields, k) unless @unnamed

data/lib/scout/tsv/traverse.rb

@@ -1,6 +1,6 @@
 require_relative 'parser'
 module TSV
-  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
+  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, uniq: false, &block)
     key_field = key_field_pos if key_field.nil?
     fields = fields_pos.dup if fields.nil?
     type = @type if type.nil?
@@ -74,6 +74,7 @@ module TSV
           values = TSV.cast_value(values, cast) if cast
 
           if Array === key 
+            key = key.uniq if uniq
             if @type == :double && one2one
               if one2one == :strict
                 key.each_with_index do |key_i,i|

data/lib/scout/tsv.rb

@@ -11,13 +11,12 @@ require_relative 'tsv/open'
 require_relative 'tsv/attach'
 require_relative 'tsv/change_id'
 require_relative 'tsv/stream'
-require_relative 'tsv/entity'
 require_relative 'tsv/annotation'
 require_relative 'tsv/csv'
 
 module TSV
   extend Annotation
-  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :entity_options, :serializer
+  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer
 
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
@@ -122,8 +121,6 @@ module TSV
         end
       end
 
-      tsv.entity_options = entity_options
-
       tsv
     end
   end

data/lib/scout/work_queue.rb

@@ -138,17 +138,19 @@ class WorkQueue
   end
 
   def abort
+    @aborted = true
     Log.low "Aborting #{@workers.length} workers in queue #{queue_id}"
     @worker_mutex.synchronize do
       @workers.each do |w| 
-        ScoutSemaphore.post_semaphore(@output.write_sem) if @output
-        ScoutSemaphore.post_semaphore(@input.read_sem) if @input
+        ScoutSemaphore.post_semaphore(@output.write_sem)
+        ScoutSemaphore.post_semaphore(@input.read_sem)
         w.abort 
       end
     end
   end
 
   def close
+    return if @closed || @aborted
     @closed = true
     @worker_mutex.synchronize{ @workers.length }.times do
       begin
@@ -165,6 +167,7 @@ class WorkQueue
   end
 
   def join(clean = true)
+    close
     begin
       @waiter.join if @waiter
       @reader.join if @reader

data/lib/scout/workflow/definition.rb

@@ -21,12 +21,14 @@ module Workflow
     @helpers ||= {}
   end
 
-  def helper(name, *args, &block)
+  def helper(name, *args, **kwargs, &block)
     if block_given?
       helpers[name] = block
     else
       raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
-      helpers[name].call(*args)
+      o = Object.new
+      o.extend step_module
+      o.send(name, *args, **kwargs)
     end
   end
 

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -246,6 +246,7 @@ module Workflow
           workload = new_workload
           sleep timer
         end
+        all_jobs.each{|s| s.join }
       rescue TryAgain
         retry
       end

data/lib/scout/workflow/step/dependencies.rb

@@ -36,7 +36,7 @@ class Step
 
     dependencies.each do |dep|
       if dep.present? && ! dep.updated?
-        Log.debug "Clean outdated #{dep.path}"
+        Log.medium "Clean outdated #{dep.path}"
         dep.clean
       end
 

data/lib/scout/workflow/step/status.rb

@@ -26,6 +26,7 @@ class Step
     newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
     newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
 
+    Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
     newer.empty?
   end
 

data/lib/scout/workflow/task/dependencies.rb

@@ -101,7 +101,7 @@ module Task
             else
               dep = _res
               dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
               non_default_inputs.concat(dep_non_default_inputs)
             end
           end

data/lib/scout/workflow/task/inputs.rb

@@ -47,7 +47,7 @@ module Task
       elsif String === value && Symbol === provided && provided.to_s == value
         same_as_default = true
       else
-      same_as_default = false
+        same_as_default = false
       end
       if ! provided.nil? && ! same_as_default
         non_default_inputs << name.to_sym

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.6.1 ruby lib
+# stub: scout-gear 10.7.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.6.1".freeze
+  s.version = "10.7.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2024-05-28"
+  s.date = "2024-06-05"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -72,7 +72,6 @@ Gem::Specification.new do |s|
     "lib/scout/tsv/change_id/translate.rb",
     "lib/scout/tsv/csv.rb",
     "lib/scout/tsv/dumper.rb",
-    "lib/scout/tsv/entity.rb",
     "lib/scout/tsv/index.rb",
     "lib/scout/tsv/open.rb",
     "lib/scout/tsv/parser.rb",
@@ -184,7 +183,6 @@ Gem::Specification.new do |s|
     "test/scout/tsv/test_change_id.rb",
     "test/scout/tsv/test_csv.rb",
     "test/scout/tsv/test_dumper.rb",
-    "test/scout/tsv/test_entity.rb",
     "test/scout/tsv/test_index.rb",
     "test/scout/tsv/test_open.rb",
     "test/scout/tsv/test_parser.rb",

data/test/scout/test_work_queue.rb

@@ -117,6 +117,7 @@ class TestWorkQueue < Test::Unit::TestCase
 
         assert_raise ScoutException do
           begin
+            t.join
             q.join(false)
           rescue
             t.raise($!)
@@ -158,6 +159,7 @@ class TestWorkQueue < Test::Unit::TestCase
 
         assert_raise ScoutException do
           begin
+            t.join
             q.join(false)
           rescue Exception
             t.raise($!)

data/test/scout/test_workflow.rb

@@ -85,4 +85,20 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal "Baking batter (Mixing base (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour)) with mixer (share/pantry/blueberries))",
       Baking.job(:bake_muffin_tray, "Blueberry muffin", :add_bluberries => true).run
   end
+
+  def test_call_helper
+    wf = Module.new do
+      extend Workflow
+
+      helper :m1 do
+        "TEST"
+      end
+
+      helper :m2 do
+        m1
+      end
+    end
+
+    assert_equal "TEST", wf.helper(:m2)
+  end
 end

data/test/scout/tsv/test_attach.rb

@@ -449,4 +449,33 @@ A    Id3
     assert res["row2"].include? "Id3"
     assert ! res["row2"].include?("b")
   end
+
+  def test_attach_list_to_list_with_complete
+    content1 =<<-EOF
+#Id    ValueA    ValueB
+row1    a    b
+row2    A    B
+    EOF
+
+    content2 =<<-EOF
+#Id    ValueC
+row1    c
+row2    C
+row3    CC
+    EOF
+
+    tsv1 = tsv2 = index = nil
+    TmpFile.with_file(content1) do |filename|
+      tsv1 = TSV.open(File.open(filename), type: :list, fields: ["ValueA"], sep: /\s+/)
+    end
+
+    TmpFile.with_file(content2) do |filename|
+      tsv2 = TSV.open(File.open(filename), type: :list, sep: /\s+/)
+    end
+
+    res = tsv1.attach tsv2, :fields => ["ValueC"], complete: true
+    assert res["row2"].include?("C")
+    refute res["row2"].include?("b")
+    assert res["row3"].include?("CC")
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.6.1
+  version: 10.7.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-05-28 00:00:00.000000000 Z
+date: 2024-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -184,7 +184,6 @@ files:
 - lib/scout/tsv/change_id/translate.rb
 - lib/scout/tsv/csv.rb
 - lib/scout/tsv/dumper.rb
-- lib/scout/tsv/entity.rb
 - lib/scout/tsv/index.rb
 - lib/scout/tsv/open.rb
 - lib/scout/tsv/parser.rb
@@ -296,7 +295,6 @@ files:
 - test/scout/tsv/test_change_id.rb
 - test/scout/tsv/test_csv.rb
 - test/scout/tsv/test_dumper.rb
-- test/scout/tsv/test_entity.rb
 - test/scout/tsv/test_index.rb
 - test/scout/tsv/test_open.rb
 - test/scout/tsv/test_parser.rb

data/lib/scout/tsv/entity.rb

@@ -1,5 +0,0 @@
-module TSV
-  def prepare_entity(obj, *args)
-    obj
-  end
-end