scout-gear 10.7.1 → 10.7.2

data/lib/scout/workflow/task/dependencies.rb

@@ -9,22 +9,33 @@ module Task
     load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
       task = step_options.delete(:task) if step_options.include?(:task)
       workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
-      id = step_options.delete(:id) if step_options.include?(:id)
-      id = step_options.delete(:jobname) if step_options.include?(:jobname)
+
+      step_id = id
+      step_id = step_options.delete(:jobname) if step_options.include?(:jobname)
+      step_id = nil if step_id == Task::DEFAULT_NAME
 
       step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
-      step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
+      step_inputs = IndiferentHash.add_defaults step_inputs.dup, definition_options
 
       resolved_inputs = {}
       step_inputs.each do |k,v|
         if Symbol === v
           input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || v
+          resolved_inputs[k] = if input_dep
+                                 input_dep
+                               elsif provided_inputs.include?(v) && self.inputs.collect(&:first).include?(v)
+                                 provided_inputs[v]
+                               elsif step_inputs.include?(v) && self.inputs.collect(&:first).include?(v)
+                                 step_inputs[v]
+                               else
+                                 v
+                               end
         else
           resolved_inputs[k] = v
         end
       end
-      job = workflow.job(task, id, resolved_inputs)
+
+      job = workflow.job(task, step_id, resolved_inputs)
       compute_options = definition_options[:compute] || []
       compute_options = [compute_options] unless Array === compute_options
       compute_options << :canfail if definition_options[:canfail]
@@ -33,26 +44,25 @@ module Task
       compute[job.path] = compute_options if compute_options.any?
 
       job.overriden = false if definition_options[:not_overriden]
+      job.compute = job.compute.nil? ? compute : job.compute.merge(compute)
 
       [job, step_inputs]
     end
 
     # Helper function
-    find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
+    filter_dep_non_default_inputs = proc do |dep,definition_options|
       dep_non_default_inputs = dep.non_default_inputs
-      dep_non_default_inputs.select do |name|
-        step_inputs.include?(name)  
-      end
       dep_non_default_inputs.reject! do |name|
-        definition_options.include?(name) && 
-          (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
+        definition_options.include?(name)
       end
 
       dep_non_default_inputs
     end
 
     deps.each do |workflow,task,definition_options,block=nil|
-      definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
+      definition_options = definition_options.nil? ? {} : definition_options.dup
+
+      dep_id = definition_options.include?(:jobname) ? definition_options.delete(:jobname) : id
 
       if provided_inputs.include?(overriden = [workflow.name, task] * "#")
         dep = provided_inputs[overriden]
@@ -66,51 +76,50 @@ module Task
         next
       end
 
-      definition_options ||= {}
 
       if block
-        fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
-        self.inputs.each do |name,type,desc,value|
+        fixed_provided_inputs = self.assign_inputs(provided_inputs, dep_id).first.to_hash
+        self.inputs.each do |name,type,desc,value,options|
           fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
         end
         fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
         block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
 
-        res = block.call id, block_options, dependencies
+        res = block.call dep_id, block_options, dependencies
 
         case res
         when Step
           dep = res
           dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
+          dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
           non_default_inputs.concat(dep_non_default_inputs)
         when Hash
           step_options = block_options.merge(res)
-          dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+          dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options.dup, block_options, dependencies)
           dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+          dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
           non_default_inputs.concat(dep_non_default_inputs)
         when Array
           res.each do |_res|
             if Hash === _res
               step_options = block_options.merge(_res)
-              dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
+              dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options.dup, block_options, dependencies)
               dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+              dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
               non_default_inputs.concat(dep_non_default_inputs)
             else
               dep = _res
               dependencies << dep
-              dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
+              dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
               non_default_inputs.concat(dep_non_default_inputs)
             end
           end
         end
       else
         step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
-        dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
+        dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options, definition_options, dependencies)
         dependencies << dep
-        dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
+        dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
         non_default_inputs.concat(dep_non_default_inputs)
       end
     end

data/lib/scout/workflow/task/inputs.rb

@@ -3,7 +3,7 @@ module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value
 
-    if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+    if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
       if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else
@@ -35,11 +35,13 @@ module Task
     input_array = []
     input_names = []
     non_default_inputs = []
+    jobname_input = nil
     self.inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
       input_names << name
       provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
       provided = Task.format_input(provided, type, options || {})
+
       if provided == value
         same_as_default = true
       elsif String === provided && Symbol === value && provided == value.to_s
@@ -49,25 +51,39 @@ module Task
       else
         same_as_default = false
       end
-      if ! provided.nil? && ! same_as_default
-        non_default_inputs << name.to_sym
-        input_array << provided
-      elsif options && options[:jobname]
-        input_array << id
-      else
-        input_array << value
+
+      if options && options[:jobname] && id == provided
+        same_as_jobname = true
       end
+
+      jobname_input = name if same_as_jobname
+
+      final = if ! provided.nil? && ! same_as_default && ! same_as_jobname
+                non_default_inputs << name.to_sym
+                provided
+              elsif options && options[:jobname] && id
+                non_default_inputs << name.to_sym
+                provided || id
+              else
+                value
+              end
+
+      final = Path.setup(final.dup) if String === final && ! (Path === final) && (type == :file || type == :path || (options && options[:asfile]))
+
+      final = final.find if (Path === final) && (type == :file)
+
+      input_array << final
     end
 
     NamedArray.setup(input_array, input_names)
 
-    [input_array, non_default_inputs]
+    [input_array, non_default_inputs, jobname_input]
   end
 
   def process_inputs(provided_inputs = {}, id = nil)
-    input_array, non_default_inputs = assign_inputs provided_inputs, id
+    input_array, non_default_inputs, jobname_input = assign_inputs provided_inputs, id
     digest_str = Misc.digest_str(input_array)
-    [input_array, non_default_inputs, digest_str]
+    [input_array, non_default_inputs, digest_str, jobname_input]
   end
 
   def self.save_file_input(orig_file, directory)
@@ -139,7 +155,7 @@ module Task
       elsif filename.end_with?('.as_path')
         value = Open.read(filename).strip
         Path.setup value
-      elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+      elsif (options &&  (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
         filename
       else
         Persist.load(filename, type)
@@ -150,12 +166,24 @@ module Task
   end
 
   def load_inputs(directory)
+    if Open.exists?(directory) && ! Open.directory?(directory)
+      TmpFile.with_file do |tmp_directory|
+        Misc.in_dir tmp_directory do
+          CMD.cmd("tar xvfz '#{directory}'")
+        end
+        return load_inputs(tmp_directory)
+      end
+    end
+
     inputs = IndiferentHash.setup({})
+    seen = []
     self.recursive_inputs.each do |p|
       name, type, desc, value, options = p
+      next if seen.include?(name)
       filename = File.join(directory, name.to_s) 
       value = Task.load_input_from_file(filename, type, options)
       inputs[name] = value unless value.nil?
+      seen << name
     end
 
     Dir.glob(File.join(directory, "*#*")).each do |file|

data/lib/scout/workflow/task.rb

@@ -31,19 +31,29 @@ module Task
   end
 
   def job(id = nil, provided_inputs = nil)
-    Persist.memory("Task job", other_options: {task: self, id: id, provided_inputs: provided_inputs}) do 
+
+    if Hash === provided_inputs
+      memory_inputs = provided_inputs.values_at *self.recursive_inputs.collect{|t| t.first }.uniq
+      memory_inputs += provided_inputs.select{|k,v| k.to_s.include?("#") }.collect{|p| p * "=" }
+      memory_inputs << provided_inputs[:load_inputs]
+    else
+      memory_inputs = provided_inputs
+    end
+
+    Persist.memory("Task job #{self.name} #{id}", other_options: {task: self, id: id, provided_inputs: memory_inputs}) do 
       provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
       provided_inputs = {} if provided_inputs.nil?
       IndiferentHash.setup(provided_inputs)
 
-      if id.nil?
-        inputs.each do |name,type,desc,default,input_options|
-          next unless input_options && input_options[:jobname]
-          id = provided_inputs[name] || default
-        end
-        id = DEFAULT_NAME if id.nil?
+      jobname_input = nil
+      inputs.each do |name,type,desc,default,input_options|
+        next unless input_options && input_options[:jobname]
+        jobname_input = name
       end
 
+      id = provided_inputs[jobname_input] if jobname_input && id.nil?
+      #id = provided_inputs[:id] if provided_inputs.include?(:id)
+
       missing_inputs = []
       self.inputs.each do |input,type,desc,val,options|
         next unless options && options[:required]
@@ -65,11 +75,13 @@ module Task
       compute = {}
       dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
-      #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
-
       non_default_inputs.uniq!
 
-      if non_default_inputs.any?
+      non_default_inputs.delete_if{|k| k.to_s.include? "#" } unless dependencies.select{|d| d.overriden? }.any?
+
+      id = DEFAULT_NAME if id.nil?
+
+      if non_default_inputs.any? && !(non_default_inputs == [jobname_input] && provided_inputs[jobname_input] == id)
         hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
         name = [id, hash] * "_"
       else

data/lib/scout/workflow/usage.rb

@@ -162,7 +162,7 @@ module Workflow
 
   def _prov_tasks(tree)
     tasks = [] 
-    heap = [tree]
+    heap = tree.values
     while heap.any?
       t = heap.pop
       t.each do |k,v|
@@ -271,7 +271,7 @@ module Workflow
         description = description.split("\n\n").first
 
         next if abridge && ! final.include?(name)
-        str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, :yellow)
+        str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, color: :yellow)
 
         prov_string = prov_string(dep_tree(name))
         str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?

data/lib/scout/workflow.rb

@@ -155,7 +155,7 @@ module Workflow
 
   def job(name, *args)
     task = tasks[name]
-    raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
+    raise TaskNotFound, "Task #{name} in #{self.to_s}" if task.nil?
     step = task.job(*args)
     step.extend step_module
     step

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.7.1 ruby lib
+# stub: scout-gear 10.7.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.7.1".freeze
+  s.version = "10.7.2".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2024-07-01"
+  s.date = "2024-12-10"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -41,6 +41,7 @@ Gem::Specification.new do |s|
     "lib/scout/entity.rb",
     "lib/scout/entity/format.rb",
     "lib/scout/entity/identifiers.rb",
+    "lib/scout/entity/named_array.rb",
     "lib/scout/entity/object.rb",
     "lib/scout/entity/property.rb",
     "lib/scout/offsite.rb",
@@ -121,6 +122,9 @@ Gem::Specification.new do |s|
     "scout_commands/alias",
     "scout_commands/batch/clean",
     "scout_commands/batch/list",
+    "scout_commands/db/query",
+    "scout_commands/db/register",
+    "scout_commands/db/show",
     "scout_commands/doc",
     "scout_commands/find",
     "scout_commands/glob",
@@ -151,6 +155,7 @@ Gem::Specification.new do |s|
     "test/scout/association/test_item.rb",
     "test/scout/entity/test_format.rb",
     "test/scout/entity/test_identifiers.rb",
+    "test/scout/entity/test_named_array.rb",
     "test/scout/entity/test_object.rb",
     "test/scout/entity/test_property.rb",
     "test/scout/offsite/test_ssh.rb",
@@ -188,6 +193,7 @@ Gem::Specification.new do |s|
     "test/scout/tsv/test_index.rb",
     "test/scout/tsv/test_open.rb",
     "test/scout/tsv/test_parser.rb",
+    "test/scout/tsv/test_path.rb",
     "test/scout/tsv/test_stream.rb",
     "test/scout/tsv/test_transformer.rb",
     "test/scout/tsv/test_traverse.rb",
@@ -225,7 +231,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.5.10".freeze
+  s.rubygems_version = "3.5.23".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/db/query

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/association'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Query a database
+
+$ #{$0} [<options>] <name> <entity>
+
+-h--help Print this help
+-l--list Only list matches
+-s--source* Source description
+-t--target* Target description
+-n--namespace* Namespace
+-i--identifiers* Identifiers
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name, entity = ARGV
+
+raise MissingParameterException, :name if name.nil?
+raise MissingParameterException, :entity if entity.nil?
+
+registry = begin
+             Scout.var.databases.registry.yaml
+           rescue
+             {}
+           end
+
+list = IndiferentHash.process_options options, :list
+
+raise ParameterException "Database #{name} not found Options: #{Log.fingerprint registry.keys}" unless registry.include? name
+
+file, db_options = registry[name]
+options = db_options.merge(options)
+
+IndiferentHash.setup(options)
+options.keys_to_sym!
+
+index = Association.index(file, **options)
+
+if entity.end_with?("~")
+  matches = index.match(entity[0..-2])
+elsif entity.start_with?("~")
+  index = index.reverse
+  matches = index.match(entity[1..-1])
+  reverse = true
+elsif entity.include?("~")
+  matches = [entity]
+else
+  matches = index.match(entity)
+end
+
+AssociationItem.setup(matches) if reverse
+
+if matches.any?
+  if list
+    if reverse
+      puts matches.collect(&:invert) * "\n"
+    else
+      puts matches * "\n"
+    end
+  else
+    matches.each do |match|
+      puts Log.color :magenta, (reverse ? match.invert : match)
+      puts index[match].prety_print 
+    end
+  end
+else
+  STDERR.puts "Query #{entity} return no results"
+end
+

data/scout_commands/db/register

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Register a database
+
+$ #{$0} [<options>] <name> <filename>
+
+-h--help Print this help
+-s--source* Source description
+-t--target* Target description
+-n--namespace* Namespace
+-i--identifiers* Identifiers
+EOF
+if options.delete :help
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name, file = ARGV
+
+raise MissingParameterException, :name if name.nil?
+raise MissingParameterException, :file if file.nil?
+
+registry = begin
+             Scout.var.databases.registry.yaml
+           rescue
+             {}
+           end
+
+file = Scout.identify(File.expand_path(file))
+registry[name] = [Scout.identify(file), options]
+
+Scout.var.databases.registry.write(registry.to_yaml)
+

data/scout_commands/db/show

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/association'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Show a database
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+registry = begin
+             Scout.var.databases.registry.yaml
+           rescue
+             {}
+           end
+
+name = ARGV.first
+
+if name.nil?
+  puts registry.keys * "\n"
+
+else
+
+  raise ParameterException "Database #{name} not found Options: #{Log.fingerprint registry.keys}" unless registry.include? name
+
+  file, options = registry[name]
+
+  IndiferentHash.setup(options)
+  options.keys_to_sym!
+
+  Log.tsv Association.database(file, **options)
+end

data/test/scout/entity/test_named_array.rb

@@ -0,0 +1,21 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestNamedArrayEntity < Test::Unit::TestCase
+  def setup
+    m = Module.new do
+      extend Entity
+      self.format = "SomeEntity"
+
+      property :prop do
+        "PROP: #{self}"
+      end
+    end
+  end
+
+  def test_true
+    a = NamedArray.setup(["a", "b"], %w(SomeEntity Other))
+    assert a["SomeEntity"].respond_to?(:prop)
+  end
+end
+

data/test/scout/persist/test_tsv.rb

@@ -40,6 +40,7 @@ row2    a    a    id3
       end
     end
 
+    assert_include tsv.fields, "ValueA"
     assert_include tsv.keys, 'row1'
     assert_include tsv.keys, 'row2'
   end
@@ -67,6 +68,7 @@ row2    a    a    id3
       end
       refute Open.exists?(tk)
       assert Open.exists?(data.persistence_path)
+      refute tsv.fields.nil?
     end
 
     assert_include tsv.keys, 'row1'
@@ -142,5 +144,23 @@ row2    a    a    id3
     assert_include tsv.keys, 'row1'
     assert_include tsv.keys, 'row2'
   end
+
+  def test_tsv_open_persist
+    content =<<-'EOF'
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+
+    tsv = nil
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, sep: /\s+/, type: :double, persist: true, merge: true)
+      assert Array === tsv.fields
+      tsv = TSV.open(filename, sep: /\s+/, type: :double, persist: true, merge: true)
+      assert Array === tsv.fields
+    end
+  end
 end
 

data/test/scout/persist/tsv/adapter/test_base.rb

@@ -39,5 +39,25 @@ class TestTSVAdapter < Test::Unit::TestCase
     assert_equal [1, 2, 3], tsv["a"]
     assert_equal [1, 2, 3], Marshal.load(tsv.orig_get("a"))
   end
+
+  def test_persist_annotations
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    TmpFile.with_file(content) do |tsv_file|
+      pid = Process.fork do
+        tsv = TSV.open tsv_file, persist: true
+      end
+      Process.waitpid pid
+      tsv = TSV.open tsv_file, persist: true
+      refute tsv.fields.nil?
+    end
+  end
+
 end