scout-gear 10.7.1 → 10.7.3

data/lib/scout/workflow/task/inputs.rb

@@ -3,7 +3,7 @@ module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value
 
-    if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+    if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
       if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else
@@ -35,11 +35,13 @@ module Task
     input_array = []
     input_names = []
     non_default_inputs = []
+    jobname_input = nil
     self.inputs.each_with_index do |p,i|
       name, type, desc, value, options = p
       input_names << name
       provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
       provided = Task.format_input(provided, type, options || {})
+
       if provided == value
         same_as_default = true
       elsif String === provided && Symbol === value && provided == value.to_s
@@ -49,25 +51,39 @@ module Task
       else
         same_as_default = false
       end
-      if ! provided.nil? && ! same_as_default
-        non_default_inputs << name.to_sym
-        input_array << provided
-      elsif options && options[:jobname]
-        input_array << id
-      else
-        input_array << value
+
+      if options && options[:jobname] && id == provided
+        same_as_jobname = true
       end
+
+      jobname_input = name if same_as_jobname
+
+      final = if ! provided.nil? && ! same_as_default && ! same_as_jobname
+                non_default_inputs << name.to_sym
+                provided
+              elsif options && options[:jobname] && id
+                non_default_inputs << name.to_sym
+                provided || id
+              else
+                value
+              end
+
+      final = Path.setup(final.dup) if String === final && ! (Path === final) && (type == :file || type == :path || (options && options[:asfile]))
+
+      final = final.find if (Path === final) && (type == :file)
+
+      input_array << final
     end
 
     NamedArray.setup(input_array, input_names)
 
-    [input_array, non_default_inputs]
+    [input_array, non_default_inputs, jobname_input]
   end
 
   def process_inputs(provided_inputs = {}, id = nil)
-    input_array, non_default_inputs = assign_inputs provided_inputs, id
+    input_array, non_default_inputs, jobname_input = assign_inputs provided_inputs, id
     digest_str = Misc.digest_str(input_array)
-    [input_array, non_default_inputs, digest_str]
+    [input_array, non_default_inputs, digest_str, jobname_input]
   end
 
   def self.save_file_input(orig_file, directory)
@@ -139,7 +155,7 @@ module Task
       elsif filename.end_with?('.as_path')
         value = Open.read(filename).strip
         Path.setup value
-      elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+      elsif (options &&  (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
         filename
       else
         Persist.load(filename, type)
@@ -150,12 +166,24 @@ module Task
   end
 
   def load_inputs(directory)
+    if Open.exists?(directory) && ! Open.directory?(directory)
+      TmpFile.with_file do |tmp_directory|
+        Misc.in_dir tmp_directory do
+          CMD.cmd("tar xvfz '#{directory}'")
+        end
+        return load_inputs(tmp_directory)
+      end
+    end
+
     inputs = IndiferentHash.setup({})
+    seen = []
     self.recursive_inputs.each do |p|
       name, type, desc, value, options = p
+      next if seen.include?(name)
       filename = File.join(directory, name.to_s) 
       value = Task.load_input_from_file(filename, type, options)
       inputs[name] = value unless value.nil?
+      seen << name
     end
 
     Dir.glob(File.join(directory, "*#*")).each do |file|

data/lib/scout/workflow/task.rb

@@ -31,19 +31,29 @@ module Task
   end
 
   def job(id = nil, provided_inputs = nil)
-    Persist.memory("Task job", other_options: {task: self, id: id, provided_inputs: provided_inputs}) do 
+
+    if Hash === provided_inputs
+      memory_inputs = provided_inputs.values_at *self.recursive_inputs.collect{|t| t.first }.uniq
+      memory_inputs += provided_inputs.select{|k,v| k.to_s.include?("#") }.collect{|p| p * "=" }
+      memory_inputs << provided_inputs[:load_inputs]
+    else
+      memory_inputs = provided_inputs
+    end
+
+    Persist.memory("Task job #{self.name} #{id}", other_options: {task: self, id: id, provided_inputs: memory_inputs}) do 
       provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
       provided_inputs = {} if provided_inputs.nil?
       IndiferentHash.setup(provided_inputs)
 
-      if id.nil?
-        inputs.each do |name,type,desc,default,input_options|
-          next unless input_options && input_options[:jobname]
-          id = provided_inputs[name] || default
-        end
-        id = DEFAULT_NAME if id.nil?
+      jobname_input = nil
+      inputs.each do |name,type,desc,default,input_options|
+        next unless input_options && input_options[:jobname]
+        jobname_input = name
       end
 
+      id = provided_inputs[jobname_input] if jobname_input && id.nil?
+      #id = provided_inputs[:id] if provided_inputs.include?(:id)
+
       missing_inputs = []
       self.inputs.each do |input,type,desc,val,options|
         next unless options && options[:required]
@@ -65,11 +75,13 @@ module Task
       compute = {}
       dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
-      #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
-
       non_default_inputs.uniq!
 
-      if non_default_inputs.any?
+      non_default_inputs.delete_if{|k| k.to_s.include? "#" } unless dependencies.select{|d| d.overriden? }.any?
+
+      id = DEFAULT_NAME if id.nil?
+
+      if non_default_inputs.any? && !(non_default_inputs == [jobname_input] && provided_inputs[jobname_input] == id)
         hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
         name = [id, hash] * "_"
       else

data/lib/scout/workflow/usage.rb

@@ -162,7 +162,7 @@ module Workflow
 
   def _prov_tasks(tree)
     tasks = [] 
-    heap = [tree]
+    heap = tree.values
     while heap.any?
       t = heap.pop
       t.each do |k,v|
@@ -271,7 +271,7 @@ module Workflow
         description = description.split("\n\n").first
 
         next if abridge && ! final.include?(name)
-        str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, :yellow)
+        str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, color: :yellow)
 
         prov_string = prov_string(dep_tree(name))
         str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?

data/lib/scout/workflow.rb

@@ -155,7 +155,7 @@ module Workflow
 
   def job(name, *args)
     task = tasks[name]
-    raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
+    raise TaskNotFound, "Task #{name} in #{self.to_s}" if task.nil?
     step = task.job(*args)
     step.extend step_module
     step

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.7.1 ruby lib
+# stub: scout-gear 10.7.3 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.7.1".freeze
+  s.version = "10.7.3".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2024-07-01"
+  s.date = "2025-01-17"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -41,8 +41,16 @@ Gem::Specification.new do |s|
     "lib/scout/entity.rb",
     "lib/scout/entity/format.rb",
     "lib/scout/entity/identifiers.rb",
+    "lib/scout/entity/named_array.rb",
     "lib/scout/entity/object.rb",
     "lib/scout/entity/property.rb",
+    "lib/scout/knowledge_base.rb",
+    "lib/scout/knowledge_base/enrichment.rb",
+    "lib/scout/knowledge_base/entity.rb",
+    "lib/scout/knowledge_base/list.rb",
+    "lib/scout/knowledge_base/query.rb",
+    "lib/scout/knowledge_base/registry.rb",
+    "lib/scout/knowledge_base/traverse.rb",
     "lib/scout/offsite.rb",
     "lib/scout/offsite/exceptions.rb",
     "lib/scout/offsite/ssh.rb",
@@ -124,6 +132,13 @@ Gem::Specification.new do |s|
     "scout_commands/doc",
     "scout_commands/find",
     "scout_commands/glob",
+    "scout_commands/kb/config",
+    "scout_commands/kb/entities",
+    "scout_commands/kb/list",
+    "scout_commands/kb/query",
+    "scout_commands/kb/register",
+    "scout_commands/kb/show",
+    "scout_commands/kb/traverse",
     "scout_commands/log",
     "scout_commands/offsite",
     "scout_commands/rbbt",
@@ -151,8 +166,15 @@ Gem::Specification.new do |s|
     "test/scout/association/test_item.rb",
     "test/scout/entity/test_format.rb",
     "test/scout/entity/test_identifiers.rb",
+    "test/scout/entity/test_named_array.rb",
     "test/scout/entity/test_object.rb",
     "test/scout/entity/test_property.rb",
+    "test/scout/knowledge_base/test_enrichment.rb",
+    "test/scout/knowledge_base/test_entity.rb",
+    "test/scout/knowledge_base/test_list.rb",
+    "test/scout/knowledge_base/test_query.rb",
+    "test/scout/knowledge_base/test_registry.rb",
+    "test/scout/knowledge_base/test_traverse.rb",
     "test/scout/offsite/test_ssh.rb",
     "test/scout/offsite/test_step.rb",
     "test/scout/offsite/test_sync.rb",
@@ -173,6 +195,7 @@ Gem::Specification.new do |s|
     "test/scout/persist/tsv/test_serialize.rb",
     "test/scout/test_association.rb",
     "test/scout/test_entity.rb",
+    "test/scout/test_knowledge_base.rb",
     "test/scout/test_offsite.rb",
     "test/scout/test_semaphore.rb",
     "test/scout/test_tsv.rb",
@@ -188,6 +211,7 @@ Gem::Specification.new do |s|
     "test/scout/tsv/test_index.rb",
     "test/scout/tsv/test_open.rb",
     "test/scout/tsv/test_parser.rb",
+    "test/scout/tsv/test_path.rb",
     "test/scout/tsv/test_stream.rb",
     "test/scout/tsv/test_transformer.rb",
     "test/scout/tsv/test_traverse.rb",
@@ -225,7 +249,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.5.10".freeze
+  s.rubygems_version = "3.5.23".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/kb/config

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Configure a knowlege base
+
+$ #{$0} [<options>] <name> [knowledge_base]
+
+-h--help Print this help
+-i--identifier_files* Identifier files separated by ','
+-n--namespace* Namespace
+EOF
+if options.delete :help
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+knowledge_base_name = ARGV.first
+
+knowledge_base = KnowledgeBase.load knowledge_base_name || :default
+
+knowledge_base.namespace = options[:namespace] if options.include? :namespace
+knowledge_base.identifier_files += options[:identifier_files].split(",") if options.include? :identifier_files
+knowledge_base.save

data/scout_commands/kb/entities

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Declare a type of entity and assing an identifiers file
+
+$ #{$0} [<options>] <entity> <identifier_files>
+
+-h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
+EOF
+if options.delete :help
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+entity, identifier_files = ARGV
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+
+knowledge_base.entity_options[entity] ||= {}
+knowledge_base.entity_options[entity][:identifiers] ||= []
+knowledge_base.entity_options[entity][:identifiers] = [knowledge_base.entity_options[entity][:identifiers]] unless Array === knowledge_base.entity_options[entity][:identifiers]
+knowledge_base.entity_options[entity][:identifiers] += identifier_files.split(",")
+knowledge_base.entity_options[entity][:identifiers].uniq!
+knowledge_base.save

data/scout_commands/kb/list

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <filename> [<other|->]*
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+knowledge_base.define_entity_modules
+
+list_name = ARGV.first
+
+if list_name.nil?
+  knowledge_base.lists.each do |type,lists|
+    puts Log.color :title, type
+    puts lists * "\n"
+  end
+else
+  list = knowledge_base.load_list(list_name)
+
+  puts list * "\n"
+end

data/scout_commands/kb/query

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Query a database
+
+$ #{$0} [<options>] <name> <entity>
+
+-h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
+-l--list Only list matches
+-s--source* Source description
+-t--target* Target description
+-n--namespace* Namespace
+-i--identifiers* Identifiers
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name, entity = ARGV
+
+raise MissingParameterException, :name if name.nil?
+raise MissingParameterException, :entity if entity.nil?
+
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+
+list = IndiferentHash.process_options options, :list
+
+raise ParameterException "Database #{name} not found Options: #{Log.fingerprint knowledge_base.all_databases.keys}" unless knowledge_base.include? name
+
+IndiferentHash.setup(options)
+options.keys_to_sym!
+
+index = knowledge_base.get_index(name, options)
+
+if entity.end_with?("~")
+  matches = index.match(entity[0..-2])
+elsif entity.start_with?("~")
+  index = index.reverse
+  matches = index.match(entity[1..-1])
+  reverse = true
+elsif entity.include?("~")
+  matches = [entity]
+else
+  matches = index.match(entity)
+end
+
+AssociationItem.setup(matches) if reverse
+
+if matches.any?
+  if list
+    if reverse
+      puts matches.collect(&:invert) * "\n"
+    else
+      puts matches * "\n"
+    end
+  else
+    matches.each do |match|
+      puts Log.color :magenta, (reverse ? match.invert : match)
+      puts index[match].prety_print 
+    end
+  end
+else
+  STDERR.puts "Query #{entity} return no results"
+end
+

data/scout_commands/kb/register

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Register a database
+
+$ #{$0} [<options>] <name> <filename>
+
+-h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
+-s--source* Source description
+-t--target* Target description
+-f--fields* Fields to consider
+-n--namespace* Namespace
+-i--identifiers* Identifiers
+-u--undirected
+EOF
+if options.delete :help
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name, file = ARGV
+
+
+raise MissingParameterException, :name if name.nil?
+raise MissingParameterException, :file if file.nil?
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+
+options[:fields] = options[:fields].split(/,\s*/) if options[:fields]
+file = Scout.identify(File.expand_path(file))
+knowledge_base.register name, Scout.identify(file), options
+
+knowledge_base.save

data/scout_commands/kb/show

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Show a database
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+
+name = ARGV.first
+
+if name.nil?
+  puts knowledge_base.all_databases * "\n"
+else
+
+  raise ParameterException "Database #{name} not found Options: #{Log.fingerprint knowledge_base.all_databases}" unless knowledge_base.include? name
+
+  Log.tsv knowledge_base.get_database(name, options)
+end

data/scout_commands/kb/traverse

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Traverse databases
+
+$ #{$0} [<options>] <traversal>
+
+-h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
+-p--paths Only list paths
+-e--entities Only list entities
+-l--list* List entities matching capture
+-ln--list_name* Save list of matches with a name
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+traversal = ARGV.first
+
+raise MissingParameterException, :traversal if traversal.nil?
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+
+only_paths, only_entities, list, list_name = IndiferentHash.process_options options, :paths, :entities, :list, :list_name
+
+rules = traversal.split(/,\s*/)
+
+knowledge_base.define_entity_modules
+
+entities, paths = knowledge_base.traverse(rules, only_entities || list)
+
+if list
+  entity_list = entities["?" + list] || []
+  puts entity_list * "\n"
+  knowledge_base.save_list list_name, entity_list if list_name
+else
+  paths = nil if only_entities && ! only_paths
+  entities = nil if only_paths && ! only_entities
+
+  if entities
+    entities.each do |type,list|
+      puts Log.color(:magenta, type[1..-1] + ": ") + Log.fingerprint(list * ", ")
+    end
+  end
+
+  if paths
+    paths.each do |path_list|
+      path_list.each do |match|
+        puts Log.color :magenta, (match.reverse ? match.invert : match)
+        puts IndiferentHash.setup(match.info).prety_print if match.info && match.info.any?
+      end
+    end
+  end
+end

data/test/data/person/brothers

@@ -1,4 +1,4 @@
 #: :sep=,#:type=:list
-#Older,Younger
+#Older (Alias),Younger (Alias)
 Clei,Guille
 Isa,Miki

data/test/scout/entity/test_identifiers.rb

@@ -16,9 +16,9 @@ class TestEntityIdentifiers < Test::Unit::TestCase
 
   Person.add_identifiers datafile_test(Entity::Identified::NAMESPACE_TAG + '/identifiers'), "Name", "Alias"
 
-  teardown do
-    Entity.formats.clear
-  end
+  #teardown do
+  #  Entity.formats.clear
+  #end
 
   def test_alias
     miguel = Person.setup("Miguel", namespace: :person)

data/test/scout/entity/test_named_array.rb

@@ -0,0 +1,21 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestNamedArrayEntity < Test::Unit::TestCase
+  def setup
+    m = Module.new do
+      extend Entity
+      self.format = "SomeEntity"
+
+      property :prop do
+        "PROP: #{self}"
+      end
+    end
+  end
+
+  def test_true
+    a = NamedArray.setup(["a", "b"], %w(SomeEntity Other))
+    assert a["SomeEntity"].respond_to?(:prop)
+  end
+end
+