scout-gear 10.7.2 → 10.7.4

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.7.2 ruby lib
+# stub: scout-gear 10.7.4 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.7.2".freeze
+  s.version = "10.7.4".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2024-12-10"
+  s.date = "2025-03-10"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -44,6 +44,13 @@ Gem::Specification.new do |s|
     "lib/scout/entity/named_array.rb",
     "lib/scout/entity/object.rb",
     "lib/scout/entity/property.rb",
+    "lib/scout/knowledge_base.rb",
+    "lib/scout/knowledge_base/enrichment.rb",
+    "lib/scout/knowledge_base/entity.rb",
+    "lib/scout/knowledge_base/list.rb",
+    "lib/scout/knowledge_base/query.rb",
+    "lib/scout/knowledge_base/registry.rb",
+    "lib/scout/knowledge_base/traverse.rb",
     "lib/scout/offsite.rb",
     "lib/scout/offsite/exceptions.rb",
     "lib/scout/offsite/ssh.rb",
@@ -114,6 +121,7 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/step/status.rb",
     "lib/scout/workflow/task.rb",
     "lib/scout/workflow/task/dependencies.rb",
+    "lib/scout/workflow/task/info.rb",
     "lib/scout/workflow/task/inputs.rb",
     "lib/scout/workflow/usage.rb",
     "lib/scout/workflow/util.rb",
@@ -122,12 +130,16 @@ Gem::Specification.new do |s|
     "scout_commands/alias",
     "scout_commands/batch/clean",
     "scout_commands/batch/list",
-    "scout_commands/db/query",
-    "scout_commands/db/register",
-    "scout_commands/db/show",
     "scout_commands/doc",
     "scout_commands/find",
     "scout_commands/glob",
+    "scout_commands/kb/config",
+    "scout_commands/kb/entities",
+    "scout_commands/kb/list",
+    "scout_commands/kb/query",
+    "scout_commands/kb/register",
+    "scout_commands/kb/show",
+    "scout_commands/kb/traverse",
     "scout_commands/log",
     "scout_commands/offsite",
     "scout_commands/rbbt",
@@ -158,6 +170,12 @@ Gem::Specification.new do |s|
     "test/scout/entity/test_named_array.rb",
     "test/scout/entity/test_object.rb",
     "test/scout/entity/test_property.rb",
+    "test/scout/knowledge_base/test_enrichment.rb",
+    "test/scout/knowledge_base/test_entity.rb",
+    "test/scout/knowledge_base/test_list.rb",
+    "test/scout/knowledge_base/test_query.rb",
+    "test/scout/knowledge_base/test_registry.rb",
+    "test/scout/knowledge_base/test_traverse.rb",
     "test/scout/offsite/test_ssh.rb",
     "test/scout/offsite/test_step.rb",
     "test/scout/offsite/test_sync.rb",
@@ -178,6 +196,7 @@ Gem::Specification.new do |s|
     "test/scout/persist/tsv/test_serialize.rb",
     "test/scout/test_association.rb",
     "test/scout/test_entity.rb",
+    "test/scout/test_knowledge_base.rb",
     "test/scout/test_offsite.rb",
     "test/scout/test_semaphore.rb",
     "test/scout/test_tsv.rb",
@@ -217,6 +236,7 @@ Gem::Specification.new do |s|
     "test/scout/workflow/step/test_provenance.rb",
     "test/scout/workflow/step/test_status.rb",
     "test/scout/workflow/task/test_dependencies.rb",
+    "test/scout/workflow/task/test_info.rb",
     "test/scout/workflow/task/test_inputs.rb",
     "test/scout/workflow/test_definition.rb",
     "test/scout/workflow/test_documentation.rb",
@@ -231,7 +251,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.5.23".freeze
+  s.rubygems_version = "3.6.5".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/doc

@@ -26,12 +26,12 @@ end
 
 module_name = ARGV.first
 if module_name.nil?
-  puts Rbbt.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
+  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
 else
   begin
-    puts Rbbt.doc.lib.scout[module_name].find_with_extension('md', :lib).read
+    puts Scout.doc.lib.scout[module_name].find_with_extension('md', :lib).read
   rescue
-    puts Rbbt.doc.lib.scout.glob("**/*.md").select{|f| File.basename(f.unset_extension) == module_name }.first.read
+    puts Scout.doc.lib.scout.glob("**/*.md").select{|f| File.basename(f.unset_extension) == module_name }.first.read
   end
 end
 

data/scout_commands/kb/config

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Configure a knowlege base
+
+$ #{$0} [<options>] <name> [knowledge_base]
+
+-h--help Print this help
+-i--identifier_files* Identifier files separated by ','
+-n--namespace* Namespace
+EOF
+if options.delete :help
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+knowledge_base_name = ARGV.first
+
+knowledge_base = KnowledgeBase.load knowledge_base_name || :default
+
+knowledge_base.namespace = options[:namespace] if options.include? :namespace
+knowledge_base.identifier_files += options[:identifier_files].split(",") if options.include? :identifier_files
+knowledge_base.save

data/scout_commands/kb/entities

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Declare a type of entity and assing an identifiers file
+
+$ #{$0} [<options>] <entity> <identifier_files>
+
+-h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
+EOF
+if options.delete :help
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+entity, identifier_files = ARGV
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+
+knowledge_base.entity_options[entity] ||= {}
+knowledge_base.entity_options[entity][:identifiers] ||= []
+knowledge_base.entity_options[entity][:identifiers] = [knowledge_base.entity_options[entity][:identifiers]] unless Array === knowledge_base.entity_options[entity][:identifiers]
+knowledge_base.entity_options[entity][:identifiers] += identifier_files.split(",")
+knowledge_base.entity_options[entity][:identifiers].uniq!
+knowledge_base.save

data/scout_commands/kb/list

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <filename> [<other|->]*
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+knowledge_base.define_entity_modules
+
+list_name = ARGV.first
+
+if list_name.nil?
+  knowledge_base.lists.each do |type,lists|
+    puts Log.color :title, type
+    puts lists * "\n"
+  end
+else
+  list = knowledge_base.load_list(list_name)
+
+  puts list * "\n"
+end

data/scout_commands/{db → kb}/query

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'scout'
-require 'scout/association'
+require 'scout/knowledge_base'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
@@ -12,6 +12,7 @@ Query a database
 $ #{$0} [<options>] <name> <entity>
 
 -h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
 -l--list Only list matches
 -s--source* Source description
 -t--target* Target description
@@ -32,23 +33,17 @@ name, entity = ARGV
 raise MissingParameterException, :name if name.nil?
 raise MissingParameterException, :entity if entity.nil?
 
-registry = begin
-             Scout.var.databases.registry.yaml
-           rescue
-             {}
-           end
 
-list = IndiferentHash.process_options options, :list
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
 
-raise ParameterException "Database #{name} not found Options: #{Log.fingerprint registry.keys}" unless registry.include? name
+list = IndiferentHash.process_options options, :list
 
-file, db_options = registry[name]
-options = db_options.merge(options)
+raise ParameterException "Database #{name} not found Options: #{Log.fingerprint knowledge_base.all_databases.keys}" unless knowledge_base.include? name
 
 IndiferentHash.setup(options)
 options.keys_to_sym!
 
-index = Association.index(file, **options)
+index = knowledge_base.get_index(name, options)
 
 if entity.end_with?("~")
   matches = index.match(entity[0..-2])

data/scout_commands/{db → kb}/register

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'scout'
+require 'scout/knowledge_base'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
@@ -11,10 +12,13 @@ Register a database
 $ #{$0} [<options>] <name> <filename>
 
 -h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
 -s--source* Source description
 -t--target* Target description
+-f--fields* Fields to consider
 -n--namespace* Namespace
 -i--identifiers* Identifiers
+-u--undirected
 EOF
 if options.delete :help
   if defined? scout_usage
@@ -27,17 +31,14 @@ end
 
 name, file = ARGV
 
+
 raise MissingParameterException, :name if name.nil?
 raise MissingParameterException, :file if file.nil?
 
-registry = begin
-             Scout.var.databases.registry.yaml
-           rescue
-             {}
-           end
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
 
+options[:fields] = options[:fields].split(/,\s*/) if options[:fields]
 file = Scout.identify(File.expand_path(file))
-registry[name] = [Scout.identify(file), options]
-
-Scout.var.databases.registry.write(registry.to_yaml)
+knowledge_base.register name, Scout.identify(file), options
 
+knowledge_base.save

data/scout_commands/{db → kb}/show

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'scout'
-require 'scout/association'
+require 'scout/knowledge_base'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
@@ -12,6 +12,7 @@ Show a database
 $ #{$0} [<options>] <name>
 
 -h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
 EOF
 if options[:help]
   if defined? scout_usage
@@ -22,26 +23,15 @@ if options[:help]
   exit 0
 end
 
-
-registry = begin
-             Scout.var.databases.registry.yaml
-           rescue
-             {}
-           end
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
 
 name = ARGV.first
 
 if name.nil?
-  puts registry.keys * "\n"
-
+  puts knowledge_base.all_databases * "\n"
 else
 
-  raise ParameterException "Database #{name} not found Options: #{Log.fingerprint registry.keys}" unless registry.include? name
-
-  file, options = registry[name]
-
-  IndiferentHash.setup(options)
-  options.keys_to_sym!
+  raise ParameterException "Database #{name} not found Options: #{Log.fingerprint knowledge_base.all_databases}" unless knowledge_base.include? name
 
-  Log.tsv Association.database(file, **options)
+  Log.tsv knowledge_base.get_database(name, options)
 end

data/scout_commands/kb/traverse

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/knowledge_base'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Traverse databases
+
+$ #{$0} [<options>] <traversal>
+
+-h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
+-p--paths Only list paths
+-e--entities Only list entities
+-l--list* List entities matching capture
+-ln--list_name* Save list of matches with a name
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+traversal = ARGV.first
+
+raise MissingParameterException, :traversal if traversal.nil?
+
+knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+
+only_paths, only_entities, list, list_name = IndiferentHash.process_options options, :paths, :entities, :list, :list_name
+
+rules = traversal.split(/,\s*/)
+
+knowledge_base.define_entity_modules
+
+entities, paths = knowledge_base.traverse(rules, only_entities || list)
+
+if list
+  entity_list = entities["?" + list] || []
+  puts entity_list * "\n"
+  knowledge_base.save_list list_name, entity_list if list_name
+else
+  paths = nil if only_entities && ! only_paths
+  entities = nil if only_paths && ! only_entities
+
+  if entities
+    entities.each do |type,list|
+      puts Log.color(:magenta, type[1..-1] + ": ") + Log.fingerprint(list * ", ")
+    end
+  end
+
+  if paths
+    paths.each do |path_list|
+      path_list.each do |match|
+        puts Log.color :magenta, (match.reverse ? match.invert : match)
+        puts IndiferentHash.setup(match.info).prety_print if match.info && match.info.any?
+      end
+    end
+  end
+end

data/scout_commands/workflow/task

@@ -19,6 +19,7 @@ $ #{$0} [<options>] <workflow> <task>
 --nostream Disable job streaming
 --update Update jobs with newer dependencies
 --deploy* Deploy mode: serial, local, or SLURM (default 'serial')
+--fork Fork and return path
 -jn--jobname* Name to use as job identifier
 -li--load_inputs* Directory with inputs files to load
 -pf--printpath Print the file path
@@ -39,8 +40,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, printpath, deploy, override_deps = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :printpath, :deploy, :override_deps,
+help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, printpath, deploy, override_deps, do_fork = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :printpath, :deploy, :override_deps, :fork,
   :deploy => 'serial'
 
 if help
@@ -92,6 +93,10 @@ end
 
 if provenance
   puts Step.prov_report(job)
+elsif do_fork
+  job.fork
+  puts job.path
+  exit 0
 else
   case deploy
   when "serial"

data/test/data/person/brothers

@@ -1,4 +1,4 @@
 #: :sep=,#:type=:list
-#Older,Younger
+#Older (Alias),Younger (Alias)
 Clei,Guille
 Isa,Miki

data/test/scout/entity/test_identifiers.rb

@@ -16,9 +16,9 @@ class TestEntityIdentifiers < Test::Unit::TestCase
 
   Person.add_identifiers datafile_test(Entity::Identified::NAMESPACE_TAG + '/identifiers'), "Name", "Alias"
 
-  teardown do
-    Entity.formats.clear
-  end
+  #teardown do
+  #  Entity.formats.clear
+  #end
 
   def test_alias
     miguel = Person.setup("Miguel", namespace: :person)

data/test/scout/knowledge_base/test_entity.rb

@@ -0,0 +1,38 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/knowledge_base'
+
+
+class TestKnowledgeBaseQuery < Test::Unit::TestCase
+  def test_types
+    TmpFile.with_dir do |dir|
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, datafile_test(:person).brothers, undirected: true
+      kb.register :parents, datafile_test(:person).parents
+
+      assert_include kb.all_databases, :brothers
+
+      assert_equal Person, kb.target_type(:parents)
+    end
+  end
+
+  def test_options
+    TmpFile.with_dir do |dir|
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, datafile_test(:person).brothers, undirected: true
+      kb.entity_options = { "Person" => {language: "es"} }
+
+      assert_include kb.entity_options_for("Person"), :language
+    end
+  end
+
+  def test_identify
+    TmpFile.with_dir do |dir|
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, datafile_test(:person).brothers, undirected: true
+
+      assert_equal "Miki", kb.identify(:brothers, "001")
+    end
+  end
+end

data/test/scout/knowledge_base/test_list.rb

@@ -0,0 +1,40 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/knowledge_base'
+class TestKnowledgeBaseQuery < Test::Unit::TestCase
+  def test_entity_list
+    TmpFile.with_dir do |dir|
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, datafile_test(:person).brothers, undirected: true
+      kb.register :parents, datafile_test(:person).parents
+
+      list = kb.subset(:brothers, :all).target_entity
+
+      kb.save_list("bro_and_sis", list)
+      assert_equal list, kb.load_list("bro_and_sis")
+
+      assert_include kb.lists["Person"], "bro_and_sis"
+      kb.delete_list("bro_and_sis")
+
+      refute kb.lists["simple"]
+    end
+  end
+
+  def test_simple_list
+    list = ["Miki", "Isa"]
+    TmpFile.with_dir do |dir|
+      kb = KnowledgeBase.new dir
+
+      kb.save_list("bro_and_sis", list)
+
+      assert_equal list, kb.load_list("bro_and_sis")
+
+      assert_include kb.lists["simple"], "bro_and_sis"
+
+      kb.delete_list("bro_and_sis")
+
+      refute kb.lists["simple"].any?
+    end
+  end
+end

data/test/scout/knowledge_base/test_query.rb

@@ -0,0 +1,39 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/knowledge_base'
+
+
+class TestKnowledgeBaseQuery < Test::Unit::TestCase
+  def test_query
+    TmpFile.with_dir do |dir|
+      kb = KnowledgeBase.new dir
+      kb.entity_options = {"Person" => {language: "es"}}
+
+      kb.register :brothers, datafile_test(:person).brothers, undirected: true
+      kb.register :parents, datafile_test(:person).parents, entity_options: {"Person" => {language: "en"}}
+
+      assert_include kb.all_databases, :brothers
+
+      matches = kb.subset(:parents, :all)
+      assert_include matches, "Clei~Domingo"
+
+      matches = kb.subset(:parents, target: :all, source: ["Miki"])
+      assert_include matches, "Miki~Juan"
+
+
+      assert_include kb.children(:parents, "Miki").target, "Juan"
+      assert_include kb.children(:brothers, "Miki").target, "Isa"
+
+      parents = matches.target_entity
+
+      assert_include parents, "Juan"
+      assert Person === parents.first
+      assert_equal "en", parents.first.language
+
+
+      matches = kb.subset(:brothers, target: :all, source: ["Miki"])
+      assert_equal "es", matches.first.source_entity.language
+    end
+  end
+end

data/test/scout/knowledge_base/test_registry.rb

@@ -0,0 +1,35 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/knowledge_base'
+
+class TestKnowlegeBase < Test::Unit::TestCase
+  def test_registry
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      assert_include kb.all_databases, :brothers
+    end
+  end
+
+  def test_registry_identifiers
+    identifier =<<-EOF
+#Alias,Initials
+Clei,CC
+Miki,MV
+Guille,GC
+Isa,IV
+    EOF
+    TmpFile.with_dir do |dir|
+      TmpFile.with_file(identifier) do |identifier_file|
+        identifiers = TSV.open(identifier_file, sep: ",", type: :single)
+        brothers = datafile_test(:person).brothers
+        kb = KnowledgeBase.new dir
+        kb.register :brothers, brothers, identifiers: identifiers
+        assert_include kb.get_index(:brothers, source: "=>Initials"), "CC~Guille"
+      end
+    end
+  end
+end
+