scout-gear 10.7.5 → 10.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60a8c5a91d83e96aaa721154fa4f7fdd63d2699c691b8c99962c6ba5561365eb
-  data.tar.gz: 4b2303f3db25c1316286e8c6edbd8d76be66ba8b851626393f0c0b5ea34aac1d
+  metadata.gz: 27ef5b8aa113a998b4dde3d2eba3b8129e29b47b5a96af51e1bb8080d82cb671
+  data.tar.gz: 6e4cba134a145b426024f03d247550573f4dbf879cc600ad951a2729255ce2f3
 SHA512:
-  metadata.gz: 0eb9761f9cc9acab8842372de6bbe9f69413bc9d6b2c1b05ebae6c51e40ca292fbce9edce27d1c5d26a7e298273e700881bdd39f8bac3441b1e9af604c443eb0
-  data.tar.gz: cb46e73a548bca55f41e088167eff7494691a6fdd61ed17f7a06f6a1a0257679441b7d62a56e67a376943e4e0a96acd39d3a4ee5888550c1721c5d79e4f7e6a4
+  metadata.gz: c31507a2e82ca6d1b308661f586633f08a917dcd8b794cb7902d86e4c022f1f6ad1d4e423da0e3d8633464feeda0fd938b1e58779f72ad12e9685c67d244b4bc
+  data.tar.gz: 1bc95a9439b34ce670ebdbd893c79793c03d339c49b4e7e6877b24dd18e05f189875e95779e8bc3caea9abd3fb8491ec73888c74a03ffb1a859c7d35e397a0c9

data/.vimproject

@@ -10,6 +10,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     exceptions.rb
     export.rb
     definition.rb
+    persist.rb
     documentation.rb
     usage.rb
     util.rb
@@ -36,6 +37,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     }
     deployment.rb
     deployment=deployment{
+     queue.rb
      trace.rb
      orchestrator.rb
     }
@@ -123,6 +125,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     traverse.rb
     enrichment.rb
     list.rb
+    description.rb
    }
    semaphore.rb
   }
@@ -157,6 +160,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    install
    trace
    prov
+   queue
+   process
+   cmd
   }
   batch=batch{
    list
@@ -164,6 +170,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   }
   resource=resource{
    produce
+   sync
   }
  }
  test=test {
@@ -172,6 +179,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   test_scout.rb
   data=data filter="*"{
    person=person{
+    README.md
     brothers
     identifiers
     marriages

data/VERSION

@@ -1 +1 @@
-10.7.5
+10.7.7

data/lib/scout/association/index.rb

@@ -1,7 +1,7 @@
 require 'scout/annotation'
 module Association
 
-  def self.index(file, source: nil, target: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
+  def self.index(file, source: nil, target: nil, source_format: nil, target_format: nil, format: nil, database: nil, **kwargs)
     IndiferentHash.setup(kwargs)
     source = kwargs.delete :source if kwargs.include?(:source)
     target = kwargs.delete :target if kwargs.include?(:target)
@@ -14,7 +14,7 @@ module Association
     index = Persist.tsv(file, kwargs, engine: "BDB", persist_options: index_persist_options) do |data|
       recycle, undirected = IndiferentHash.process_options kwargs, :recycle, :undirected
 
-      database = Association.open(file, source: source, target: target, source_format: source_format, target_format: target_format, **kwargs.merge(persist_prefix: "Association::Database"))
+      database ||= Association.open(file, source: source, target: target, source_format: source_format, target_format: target_format, **kwargs.merge(persist_prefix: "Association::Database"))
 
       source_field = database.key_field
       target_field, *fields = database.fields

data/lib/scout/knowledge_base/description.rb

@@ -0,0 +1,108 @@
+class KnowledgeBase
+  def self.doc_parse_up_to(str, pattern, keep = false)
+    pre, _pat, _post = str.partition pattern
+    if _pat
+      [pre, (keep ? _pat << _post : _post)]
+    else
+      _post
+    end
+  end
+
+  def self.doc_parse_chunks(str, pattern)
+    parts = str.split(pattern)
+    return {} if parts.length < 2
+    databases = Hash[*parts[1..-1].collect{|v| v.strip }]
+    databases.delete_if{|t,d| d.empty?}
+    databases.transform_keys!(&:downcase)
+    databases
+  end
+
+  def self.parse_knowledge_base_doc(doc)
+    description, db_description = doc_parse_up_to doc, /^#/, true
+    databases = doc_parse_chunks db_description, /^# (.*)/ 
+    IndiferentHash.setup({:description => description.strip, :databases => databases})
+  end
+
+  def documentation_markdown
+    return "" if @libdir.nil?
+    file = @libdir['README.md'].find unless file.exists?
+    if file.exists?
+      file.read
+    else
+      ""
+    end
+  end
+  
+  def database_description_file(name)
+    dir[name.to_s + '.md']
+  end
+  
+  def knowledge_base_description_file(name)
+    file = dir['README.md']
+    return file if file.exists?
+
+    file, options = registry[name]
+    file = Path.setup(file.dup) unless file.nil? or Path === file
+    source_readme = file.dirname['README.md'] if file
+    return source_readme if source_readme  && source_readme.exists?
+  end
+  
+  def description(name)
+    return registered_options(name)[:description] if registered_options(name)[:description]
+    
+    return database_description_file(name).read if database_description_file(name).exist? 
+
+    if knowledge_base_description_file(name)
+      KnowledgeBase.parse_knowledge_base_doc(knowledge_base_description_file(name).read)[:databases][name.to_s.downcase]
+    end
+  end
+
+  def markdown(name)
+    description = description(name)
+    source_type = source_type(name)
+    target_type = target_type(name)
+
+    full_description = []
+    empty_line = ''
+    full_description << ("# " << Misc.humanize(name))
+    full_description << empty_line
+
+    source_formats = begin
+                       source_index(name).key_field.split(',')
+                     rescue
+                       []
+                     end
+
+    target_formats = begin
+                       target_index(name).key_field.split(',')
+                     rescue
+                       []
+                     end
+
+    if source_type 
+      full_description << "Source: #{source_type} - #{source(name)}"
+    else
+      full_description << "Source: #{source(name)}"
+    end
+    #full_description.last << ". Accepted formats: #{source_formats*", "}" if source_formats.any?
+
+    if target_type 
+      full_description << "Target: #{target_type} - #{target(name)}"
+    else
+      full_description << "Target: #{target(name)}"
+    end
+    #full_description.last << ". Accepted formats: #{target_formats*", "}" if target_formats.any?
+    
+    if undirected?(name) 
+      full_description << "Undirected database, source and target can be reversed."
+    end
+
+    if description
+      full_description << empty_line
+      full_description << description
+      full_description << empty_line
+    end
+
+    full_description * "\n"
+  end
+end

data/lib/scout/knowledge_base/entity.rb

@@ -80,6 +80,7 @@ class KnowledgeBase
       end
       identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
       identifier_files.uniq!
+      identifier_files.collect!{|f| (Path === f) ? f : Path.setup(f.dup) }
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, db_namespace(name)))} if not namespace and db_namespace(name)
       identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
@@ -89,7 +90,11 @@ class KnowledgeBase
   
   def target_index(name)
     Persist.memory("Target index #{name}: KB directory #{dir}") do
-      identifier_files = identifier_files(name)
+      if @identifier_files && @identifier_files.any?
+        identifier_files = @identifier_files
+      else
+        identifier_files = database_identifier_files(name)
+      end
       identifier_files.concat Entity.identifier_files(target(name)) if defined? Entity
       identifier_files.uniq!
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if self.namespace

data/lib/scout/knowledge_base/registry.rb

@@ -15,9 +15,29 @@ class KnowledgeBase
     end
   end
 
+  def present_databases
+    dir.glob("*.database").collect{|file| File.basename(file, '.database')}
+  end
+
   def all_databases
     return [] unless @registry
-    @registry.keys 
+    (@registry.keys + present_databases).uniq
+  end
+  
+  def database_file(name)
+    if @registry[name].nil?
+      nil
+    else
+      @registry[name].first
+    end
+  end
+
+  def registered_options(name)
+    if @registry[name].nil?
+      IndiferentHash.setup({})
+    else
+      IndiferentHash.setup(@registry[name].last)
+    end
   end
 
   def include?(name)
@@ -29,21 +49,21 @@ class KnowledgeBase
     @fields[name] ||= get_index(name).fields
   end
 
-  def description(name)
-    @descriptions ||= {}
-    @descriptions[name] ||= get_index(name).key_field.split("~")
+  def pair(name)
+    @pairs ||= {}
+    @pairs[name] ||= get_index(name).key_field.split("~")
   end
 
   def source(name)
-    description(name)[0]
+    pair(name)[0]
   end
 
   def target(name)
-    description(name)[1]
+    pair(name)[1]
   end
 
   def undirected(name)
-    description(name).length == 3
+    pair(name).length == 3
   end
 
   alias undirected? undirected
@@ -69,7 +89,12 @@ class KnowledgeBase
 
           persist_dir = dir
           persist_path = persist_dir[key].find
-          file, registered_options = registry[name]
+
+          file = database_file(name)
+          registered_options = registered_options(name)
+          registered_options =  IndiferentHash.setup(registered_options).except(:description)
+
+          registered_options = IndiferentHash.add_defaults registered_options, identifiers: self.identifier_files if registered_options
 
           options = IndiferentHash.add_defaults options, registered_options if registered_options and registered_options.any?
           options = IndiferentHash.add_defaults options, :persist_path => persist_path, :persist_dir => persist_dir, :persist => true
@@ -89,11 +114,11 @@ class KnowledgeBase
                     Log.low "Re-opening index #{ name } from #{ Log.fingerprint persist_path }. #{options}"
                     Association.index(file, **options, persist_options: persist_options.dup)
                   else
-                    options = IndiferentHash.add_defaults options, registered_options if registered_options
-                    raise "Repo #{ name } not found and not registered" if file.nil?
-                    Log.medium "Opening index #{ name } from #{ Log.fingerprint file }. #{options}"
+                    database = get_database name if file.nil?
                     file = file.call if Proc === file
-                    Association.index(file, **options, persist_options: persist_options.dup)
+                    Log.medium "Opening index #{ name } from #{ Log.fingerprint database }. #{options}"
+                    options = IndiferentHash.add_defaults options, registered_options if registered_options
+                    Association.index(file, **options, persist_options: persist_options.dup, database: database)
                   end
 
           index.namespace = self.namespace unless self.namespace
@@ -128,7 +153,12 @@ class KnowledgeBase
 
           persist_dir = dir
           persist_path = persist_dir[key].find
-          file, registered_options = registry[name]
+
+          file = database_file(name)
+          registered_options = registered_options(name)
+          registered_options =  IndiferentHash.setup(registered_options).except(:description)
+
+          registered_options = IndiferentHash.add_defaults registered_options, identifiers: self.identifier_files if registered_options
 
           options = IndiferentHash.add_defaults options, registered_options if registered_options and registered_options.any?
           options = IndiferentHash.add_defaults options, :persist_path => persist_path, :persist => true
@@ -145,7 +175,6 @@ class KnowledgeBase
 
           database = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
                        Log.low "Re-opening database #{ name } from #{ Log.fingerprint persist_path }. #{options}"
-                       #Association.database(file, **options, persist_options: persist_options)
                        Association.database(file, **options.merge(persist_options: persist_options))
                      else
                        options = IndiferentHash.add_defaults options, registered_options if registered_options
@@ -153,7 +182,6 @@ class KnowledgeBase
                        raise "Repo #{ name } not found and not registered" if file.nil?
                        Log.medium "Opening database #{ name } from #{ Log.fingerprint file }. #{options}"
                        file = file.call if Proc === file
-                       #Association.database(file, **options, persist_options: persist_options)
                        Association.database(file, **options.merge(persist_options: persist_options))
                      end
 

data/lib/scout/knowledge_base.rb

@@ -5,6 +5,7 @@ require_relative 'knowledge_base/entity'
 require_relative 'knowledge_base/query'
 require_relative 'knowledge_base/traverse'
 require_relative 'knowledge_base/list'
+require_relative 'knowledge_base/description'
 #require 'scout/knowledge_base/query'
 #require 'scout/knowledge_base/syndicate'
 
@@ -23,12 +24,12 @@ class KnowledgeBase
     @entity_options ||= IndiferentHash.setup({})
 
     @format         ||= IndiferentHash.setup({})
-    @descriptions   ||= IndiferentHash.setup({})
+    pairs          ||= IndiferentHash.setup({})
     @indices        ||= IndiferentHash.setup({})
   end
 
   def config_file(name)
-    @dir.config[name.to_s]
+    @dir['config'][name.to_s]
   end
 
   def save_variable(name)

data/lib/scout/tsv/annotation.rb

@@ -75,17 +75,17 @@ module Annotation
     when Annotation.is_annotated?(objs)
       tsv.key_field = "List"
 
-      tsv[objs.annotation_id] = self.list_tsv_values(objs, fields).dup
+      tsv[objs.id] = self.list_tsv_values(objs, fields).dup
     when Array === objs 
       tsv.key_field = "ID"
 
       if Annotation.is_annotated?(objs.compact.first)
         objs.compact.each_with_index do |obj,i|
-          tsv[obj.annotation_id + "#" << i.to_s] = self.obj_tsv_values(obj, fields).dup
+          tsv[obj.id + "#" << i.to_s] = self.obj_tsv_values(obj, fields).dup
         end
       elsif (objs.any? && Annotation.is_annotated?(objs.compact.first.compact.first))
         objs.flatten.compact.each_with_index do |obj,i|
-          tsv[obj.annotation_id + "#" << i.to_s] = self.obj_tsv_values(obj, fields).dup
+          tsv[obj.id + "#" << i.to_s] = self.obj_tsv_values(obj, fields).dup
         end
       end
     else

data/lib/scout/tsv/change_id/translate.rb

@@ -58,6 +58,7 @@ module TSV
       #next if Path === file && ! Open.exist?(file)
       begin
         file = file.produce if Path === file
+        raise "Could no produce file" if FalseClass === file
       rescue
         Log.warn $!.message
         next

data/lib/scout/tsv/open.rb

@@ -153,6 +153,14 @@ module Open
                 end
                 obj.close
                 obj.join if obj.respond_to? :join
+              elsif options[:type] == :matrix
+                Log.low "Traverse stream by lines #{Log.fingerprint obj}"
+                parser = options[:sep].nil? ? TSV::Parser.new(obj) : TSV::Parser.new(obj, sep: options[:sep])
+                parser.traverse **options do |parts|
+                  res = block.call parts
+                  callback.call res if callback
+                  nil
+                end
               else
                 Log.low "Traverse stream with parser #{Log.fingerprint obj}"
                 parser = options[:sep].nil? ? TSV::Parser.new(obj) : TSV::Parser.new(obj, sep: options[:sep])

data/lib/scout/tsv/parser.rb

@@ -31,6 +31,12 @@ module TSV
 
     return nil if select && ! TSV.select(items[0], items[1..-1], select, fields: field_names, type: type, sep: sep2)
 
+    if String === key
+      raise "Key by name, but no field names" if field_names.nil?
+      key = field_names.index key
+      raise "Key #{key} not found in field names #{Log.fingerprint field_names}" if key.nil?
+    end
+
     if positions.nil? && key == 0
       key = items.shift
     elsif positions.nil?
@@ -65,7 +71,7 @@ module TSV
     [key, items]
   end
 
-  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, head: nil, **kwargs, &block)
+  def self.parse_stream(stream, data: nil, source_type: nil, sep: "\t", type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, head: nil, **kwargs, &block)
     begin
       bar = "Parsing #{Log.fingerprint stream}" if TrueClass === bar
       bar = Log::ProgressBar.get_obj_bar(stream, bar) if bar
@@ -81,7 +87,7 @@ module TSV
           data.serializer.to_s.include?("String") &&
           same_type && 
           ! (head || kwargs[:cast] || kwargs[:positions] || (kwargs[:key] && kwargs[:key] != 0) || Proc === fix ) &&
-          (kwargs[:sep].nil? || kwargs[:sep] == "\t")
+          (sep.nil? || sep == "\t")
 
 
         Log.debug "Loading #{Log.fingerprint stream} directly into #{Log.fingerprint data}"
@@ -114,12 +120,17 @@ module TSV
             line = Misc.fixutf8(line)
           end
           bar.tick if bar
+
           if type == :array || type == :line
             block.call line
             next
+          elsif type == :matrix
+            parts = line.split(sep)
+            block.call parts
+            next
           end
 
-          key, items = parse_line(line, type: source_type, field_names: field_names, **kwargs)
+          key, items = parse_line(line, type: source_type, field_names: field_names, sep: sep, **kwargs)
 
           next if key.nil?
 

data/lib/scout/workflow/definition.rb

@@ -29,7 +29,7 @@ module Workflow
     if block_given?
       helpers[name] = block
     else
-      raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
+      raise ScoutException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
       o = Object.new
       o.extend step_module
       o.send(name, *args, **kwargs)
@@ -164,7 +164,7 @@ module Workflow
       type = task_proc.type 
     end
     task name => type do
-      raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
+      raise ScoutException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -1,6 +1,8 @@
 module Workflow
   class Orchestrator
 
+    class NoWork < Exception; end
+
     def self.job_workload(job)
       workload = {job => []}
       return workload if job.done? && job.updated?
@@ -202,7 +204,7 @@ module Workflow
 
           candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
           candidates.uniq!
-          raise "No candidates and no running jobs" if candidates.empty?
+          raise NoWork, "No candidates and no running jobs" if candidates.empty?
 
           candidates.each do |job|
             case 
@@ -272,4 +274,10 @@ module Workflow
     orchestrator.process({}, produce_list)
     produce_list
   end
+
+  def self.produce(jobs, produce_cpus: Etc.nprocessors, produce_timer: 5)
+    jobs = [jobs] unless Array === jobs
+    orchestrator = Orchestrator.new produce_timer.to_i, cpus: produce_cpus.to_i
+    orchestrator.process({}, jobs)
+  end
 end

data/lib/scout/workflow/deployment/queue.rb

@@ -0,0 +1,26 @@
+module Workflow
+  def self.name2clean_name(name)
+    name.reverse.partition("_").last.reverse
+  end
+
+  def self.queue_job(file)
+    workflow, task, name = file.split("/").values_at(-3, -2, -1) if file
+    workflow = Workflow.require_workflow workflow
+
+    if Open.directory?(file)
+      clean_name = name2clean_name name
+      inputs = workflow.tasks[task].load_inputs(file)
+      workflow.job(task, clean_name, inputs)
+    else
+      workflow.job(task, name)
+    end
+  end
+
+  def self.unqueue(file, &block)
+    Open.lock file do
+      job = queue_job(file)
+      puts job.run
+      Open.rm_rf file
+    end
+  end
+end

data/lib/scout/workflow/export.rb

@@ -18,6 +18,12 @@ module Workflow
     @exec_exports ||= []
   end
 
+  def clear_exports
+    asynchronous_exports.clear
+    synchronous_exports.clear
+    exec_exports.clear
+    stream_exports.clear
+  end
 
   def all_exports
     asynchronous_exports + synchronous_exports + exec_exports + stream_exports

data/lib/scout/workflow/persist.rb

@@ -0,0 +1,6 @@
+module Workflow
+  def persist(name, type = :serializer, options = {}, &block)
+    options = IndiferentHash.add_defaults options, dir: Scout.var.workflows[self.name].persist
+    Persist.persist(name, type, options, &block)
+  end
+end

data/lib/scout/workflow/step/info.rb

@@ -1,3 +1,4 @@
+require 'time'
 require 'scout/config'
 class Step
   SERIALIZER = Scout::Config.get(:serializer, :step_info, :info, :step, env: "SCOUT_SERIALIZER", default: :json)

data/lib/scout/workflow/step/inputs.rb

@@ -1,5 +1,15 @@
 class Step
   def save_inputs(inputs_dir)
-    self.task.save_inputs(inputs_dir, provided_inputs)
+    if clean_name != name
+      hash = name[clean_name.length..-1]
+      inputs_dir += hash
+      Log.medium "Saving job inputs to: #{Log.fingerprint inputs_dir}"
+      self.task.save_inputs(inputs_dir, provided_inputs)
+      inputs_dir
+    else
+      Log.medium "Saving no input job: #{Log.fingerprint inputs_dir}"
+      Open.touch(inputs_dir)
+      inputs_dir
+    end
   end
 end

data/lib/scout/workflow/task.rb

@@ -34,6 +34,7 @@ module Task
   def job(id = nil, provided_inputs = nil)
 
     if Hash === provided_inputs
+      IndiferentHash.setup provided_inputs
       memory_inputs = provided_inputs.values_at *self.recursive_inputs.collect{|t| t.first }.uniq
       memory_inputs += provided_inputs.select{|k,v| k.to_s.include?("#") }.collect{|p| p * "=" }
       memory_inputs << provided_inputs[:load_inputs]
@@ -41,7 +42,7 @@ module Task
       memory_inputs = provided_inputs
     end
 
-    Persist.memory("Task job #{self.name} #{id}", other_options: {task: self, id: id, provided_inputs: memory_inputs}) do 
+    Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {task: self.name, id: id, provided_inputs: memory_inputs}) do
       provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
       provided_inputs = {} if provided_inputs.nil?
       IndiferentHash.setup(provided_inputs)

data/lib/scout/workflow.rb

@@ -9,13 +9,19 @@ require_relative 'workflow/exceptions'
 require_relative 'workflow/path'
 require_relative 'workflow/entity'
 require_relative 'workflow/export'
+require_relative 'workflow/persist'
 
 require 'scout/resource'
 require 'scout/resource/scout'
 
 module Workflow
   class << self
-    attr_accessor :workflows, :main, :workflow_dir, :autoinstall, :workflow_repo
+    attr_accessor :workflows, :main, :workflow_dir, :autoinstall, :workflow_repo, :job_cache
+
+    def job_cache
+      @job_cache ||={}
+    end
+
     def workflows
       @workflows ||= []
     end

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.7.5 ruby lib
+# stub: scout-gear 10.7.7 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.7.5".freeze
+  s.version = "10.7.7".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-03-19"
+  s.date = "2025-04-11"
   s.description = "Scout gear: workflow, TSVs, persistence, entities, associations, and knowledge_bases.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -45,6 +45,7 @@ Gem::Specification.new do |s|
     "lib/scout/entity/object.rb",
     "lib/scout/entity/property.rb",
     "lib/scout/knowledge_base.rb",
+    "lib/scout/knowledge_base/description.rb",
     "lib/scout/knowledge_base/enrichment.rb",
     "lib/scout/knowledge_base/entity.rb",
     "lib/scout/knowledge_base/list.rb",
@@ -98,12 +99,14 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/definition.rb",
     "lib/scout/workflow/deployment.rb",
     "lib/scout/workflow/deployment/orchestrator.rb",
+    "lib/scout/workflow/deployment/queue.rb",
     "lib/scout/workflow/deployment/trace.rb",
     "lib/scout/workflow/documentation.rb",
     "lib/scout/workflow/entity.rb",
     "lib/scout/workflow/exceptions.rb",
     "lib/scout/workflow/export.rb",
     "lib/scout/workflow/path.rb",
+    "lib/scout/workflow/persist.rb",
     "lib/scout/workflow/step.rb",
     "lib/scout/workflow/step/archive.rb",
     "lib/scout/workflow/step/children.rb",
@@ -140,11 +143,14 @@ Gem::Specification.new do |s|
     "scout_commands/log",
     "scout_commands/rbbt",
     "scout_commands/resource/produce",
+    "scout_commands/resource/sync",
     "scout_commands/template",
     "scout_commands/update",
+    "scout_commands/workflow/cmd",
     "scout_commands/workflow/info",
     "scout_commands/workflow/install",
     "scout_commands/workflow/list",
+    "scout_commands/workflow/process",
     "scout_commands/workflow/prov",
     "scout_commands/workflow/task",
     "scout_commands/workflow/trace",
@@ -154,6 +160,7 @@ Gem::Specification.new do |s|
     "share/software/install_helpers",
     "share/templates/command",
     "share/templates/workflow.rb",
+    "test/data/person/README.md",
     "test/data/person/brothers",
     "test/data/person/identifiers",
     "test/data/person/marriages",
@@ -166,6 +173,7 @@ Gem::Specification.new do |s|
     "test/scout/entity/test_named_array.rb",
     "test/scout/entity/test_object.rb",
     "test/scout/entity/test_property.rb",
+    "test/scout/knowledge_base/test_description.rb",
     "test/scout/knowledge_base/test_enrichment.rb",
     "test/scout/knowledge_base/test_entity.rb",
     "test/scout/knowledge_base/test_list.rb",

data/scout_commands/kb/config

@@ -12,6 +12,7 @@ Configure a knowlege base
 $ #{$0} [<options>] <name> [knowledge_base]
 
 -h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
 -i--identifier_files* Identifier files separated by ','
 -n--namespace* Namespace
 EOF
@@ -26,8 +27,10 @@ end
 
 knowledge_base_name = ARGV.first
 
+knowledge_base_name ||= options[:knowledge_base] || :default
 knowledge_base = KnowledgeBase.load knowledge_base_name || :default
 
 knowledge_base.namespace = options[:namespace] if options.include? :namespace
 knowledge_base.identifier_files += options[:identifier_files].split(",") if options.include? :identifier_files
+knowledge_base.identifier_files.uniq!
 knowledge_base.save

data/scout_commands/kb/list

@@ -12,6 +12,7 @@ Description of the tool
 $ #{$0} [<options>] <filename> [<other|->]*
 
 -h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
 EOF
 if options[:help]
   if defined? scout_usage

data/scout_commands/kb/query

@@ -34,7 +34,8 @@ raise MissingParameterException, :name if name.nil?
 raise MissingParameterException, :entity if entity.nil?
 
 
-knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+knowledge_base = IndiferentHash.process_options options, :knowledge_base
+knowledge_base = KnowledgeBase.load knowledge_base || :default
 
 list = IndiferentHash.process_options options, :list
 

data/scout_commands/kb/register

@@ -19,6 +19,7 @@ $ #{$0} [<options>] <name> <filename>
 -n--namespace* Namespace
 -i--identifiers* Identifiers
 -u--undirected
+-d--description* Database description
 EOF
 if options.delete :help
   if defined? scout_usage
@@ -35,7 +36,8 @@ name, file = ARGV
 raise MissingParameterException, :name if name.nil?
 raise MissingParameterException, :file if file.nil?
 
-knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+knowledge_base = IndiferentHash.process_options options, :knowledge_base
+knowledge_base = KnowledgeBase.load knowledge_base || :default
 
 options[:fields] = options[:fields].split(/,\s*/) if options[:fields]
 file = Scout.identify(File.expand_path(file))

data/scout_commands/kb/show

@@ -23,7 +23,8 @@ if options[:help]
   exit 0
 end
 
-knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+knowledge_base = IndiferentHash.process_options options, :knowledge_base
+knowledge_base = KnowledgeBase.load knowledge_base || :default
 
 name = ARGV.first
 
@@ -31,7 +32,8 @@ if name.nil?
   puts knowledge_base.all_databases * "\n"
 else
 
-  raise ParameterException "Database #{name} not found Options: #{Log.fingerprint knowledge_base.all_databases}" unless knowledge_base.include? name
+  raise ParameterException, "Database #{name} not found Options: #{Log.fingerprint knowledge_base.all_databases}" unless knowledge_base.include? name
 
+  puts knowledge_base.markdown(name)
   Log.tsv knowledge_base.get_database(name, options)
 end

data/scout_commands/resource/sync

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Sync to resources between path maps or between hosts
+
+$ #{$0} [<options>] <path> [<path_map>]
+
+Path map defaults to :user
+
+-h--help Print this help
+-s--source* Host of source path
+-t--target* Host of target path
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+path, path_map = ARGV
+
+raise MissingParameterException, :path if path.nil?
+
+options.keys_to_sym!
+Resource.sync(Path.setup(path.dup), path_map, **options)

data/scout_commands/workflow/cmd

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Run a workflow command
+
+$ #{$0} <workflow> <command> [<subcommands>] [<options>] [<arg> ...]
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+workflow = ARGV.shift
+
+if workflow == '-h'
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+raise ParameterException, "No workflow specified" if workflow.nil?
+
+require 'scout/workflow'
+
+wf = Workflow.require_workflow workflow
+dir = $command_dir = wf.libdir.share.scout_commands.find
+
+def prev_dir(prev)
+    scout_command_dir = $command_dir
+
+    prev.each do |previous_command|
+        scout_command_dir = scout_command_dir[previous_command]
+    end
+
+    scout_command_dir
+end
+
+def commands(prev)
+    scout_command_dir = prev_dir(prev)
+
+    command_file_dirs = scout_command_dir.find_all
+    command_files = command_file_dirs.collect{|d| d.glob('*') }.flatten
+    command_files.collect{|p| File.basename(p) }.uniq.reject{|p| p =~ /\.desc$/}.sort
+end
+
+
+prev = []
+
+$previous_commands << ' cmd'
+$previous_commands << ' '  << workflow
+
+begin
+  while ARGV.any?
+    command = ARGV.shift
+    case
+    when File.directory?(dir[command].find)
+      prev << command
+      $previous_commands << command
+      dir = dir[command]
+    when File.directory?(dir[command].find)
+      prev << command
+      dir = dir[command]
+    when dir[command].exists?
+      load dir[command].find
+      exit 0
+    else
+      if command == 'bootstrap'
+        if wf.libdir["test_workflow.rb"].exists?
+          Log.info "No bootstrap for #{ workflow }, running test_workflow.rb instead"
+          CMD.cmd_log('ruby', wf.libdir["test_workflow.rb"].find)
+        else
+          Log.info "No bootstrap for #{ workflow }, running examples instead"
+          CMD.cmd_log("scout workflow example #{ workflow }")
+          exit 0
+        end
+      end
+
+      raise ParameterException, "Error: Command not understood: #{command}"
+    end
+  end
+end
+
+puts SOPT.doc
+puts
+puts Log.color :magenta, "## COMMANDS"
+puts
+puts Log.color :magenta, "Command:"
+puts 
+puts " scout #{$previous_commands * " "} "
+puts 
+puts Log.color :magenta, "Subcommands:"
+puts 
+
+commands(prev).each do |command|
+  directory = File.directory? dir[command].find
+  if directory
+    puts "  " << Log.color(:blue, command)
+  else
+    puts "  " << command
+  end
+end
+    

data/scout_commands/workflow/process

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/workflow/deployment/queue'
+require 'scout/workflow/deployment/orchestrator'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Process the queue
+
+$ #{$0} [<options>] ([<workflow>] [<task>] [<name>] | <filename>)
+
+-h--help Print this help
+-l--list List queue jobs
+--continuous Process continuously
+--produce_timer* Produce timer for sleeping
+--produce_cpus* Cpus to use concurrently producing jobs
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+list, continuous = IndiferentHash.process_options options, :list, :continuous
+
+queue_dir = Scout.var.queue
+
+class TrayAgain < Exception; end
+
+begin
+  if ARGV.empty?
+    files = queue_dir.glob("*/*/*")
+    if list
+      puts files * "\n"
+      exit
+    end
+  else
+    workflow, task, name = ARGV
+
+    # First deal with fixed files
+    if task.nil?
+      if Open.exists?(workflow)
+        Workflow.unqueue files.first
+        exit
+
+      else
+        files = queue_dir.glob("#{workflow}/*/*")
+      end
+    elsif name.nil?
+      files = queue_dir.glob("#{workflow}/#{task}/*")
+    else
+      files = queue_dir.glob("#{workflow}/#{task}/#{name}*")
+    end
+  end
+
+  jobs = files.collect{|file| Workflow.queue_job(file) }
+  begin
+    options.keys.each do |key|
+      options[key.to_sym] = options.delete(key)
+    end
+    Workflow.produce(jobs, **options)
+  rescue Workflow::Orchestrator::NoWork
+  end
+
+  files.each do |f| Open.rm_rf f end
+
+  Workflow.job_cache.clear
+
+  if continuous
+    Log.debug "Continuous processing"
+    sleep 1
+    raise Workflow::Orchestrator::NoWork
+  end
+rescue Workflow::Orchestrator::NoWork
+  retry
+end
+

data/scout_commands/workflow/task

@@ -20,8 +20,9 @@ $ #{$0} [<options>] <workflow> <task>
 --update Update jobs with newer dependencies
 --deploy* Deploy mode: serial, local, or SLURM (default 'serial')
 --fork Fork and return path
+--load_inputs* Directory or file with inputs files to load
+--save_inputs* Directory or tar.gz file path to store inputs
 -jn--jobname* Name to use as job identifier
--li--load_inputs* Directory with inputs files to load
 -pf--printpath Print the file path
 -prov--provenance Print the step provenance
 -cl--clean Clean the last step
@@ -40,8 +41,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, printpath, deploy, override_deps, do_fork = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :printpath, :deploy, :override_deps, :fork,
+help, provenance, clean, recursive_clean, clean_task, load_inputs, save_inputs, jobname, printpath, deploy, override_deps, do_fork = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :save_inputs, :jobname, :printpath, :deploy, :override_deps, :fork,
   :deploy => 'serial'
 
 if help
@@ -93,12 +94,23 @@ end
 
 if provenance
   puts Step.prov_report(job)
+
 elsif do_fork
   job.fork
   puts job.path
   exit 0
+elsif save_inputs
+  puts job.save_inputs(save_inputs)
+  exit 0
 else
+
   case deploy
+  when "queue"
+    if ! job.done?
+      save_inputs = Scout.var.queue[workflow.to_s][task_name][job.clean_name].find
+      puts job.save_inputs(save_inputs)
+      exit
+    end
   when "serial"
     job.run(true) 
   when "local"

data/test/data/person/README.md

@@ -0,0 +1,17 @@
+Databases describing people and their relationships.
+
+# Brothers
+
+Sibling relationships. The first of the pair is the Older 
+
+# Marriages
+
+Marriages between people. 
+
+Date: Date when the marriage occurred
+
+# Parents
+
+List parents.
+
+Type of parent: father or mother

data/test/scout/knowledge_base/test_description.rb

@@ -0,0 +1,59 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/knowledge_base'
+class TestKnowledgebaseDesc < Test::Unit::TestCase
+  def test_brothers_registry
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers, description: "Sibling relationships."
+      assert_equal "Sibling relationships.", kb.description(:brothers)
+    end
+  end
+
+  def test_brothers_README
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      kb.dir['brothers.md'].write "Sibling relationships."
+      assert_equal "Sibling relationships.", kb.description(:brothers)
+    end
+  end
+
+  def test_broters_kb_README
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      kb.dir['README.md'].write <<-EOF
+Databases describing people and their relationships.
+
+# Brothers
+
+Sibling relationships.
+      EOF
+      assert_equal "Sibling relationships.", kb.description(:brothers)
+    end
+  end
+
+  def test_brothers_kb_source_README
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      assert_include kb.description(:brothers), "Sibling relationships."
+    end
+  end
+ 
+  def test_full_description
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      assert_include kb.markdown(:brothers), "Older"
+    end
+  end
+end
+

data/test/scout/workflow/test_entity.rb

@@ -12,15 +12,15 @@ class TestWorkflowEntity < Test::Unit::TestCase
         "Mi name is #{self}"
       end
 
-      entity_job hi: :string do
+      entity_task hi: :string do
         "Hi. #{entity.introduction}"
       end
 
-      list_job group_hi: :string do
+      list_task group_hi: :string do
         "Here is the group: " + entity_list.hi * "; "
       end
 
-      list_job bye: :array do
+      list_task bye: :array do
         entity_list.collect do |e|
           "Bye from #{e}"
         end

data/test/test_helper.rb

@@ -20,6 +20,7 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 #require 'scout/helper/misc/development'
+require 'scout-gear'
 require 'scout/tmpfile'
 require 'scout/log'
 require 'scout/open'
@@ -54,6 +55,7 @@ class Test::Unit::TestCase
   
   teardown do
     Open.rm_rf tmpdir
+    Workflow.job_cache.clear
   end
 
   def self.datadir_test

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.7.5
+  version: 10.7.7
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-03-19 00:00:00.000000000 Z
+date: 2025-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -114,6 +114,7 @@ files:
 - lib/scout/entity/object.rb
 - lib/scout/entity/property.rb
 - lib/scout/knowledge_base.rb
+- lib/scout/knowledge_base/description.rb
 - lib/scout/knowledge_base/enrichment.rb
 - lib/scout/knowledge_base/entity.rb
 - lib/scout/knowledge_base/list.rb
@@ -167,12 +168,14 @@ files:
 - lib/scout/workflow/definition.rb
 - lib/scout/workflow/deployment.rb
 - lib/scout/workflow/deployment/orchestrator.rb
+- lib/scout/workflow/deployment/queue.rb
 - lib/scout/workflow/deployment/trace.rb
 - lib/scout/workflow/documentation.rb
 - lib/scout/workflow/entity.rb
 - lib/scout/workflow/exceptions.rb
 - lib/scout/workflow/export.rb
 - lib/scout/workflow/path.rb
+- lib/scout/workflow/persist.rb
 - lib/scout/workflow/step.rb
 - lib/scout/workflow/step/archive.rb
 - lib/scout/workflow/step/children.rb
@@ -209,11 +212,14 @@ files:
 - scout_commands/log
 - scout_commands/rbbt
 - scout_commands/resource/produce
+- scout_commands/resource/sync
 - scout_commands/template
 - scout_commands/update
+- scout_commands/workflow/cmd
 - scout_commands/workflow/info
 - scout_commands/workflow/install
 - scout_commands/workflow/list
+- scout_commands/workflow/process
 - scout_commands/workflow/prov
 - scout_commands/workflow/task
 - scout_commands/workflow/trace
@@ -223,6 +229,7 @@ files:
 - share/software/install_helpers
 - share/templates/command
 - share/templates/workflow.rb
+- test/data/person/README.md
 - test/data/person/brothers
 - test/data/person/identifiers
 - test/data/person/marriages
@@ -235,6 +242,7 @@ files:
 - test/scout/entity/test_named_array.rb
 - test/scout/entity/test_object.rb
 - test/scout/entity/test_property.rb
+- test/scout/knowledge_base/test_description.rb
 - test/scout/knowledge_base/test_enrichment.rb
 - test/scout/knowledge_base/test_entity.rb
 - test/scout/knowledge_base/test_list.rb