scout-gear 10.7.4 → 10.7.6

data/lib/scout/workflow/entity.rb

@@ -0,0 +1,99 @@
+require 'scout/entity'
+require 'scout/workflow'
+
+module EntityWorkflow
+
+  def self.extended(base)
+    base.extend Workflow
+    base.extend Entity
+
+    base.instance_variable_set(:@annotation_inputs, IndiferentHash.setup({})) unless base.instance_variables.include?(:@annotation_inputs)
+    class << base
+      def annotation_input(name, type=nil, desc=nil, default=nil, options = {})
+        annotation name
+        annotation_inputs = self.instance_variable_get("@annotation_inputs")
+        annotation_inputs[name] = [type, desc, default, options]
+      end
+    end
+
+    base.helper :entity do
+      base.setup(clean_name.dup, inputs.to_hash)
+    end
+
+    base.helper :entity_list do 
+      list = inputs.last
+      list = list.load if Step === list
+      base.setup(list, inputs.to_hash)
+    end
+
+    base.property job: :both  do |task_name,options={}|
+      if Array === self && AnnotatedArray === self
+        base.job(task_name, "Default", options.merge(list: self))
+      else
+        base.job(task_name, self, options)
+      end
+    end
+  end
+
+  def property_task(task_name, property_type=:single, *args, &block)
+    task_name, result_type = task_name.keys.first, task_name.values.first if Hash === task_name
+
+    annotation_inputs = self.instance_variable_get("@annotation_inputs")
+    self.annotations.each do |annotation|
+      if annotation_inputs[annotation]
+        input annotation, *annotation_inputs[annotation]
+      else
+        input annotation
+      end
+    end
+    case property_type
+    when :single, :single2array
+      input :entity, :string, "#{self.to_s} identifier", nil, jobname: true
+      task(task_name => result_type, &block)
+    when :both
+      input :entity, :string, "#{self.to_s} identifier", nil, jobname: true
+      input :list, :array, "#{self.to_s} identifier list"
+      task(task_name => result_type, &block)
+    else
+      input :list, :array, "#{self.to_s} identifier list"
+      task(task_name => result_type, &block)
+    end
+
+    property task_name => property_type do |*args|
+      job = job(task_name, *args)
+      Array === job ? job.collect(&:run) : job.run
+    end
+  end
+
+  def entity_task(task_name, *args, &block)
+    property_task(task_name, :single, *args, &block)
+  end
+
+  def list_task(task_name, *args, &block)
+    property_task(task_name, :array, *args, &block)
+  end
+
+  def multiple_task(task_name, *args, &block)
+    property_task(task_name, :multiple, *args, &block)
+  end
+
+  def property_task_alias(task_name, property_type=:single, *args)
+    task_alias task_name, *args
+    property task_name => property_type do |*args|
+      job = job(task_name, *args)
+      Array === job ? job.collect(&:run) : job.run
+    end
+  end
+
+  def entity_task_alias(task_name, *args)
+    property_task_alias(task_name, :single, *args)
+  end
+
+  def list_task_alias(task_name, *args)
+    property_task_alias(task_name, :array, *args)
+  end
+
+  def multiple_task_alias(task_name, *args)
+    property_task_alias(task_name, :multiple, *args)
+  end
+end

data/lib/scout/workflow/export.rb

@@ -0,0 +1,72 @@
+module Workflow
+
+  annotation :asynchronous_exports, :synchronous_exports, :exec_exports, :stream_exports
+
+  def asynchronous_exports
+    @asynchronous_exports ||= []
+  end
+
+  def synchronous_exports
+    @synchronous_exports ||= []
+  end
+
+  def exec_exports
+    @exec_exports ||= []
+  end
+
+  def stream_exports
+    @exec_exports ||= []
+  end
+
+  def clear_exports
+    asynchronous_exports.clear
+    synchronous_exports.clear
+    exec_exports.clear
+    stream_exports.clear
+  end
+
+  def all_exports
+    asynchronous_exports + synchronous_exports + exec_exports + stream_exports
+  end
+
+  alias task_exports all_exports
+
+  def unexport(*names)
+    names = names.collect{|n| n.to_s} + names.collect{|n| n.to_sym}
+    names.uniq!
+    exec_exports.replace exec_exports - names if exec_exports
+    synchronous_exports.replace synchronous_exports - names if synchronous_exports
+    asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
+    stream_exports.replace stream_exports - names if stream_exports
+  end
+  
+  def export_exec(*names)
+    unexport *names
+    exec_exports.concat names
+    exec_exports.uniq!
+    exec_exports
+  end
+
+  def export_synchronous(*names)
+    unexport *names
+    synchronous_exports.concat names
+    synchronous_exports.uniq!
+    synchronous_exports
+  end
+
+  def export_asynchronous(*names)
+    unexport *names
+    asynchronous_exports.concat names
+    asynchronous_exports.uniq!
+    asynchronous_exports
+  end
+
+  def export_stream(*names)
+    unexport *names
+    stream_exports.concat names
+    stream_exports.uniq!
+    stream_exports
+  end
+
+  alias export export_asynchronous
+end

data/lib/scout/workflow/persist.rb

@@ -0,0 +1,6 @@
+module Workflow
+  def persist(name, type = :serializer, options = {}, &block)
+    options = IndiferentHash.add_defaults options, dir: Scout.var.workflows[self.name].persist
+    Persist.persist(name, type, options, &block)
+  end
+end

data/lib/scout/workflow/step/file.rb

@@ -21,13 +21,13 @@ class Step
   end
 
   def files
-    Dir.glob(File.join(files_dir, '**', '*')).reject{|path| File.directory? path }.collect do |path| 
-      Misc.path_relative_to(files_dir, path) 
+    files_dir.glob("**/*").reject{|path| File.directory? path }.collect do |path|
+      Misc.path_relative_to(files_dir, path)
     end
   end
 
   def bundle_files
-    [path, info_file, Dir.glob(File.join(files_dir,"**/*"))].flatten.select{|f| Open.exist?(f) }
+    [path, info_file, files_dir.glob("**/*")].flatten.select{|f| Open.exist?(f) }
   end
 
   def copy_linked_files_dir

data/lib/scout/workflow/step/info.rb

@@ -1,3 +1,4 @@
+require 'time'
 require 'scout/config'
 class Step
   SERIALIZER = Scout::Config.get(:serializer, :step_info, :info, :step, env: "SCOUT_SERIALIZER", default: :json)
@@ -74,6 +75,11 @@ class Step
           issued = info[:issued]
           start = info[:start]
           eend = new_info[:end]
+
+          start = Time.parse start if String === start
+          eend = Time.parse eend if String === eend
+          issued = Time.parse issued if String === issued
+
           if start && eend
             time = eend - start
             total_time = eend - issued

data/lib/scout/workflow/step/inputs.rb

@@ -1,5 +1,15 @@
 class Step
   def save_inputs(inputs_dir)
-    self.task.save_inputs(inputs_dir, provided_inputs)
+    if clean_name != name
+      hash = name[clean_name.length..-1]
+      inputs_dir += hash
+      Log.medium "Saving job inputs to: #{Log.fingerprint inputs_dir}"
+      self.task.save_inputs(inputs_dir, provided_inputs)
+      inputs_dir
+    else
+      Log.medium "Saving no input job: #{Log.fingerprint inputs_dir}"
+      Open.touch(inputs_dir)
+      inputs_dir
+    end
   end
 end

data/lib/scout/workflow/step/provenance.rb

@@ -108,12 +108,11 @@ class Step
     status = info[:status] || :missing
     status = status.to_sym if String === status
     status = :noinfo if status == :missing && Open.exist?(path)
-    status = "remote" if Open.remote?(path) || Open.ssh?(path)
+    status = :remote if Open.remote?(path) || Open.ssh?(path)
     name = info[:name] || File.basename(path)
     status = :unsync if status == :done and not Open.exist?(path)
     status = :notfound if status == :noinfo and not Open.exist?(path)
 
-
     this_step_msg = prov_report_msg(status, name, path, info, input)
 
     input_dependencies ||= {}

data/lib/scout/workflow/step/status.rb

@@ -42,6 +42,7 @@ class Step
     @result = nil
     @info = nil
     @info_load_time = nil
+    @done = nil
     Open.rm path if Open.exist_or_link?(path)
     Open.rm tmp_path if Open.exist_or_link?(tmp_path)
     Open.rm info_file if Open.exist_or_link?(info_file)

data/lib/scout/workflow/step.rb

@@ -75,19 +75,21 @@ class Step
   def clean_name
     return @id if @id
     return info[:clean_name] if info.include? :clean_name
-    if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
+    if m = name.match(/(.+?)(?:_[a-z0-9]{32})?(?:\..*)?$/)
       return m[1] 
     end
     return name.split(".").first
   end
 
   def task_name
+    return @task_name if @task_name
     @task_name ||= @task.name if @task.respond_to?(:name)
     @task_name ||= info[:task_name] if Open.exist?(info_file)
     @task_name ||= path.split("/")[-2]
   end
 
   def workflow
+    return @workflow if @workflow
     @workflow ||= @task.workflow if Task === @task
     @workflow ||= info[:workflow] if info_file && Open.exist?(info_file)
     @workflow ||= path.split("/")[-3]
@@ -181,7 +183,7 @@ class Step
         log :start
         @exec_result = exec
 
-        if @exec_result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
+        if @exec_result.nil? && Open.exist?(self.tmp_path) && ! Open.exist?(self.path)
           Open.mv self.tmp_path, self.path
         else
           @exec_result = @exec_result.stream if @exec_result.respond_to?(:stream) && ! (TSV === @exec_result)
@@ -263,7 +265,7 @@ class Step
   end
 
   def done?
-    Open.exist?(path)
+    @done ||= Open.exist?(path)
   end
 
   def streaming?

data/lib/scout/workflow/task/inputs.rb

@@ -186,7 +186,8 @@ module Task
       seen << name
     end
 
-    Dir.glob(File.join(directory, "*#*")).each do |file|
+    directory = Path.setup(directory.dup) unless Path === directory
+    directory.glob("*#*").each do |file|
       override_dep, _, extension = File.basename(file).partition(".")
 
       inputs[override_dep] = Task.load_input_from_file(file, :file)

data/lib/scout/workflow/task.rb

@@ -34,6 +34,7 @@ module Task
   def job(id = nil, provided_inputs = nil)
 
     if Hash === provided_inputs
+      IndiferentHash.setup provided_inputs
       memory_inputs = provided_inputs.values_at *self.recursive_inputs.collect{|t| t.first }.uniq
       memory_inputs += provided_inputs.select{|k,v| k.to_s.include?("#") }.collect{|p| p * "=" }
       memory_inputs << provided_inputs[:load_inputs]
@@ -41,7 +42,7 @@ module Task
       memory_inputs = provided_inputs
     end
 
-    Persist.memory("Task job #{self.name} #{id}", other_options: {task: self, id: id, provided_inputs: memory_inputs}) do 
+    Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {task: self.name, id: id, provided_inputs: memory_inputs}) do
       provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
       provided_inputs = {} if provided_inputs.nil?
       IndiferentHash.setup(provided_inputs)

data/lib/scout/workflow.rb

@@ -7,13 +7,21 @@ require_relative 'workflow/usage'
 require_relative 'workflow/deployment'
 require_relative 'workflow/exceptions'
 require_relative 'workflow/path'
+require_relative 'workflow/entity'
+require_relative 'workflow/export'
+require_relative 'workflow/persist'
 
 require 'scout/resource'
 require 'scout/resource/scout'
 
 module Workflow
   class << self
-    attr_accessor :workflows, :main, :workflow_dir, :autoinstall, :workflow_repo
+    attr_accessor :workflows, :main, :workflow_dir, :autoinstall, :workflow_repo, :job_cache
+
+    def job_cache
+      @job_cache ||={}
+    end
+
     def workflows
       @workflows ||= []
     end
@@ -50,7 +58,7 @@ module Workflow
 
   end
 
-  attr_accessor :libdir
+  attr_accessor :libdir, :knowledge_base
 
   def self.extended(base)
     self.workflows << base
@@ -70,7 +78,7 @@ module Workflow
 
   def self.install_workflow(workflow, base_repo_url = nil)
     case
-    when File.exist?(workflow)
+    when Open.exist?(workflow)
       update_workflow_dir(workflow)
     else
       Misc.in_dir(self.workflow_dir) do

data/lib/scout-gear.rb

@@ -1,5 +1,9 @@
 require 'scout-essentials'
 require_relative 'scout/tsv'
-require_relative 'scout/offsite'
 
 Path.path_maps[:scout_gear] = File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
+
+Persist.cache_dir     = Scout.var.cache.persistence
+TmpFile.tmpdir        = Scout.tmp.find :user
+Resource.default_resource = Scout
+

data/scout-gear.gemspec

@@ -2,17 +2,17 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.7.4 ruby lib
+# stub: scout-gear 10.7.6 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.7.4".freeze
+  s.version = "10.7.6".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-03-10"
-  s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
+  s.date = "2025-04-08"
+  s.description = "Scout gear: workflow, TSVs, persistence, entities, associations, and knowledge_bases.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
   s.extra_rdoc_files = [
@@ -45,17 +45,13 @@ Gem::Specification.new do |s|
     "lib/scout/entity/object.rb",
     "lib/scout/entity/property.rb",
     "lib/scout/knowledge_base.rb",
+    "lib/scout/knowledge_base/description.rb",
     "lib/scout/knowledge_base/enrichment.rb",
     "lib/scout/knowledge_base/entity.rb",
     "lib/scout/knowledge_base/list.rb",
     "lib/scout/knowledge_base/query.rb",
     "lib/scout/knowledge_base/registry.rb",
     "lib/scout/knowledge_base/traverse.rb",
-    "lib/scout/offsite.rb",
-    "lib/scout/offsite/exceptions.rb",
-    "lib/scout/offsite/ssh.rb",
-    "lib/scout/offsite/step.rb",
-    "lib/scout/offsite/sync.rb",
     "lib/scout/persist/engine.rb",
     "lib/scout/persist/engine/fix_width_table.rb",
     "lib/scout/persist/engine/packed_index.rb",
@@ -103,10 +99,14 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/definition.rb",
     "lib/scout/workflow/deployment.rb",
     "lib/scout/workflow/deployment/orchestrator.rb",
+    "lib/scout/workflow/deployment/queue.rb",
     "lib/scout/workflow/deployment/trace.rb",
     "lib/scout/workflow/documentation.rb",
+    "lib/scout/workflow/entity.rb",
     "lib/scout/workflow/exceptions.rb",
+    "lib/scout/workflow/export.rb",
     "lib/scout/workflow/path.rb",
+    "lib/scout/workflow/persist.rb",
     "lib/scout/workflow/step.rb",
     "lib/scout/workflow/step/archive.rb",
     "lib/scout/workflow/step/children.rb",
@@ -141,14 +141,15 @@ Gem::Specification.new do |s|
     "scout_commands/kb/show",
     "scout_commands/kb/traverse",
     "scout_commands/log",
-    "scout_commands/offsite",
     "scout_commands/rbbt",
     "scout_commands/resource/produce",
     "scout_commands/template",
     "scout_commands/update",
+    "scout_commands/workflow/cmd",
     "scout_commands/workflow/info",
     "scout_commands/workflow/install",
     "scout_commands/workflow/list",
+    "scout_commands/workflow/process",
     "scout_commands/workflow/prov",
     "scout_commands/workflow/task",
     "scout_commands/workflow/trace",
@@ -158,6 +159,7 @@ Gem::Specification.new do |s|
     "share/software/install_helpers",
     "share/templates/command",
     "share/templates/workflow.rb",
+    "test/data/person/README.md",
     "test/data/person/brothers",
     "test/data/person/identifiers",
     "test/data/person/marriages",
@@ -170,16 +172,13 @@ Gem::Specification.new do |s|
     "test/scout/entity/test_named_array.rb",
     "test/scout/entity/test_object.rb",
     "test/scout/entity/test_property.rb",
+    "test/scout/knowledge_base/test_description.rb",
     "test/scout/knowledge_base/test_enrichment.rb",
     "test/scout/knowledge_base/test_entity.rb",
     "test/scout/knowledge_base/test_list.rb",
     "test/scout/knowledge_base/test_query.rb",
     "test/scout/knowledge_base/test_registry.rb",
     "test/scout/knowledge_base/test_traverse.rb",
-    "test/scout/offsite/test_ssh.rb",
-    "test/scout/offsite/test_step.rb",
-    "test/scout/offsite/test_sync.rb",
-    "test/scout/offsite/test_task.rb",
     "test/scout/persist/engine/test_fix_width_table.rb",
     "test/scout/persist/engine/test_packed_index.rb",
     "test/scout/persist/engine/test_sharder.rb",
@@ -197,7 +196,6 @@ Gem::Specification.new do |s|
     "test/scout/test_association.rb",
     "test/scout/test_entity.rb",
     "test/scout/test_knowledge_base.rb",
-    "test/scout/test_offsite.rb",
     "test/scout/test_semaphore.rb",
     "test/scout/test_tsv.rb",
     "test/scout/test_work_queue.rb",
@@ -240,6 +238,7 @@ Gem::Specification.new do |s|
     "test/scout/workflow/task/test_inputs.rb",
     "test/scout/workflow/test_definition.rb",
     "test/scout/workflow/test_documentation.rb",
+    "test/scout/workflow/test_entity.rb",
     "test/scout/workflow/test_path.rb",
     "test/scout/workflow/test_step.rb",
     "test/scout/workflow/test_task.rb",
@@ -260,9 +259,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0".freeze])
   s.add_runtime_dependency(%q<matrix>.freeze, [">= 0".freeze])
   s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0".freeze])
-  s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12".freeze])
-  s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0".freeze])
   s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0".freeze])
-  s.add_development_dependency(%q<simplecov>.freeze, [">= 0".freeze])
 end
 

data/scout_commands/kb/config

@@ -12,6 +12,7 @@ Configure a knowlege base
 $ #{$0} [<options>] <name> [knowledge_base]
 
 -h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
 -i--identifier_files* Identifier files separated by ','
 -n--namespace* Namespace
 EOF
@@ -26,8 +27,10 @@ end
 
 knowledge_base_name = ARGV.first
 
+knowledge_base_name ||= options[:knowledge_base] || :default
 knowledge_base = KnowledgeBase.load knowledge_base_name || :default
 
 knowledge_base.namespace = options[:namespace] if options.include? :namespace
 knowledge_base.identifier_files += options[:identifier_files].split(",") if options.include? :identifier_files
+knowledge_base.identifier_files.uniq!
 knowledge_base.save

data/scout_commands/kb/list

@@ -12,6 +12,7 @@ Description of the tool
 $ #{$0} [<options>] <filename> [<other|->]*
 
 -h--help Print this help
+-kb--knowledge_base* Knowlege base name (or :default)
 EOF
 if options[:help]
   if defined? scout_usage

data/scout_commands/kb/query

@@ -34,7 +34,8 @@ raise MissingParameterException, :name if name.nil?
 raise MissingParameterException, :entity if entity.nil?
 
 
-knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+knowledge_base = IndiferentHash.process_options options, :knowledge_base
+knowledge_base = KnowledgeBase.load knowledge_base || :default
 
 list = IndiferentHash.process_options options, :list
 

data/scout_commands/kb/register

@@ -19,6 +19,7 @@ $ #{$0} [<options>] <name> <filename>
 -n--namespace* Namespace
 -i--identifiers* Identifiers
 -u--undirected
+-d--description* Database description
 EOF
 if options.delete :help
   if defined? scout_usage
@@ -35,7 +36,8 @@ name, file = ARGV
 raise MissingParameterException, :name if name.nil?
 raise MissingParameterException, :file if file.nil?
 
-knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+knowledge_base = IndiferentHash.process_options options, :knowledge_base
+knowledge_base = KnowledgeBase.load knowledge_base || :default
 
 options[:fields] = options[:fields].split(/,\s*/) if options[:fields]
 file = Scout.identify(File.expand_path(file))

data/scout_commands/kb/show

@@ -23,7 +23,8 @@ if options[:help]
   exit 0
 end
 
-knowledge_base = KnowledgeBase.load options[:knowledge_base] || :default
+knowledge_base = IndiferentHash.process_options options, :knowledge_base
+knowledge_base = KnowledgeBase.load knowledge_base || :default
 
 name = ARGV.first
 
@@ -31,7 +32,8 @@ if name.nil?
   puts knowledge_base.all_databases * "\n"
 else
 
-  raise ParameterException "Database #{name} not found Options: #{Log.fingerprint knowledge_base.all_databases}" unless knowledge_base.include? name
+  raise ParameterException, "Database #{name} not found Options: #{Log.fingerprint knowledge_base.all_databases}" unless knowledge_base.include? name
 
+  puts knowledge_base.markdown(name)
   Log.tsv knowledge_base.get_database(name, options)
 end