scout-gear 10.4.0 → 10.7.0

data/lib/scout/workflow/task.rb

@@ -1,12 +1,12 @@
-require 'scout/meta_extension'
+require 'scout/annotation'
 require 'scout/named_array'
 require_relative 'step'
 require_relative 'task/inputs'
 require_relative 'task/dependencies'
 
 module Task
-  extend MetaExtension
-  extension_attr :name, :type, :inputs, :deps, :directory, :description, :returns, :extension, :workflow
+  extend Annotation
+  annotation :name, :type, :inputs, :deps, :directory, :description, :returns, :extension, :workflow
 
   DEFAULT_NAME = "Default"
 
@@ -22,15 +22,6 @@ module Task
     @inputs ||= []
   end
 
-  def recursive_inputs
-    return inputs if deps.nil?
-    deps.inject(inputs) do |acc,dep|
-      workflow, task = dep
-      next acc if workflow.nil? || task.nil?
-      acc += workflow.tasks[task].recursive_inputs
-    end
-  end
-
   def directory
     @directory ||= Task.default_directory
   end
@@ -39,75 +30,81 @@ module Task
     binding.instance_exec(*inputs, &self)
   end
 
-  def job(id = DEFAULT_NAME, provided_inputs = nil )
-    provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
-    provided_inputs = {} if provided_inputs.nil?
-    IndiferentHash.setup(provided_inputs)
-    id = DEFAULT_NAME if id.nil?
+  def job(id = nil, provided_inputs = nil)
+    Persist.memory("Task job", other_options: {task: self, id: id, provided_inputs: provided_inputs}) do 
+      provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
+      provided_inputs = {} if provided_inputs.nil?
+      IndiferentHash.setup(provided_inputs)
 
+      if id.nil?
+        inputs.each do |name,type,desc,default,input_options|
+          next unless input_options && input_options[:jobname]
+          id = provided_inputs[name] || default
+        end
+        id = DEFAULT_NAME if id.nil?
+      end
 
-    missing_inputs = []
-    self.inputs.each do |input,type,desc,val,options|
-      next unless options && options[:required]
-      missing_inputs << input unless provided_inputs.include?(input)
-    end if self.inputs
+      missing_inputs = []
+      self.inputs.each do |input,type,desc,val,options|
+        next unless options && options[:required]
+        missing_inputs << input unless provided_inputs.include?(input)
+      end if self.inputs
 
-    if missing_inputs.length == 1
-      raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
-    end
+      if missing_inputs.length == 1
+        raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
+      end
 
-    if missing_inputs.length > 1
-      raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
-    end
+      if missing_inputs.length > 1
+        raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
+      end
 
-    provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
+      provided_inputs = load_inputs(provided_inputs.delete(:load_inputs)).merge(provided_inputs) if Hash === provided_inputs && provided_inputs[:load_inputs]
 
-    inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
+      job_inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
 
-    compute = {}
-    dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
+      compute = {}
+      dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
-    #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
+      #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
-    non_default_inputs.uniq!
+      non_default_inputs.uniq!
 
-    if non_default_inputs.any?
-      hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
-      name = [id, hash] * "_"
-    else
-      name = id
-    end
+      if non_default_inputs.any?
+        hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
+        name = [id, hash] * "_"
+      else
+        name = id
+      end
 
-    extension = self.extension
-    if extension == :dep_task
-      extension = nil
-      if dependencies.any?
-        dep_basename = File.basename(dependencies.last.path)
-        if dep_basename.include? "."
-          parts = dep_basename.split(".")
-          extension = [parts.pop]
-          while parts.last.length <= 4
-            extension << parts.pop
+      extension = self.extension
+      if extension == :dep_task
+        extension = nil
+        if dependencies.any?
+          dep_basename = File.basename(dependencies.last.path)
+          if dep_basename.include? "."
+            parts = dep_basename.split(".")
+            extension = [parts.pop]
+            while parts.last.length <= 4
+              extension << parts.pop
+            end
+            extension = extension.reverse * "."
           end
-          extension = extension.reverse * "."
         end
       end
-    end
 
 
-    path = directory[name]
+      path = directory[name]
 
-    path = path.set_extension(extension) if extension
+      path = path.set_extension(extension) if extension
 
-    Persist.memory(path) do 
       if hash
         Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
       else
         Log.debug "ID #{self.name} #{id} - Clean"
       end
-      NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
+      NamedArray.setup(job_inputs, @inputs.collect{|i| i[0] }) if @inputs
       step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
-      Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
+      Step.new path.find, job_inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
     end
   end
 

data/lib/scout/workflow/usage.rb

@@ -114,6 +114,7 @@ module Task
   def get_SOPT
     sopt_option_string = self.SOPT_str
     job_options = SOPT.get sopt_option_string
+
     recursive_inputs.uniq.each do |name,type|
       next unless type.to_s.include?('array')
       if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
@@ -285,7 +286,6 @@ module Workflow
         task = self.tasks[task_name]
       end
 
-      #dependencies = self.rec_dependencies(task_name).collect{|dep_name| Array === dep_name ? dep_name.first.tasks[dep_name[1].to_sym] : self.tasks[dep_name.to_sym]}
       str.puts task.usage(self, self.recursive_deps(task_name))
 
       dep_tree = {[self, task_name] => dep_tree(task_name)}

data/lib/scout/workflow/util.rb

@@ -8,6 +8,10 @@ module Workflow
     mod
   end
 
+  def self.installed_workflows
+    Path.setup("workflows").glob_all("*").collect{|f| File.basename(f) }.uniq
+  end
+
   def find_in_dependencies(name, dependencies)
     name = name.to_sym
     dependencies.select{|dep| dep.task_name.to_sym == name }

data/lib/scout/workflow.rb

@@ -11,13 +11,45 @@ require 'scout/resource/scout'
 
 module Workflow
   class << self
-    attr_accessor :workflows, :main
+    attr_accessor :workflows, :main, :workflow_dir, :autoinstall, :workflow_repo
     def workflows
       @workflows ||= []
     end
+
+    def workflow_dir
+      @workflow_dir || 
+        ENV["SCOUT_WORKFLOW_DIR"] || 
+        begin 
+          workflow_dir_config = Path.setup("etc/workflow_dir")
+          if workflow_dir_config.exists?
+            Path.setup(workflow_dir_config.read.strip)
+          else
+            Path.setup('workflows').find(:user)
+          end
+        end
+    end
+
+    def workflow_repo
+      @workflow_repo || 
+        ENV["SCOUT_WORKFLOW_REPO"] || 
+        begin 
+          workflow_repo_config = Path.setup("etc/workflow_repo")
+          if workflow_repo_config.exists?
+            workflow_repo_config.read.strip
+          else
+            'https://github.com/Scout-Workflows/'
+          end
+        end
+    end
+
+    def autoinstall
+      @autoinstall || ENV["SCOUT_WORKFLOW_AUTOINSTALL"] = "true"
+    end
+
   end
 
   attr_accessor :libdir
+
   def self.extended(base)
     self.workflows << base
     libdir = Path.caller_lib_dir
@@ -25,22 +57,87 @@ module Workflow
     base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
   end
 
+  def self.update_workflow_dir(workflow_dir)
+    Misc.in_dir(workflow_dir) do
+      Log.info "Updating: " + workflow_dir
+      `git pull`
+      `git submodule init`
+      `git submodule update`
+    end
+  end
+
+  def self.install_workflow(workflow, base_repo_url = nil)
+    case
+    when File.exist?(workflow)
+      update_workflow_dir(workflow)
+    else
+      Misc.in_dir(self.workflow_dir) do
+        Log.info "Installing: " + workflow
+
+        repo_base_url ||= self.workflow_repo
+
+
+        if repo_base_url.include?(workflow) or repo_base_url.include?(Misc.snake_case(workflow))
+          repo = repo_base_url
+        else
+          begin
+            repo = File.join(repo_base_url, workflow + '.git')
+            CMD.cmd("wget '#{repo}' -O /dev/null").read
+          rescue
+            Log.debug "Workflow repo does not exist, trying snake_case: #{ repo }"
+            begin
+              repo = File.join(repo_base_url, Misc.snake_case(workflow) + '.git')
+              CMD.cmd("wget '#{repo}' -O /dev/null").read
+            rescue
+              raise "Workflow repo does not exist: #{ repo }"
+            end
+          end
+        end
+
+        Log.warn "Cloning #{ repo }"
+        Misc.insist do
+          `git clone "#{repo}" #{ Misc.snake_case(workflow) }`
+          raise unless $?.success?
+        end
+        Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
+        Misc.in_dir(Misc.snake_case(workflow)) do
+          `git submodule init`
+          `git submodule update`
+        end
+      end
+    end
+  end
+
+  def self.require_workflow_file(file)
+    file = file.find if Path === file
+    $LOAD_PATH.unshift(File.join(File.dirname(file), 'lib'))
+    load file
+  end
+
   def self.require_workflow(workflow_name_orig)
     first = nil
     workflow_name_orig.split("+").each do |complete_workflow_name|
       self.main = nil
-      workflow_name, *subworkflows = complete_workflow_name.split("::")
-      workflow = workflow_name
-      workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
-      workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
-      workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
-      if Open.exists?(workflow)
-        self.main = nil
-        workflow = workflow.find if Path === workflow
-        $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
-        load workflow
-      else
-        raise "Workflow #{workflow_name} not found"
+
+      Persist.memory(complete_workflow_name, prefix: "Workflow") do
+        begin
+          workflow_name, *subworkflows = complete_workflow_name.split("::")
+          workflow_file = workflow_name
+          workflow_file = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow_file)
+          workflow_file = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file)
+          workflow_file = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow_file)
+          if Open.exists?(workflow_file)
+            self.main = nil
+            require_workflow_file(workflow_file)
+          elsif autoinstall
+            install_workflow(workflow_name)
+            raise TryAgain
+          else
+            raise "Workflow #{workflow_name} not found"
+          end
+        rescue TryAgain
+          retry
+        end
       end
 
       current = begin

data/lib/scout-gear.rb

@@ -1,3 +1,5 @@
 require 'scout-essentials'
 require_relative 'scout/tsv'
 require_relative 'scout/offsite'
+
+Path.path_maps[:scout_gear] = File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")

data/lib/scout.rb

@@ -1,5 +1,4 @@
 require 'workflow-scout'
-require 'rbbt-scout'
 
 module Scout
   def self.version

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.4.0 ruby lib
+# stub: scout-gear 10.7.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.4.0"
+  s.version = "10.7.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-08-09"
+  s.date = "2024-06-05"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -33,26 +33,49 @@ Gem::Specification.new do |s|
     "lib/rbbt-scout.rb",
     "lib/scout-gear.rb",
     "lib/scout.rb",
+    "lib/scout/association.rb",
+    "lib/scout/association/fields.rb",
+    "lib/scout/association/index.rb",
+    "lib/scout/association/item.rb",
+    "lib/scout/association/util.rb",
+    "lib/scout/entity.rb",
+    "lib/scout/entity/format.rb",
+    "lib/scout/entity/identifiers.rb",
+    "lib/scout/entity/object.rb",
+    "lib/scout/entity/property.rb",
     "lib/scout/offsite.rb",
     "lib/scout/offsite/exceptions.rb",
     "lib/scout/offsite/ssh.rb",
     "lib/scout/offsite/step.rb",
     "lib/scout/offsite/sync.rb",
+    "lib/scout/persist/engine.rb",
+    "lib/scout/persist/engine/fix_width_table.rb",
+    "lib/scout/persist/engine/packed_index.rb",
+    "lib/scout/persist/engine/sharder.rb",
+    "lib/scout/persist/engine/tkrzw.rb",
+    "lib/scout/persist/engine/tokyocabinet.rb",
+    "lib/scout/persist/tsv.rb",
+    "lib/scout/persist/tsv/adapter.rb",
+    "lib/scout/persist/tsv/adapter/base.rb",
+    "lib/scout/persist/tsv/adapter/fix_width_table.rb",
+    "lib/scout/persist/tsv/adapter/packed_index.rb",
+    "lib/scout/persist/tsv/adapter/sharder.rb",
+    "lib/scout/persist/tsv/adapter/tkrzw.rb",
+    "lib/scout/persist/tsv/adapter/tokyocabinet.rb",
+    "lib/scout/persist/tsv/serialize.rb",
     "lib/scout/semaphore.rb",
     "lib/scout/tsv.rb",
+    "lib/scout/tsv/annotation.rb",
+    "lib/scout/tsv/annotation/repo.rb",
     "lib/scout/tsv/attach.rb",
     "lib/scout/tsv/change_id.rb",
+    "lib/scout/tsv/change_id/translate.rb",
+    "lib/scout/tsv/csv.rb",
     "lib/scout/tsv/dumper.rb",
     "lib/scout/tsv/index.rb",
     "lib/scout/tsv/open.rb",
     "lib/scout/tsv/parser.rb",
     "lib/scout/tsv/path.rb",
-    "lib/scout/tsv/persist.rb",
-    "lib/scout/tsv/persist/adapter.rb",
-    "lib/scout/tsv/persist/fix_width_table.rb",
-    "lib/scout/tsv/persist/serialize.rb",
-    "lib/scout/tsv/persist/tkrzw.rb",
-    "lib/scout/tsv/persist/tokyocabinet.rb",
     "lib/scout/tsv/stream.rb",
     "lib/scout/tsv/transformer.rb",
     "lib/scout/tsv/traverse.rb",
@@ -72,8 +95,11 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/definition.rb",
     "lib/scout/workflow/deployment.rb",
     "lib/scout/workflow/deployment/orchestrator.rb",
+    "lib/scout/workflow/deployment/trace.rb",
     "lib/scout/workflow/documentation.rb",
     "lib/scout/workflow/step.rb",
+    "lib/scout/workflow/step/archive.rb",
+    "lib/scout/workflow/step/children.rb",
     "lib/scout/workflow/step/config.rb",
     "lib/scout/workflow/step/dependencies.rb",
     "lib/scout/workflow/step/file.rb",
@@ -114,26 +140,52 @@ Gem::Specification.new do |s|
     "share/software/install_helpers",
     "share/templates/command",
     "share/templates/workflow.rb",
+    "test/data/person/brothers",
+    "test/data/person/identifiers",
+    "test/data/person/marriages",
+    "test/data/person/parents",
+    "test/scout/association/test_fields.rb",
+    "test/scout/association/test_index.rb",
+    "test/scout/association/test_item.rb",
+    "test/scout/entity/test_format.rb",
+    "test/scout/entity/test_identifiers.rb",
+    "test/scout/entity/test_object.rb",
+    "test/scout/entity/test_property.rb",
     "test/scout/offsite/test_ssh.rb",
     "test/scout/offsite/test_step.rb",
     "test/scout/offsite/test_sync.rb",
     "test/scout/offsite/test_task.rb",
+    "test/scout/persist/engine/test_fix_width_table.rb",
+    "test/scout/persist/engine/test_packed_index.rb",
+    "test/scout/persist/engine/test_sharder.rb",
+    "test/scout/persist/engine/test_tkrzw.rb",
+    "test/scout/persist/engine/test_tokyocabinet.rb",
+    "test/scout/persist/test_tsv.rb",
+    "test/scout/persist/tsv/adapter/test_base.rb",
+    "test/scout/persist/tsv/adapter/test_fix_width_table.rb",
+    "test/scout/persist/tsv/adapter/test_packed_index.rb",
+    "test/scout/persist/tsv/adapter/test_serialize.rb",
+    "test/scout/persist/tsv/adapter/test_sharder.rb",
+    "test/scout/persist/tsv/adapter/test_tkrzw.rb",
+    "test/scout/persist/tsv/adapter/test_tokyocabinet.rb",
+    "test/scout/persist/tsv/test_serialize.rb",
+    "test/scout/test_association.rb",
+    "test/scout/test_entity.rb",
     "test/scout/test_offsite.rb",
     "test/scout/test_semaphore.rb",
     "test/scout/test_tsv.rb",
     "test/scout/test_work_queue.rb",
     "test/scout/test_workflow.rb",
-    "test/scout/tsv/persist/test_adapter.rb",
-    "test/scout/tsv/persist/test_fix_width_table.rb",
-    "test/scout/tsv/persist/test_tkrzw.rb",
-    "test/scout/tsv/persist/test_tokyocabinet.rb",
+    "test/scout/tsv/annotation/test_repo.rb",
+    "test/scout/tsv/change_id/test_translate.rb",
+    "test/scout/tsv/test_annotation.rb",
     "test/scout/tsv/test_attach.rb",
     "test/scout/tsv/test_change_id.rb",
+    "test/scout/tsv/test_csv.rb",
     "test/scout/tsv/test_dumper.rb",
     "test/scout/tsv/test_index.rb",
     "test/scout/tsv/test_open.rb",
     "test/scout/tsv/test_parser.rb",
-    "test/scout/tsv/test_persist.rb",
     "test/scout/tsv/test_stream.rb",
     "test/scout/tsv/test_transformer.rb",
     "test/scout/tsv/test_traverse.rb",
@@ -148,6 +200,9 @@ Gem::Specification.new do |s|
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
     "test/scout/workflow/deployment/test_orchestrator.rb",
+    "test/scout/workflow/deployment/test_trace.rb",
+    "test/scout/workflow/step/test_archive.rb",
+    "test/scout/workflow/step/test_children.rb",
     "test/scout/workflow/step/test_dependencies.rb",
     "test/scout/workflow/step/test_info.rb",
     "test/scout/workflow/step/test_load.rb",
@@ -167,18 +222,18 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.5.0.dev".freeze
+  s.rubygems_version = "3.5.10".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4
 
-  s.add_runtime_dependency(%q<scout-essentials>.freeze, [">= 0"])
-  s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0"])
-  s.add_runtime_dependency(%q<matrix>.freeze, [">= 0"])
-  s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0"])
-  s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
-  s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
-  s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
-  s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<scout-essentials>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<matrix>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0".freeze])
+  s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12".freeze])
+  s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0".freeze])
+  s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0".freeze])
+  s.add_development_dependency(%q<simplecov>.freeze, [">= 0".freeze])
 end
 

data/scout_commands/rbbt

@@ -1,5 +1,7 @@
 #!/usr/bin/env ruby
 
+require 'rbbt-scout'
+
 case
 when File.exist?(file = File.join(__dir__, '../modules/rbbt-util/bin/rbbt'))
   $LOAD_PATH.unshift(lib_dir = File.join(file, '../..', 'lib'))

data/test/data/person/brothers

@@ -0,0 +1,4 @@
+#: :sep=,#:type=:list
+#Older,Younger
+Clei,Guille
+Isa,Miki

data/test/data/person/identifiers

@@ -0,0 +1,10 @@
+#: :sep=,#:type=:list
+#Name,Alias,ID
+Miguel,Miki,001
+Cleia,Clei,002
+Isabel,Isa,003
+Guillermo,Guille,004
+Juan Luis,Juan,005
+Maria Luz,Mariluz,006
+Domingo,Domingo,007
+Gloria,Gloria,008

data/test/data/person/marriages

@@ -0,0 +1,3 @@
+#: :type=:list
+#Husband (ID)	Wife (ID)	Date
+001	002	2021

data/test/data/person/parents

@@ -0,0 +1,6 @@
+#: :sep=,#:type=:double
+#Child (Alias),Parent (Alias),Type of parent
+Miki,Juan|Mariluz,father|mother
+Isa,Juan|Mariluz,father|mother
+Guille,Domingo|Gloria,father|mother
+Clei,Domingo|Gloria,father|mother