scout-gear 10.8.4 → 10.9.0

data/lib/scout/knowledge_base/list.rb

@@ -12,7 +12,7 @@ class KnowledgeBase
       if entity_type.to_s == "simple"
         path = dir.lists[entity_type.to_s][id]
       else
-        path = dir.lists[entity_type.to_s][id + ".tsv"]
+        path = dir.lists[entity_type.to_s][id].find_with_extension("tsv")
       end
     else
       path = dir.lists.glob("*/#{id}").first
@@ -33,6 +33,7 @@ class KnowledgeBase
     Open.lock path do
       begin
         if AnnotatedArray === list
+          path = path.set_extension('tsv')
           Open.write(path, Annotation.tsv(list, :all).to_s)
         else
           Open.write(path, list * "\n")
@@ -60,7 +61,11 @@ class KnowledgeBase
         list.extend AnnotatedArray
         list
       else
-        path.list
+        list = path.list
+        if entity_type
+          Entity.prepare_entity(list, entity_type)
+        end
+        list
       end
     rescue
       Log.exception $!

data/lib/scout/knowledge_base/registry.rb

@@ -84,7 +84,7 @@ class KnowledgeBase
           key = name.to_s + "_" + fp
         end
 
-        Persist.memory("Index:" << [key, dir] * "@") do
+        Persist.memory("Index:" + [key, dir] * "@") do
           options = options.dup
 
           persist_dir = dir
@@ -148,7 +148,7 @@ class KnowledgeBase
         options[:namespace] ||= self.namespace unless self.namespace.nil?
 
         key += '.database'
-        Persist.memory("Database:" << [key, dir] * "@") do
+        Persist.memory("Database:" + [key, dir] * "@") do
           options = options.dup
 
           persist_dir = dir

data/lib/scout/knowledge_base.rb

@@ -59,8 +59,26 @@ class KnowledgeBase
   end
 
   def self.load(dir)
-    dir = Path.setup("var").knowledge_base[dir.to_s] if Symbol === dir
-    kb = KnowledgeBase.new dir
+    kb = case dir
+         when Path
+           KnowledgeBase.new dir
+         when Symbol
+           dir = Path.setup("var").knowledge_base[dir.to_s] if Symbol === dir
+           kb = KnowledgeBase.new dir
+         when Workflow
+           raise if dir.knowledge_base.nil?
+           kb = dir.knowledge_base
+         when String
+           if Workflow.list.include? dir
+             workflow = Workflow.require_workflow dir
+             kb = workflow.knowledge_base
+           elsif dir =~ /^\w+$/
+             load(dir.to_sym)
+           else
+             kb = KnowledgeBase.new dir
+           end
+         end
+
     kb.load
     kb
   end

data/lib/scout/monitor.rb

@@ -2,14 +2,18 @@ require 'scout'
 
 module Scout
 
-  LOCK_DIRS = Scout.tmp.tsv_open_locks.find_all + Scout.tmp.persist_locks.find_all + Scout.tmp.sensiblewrite_locks.find_all +
-    Scout.tmp.produce_locks.find_all + Scout.tmp.step_info_locks.find_all
-
   SENSIBLE_WRITE_DIRS = Open.sensible_write_dir.find_all
 
-  PERSIST_DIRS    = Scout.share.find_all  + Scout.var.cache.persistence.find_all
+  LOCK_DIRS = Path.setup('tmp/tsv_open_locks').find_all +
+    Path.setup('tmp/tsv_locks').find_all +
+    Path.setup('tmp/persist_locks').find_all +
+    Path.setup('tmp/sensible_write_locks').find_all +
+    Path.setup('tmp/produce_locks').find_all +
+    Path.setup('tmp/step_info_locks').find_all
+
+  PERSIST_DIRS    = Path.setup('share').find_all  + Path.setup('var/cache/persistence').find_all
 
-  JOB_DIRS = Scout.var.jobs.find_all
+  JOB_DIRS = Path.setup('var/jobs').find_all
 
   MUTEX_FOR_THREAD_EXCLUSIVE = Mutex.new
 
@@ -68,7 +72,7 @@ module Scout
           lock_info[f][:ppid] = info[:ppid]
         end
       rescue Exception
-        Log.exception $!
+        Log.warn $!.message
       end
     end
     lock_info

data/lib/scout/persist/engine/packed_index.rb

@@ -9,7 +9,7 @@ class PackedIndex
   }
 
   def self.process_mask(mask)
-    str = ""
+    str = "".dup
     size = 0
     mask.each do |e|
       if ELEMS.include? e
@@ -50,7 +50,7 @@ class PackedIndex
       @mask = @stream.read(mask_length)
       @offset = @mask.length + 8
     end
-    @nil_string = "NIL" << ("-" * (@item_size - 3))
+    @nil_string = "NIL" + ("-" * (@item_size - 3))
   end
 
   def file

data/lib/scout/persist/engine/sharder.rb

@@ -37,7 +37,7 @@ class Sharder
       databases[shard]
     else
       database = databases[shard] ||= begin
-                                        path = File.join(persistence_path, 'shard-' << shard.to_s)
+                                        path = File.join(persistence_path, 'shard-' + shard.to_s)
                                         (writable or File.exist?(path)) ? Persist.open_database(path, (File.exist?(path) ? false : writable), :clean, db_type, @persist_options) : nil
                                       end
       Log.warn "Database #{ path } missing" if database.nil?

data/lib/scout/persist/tsv.rb

@@ -63,6 +63,7 @@ module Persist
 
         yield(data)
         data.save_annotation_hash if Annotation.is_annotated?(data)
+        data.read if data.respond_to?(:read)
         data
       end
     end

data/lib/scout/semaphore.rb

@@ -75,7 +75,7 @@ if continue
 
     def self.with_semaphore(size, file = nil)
       if file.nil?
-        file = "/scout-" << Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
+        file = "/scout-" + Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
       else
         file = file.gsub('/', '_') if file
       end

data/lib/scout/tsv/dumper.rb

@@ -16,7 +16,7 @@ module TSV
       if String === preamble
         preamble_str = preamble
       elsif preamble && options.values.compact.any?
-        preamble_str = "#: " << IndiferentHash.hash2string(options.merge(serializer: nil))
+        preamble_str = "#: " + IndiferentHash.hash2string(options.merge(serializer: nil))
       else
         preamble_str = nil
       end
@@ -81,7 +81,7 @@ module TSV
       header = Dumper.header(@options.merge(type: @type, sep: @sep, preamble: preamble))
       @mutex.synchronize do
         @initialized = true
-        @sin << header << "\n" if header and ! header.empty?
+        @sin << header + "\n" if header and ! header.empty?
       end
     end
 
@@ -134,7 +134,7 @@ module TSV
     end
 
     def fingerprint
-      "Dumper:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+      "Dumper:{" + Log.fingerprint(self.all_fields|| []) << "}"
     end
 
     def digest_str

data/lib/scout/tsv/open.rb

@@ -61,6 +61,7 @@ module Open
       end
 
       if into.respond_to?(:close)
+        obj = obj.find if Path === obj
         into_thread = Thread.new do 
           Thread.current.report_on_exception = false
           Thread.current["name"] = "Traverse into"

data/lib/scout/tsv/parser.rb

@@ -447,7 +447,7 @@ module TSV
     end
 
     def fingerprint
-      "Parser:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+      "Parser:{" + Log.fingerprint(self.all_fields|| []) << "}"
     end
 
     def digest_str

data/lib/scout/tsv/transformer.rb

@@ -3,6 +3,7 @@ module TSV
     attr_accessor :unnamed, :parser, :dumper, :namespace
 
     def initialize(parser, dumper = nil, unnamed: nil, namespace: nil)
+
       if TSV::Parser === parser
         @parser = parser
       elsif TSV === parser

data/lib/scout/tsv/util.rb

@@ -165,11 +165,11 @@ Example:
   end
 
   def fingerprint
-    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
+    "TSV:{" + Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
   end
 
   def digest_str
-    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << ";" << Log.fingerprint(self.values) << "}"
+    "TSV:{" + Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << ";" << Log.fingerprint(self.values) << "}"
   end
 
   def inspect

data/lib/scout/work_queue/socket.rb

@@ -9,7 +9,7 @@ class WorkQueue
 
       @serializer = serializer || Marshal
 
-      @key = "/" << rand(1000000000).to_s << '.' << Process.pid.to_s;
+      @key = "/" + rand(1000000000).to_s << '.' << Process.pid.to_s;
       @write_sem = @key + '.in'
       @read_sem = @key + '.out'
       Log.debug "Creating socket semaphores: #{@key}"

data/lib/scout/work_queue/worker.rb

@@ -27,9 +27,9 @@ class WorkQueue
       run do
         begin
           if output
-            Open.purge_pipes(output.swrite)
+            Open.purge_pipes(input.sread, output.swrite)
           else
-            Open.purge_pipes
+            Open.purge_pipes(input.sread)
           end
 
           while obj = input.read
@@ -43,9 +43,11 @@ class WorkQueue
         rescue DoneProcessing
         rescue Interrupt
         rescue Exception
-          output.write WorkerException.new($!, Process.pid)
-          exit -1
-        ensure
+          begin
+            output.write WorkerException.new($!, Process.pid)
+          ensure
+            exit -1
+          end
         end
         exit 0
       end

data/lib/scout/workflow/entity.rb

@@ -71,10 +71,31 @@ module EntityWorkflow
     end
 
     property_name = task_name.to_s.sub(/^(#{entity_name}_list|#{entity_name}|list)_/, '')
+    property_job_name = property_name + '_job'
+
+    property property_job_name => property_type do |*args|
+      job(task_name, *args)
+    end
+
     property property_name => property_type do |*args|
-      job = job(task_name, *args)
+      job = self.send(property_job_name)
+
+      job.join if job.running?
+
+      if job.error?
+        if job.recoverable_error?
+          job.clean
+        else
+          raise job.exception
+        end
+      end
+
+      job.run unless job.done?
+
+      job.load
       Array === job ? job.collect(&:run) : job.run
     end
+
   end
 
   def entity_task(task_name, *args, &block)

data/lib/scout/workflow/step/config.rb

@@ -8,10 +8,10 @@ class Step
     new_tokens = []
     if workflow
       workflow_name = workflow.name
-      new_tokens << ("workflow:" << workflow_name)
-      new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
+      new_tokens << ("workflow:" + workflow_name)
+      new_tokens << ("task:" + workflow_name << "#" << task_name.to_s)
     end
-    new_tokens << ("task:" << task_name.to_s)
+    new_tokens << ("task:" + task_name.to_s)
 
     Scout::Config.get(key, tokens + new_tokens, options)
   end

data/lib/scout/workflow/step/file.rb

@@ -13,6 +13,10 @@ class Step
                    end
   end
 
+  def files_dir=(dir)
+    @files_dir = dir
+  end
+
   def file(file = nil)
     dir = files_dir
     Path.setup(dir) unless Path === dir

data/lib/scout/workflow/step/info.rb

@@ -120,7 +120,7 @@ class Step
       if info.include?(key)
         case info[key]
         when Array
-          info[key].concat Array === value ? value : [value]
+          info[key].concat(Array === value ? value : [value])
         when Hash
           info[key].merge! value
         else
@@ -201,7 +201,13 @@ class Step
   end
 
   def exception
-    info[:exception]
+    return nil unless info[:exception]
+    begin
+      Marshal.load(Base64.decode64(info[:exception]))
+    rescue
+      Log.exception $!
+      nil
+    end
   end
 
   # Marshal Step

data/lib/scout/workflow/step.rb

@@ -16,7 +16,7 @@ require_relative 'step/archive'
 class Step 
 
   attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context = nil, &task)
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context: nil, &task)
     @path = path
     @inputs = inputs
     @dependencies = dependencies
@@ -112,17 +112,22 @@ class Step
 
   def exec
 
-    if inputs 
+    if inputs
       if Task === task
         types = task.inputs.collect{|name,type| type }
-        new_inputs = inputs.zip(types).collect{|input,info|  
+        new_inputs = inputs.zip(types).collect{|input,info|
           type, desc, default, options = info
           next input unless Step === input
           input.join if input.streaming?
           Task.format_input(input.join.path, type, options)
         }
       else
-        new_inputs = inputs.collect{|input|  
+        if Hash === inputs
+          new_inputs = inputs.values
+        else
+          new_inputs = inputs
+        end
+        new_inputs = new_inputs.collect{|input|
           Step === input ? input.load : input
         }
       end
@@ -218,7 +223,7 @@ class Step
 
       @result
     rescue Exception => e
-      merge_info :status => :error, :exception => e, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
+      merge_info :status => :error, :exception => Base64.encode64(Marshal.dump(e)), :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
       begin
         abort_dependencies
       ensure

data/lib/scout/workflow/task/inputs.rb

@@ -169,7 +169,7 @@ module Task
   end
 
   def load_inputs(directory)
-    if Open.exists?(directory) && ! Open.directory?(directory)
+    if Open.exists?(directory) && ! Open.directory?(directory) && ! Open.size(directory) == 0
       TmpFile.with_file do |tmp_directory|
         Misc.in_dir tmp_directory do
           CMD.cmd("tar xvfz '#{directory}'")

data/lib/scout/workflow/usage.rb

@@ -35,6 +35,7 @@ module Task
       str.puts SOPT.input_array_doc(inputs)
 
       inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
+        next unless options
         selects << [name, options[:select_options]] if options[:select_options]
       end
     end
@@ -194,9 +195,9 @@ module Workflow
       if child
         #·description << "->" << task_name.to_s
       elsif first
-        description << "" << task_name.to_s
+        description += "" + task_name.to_s
       else
-        description << ";" << task_name.to_s
+        description += ";" + task_name.to_s
       end
       
       seen << [workflow, task_name]

data/lib/scout/workflow/util.rb

@@ -16,5 +16,27 @@ module Workflow
     name = name.to_sym
     dependencies.select{|dep| dep.task_name.to_sym == name }
   end
+
+  def all_tasks
+    tasks.keys
+  end
+
+  def self.list
+    Path.setup('workflows').glob('*').collect{|p| p.basename }
+  end
+
+  def task_jobs_files(task_name)
+    self.directory[task_name].glob("**").
+      collect{|f| %w(info files).include?(f.get_extension) ? f.unset_extension : f }.
+      uniq
+  end
+
+  def task_jobs(task_name)
+    task_jobs_files(task_name).collect{|f| Step.load f }
+  end
+
+  def load_job(task_name, name)
+    Step.new self.directory[task_name][name]
+  end
 end
 

data/scout-gear.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.8.4 ruby lib
+# stub: scout-gear 10.9.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.8.4".freeze
+  s.version = "10.9.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.executables = ["scout".freeze]
   s.extra_rdoc_files = [
     "LICENSE.txt",
+    "README.md",
     "README.rdoc"
   ]
   s.files = [
@@ -24,12 +25,19 @@ Gem::Specification.new do |s|
     ".gitmodules",
     ".vimproject",
     "LICENSE.txt",
+    "README.md",
     "README.rdoc",
     "Rakefile",
     "VERSION",
     "bin/scout",
-    "doc/lib/scout/path.md",
-    "doc/lib/scout/workflow/task.md",
+    "doc/Association.md",
+    "doc/Entity.md",
+    "doc/KnowledgeBase.md",
+    "doc/Persist.md",
+    "doc/Semaphore.md",
+    "doc/TSV.md",
+    "doc/WorkQueue.md",
+    "doc/Workflow.md",
     "lib/scout-gear.rb",
     "lib/scout.rb",
     "lib/scout/association.rb",
@@ -130,7 +138,9 @@ Gem::Specification.new do |s|
     "scout_commands/alias",
     "scout_commands/batch/clean",
     "scout_commands/batch/list",
+    "scout_commands/cat",
     "scout_commands/doc",
+    "scout_commands/entity",
     "scout_commands/find",
     "scout_commands/glob",
     "scout_commands/kb/config",
@@ -145,6 +155,7 @@ Gem::Specification.new do |s|
     "scout_commands/resource/produce",
     "scout_commands/resource/sync",
     "scout_commands/system/clean",
+    "scout_commands/system/status",
     "scout_commands/template",
     "scout_commands/update",
     "scout_commands/workflow/cmd",
@@ -252,7 +263,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.7".freeze
+  s.rubygems_version = "3.7.0.dev".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/cat

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+
+require 'scout-gear'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Cat a file
+
+$ #{$0} [<options>] (<resource> <path>|<path>)
+
+-h--help Print this help
+-r--requires* Files to require; 'all' for all in Scout.etc.requires
+-lw--load_workflow* Files to require; 'all' for all in Scout.etc.requires
+-w--where* Where to look for the path
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+resource, path = ARGV
+path, resource = resource, nil if path.nil?
+
+raise MissingParameterException.new(:path) if path.nil?
+
+case options[:workflows]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.workflows.list.each do |workflow|
+    Workflow.require_workflow file
+  end if Scout.etc.workflows.exists?
+else
+  options[:workflows].split(/[ ,;|]/).each do |workflow|
+    Workflow.require_workflow workflow
+  end
+end
+
+case options[:requires]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.requires.list.each do |file|
+    require file
+  end if Scout.etc.requires.exists?
+else
+  options[:requires].split(/[ ,;|]/).each do |file|
+    require file
+  end
+end
+
+location = nil
+where = options[:where]
+all = options[:all]
+
+begin
+  resource = Workflow.require_workflow resource
+rescue
+  begin
+    resource = Kernel.const_get(resource)
+  rescue
+    raise "Resource not found: #{ resource }"
+  end
+end if resource
+
+path = (resource || Scout)[path.dup]
+
+if where.nil? || where == 'all' || path.path_maps.include?(where.to_sym)
+  location = path.find(where)
+
+  if Array === location
+    file = location.first
+    puts location.first.read
+    puts location * "\n"
+  else
+    file = Open.exists?(location)? location : nil
+  end
+
+  puts file.read if file
+else
+  raise ParameterException, "Where '#{where}' not identified. Try scout-camp if looking for a remote file"
+end

data/scout_commands/doc

@@ -26,7 +26,9 @@ end
 
 module_name = ARGV.first
 if module_name.nil?
-  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
+  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f| 
+    f.find.unset_extension.sub(Scout.doc.lib.scout.find + "/", '')
+  } * "\n"
 else
   begin
     puts Scout.doc.lib.scout[module_name].find_with_extension('md', :lib).read