scout-gear 9.1.0 → 10.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bbe916f2d82030e26b1049319216f45341b7e7f7410df65e3953c0a733af2c21
-  data.tar.gz: 152b31b552f90b2bb06aed9bc3cdd12a3bff48746468e52b39e5754977a40f90
+  metadata.gz: ab089af23ec3cb6d23db1190f712d2a981db96e818572aa5b561227a9601b9b8
+  data.tar.gz: d72662def6d6b7207d015d7cea613a1882578efe201ed391f79d37027ab7c70e
 SHA512:
-  metadata.gz: 51f41961b2a95d847e4a60a70a2774c34098c4fa502818be8b9d5ec30ced53fce81d02862a1104498060c04a88138ca329ae82ba09fa264b6d9ec0daca80a3a4
-  data.tar.gz: 113d51ff9798416073a86debb07ce47708254a4594e5785cfc4543794c3ff4e09da8809c29d67f64d57aeacd30c1c2ad49998d5eca4f544f3381991c296db1c5
+  metadata.gz: d5b67fdc030ad2ed3dd4d9da47d7f12068b71f14a1ae390f65c411bc2e835ac2da1648fccb554cbda4a3a56bdf6b16fc63080620453e75481590622cfd883247
+  data.tar.gz: 97fa38ef8db895d3a8aa05d2e4e521bb98d28924b389429994f6e98bceb7bfa7ad74920676c17b8b124bf288700e80078909ea20a84c3c0b24efd0cf7b0632cb

data/.vimproject

@@ -28,81 +28,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   rbbt-scout.rb
   scout.rb
   scout=scout{
-   meta_extension.rb
-   exceptions.rb
-
-   misc.rb
-   misc=misc{
-    format.rb
-    insist.rb
-    digest.rb
-    filesystem.rb
-    monitor.rb
-    system.rb
-   }
-   named_array.rb
-   indiferent_hash.rb
-   indiferent_hash=indiferent_hash{
-    case_insensitive.rb
-    options.rb
-   }
-   log.rb
-   log=log{
-    color.rb
-    color_class.rb
-    fingerprint.rb
-    progress.rb
-    progress=progress{
-     report.rb
-     util.rb
-    }
-    trap.rb
-   }
-   tmpfile.rb
-   simple_opt.rb
-   simple_opt=simple_opt{
-    accessor.rb
-    doc.rb
-    parse.rb
-    get.rb
-    setup.rb
-   }
-   path.rb
-   path=path{
-    find.rb
-    util.rb
-    tmpfile.rb
-   }
-   concurrent_stream.rb
-   cmd.rb
-   open.rb
-   open=open{
-    lock.rb
-    lock=lock{
-     lockfile.rb
-    }
-    stream.rb
-    remote.rb
-    util.rb
-   }
-   resource.rb
-   resource=resource{
-    produce.rb
-    produce=produce{
-     rake.rb
-    }
-    software.rb
-    scout.rb
-    util.rb
-    path.rb
-   }
-   config.rb
-   persist.rb
-   persist=persist{
-    serialize.rb
-    open.rb
-    path.rb
-   }
    workflow.rb
    workflow=workflow{
     definition.rb
@@ -146,6 +71,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      process.rb
      unzip.rb
      reorder.rb
+     sort.rb
     }
     parser.rb
     dumper.rb
@@ -173,9 +99,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     sync.rb
     step.rb
    }
-   hpc=hpc{
-    slurm.rb
-   }
   }
  }
  test=test {

data/Rakefile

@@ -20,10 +20,9 @@ Juwelier::Tasks.new do |gem|
   # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
   #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
   #  gem.add_development_dependency 'rspec', '> 1.2.3'
-  gem.add_runtime_dependency 'term-ansicolor'
+  gem.add_runtime_dependency 'scout-essentials'
   gem.add_runtime_dependency 'net-ssh'
   gem.add_runtime_dependency 'matrix'
-  gem.add_runtime_dependency 'sys-proctable'
   gem.add_runtime_dependency 'RubyInline'
   #gem.add_runtime_dependency 'tokyocabinet'
 
@@ -36,6 +35,7 @@ Juwelier::RubygemsDotOrgTasks.new
 
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
+  ENV["SCOUT_LOG"] = "NONE"
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true

data/VERSION

@@ -1 +1 @@
-9.1.0
+10.1.0

data/lib/scout/offsite/exceptions.rb

@@ -0,0 +1,9 @@
+class SSHProcessFailed < StandardError
+  attr_accessor :host, :cmd
+  def initialize(host, cmd)
+    @host = host
+    @cmd = cmd
+    message = "SSH server #{host} failed cmd '#{cmd}'" 
+    super(message)
+  end
+end

data/lib/scout/offsite/ssh.rb

@@ -1,4 +1,5 @@
 require 'net/ssh'
+require_relative 'exceptions'
 
 class SSHLine
   class << self

data/lib/scout/semaphore.rb

@@ -8,6 +8,8 @@ end
 
 if continue
   module ScoutSemaphore
+    class SemaphoreInterrupted < TryAgain; end
+
     inline(:C) do |builder|
       builder.prefix <<-EOF
   #include <unistd.h>

data/lib/scout/tsv/open.rb

@@ -1,4 +1,4 @@
-require_relative '../open'
+require 'scout/open'
 require_relative '../work_queue'
 
 module MultipleResult

data/lib/scout/tsv/parser.rb

@@ -1,4 +1,4 @@
-require_relative '../named_array'
+require 'scout/named_array'
 module TSV
   def self.cast_value(value, cast)
     if Array === value

data/lib/scout/tsv/persist/adapter.rb

@@ -1,5 +1,5 @@
-require_relative '../../open/lock'
-require_relative '../../meta_extension'
+require 'scout/open/lock'
+require 'scout/meta_extension'
 require_relative 'serialize'
 
 module TSVAdapter

data/lib/scout/tsv/persist.rb

@@ -1,3 +1,4 @@
+require 'scout/persist'
 require_relative 'persist/adapter'
 require_relative 'persist/tokyocabinet'
 

data/lib/scout/tsv/util/sort.rb

@@ -0,0 +1,74 @@
+module TSV
+  def sort_by(field = nil, just_keys = false, &block)
+    field = :all if field.nil?
+
+    if field == :all
+      elems = collect
+    else
+      elems = []
+      case type
+      when :single
+        through :key, field do |key, field|
+          elems << [key, field]
+        end
+      when :list, :flat
+        through :key, field do |key, fields|
+          elems << [key, fields.first]
+        end
+      when :double
+        through :key, field do |key, fields|
+          elems << [key, fields.first]
+        end
+      end
+    end
+
+    if not block_given?
+      if fields == :all
+        if just_keys
+          keys = elems.sort_by{|key, value| key }.collect{|key, values| key}
+          keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
+        else
+          elems.sort_by{|key, value| key }
+        end
+      else
+        sorted = elems.sort do |a, b| 
+          a_value = a.last
+          b_value = b.last
+          a_empty = a_value.nil? or (a_value.respond_to?(:empty?) and a_value.empty?)
+          b_empty = b_value.nil? or (b_value.respond_to?(:empty?) and b_value.empty?)
+          case
+          when (a_empty and b_empty)
+            0
+          when a_empty
+            -1
+          when b_empty
+            1
+          when Array === a_value
+            if a_value.length == 1 and b_value.length == 1
+              a_value.first <=> b_value.first
+            else
+              a_value.length <=> b_value.length
+            end
+          else
+            a_value <=> b_value
+          end
+        end
+        if just_keys
+          keys = sorted.collect{|key, value| key}
+          keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
+          keys
+        else
+          sorted.collect{|key, value| [key, self[key]]}
+        end
+      end
+    else
+      if just_keys
+        keys = elems.sort_by(&block).collect{|key, value| key}
+        keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
+        keys
+      else
+        elems.sort_by(&block).collect{|key, value| [key, self[key]]}
+      end
+    end
+  end
+end

data/lib/scout/tsv/util.rb

@@ -6,6 +6,7 @@ require_relative 'util/process'
 require_relative 'util/select'
 require_relative 'util/reorder'
 require_relative 'util/unzip'
+require_relative 'util/sort'
 module TSV
   def self.identify_field(key_field, fields, name, strict: nil)
     return :key if name == :key || (! strict && NamedArray.field_match(key_field, name))

data/lib/scout/tsv.rb

@@ -1,4 +1,4 @@
-require_relative 'meta_extension'
+require 'scout/meta_extension'
 require_relative 'tsv/util'
 require_relative 'tsv/parser'
 require_relative 'tsv/dumper'

data/lib/scout/work_queue/exceptions.rb

@@ -0,0 +1,18 @@
+class DoneProcessing < Exception
+  attr_accessor :pid
+  def initialize(pid = Process.pid)
+    @pid = pid
+  end
+
+  def message
+    "Done processing pid #{pid}"
+  end
+end
+
+class WorkerException < ScoutException
+  attr_accessor :worker_exception, :pid
+  def initialize(worker_exception, pid)
+    @worker_exception = worker_exception
+    @pid = pid
+  end
+end

data/lib/scout/work_queue.rb

@@ -1,5 +1,6 @@
 require_relative 'work_queue/socket'
 require_relative 'work_queue/worker'
+require_relative 'work_queue/exceptions'
 require 'timeout'
 
 class WorkQueue

data/lib/scout/workflow/definition.rb

@@ -1,4 +1,4 @@
-require_relative '../meta_extension'
+require 'scout/meta_extension'
 
 module Workflow
   extend MetaExtension

data/lib/scout/workflow/step/config.rb

@@ -1,4 +1,4 @@
-require_relative '../../config'
+require 'scout/config'
 
 class Step
   def config(key, *tokens)

data/lib/scout/workflow/step/status.rb

@@ -31,6 +31,7 @@ class Step
     Open.rm tmp_path if Open.exist?(tmp_path)
     Open.rm info_file if Open.exist?(info_file)
     Open.rm_rf files_dir if Open.exist?(files_dir)
+    self
   end
 
   def self.clean(file)

data/lib/scout/workflow/step.rb

@@ -1,5 +1,5 @@
-require_relative '../path'
-require_relative '../persist'
+require 'scout/path'
+require 'scout/persist'
 require_relative 'step/info'
 require_relative 'step/status'
 require_relative 'step/load'
@@ -124,62 +124,62 @@ class Step
     return @result || self.load if done?
     prepare_dependencies
     run_dependencies
-    @result = 
-      begin
-        Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
-          clear_info
-          input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
-          merge_info :status => :start, :start => Time.now,
-            :pid => Process.pid, :pid_hostname => Misc.hostname, 
-            :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
-            :dependencies => dependencies.collect{|d| d.path }
-
-          @result = exec
-
-          if @result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
-            Open.mv self.tmp_path, self.path
-          else
-            @result = @result.stream if @result.respond_to?(:stream)
-          end
+    begin
+      @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
+        clear_info
+        input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
+        merge_info :status => :start, :start => Time.now,
+          :pid => Process.pid, :pid_hostname => Misc.hostname, 
+          :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
+          :dependencies => dependencies.collect{|d| d.path }
+
+        @exec_result = exec
+
+        if @exec_result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
+          Open.mv self.tmp_path, self.path
+        else
+          @exec_result = @exec_result.stream if @exec_result.respond_to?(:stream)
+        end
 
-          @result
+        @exec_result
 
-          if (IO === @result || StringIO === @result) && (ENV["SCOUT_NO_STREAM"] == "true" || ! stream)
-            Open.sensible_write(self.path, @result)
-            @result = nil
-          else
-            @result
-          end
+        if (IO === @exec_result || StringIO === @exec_result) && (ENV["SCOUT_NO_STREAM"] == "true" || ! stream)
+          Open.sensible_write(self.path, @exec_result)
+          @exec_result = nil
+        else
+          @exec_result
         end
-      rescue Exception => e
-        merge_info :status => :error, :exception => e, :end => Time.now
-        abort_dependencies
-        raise e
-      ensure
-        if ! (error? || aborted?)
-          if streaming?
-            ConcurrentStream.setup(@result) do
-              merge_info :status => :done, :end => Time.now
-            end
+      end
+    rescue Exception => e
+      merge_info :status => :error, :exception => e, :end => Time.now
+      abort_dependencies
+      raise e
+    ensure
+      if ! (error? || aborted?)
+        if streaming?
+          ConcurrentStream.setup(@result) do
+            merge_info :status => :done, :end => Time.now
+          end
 
-            @result.abort_callback = proc do |exception|
-              if exception.nil? || Aborted === exception || Interrupt === exception
-                merge_info :status => :aborted, :end => Time.now
-              else
-                begin
-                  merge_info :status => :error, :exception => exception, :end => Time.now
-                rescue Exception
-                  Log.exception $!
-                end
+          @result.abort_callback = proc do |exception|
+            if exception.nil? || Aborted === exception || Interrupt === exception
+              merge_info :status => :aborted, :end => Time.now
+            else
+              begin
+                merge_info :status => :error, :exception => exception, :end => Time.now
+              rescue Exception
+                Log.exception $!
               end
             end
-
-            log :streaming
-          else
-            merge_info :status => :done, :end => Time.now
           end
+
+
+          log :streaming
+        else
+          merge_info :status => :done, :end => Time.now
         end
       end
+    end
   end
 
   def fork

data/lib/scout/workflow/task/inputs.rb

@@ -1,4 +1,4 @@
-require_relative '../../named_array'
+require 'scout/named_array'
 module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value

data/lib/scout/workflow/task.rb

@@ -1,5 +1,5 @@
-require_relative '../meta_extension'
-require_relative '../named_array'
+require 'scout/meta_extension'
+require 'scout/named_array'
 require_relative 'step'
 require_relative 'task/inputs'
 require_relative 'task/dependencies'

data/lib/scout/workflow.rb

@@ -6,8 +6,8 @@ require_relative 'workflow/documentation'
 require_relative 'workflow/usage'
 require_relative 'workflow/deployment'
 
-require_relative 'resource'
-require_relative 'resource/scout'
+require 'scout/resource'
+require 'scout/resource/scout'
 
 module Workflow
   class << self

data/lib/scout-gear.rb

@@ -1,12 +1,3 @@
-require_relative 'scout/exceptions'
-require_relative 'scout/indiferent_hash'
-require_relative 'scout/tmpfile'
-require_relative 'scout/log'
-require_relative 'scout/path'
-require_relative 'scout/simple_opt'
-require_relative 'scout/resource'
-require_relative 'scout/resource/scout'
-require_relative 'scout/persist'
+require 'scout-essentials'
 require_relative 'scout/tsv'
-require_relative 'scout/config'
 require_relative 'scout/offsite'

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 9.1.0 ruby lib
+# stub: scout-gear 10.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "9.1.0"
+  s.version = "10.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-06-30"
+  s.date = "2023-07-07"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -33,65 +33,12 @@ Gem::Specification.new do |s|
     "lib/rbbt-scout.rb",
     "lib/scout-gear.rb",
     "lib/scout.rb",
-    "lib/scout/cmd.rb",
-    "lib/scout/concurrent_stream.rb",
-    "lib/scout/config.rb",
-    "lib/scout/exceptions.rb",
-    "lib/scout/indiferent_hash.rb",
-    "lib/scout/indiferent_hash/case_insensitive.rb",
-    "lib/scout/indiferent_hash/options.rb",
-    "lib/scout/log.rb",
-    "lib/scout/log/color.rb",
-    "lib/scout/log/color_class.rb",
-    "lib/scout/log/fingerprint.rb",
-    "lib/scout/log/progress.rb",
-    "lib/scout/log/progress/report.rb",
-    "lib/scout/log/progress/util.rb",
-    "lib/scout/log/trap.rb",
-    "lib/scout/meta_extension.rb",
-    "lib/scout/misc.rb",
-    "lib/scout/misc/digest.rb",
-    "lib/scout/misc/filesystem.rb",
-    "lib/scout/misc/format.rb",
-    "lib/scout/misc/helper.rb",
-    "lib/scout/misc/insist.rb",
-    "lib/scout/misc/monitor.rb",
-    "lib/scout/misc/system.rb",
-    "lib/scout/named_array.rb",
     "lib/scout/offsite.rb",
+    "lib/scout/offsite/exceptions.rb",
     "lib/scout/offsite/ssh.rb",
     "lib/scout/offsite/step.rb",
     "lib/scout/offsite/sync.rb",
-    "lib/scout/open.rb",
-    "lib/scout/open/lock.rb",
-    "lib/scout/open/lock/lockfile.rb",
-    "lib/scout/open/remote.rb",
-    "lib/scout/open/stream.rb",
-    "lib/scout/open/util.rb",
-    "lib/scout/path.rb",
-    "lib/scout/path/find.rb",
-    "lib/scout/path/tmpfile.rb",
-    "lib/scout/path/util.rb",
-    "lib/scout/persist.rb",
-    "lib/scout/persist/open.rb",
-    "lib/scout/persist/path.rb",
-    "lib/scout/persist/serialize.rb",
-    "lib/scout/resource.rb",
-    "lib/scout/resource/open.rb",
-    "lib/scout/resource/path.rb",
-    "lib/scout/resource/produce.rb",
-    "lib/scout/resource/produce/rake.rb",
-    "lib/scout/resource/scout.rb",
-    "lib/scout/resource/software.rb",
-    "lib/scout/resource/util.rb",
     "lib/scout/semaphore.rb",
-    "lib/scout/simple_opt.rb",
-    "lib/scout/simple_opt/accessor.rb",
-    "lib/scout/simple_opt/doc.rb",
-    "lib/scout/simple_opt/get.rb",
-    "lib/scout/simple_opt/parse.rb",
-    "lib/scout/simple_opt/setup.rb",
-    "lib/scout/tmpfile.rb",
     "lib/scout/tsv.rb",
     "lib/scout/tsv/attach.rb",
     "lib/scout/tsv/change_id.rb",
@@ -113,8 +60,10 @@ Gem::Specification.new do |s|
     "lib/scout/tsv/util/process.rb",
     "lib/scout/tsv/util/reorder.rb",
     "lib/scout/tsv/util/select.rb",
+    "lib/scout/tsv/util/sort.rb",
     "lib/scout/tsv/util/unzip.rb",
     "lib/scout/work_queue.rb",
+    "lib/scout/work_queue/exceptions.rb",
     "lib/scout/work_queue/socket.rb",
     "lib/scout/work_queue/worker.rb",
     "lib/scout/workflow.rb",
@@ -157,49 +106,12 @@ Gem::Specification.new do |s|
     "share/software/install_helpers",
     "share/templates/command",
     "share/templates/workflow.rb",
-    "test/scout/indiferent_hash/test_case_insensitive.rb",
-    "test/scout/indiferent_hash/test_options.rb",
-    "test/scout/log/test_color.rb",
-    "test/scout/log/test_progress.rb",
-    "test/scout/misc/test_digest.rb",
-    "test/scout/misc/test_filesystem.rb",
-    "test/scout/misc/test_insist.rb",
-    "test/scout/misc/test_system.rb",
     "test/scout/offsite/test_ssh.rb",
     "test/scout/offsite/test_step.rb",
     "test/scout/offsite/test_sync.rb",
     "test/scout/offsite/test_task.rb",
-    "test/scout/open/test_lock.rb",
-    "test/scout/open/test_remote.rb",
-    "test/scout/open/test_stream.rb",
-    "test/scout/open/test_util.rb",
-    "test/scout/path/test_find.rb",
-    "test/scout/path/test_util.rb",
-    "test/scout/persist/test_open.rb",
-    "test/scout/persist/test_path.rb",
-    "test/scout/persist/test_serialize.rb",
-    "test/scout/resource/test_path.rb",
-    "test/scout/resource/test_produce.rb",
-    "test/scout/resource/test_software.rb",
-    "test/scout/resource/test_util.rb",
-    "test/scout/simple_opt/test_doc.rb",
-    "test/scout/simple_opt/test_get.rb",
-    "test/scout/simple_opt/test_parse.rb",
-    "test/scout/simple_opt/test_setup.rb",
-    "test/scout/test_cmd.rb",
-    "test/scout/test_concurrent_stream.rb",
-    "test/scout/test_config.rb",
-    "test/scout/test_indiferent_hash.rb",
-    "test/scout/test_log.rb",
-    "test/scout/test_meta_extension.rb",
-    "test/scout/test_misc.rb",
-    "test/scout/test_named_array.rb",
-    "test/scout/test_open.rb",
-    "test/scout/test_path.rb",
-    "test/scout/test_persist.rb",
-    "test/scout/test_resource.rb",
+    "test/scout/test_offsite.rb",
     "test/scout/test_semaphore.rb",
-    "test/scout/test_tmpfile.rb",
     "test/scout/test_tsv.rb",
     "test/scout/test_work_queue.rb",
     "test/scout/test_workflow.rb",
@@ -221,6 +133,7 @@ Gem::Specification.new do |s|
     "test/scout/tsv/util/test_process.rb",
     "test/scout/tsv/util/test_reorder.rb",
     "test/scout/tsv/util/test_select.rb",
+    "test/scout/tsv/util/test_sort.rb",
     "test/scout/tsv/util/test_unzip.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
@@ -249,10 +162,9 @@ Gem::Specification.new do |s|
 
   s.specification_version = 4
 
-  s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<scout-essentials>.freeze, [">= 0"])
   s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0"])
   s.add_runtime_dependency(%q<matrix>.freeze, [">= 0"])
-  s.add_runtime_dependency(%q<sys-proctable>.freeze, [">= 0"])
   s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0"])
   s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
   s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])