scout-gear 5.1.1 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '070801d66b058318a5be03704b424158222038331dac4d7984fab415ca1680ee'
-  data.tar.gz: e58bcd314c0fa55acb722d12f8e77e69ade25f182501823db9a14691eaf3a887
+  metadata.gz: c39189bc9fe8816a3c0494c78966486f28383e8e15bf38ab33fe4dcde6739f3c
+  data.tar.gz: 5f2753e6810a68be44589f9392de1eff2f8592dd2704b05a0ccdf866f07f6a1b
 SHA512:
-  metadata.gz: 81146847ebd0802456b42f152d2fe6e96d1a4ab434b20111afd2f9bd2bf7e1006d8ce2108ba22db78ae65e23ba7e69d6e4e28543aaa3c55a0482727b2472ed7c
-  data.tar.gz: 59d1574833255c0336472a5297bb00e21319b044bca684b018512557de65cc8ff34091b2bc9adcd5f4758833793cb0b4a8092927835a328f76170fa8c60be115
+  metadata.gz: 820d17306f468e7b206672b717489f78eb6fbfa37c843441202ce124c54cb7b84b89062a951c14a2db65ada47faf073dbc6a1b863294388ce66a8c1f7dd36e9f
+  data.tar.gz: c0fe23608355128c0e709db78c5598c5ece26e885eaa8788872878b7d55e29fc50bcb8215d3b4080f9e8d89456630e2739b087ee7d8b4b57498b299df3dc89d2

data/.vimproject

@@ -1,16 +1,19 @@
 scout-gear=/$PWD filter="*.rb *.yaml" {
  Rakefile
- bin=bin  filter="*"{
+ bin=bin filter="*"{
   scout
-  scout_rbbt
  }
  scout_commands=scout_commands filter="*"{
+  rbbt
+  alias
   find
   glob
   workflow=workflow{
+   task_old
    task
+   list
+   info
   }
-  alias
  }
  lib=lib {
   scout-gear.rb
@@ -26,6 +29,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     insist.rb
     digest.rb
     filesystem.rb
+    monitor.rb
    }
    indiferent_hash.rb
    indiferent_hash=indiferent_hash{
@@ -73,8 +77,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     produce=produce{
      rake.rb
     }
-    path.rb
     scout.rb
+    util.rb
+    path.rb
    }
    persist.rb
    persist=persist{
@@ -88,6 +93,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     step.rb
     step=step{
      info.rb
+     load.rb
     }
     task.rb
     task=task{

data/Rakefile

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+ENV["BRANCH"] = 'main'
+
 require 'rubygems'
 require 'rake'
 

data/VERSION

@@ -1 +1 @@
-5.1.1
+5.2.0

data/bin/scout

@@ -39,6 +39,8 @@ if dev_dir
   end
 end
 
+Log.nocolor = true if ARGV.include? "--nocolor"
+
 require 'scout/simple_opt'
 
 options = SOPT.setup <<EOF

data/lib/scout/meta_extension.rb

@@ -24,8 +24,10 @@ module MetaExtension
       return if attrs.nil? || attrs.empty?
 
       if rest.length == 1 && Hash === (rlast = rest.last) && 
-          ! (rlkey = rlast.keys.first).nil? &&
-          attrs.include?(rlkey.to_sym)
+          ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
+           (! attrs.length != 1 ))
+
+          
 
         pairs = rlast
       else

data/lib/scout/misc/format.rb

@@ -25,7 +25,14 @@ module Misc
     str
   end
 
-  def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
+  
+  MAX_WIDTH = 100
+  def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
+    size ||= Log.tty_size || MAX_WIDTH
+    size = MAX_WIDTH if size > MAX_WIDTH
+    indent ||= 0
+    offset ||= 0
+
     i = 0
     size = size + offset + indent
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
@@ -55,7 +62,10 @@ module Misc
       end*""
   end
 
-  def self.format_definition_list_item(dt, dd, size = 80, indent = 20, color = :yellow)
+  def self.format_definition_list_item(dt, dd,  indent = nil, size = nil, color = :yellow)
+    size ||= Log.tty_size || MAX_WIDTH
+    size = MAX_WIDTH if size > MAX_WIDTH
+    indent ||= size / 3
     dd = "" if dd.nil?
     dt = Log.color color, dt if color
     dt = dt.to_s  unless dd.empty?
@@ -73,10 +83,12 @@ module Misc
     text
   end
 
-  def self.format_definition_list(defs, size = 80, indent = 20, color = :yellow, sep = "\n\n")
+  def self.format_definition_list(defs, indent = nil, size = nil, color = :yellow, sep = "\n\n")
+    size ||= Log.tty_size || MAX_WIDTH
+    indent ||= 30
     entries = []
     defs.each do |dt,dd|
-      text = format_definition_list_item(dt,dd,size,indent,color)
+      text = format_definition_list_item(dt,dd,indent, size,color)
       entries << text
     end
     entries * sep 

data/lib/scout/misc/monitor.rb

@@ -0,0 +1,23 @@
+module Misc
+  def self.benchmark(repeats = 1, message = nil)
+    require 'benchmark'
+    res = nil
+    begin
+      measure = Benchmark.measure do
+        repeats.times do
+          res = yield
+        end
+      end
+      if message
+        puts "#{message }: #{ repeats } repeats"
+      else
+        puts "Benchmark for #{ repeats } repeats"
+      end
+      puts measure
+    rescue Exception
+      puts "Benchmark aborted"
+      raise $!
+    end
+    res
+  end
+end

data/lib/scout/misc.rb

@@ -2,6 +2,7 @@ require_relative 'misc/format'
 require_relative 'misc/insist'
 require_relative 'misc/digest'
 require_relative 'misc/filesystem'
+require_relative 'misc/monitor'
 
 module Misc
 end

data/lib/scout/open/stream.rb

@@ -68,6 +68,7 @@ module Open
         block.call if block_given?
 
         Log.high "Done consuming stream #{Log.fingerprint io} into #{into_path || into}"
+        c
       rescue Aborted
         Log.high "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
         io.abort $! if io.respond_to? :abort

data/lib/scout/path/find.rb

@@ -123,7 +123,7 @@ module Path
 
   def follow(map_name = :default, annotate = true)
     IndiferentHash.setup(path_maps)
-    map = path_maps[map_name]
+    map = path_maps[map_name] || Path.path_maps[map_name]
     raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
     while Symbol === map
       map_name = map
@@ -163,4 +163,5 @@ module Path
       .collect{|where| find(where) }
       .select{|file| file.exist? }.uniq
   end
+
 end

data/lib/scout/path.rb

@@ -24,7 +24,7 @@ module Path
   end
 
   def path_maps
-    @path_maps ||= Path.path_maps
+    @path_maps ||= Path.path_maps.dup
   end
 
   def join(subpath, prevpath = nil)

data/lib/scout/persist/serialize.rb

@@ -3,6 +3,7 @@ require_relative 'open'
 
 module Persist
   TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
+  SERIALIZER = :json
 
   class << self
     attr_accessor :save_drivers, :load_drivers
@@ -15,6 +16,8 @@ module Persist
   end
 
   def self.serialize(content, type)
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
     case type
     when nil, :string, :integer, :float, :boolean, :file, :path
       if IO === content || StringIO === content
@@ -28,7 +31,7 @@ module Persist
       content.to_yaml
     when :json
       content.to_json
-    when :marshal, :serializer
+    when :marshal
       Marshal.dump(content)
     else
       if m = type.to_s.match(/(.*)_array/)
@@ -41,6 +44,8 @@ module Persist
   end
 
   def self.deserialize(serialized, type)
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
     case type
     when nil, :string, :file, :stream
       serialized
@@ -58,7 +63,7 @@ module Persist
       YAML.parse(serialized)
     when :json
       JSON.parse(serialized)
-    when :marshal, :serializer
+    when :marshal
       Marshal.load(serialized)
     else
       if m = type.to_s.match(/(.*)_array/)
@@ -73,6 +78,10 @@ module Persist
 
   MEMORY = {}
   def self.save(content, file, type = :serializer)
+    type = :serializer if type.nil?
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    type = MEMORY if type == :memory
     return if content.nil?
     type = MEMORY if type == :memory
     type = :serializer if type.nil?
@@ -97,16 +106,18 @@ module Persist
       ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
     else
       serialized = serialize(content, type)
-      Open.sensible_write(file, serialized)
+      Open.sensible_write(file, serialized, :force => true)
       content
     end
   end
 
   def self.load(file, type = :serializer)
     file = file.find if Path === file
+    type = :serializer if type.nil?
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
     type = MEMORY if type == :memory
     return unless Hash === type || Open.exist?(file) 
-    type = :serializer if type.nil?
 
     Log.debug "Load #{Log.fingerprint type} on #{file}"
     if load_drivers[type]

data/lib/scout/resource/path.rb

@@ -1,4 +1,9 @@
 module Path
+  def relocate
+    return self if Open.exists?(self)
+    Resource.relocate(self)
+  end
+
   def open(*args, &block)
     produce
     Open.open(self, *args, &block)

data/lib/scout/resource/util.rb

@@ -0,0 +1,48 @@
+module Resource
+  def identify(path)
+    return path unless path.start_with?("/")
+    path_maps = path.path_maps || self.path_maps || Path.path_maps
+    path = File.expand_path(path)
+    path += "/" if File.directory?(path)
+
+    map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
+    map_order -= [:current, "current"]
+    map_order << :current
+
+    choices = []
+    map_order.uniq.each do |name|
+      pattern = path_maps[name]
+      pattern = path_maps[pattern] while Symbol === pattern
+      next if pattern.nil?
+
+      pattern = pattern.sub('{PWD}', Dir.pwd)
+      if String ===  pattern and pattern.include?('{')
+        regexp = "^" + pattern
+          .gsub(/{(TOPLEVEL)}/,'(?<\1>[^/]+)')
+          .gsub(/{([^}]+)}/,'(?<\1>[^/]+)?') +
+        "(?:/(?<REST>.*))?/?$"
+        if m = path.match(regexp) 
+          if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == self.pkgdir
+            unlocated = %w(TOPLEVEL SUBPATH PATH REST).collect{|c| 
+              m.named_captures.include?(c) ? m[c] : nil
+            }.compact * "/"
+            unlocated.gsub!(/\/+/,'/')
+            unlocated[self.subdir] = "" if self.subdir
+            choices << self.annotate(unlocated)
+          end
+        end
+      end
+    end
+
+    Path.setup(choices.sort_by{|s| s.length }.first, self, nil, path_maps)
+  end
+
+  def self.relocate(path)
+    return path if Open.exists?(path)
+    resource = path.pkgdir if Path === path
+    resource = Scout unless Resource === resource
+    unlocated = resource.identify path
+    unlocated.find
+  end
+end
+

data/lib/scout/resource.rb

@@ -1,6 +1,8 @@
 require_relative 'log'
 require_relative 'path'
 require_relative 'resource/produce'
+require_relative 'resource/path'
+require_relative 'resource/util'
 
 module Resource
   extend MetaExtension

data/lib/scout/simple_opt/doc.rb

@@ -42,10 +42,33 @@ module SOPT
       "=<#{ type }>"
     end
     #extra << " (default: #{Array === default ? (default.length > 3 ? default[0..2]*", " + ', ...' : default*", " ): default})" if default != nil
-    extra << " (default: #{Misc.fingerprint(default)})" if default != nil
+    extra << " (default: #{Log.fingerprint(default)})" if default != nil
     input_str << Log.color(:green, extra)
   end
 
+  def self.input_array_doc(input_array)
+    input_array.collect do |name,type,description,default,options|
+      type = :string if type.nil?
+
+      name = name.to_s
+      shortcut, options = options, nil if String === options || Symbol === options
+
+      case options && options[:shortcut]
+      when FalseClass
+        shortcut = nil
+      when TrueClass, nil
+        shortcut = fix_shortcut(name[0], name)
+      else
+        shortcut = options[:shortcut]
+      end unless shortcut
+
+      shortcut = fix_shortcut(shortcut, name)
+      register(shortcut, name, type, description) unless self.inputs.include? name
+      name  = SOPT.input_format(name, type.to_sym, default, shortcut ) 
+      Misc.format_definition_list_item(name, description)
+    end * "\n"
+  end
+
   def self.input_doc(inputs, input_types = nil, input_descriptions = nil, input_defaults = nil, input_shortcuts = nil)
     type = description = default = nil
     shortcut = ""
@@ -73,10 +96,11 @@ module SOPT
       register(shortcut, name, type, description) unless self.inputs.include? name
 
       name  = SOPT.input_format(name, type.to_sym, default, shortcut) 
-      Misc.format_definition_list_item(name, description, 80, 31, nil)
+      Misc.format_definition_list_item(name, description)
     end * "\n"
   end
 
+
   def self.doc
     doc =<<-EOF
 #{Log.color :magenta}#{command}(1) -- #{summary}

data/lib/scout/workflow/definition.rb

@@ -33,6 +33,11 @@ module Workflow
     @annotate_next_task[type] << obj
   end
 
+  def annotate_next_task_single(type, obj)
+    @annotate_next_task ||= {}
+    @annotate_next_task[type] = obj
+  end
+
   def dep(*args, &block)
     case args.length
     when 3
@@ -60,13 +65,14 @@ module Workflow
     @tasks ||= IndiferentHash.setup({})
     begin
       @annotate_next_task ||= {}
-      @tasks[name] = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
+      task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
+      @tasks[name] = task
     ensure
       @annotate_next_task = {}
     end
   end
 
   def desc(description)
-    annotate_next_task(:desc, description)
+    annotate_next_task_single(:description, description)
   end
 end

data/lib/scout/workflow/documentation.rb

@@ -1,4 +1,5 @@
 module Workflow
+  attr_accessor :title, :description
 
   def self.doc_parse_first_line(str)
     if str.match(/^([^\n]*)\n\n(.*)/sm)
@@ -45,33 +46,38 @@ module Workflow
     end
   end
 
-  def load_documentation
-    return if @documentation
-    @documentation ||= Workflow.parse_workflow_doc documentation_markdown
-    @documentation[:tasks].each do |task, description|
-      if task.include? "#"
-        workflow, task = task.split("#")
-        workflow = begin
-                     Kernel.const_get workflow
-                   rescue
-                     next
-                   end
-      else
-        workflow = self
-      end
-
-      task = task.to_sym
-      if workflow.tasks.include? task
-        workflow.tasks[task].description = description
-      else
-        Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
-      end
-    end
-  end
-
   attr_accessor :documentation
   def documentation
-    load_documentation if @documentation.nil?
-    @documentation 
+    @documentation ||= begin
+                         documentation = Workflow.parse_workflow_doc documentation_markdown
+
+                         if @description && (documentation[:description].nil? || documentation[:description].empty?)
+                           documentation[:description] = @description 
+                         end
+
+                         if @title && (documentation[:title].nil? || documentation[:title].empty?)
+                           documentation[:title] = @title 
+                         end
+                         documentation[:tasks].each do |task, description|
+                           if task.include? "#"
+                             workflow, task = task.split("#")
+                             workflow = begin
+                                          Kernel.const_get workflow
+                                        rescue
+                                          next
+                                        end
+                           else
+                             workflow = self
+                           end
+
+                           task = task.to_sym
+                           if workflow.tasks.include? task
+                             workflow.tasks[task].description = description
+                           else
+                             Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
+                           end
+                         end
+                         documentation
+                       end
   end
 end

data/lib/scout/workflow/step/info.rb

@@ -1,15 +1,21 @@
 class Step
+  SERIALIZER = :json
   def info_file
-    @info_file ||= @path + ".info"
+    @info_file ||= begin
+                     info_file = @path + ".info"
+                     @path.annotate info_file if Path === @path
+                     info_file
+                   end
   end
 
   def load_info
-    @info = Persist.load(info_file, :marshal) || {}
+    @info = Persist.load(info_file, SERIALIZER) || {}
+    IndiferentHash.setup(@info)
     @info_load_time = Time.now
   end
 
-  def save_info
-    Persist.save(@info, info_file, :marshal)
+  def save_info(info = nil)
+    Persist.save(info, info_file, SERIALIZER)
     @info_load_time = Time.now
   end
 
@@ -40,19 +46,13 @@ class Step
         info[key] = value
       end
     end
-    save_info
+    save_info(info)
   end
 
   def set_info(key, value)
     merge_info(key => value)
   end
   
-  def init_info
-    @info = {
-      :status => :waiting
-    }
-  end
-
   def report_status(status, message = nil)
     if message.nil?
       Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)

data/lib/scout/workflow/step/load.rb

@@ -0,0 +1,18 @@
+class Step
+  def self.relocate(path)
+    return path if Open.exists?(path)
+    Path.setup(path) unless Path === path
+    relocated = path.relocate
+    return relocated if Open.exists?(relocated)
+    subpath = path.split("/")[-3..-1] * "/"
+    relocated = Path.setup("var/jobs")[subpath]
+    return relocated if Open.exists?(relocated)
+    path
+  end
+
+  def self.load(path)
+    path = relocate(path) unless Open.exists?(path)
+    raise "Could not load #{path}" unless Open.exists?(path)
+    s = Step.new path
+  end
+end

data/lib/scout/workflow/step.rb

@@ -1,20 +1,43 @@
 require_relative '../path'
 require_relative '../persist'
 require_relative 'step/info'
+require_relative 'step/load'
 
 class Step 
 
   attr_accessor :path, :inputs, :dependencies, :task
-  def initialize(path, inputs = nil, dependencies = [], &task) 
+  def initialize(path, inputs = nil, dependencies = nil, &task) 
     @path = path
     @inputs = inputs
     @dependencies = dependencies
     @task = task
   end
 
+  def inputs
+    @inputs ||= begin
+                  if Open.exists?(info_file)
+                    info[:inputs]
+                  else
+                    []
+                  end
+                end
+  end
+
+  def dependencies
+    @dependencies ||= begin
+                        if Open.exists?(info_file)
+                          info[:dependencies].collect do |path|
+                            Step.load(path)
+                          end
+                        else
+                          []
+                        end
+                      end
+  end
+
   attr_accessor :type
   def type
-    @type ||= @task.respond_to?(:type) ? @task.type : nil
+    @type ||= @task.respond_to?(:type) ? @task.type : info[:type]
   end
 
   def name
@@ -31,14 +54,15 @@ class Step
 
   attr_reader :result
   def run
+    return @result || self.load if done?
+    dependencies.each{|dep| dep.run }
     @result = Persist.persist(name, type, :path => path) do
       begin
         merge_info :status => :start, :start => Time.now,
           :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
-          :inputs => inputs,
+          :inputs => inputs, :type => type,
           :dependencies => dependencies.collect{|d| d.path }
 
-        dependencies.each{|dep| dep.run }
         @result = exec
       ensure
         if streaming?
@@ -61,6 +85,11 @@ class Step
     IO === @result || StringIO === @result
   end
 
+  def stream
+    join
+    streaming? ? @result : Open.open(path)
+  end
+
   def join
     if streaming?
       Open.consume_stream(@result, false) 
@@ -83,6 +112,13 @@ class Step
     FileUtils.rm path.find if path.exist?
   end
 
+  def recursive_clean
+    dependencies.each do |dep|
+      dep.recursive_clean
+    end
+    clean
+  end
+
   def step(task_name)
     dependencies.each do |dep|
       return dep if dep.task_name == task_name

data/lib/scout/workflow/task/inputs.rb

@@ -22,7 +22,7 @@ module Task
   end
 
   def assign_inputs(provided_inputs = {})
-    if self.inputs.nil?
+    if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
       case provided_inputs
       when Array
         return [provided_inputs, provided_inputs]
@@ -31,6 +31,8 @@ module Task
       end
     end
 
+    IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
+
     input_array = []
     non_default_inputs = []
     self.inputs.each_with_index do |p,i|
@@ -58,7 +60,7 @@ module Task
   end
   
   def process_inputs(provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    input_array, non_default_inputs = assign_inputs provided_inputs
     digest = Misc.digest(input_array)
     [input_array, non_default_inputs, digest]
   end