scout-gear 5.1.1 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '070801d66b058318a5be03704b424158222038331dac4d7984fab415ca1680ee'
-  data.tar.gz: e58bcd314c0fa55acb722d12f8e77e69ade25f182501823db9a14691eaf3a887
+  metadata.gz: 919678993bd756d6a67712023120e562c01a69942faf8ad23eb088f29559b1fa
+  data.tar.gz: a233c808dc9af64381196bce97c74727c9064c9b1a4c2a129b364a5d776b01e1
 SHA512:
-  metadata.gz: 81146847ebd0802456b42f152d2fe6e96d1a4ab434b20111afd2f9bd2bf7e1006d8ce2108ba22db78ae65e23ba7e69d6e4e28543aaa3c55a0482727b2472ed7c
-  data.tar.gz: 59d1574833255c0336472a5297bb00e21319b044bca684b018512557de65cc8ff34091b2bc9adcd5f4758833793cb0b4a8092927835a328f76170fa8c60be115
+  metadata.gz: 842323af61a5c66c7bf4433cbc0d23e6484e562c88601c64ac7a47ea2c1e217e6f7b590bd7d613d62614ebb2ab228246a2ac150d4d263f9425d9c1a2b4333e65
+  data.tar.gz: 2a4a7d00279d0cbc3153d7b116405fb5d02f8656249b4e0bdf417d01599749686242fb30db183b2baff856e92dcd5c83317161e73bb5bb1dbc861ce10d0086de

data/.vimproject

@@ -1,16 +1,19 @@
 scout-gear=/$PWD filter="*.rb *.yaml" {
  Rakefile
- bin=bin  filter="*"{
+ bin=bin filter="*"{
   scout
-  scout_rbbt
  }
  scout_commands=scout_commands filter="*"{
+  rbbt
+  alias
   find
   glob
   workflow=workflow{
+   task_old
    task
+   list
+   info
   }
-  alias
  }
  lib=lib {
   scout-gear.rb
@@ -26,6 +29,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     insist.rb
     digest.rb
     filesystem.rb
+    monitor.rb
    }
    indiferent_hash.rb
    indiferent_hash=indiferent_hash{
@@ -73,8 +77,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     produce=produce{
      rake.rb
     }
-    path.rb
     scout.rb
+    util.rb
+    path.rb
    }
    persist.rb
    persist=persist{
@@ -88,6 +93,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     step.rb
     step=step{
      info.rb
+     load.rb
     }
     task.rb
     task=task{
@@ -97,6 +103,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     usage.rb
     util.rb
    }
+   semaphore.rb
+   work_queue.rb
+   work_queue=work_queue{
+    socket.rb
+    worker.rb
+   }
   }
  }
  test=test {
@@ -218,16 +230,16 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
       concurrency=concurrency{
        processes.rb
        threads.rb
-     processes=processes{
-      socket.rb
-      worker.rb
-     }
-    }
+       processes=processes{
+        socket.rb
+        worker.rb
+       }
+      }
 
-    procpath.rb
+      procpath.rb
 
-    migrate.rb
-   }
+      migrate.rb
+     }
 
    monitor.rb
 

data/Rakefile

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+ENV["BRANCH"] = 'main'
+
 require 'rubygems'
 require 'rake'
 

data/VERSION

@@ -1 +1 @@
-5.1.1
+6.0.0

data/bin/scout

@@ -39,6 +39,8 @@ if dev_dir
   end
 end
 
+Log.nocolor = true if ARGV.include? "--nocolor"
+
 require 'scout/simple_opt'
 
 options = SOPT.setup <<EOF

data/lib/scout/exceptions.rb

@@ -71,8 +71,20 @@ end
 
 class LockInterrupted < TryAgain; end
 
-#
-#class ClosedStream < StandardError; end
+class ClosedStream < StandardError; end
+
+class DoneProcessing < Exception
+  attr_accessor :pid
+  def initialize(pid = Process.pid)
+    @pid = pid
+  end
+
+  def message
+    "Done processing pid #{pid}"
+  end
+end
+
+
 #class OpenGzipError < StandardError; end
 #
 #

data/lib/scout/log/color.rb

@@ -2,6 +2,7 @@ require_relative 'color_class'
 require_relative '../indiferent_hash'
 
 require 'term/ansicolor'
+require 'colorist'
 
 module Colorize
   def self.colors=(colors)
@@ -9,7 +10,13 @@ module Colorize
   end
 
   def self.colors
-    @colors ||= IndiferentHash.setup({green: "#00cd00" , red: "#cd0000" , yellow: "#ffd700" })
+    @colors ||= IndiferentHash.setup(Hash[<<-EOF.split("\n").collect{|l| l.split(" ")}])
+green #00cd00 
+red #cd0000 
+yellow #ffd700
+blue #0000cd
+path blue
+EOF
   end
 
   def self.diverging_colors=(colors)
@@ -33,7 +40,6 @@ module Colorize
     EOF
   end
 
-
   def self.from_name(color)
     return color if color =~ /^#?[0-9A-F]+$/i
     return colors[color.to_s] if colors.include?(color.to_s)
@@ -52,7 +58,7 @@ module Colorize
     when 'blue'
       colors["RoyalBlue"] 
     else
-      colors[color.to_s] || color
+      colors[color.to_s] || color.to_s
     end
   end
 
@@ -134,8 +140,21 @@ module Log
   WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
 
   SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
+  CONCEPT_COLORS = IndiferentHash.setup({
+    :title => magenta,
+    :path => blue,
+    :input => blue,
+    :value => green,
+    :integer => green,
+    :negative => red,
+    :float => green,
+    :waiting => yellow,
+    :started => blue,
+    :done => green,
+    :error => red,
+  })
   HIGHLIGHT = "\033[1m"
-
+  
   def self.uncolor(str)
     "" << Term::ANSIColor.uncolor(str)
   end
@@ -144,15 +163,20 @@ module Log
     reset
   end
 
-  def self.color(severity, str = nil, reset = false)
+  def self.color(color, str = nil, reset = false)
     return str.dup || "" if nocolor 
-    color = reset ? Term::ANSIColor.reset : ""
-    color << SEVERITY_COLOR[severity] if Integer === severity
-    color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity 
+
+    color = SEVERITY_COLOR[color] if Integer === color
+    color = CONCEPT_COLORS[color] if CONCEPT_COLORS.include?(color)
+    color = Term::ANSIColor.send(color) if Symbol === color and Term::ANSIColor.respond_to?(color)
+
+    return str if Symbol === color
+    color_str = reset ? Term::ANSIColor.reset : ""
+    color_str << color
     if str.nil?
-      color
+      color_str
     else
-      color + str.to_s + self.color(0)
+      color_str + str.to_s + Term::ANSIColor.reset
     end
   end
 

data/lib/scout/log/progress/report.rb

@@ -195,15 +195,16 @@ module Log
     def done(io = STDERR)
       done_msg = Log.color(:magenta, "· ") << Log.color(:green, "done")
       if @start
-        ellapsed = (Time.now - @start).to_i
+        ellapsed = (Time.now - @start)
       else
         ellapsed = 0
       end
-      ellapsed = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
-      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed)
+      ellapsed_str = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
+      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed_str)
       @last_count = 0
       @last_time = @start
-      done_msg << " - " << thr_msg 
+      thr = ellapsed > 0 ? (@ticks / ellapsed).to_i.to_s : 0
+      done_msg << " - " << Log.color(:blue, thr) << " per second"
       done_msg << Log.color(:magenta, " · " << desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
 

data/lib/scout/meta_extension.rb

@@ -24,8 +24,10 @@ module MetaExtension
       return if attrs.nil? || attrs.empty?
 
       if rest.length == 1 && Hash === (rlast = rest.last) && 
-          ! (rlkey = rlast.keys.first).nil? &&
-          attrs.include?(rlkey.to_sym)
+          ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
+           (! attrs.length != 1 ))
+
+          
 
         pairs = rlast
       else

data/lib/scout/misc/format.rb

@@ -25,7 +25,14 @@ module Misc
     str
   end
 
-  def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
+  
+  MAX_WIDTH = 100
+  def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
+    size ||= Log.tty_size || MAX_WIDTH
+    size = MAX_WIDTH if size > MAX_WIDTH
+    indent ||= 0
+    offset ||= 0
+
     i = 0
     size = size + offset + indent
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
@@ -55,7 +62,10 @@ module Misc
       end*""
   end
 
-  def self.format_definition_list_item(dt, dd, size = 80, indent = 20, color = :yellow)
+  def self.format_definition_list_item(dt, dd,  indent = nil, size = nil, color = :yellow)
+    size ||= Log.tty_size || MAX_WIDTH
+    size = MAX_WIDTH if size > MAX_WIDTH
+    indent ||= size / 3
     dd = "" if dd.nil?
     dt = Log.color color, dt if color
     dt = dt.to_s  unless dd.empty?
@@ -73,10 +83,12 @@ module Misc
     text
   end
 
-  def self.format_definition_list(defs, size = 80, indent = 20, color = :yellow, sep = "\n\n")
+  def self.format_definition_list(defs, indent = nil, size = nil, color = :yellow, sep = "\n\n")
+    size ||= Log.tty_size || MAX_WIDTH
+    indent ||= 30
     entries = []
     defs.each do |dt,dd|
-      text = format_definition_list_item(dt,dd,size,indent,color)
+      text = format_definition_list_item(dt,dd,indent, size,color)
       entries << text
     end
     entries * sep 

data/lib/scout/misc/monitor.rb

@@ -0,0 +1,41 @@
+module Misc
+  def self.benchmark(repeats = 1, message = nil)
+    require 'benchmark'
+    res = nil
+    begin
+      measure = Benchmark.measure do
+        repeats.times do
+          res = yield
+        end
+      end
+      if message
+        puts "#{message }: #{ repeats } repeats"
+      else
+        puts "Benchmark for #{ repeats } repeats"
+      end
+      puts measure
+    rescue Exception
+      puts "Benchmark aborted"
+      raise $!
+    end
+    res
+  end
+
+  def self.profile(options = {})
+    require 'ruby-prof'
+    profiler = RubyProf::Profile.new
+    profiler.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = profiler.stop
+      printer = RubyProf::FlatPrinter.new(result)
+      printer.print(STDOUT, options)
+    end
+
+    res
+  end
+end

data/lib/scout/misc.rb

@@ -2,6 +2,7 @@ require_relative 'misc/format'
 require_relative 'misc/insist'
 require_relative 'misc/digest'
 require_relative 'misc/filesystem'
+require_relative 'misc/monitor'
 
 module Misc
 end

data/lib/scout/open/stream.rb

@@ -68,6 +68,7 @@ module Open
         block.call if block_given?
 
         Log.high "Done consuming stream #{Log.fingerprint io} into #{into_path || into}"
+        c
       rescue Aborted
         Log.high "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
         io.abort $! if io.respond_to? :abort
@@ -369,4 +370,34 @@ module Open
   def self.tee_stream(stream)
     tee_stream_thread(stream)
   end
+
+  def self.read_stream(stream, size)
+    str = nil
+    Thread.pass while IO.select([stream],nil,nil,1).nil?
+    while not str = stream.read(size)
+      IO.select([stream],nil,nil,1) 
+      Thread.pass
+      raise ClosedStream if stream.eof?
+    end
+
+    while str.length < size
+      raise ClosedStream if stream.eof?
+      IO.select([stream],nil,nil,1)
+      if new = stream.read(size-str.length)
+        str << new
+      end
+    end
+    str
+  end
+
+  def self.read_stream(stream, size)
+    str = ""
+    while str.length < size
+      missing = size - str.length
+      more = stream.read(missing)
+      str << more
+    end
+    str
+  end
+
 end

data/lib/scout/path/find.rb

@@ -123,7 +123,7 @@ module Path
 
   def follow(map_name = :default, annotate = true)
     IndiferentHash.setup(path_maps)
-    map = path_maps[map_name]
+    map = path_maps[map_name] || Path.path_maps[map_name]
     raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
     while Symbol === map
       map_name = map
@@ -163,4 +163,5 @@ module Path
       .collect{|where| find(where) }
       .select{|file| file.exist? }.uniq
   end
+
 end

data/lib/scout/path.rb

@@ -24,7 +24,7 @@ module Path
   end
 
   def path_maps
-    @path_maps ||= Path.path_maps
+    @path_maps ||= Path.path_maps.dup
   end
 
   def join(subpath, prevpath = nil)

data/lib/scout/persist/serialize.rb

@@ -3,6 +3,7 @@ require_relative 'open'
 
 module Persist
   TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
+  SERIALIZER = :json
 
   class << self
     attr_accessor :save_drivers, :load_drivers
@@ -15,6 +16,8 @@ module Persist
   end
 
   def self.serialize(content, type)
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
     case type
     when nil, :string, :integer, :float, :boolean, :file, :path
       if IO === content || StringIO === content
@@ -28,7 +31,7 @@ module Persist
       content.to_yaml
     when :json
       content.to_json
-    when :marshal, :serializer
+    when :marshal
       Marshal.dump(content)
     else
       if m = type.to_s.match(/(.*)_array/)
@@ -41,6 +44,8 @@ module Persist
   end
 
   def self.deserialize(serialized, type)
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
     case type
     when nil, :string, :file, :stream
       serialized
@@ -58,7 +63,7 @@ module Persist
       YAML.parse(serialized)
     when :json
       JSON.parse(serialized)
-    when :marshal, :serializer
+    when :marshal
       Marshal.load(serialized)
     else
       if m = type.to_s.match(/(.*)_array/)
@@ -73,6 +78,10 @@ module Persist
 
   MEMORY = {}
   def self.save(content, file, type = :serializer)
+    type = :serializer if type.nil?
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    type = MEMORY if type == :memory
     return if content.nil?
     type = MEMORY if type == :memory
     type = :serializer if type.nil?
@@ -97,16 +106,18 @@ module Persist
       ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
     else
       serialized = serialize(content, type)
-      Open.sensible_write(file, serialized)
+      Open.sensible_write(file, serialized, :force => true)
       content
     end
   end
 
   def self.load(file, type = :serializer)
     file = file.find if Path === file
+    type = :serializer if type.nil?
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
     type = MEMORY if type == :memory
     return unless Hash === type || Open.exist?(file) 
-    type = :serializer if type.nil?
 
     Log.debug "Load #{Log.fingerprint type} on #{file}"
     if load_drivers[type]

data/lib/scout/resource/path.rb

@@ -1,4 +1,9 @@
 module Path
+  def relocate
+    return self if Open.exists?(self)
+    Resource.relocate(self)
+  end
+
   def open(*args, &block)
     produce
     Open.open(self, *args, &block)

data/lib/scout/resource/util.rb

@@ -0,0 +1,48 @@
+module Resource
+  def identify(path)
+    return path unless path.start_with?("/")
+    path_maps = path.path_maps || self.path_maps || Path.path_maps
+    path = File.expand_path(path)
+    path += "/" if File.directory?(path)
+
+    map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
+    map_order -= [:current, "current"]
+    map_order << :current
+
+    choices = []
+    map_order.uniq.each do |name|
+      pattern = path_maps[name]
+      pattern = path_maps[pattern] while Symbol === pattern
+      next if pattern.nil?
+
+      pattern = pattern.sub('{PWD}', Dir.pwd)
+      if String ===  pattern and pattern.include?('{')
+        regexp = "^" + pattern
+          .gsub(/{(TOPLEVEL)}/,'(?<\1>[^/]+)')
+          .gsub(/{([^}]+)}/,'(?<\1>[^/]+)?') +
+        "(?:/(?<REST>.*))?/?$"
+        if m = path.match(regexp) 
+          if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == self.pkgdir
+            unlocated = %w(TOPLEVEL SUBPATH PATH REST).collect{|c| 
+              m.named_captures.include?(c) ? m[c] : nil
+            }.compact * "/"
+            unlocated.gsub!(/\/+/,'/')
+            unlocated[self.subdir] = "" if self.subdir
+            choices << self.annotate(unlocated)
+          end
+        end
+      end
+    end
+
+    Path.setup(choices.sort_by{|s| s.length }.first, self, nil, path_maps)
+  end
+
+  def self.relocate(path)
+    return path if Open.exists?(path)
+    resource = path.pkgdir if Path === path
+    resource = Scout unless Resource === resource
+    unlocated = resource.identify path
+    unlocated.find
+  end
+end
+

data/lib/scout/resource.rb

@@ -1,6 +1,8 @@
 require_relative 'log'
 require_relative 'path'
 require_relative 'resource/produce'
+require_relative 'resource/path'
+require_relative 'resource/util'
 
 module Resource
   extend MetaExtension