scout-essentials 1.7.1 → 1.8.0

data/lib/scout/log/progress/report.rb

@@ -120,18 +120,18 @@ module Log
       str = Log.color(:magenta, "·")
       if @ticks == 0
         if @max
-          return str << " " << Log.color(:magenta, "waiting on #{@max} #{bytes ? 'bytes' : 'items'}") <<  Log.color(:magenta, " · " << desc)
+          return str + " " << Log.color(:magenta, "waiting on #{@max} #{bytes ? 'bytes' : 'items'}") <<  Log.color(:magenta, " · " + desc)
         else
-          return str << " " << Log.color(:magenta, "waiting - PID: #{Process.pid}") <<  Log.color(:magenta, " · " << desc)
+          return str + " " << Log.color(:magenta, "waiting - PID: #{Process.pid}") <<  Log.color(:magenta, " · " + desc)
         end
       end
-      str << " " << thr_msg
+      str += " " + thr_msg
       if max
         str << Log.color(:blue, " -- ") << eta_msg
       else
         str << Log.color(:blue, " -- ") << ticks.to_s << " #{bytes ? 'bytes' : 'items'}"
       end
-      str <<  Log.color(:magenta, " · " << desc)
+      str <<  Log.color(:magenta, " · " + desc)
       str
     end
 
@@ -208,8 +208,8 @@ module Log
       @last_count = 0
       @last_time = @start
       thr = ellapsed > 0 ? (@ticks / ellapsed).to_i.to_s : 0
-      done_msg << " - " << Log.color(:blue, thr) << " per second"
-      done_msg << Log.color(:magenta, " · " << desc)
+      done_msg << " - " + Log.color(:blue, thr) << " per second"
+      done_msg << Log.color(:magenta, " · " + desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
 
       FileUtils.rm file if file and File.exist?(file)

data/lib/scout/log.rb

@@ -59,7 +59,7 @@ module Log
                IO.console.winsize.last
              rescue Exception
                begin
-                 res = `tput li`
+                 res = `tput cols`
                  res = nil if res == ""
                  res || ENV["TTY_SIZE"] || 80
                rescue Exception
@@ -165,7 +165,7 @@ module Log
     end
   end
 
-  LAST = "log"
+  LAST = "log".dup
   def self.logn(message = nil, severity = MEDIUM, &block)
     return if severity < self.severity
     message ||= block.call if block_given?
@@ -180,8 +180,8 @@ module Log
     else
       prefix = time << color(severity) << "["  << sev_str << "]" << color(0)
     end
-    message = "" << highlight << message << color(0) if severity >= INFO
-    str = prefix << " " << message.to_s
+    message = [highlight, message, color(0)].join if severity >= INFO
+    str = prefix + " " + message.to_s
 
     log_write str
 
@@ -288,9 +288,9 @@ module Log
   end
 
   def self.tsv(tsv, example = false)
-    log_puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
-    log_puts Log.color(:blue, "=> "<< Log.fingerprint(tsv), true)
-    log_puts Log.color(:cyan, "=> " << tsv.summary)
+    log_puts Log.color :magenta, "TSV log: " + Log.last_caller(caller).gsub('`',"'")
+    log_puts Log.color(:blue, "=> " + Log.fingerprint(tsv), true)
+    log_puts Log.color(:cyan, "=> " + tsv.summary)
     if example && ! tsv.empty?
       key = case example
             when TrueClass, :first, "first"

data/lib/scout/misc/digest.rb

@@ -8,15 +8,7 @@ module Misc
       case obj
       when String
         if Path.is_filename?(obj) && Open.exists?(obj)
-          if File.directory?(obj)
-            if Path === obj
-              "Directory MD5: #{digest_str(obj.glob("*"))}"
-            else
-              "Directory MD5: #{digest_str(Dir.glob(File.join(obj, "*")))}"
-            end
-          else
-            "File MD5: #{Misc.digest_file(obj)}"
-          end
+          Path.setup(obj).digest_str
         else
           obj.dup
         end
@@ -27,12 +19,12 @@ module Misc
           length = obj.length
           mid = length/2
           sample_pos = [1, 2, mid, length-2, length-1]
-          "[#{length}:" << obj.values_at(*sample_pos).inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+          "[#{length}:" + obj.values_at(*sample_pos).inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
         else
-          '[' << obj.inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+          '[' + obj.inject("".dup){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
         end
       when Hash
-        '{' << obj.inject(""){|acc,p| s = Misc.digest_str(p.first) << "=" << Misc.digest_str(p.last); acc.empty? ? s : acc << ', ' << s } << '}'
+        '{' + obj.inject(""){|acc,p| s = Misc.digest_str(p.first) + "=" << Misc.digest_str(p.last); acc.empty? ? s : acc << ', ' << s } << '}'
       when Integer
         obj.to_s
       when Float
@@ -49,6 +41,8 @@ module Misc
         "true"
       when FalseClass
         "false"
+      when Proc
+        Misc.digest obj.source_location
       else
         obj.inspect
       end
@@ -63,7 +57,11 @@ module Misc
   def self.file_md5(file)
     file = file.find if Path === file
     file = File.expand_path(file)
-    Digest::MD5.file(file).hexdigest
+    begin
+      Digest::MD5.file(file).hexdigest
+    rescue
+      Digest::MD5.hexdigest(file)
+    end
   end
 
   def self.fast_file_md5(file, sample = 3_000_000)

data/lib/scout/misc/format.rb

@@ -44,7 +44,7 @@ module Misc
     offset ||= 0
 
     i = 0
-    size = size + offset + indent
+    #size = size + offset + indent
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
       text.split(re).collect do |paragraph|
       i += 1
@@ -74,7 +74,7 @@ module Misc
 
   def self.format_definition_list_item(dt, dd,  indent = nil, size = nil, color: :yellow)
     if size.nil?
-      base_size = MAX_TTY_LINE_WIDTH
+      base_size = Log.tty_size || MAX_TTY_LINE_WIDTH
       base_indent = indent || (base_size / 3)
       size = base_size - base_indent
     end

data/lib/scout/misc/system.rb

@@ -1,5 +1,10 @@
 module Misc
 
+  def self.add_libdir(dir=nil)
+    dir ||= File.join(Path.caller_lib_dir(caller.first), 'lib')
+    $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include? dir
+  end
+
   def self.hostname
     @@hostname ||= begin
                      (ENV["HOSTNAME"] || `hostname`).strip

data/lib/scout/open/final.rb

@@ -70,6 +70,12 @@ module Open
     notify_write(file)
   end
 
+  def append(file, content, options = {})
+    options = IndiferentHash.setup options
+    options[:mode] = "a"
+    write(file, content, options)
+  end
+
   def self.mv(source, target, options = {})
     target = target.find if Path === target
     source = source.find if Path === source
@@ -81,6 +87,7 @@ module Open
   end
 
   def self.rm(file)
+    file = file.find if Path === file
     FileUtils.rm(file) if File.exist?(file) || Open.broken_link?(file)
   end
 
@@ -101,6 +108,14 @@ module Open
     end
   end
 
+  def self.mkfiledir(target)
+    target = target.find if Path === target
+    dir = File.dirname(target)
+    if ! File.exist?(dir)
+      FileUtils.mkdir_p dir
+    end
+  end
+
   def self.cp(source, target, options = {})
     source = source.find if Path === source
     target = target.find if Path === target
@@ -188,7 +203,7 @@ module Open
     begin
       Open.ln(source, target, options)
     rescue
-      Log.debug "Could not make regular link, trying symbolic: #{Misc.fingerprint(source)} -> #{Misc.fingerprint(target)}"
+      Log.debug "Could not make regular link, trying symbolic: #{Log.fingerprint(source)} -> #{Log.fingerprint(target)}"
       Open.ln_s(source, target, options)
     end
     nil

data/lib/scout/open/remote.rb

@@ -92,7 +92,6 @@ module Open
         open_cache(url, options)
       end
     rescue
-     STDERR.puts $!.backtrace.inspect
      raise OpenURLError, "Error reading remote url: #{ url }.\n#{$!.message}"
     end
   end

data/lib/scout/open/stream.rb

@@ -204,6 +204,7 @@ module Open
 
   def self.purge_pipes(*save)
     PIPE_MUTEX.synchronize do
+      save = save.flatten
       OPEN_PIPE_IN.each do |pipe|
         next if save.include? pipe
         pipe.close unless pipe.closed?
@@ -361,7 +362,7 @@ module Open
     end
 
     main_pipe.abort_callback = proc do |exception|
-      stream.abort(exception)
+      stream.abort(exception) if ConcurrentStream === stream
       out_pipes[1..-1].each do |sout|
         sout.abort(exception)
       end
@@ -398,10 +399,11 @@ module Open
   end
 
   def self.read_stream(stream, size)
-    str = ""
+    str = "".dup
     while str.length < size
       missing = size - str.length
       more = stream.read(missing)
+      raise ClosedStream if more.nil?
       str << more
     end
     str
@@ -464,15 +466,15 @@ module Open
               if compact
                 current_parts[i] = part.dup
               else
-                current_parts[i] = "|" << part
+                current_parts[i] = "|" + part
               end
             else
-              current_parts[i] = current_parts[i] << "|" << part
+              current_parts[i] = current_parts[i] + "|" << part
             end
           end
 
           (parts.length..current_parts.length-1).to_a.each do |pos|
-            current_parts[pos] = current_parts[pos] << "|" << ""
+            current_parts[pos] = current_parts[pos] + "|" << ""
           end
         when current_key.nil?
           current_key = key
@@ -498,4 +500,27 @@ module Open
       end unless current_key.nil?
     end
   end
+
+  def self.line_monitor_stream(stream, &block)
+    monitor, out = tee_stream stream
+    monitor_thread = Thread.new do
+      begin
+        while line = monitor.gets
+          block.call line
+        end
+      rescue
+        monitor.abort $!
+        monitor.close unless monitor.closed?
+        monitor.join if monitor.respond_to?(:join) && ! monitor.aborted?
+        out.raise $! if out.respond_to?(:raise)
+      ensure
+        monitor.close unless monitor.closed?
+        monitor.join if monitor.respond_to?(:join) && ! monitor.aborted?
+      end
+    end
+    monitor_thread.report_on_exception = false
+
+    stream.annotate out if stream.respond_to? :annotate
+    ConcurrentStream.setup out, :threads => monitor_thread
+  end
 end

data/lib/scout/open/util.rb

@@ -1,3 +1,4 @@
+require 'pathname'
 module Open
   GREP_CMD = begin
                if ENV["GREP_CMD"] 
@@ -129,4 +130,35 @@ module Open
     file = file.produce_and_find if Path === file
     Open.read(file).split("\n")
   end
+
+  def self.wait_for(path, timeout: 10)
+    return if Open.exists?(path)
+
+    path = path.find if Path === path
+
+    require 'listen'
+    require 'timeout'
+
+    dir  = File.dirname(path)
+    name = File.basename(path)
+
+    queue = Queue.new
+    listener = Listen.to(dir) do |_modified, added, _removed|
+      queue << :found if added.include?(path)
+    end
+
+    listener.start
+
+    begin
+      Timeout.timeout(timeout) do
+        # This blocks until something is pushed to the queue
+        queue.pop
+      end
+      true
+    rescue Timeout::Error
+      false
+    ensure
+      listener.stop
+    end
+  end
 end

data/lib/scout/path/digest.rb

@@ -3,11 +3,21 @@ module Path
   def digest_str
     case
     when File.directory?(self)
-      "Directory MD5: #{Misc.digest_str(self.glob("*"))}"
+      files = self.glob("*")
+
+      files = files.reject{|f| File.directory?(f) }
+
+      files = Annotation.purge files
+
+      if files.length > 10
+        "Directory MD5: #{files.length} #{Misc.digest_str(files*"\n")}"
+      else
+        "Directory MD5: #{Misc.digest_str(files)}"
+      end
     when self.located? && File.exist?(self)
       "File MD5: #{Misc.digest_file(self)}"
     else
-      '\'' << self << '\''
+      '\'' + self << '\''
     end
   end
 end

data/lib/scout/path/find.rb

@@ -6,12 +6,15 @@ module Path
       caller_dup = caller.dup
       while file = caller_dup.shift
         break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
-          file =~ /(?:scout|rbbt)\/(?:.*\/)?(path|open|final|tsv|refactor)\.rb/ or
+          file =~ /(?:scout|rbbt)\/(?:.*\/)?(path|open|final|tsv|refactor|hook|parser)\.rb/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
           file =~ /(?:scout|rbbt)\/persist.rb/ or
+          file =~ /(?:scout|rbbt)\/(persist|workflow)\// or
           file =~ /scout\/resource\/produce.rb/ or
-          file =~ /modules\/rbbt-util/
+          file =~ /modules\/rbbt-util/ or
+          file =~ /^<internal:/
       end
+
       return nil if file.nil?
       file = file.sub(/\.rb[^\w].*/,'.rb')
     end
@@ -65,6 +68,8 @@ module Path
       sub('{REMOVE}/', '').
       sub('{REMOVE}', '')
 
+    file.sub!(/\/$/,'') if path._subpath.nil?
+
     while true
       file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
         key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
@@ -281,10 +286,18 @@ module Path
       .select{|file| file.exist? }.uniq
   end
 
-  def find_with_extension(extension, *args)
+  def find_with_extension(extension, *args, produce: true)
     found = self.find(*args)
-    return found if found.exists?
-    found_with_extension = self.set_extension(extension).find
-    found_with_extension.exists? ? found_with_extension : found
+    return found if found.exists?(produce: produce) && ! found.directory?
+    if Array === extension
+      extension.each do |ext|
+        found_with_extension = self.set_extension(ext).find
+        return found_with_extension if found_with_extension.exists?(produce: produce)
+      end
+    else
+      found_with_extension = self.set_extension(extension).find
+      return found_with_extension if found_with_extension.exists?(produce: produce)
+    end
+    return found
   end
 end

data/lib/scout/path/util.rb

@@ -15,6 +15,14 @@ module Path
     return false
   end
 
+  def self.can_read?(string)
+    return true if Open.remote?(string)
+    return false unless Path.is_filename?(string)
+    string = string.find if Path === string
+    return false if File.directory?(string)
+    return true
+  end
+
   def self.sanitize_filename(filename, length = 254)
     if filename.length > length
       if filename =~ /(\..{2,9})$/
@@ -37,6 +45,14 @@ module Path
     File.directory?(self.find)
   end
 
+  def realpath
+    File.realpath(self.find)
+  end
+
+  def relative_to(dir)
+    Misc.path_relative_to(File.expand_path(dir), self.find)
+  end
+
   def sub(*args)
     self.annotate super(*args)
   end
@@ -71,12 +87,17 @@ module Path
     end
   end
 
+  def glob_names(...)
+    glob(...).collect{|f| f.basename }
+  end
+
   def glob_all(pattern = nil, caller_lib = nil, search_paths = nil)
     search_paths ||= self.path_maps || Path.path_maps
     search_paths = search_paths.dup
 
     search_paths.keys.collect do |where| 
       found = find(where)
+      next unless found.located?
 
       paths = pattern ? found.glob(pattern) : found.glob
 
@@ -135,8 +156,17 @@ module Path
     self.annotate(new_path)
   end
 
+  def newer_files(*files)
+    files.flatten.select do |file|
+      Path.newer?(self, file)
+    end
+  end
 
-  # Is 'file' newer than 'path'? return non-true if path is newer than file
+  def outdated?(...)
+    newer_files(...).any?
+  end
+
+  # Is 'file' newer than 'path'? return false if path is newer than file
   def self.newer?(path, file, by_link = false)
     return true if not Open.exists?(file)
     path = path.find if Path === path
@@ -153,4 +183,10 @@ module Path
     return diff if diff < 0
     return false
   end
+
+  def final_pkgdir
+    final_pkgdir = self.pkgdir
+    final_pkgdir = final_pkgdir.pkgdir while final_pkgdir.respond_to?(:pkgdir)
+    final_pkgdir
+  end
 end

data/lib/scout/persist/open.rb

@@ -4,10 +4,12 @@ require 'yaml'
 
 module Open
   def self.json(file)
+    file = file.find_with_extension :json if Path === file
     Open.open(file){|f| JSON.load(f) }
   end
 
   def self.yaml(file)
+    file = file.find_with_extension :yaml if Path === file
     Open.open(file){|f| YAML.unsafe_load(f) }
   end
 

data/lib/scout/persist.rb

@@ -8,6 +8,7 @@ module Persist
     def cache_dir=(cache_dir)
       @cache_dir = Path === cache_dir ? cache_dir : Path.setup(cache_dir)
     end
+
     def cache_dir
       @cache_dir ||= Path.setup("var/cache/persistence")
     end
@@ -34,11 +35,16 @@ module Persist
 
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
     data = persist_options[:data] || options[:data]
+    check = persist_options[:check] || options[:check]
     no_load = persist_options[:no_load] || options[:no_load]
 
     update = options[:update] || persist_options[:update]
     update = Open.mtime(update) if Path === update
-    update = Open.mtime(file) >= update ? false : true if Time === update
+
+    update = true if file.outdated?(check) if Path === file && check
+    file_mtime = Open.mtime(file)
+    update = file_mtime >= update ? false : true if file_mtime && Time === update
+    update = file_mtime >= (Time.now - update) ? false : true if file_mtime && Numeric === update
 
     if type == :memory
       repo = options[:memory] || options[:repo] || MEMORY_CACHE

data/lib/scout/resource/path.rb

@@ -74,8 +74,8 @@ module Path
     Open.list(found)
   end
 
-  def exists?
+  def exists?(produce: true)
     return true if Open.exists?(self.find)
-    self.produce
+    self.produce if produce
   end
 end

data/lib/scout/resource/util.rb

@@ -15,6 +15,8 @@ module Resource
     path = File.expand_path(path) if path.start_with?('/')
     path += "/" if File.directory?(path) and not path.end_with?('/')
 
+    self_pkgdir = self.final_pkgdir
+
     choices = []
     map_order.uniq.each do |name|
       pattern = path_maps[name]
@@ -31,7 +33,7 @@ module Resource
           .gsub(/\/{([^}]+)}/,'(?:/(?<\1>[^/]+))?') +
         "(?:/(?<REST>.+))?/?$"
         if m = path.match(regexp) 
-          if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == self.pkgdir
+          if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == self_pkgdir
 
             unlocated = %w(TOPLEVEL SUBPATH PATH REST).collect{|c| 
               m.named_captures.include?(c) ? m[c] : nil
@@ -40,10 +42,11 @@ module Resource
             if self.subdir && ! self.subdir.empty?
               subdir = self.subdir
               subdir += "/" unless subdir.end_with?("/")
-              unlocated[subdir] = "" 
+              unlocated[subdir] = ""
             end
 
-            choices << self.annotate(unlocated)
+            unlocated = path.annotate(unlocated) if Path === path
+            choices << unlocated
           end
         end
       end
@@ -51,13 +54,24 @@ module Resource
 
     identified = choices.sort_by{|s| s.length }.first
 
-    Path.setup(identified || path, self, nil, path_maps)
+    identified ||= path
+
+    home = ENV['HOME']
+    if home
+      home += home[0..-2] if home.end_with?('/')
+      identified = identified.sub(/^#{home}(\/|$)/, '~\1')
+    end
+
+    Path.setup(identified, self, nil, path_maps) unless Path === identified
+
+    identified
   end
 
   def self.identify(path)
     resource = path.pkgdir if Path === path
     resource = Resource.default_resource unless Resource === resource
     unlocated = resource.identify path
+    unlocated
   end
 
   def self.relocate(path)

data/lib/scout/resource.rb

@@ -33,7 +33,21 @@ module Resource
   end
 
   def path_maps
-    @path_maps ||= Path.path_maps
+    @path_maps ||= Path.path_maps.dup
+  end
+
+  def map_order
+    @map_order ||= Path.map_order.dup
+  end
+
+  def prepend_path(name, map)
+    path_maps[name] = map
+    map_order.unshift(name.to_sym)
+  end
+
+  def append_path(name, map)
+    path_maps[name] = map
+    map_order.push(name.to_sym)
   end
 
   def subdir

data/lib/scout/simple_opt/parse.rb

@@ -35,6 +35,8 @@ module SOPT
   end
 
   def self.register(short, long, asterisk, description)
+    long = long.to_s if Symbol === long
+    short = short.to_s if Symbol === short
     short = fix_shortcut(short, long)
     shortcuts[short] = long if short
     inputs << long

data/lib/scout/tmpfile.rb

@@ -124,7 +124,7 @@ module TmpFile
 
     filename = filename[0..MAX_FILE_LENGTH] << Misc.digest(filename[MAX_FILE_LENGTH+1..-1]) if filename.length > MAX_FILE_LENGTH + 10
 
-    filename  += ":" << Misc.digest(clean_options) unless clean_options.empty?
+    filename  += ":" + Misc.digest(clean_options) unless clean_options.empty?
 
     persistence_dir[filename]
   end