scout-essentials 1.3.1 → 1.5.0

data/lib/scout/misc/math.rb

@@ -0,0 +1,109 @@
+module Misc
+
+  Log2Multiplier = 1.0 / Math.log(2.0)
+  Log10Multiplier = 1.0 / Math.log(10.0)
+  def self.log2(x)
+    Math.log(x) * Log2Multiplier
+  end
+
+  def self.log10(x)
+    Math.log(x) * Log10Multiplier
+  end
+
+  def self.max(list)
+    max = nil
+    list.each do |v|
+      next if v.nil?
+      max = v if max.nil? or v > max
+    end
+    max
+  end
+
+  def self.min(list)
+    min = nil
+    list.each do |v|
+      next if v.nil?
+      min = v if min.nil? or v < min
+    end
+    min
+  end
+
+  def self.std_num_vector(v, min, max)
+    v_min = Misc.min(v)
+    v_max = Misc.max(v)
+    v_range = v_max - v_min
+    range = max.to_f - min.to_f
+
+    v.collect{|e| (e.nil? || e.nan?) ? e : min + range * (e.to_f - v_min) / v_range } 
+  end
+
+  def self.sum(list)
+    list.compact.inject(0.0){|acc,e| acc += e }
+  end
+
+  def self.mean(list)
+    sum(list.compact.collect{|v| v.to_f } ) / list.compact.length
+  end
+
+  def self.median(array)
+    sorted = array.sort
+    len = sorted.length
+    (sorted[(len - 1) / 2] + sorted[len / 2]).to_f / 2
+  end
+
+  def self.variance(list)
+    return nil if list.length < 3
+    mean = mean(list)
+    list = list.compact
+    list_length = list.length
+
+    total_square_distance = 0.0
+    list.each do |value|
+      distance = value.to_f - mean
+      total_square_distance += distance * distance
+    end
+
+    total_square_distance / (list_length - 1)
+  end
+
+
+  def self.sd(list)
+    return nil if list.length < 3
+    variance = self.variance(list)
+    Math.sqrt(variance)
+  end
+
+  def self.counts(array)
+    counts = {}
+    array.each do |e|
+      counts[e] ||= 0
+      counts[e] += 1
+    end
+
+    counts
+  end
+
+  def self.proportions(array)
+    total = array.length
+
+    proportions = Hash.new 0
+
+    array.each do |e|
+      proportions[e] += 1.0 / total
+    end
+
+    class << proportions; self; end.class_eval do
+      def to_s
+        sort{|a,b| a[1] == b[1] ? a[0] <=> b[0] : a[1] <=> b[1]}.collect{|k,c| "%3d\t%s" % [c, k]} * "\n"
+      end
+    end
+
+    proportions
+  end
+
+  def self.zscore(e, list)
+    m = Misc.mean(list)
+    sd = Misc.sd(list)
+    (e.to_f - m) / sd
+  end
+end

data/lib/scout/misc/system.rb

@@ -13,6 +13,22 @@ module Misc
     Sys::ProcTable.ps.select{ |pe| pe.ppid == ppid }
   end
 
+  def self.wait_child(pid)
+    begin
+      Process.waitpid2 pid.to_i
+    rescue Errno::ECHILD
+    end
+  end
+
+  def self.abort_child(pid, wait = true)
+    begin
+      Process.kill("TERM", pid.to_i)
+      wait_child(pid) if wait
+    rescue
+      Log.debug("Process #{pid} was not killed: #{$!.message}")
+    end
+  end
+
   def self.env_add(var, value, sep = ":", prepend = true)
     if ENV[var].nil?
       ENV[var] = value
@@ -37,8 +53,8 @@ module Misc
     end
   end
   
-  def self.update_git(gem_name = 'scout-gear')
-    gem_name = 'scout-gear' if gem_name.nil?
+  def self.update_git(gem_name = 'scout-essentials')
+    gem_name = 'scout-essentials' if gem_name.nil?
     dir = File.join(__dir__, '../../../../', gem_name)
     return unless Open.exist?(dir)
     Misc.in_dir dir do

data/lib/scout/misc.rb

@@ -5,6 +5,7 @@ require_relative 'misc/filesystem'
 require_relative 'misc/monitor'
 require_relative 'misc/system'
 require_relative 'misc/helper'
+require_relative 'misc/math'
 
 module Misc
 end

data/lib/scout/named_array.rb

@@ -1,7 +1,7 @@
-require_relative 'meta_extension'
+require_relative 'annotation'
 module NamedArray
-  extend MetaExtension
-  extension_attr :fields, :key
+  extend Annotation
+  annotation :fields, :key
 
   def all_fields
     [key, fields].compact.flatten
@@ -9,9 +9,11 @@ module NamedArray
 
   def self.field_match(field, name)
     if (String === field) && (String === name)
-      field == name || 
-        field.start_with?(name) || field.include?("(" + name + ")") ||
-        name.start_with?(field) || name.include?("(" + field + ")")
+      return true if field == name
+      return true if field.include?("(" + name + ")") 
+      return true if name.include?("(" + field + ")")
+      return true if field.start_with?(name + " ")
+      return true if name.start_with?(field + " ")
     else
       field == name
     end

data/lib/scout/open/remote.rb

@@ -87,7 +87,12 @@ module Open
   end
 
   def self.download(url, file)
-    CMD.cmd_log(:wget, "'#{url}' -O '#{file}'")
+    begin
+      CMD.cmd_log(:wget, "'#{url}' -O '#{file}'")
+    rescue
+      FileUtils.rm(file) if File.exist?(file)
+      raise $!
+    end
   end
 
   def self.digest_url(url, options = {})
@@ -107,7 +112,7 @@ module Open
       nil
     end
   end
- 
+
   def self.remove_from_cache(url, options = {})
     filename = cache_file(url, options)
     if File.exist? filename
@@ -116,7 +121,7 @@ module Open
       nil
     end
   end
-  
+
   def self.add_cache(url, data, options = {})
     filename = cache_file(url, options)
     Open.sensible_write(filename, data, :force => true)

data/lib/scout/open/stream.rb

@@ -126,7 +126,11 @@ module Open
               end
             end
           else
-            File.open(tmp_path, 'wb') do |f|  end
+            if content.respond_to?(:write_file)
+              content.write_file(tmp_path)
+            else
+              File.open(tmp_path, 'wb') do |f| f.write content.to_s  end
+            end
           end
 
           begin

data/lib/scout/open/util.rb

@@ -22,7 +22,7 @@ module Open
         end
       end
     else
-      CMD.cmd("#{GREP_CMD} #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
+      CMD.cmd("#{GREP_CMD} #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :nofail => true, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
     end
   end
 

data/lib/scout/path/find.rb

@@ -7,7 +7,7 @@ module Path
       caller_dup = caller.dup
       while file = caller_dup.shift
         break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
-          file =~ /(?:scout|rbbt)\/(?:.*\/)?(path|open)\.rb/ or
+          file =~ /(?:scout|rbbt)\/(?:.*\/)?(path|open|tsv|refactor)\.rb/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
           file =~ /(?:scout|rbbt)\/persist.rb/ or
           file =~ /scout\/resource\/produce.rb/ or
@@ -33,7 +33,13 @@ module Path
   end
 
   def self.follow(path, map, map_name = nil)
-    file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
+    map = File.join(map, '{PATH}') unless map.include?("{")
+    if path.respond_to?(:pkgdir)
+      pkgdir = path.pkgdir
+      pkgdir = pkgdir.pkgdir while pkgdir.respond_to?(:pkgdir)
+    end
+    pkgdir = Path.default_pkgdir if pkgdir.nil?
+    file = map.sub('{PKGDIR}', pkgdir).
       sub('{HOME}', ENV["HOME"]).
       sub('{RESOURCE}', path.pkgdir.to_s).
       sub('{PWD}', FileUtils.pwd).
@@ -69,7 +75,7 @@ module Path
       :cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
       :bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
       :lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
-      :scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
+      :scout_essentials => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
       :tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
       :default => :user
     })
@@ -84,10 +90,30 @@ module Path
   end
 
   def self.add_path(name, map)
-    @@path_maps[name] = map
+    path_maps[name] = map
     @@map_order = nil
   end
 
+  def self.prepend_path(name, map)
+    path_maps[name] = map
+    map_order.unshift(name.to_sym)
+  end
+
+  def self.append_path(name, map)
+    path_maps[name] = map
+    map_order.push(name.to_sym)
+  end
+
+  def prepend_path(name, map)
+    path_maps[name] = map
+    map_order.unshift(name.to_sym)
+  end
+
+  def append_path(name, map)
+    path_maps[name] = map
+    map_order.push(name.to_sym)
+  end
+
   def self.load_path_maps(filename)
     Path.setup(filename) unless Path === filename
     if filename.exist?
@@ -135,10 +161,6 @@ module Path
     @original
   end
 
-  def map_order
-    @map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
-  end
-
   def follow(map_name = :default, annotate = true)
     IndiferentHash.setup(path_maps)
     map = path_maps[map_name] || Path.path_maps[map_name]

data/lib/scout/path/util.rb

@@ -64,7 +64,7 @@ module Path
   end
 
   def glob_all(pattern = nil, caller_lib = nil, search_paths = nil)
-    search_paths ||= Path.path_maps
+    search_paths ||= self.path_maps || Path.path_maps
     search_paths = search_paths.dup
 
     search_paths.keys.collect do |where| 

data/lib/scout/path.rb

@@ -1,11 +1,11 @@
-require_relative 'meta_extension'
+require_relative 'annotation'
 require_relative 'path/find'
 require_relative 'path/util'
 require_relative 'path/tmpfile'
 
 module Path
-  extend MetaExtension
-  extension_attr :pkgdir, :libdir, :path_maps
+  extend Annotation
+  annotation :pkgdir, :libdir, :path_maps, :map_order
 
   def self.default_pkgdir
     @@default_pkgdir ||= 'scout'
@@ -27,7 +27,13 @@ module Path
     @path_maps ||= Path.path_maps.dup
   end
 
-  def join(subpath, prevpath = nil)
+  def map_order
+    @map_order ||= (path_maps.keys & Path.map_order) + (path_maps.keys - Path.map_order)
+  end
+
+  def join(subpath = nil, prevpath = nil)
+    return self if subpath.nil?
+
     subpath = subpath.to_s if Symbol === subpath
     prevpath = prevpath.to_s if Symbol === prevpath
 

data/lib/scout/persist/open.rb

@@ -8,7 +8,7 @@ module Open
   end
 
   def self.yaml(file)
-    Open.open(file){|f| YAML.load(f) }
+    Open.open(file){|f| YAML.unsafe_load(f) }
   end
 
   def self.marshal(file)

data/lib/scout/persist/serialize.rb

@@ -34,6 +34,8 @@ module Persist
       content.to_json
     when :marshal
       Marshal.dump(content)
+    when :annotation, :annotations
+      Annotation.tsv(content, :all).to_s
     else
       if m = type.to_s.match(/(.*)_array/)
         type = m[1].to_sym
@@ -69,6 +71,8 @@ module Persist
       JSON.parse(serialized)
     when :marshal
       Marshal.load(serialized)
+    when :annotation, :annotations
+      Annotation.load_tsv(TSV.open(serialized))
     else
       if m = type.to_s.match(/(.*)_array/)
         type = m[1].to_sym
@@ -110,7 +114,7 @@ module Persist
         f.puts content
       end
       content
-   else
+    else
       serialized = serialize(content, type)
       Open.sensible_write(file, serialized, :force => true)
       return nil

data/lib/scout/persist.rb

@@ -31,41 +31,65 @@ module Persist
   def self.persist(name, type = :serializer, options = {}, &block)
     persist_options = IndiferentHash.pull_keys options, :persist 
     return yield if FalseClass === persist_options[:persist]
+
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
+    data = persist_options[:data] || options[:data]
+    no_load = persist_options[:no_load] || options[:no_load]
+
+    update = options[:update] || persist_options[:update]
+    update = Open.mtime(update) if Path === update
+    update = Open.mtime(file) >= update ? false : true if Time === update
 
     if type == :memory
       repo = options[:memory] || options[:repo] || MEMORY_CACHE
-      repo[file] ||= yield
+      if update
+        repo[file] = yield
+      else
+        repo[file] ||= yield
+      end
       return repo[file]
     end
 
-    update = options[:update] || persist_options[:update]
-    update = Open.mtime(update) if Path === update
-    update = Open.mtime(file) >= update ? false : true if Time === update
-
-    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
+    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir}) if String === file
 
     Open.lock lockfile do |lock|
       if Open.exist?(file) && ! update
-        Persist.load(file, type)
+        if TrueClass === no_load
+          file
+        else
+          Persist.load(file, type)
+        end
       else
         begin
+          Open.rm(file.find) if update && Open.exists?(file)
+
           file = file.find if Path === file
-          return yield(file) if block.arity == 1
-          res = yield
+          if block.arity == 1
+            if data
+              yield(data)
+              res = data
+            else
+              return yield(file)
+            end
+          else
+            res = yield
+          end
 
           if res.nil?
-            if type.nil?
-              Log.debug "Empty result and no persist type; not loading result file"
-              return nil
+            if no_load
+              Log.debug "Empty result and no_load is '#{no_load}'"
+              return file
             else
-              Log.debug "Empty result; loading #{type} result from file"
-              return Persist.load(file, type)
+              if type.nil?
+                Log.debug "Empty result and no persist type; not loading result file"
+                return nil
+              else
+                Log.debug "Empty result; loading #{type} result from file"
+                return Persist.load(file, type)
+              end
             end
           end
 
-          Open.rm(file)
-
           if IO === res || StringIO === res
             tee_copies = options[:tee_copies] || 1
             main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
@@ -90,20 +114,25 @@ module Persist
           Thread.handle_interrupt(Exception => :never) do
             if Open.exist?(file)
               Log.debug "Failed persistence #{file} - erasing"
-              Open.rm file
+              Open.rm_rf file
             else
               Log.debug "Failed persistence #{file}"
             end
-          end
+          end unless DontPersist === $!
           raise $! unless options[:canfail]
         end
-        res
+        
+        if TrueClass === no_load
+          file
+        else
+          res
+        end
       end
     end
   end
 
   def self.memory(name, options = {}, &block)
-    options[:persist_path] ||= options[:path] ||= [name, options[:key]].compact * ":"
+    options[:persist_path] ||= options[:path] ||= [name, options[:key]].compact * ":" if options[:key]
     self.persist(name, :memory, options, &block)
   end
 

data/lib/scout/resource/path.rb

@@ -1,7 +1,7 @@
 module Path
   def produce(force = false)
     raise @produced if Exception === @produced
-    return self if ! force && (Open.exist?(self) || @produced)
+    return self if ! force && (Open.exist?(self.find) || @produced)
     begin
       if Resource === self.pkgdir
         self.pkgdir.produce self, force

data/lib/scout/resource/scout.rb

@@ -1,5 +1,7 @@
 module Scout
   extend Resource
+
+  self.pkgdir = 'scout'
 end
 
 Path.load_path_maps(Scout.etc["path_maps"])

data/lib/scout/resource/util.rb

@@ -20,19 +20,24 @@ module Resource
       if String ===  pattern and pattern.include?('{')
         regexp = "^" + pattern
           .gsub(/{(TOPLEVEL)}/,'(?<\1>[^/]+)')
-          .gsub(/{([^}]+)}/,'(?<\1>[^/]+)?') +
-        "(?:/(?<REST>.*))?/?$"
+          .gsub(/\.{(PKGDIR)}/,'\.(?<\1>[^/]+)')
+          .gsub(/\/{([^}]+)}/,'(?:/(?<\1>[^/]+))?') +
+        "(?:/(?<REST>.+))?/?$"
         if m = path.match(regexp) 
           if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == self.pkgdir
+
             unlocated = %w(TOPLEVEL SUBPATH PATH REST).collect{|c| 
               m.named_captures.include?(c) ? m[c] : nil
             }.compact * "/"
+
             unlocated.gsub!(/\/+/,'/')
+
             if self.subdir && ! self.subdir.empty?
               subdir = self.subdir
               subdir += "/" unless subdir.end_with?("/")
               unlocated[subdir] = "" 
             end
+
             choices << self.annotate(unlocated)
           end
         end
@@ -46,7 +51,7 @@ module Resource
 
   def self.identify(path)
     resource = path.pkgdir if Path === path
-    resource = Scout unless Resource === resource
+    resource = Resource.default_resource unless Resource === resource
     unlocated = resource.identify path
   end
 

data/lib/scout/resource.rb

@@ -7,13 +7,25 @@ require_relative 'resource/util'
 require_relative 'resource/software'
 
 module Resource
-  extend MetaExtension
-  extension_attr :pkgdir, :libdir, :subdir, :resources, :rake_dirs, :path_maps, :lock_dir
+  extend Annotation
+  annotation :pkgdir, :libdir, :subdir, :resources, :rake_dirs, :path_maps, :map_order, :lock_dir
+
+  class << Resource
+    attr_accessor :default_resource
+
+    def default_resource
+      @default_resource ||= Scout
+    end
+  end
 
   def self.default_lock_dir
     Path.setup('tmp/produce_locks').find
   end
 
+  def path_maps
+    @path_maps ||= Path.path_maps.dup
+  end
+
   def subdir
     @subdir ||= ""
   end
@@ -27,7 +39,7 @@ module Resource
   end
 
   def root
-    Path.setup(subdir, self, self.libdir, @path_maps)
+    Path.setup(subdir.dup, self, self.libdir, @path_maps, @map_order)
   end
 
   def method_missing(name, prev = nil, *args)