scout-gear 1.2.0 → 5.1.1

data/lib/scout/open.rb

@@ -0,0 +1,169 @@
+require_relative 'tmpfile'
+require_relative 'path'
+require_relative 'cmd'
+
+require_relative 'open/stream'
+require_relative 'open/util'
+require_relative 'open/remote'
+require_relative 'open/lock'
+
+module Open
+  module NamedStream
+    attr_accessor :filename
+
+    def digest_str
+      if Path === filename && ! filename.located?
+        filename
+      else
+        Misc.file_md5(filename)
+      end
+    end
+  end
+
+  def self.get_stream(file, mode = 'r')
+    file = file.find if Path === file
+
+    return Open.ssh(file) if Open.ssh?(file)
+    return Open.wget(file) if Open.remote?(file)
+
+    File.open(file, mode)
+  end
+
+  def self.file_open(file, grep = false, mode = 'r', invert_grep = false)
+    Open.mkdir File.dirname(file) if mode.include? 'w'
+
+    stream = get_stream(file, mode)
+
+    if grep
+      grep(stream, grep, invert_grep)
+    else
+      stream
+    end
+  end
+
+  def self.file_write(file, content, mode = 'w')
+    File.open(file, mode) do |f|
+      begin
+        f.flock(File::LOCK_EX)
+        f.write content 
+        f.flock(File::LOCK_UN)
+      ensure
+        f.close unless f.closed?
+      end
+    end
+  end
+
+  def self.open(file, options = {})
+    if IO === file || StringIO === file
+      if block_given?
+        res = yield file 
+        file.close
+        return res
+      else
+        return file 
+      end
+    end
+
+    options = IndiferentHash.add_defaults options, :noz => false, :mode => 'r'
+
+    mode = IndiferentHash.process_options options, :mode
+
+    options[:noz] = true if mode.include? "w"
+
+    io = file_open(file, options[:grep], mode, options[:invert_grep])
+
+    io = unzip(io)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
+    io = gunzip(io)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
+    io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
+
+    io.extend NamedStream
+    io.filename = file
+
+    if block_given?
+      res = nil
+      begin
+        res = yield(io)
+      rescue DontClose
+        res = $!.payload
+      rescue Exception
+        io.abort if io.respond_to? :abort
+        io.join if io.respond_to? :join
+        raise $!
+      ensure
+        io.close if io.respond_to? :close and not io.closed?
+        io.join if io.respond_to? :join
+      end
+      res
+    else
+      io
+    end
+  end
+
+  def self.read(file, options = {}, &block)
+    open(file, options) do |f|
+      if block_given?
+        res = []
+        while not f.eof?
+          l = f.gets
+          l = Misc.fixutf8(l) unless options[:nofix]
+          res << yield(l)
+        end
+        res
+      else
+        if options[:nofix]
+          f.read
+        else
+          Misc.fixutf8(f.read)
+        end
+      end
+    end
+  end
+
+  def self.write(file, content = nil, options = {})
+    options = IndiferentHash.add_defaults options, :mode => 'w'
+
+    file = file.find(options[:where]) if Path === file
+    mode = IndiferentHash.process_options options, :mode
+
+    FileUtils.mkdir_p File.dirname(file)
+
+    case
+    when block_given?
+      begin
+        f = File.open(file, mode)
+        begin
+          yield f
+        ensure
+          f.close unless f.closed?
+        end
+      rescue Exception
+        FileUtils.rm file if File.exist? file
+        raise $!
+      end
+    when content.nil?
+      file_write(file, "", mode)
+    when String === content
+      file_write(file, content, mode)
+    when (IO === content || StringIO === content)
+      begin
+        File.open(file, mode) do |f| 
+          f.flock(File::LOCK_EX)
+          while block = content.read(Open::BLOCK_SIZE)
+            f.write block
+          end
+          f.flock(File::LOCK_UN)
+        end
+      rescue Exception
+        FileUtils.rm_rf file if File.exist? file
+        raise $!
+      end
+      content.close unless content.closed?
+      content.join if content.respond_to? :join
+    else
+      raise "Content unknown #{Log.fingerprint content}"
+    end
+
+    notify_write(file) 
+  end
+
+end

data/lib/scout/path/find.rb

@@ -1,27 +1,57 @@
+require_relative '../indiferent_hash'
 module Path
-  def _parts
-    @_parts ||= self.split("/")
-  end
 
-  def _subpath
-    @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0]
-  end
-  
-  def _toplevel
-    @toplevel ||= _parts.length > 1 ? _parts[0] : nil
+  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
+    
+    if file.nil?
+      caller_dup = caller.dup
+      while file = caller_dup.shift
+        break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
+          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
+          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
+          file =~ /(?:scout|rbbt)\/persist.rb/
+      end
+      file = file.sub(/\.rb[^\w].*/,'.rb')
+    end
+
+    relative_to = [relative_to] unless Array === relative_to
+    file = File.expand_path(file)
+    return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
+
+    while file != '/'
+      dir = File.dirname file
+
+      return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
+
+      file = File.dirname file
+    end
+
+    return nil
   end
 
-  def self.follow(path, map)
-    map.sub('{PKGDIR}', path.pkgdir || Path.default_pkgdir).
-      sub('{RESOURCE}', path.to_s).
+  def self.follow(path, map, map_name = nil)
+    file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
+      sub('{RESOURCE}', path.pkgdir.to_s).
       sub('{PWD}', FileUtils.pwd).
       sub('{TOPLEVEL}', path._toplevel).
       sub('{SUBPATH}', path._subpath).
       sub('{BASENAME}', File.basename(path)).
       sub('{PATH}', path).
-      sub('{LIBDIR}', path.libdir || Path.caller_lib_dir).
+      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
+      sub('{MAPNAME}', map_name.to_s).
       sub('{REMOVE}/', '').
       sub('{REMOVE}', '').gsub(/\/+/,'/')
+
+    while true
+      file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
+        key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
+        key_text = follow(path, "#{key}}", map_name)
+        key_text[orig] = replace[0..-2] if key_text.include?(orig)
+        key_text
+      end || break
+    end
+
+    file
   end
 
   def self.path_maps
@@ -40,19 +70,36 @@ module Path
     })
   end
 
-  def self.map_search
-    @@map_search ||= %w(current workflow user local global lib fast cache bulk)
+  def self.basic_map_order
+    @@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
   end
 
-  def self.search_order
-    @@search_order ||= (path_maps.keys & map_search) + (path_maps.keys - map_search)
+  def self.map_order
+    @@map_order ||= (path_maps.keys & basic_map_order) + (path_maps.keys - basic_map_order)
+  end
+
+  def self.add_path(name, map)
+    @@path_maps[name] = map
+    @@map_order = nil
+  end
+
+  def _parts
+    @_parts ||= self.split("/")
+  end
+
+  def _subpath
+    @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
+  end
+  
+  def _toplevel
+    @toplevel ||= _parts.length > 1 ? _parts[0] : ""
   end
 
   SLASH = "/"[0]
   DOT = "."[0]
   def located?
     # OPEN RESOURCE
-    self.slice(0,1) == SLASH || (self.char(0,1) == DOT && self.char(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
+    self.slice(0,1) == SLASH || (self.slice(0,1) == DOT && self.slice(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
   end
 
   def annotate_found_where(found, where)
@@ -70,13 +117,19 @@ module Path
     @original
   end
 
+  def map_order
+    @map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
+  end
+
   def follow(map_name = :default, annotate = true)
-    map = Path.path_maps[map_name]
+    IndiferentHash.setup(path_maps)
+    map = path_maps[map_name]
+    raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
     while Symbol === map
       map_name = map
-      map = Path.path_maps[map_name]
+      map = path_maps[map_name]
     end
-    found = Path.follow(self, map)
+    found = Path.follow(self, map, map_name)
 
     annotate_found_where(found, map_name)  if annotate
 
@@ -85,13 +138,13 @@ module Path
 
   def find(where = nil)
     return self if located?
+    return find_all if where == 'all' || where == :all
     return follow(where) if where
 
-
-    Path.search_order.each do |map_name|
+    map_order.each do |map_name|
       found = follow(map_name, false)
 
-      return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(real_path)
+      return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
     end
 
     return follow(:default)
@@ -106,9 +159,8 @@ module Path
   alias exists? exist?
 
   def find_all(caller_lib = nil, search_paths = nil)
-    Path.search_order
+    map_order
       .collect{|where| find(where) }
       .select{|file| file.exist? }.uniq
   end
-
 end

data/lib/scout/path/tmpfile.rb

@@ -0,0 +1,8 @@
+module TmpFile
+  def self.with_path(*args, &block)
+    TmpFile.with_file(*args) do |file|
+      Path.setup(file)
+      yield file
+    end
+  end
+end

data/lib/scout/path/util.rb

@@ -1,4 +1,12 @@
 module Path
+
+  def self.is_filename?(string, need_to_exists = true)
+    return false if string.nil?
+    return true if Path === string
+    return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265 }.empty? and (! need_to_exists || File.exist?(string))
+    return false
+  end
+
   def directory?
     return nil unless self.exist?
     File.directory?(self.find)
@@ -12,7 +20,6 @@ module Path
     self.annotate(File.basename(self))
   end
 
-
   def glob(pattern = '*')
     if self.include? "*"
       self.glob_all
@@ -31,12 +38,11 @@ module Path
   end
 
   def glob_all(pattern = nil, caller_lib = nil, search_paths = nil)
-    search_paths ||= Path.search_paths
+    search_paths ||= Path.path_maps
     search_paths = search_paths.dup
 
-    location_paths = {}
     search_paths.keys.collect do |where| 
-      found = find(where, Path.caller_lib_dir, search_paths)
+      found = find(where)
       paths = pattern ? Dir.glob(File.join(found, pattern)) : Dir.glob(found) 
 
       paths = paths.collect{|p| self.annotate p }
@@ -46,7 +52,13 @@ module Path
         p.where = where
       end if found.original and pattern
 
-      location_paths[where] = paths
+      paths
+    end.flatten.uniq
+  end
+
+  def no_method_missing
+    class << self
+      undef_method :method_missing
     end
   end
 end

data/lib/scout/path.rb

@@ -1,40 +1,18 @@
 require_relative 'meta_extension'
 require_relative 'path/find'
 require_relative 'path/util'
+require_relative 'path/tmpfile'
 
 module Path
   extend MetaExtension
-  extension_attr :pkgdir, :libdir
-
-  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
-    
-    if file.nil?
-      caller_dup = caller.dup
-      while file = caller_dup.shift
-        break unless file =~ /scout\/(?:resource\.rb|workflow\.rb)/ or
-          file =~ /scout\/path\.rb/ or
-          file =~ /scout\/persist.rb/
-      end
-      file = file.sub(/\.rb[^\w].*/,'.rb')
-    end
-
-    relative_to = [relative_to] unless Array === relative_to
-    file = File.expand_path(file)
-    return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
-
-    while file != '/'
-      dir = File.dirname file
-
-      return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
-
-      file = File.dirname file
-    end
-
-    return nil
-  end
+  extension_attr :pkgdir, :libdir, :path_maps
 
   def self.default_pkgdir
-    @@default_pkgdir = 'scout'
+    @@default_pkgdir ||= 'scout'
+  end
+  
+  def self.default_pkgdir=(pkgdir)
+    @@default_pkgdir = pkgdir
   end
 
   def pkgdir
@@ -42,7 +20,11 @@ module Path
   end
 
   def libdir
-    @libdir ||= Path.caller_lib_dir
+    @libdir || Path.caller_lib_dir
+  end
+
+  def path_maps
+    @path_maps ||= Path.path_maps
   end
 
   def join(subpath, prevpath = nil)
@@ -50,7 +32,7 @@ module Path
     prevpath = prevpath.to_s if Symbol === prevpath
 
     subpath = File.join(prevpath.to_s, subpath) if prevpath
-    new = File.join(self, subpath)
+    new = self.empty? ? subpath : File.join(self, subpath)
     self.annotate(new)
     new
   end

data/lib/scout/persist/open.rb

@@ -0,0 +1,17 @@
+require_relative '../open'
+require 'json'
+require 'yaml'
+
+module Open
+  def self.json(file)
+    Open.open(file){|f| JSON.load(f) }
+  end
+
+  def self.yaml(file)
+    Open.open(file){|f| YAML.load(f) }
+  end
+
+  def self.marshal(file)
+    Open.open(file){|f| Marshal.load(f) }
+  end
+end

data/lib/scout/persist/path.rb

@@ -0,0 +1,15 @@
+require_relative 'open'
+
+module Path
+  def yaml(*rest, &block)
+    Open.yaml(self, *rest, &block)
+  end
+
+  def json(*rest, &block)
+    Open.json(self, *rest, &block)
+  end
+
+  def marshal(*rest, &block)
+    Open.marshal(self, *rest, &block)
+  end
+end

data/lib/scout/persist/serialize.rb

@@ -0,0 +1,140 @@
+require_relative '../open'
+require_relative 'open'
+
+module Persist
+  TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
+
+  class << self
+    attr_accessor :save_drivers, :load_drivers
+    def save_drivers
+      @save_drivers ||= {}
+    end
+    def load_drivers
+      @load_drivers ||= {}
+    end
+  end
+
+  def self.serialize(content, type)
+    case type
+    when nil, :string, :integer, :float, :boolean, :file, :path
+      if IO === content || StringIO === content
+        content.read
+      else
+        content.to_s
+      end
+    when :array
+      content * "\n"
+    when :yaml
+      content.to_yaml
+    when :json
+      content.to_json
+    when :marshal, :serializer
+      Marshal.dump(content)
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        type = m[1].to_sym
+        content.collect{|c| serialize(c, type) } * "\n"
+      else
+        raise "Persist does not know #{Log.fingerprint type}"
+      end
+    end
+  end
+
+  def self.deserialize(serialized, type)
+    case type
+    when nil, :string, :file, :stream
+      serialized
+    when :path
+      Path.setup(serialized)
+    when :integer
+      serialized.to_i
+    when :float
+      serialized.to_f
+    when :boolean
+      TRUE_STRINGS.include? serialized
+    when :array
+      serialized.split("\n")
+    when :yaml
+      YAML.parse(serialized)
+    when :json
+      JSON.parse(serialized)
+    when :marshal, :serializer
+      Marshal.load(serialized)
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        type = m[1].to_sym
+        new_content = serialized.split("\n")
+        new_content.collect{|c| deserialize(c, type) }
+      else
+        raise "Persist does not know #{Log.fingerprint type}"
+      end
+    end
+  end
+
+  MEMORY = {}
+  def self.save(content, file, type = :serializer)
+    return if content.nil?
+    type = MEMORY if type == :memory
+    type = :serializer if type.nil?
+
+    if Hash === type
+      type[file] = content
+      return
+    end
+
+    Log.debug "Save #{Log.fingerprint type} on #{file}"
+    if save_drivers[type]
+      Open.write(file, save_drivers[type].call(content))
+      return
+    end
+
+    if IO === content || StringIO === content
+      main, copy = Open.tee_stream_thread content
+      t = Thread.new do
+        Thread.current["name"] = "file saver: " + file
+        Open.sensible_write(file, main)
+      end
+      ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
+    else
+      serialized = serialize(content, type)
+      Open.sensible_write(file, serialized)
+      content
+    end
+  end
+
+  def self.load(file, type = :serializer)
+    file = file.find if Path === file
+    type = MEMORY if type == :memory
+    return unless Hash === type || Open.exist?(file) 
+    type = :serializer if type.nil?
+
+    Log.debug "Load #{Log.fingerprint type} on #{file}"
+    if load_drivers[type]
+      return load_drivers[type].call(file)
+    end
+
+    case type
+    when :yaml
+      Open.yaml(file)
+    when :json
+      Open.json(file)
+    when :marshal, :serializer
+      Open.marshal(file)
+    when :stream
+      Open.open(file)
+    when :file
+      value = Open.read(file)
+      value.sub!(/^\./, File.dirname(file)) if value.start_with?("./")
+      value
+    when :file_array
+      Open.read(file).split("\n").collect do |f|
+        f.sub!(/^\./, File.dirname(file)) if f.start_with?("./")
+        f
+      end
+    when Hash
+      type[file]
+    else
+      deserialize(Open.read(file), type)
+    end
+  end
+end

data/lib/scout/persist.rb

@@ -0,0 +1,54 @@
+require_relative 'persist/serialize'
+require_relative 'persist/open'
+require_relative 'persist/path'
+
+module Persist
+  class << self
+    attr :cache_dir
+    def cache_dir=(cache_dir)
+      @cache_dir = Path === cache_dir ? cache_dir : Path.setup(cache_dir)
+    end
+    def cache_dir
+      @cache_dir ||= Path.setup("var/cache/persistence")
+    end
+
+    attr_writer :lock_dir
+    def lock_dir
+      @lock_dir ||= Path.setup("var/cache/persist_locks")
+    end
+  end
+
+  def self.persistence_path(name, options = {})
+    options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
+    other_options = IndiferentHash.pull_keys options, :other
+    TmpFile.tmp_for_file(name, options, other_options)
+  end
+
+  def self.persist(name, type = :serializer, options = {}, &block)
+    persist_options = IndiferentHash.pull_keys options, :persist 
+    file = persist_options[:path] || options[:path] || persistence_path(name, options)
+
+    update = options[:update] || persist_options[:update]
+    update = Open.mtime(update) if Path === update
+    update = Open.mtime(file) >= update ? false : true if Time === update
+
+    if Open.exist?(file) && ! update
+      Persist.load(file, type)
+    else
+      res = yield
+      begin
+        Open.rm(file)
+        res = Persist.save(res, file, type)
+      rescue
+        raise $! unless options[:canfail]
+        Log.debug "Could not persist #{type} on #{file}"
+      end
+      res
+    end
+  end
+
+  def self.memory(name, *args, &block)
+    self.persist(name, :memory, *args, &block)
+  end
+
+end

data/lib/scout/resource/path.rb

@@ -0,0 +1,15 @@
+module Path
+  def open(*args, &block)
+    produce
+    Open.open(self, *args, &block)
+  end
+
+  def read
+    produce
+    Open.read(self)
+  end
+
+  def write(*args, &block)
+    Open.write(self.find, *args, &block)
+  end
+end