scout-gear 2.0.0 → 5.2.0

data/lib/scout/open.rb

@@ -0,0 +1,169 @@
+require_relative 'tmpfile'
+require_relative 'path'
+require_relative 'cmd'
+
+require_relative 'open/stream'
+require_relative 'open/util'
+require_relative 'open/remote'
+require_relative 'open/lock'
+
+module Open
+  module NamedStream
+    attr_accessor :filename
+
+    def digest_str
+      if Path === filename && ! filename.located?
+        filename
+      else
+        Misc.file_md5(filename)
+      end
+    end
+  end
+
+  def self.get_stream(file, mode = 'r')
+    file = file.find if Path === file
+
+    return Open.ssh(file) if Open.ssh?(file)
+    return Open.wget(file) if Open.remote?(file)
+
+    File.open(file, mode)
+  end
+
+  def self.file_open(file, grep = false, mode = 'r', invert_grep = false)
+    Open.mkdir File.dirname(file) if mode.include? 'w'
+
+    stream = get_stream(file, mode)
+
+    if grep
+      grep(stream, grep, invert_grep)
+    else
+      stream
+    end
+  end
+
+  def self.file_write(file, content, mode = 'w')
+    File.open(file, mode) do |f|
+      begin
+        f.flock(File::LOCK_EX)
+        f.write content 
+        f.flock(File::LOCK_UN)
+      ensure
+        f.close unless f.closed?
+      end
+    end
+  end
+
+  def self.open(file, options = {})
+    if IO === file || StringIO === file
+      if block_given?
+        res = yield file 
+        file.close
+        return res
+      else
+        return file 
+      end
+    end
+
+    options = IndiferentHash.add_defaults options, :noz => false, :mode => 'r'
+
+    mode = IndiferentHash.process_options options, :mode
+
+    options[:noz] = true if mode.include? "w"
+
+    io = file_open(file, options[:grep], mode, options[:invert_grep])
+
+    io = unzip(io)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
+    io = gunzip(io)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
+    io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
+
+    io.extend NamedStream
+    io.filename = file
+
+    if block_given?
+      res = nil
+      begin
+        res = yield(io)
+      rescue DontClose
+        res = $!.payload
+      rescue Exception
+        io.abort if io.respond_to? :abort
+        io.join if io.respond_to? :join
+        raise $!
+      ensure
+        io.close if io.respond_to? :close and not io.closed?
+        io.join if io.respond_to? :join
+      end
+      res
+    else
+      io
+    end
+  end
+
+  def self.read(file, options = {}, &block)
+    open(file, options) do |f|
+      if block_given?
+        res = []
+        while not f.eof?
+          l = f.gets
+          l = Misc.fixutf8(l) unless options[:nofix]
+          res << yield(l)
+        end
+        res
+      else
+        if options[:nofix]
+          f.read
+        else
+          Misc.fixutf8(f.read)
+        end
+      end
+    end
+  end
+
+  def self.write(file, content = nil, options = {})
+    options = IndiferentHash.add_defaults options, :mode => 'w'
+
+    file = file.find(options[:where]) if Path === file
+    mode = IndiferentHash.process_options options, :mode
+
+    FileUtils.mkdir_p File.dirname(file)
+
+    case
+    when block_given?
+      begin
+        f = File.open(file, mode)
+        begin
+          yield f
+        ensure
+          f.close unless f.closed?
+        end
+      rescue Exception
+        FileUtils.rm file if File.exist? file
+        raise $!
+      end
+    when content.nil?
+      file_write(file, "", mode)
+    when String === content
+      file_write(file, content, mode)
+    when (IO === content || StringIO === content)
+      begin
+        File.open(file, mode) do |f| 
+          f.flock(File::LOCK_EX)
+          while block = content.read(Open::BLOCK_SIZE)
+            f.write block
+          end
+          f.flock(File::LOCK_UN)
+        end
+      rescue Exception
+        FileUtils.rm_rf file if File.exist? file
+        raise $!
+      end
+      content.close unless content.closed?
+      content.join if content.respond_to? :join
+    else
+      raise "Content unknown #{Log.fingerprint content}"
+    end
+
+    notify_write(file) 
+  end
+
+end

data/lib/scout/path/find.rb

@@ -1,20 +1,36 @@
 require_relative '../indiferent_hash'
 module Path
 
-  def _parts
-    @_parts ||= self.split("/")
-  end
+  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
+    
+    if file.nil?
+      caller_dup = caller.dup
+      while file = caller_dup.shift
+        break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
+          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
+          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
+          file =~ /(?:scout|rbbt)\/persist.rb/
+      end
+      file = file.sub(/\.rb[^\w].*/,'.rb')
+    end
 
-  def _subpath
-    @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
-  end
-  
-  def _toplevel
-    @toplevel ||= _parts.length > 1 ? _parts[0] : ""
+    relative_to = [relative_to] unless Array === relative_to
+    file = File.expand_path(file)
+    return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
+
+    while file != '/'
+      dir = File.dirname file
+
+      return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
+
+      file = File.dirname file
+    end
+
+    return nil
   end
 
-  def self.follow(path, map)
-    map.sub('{PKGDIR}', path.pkgdir.respond_to?(:subdir) ? path.pkgdir.pkgdir : path.pkgdir || Path.default_pkgdir).
+  def self.follow(path, map, map_name = nil)
+    file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
       sub('{RESOURCE}', path.pkgdir.to_s).
       sub('{PWD}', FileUtils.pwd).
       sub('{TOPLEVEL}', path._toplevel).
@@ -22,8 +38,20 @@ module Path
       sub('{BASENAME}', File.basename(path)).
       sub('{PATH}', path).
       sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
+      sub('{MAPNAME}', map_name.to_s).
       sub('{REMOVE}/', '').
       sub('{REMOVE}', '').gsub(/\/+/,'/')
+
+    while true
+      file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
+        key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
+        key_text = follow(path, "#{key}}", map_name)
+        key_text[orig] = replace[0..-2] if key_text.include?(orig)
+        key_text
+      end || break
+    end
+
+    file
   end
 
   def self.path_maps
@@ -42,17 +70,29 @@ module Path
     })
   end
 
-  def self.map_search
-    @@map_search ||= %w(current workflow user local global lib fast cache bulk)
+  def self.basic_map_order
+    @@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
   end
 
-  def self.search_order
-    @@search_order ||= (path_maps.keys & map_search) + (path_maps.keys - map_search)
+  def self.map_order
+    @@map_order ||= (path_maps.keys & basic_map_order) + (path_maps.keys - basic_map_order)
   end
 
   def self.add_path(name, map)
     @@path_maps[name] = map
-    @@search_order = nil
+    @@map_order = nil
+  end
+
+  def _parts
+    @_parts ||= self.split("/")
+  end
+
+  def _subpath
+    @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
+  end
+  
+  def _toplevel
+    @toplevel ||= _parts.length > 1 ? _parts[0] : ""
   end
 
   SLASH = "/"[0]
@@ -77,13 +117,19 @@ module Path
     @original
   end
 
+  def map_order
+    @map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
+  end
+
   def follow(map_name = :default, annotate = true)
-    map = Path.path_maps[map_name]
+    IndiferentHash.setup(path_maps)
+    map = path_maps[map_name] || Path.path_maps[map_name]
+    raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
     while Symbol === map
       map_name = map
-      map = Path.path_maps[map_name]
+      map = path_maps[map_name]
     end
-    found = Path.follow(self, map)
+    found = Path.follow(self, map, map_name)
 
     annotate_found_where(found, map_name)  if annotate
 
@@ -95,7 +141,7 @@ module Path
     return find_all if where == 'all' || where == :all
     return follow(where) if where
 
-    Path.search_order.each do |map_name|
+    map_order.each do |map_name|
       found = follow(map_name, false)
 
       return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
@@ -113,8 +159,9 @@ module Path
   alias exists? exist?
 
   def find_all(caller_lib = nil, search_paths = nil)
-    Path.search_order
+    map_order
       .collect{|where| find(where) }
       .select{|file| file.exist? }.uniq
   end
+
 end

data/lib/scout/path/tmpfile.rb

@@ -0,0 +1,8 @@
+module TmpFile
+  def self.with_path(*args, &block)
+    TmpFile.with_file(*args) do |file|
+      Path.setup(file)
+      yield file
+    end
+  end
+end

data/lib/scout/path/util.rb

@@ -1,4 +1,12 @@
 module Path
+
+  def self.is_filename?(string, need_to_exists = true)
+    return false if string.nil?
+    return true if Path === string
+    return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265 }.empty? and (! need_to_exists || File.exist?(string))
+    return false
+  end
+
   def directory?
     return nil unless self.exist?
     File.directory?(self.find)
@@ -12,7 +20,6 @@ module Path
     self.annotate(File.basename(self))
   end
 
-
   def glob(pattern = '*')
     if self.include? "*"
       self.glob_all
@@ -48,4 +55,10 @@ module Path
       paths
     end.flatten.uniq
   end
+
+  def no_method_missing
+    class << self
+      undef_method :method_missing
+    end
+  end
 end

data/lib/scout/path.rb

@@ -1,39 +1,12 @@
 require_relative 'meta_extension'
 require_relative 'path/find'
 require_relative 'path/util'
+require_relative 'path/tmpfile'
 
 module Path
   extend MetaExtension
   extension_attr :pkgdir, :libdir, :path_maps
 
-  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
-    
-    if file.nil?
-      caller_dup = caller.dup
-      while file = caller_dup.shift
-        break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
-          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
-          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
-          file =~ /(?:scout|rbbt)\/persist.rb/
-      end
-      file = file.sub(/\.rb[^\w].*/,'.rb')
-    end
-
-    relative_to = [relative_to] unless Array === relative_to
-    file = File.expand_path(file)
-    return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
-
-    while file != '/'
-      dir = File.dirname file
-
-      return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
-
-      file = File.dirname file
-    end
-
-    return nil
-  end
-
   def self.default_pkgdir
     @@default_pkgdir ||= 'scout'
   end
@@ -42,13 +15,16 @@ module Path
     @@default_pkgdir = pkgdir
   end
 
-
   def pkgdir
     @pkgdir ||= Path.default_pkgdir
   end
 
   def libdir
-    @libdir
+    @libdir || Path.caller_lib_dir
+  end
+
+  def path_maps
+    @path_maps ||= Path.path_maps.dup
   end
 
   def join(subpath, prevpath = nil)

data/lib/scout/persist/open.rb

@@ -0,0 +1,17 @@
+require_relative '../open'
+require 'json'
+require 'yaml'
+
+module Open
+  def self.json(file)
+    Open.open(file){|f| JSON.load(f) }
+  end
+
+  def self.yaml(file)
+    Open.open(file){|f| YAML.load(f) }
+  end
+
+  def self.marshal(file)
+    Open.open(file){|f| Marshal.load(f) }
+  end
+end

data/lib/scout/persist/path.rb

@@ -0,0 +1,15 @@
+require_relative 'open'
+
+module Path
+  def yaml(*rest, &block)
+    Open.yaml(self, *rest, &block)
+  end
+
+  def json(*rest, &block)
+    Open.json(self, *rest, &block)
+  end
+
+  def marshal(*rest, &block)
+    Open.marshal(self, *rest, &block)
+  end
+end

data/lib/scout/persist/serialize.rb

@@ -0,0 +1,151 @@
+require_relative '../open'
+require_relative 'open'
+
+module Persist
+  TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
+  SERIALIZER = :json
+
+  class << self
+    attr_accessor :save_drivers, :load_drivers
+    def save_drivers
+      @save_drivers ||= {}
+    end
+    def load_drivers
+      @load_drivers ||= {}
+    end
+  end
+
+  def self.serialize(content, type)
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    case type
+    when nil, :string, :integer, :float, :boolean, :file, :path
+      if IO === content || StringIO === content
+        content.read
+      else
+        content.to_s
+      end
+    when :array
+      content * "\n"
+    when :yaml
+      content.to_yaml
+    when :json
+      content.to_json
+    when :marshal
+      Marshal.dump(content)
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        type = m[1].to_sym
+        content.collect{|c| serialize(c, type) } * "\n"
+      else
+        raise "Persist does not know #{Log.fingerprint type}"
+      end
+    end
+  end
+
+  def self.deserialize(serialized, type)
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    case type
+    when nil, :string, :file, :stream
+      serialized
+    when :path
+      Path.setup(serialized)
+    when :integer
+      serialized.to_i
+    when :float
+      serialized.to_f
+    when :boolean
+      TRUE_STRINGS.include? serialized
+    when :array
+      serialized.split("\n")
+    when :yaml
+      YAML.parse(serialized)
+    when :json
+      JSON.parse(serialized)
+    when :marshal
+      Marshal.load(serialized)
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        type = m[1].to_sym
+        new_content = serialized.split("\n")
+        new_content.collect{|c| deserialize(c, type) }
+      else
+        raise "Persist does not know #{Log.fingerprint type}"
+      end
+    end
+  end
+
+  MEMORY = {}
+  def self.save(content, file, type = :serializer)
+    type = :serializer if type.nil?
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    type = MEMORY if type == :memory
+    return if content.nil?
+    type = MEMORY if type == :memory
+    type = :serializer if type.nil?
+
+    if Hash === type
+      type[file] = content
+      return
+    end
+
+    Log.debug "Save #{Log.fingerprint type} on #{file}"
+    if save_drivers[type]
+      Open.write(file, save_drivers[type].call(content))
+      return
+    end
+
+    if IO === content || StringIO === content
+      main, copy = Open.tee_stream_thread content
+      t = Thread.new do
+        Thread.current["name"] = "file saver: " + file
+        Open.sensible_write(file, main)
+      end
+      ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
+    else
+      serialized = serialize(content, type)
+      Open.sensible_write(file, serialized, :force => true)
+      content
+    end
+  end
+
+  def self.load(file, type = :serializer)
+    file = file.find if Path === file
+    type = :serializer if type.nil?
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    type = MEMORY if type == :memory
+    return unless Hash === type || Open.exist?(file) 
+
+    Log.debug "Load #{Log.fingerprint type} on #{file}"
+    if load_drivers[type]
+      return load_drivers[type].call(file)
+    end
+
+    case type
+    when :yaml
+      Open.yaml(file)
+    when :json
+      Open.json(file)
+    when :marshal, :serializer
+      Open.marshal(file)
+    when :stream
+      Open.open(file)
+    when :file
+      value = Open.read(file)
+      value.sub!(/^\./, File.dirname(file)) if value.start_with?("./")
+      value
+    when :file_array
+      Open.read(file).split("\n").collect do |f|
+        f.sub!(/^\./, File.dirname(file)) if f.start_with?("./")
+        f
+      end
+    when Hash
+      type[file]
+    else
+      deserialize(Open.read(file), type)
+    end
+  end
+end

data/lib/scout/persist.rb

@@ -0,0 +1,54 @@
+require_relative 'persist/serialize'
+require_relative 'persist/open'
+require_relative 'persist/path'
+
+module Persist
+  class << self
+    attr :cache_dir
+    def cache_dir=(cache_dir)
+      @cache_dir = Path === cache_dir ? cache_dir : Path.setup(cache_dir)
+    end
+    def cache_dir
+      @cache_dir ||= Path.setup("var/cache/persistence")
+    end
+
+    attr_writer :lock_dir
+    def lock_dir
+      @lock_dir ||= Path.setup("var/cache/persist_locks")
+    end
+  end
+
+  def self.persistence_path(name, options = {})
+    options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
+    other_options = IndiferentHash.pull_keys options, :other
+    TmpFile.tmp_for_file(name, options, other_options)
+  end
+
+  def self.persist(name, type = :serializer, options = {}, &block)
+    persist_options = IndiferentHash.pull_keys options, :persist 
+    file = persist_options[:path] || options[:path] || persistence_path(name, options)
+
+    update = options[:update] || persist_options[:update]
+    update = Open.mtime(update) if Path === update
+    update = Open.mtime(file) >= update ? false : true if Time === update
+
+    if Open.exist?(file) && ! update
+      Persist.load(file, type)
+    else
+      res = yield
+      begin
+        Open.rm(file)
+        res = Persist.save(res, file, type)
+      rescue
+        raise $! unless options[:canfail]
+        Log.debug "Could not persist #{type} on #{file}"
+      end
+      res
+    end
+  end
+
+  def self.memory(name, *args, &block)
+    self.persist(name, :memory, *args, &block)
+  end
+
+end

data/lib/scout/resource/path.rb

@@ -0,0 +1,20 @@
+module Path
+  def relocate
+    return self if Open.exists?(self)
+    Resource.relocate(self)
+  end
+
+  def open(*args, &block)
+    produce
+    Open.open(self, *args, &block)
+  end
+
+  def read
+    produce
+    Open.read(self)
+  end
+
+  def write(*args, &block)
+    Open.write(self.find, *args, &block)
+  end
+end