scout-gear 9.1.0 → 10.0.1

data/lib/scout/open/util.rb

@@ -1,244 +0,0 @@
-module Open
-  GREP_CMD = begin
-               if ENV["GREP_CMD"] 
-                 ENV["GREP_CMD"]
-               elsif File.exist?('/bin/grep')
-                 "/bin/grep"
-               elsif File.exist?('/usr/bin/grep')
-                 "/usr/bin/grep"
-               else
-                 "grep"
-               end
-             end
-
-  def self.grep(stream, grep, invert = false, fixed = nil)
-    case 
-    when Array === grep
-      TmpFile.with_file(grep * "\n", false) do |f|
-        if FalseClass === fixed
-          CMD.cmd("#{GREP_CMD} #{invert ? '-v' : ''} -", "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
-        else
-          CMD.cmd("#{GREP_CMD} #{invert ? '-v' : ''} -", "-w" => true, "-F" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
-        end
-      end
-    else
-      CMD.cmd("#{GREP_CMD} #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
-    end
-  end
-
-  def self.gzip_pipe(file)
-    Open.gzip?(file) ? "<(gunzip -c '#{file}')" : "'#{file}'"
-  end
-
-  def self.bgunzip(stream)
-    Bgzf.setup stream
-  end
-   
-  def self.gunzip(stream)
-    CMD.cmd('zcat', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
-  end
-
-  def self.gzip(stream)
-    CMD.cmd('gzip', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
-  end
-
-  def self.bgzip(stream)
-    CMD.cmd('bgzip', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
-  end
-
-  def self.unzip(stream)
-    TmpFile.with_file(stream.read) do |filename|
-      StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", "-p" => true, :pipe => true).read)
-    end
-  end
-
-  # Questions
-  def self.gzip?(file)
-    file = file.find if Path === file
-    !! (file =~ /\.gz$/)
-  end
-
-  def self.bgzip?(file)
-    file = file.find if Path === file
-    !! (file =~ /\.bgz$/)
-  end
-
-  def self.zip?(file)
-    file = file.find if Path === file
-    !! (file =~ /\.zip$/)
-  end
-
-  def self.is_stream?(obj)
-    IO === obj || StringIO === obj
-  end
-
-  def self.has_stream?(obj)
-    obj.respond_to?(:stream)
-  end
-
-
-  def self.notify_write(file)
-    begin
-      notification_file = file + '.notify'
-      if Open.exists? notification_file
-        key = Open.read(notification_file).strip
-        key = nil if key.empty?
-        if key && key.include?("@")
-          to = from = key
-          subject = "Wrote " << file
-          message = "Content attached"
-          Misc.send_email(from, to, subject, message, :files => [file])
-        else
-          Misc.notify("Wrote " << file, nil, key)
-        end
-        Open.rm notification_file
-      end
-    rescue
-      Log.exception $!
-      Log.warn "Error notifying write of #{ file }"
-    end
-  end
-
-  def self.broken_link?(path)
-    File.symlink?(path) && ! File.exist?(File.readlink(path))
-  end
-
-  def self.directory?(file)
-    file = file.find if Path === file
-    File.directory?(file)
-  end
-
-  def self.exists?(file)
-    file = file.find if Path === file
-    File.exist?(file)
-  end
-  class << self; alias exist? exists? end
-
-  def self.mv(source, target, options = {})
-    target = target.find if Path === target
-    source = source.find if Path === source
-    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    tmp_target = File.join(File.dirname(target), '.tmp_mv.' + File.basename(target))
-    FileUtils.mv source, tmp_target
-    FileUtils.mv tmp_target, target
-    return nil
-  end
-
-  def self.rm(file)
-    FileUtils.rm(file) if File.exist?(file) || Open.broken_link?(file)
-  end
-
-  def self.rm_rf(file)
-    FileUtils.rm_rf(file)
-  end
-
-  def self.touch(file)
-    FileUtils.touch(file)
-  end
-
-  def self.mkdir(target)
-    target = target.find if Path === target
-    if ! File.exist?(target)
-      FileUtils.mkdir_p target
-    end
-  end
-
-  def self.writable?(path)
-    path = path.find if Path === path
-    if File.symlink?(path)
-      File.writable?(File.dirname(path))
-    elsif File.exist?(path)
-      File.writable?(path)
-    else
-      File.writable?(File.dirname(File.expand_path(path)))
-    end
-  end
-
-  def self.ctime(file)
-    file = file.find if Path === file
-    File.ctime(file)
-  end
-
-  def self.realpath(file)
-    file = file.find if Path === file
-    Pathname.new(File.expand_path(file)).realpath.to_s 
-  end
-
-  def self.mtime(file)
-    file = file.find if Path === file
-    begin
-      if File.symlink?(file) || File.stat(file).nlink > 1
-        if File.exist?(file + '.info') && defined?(Step)
-          done = Persist.load(file + '.info', Step::SERIALIZER)[:done]
-          return done if done
-        end
-
-        file = Pathname.new(file).realpath.to_s 
-      end
-      return nil unless File.exist?(file)
-      File.mtime(file)
-    rescue
-      nil
-    end
-  end
-
-  def self.cp(source, target, options = {})
-    source = source.find if Path === source
-    target = target.find if Path === target
-
-    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm_rf target if File.exist?(target)
-    FileUtils.cp_r source, target
-  end
-
-
-  def self.ln_s(source, target, options = {})
-    source = source.find if Path === source
-    target = target.find if Path === target
-
-    target = File.join(target, File.basename(source)) if File.directory? target
-    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
-    FileUtils.rm target if File.symlink?(target)
-    FileUtils.ln_s source, target
-  end
-
-  def self.ln(source, target, options = {})
-    source = source.find if Path === source
-    target = target.find if Path === target
-    source = File.realpath(source) if File.symlink?(source)
-
-    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
-    FileUtils.rm target if File.symlink?(target)
-    FileUtils.ln source, target
-  end
-
-  def self.ln_h(source, target, options = {})
-    source = source.find if Path === source
-    target = target.find if Path === target
-
-    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
-    begin
-      CMD.cmd("ln -L '#{ source }' '#{ target }'")
-    rescue ProcessFailed
-      Log.debug "Could not hard link #{source} and #{target}: #{$!.message.gsub("\n", '. ')}"
-      CMD.cmd("cp -L '#{ source }' '#{ target }'")
-    end
-  end
-
-  def self.link(source, target, options = {})
-    begin
-      Open.ln(source, target, options)
-    rescue
-      Open.ln_s(source, target, options)
-    end
-    nil
-  end
-
-  def self.list(file)
-    file = file.produce_and_find if Path === file
-    Open.read(file).split("\n")
-  end
-end

data/lib/scout/open.rb

@@ -1,170 +0,0 @@
-require_relative 'tmpfile'
-require_relative 'path'
-require_relative 'cmd'
-
-require_relative 'open/stream'
-require_relative 'open/util'
-require_relative 'open/remote'
-require_relative 'open/lock'
-
-module Open
-  module NamedStream
-    attr_accessor :filename
-
-    def digest_str
-      if Path === filename && ! filename.located?
-        filename
-      else
-        Misc.file_md5(filename)
-      end
-    end
-  end
-
-  def self.get_stream(file, mode = 'r', options = {})
-    return file if Open.is_stream?(file)
-    return file.stream if Open.has_stream?(file)
-    file = file.find if Path === file
-
-    return Open.ssh(file, options) if Open.ssh?(file)
-    return Open.wget(file, options) if Open.remote?(file)
-
-    File.open(file, mode)
-  end
-
-  def self.file_open(file, grep = false, mode = 'r', invert_grep = false, options = {})
-    Open.mkdir File.dirname(file) if mode.include? 'w'
-
-    stream = get_stream(file, mode, options)
-
-    if grep
-      grep(stream, grep, invert_grep)
-    else
-      stream
-    end
-  end
-
-  def self.file_write(file, content, mode = 'w')
-    File.open(file, mode) do |f|
-      begin
-        f.flock(File::LOCK_EX)
-        f.write content 
-        f.flock(File::LOCK_UN)
-      ensure
-        f.close unless f.closed?
-      end
-    end
-  end
-
-  def self.open(file, options = {})
-    if IO === file || StringIO === file
-      if block_given?
-        res = yield file, options
-        file.close
-        return res
-      else
-        return file
-      end
-    end
-
-    options = IndiferentHash.add_defaults options, :noz => false, :mode => 'r'
-
-    mode = IndiferentHash.process_options options, :mode
-
-    options[:noz] = true if mode.include? "w"
-
-    io = file_open(file, options[:grep], mode, options[:invert_grep], options)
-
-    io = unzip(io)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
-    io = gunzip(io)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
-    io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
-
-    io.extend NamedStream
-    io.filename = file
-
-    if block_given?
-      res = nil
-      begin
-        res = yield(io)
-      rescue DontClose
-        res = $!.payload
-      rescue Exception
-        io.abort $! if io.respond_to? :abort
-        io.join if io.respond_to? :join
-        raise $!
-      ensure
-        io.close if io.respond_to? :close and not io.closed?
-        io.join if io.respond_to? :join
-      end
-      res
-    else
-      io
-    end
-  end
-
-  def self.read(file, options = {}, &block)
-    open(file, options) do |f|
-      if block_given?
-        res = []
-        while not f.eof?
-          l = f.gets
-          l = Misc.fixutf8(l) unless options[:nofix]
-          res << yield(l)
-        end
-        res
-      else
-        if options[:nofix]
-          f.read
-        else
-          Misc.fixutf8(f.read)
-        end
-      end
-    end
-  end
-
-  def self.write(file, content = nil, options = {})
-    options = IndiferentHash.add_defaults options, :mode => 'w'
-
-    file = file.find(options[:where]) if Path === file
-    mode = IndiferentHash.process_options options, :mode
-
-    FileUtils.mkdir_p File.dirname(file)
-
-    case
-    when block_given?
-      begin
-        f = File.open(file, mode)
-        begin
-          yield f
-        ensure
-          f.close unless f.closed?
-        end
-      rescue Exception
-        FileUtils.rm file if File.exist? file
-        raise $!
-      end
-    when content.nil?
-      file_write(file, "", mode)
-    when String === content
-      file_write(file, content, mode)
-    when (IO === content || StringIO === content)
-      begin
-        File.open(file, mode) do |f| 
-          f.flock(File::LOCK_EX)
-          while block = content.read(Open::BLOCK_SIZE)
-            f.write block
-          end
-          f.flock(File::LOCK_UN)
-        end
-      rescue Exception
-        FileUtils.rm_rf file if File.exist? file
-        raise $!
-      end
-      content.close unless content.closed?
-      content.join if content.respond_to? :join
-    else
-      raise "Content unknown #{Log.fingerprint content}"
-    end
-
-    notify_write(file)
-  end
-end

data/lib/scout/path/find.rb

@@ -1,204 +0,0 @@
-require_relative '../indiferent_hash'
-module Path
-
-  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
-    
-    if file.nil?
-      caller_dup = caller.dup
-      while file = caller_dup.shift
-        break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
-          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
-          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
-          file =~ /(?:scout|rbbt)\/persist.rb/ or
-          file =~ /scout\/resource\/produce.rb/ or
-          file =~ /modules\/rbbt-util/
-      end
-      return nil if file.nil?
-      file = file.sub(/\.rb[^\w].*/,'.rb')
-    end
-
-    relative_to = [relative_to] unless Array === relative_to
-    file = File.expand_path(file)
-    return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
-
-    while file != '/'
-      dir = File.dirname file
-
-      return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
-
-      file = File.dirname file
-    end
-
-    return nil
-  end
-
-  def self.follow(path, map, map_name = nil)
-    file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
-      sub('{HOME}', ENV["HOME"]).
-      sub('{RESOURCE}', path.pkgdir.to_s).
-      sub('{PWD}', FileUtils.pwd).
-      sub('{TOPLEVEL}', path._toplevel).
-      sub('{SUBPATH}', path._subpath).
-      sub('{BASENAME}', File.basename(path)).
-      sub('{PATH}', path).
-      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir || "NOLIBDIR").
-      sub('{MAPNAME}', map_name.to_s).
-      sub('{REMOVE}/', '').
-      sub('{REMOVE}', '').gsub(/\/+/,'/')
-
-    while true
-      file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
-        key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
-        key_text = follow(path, "#{key}}", map_name)
-        key_text[orig] = replace[0..-2] if key_text.include?(orig)
-        key_text
-      end || break
-    end
-
-    file
-  end
-
-  def self.path_maps
-    @@path_maps ||= IndiferentHash.setup({
-      :current => "{PWD}/{TOPLEVEL}/{SUBPATH}",
-      :user    => "{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}",
-      :global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-      :usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-      :local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-      :fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-      :cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-      :bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-      :lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
-      :scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
-      :tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
-      :default => :user
-    })
-  end
-
-  def self.basic_map_order
-    @@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
-  end
-
-  def self.map_order
-    @@map_order ||= (path_maps.keys & basic_map_order) + (path_maps.keys - basic_map_order)
-  end
-
-  def self.add_path(name, map)
-    @@path_maps[name] = map
-    @@map_order = nil
-  end
-
-  def _parts
-    @_parts ||= self.split("/")
-  end
-
-  def _subpath
-    @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
-  end
-  
-  def _toplevel
-    @toplevel ||= _parts.length > 1 ? _parts[0] : ""
-  end
-
-  SLASH = "/"[0]
-  DOT = "."[0]
-  def located?
-    # OPEN RESOURCE
-    self.slice(0,1) == SLASH || (self.slice(0,1) == DOT && self.slice(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
-  end
-
-  def annotate_found_where(found, where)
-    self.annotate(found).tap{|p| 
-      p.instance_variable_set("@where", where) 
-      p.instance_variable_set("@original", self.dup) 
-    }
-  end
-
-  def where
-    @where
-  end
-
-  def original
-    @original
-  end
-
-  def map_order
-    @map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
-  end
-
-  def follow(map_name = :default, annotate = true)
-    IndiferentHash.setup(path_maps)
-    map = path_maps[map_name] || Path.path_maps[map_name]
-    if map.nil? && String === map_name
-      map = File.join(map_name, '{TOPLEVEL}/{SUBPATH}')
-    end
-    raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
-    while Symbol === map
-      map_name = map
-      map = path_maps[map_name]
-    end
-    found = Path.follow(self, map, map_name)
-
-    annotate_found_where(found, map_name)  if annotate
-
-    found
-  end
-
-  def self.exists_file_or_alternatives(file)
-    return file if File.exist?(file) or File.directory?(file)
-    %w(gz bgz zip).each do |extension|
-      alt_file = file + '.' + extension
-      return alt_file if File.exist?(alt_file) or File.directory?(alt_file)
-    end
-    nil
-  end
-
-  def find(where = nil)
-    if located?
-      if File.exist?(self)
-        return self if located?
-      else
-        found = Path.exists_file_or_alternatives(self)
-        if found
-          return self.annotate(found)
-        else
-          return self if located?
-        end
-      end
-    end
-
-    return find_all if where == 'all' || where == :all
-
-    return follow(where) if where
-
-    map_order.each do |map_name|
-      found = follow(map_name, false)
-
-      found = Path.exists_file_or_alternatives(found)
-      return annotate_found_where(found, map_name) if found
-    end
-
-    return follow(:default)
-  end
-
-  def exist?
-    # OPEN
-    found = self.find
-    File.exist?(found) || File.directory?(found)
-  end
-
-  alias exists? exist?
-
-  def find_all(caller_lib = nil, search_paths = nil)
-    map_order
-      .collect{|where| find(where) }
-      .select{|file| file.exist? }.uniq
-  end
-
-  def find_with_extension(extension, *args)
-    found = self.find(*args)
-    return found if found.exists?
-    found_with_extension = self.set_extension(extension).find
-    found_with_extension.exists? ? found_with_extension : found
-  end
-end

data/lib/scout/path/tmpfile.rb

@@ -1,8 +0,0 @@
-module TmpFile
-  def self.with_path(*args, &block)
-    TmpFile.with_file(*args) do |file|
-      Path.setup(file)
-      yield file
-    end
-  end
-end