scout-essentials 1.0.0

data/lib/scout/open/util.rb

@@ -0,0 +1,244 @@
+module Open
+  GREP_CMD = begin
+               if ENV["GREP_CMD"] 
+                 ENV["GREP_CMD"]
+               elsif File.exist?('/bin/grep')
+                 "/bin/grep"
+               elsif File.exist?('/usr/bin/grep')
+                 "/usr/bin/grep"
+               else
+                 "grep"
+               end
+             end
+
+  def self.grep(stream, grep, invert = false, fixed = nil)
+    case 
+    when Array === grep
+      TmpFile.with_file(grep * "\n", false) do |f|
+        if FalseClass === fixed
+          CMD.cmd("#{GREP_CMD} #{invert ? '-v' : ''} -", "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        else
+          CMD.cmd("#{GREP_CMD} #{invert ? '-v' : ''} -", "-w" => true, "-F" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        end
+      end
+    else
+      CMD.cmd("#{GREP_CMD} #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
+    end
+  end
+
+  def self.gzip_pipe(file)
+    Open.gzip?(file) ? "<(gunzip -c '#{file}')" : "'#{file}'"
+  end
+
+  def self.bgunzip(stream)
+    Bgzf.setup stream
+  end
+   
+  def self.gunzip(stream)
+    CMD.cmd('zcat', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  end
+
+  def self.gzip(stream)
+    CMD.cmd('gzip', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  end
+
+  def self.bgzip(stream)
+    CMD.cmd('bgzip', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  end
+
+  def self.unzip(stream)
+    TmpFile.with_file(stream.read) do |filename|
+      StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", "-p" => true, :pipe => true).read)
+    end
+  end
+
+  # Questions
+  def self.gzip?(file)
+    file = file.find if Path === file
+    !! (file =~ /\.gz$/)
+  end
+
+  def self.bgzip?(file)
+    file = file.find if Path === file
+    !! (file =~ /\.bgz$/)
+  end
+
+  def self.zip?(file)
+    file = file.find if Path === file
+    !! (file =~ /\.zip$/)
+  end
+
+  def self.is_stream?(obj)
+    IO === obj || StringIO === obj
+  end
+
+  def self.has_stream?(obj)
+    obj.respond_to?(:stream)
+  end
+
+
+  def self.notify_write(file)
+    begin
+      notification_file = file + '.notify'
+      if Open.exists? notification_file
+        key = Open.read(notification_file).strip
+        key = nil if key.empty?
+        if key && key.include?("@")
+          to = from = key
+          subject = "Wrote " << file
+          message = "Content attached"
+          Misc.send_email(from, to, subject, message, :files => [file])
+        else
+          Misc.notify("Wrote " << file, nil, key)
+        end
+        Open.rm notification_file
+      end
+    rescue
+      Log.exception $!
+      Log.warn "Error notifying write of #{ file }"
+    end
+  end
+
+  def self.broken_link?(path)
+    File.symlink?(path) && ! File.exist?(File.readlink(path))
+  end
+
+  def self.directory?(file)
+    file = file.find if Path === file
+    File.directory?(file)
+  end
+
+  def self.exists?(file)
+    file = file.find if Path === file
+    File.exist?(file)
+  end
+  class << self; alias exist? exists? end
+
+  def self.mv(source, target, options = {})
+    target = target.find if Path === target
+    source = source.find if Path === source
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    tmp_target = File.join(File.dirname(target), '.tmp_mv.' + File.basename(target))
+    FileUtils.mv source, tmp_target
+    FileUtils.mv tmp_target, target
+    return nil
+  end
+
+  def self.rm(file)
+    FileUtils.rm(file) if File.exist?(file) || Open.broken_link?(file)
+  end
+
+  def self.rm_rf(file)
+    FileUtils.rm_rf(file)
+  end
+
+  def self.touch(file)
+    FileUtils.touch(file)
+  end
+
+  def self.mkdir(target)
+    target = target.find if Path === target
+    if ! File.exist?(target)
+      FileUtils.mkdir_p target
+    end
+  end
+
+  def self.writable?(path)
+    path = path.find if Path === path
+    if File.symlink?(path)
+      File.writable?(File.dirname(path))
+    elsif File.exist?(path)
+      File.writable?(path)
+    else
+      File.writable?(File.dirname(File.expand_path(path)))
+    end
+  end
+
+  def self.ctime(file)
+    file = file.find if Path === file
+    File.ctime(file)
+  end
+
+  def self.realpath(file)
+    file = file.find if Path === file
+    Pathname.new(File.expand_path(file)).realpath.to_s 
+  end
+
+  def self.mtime(file)
+    file = file.find if Path === file
+    begin
+      if File.symlink?(file) || File.stat(file).nlink > 1
+        if File.exist?(file + '.info') && defined?(Step)
+          done = Persist.load(file + '.info', Step::SERIALIZER)[:done]
+          return done if done
+        end
+
+        file = Pathname.new(file).realpath.to_s 
+      end
+      return nil unless File.exist?(file)
+      File.mtime(file)
+    rescue
+      nil
+    end
+  end
+
+  def self.cp(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm_rf target if File.exist?(target)
+    FileUtils.cp_r source, target
+  end
+
+
+  def self.ln_s(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    target = File.join(target, File.basename(source)) if File.directory? target
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
+    FileUtils.rm target if File.symlink?(target)
+    FileUtils.ln_s source, target
+  end
+
+  def self.ln(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+    source = File.realpath(source) if File.symlink?(source)
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
+    FileUtils.rm target if File.symlink?(target)
+    FileUtils.ln source, target
+  end
+
+  def self.ln_h(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
+    begin
+      CMD.cmd("ln -L '#{ source }' '#{ target }'")
+    rescue ProcessFailed
+      Log.debug "Could not hard link #{source} and #{target}: #{$!.message.gsub("\n", '. ')}"
+      CMD.cmd("cp -L '#{ source }' '#{ target }'")
+    end
+  end
+
+  def self.link(source, target, options = {})
+    begin
+      Open.ln(source, target, options)
+    rescue
+      Open.ln_s(source, target, options)
+    end
+    nil
+  end
+
+  def self.list(file)
+    file = file.produce_and_find if Path === file
+    Open.read(file).split("\n")
+  end
+end

data/lib/scout/open.rb

@@ -0,0 +1,170 @@
+require_relative 'tmpfile'
+require_relative 'path'
+require_relative 'cmd'
+
+require_relative 'open/stream'
+require_relative 'open/util'
+require_relative 'open/remote'
+require_relative 'open/lock'
+
+module Open
+  module NamedStream
+    attr_accessor :filename
+
+    def digest_str
+      if Path === filename && ! filename.located?
+        filename
+      else
+        Misc.file_md5(filename)
+      end
+    end
+  end
+
+  def self.get_stream(file, mode = 'r', options = {})
+    return file if Open.is_stream?(file)
+    return file.stream if Open.has_stream?(file)
+    file = file.find if Path === file
+
+    return Open.ssh(file, options) if Open.ssh?(file)
+    return Open.wget(file, options) if Open.remote?(file)
+
+    File.open(file, mode)
+  end
+
+  def self.file_open(file, grep = false, mode = 'r', invert_grep = false, options = {})
+    Open.mkdir File.dirname(file) if mode.include? 'w'
+
+    stream = get_stream(file, mode, options)
+
+    if grep
+      grep(stream, grep, invert_grep)
+    else
+      stream
+    end
+  end
+
+  def self.file_write(file, content, mode = 'w')
+    File.open(file, mode) do |f|
+      begin
+        f.flock(File::LOCK_EX)
+        f.write content 
+        f.flock(File::LOCK_UN)
+      ensure
+        f.close unless f.closed?
+      end
+    end
+  end
+
+  def self.open(file, options = {})
+    if IO === file || StringIO === file
+      if block_given?
+        res = yield file, options
+        file.close
+        return res
+      else
+        return file
+      end
+    end
+
+    options = IndiferentHash.add_defaults options, :noz => false, :mode => 'r'
+
+    mode = IndiferentHash.process_options options, :mode
+
+    options[:noz] = true if mode.include? "w"
+
+    io = file_open(file, options[:grep], mode, options[:invert_grep], options)
+
+    io = unzip(io)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
+    io = gunzip(io)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
+    io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
+
+    io.extend NamedStream
+    io.filename = file
+
+    if block_given?
+      res = nil
+      begin
+        res = yield(io)
+      rescue DontClose
+        res = $!.payload
+      rescue Exception
+        io.abort $! if io.respond_to? :abort
+        io.join if io.respond_to? :join
+        raise $!
+      ensure
+        io.close if io.respond_to? :close and not io.closed?
+        io.join if io.respond_to? :join
+      end
+      res
+    else
+      io
+    end
+  end
+
+  def self.read(file, options = {}, &block)
+    open(file, options) do |f|
+      if block_given?
+        res = []
+        while not f.eof?
+          l = f.gets
+          l = Misc.fixutf8(l) unless options[:nofix]
+          res << yield(l)
+        end
+        res
+      else
+        if options[:nofix]
+          f.read
+        else
+          Misc.fixutf8(f.read)
+        end
+      end
+    end
+  end
+
+  def self.write(file, content = nil, options = {})
+    options = IndiferentHash.add_defaults options, :mode => 'w'
+
+    file = file.find(options[:where]) if Path === file
+    mode = IndiferentHash.process_options options, :mode
+
+    FileUtils.mkdir_p File.dirname(file)
+
+    case
+    when block_given?
+      begin
+        f = File.open(file, mode)
+        begin
+          yield f
+        ensure
+          f.close unless f.closed?
+        end
+      rescue Exception
+        FileUtils.rm file if File.exist? file
+        raise $!
+      end
+    when content.nil?
+      file_write(file, "", mode)
+    when String === content
+      file_write(file, content, mode)
+    when (IO === content || StringIO === content)
+      begin
+        File.open(file, mode) do |f| 
+          f.flock(File::LOCK_EX)
+          while block = content.read(Open::BLOCK_SIZE)
+            f.write block
+          end
+          f.flock(File::LOCK_UN)
+        end
+      rescue Exception
+        FileUtils.rm_rf file if File.exist? file
+        raise $!
+      end
+      content.close unless content.closed?
+      content.join if content.respond_to? :join
+    else
+      raise "Content unknown #{Log.fingerprint content}"
+    end
+
+    notify_write(file)
+  end
+end

data/lib/scout/path/find.rb

@@ -0,0 +1,204 @@
+require_relative '../indiferent_hash'
+module Path
+
+  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
+    
+    if file.nil?
+      caller_dup = caller.dup
+      while file = caller_dup.shift
+        break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
+          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
+          file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
+          file =~ /(?:scout|rbbt)\/persist.rb/ or
+          file =~ /scout\/resource\/produce.rb/ or
+          file =~ /modules\/rbbt-util/
+      end
+      return nil if file.nil?
+      file = file.sub(/\.rb[^\w].*/,'.rb')
+    end
+
+    relative_to = [relative_to] unless Array === relative_to
+    file = File.expand_path(file)
+    return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
+
+    while file != '/'
+      dir = File.dirname file
+
+      return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
+
+      file = File.dirname file
+    end
+
+    return nil
+  end
+
+  def self.follow(path, map, map_name = nil)
+    file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
+      sub('{HOME}', ENV["HOME"]).
+      sub('{RESOURCE}', path.pkgdir.to_s).
+      sub('{PWD}', FileUtils.pwd).
+      sub('{TOPLEVEL}', path._toplevel).
+      sub('{SUBPATH}', path._subpath).
+      sub('{BASENAME}', File.basename(path)).
+      sub('{PATH}', path).
+      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir || "NOLIBDIR").
+      sub('{MAPNAME}', map_name.to_s).
+      sub('{REMOVE}/', '').
+      sub('{REMOVE}', '').gsub(/\/+/,'/')
+
+    while true
+      file.gsub!(/\{(.+)(?<!\\)\/(.+)(?<!\\)\/(.+)\}/) do |m|
+        key, orig, replace = m.split(/(?<!\\)\//).collect{|p| p.gsub('\/','/') }
+        key_text = follow(path, "#{key}}", map_name)
+        key_text[orig] = replace[0..-2] if key_text.include?(orig)
+        key_text
+      end || break
+    end
+
+    file
+  end
+
+  def self.path_maps
+    @@path_maps ||= IndiferentHash.setup({
+      :current => "{PWD}/{TOPLEVEL}/{SUBPATH}",
+      :user    => "{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}",
+      :global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
+      :scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
+      :tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
+      :default => :user
+    })
+  end
+
+  def self.basic_map_order
+    @@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
+  end
+
+  def self.map_order
+    @@map_order ||= (path_maps.keys & basic_map_order) + (path_maps.keys - basic_map_order)
+  end
+
+  def self.add_path(name, map)
+    @@path_maps[name] = map
+    @@map_order = nil
+  end
+
+  def _parts
+    @_parts ||= self.split("/")
+  end
+
+  def _subpath
+    @subpath ||= _parts.length > 1 ? _parts[1..-1] * "/" : _parts[0] || ""
+  end
+  
+  def _toplevel
+    @toplevel ||= _parts.length > 1 ? _parts[0] : ""
+  end
+
+  SLASH = "/"[0]
+  DOT = "."[0]
+  def located?
+    # OPEN RESOURCE
+    self.slice(0,1) == SLASH || (self.slice(0,1) == DOT && self.slice(1,2) == SLASH) # || (resource != Rbbt && (Open.remote?(self) || Open.ssh?(self)))
+  end
+
+  def annotate_found_where(found, where)
+    self.annotate(found).tap{|p| 
+      p.instance_variable_set("@where", where) 
+      p.instance_variable_set("@original", self.dup) 
+    }
+  end
+
+  def where
+    @where
+  end
+
+  def original
+    @original
+  end
+
+  def map_order
+    @map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
+  end
+
+  def follow(map_name = :default, annotate = true)
+    IndiferentHash.setup(path_maps)
+    map = path_maps[map_name] || Path.path_maps[map_name]
+    if map.nil? && String === map_name
+      map = File.join(map_name, '{TOPLEVEL}/{SUBPATH}')
+    end
+    raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
+    while Symbol === map
+      map_name = map
+      map = path_maps[map_name]
+    end
+    found = Path.follow(self, map, map_name)
+
+    annotate_found_where(found, map_name)  if annotate
+
+    found
+  end
+
+  def self.exists_file_or_alternatives(file)
+    return file if File.exist?(file) or File.directory?(file)
+    %w(gz bgz zip).each do |extension|
+      alt_file = file + '.' + extension
+      return alt_file if File.exist?(alt_file) or File.directory?(alt_file)
+    end
+    nil
+  end
+
+  def find(where = nil)
+    if located?
+      if File.exist?(self)
+        return self if located?
+      else
+        found = Path.exists_file_or_alternatives(self)
+        if found
+          return self.annotate(found)
+        else
+          return self if located?
+        end
+      end
+    end
+
+    return find_all if where == 'all' || where == :all
+
+    return follow(where) if where
+
+    map_order.each do |map_name|
+      found = follow(map_name, false)
+
+      found = Path.exists_file_or_alternatives(found)
+      return annotate_found_where(found, map_name) if found
+    end
+
+    return follow(:default)
+  end
+
+  def exist?
+    # OPEN
+    found = self.find
+    File.exist?(found) || File.directory?(found)
+  end
+
+  alias exists? exist?
+
+  def find_all(caller_lib = nil, search_paths = nil)
+    map_order
+      .collect{|where| find(where) }
+      .select{|file| file.exist? }.uniq
+  end
+
+  def find_with_extension(extension, *args)
+    found = self.find(*args)
+    return found if found.exists?
+    found_with_extension = self.set_extension(extension).find
+    found_with_extension.exists? ? found_with_extension : found
+  end
+end

data/lib/scout/path/tmpfile.rb

@@ -0,0 +1,8 @@
+module TmpFile
+  def self.with_path(*args, &block)
+    TmpFile.with_file(*args) do |file|
+      Path.setup(file)
+      yield file
+    end
+  end
+end