scout-essentials 1.0.0

data/lib/scout/path/util.rb

@@ -0,0 +1,127 @@
+module Path
+  def no_method_missing
+    class << self
+      undef_method :method_missing
+    end
+  end
+
+  def self.is_filename?(string, need_to_exists = true)
+    return false if string.nil?
+    return true if Path === string
+    return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265 }.empty? and (! need_to_exists || File.exist?(string))
+    return false
+  end
+
+  def self.sanitize_filename(filename, length = 254)
+    if filename.length > length
+      if filename =~ /(\..{2,9})$/
+        extension = $1
+      else
+        extension = ''
+      end
+
+      post_fix = "--#{filename.length}@#{length}_#{Misc.digest(filename)[0..4]}" + extension
+
+      filename = filename[0..(length - post_fix.length - 1)] << post_fix
+    else
+      filename
+    end
+    filename
+  end
+
+  def directory?
+    return nil unless self.exist?
+    File.directory?(self.find)
+  end
+
+  def dirname
+    self.annotate(File.dirname(self))
+  end
+
+  def basename
+    self.annotate(File.basename(self))
+  end
+
+  def glob(pattern = '*')
+    if self.include? "*"
+      self.glob_all
+    else
+      return [] unless self.exist? 
+      found = self.find
+      exp = File.join(found, pattern)
+      paths = Dir.glob(exp).collect{|f| self.annotate(f) }
+
+      paths.each do |p|
+        p.original = File.join(found.original, p.sub(/^#{found}/, ''))
+      end if found.original
+
+      paths
+    end
+  end
+
+  def glob_all(pattern = nil, caller_lib = nil, search_paths = nil)
+    search_paths ||= Path.path_maps
+    search_paths = search_paths.dup
+
+    search_paths.keys.collect do |where| 
+      found = find(where)
+      paths = pattern ? Dir.glob(File.join(found, pattern)) : Dir.glob(found) 
+
+      paths = paths.collect{|p| self.annotate p }
+
+      paths = paths.each do |p|
+        p.original = File.join(found.original, p.sub(/^#{found}/, ''))
+        p.where = where
+      end if found.original and pattern
+
+      paths
+    end.flatten.uniq
+  end
+
+  def set_extension(extension)
+    self.annotate(self + ".#{extension}")
+  end
+
+  def unset_extension
+    self.annotate(self.split(".")[0..-2] * ".")
+  end
+
+  def remove_extension(extension = nil)
+    if extension.nil?
+      unset_extension
+    else
+      self.annotate(self.sub(/\.#{extension}$/,''))
+    end
+  end
+
+  def replace_extension(new_extension = nil, multiple = false)
+    if String === multiple
+      new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
+    elsif multiple
+      new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
+    else
+      new_path = self.sub(/\.[^\.\/]{1,5}$/,'')
+    end
+    new_path = new_path + "." + new_extension.to_s
+    self.annotate(new_path)
+  end
+
+
+  # Is 'file' newer than 'path'? return non-true if path is newer than file
+  def self.newer?(path, file, by_link = false)
+    return true if not Open.exists?(file)
+    path = path.find if Path === path
+    file = file.find if Path === file
+    if by_link
+      patht = File.exist?(path) ? File.lstat(path).mtime : nil
+      filet = File.exist?(file) ? File.lstat(file).mtime : nil
+    else
+      patht = Open.mtime(path)
+      filet = Open.mtime(file)
+    end
+    return true if patht.nil? || filet.nil?
+    diff = patht - filet
+    return diff if diff < 0
+    return false
+  end
+end

data/lib/scout/path.rb

@@ -0,0 +1,51 @@
+require_relative 'meta_extension'
+require_relative 'path/find'
+require_relative 'path/util'
+require_relative 'path/tmpfile'
+
+module Path
+  extend MetaExtension
+  extension_attr :pkgdir, :libdir, :path_maps
+
+  def self.default_pkgdir
+    @@default_pkgdir ||= 'scout'
+  end
+  
+  def self.default_pkgdir=(pkgdir)
+    @@default_pkgdir = pkgdir
+  end
+
+  def pkgdir
+    @pkgdir ||= Path.default_pkgdir
+  end
+
+  def libdir
+    @libdir || Path.caller_lib_dir
+  end
+
+  def path_maps
+    @path_maps ||= Path.path_maps.dup
+  end
+
+  def join(subpath, prevpath = nil)
+    subpath = subpath.to_s if Symbol === subpath
+    prevpath = prevpath.to_s if Symbol === prevpath
+
+    subpath = File.join(prevpath.to_s, subpath) if prevpath
+    new = self.empty? ? subpath.dup : File.join(self, subpath)
+    self.annotate(new)
+    new
+  end
+
+  alias [] join
+  alias / join
+
+  def method_missing(name, prev = nil, *args, &block)
+    if block_given? || name.to_s.start_with?('to_')
+      super name, prev, *args, &block
+    else
+      join(name, prev)
+    end
+  end
+
+end

data/lib/scout/persist/open.rb

@@ -0,0 +1,17 @@
+require_relative '../open'
+require 'json'
+require 'yaml'
+
+module Open
+  def self.json(file)
+    Open.open(file){|f| JSON.load(f) }
+  end
+
+  def self.yaml(file)
+    Open.open(file){|f| YAML.load(f) }
+  end
+
+  def self.marshal(file)
+    Open.open(file){|f| Marshal.load(f) }
+  end
+end

data/lib/scout/persist/path.rb

@@ -0,0 +1,15 @@
+require_relative 'open'
+
+module Path
+  def yaml(*rest, &block)
+    Open.yaml(self, *rest, &block)
+  end
+
+  def json(*rest, &block)
+    Open.json(self, *rest, &block)
+  end
+
+  def marshal(*rest, &block)
+    Open.marshal(self, *rest, &block)
+  end
+end

data/lib/scout/persist/serialize.rb

@@ -0,0 +1,157 @@
+require_relative '../open'
+require_relative 'open'
+require 'set'
+
+module Persist
+  TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
+  SERIALIZER = :json
+
+  class << self
+    attr_accessor :save_drivers, :load_drivers
+    def save_drivers
+      @save_drivers ||= {}
+    end
+    def load_drivers
+      @load_drivers ||= {}
+    end
+  end
+
+  def self.serialize(content, type)
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    case type
+    when nil, :string, :text, :integer, :float, :boolean, :file, :path, :select, :folder, :binary
+      if IO === content || StringIO === content
+        content.read
+      else
+        content.to_s
+      end
+    when :array
+      content * "\n"
+    when :yaml
+      content.to_yaml
+    when :json
+      content.to_json
+    when :marshal
+      Marshal.dump(content)
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        type = m[1].to_sym
+        content.collect{|c| serialize(c, type) } * "\n"
+      else
+        raise "Persist does not know #{Log.fingerprint type}"
+      end
+    end
+  end
+
+  def self.deserialize(serialized, type)
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+
+    case type
+    when nil, :string, :text, :file, :stream, :select, :folder
+      serialized
+    when :path
+      Path.setup(serialized)
+    when :integer
+      serialized.to_i
+    when :float
+      serialized.to_f
+    when :boolean
+      TRUE_STRINGS.include? serialized.strip
+    when :array
+      serialized.split("\n")
+    when :yaml
+      YAML.parse(serialized)
+    when :json
+      JSON.parse(serialized)
+    when :marshal
+      Marshal.load(serialized)
+    else
+      if m = type.to_s.match(/(.*)_array/)
+        type = m[1].to_sym
+        new_content = serialized.split("\n")
+        new_content.collect{|c| deserialize(c, type) }
+      else
+        raise "Persist does not know #{Log.fingerprint type}"
+      end
+    end
+  end
+
+  MEMORY = {}
+  def self.save(content, file, type = :serializer)
+    type = :serializer if type.nil?
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    type = MEMORY if type == :memory
+    return if content.nil?
+    type = MEMORY if type == :memory
+    type = :serializer if type.nil?
+
+    if Hash === type
+      type[file] = content
+      return
+    end
+
+    Log.debug "Save #{Log.fingerprint type} on #{file}"
+    if save_drivers[type]
+      if save_drivers[type].arity == 1
+        return Open.sensible_write(file, save_drivers[type].call(content))
+      else
+        return save_drivers[type].call(file, content)
+      end
+    end
+
+    if type == :binary
+      content.force_encoding("ASCII-8BIT") if content.respond_to? :force_encoding
+      Open.open(file, :mode => 'wb') do |f|
+        f.puts content
+      end
+      content
+   else
+      serialized = serialize(content, type)
+      Open.sensible_write(file, serialized, :force => true)
+      return nil
+    end
+  end
+
+  def self.load(file, type = :serializer)
+    file = file.find if Path === file
+    type = :serializer if type.nil?
+    type = type.to_sym if String === type
+    type = SERIALIZER if type == :serializer
+    type = MEMORY if type == :memory
+    return unless Hash === type || Open.exist?(file) 
+
+    Log.debug "Load #{Log.fingerprint type} on #{file}"
+    if load_drivers[type]
+      return load_drivers[type].call(file)
+    end
+
+    case type
+    when :binary
+      Open.read(file, :mode => 'rb')
+    when :yaml
+      Open.yaml(file)
+    when :json
+      Open.json(file)
+    when :marshal, :serializer
+      Open.marshal(file)
+    when :stream
+      Open.open(file)
+    when :file
+      value = Open.read(file)
+      value.sub!(/^\./, File.dirname(file)) if value.start_with?("./")
+      value
+    when :file_array
+      Open.read(file).split("\n").collect do |f|
+        f.sub!(/^\./, File.dirname(file)) if f.start_with?("./")
+        f
+      end
+    when Hash
+      type[file]
+    else
+      deserialize(Open.read(file), type)
+    end
+  end
+end

data/lib/scout/persist.rb

@@ -0,0 +1,104 @@
+require_relative 'persist/serialize'
+require_relative 'persist/open'
+require_relative 'persist/path'
+
+module Persist
+  class << self
+    attr :cache_dir
+    def cache_dir=(cache_dir)
+      @cache_dir = Path === cache_dir ? cache_dir : Path.setup(cache_dir)
+    end
+    def cache_dir
+      @cache_dir ||= Path.setup("var/cache/persistence")
+    end
+
+    attr_writer :lock_dir
+    def lock_dir
+      @lock_dir ||= Path.setup("tmp/persist_locks").find
+    end
+  end
+
+  def self.persistence_path(name, options = {})
+    options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
+    other_options = IndiferentHash.pull_keys options, :other
+    name = name.filename if name.respond_to?(:filename) && name.filename
+    persist_options = {}
+    TmpFile.tmp_for_file(name, options.merge(persist_options), other_options)
+  end
+
+  MEMORY_CACHE = {}
+  CONNECTIONS = {}
+  def self.persist(name, type = :serializer, options = {}, &block)
+    persist_options = IndiferentHash.pull_keys options, :persist 
+    return yield if FalseClass === persist_options[:persist]
+    file = persist_options[:path] || options[:path] || persistence_path(name, options)
+
+    if type == :memory
+      repo = options[:memory] || options[:repo] || MEMORY_CACHE
+      repo[file] ||= yield
+      return repo[file]
+    end
+
+    update = options[:update] || persist_options[:update]
+    update = Open.mtime(update) if Path === update
+    update = Open.mtime(file) >= update ? false : true if Time === update
+
+    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
+
+    Open.lock lockfile do |lock|
+      if Open.exist?(file) && ! update
+        Persist.load(file, type)
+      else
+        begin
+          file = file.find if Path === file
+          return yield(file) if block.arity == 1
+          res = yield
+
+          if res.nil?
+            return Persist.load(file, type)
+          end
+
+          Open.rm(file)
+
+          if IO === res || StringIO === res
+            tee_copies = options[:tee_copies] || 1
+            main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
+            main.lock = lock
+            t = Thread.new do
+              Thread.current.report_on_exception = false
+              Thread.current["name"] = "file saver: " + file
+              Open.sensible_write(file, main)
+            end
+            Thread.pass until t["name"]
+            copies.each_with_index do |copy,i|
+              next_stream = copies[i+1] if copies.length > i
+              ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
+            end
+            res = copies.first
+            raise KeepLocked.new(res)
+          else
+            pres = Persist.save(res, file, type)
+            res = pres unless pres.nil?
+          end
+        rescue Exception
+          Thread.handle_interrupt(Exception => :never) do
+            if Open.exist?(file)
+              Log.debug "Failed persistence #{file} - erasing"
+              Open.rm file
+            else
+              Log.debug "Failed persistence #{file}"
+            end
+          end
+          raise $! unless options[:canfail]
+        end
+        res
+      end
+    end
+  end
+
+  def self.memory(name, options = {}, &block)
+    options[:persist_path] ||= options[:path] ||= [name, options[:key]].compact * ":"
+    self.persist(name, :memory, options, &block)
+  end
+
+end

data/lib/scout/resource/open.rb

@@ -0,0 +1,8 @@
+class << Open
+  alias _just_open open
+
+  def open(file, *args, **kwargs, &block)
+    file.produce if Path === file
+    _just_open(file, *args, **kwargs, &block)
+  end
+end

data/lib/scout/resource/path.rb

@@ -0,0 +1,80 @@
+module Path
+  def produce(force = false)
+    return self if ! force && (Open.exist?(self) || @produced)
+    begin
+      if Resource === self.pkgdir
+        self.pkgdir.produce self, force
+      else
+        false
+      end
+    rescue ResourceNotFound
+      false
+    rescue
+      message = $!.message
+      message = "No exception message" if message.nil? || message.empty?
+      Log.warn "Error producing #{self}: #{message}"
+      raise $!
+    ensure
+      @produced = true
+    end
+  end
+
+  def produce_with_extension(extension, *args)
+    begin
+      self.produce(*args)
+    rescue Exception
+      exception = $!
+      begin
+        self.set_extension(extension).produce(*args)
+      rescue Exception
+        raise exception
+      end
+    end
+  end
+
+  def produce_and_find(extension = nil, *args)
+    found = if extension
+              found = find_with_extension(extension, *args)
+              found.exists? ? found : produce_with_extension(extension, *args)
+            else
+              found = find
+              found.exists? ? found : produce(*args)
+            end
+    raise "Not found: #{self}" unless found
+
+    found
+  end
+
+  def relocate
+    return self if Open.exists?(self)
+    Resource.relocate(self)
+  end
+
+  def identify
+    Resource.identify(self)
+  end
+
+  def open(*args, &block)
+    produce
+    Open.open(self, *args, &block)
+  end
+
+  def read
+    produce
+    Open.read(self)
+  end
+
+  def write(*args, &block)
+    Open.write(self.find, *args, &block)
+  end
+
+  def list
+    found = produce_and_find('list')
+    Open.list(found)
+  end
+
+  def exists?
+    return true if Open.exists?(self.find)
+    self.produce
+  end
+end

data/lib/scout/resource/produce/rake.rb

@@ -0,0 +1,69 @@
+require_relative '../../misc'
+require_relative '../../path'
+require 'rake'
+
+class Rake::FileTask
+  class << self
+    alias_method :old_define_task, :define_task
+  end
+
+  def self.define_task(file, *args, &block)
+    @@files ||= []
+    @@files << file
+    old_define_task(file, *args, &block)
+  end
+
+  def self.files
+    @@files
+  end
+
+  def self.clear_files
+    @@files = []
+  end
+end
+
+module ScoutRake
+  class TaskNotFound < StandardError; end
+  def self.run(rakefile, dir, task, &block)
+    old_pwd = FileUtils.pwd
+
+    Rake::Task.clear
+    Rake::FileTask.clear_files
+
+    t = nil
+    pid = Process.fork{
+      if block_given?
+        TOPLEVEL_BINDING.receiver.instance_exec &block
+      else
+        if Path.is_filename? rakefile
+          rakefile = rakefile.produce.find
+          load rakefile
+        else
+          TmpFile.with_file(rakefile) do |tmpfile|
+            load tmpfile
+          end
+        end
+      end
+
+      raise TaskNotFound if Rake::Task[task].nil?
+
+      #Misc.pre_fork
+      begin
+        Misc.in_dir(dir) do
+          Rake::Task[task].invoke
+
+          Rake::Task.clear
+          Rake::FileTask.clear_files
+        end
+      rescue Exception
+        Log.error "Error in rake: #{$!.message}"
+        Log.exception $!
+        Kernel.exit! -1
+      end
+      Kernel.exit! 0
+    }
+    Process.waitpid(pid)
+    raise "Rake failed" unless $?.success?
+
+  end
+end