rbbt-util 3.2.1 → 4.0.0

data/lib/rbbt/{util/fix_width_table.rb → fix_width_table.rb}

@@ -4,15 +4,15 @@ class FixWidthTable
   def initialize(filename, value_size = nil, range = nil, update = false)
     @filename = filename
 
-    if update or %w(memory stringio).include?(filename.to_s.downcase) or not File.exists? filename
-      Log.debug "Writing FixWidthTable at #{ @filename.inspect }"
-      @value_size = value_size
-      @range = range
+    if update or %w(memory stringio).include?(filename.to_s.downcase) or not File.exists?(filename)
+      Log.debug "FixWidthTable create: #{ filename }"
+      @value_size  = value_size
+      @range       = range
       @record_size = @value_size + (@range ? 12 : 4)
 
       if %w(memory stringio).include? filename.to_s.downcase
         @filename = :memory
-        @file = StringIO.new
+        @file     = StringIO.new
       else
         FileUtils.rm @filename if File.exists? @filename
         @file = File.open(@filename, 'wb')
@@ -22,9 +22,9 @@ class FixWidthTable
       @file.write [@range ? 1 : 0 ].pack("C")
       @size = 0
     else
-      Log.debug "Reading FixWidthTable at #{ @filename.inspect }"
-      @file = File.open(@filename, 'r')
-      @value_size = @file.read(4).unpack("L").first
+      Log.debug "FixWidthTable up-to-date: #{ filename }"
+      @file        = File.open(@filename, 'r')
+      @value_size  = @file.read(4).unpack("L").first
       @range       = @file.read(1).unpack("C").first == 1
       @record_size = @value_size + (@range ? 12 : 4)
       @size        = (File.size(@filename) - 5) / (@record_size)
@@ -36,9 +36,7 @@ class FixWidthTable
   def self.get(filename, value_size = nil, range = nil, update = false)
     return self.new(filename, value_size, range, update) if filename == :memory
     case
-    when (!File.exists?(filename) or update)
-      CONNECTIONS[filename] = self.new(filename, value_size, range, update)
-    when (not CONNECTIONS.include?(filename))
+    when (!File.exists?(filename) or update or not CONNECTIONS.include?(filename))
       CONNECTIONS[filename] = self.new(filename, value_size, range, update)
     end
 
@@ -48,9 +46,9 @@ class FixWidthTable
   def format(pos, value)
     padding = value_size - value.length
     if range
-      (pos  + [value + "\0" * padding]).pack("llla#{value_size}")
+      (pos  + [value + ("\0" * padding)]).pack("llla#{value_size}")
     else
-      [pos, value + "\0" * padding].pack("la#{value_size}")
+      [pos, value + ("\0" * padding)].pack("la#{value_size}")
     end
   end
 
@@ -236,4 +234,10 @@ class FixWidthTable
     end
   end
 
+  def values_at(*list)
+    list.collect{|pos|
+      self[pos]
+    }
+  end
+
 end

data/lib/rbbt/persist.rb

@@ -0,0 +1,164 @@
+require 'digest/md5'
+
+require 'rbbt/util/misc'
+require 'rbbt/util/open'
+
+require 'rbbt/persist/tsv'
+
+module Persist
+  CACHEDIR="/tmp/tsv_persistent_cache"
+  FileUtils.mkdir CACHEDIR unless File.exist? CACHEDIR
+
+  def self.cachedir=(cachedir)
+    CACHEDIR.replace cachedir
+    FileUtils.mkdir_p CACHEDIR unless File.exist? CACHEDIR
+  end
+
+  def self.cachedir
+    CACHEDIR
+  end
+ 
+  def self.newer?(path, file)
+    return true if not File.exists? file
+    return true if File.mtime(path) < File.mtime(file)
+    return false
+  end
+
+  def self.is_persisted?(path, persist_options = {})
+    return false if not File.exists? path
+    return false if TrueClass === persist_options[:update]
+
+    check = persist_options[:check]
+    if not check.nil?
+      if Array === check
+        return false if check.select{|file| newer? path, file}.any?
+     else
+        return false if newer? path, check
+     end
+    end
+
+    return true
+  end
+
+  def self.persistence_path(file, persist_options = {}, options = {})
+    persistence_file = Misc.process_options persist_options, :file
+    return persistence_file if not persistence_file.nil?
+
+    prefix = Misc.process_options persist_options, :prefix
+
+    if prefix.nil?
+      perfile = file.gsub(/\//, '>') 
+    else
+      perfile = prefix.to_s + ":" + file.gsub(/\//, '>') 
+    end
+
+    if options.include? :filters
+      options[:filters].each do |match,value|
+        perfile = perfile + "&F[#{match}=#{Misc.digest(value.inspect)}]"
+      end
+    end
+
+    persistence_dir = Misc.process_options(persist_options, :dir) || CACHEDIR
+
+    filename = perfile.gsub(/\s/,'_').gsub(/\//,'>')
+    options_md5 = Misc.hash2md5 options
+    filename  << ":" << options_md5 unless options_md5.empty?
+
+    File.join(persistence_dir, filename)
+  end
+
+  def self.load_file(path, type)
+    case (type || "nil").to_sym
+    when :nil
+      nil
+    when :tsv
+      TSV.open(path)
+    when :marshal_tsv
+      TSV.setup(Marshal.load(Open.open(path)))
+    when :fwt
+      FixWidthTable.get(path) 
+    when :string, :text
+      Open.read(path)
+    when :array
+      res = Open.read(path).split("\n", -1)
+      res.pop
+      res
+    when :marshal
+      Marshal.load(Open.open(path))
+    when :yaml
+      YAML.load(Open.open(path))
+    when :float
+      Open.read(path).to_f
+    when :integer
+      Open.read(path).to_i
+    when :tsv
+      TSV.open(Open.open(path))
+    else
+      raise "Unknown persistence: #{ type }"
+    end
+  end
+
+  def self.save_file(path, type, content)
+
+    return if (content.nil? and File.exists? path)
+    
+    case (type || "nil").to_sym
+    when :nil
+      nil
+    when :fwt
+      content.file.seek 0
+      Open.write(path, content.file.read)
+    when :tsv
+      Open.write(path, content.to_s)
+    when :string, :text
+      Open.write(path, content)
+    when :array
+      Open.write(path, content * "\n" + "\n")
+    when :marshal_tsv
+      Open.write(path, Marshal.dump(content.dup))
+    when :marshal
+      Open.write(path, Marshal.dump(content))
+    when :yaml
+      Open.write(path, YAML.dump(content))
+    when :float, :integer, :tsv
+      Open.write(path, content.to_s)
+    else
+      raise "Unknown persistence: #{ type }"
+    end
+  end
+
+  def self.persist(name, type = nil, persist_options = {})
+    type ||= :marshal
+    persist_options = Misc.add_defaults persist_options, :persist => true
+
+    if persist_options[:persist]
+      path = persistence_path(name, persist_options)
+      Misc.lock(path) do
+        if is_persisted?(path, persist_options)
+          Log.debug "Persist up-to-date: #{ path } - #{persist_options.inspect}"
+          return load_file(path, type) 
+        else
+          Log.debug "Persist create: #{ path } - #{persist_options.inspect}"
+        end
+        res = yield
+        save_file(path, type, res)
+        res
+      end
+    else
+      yield
+    end
+  end
+end
+
+module LocalPersist
+
+  attr_accessor :local_persist_dir
+  def local_persist(name, type = nil, options= {}, &block)
+    Persist.persist(name, type, options.merge({:dir => @local_persist_dir}), &block)
+  end
+
+  def local_persist_tsv(source, name, opt = {}, options= {}, &block)
+    Persist.persist_tsv(source, name, opt, options.merge({:dir => @local_persist_dir}), &block)
+  end
+
+end

data/lib/rbbt/persist/tsv.rb

@@ -0,0 +1,135 @@
+require 'tokyocabinet'
+
+module Persist
+  TC_CONNECTIONS = {}
+  def self.open_tokyocabinet(path, write, serializer = nil)
+    write = true if not File.exists?(path)
+    flags = (write ? TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT : TokyoCabinet::HDB::OREADER)
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = TC_CONNECTIONS[path] ||= TokyoCabinet::HDB.new
+    database.close
+
+    if !database.open(path, flags)
+      ecode = database.ecode
+      raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
+    end
+
+    if not database.respond_to? :old_close
+      class << database
+        attr_accessor :writable, :closed, :persistence_path
+
+        alias old_close close
+        def close
+          @closed = true
+          old_close
+        end
+
+        def read(force = false)
+          return if not write? and not closed and not force
+          self.close
+          if !self.open(@persistence_path, TokyoCabinet::BDB::OREADER)
+            ecode = self.ecode
+            raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+          end
+          @writable = false
+          @closed = false
+          self
+        end
+
+        def write(force = true)
+          return if write? and not closed and not force
+          self.close
+          if !self.open(@persistence_path, TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT)
+            ecode = self.ecode
+            raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+          end
+          @writable = true
+          @closed = false
+          self
+        end
+
+        def write?
+          @writable
+        end
+
+        def collect
+          res = []
+          each do |key, value|
+            res << if block_given?
+                     yield key, value
+            else
+              [key, value]
+            end
+          end
+          res
+        end
+
+        def delete(key)
+          out(key)
+        end
+
+
+        def merge!(hash)
+          hash.each do |key,values|
+            self[key] = values
+          end
+        end
+
+      end
+    end
+
+    database.persistence_path ||= path
+
+    TSV.setup database
+    database.serializer = serializer unless serializer.nil?
+
+    database
+  end
+
+  def self.persist_tsv(source, filename, options, persist_options = {})
+    persist_options[:prefix] ||= "TSV"
+
+    data = case
+           when persist_options[:data]
+             persist_options[:data]
+           when persist_options[:persist]
+
+             filename ||= source.filename if source.respond_to? :filename
+             filename ||= source.object_id.to_s
+
+             path = persistence_path(filename, persist_options, options)
+             if is_persisted? path
+               Log.debug "TSV persistence up-to-date: #{ path }"
+               return open_tokyocabinet(path, false) 
+             else
+               Log.debug "TSV persistence creating: #{ path }"
+             end
+
+             FileUtils.rm path if File.exists? path
+
+             data = open_tokyocabinet(path, true)
+             data.serializer = :type
+             data
+           else
+             data = {}
+           end
+
+    begin
+      yield data
+    rescue Exception
+      begin
+        data.close if data.respondo_to? :close
+      rescue
+      end
+      FileUtils.rm path if path and File.exists? path
+      raise $!
+    end
+
+    data.read if data.respond_to? :read and  data.respond_to? :write? and data.write?
+
+    data
+  end
+
+end

data/lib/rbbt/resource.rb

@@ -0,0 +1,100 @@
+require 'rbbt/util/open'
+require 'rbbt/util/log'
+require 'rbbt/util/chain_methods'
+require 'rbbt/resource/path'
+require 'rbbt/resource/rake'
+
+module Resource
+  extend ChainMethods
+  self.chain_prefix = :resource
+  def self.extended(base)
+    setup_chains(base)
+    if not base.respond_to? :pkgdir
+      class << base
+        attr_accessor :pkgdir, :subdir, :resources, :rake_dirs
+      end
+
+      base.pkgdir = 'rbbt'
+      base.subdir = ''
+      base.resources = {}
+      base.rake_dirs = {}
+    end
+    base
+  end
+
+  def root()
+    Path.setup @subdir || "", @pkgdir, self
+  end
+
+  def resource_method_missing(name, prev = nil, *args)
+    # Fix problem with ruby 1.9 calling methods by its own initiative. ARG
+    root.send(name, prev, *args)
+  end
+
+  def [](file = nil)
+    if file.nil?
+      root
+    else
+      root.send(file)
+    end
+  end
+
+  def claim(path, type, content = nil, &block)
+    path = path.find if path.respond_to? :find
+    if type == :rake
+      @rake_dirs[path] = content
+    else
+      @resources[path] = [type, content || block]
+
+      if type == :install
+        Log.debug "Preparing software: #{path}"
+        path.produce
+        software_dir = path.resource.root.software.find :user
+        set_software_env(software_dir)
+      end
+    end
+  end
+
+  def produce(path)
+    case
+    when @resources.include?(path)
+      type, content = @resources[path]
+    when has_rake(path)
+      type = :rake
+      rake_dir, content = rake_for(path)
+    else
+      raise "Resource #{ path } does not seem to be claimed"
+    end
+
+    case type
+    when :string
+      Open.write(path, content)
+    when :url
+      Open.write(path, Open.open(content))
+    when :proc
+      data = content.call
+      Open.write(path, data) unless File.exists? path
+    when :rake
+      run_rake(path, content, rake_dir)
+    when :install
+      Log.debug "Installing software: #{path}"
+      software_dir = path.resource.root.software.find :user
+      preamble = <<-EOF
+#!/bin/bash
+
+RBBT_SOFTWARE_DIR="#{software_dir}"
+
+INSTALL_HELPER_FILE="#{Rbbt.share.install.software.lib.install_helpers.find :lib, caller_lib_dir(__FILE__)}"
+source "$INSTALL_HELPER_FILE"
+      EOF
+
+      CMD.cmd('bash', :in => preamble + "\n" + Open.read(content))
+
+      set_software_env(software_dir)
+    else
+      raise "Could not produce #{ resource }. (#{ type }, #{ content })"
+    end
+
+    path
+  end
+end