rbbt-util 1.1.0 → 1.2.1

data/lib/rbbt/util/excel2tsv.rb

@@ -1,7 +1,6 @@
-require 'rbbt/util/tsv'
-require 'rbbt/util/open'
 require 'spreadsheet'
-
+require 'rbbt/util/tsv'
+require 'rbbt/util/tmpfile'
 class TSV
   def self.excel2tsv(file, options = {})
     sheet = options.delete :sheet

data/lib/rbbt/util/log.rb

@@ -48,3 +48,8 @@ module Log
     @@severity = ENV['RBBT_LOG'].to_i
   end
 end
+
+def ddd(message, file = $stdout)
+  Log.debug "DEVEL => " << caller.first
+  Log.debug "DEVEL: " << message.inspect
+end

data/lib/rbbt/util/misc.rb

@@ -80,6 +80,14 @@ module Misc
     new_options
   end
 
+  def self.process_options(hash, *keys)
+    if keys.length == 1
+      hash.delete keys.first.to_sym
+    else
+      keys.collect do |key| hash.delete(key.to_sym) || hash.delete(key.to_s) end
+    end
+  end
+
   def self.string2hash(string)
 
     options = {}
@@ -91,7 +99,7 @@ module Misc
       end
 
       option = option.sub(":",'').to_sym if option.chars.first == ':'
-
+      value  = value.sub(":",'').to_sym if String === value and value.chars.first == ':'
       
       if value == true
         options[option] = option.to_s.chars.first != '!' 
@@ -129,10 +137,18 @@ module Misc
     fields.each_with_index{|f,i| return i if f =~ field_re}
     raise FieldNotFoundError, "Field '#{ field }' was not found" unless quiet
   end
+
+  def self.first(list)
+    return nil if list.nil?
+    return list.first
+  end
+
 end
 
 module PDF2Text
   def self.pdf2text(filename)
+    require 'rbbt/util/cmd'
+    require 'rbbt/util/tmpfile'
     TmpFile.with_file(Open.read(filename)) do |pdf|
       CMD.cmd("pdftotext #{pdf} -", :pipe => false, :stderr => true)
     end
@@ -169,8 +185,20 @@ class NamedArray < Array
     keys = keys.collect{|k| Misc.field_position(fields, k) }
     original_values_at(*keys)
   end
+
+  def zip_fields
+    zipped = self[0].zip(*self[1..-1])
+    zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields 
+    zipped 
+  end
 end
 
+def benchmark
+  require 'benchmark'
+  puts(Benchmark.measure do
+    yield
+  end)
+end
 
 def profile 
   require 'ruby-prof'

data/lib/rbbt/util/open.rb

@@ -146,6 +146,7 @@ module Open
     !! (file =~ /\.zip/)
   end
 
+
   # Open Read Write
 
   def self.open(url, options = {})

data/lib/rbbt/util/persistence.rb

@@ -0,0 +1,109 @@
+require 'rbbt/util/misc'
+require 'rbbt/util/open'
+require 'yaml'
+
+module Persistence
+  require 'rbbt/util/tc_hash'
+  TSV = TCHash
+
+  CACHEDIR="/tmp/tsv_persistent_cache"
+  FileUtils.mkdir CACHEDIR unless File.exist? CACHEDIR
+
+  def self.cachedir=(cachedir)
+    CACHEDIR.replace cachedir
+    FileUtils.mkdir_p CACHEDIR unless File.exist? CACHEDIR
+  end
+
+  def self.cachedir
+    CACHEDIR
+  end
+ 
+  def self.get_persistence_file(file, prefix, options = {})
+    File.join(CACHEDIR, prefix.to_s.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, options].inspect))
+  end
+
+  def self.persist(file, prefix = "", persistence_type = :string, options = {})
+    options = Misc.add_defaults options, :persistence => true
+
+    persistence, persistence_file =
+      Misc.process_options options, :persistence, :persistence_file
+
+    filename = Misc.process_options options, :filename
+    filename ||= case
+                  when (String === file and File.exists? file)
+                    File.expand_path file
+                  when File === file
+                    File.expand_path file.path
+                  when TSV === file
+                    file.filename
+                  else
+                    Digest::MD5.hexdigest(file.inspect)
+                  end
+
+    if persistence
+      persistence_file ||= get_persistence_file(filename, prefix, options)
+      
+      #{{{ CREATE
+      if ! File.exists? persistence_file
+        Log.low "Creating Persistence #{ persistence_file } for #{ filename }"
+        res = yield file, options, filename, persistence_file
+        if Array === res and res.length == 2 and (Hash === res[1] or res[1].nil?)
+          data, extra = res
+        else
+          data, extra = [res, nil]
+        end
+        ddd data.filename
+
+        case persistence_type.to_sym
+        when :tsv 
+          if Hash === data or Object::TSV === data
+            Log.debug "Creating #{Persistence::TSV} for #{ persistence_file }"
+            per = Persistence::TSV.get persistence_file
+            per.write
+            data.each{|k,v| per[k.to_s] = v}
+            %w(case_insensitive fields key_field type filename). each do |key| 
+              if data.respond_to? key
+                per.send "#{key}=".to_sym, data.send(key.to_sym) 
+              else
+                per.send "#{key}=".to_sym, extra[key.to_sym]
+              end
+            end
+            per.read
+
+            data = per
+          end
+        when :string
+          Open.write(persistence_file, data.to_s)
+        when :marshal
+          Open.write(persistence_file, Marshal.dump(data))
+        when :yaml
+          Open.write(persistence_file, YAML.dump(data))
+        end
+        
+        return [data, extra]
+
+      #{{{ LOAD
+      else
+        Log.low "Opening Persistence #{ persistence_file } for #{ filename }"
+        case persistence_type.to_sym
+        when :tsv
+          data        = Persistence::TSV.get persistence_file
+          
+          extra = {}
+          %W(case_insensitive fields key_field type filename).each{|key| extra[key.to_sym] = data.send key.to_sym}
+
+          return [data, extra]
+        when :string
+          return [Open.read(persistence_type), nil]
+        when :marshal
+          return [File.open(persistence_file){|f| Marshal.load(f)}, nil]
+        when :yaml
+          return [File.open(persistence_file){|f| YAML.load(f)}, nil]
+        end
+
+      end
+    else
+      yield file, options
+    end
+  end
+end

data/lib/rbbt/util/pkg_data.rb

@@ -1,10 +1,60 @@
 require 'rbbt/util/open'
 require 'rbbt/util/tsv'
 require 'rbbt/util/log'
-require 'rake'
+require 'rbbt/util/rake'
 
 module PKGData
-  FILES = {} unless defined? FILES
+  attr_accessor :claims
+  def self.extended(base)
+    base.claims = {}
+  end
+
+  module Path
+    attr_accessor :base
+
+    def method_missing(name, *args, &block)
+      new = File.join(self.dup, name.to_s)
+      new.extend Path
+      new.base = base
+      new
+    end
+
+    def [](name)
+      new = File.join(self.dup, name.to_s)
+      new.extend Path
+      new.base = base
+      new
+    end
+
+    def tsv(options = {})
+      produce
+      TSV.new self, options
+    end
+
+    def index(field = nil, other = nil, options = {})
+      produce
+      TSV.index self, options.merge(:target => field, :others => other)
+    end
+
+    def open
+      produce
+      Open.open(self)
+    end
+
+    def read
+      produce
+      Open.read(self)
+    end
+
+    def produce
+      return if File.exists? self
+
+      Log.debug("Trying to produce '#{ self }'")
+      file, producer = base.reclaim self
+      base.produce(self, producer[:get], producer[:subdir], producer[:sharedir])
+    end
+  end
+
   class SharedirNotFoundError < StandardError; end
 
   def self.sharedir_for_file(file = __FILE__)
@@ -32,79 +82,81 @@ module PKGData
     raise SharedirNotFoundError
   end
 
-  def run_rake(path, dir, task = nil)
-    rakefile = File.join(dir, 'Rakefile')
-    return nil unless File.exists? rakefile
-    if task.nil?
-      task ||= :default
-    else
-      task.sub!(/\/$/,'') if String === task
-      path = File.dirname(path)
-    end
+  def files
+    path = datadir.dup.extend Path
+    path.base = self
+    path
+  end
 
-    load rakefile
-    old_dir = FileUtils.pwd
-    begin
-      FileUtils.mkdir_p path
-      FileUtils.chdir path
-      Rake::Task[task].invoke
-      Rake::Task[task].reenable
-    ensure
-      FileUtils.chdir old_dir
+  def in_datadir?(file)
+    if File.expand_path(file.to_s) =~ /^#{Regexp.quote File.expand_path(datadir)}/
+      true
+    else
+      false
     end
-    true
   end
 
-  def get_datafile(file, path, get, sharedir)
-    Log.log "Getting data file '#{ file }' into '#{ path }'. Get: #{get.to_s}"
-
-    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
-
-    case
-    when get.nil?
-      load File.join(sharedir, 'install', file)
-
-    when Proc === get
-      Open.write(path, get.call(file, path))
-
-    when TSV === get
-      Open.write(path, get.to_s)
+  def claim(file, get = nil, subdir = nil, sharedir = nil)
+    file = case
+           when (file.nil? or file === :all)
+             File.join(datadir, subdir.to_s)
+           when in_datadir?(file)
+             file
+           else
+             File.join(datadir, subdir.to_s, file.to_s)
+           end
+
+    sharedir ||= PKGData.get_caller_sharedir
+    claims[file] = {:get => get, :subdir => subdir, :sharedir => sharedir}
+    produce(file, get, subdir, sharedir) if TSV === get
+    produce(file, get, subdir, sharedir) if String === get and not File.exists?(get) and reclaim(file).nil? and not File.basename(get.to_s) == "Rakefile"
+  end
 
-    when String === get
-      install_dir =File.expand_path(File.join(sharedir, 'install')) 
-      rake_dir  = File.join(install_dir, File.dirname(get), file)
-      rake_task = nil
-
-      until rake_dir == install_dir
-        return if run_rake(path, rake_dir, rake_task)
-        rake_task = File.join(File.basename(rake_dir), rake_task || "")
-        rake_dir  = File.dirname(rake_dir)
-      end
+  def reclaim(file)
+    file = File.expand_path(file.dup)
+    return nil unless in_datadir? file
 
-      if (File.exists?(File.join(sharedir, get)) and not File.directory?(File.join(sharedir, get)))
-        Open.write(path, Open.open(File.join(sharedir, get)))
-      else
-        Open.write(path, Open.open(get, :wget_options => {:pipe => true}, :nocache => true))
+    while file != File.expand_path(datadir)
+      if @claims[file]
+        return [file, @claims[file]]
       end
+      file = File.dirname(file)
     end
+    nil
   end
 
-  def add_datafiles(files = {})
-    files.each do |file, info|
-      subpath, get, sharedir = info
+  def declaim(file)
+    @claims.delete file if @claims.include? file
+  end
 
-      path = File.join(datadir, subpath.to_s, file.to_s)
+  def produce_with_rake(rakefile, subdir, file)
+    task  = File.expand_path(file).sub(/^.*#{Regexp.quote(File.join(datadir, subdir))}\/?/, '')
+    RakeHelper.run(rakefile, task, File.join(File.join(datadir, subdir)))
+  end
 
-      if not File.exists?(path)
-        sharedir ||= PKGData.get_caller_sharedir
-        get_datafile(file.to_s, path, get, sharedir)
-      end
+  def produce(file, get, subdir, sharedir)
+    Log.low "Getting data file '#{ file }' into '#{ subdir }'. Get: #{get.class}"
 
-      FILES[file.to_s] = path
-    end
-  end
+    FileUtils.mkdir_p File.dirname(file) unless File.exists?(File.dirname(file))
 
-  def find_datafile(file)
-    FILES[file.to_s]
+    case 
+    when get.nil?
+      FileUtils.cp File.join(sharedir, subdir.to_s, File.basename(file.to_s)), file.to_s
+    when Proc === get
+      Open.write(file, get.call)
+    when TSV === get
+      Open.write(file, get.to_s)
+    when ((String === get or Symbol === get) and File.basename(get.to_s) == "Rakefile")
+      if Symbol === get
+        rakefile = File.join(sharedir, subdir, get.to_s)
+      else
+        rakefile = File.join(sharedir, get.to_s)
+      end
+      produce_with_rake(rakefile, subdir, file)
+    when String === get
+      Open.write(file, Open.read(get, :wget_options => {:pipe => true}, :nocache => true))
+    else
+      raise "Unknown Get: #{get.class}"
+    end
   end
 end

data/lib/rbbt/util/rake.rb

@@ -0,0 +1,78 @@
+require 'rbbt/util/tsv'
+require 'rbbt/util/open'
+require 'rbbt/util/log'
+
+module RakeHelper
+  def self.files(rakefile, task = :default, chdir = nil)
+    status = nil
+    files = nil
+    TmpFile.with_file do |f|
+      pid = Process.fork{
+        require 'rake'
+        FileUtils.chdir chdir if chdir
+
+        Rake::FileTask.module_eval do
+          class << self
+            alias_method :old_define_task, :define_task
+          end
+          def self.define_task(file, *args, &block)
+            @@files ||= []
+            @@files << file
+            old_define_task(file, *args, &block)
+          end
+
+          def self.files
+            @@files
+          end
+        end
+
+        load rakefile
+
+        Open.write(f, Rake::FileTask.files * "\n")
+        exit
+      }
+
+
+      pid, status = Process.waitpid2(pid)
+      files = Open.read(f).split("\n")
+    end
+    raise "Error getting files from Rake: #{ rakefile } " unless status.success?
+    files
+  end
+
+  def self.run(rakefile, task = :default, chdir = nil)
+    pid = Process.fork{
+      require 'rake'
+      FileUtils.chdir chdir if chdir
+
+      Rake::FileTask.module_eval do
+        class << self
+          alias_method :old_define_task, :define_task
+        end
+        def self.define_task(file, *args, &block)
+          @@files ||= []
+          @@files << file
+          old_define_task(file, *args, &block)
+        end
+
+        def self.files
+          @@files
+        end
+      end
+
+      load rakefile
+
+      task(:default) do |t|
+        Rake::FileTask.files.each do |file| Rake::Task[file].invoke end
+      end
+
+      Rake::Task[task].invoke
+      exit
+    }
+    pid, status = Process.waitpid2(pid)
+
+    raise "Error in Rake: #{ rakefile } => #{ task }" unless status.success?
+  end
+end
+
+