rbbt-util 1.2.1 → 2.0.1

data/lib/rbbt/util/open.rb

@@ -80,7 +80,7 @@ module Open
   # Cache
 
   def self.in_cache(url, options = {})
-    digest = Digest::MD5.hexdigest(url)
+    digest = Digest::MD5.hexdigest([url, options["--post-data"]].inspect)
 
     filename = File.join(REMOTE_CACHEDIR, digest)
     if File.exists? filename
@@ -91,7 +91,7 @@ module Open
   end
   
   def self.add_cache(url, data, options = {})
-    digest = Digest::MD5.hexdigest(url)
+    digest = Digest::MD5.hexdigest([url, options["--post-data"]].inspect)
     Misc.sensiblewrite(File.join(REMOTE_CACHEDIR, digest), data)
   end
 
@@ -163,19 +163,28 @@ module Open
          when (not remote?(url))
            file_open(url, options[:grep])
          when options[:nocache]
+           # What about grep?
            wget(url, wget_options)
-         when in_cache(url)
-           file_open(in_cache(url), options[:grep])
+         when in_cache(url, wget_options)
+           file_open(in_cache(url, wget_options), options[:grep])
          else
            io = wget(url, wget_options)
-           add_cache(url, io)
+           add_cache(url, io, wget_options)
            io.close
-           file_open(in_cache(url), options[:grep])
+           file_open(in_cache(url, wget_options), options[:grep])
          end
-    io = unzip(io)  if (zip?  url and not options[:noz]) or options[:zip]
-    io = gunzip(io) if (gzip? url and not options[:noz]) or options[:gzip]
+    io = unzip(io)  if (zip?(url)  and not options[:noz]) or options[:zip]
+    io = gunzip(io) if (gzip?(url) and not options[:noz]) or options[:gzip]
 
-    io
+    if block_given?
+      yield io 
+    else
+      io
+    end
+  end
+
+  def self.can_open?(file)
+    String === file and (File.exists?(file) or remote?(file))
   end
 
   def self.read(file, options = {}, &block)

data/lib/rbbt/util/path.rb

@@ -0,0 +1,152 @@
+require 'rbbt/util/misc'
+require 'rbbt/util/tsv'
+
+module Path
+  attr_accessor :pkg_module, :datadir
+
+  def self.find_files_back_to(path, target, subdir)
+    return [] if path.nil?
+    raise "Path #{ path } not in directory #{ subdir }" if not Misc.in_directory? path, subdir
+
+    pkg_module = path.pkg_module
+
+    files = []
+    while path != subdir
+      path = File.dirname(path)
+      path.extend Path
+      path.pkg_module = pkg_module
+      path.datadir    = path.datadir
+      if path[target].exists? 
+        files << path[target]
+      end
+    end
+
+    files
+  end
+
+  def self.path(string, datadir = nil, pkg_module = nil)
+    string.extend Path
+    string.datadir = datadir
+    string.pkg_module = case
+                        when pkg_module.nil?
+                          nil
+                        when String === pkg_module
+                          Misc.string2const pkg_module
+                        else
+                          pkg_module
+                        end
+    string
+  end
+
+  def method_missing(name, *args, &block)
+    new = File.join(self.dup, name.to_s)
+    new.extend Path
+    new.pkg_module = pkg_module
+    new.datadir    = datadir
+    new
+  end
+
+  def [](name)
+    new = File.join(self.dup, name.to_s)
+    new.extend Path
+    new.pkg_module = pkg_module
+    new.datadir    = datadir
+    new
+  end
+
+  def namespace
+    return nil if self.nil? or self.empty? or (not datadir.nil? and (self == datadir or File.dirname(self) == datadir))
+    if File.directory? self
+      File.basename(self)
+    else
+      File.basename(File.dirname(self))
+    end
+  end
+
+  def identifier_files
+    if datadir.nil?
+      path = File.join(File.dirname(self), 'identifiers')
+      path.extend Path
+      path.pkg_module = pkg_module
+      if path.exists?
+        [path]
+      else
+        []
+      end
+    else
+      identifier_files = Path.find_files_back_to(self, 'identifiers', datadir)
+      return identifier_files.collect{|f| Path.path(f, datadir, pkg_module)}
+    end
+  end
+
+  def tsv(key = nil, options = {})
+    if options.empty? and Hash === key
+      options, key = key, nil
+    end
+
+    produce
+    TSV.new self, key, options.merge(:datadir => datadir)
+  end
+
+  def index(options = {})
+    produce
+    TSV.index self, options
+  end
+
+  def open(options = {})
+    produce
+    Open.open(self, options)
+  end
+
+  def read(options = {})
+    produce
+    Open.read(self, options)
+  end
+
+  def fields(sep = nil, header_hash = nil)
+    produce
+    TSV.parse_header(self.open, sep, header_hash)[1].collect{|f| f.extend TSV::Field; f.namespace = namespace ;f}
+  end
+
+  def all_fields(sep = nil, header_hash = nil)
+    produce
+    key_field, fields = TSV.parse_header(self.open, sep, header_hash).values_at(0, 1).flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace; f}
+  end
+
+  def fields_in_namespace(sep = nil, header_hash = nil)
+    produce
+    TSV.parse_header(self.open, sep, header_hash)[1].collect{|f| f.extend TSV::Field; f.namespace = namespace ;f}.select{|f| f.namespace == namespace}
+  end
+
+  def all_namespace_fields(sep = nil, header_hash = nil)
+    produce
+    key_field, fields = TSV.parse_header(self.open, sep, header_hash).values_at(0, 1).flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace; f}.select{|f| f.namespace == namespace}
+  end
+
+
+  def filename
+    self.to_s
+  end 
+
+  def exists?
+    begin
+      produce
+      true
+    rescue
+      false
+    end
+  end
+
+  def produce
+    return if File.exists? self
+
+    Log.debug("Trying to produce '#{ self }'")
+    file, producer = pkg_module.reclaim self
+
+    raise "File #{self} has not been claimed, cannot produce" if file.nil? or producer.nil?
+
+    pkg_module.produce(self, producer[:get], producer[:subdir], producer[:sharedir])
+  end
+end
+
+

data/lib/rbbt/util/persistence.rb

@@ -1,3 +1,4 @@
+require 'rbbt/util/tsv'
 require 'rbbt/util/misc'
 require 'rbbt/util/open'
 require 'yaml'
@@ -19,91 +20,297 @@ module Persistence
   end
  
   def self.get_persistence_file(file, prefix, options = {})
-    File.join(CACHEDIR, prefix.to_s.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, options].inspect))
+    name = prefix.to_s << ":" << file.to_s << ":"
+    File.join(CACHEDIR, name.to_s.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, options].inspect))
   end
 
-  def self.persist(file, prefix = "", persistence_type = :string, options = {})
-    options = Misc.add_defaults options, :persistence => true
+  def self.get_filename(file)
+    case
+    when (String === file and File.exists? file)
+      File.expand_path file
+    when File === file
+      File.expand_path file.path
+    when Object::TSV === file
+      file.filename
+    when String === file
+      file
+    else
+      file.class.to_s
+    end
+  end
+
+  def self.persist_string(file, prefix = "", options = {})
+    options = 
+      Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
+    persistence_update, persistence_file, filename =
+      Misc.process_options options, :persistence_update, :persistence_file, :filename
+
+    filename         ||= get_filename(file)
+    persistence_file ||= get_persistence_file(filename, prefix, options)
+
+    if persistence_update or not File.exists? persistence_file
+      Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
+
+      res = yield file, options, filename, persistence_file
+      Open.write(persistence_file, res.to_s)
+      res
+    else
+      Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
+
+      Open.read(persistence_file)
+    end
+  end
+
+  def self.persist_marshal(file, prefix = "", options = {})
+    options = 
+      Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
+    persistence_update, persistence_file, filename =
+      Misc.process_options options, :persistence_update, :persistence_file, :filename
+
+    filename         ||= get_filename(file)
+    persistence_file ||= get_persistence_file(filename, prefix, options)
+
+    if persistence_update or not File.exists? persistence_file
+      Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
+      res = yield file, options
+      Open.write(persistence_file, Marshal.dump(res))
+      res
+    else
+      Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
+      Marshal.load(Open.open(persistence_file))
+    end
+  end
+
+  def self.persist_yaml(file, prefix = "", options = {})
+    options = 
+      Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
+    persistence_update, persistence_file, filename =
+      Misc.process_options options, :persistence_update, :persistence_file, :filename
+
+    filename         ||= get_filename(file)
+    persistence_file ||= get_persistence_file(filename, prefix, options)
+
+    if persistence_update or not File.exists? persistence_file
+      Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
+      res = yield file, options, filename, persistence_file
+      Open.write(persistence_file, YAML.dump(res))
+      res
+    else
+      Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
+      YAML.load(Open.open(persistence_file))
+    end
+  end
+
+  def self.persist_tsv_string(file, prefix = "", options = {})
+    options = 
+      Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
+    persistence_update, persistence_file, filename =
+      Misc.process_options options, :persistence_update, :persistence_file, :filename
+
+    filename         ||= get_filename(file)
+    persistence_file ||= get_persistence_file(filename, prefix, options)
+
+    if persistence_update or not File.exists? persistence_file
+      Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
+      res = yield file, options, filename, persistence_file
+      Open.write(persistence_file, res.to_s)
+      res
+    else
+      Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
+      tsv = Object::TSV.new persistence_file
+      tsv.filename = filename
+      tsv
+    end
+  end
+
+  def self.tsv_serializer(data)
+    case
+    when (not Object::TSV === data)
+      :marshal
+    when data.type == :double
+      :double
+    when data.type == :single
+      :single
+    else
+      :list
+    end
+  end
+
+  def self.persist_tsv(file, prefix = "", options = {})
+    options = 
+      Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
+    persistence_update, persistence_file, filename =
+      Misc.process_options options, :persistence_update, :persistence_file, :filename
 
-    persistence, persistence_file =
-      Misc.process_options options, :persistence, :persistence_file
-
-    filename = Misc.process_options options, :filename
-    filename ||= case
-                  when (String === file and File.exists? file)
-                    File.expand_path file
-                  when File === file
-                    File.expand_path file.path
-                  when TSV === file
-                    file.filename
-                  else
-                    Digest::MD5.hexdigest(file.inspect)
-                  end
-
-    if persistence
-      persistence_file ||= get_persistence_file(filename, prefix, options)
-      
-      #{{{ CREATE
-      if ! File.exists? persistence_file
-        Log.low "Creating Persistence #{ persistence_file } for #{ filename }"
-        res = yield file, options, filename, persistence_file
-        if Array === res and res.length == 2 and (Hash === res[1] or res[1].nil?)
-          data, extra = res
-        else
-          data, extra = [res, nil]
+    filename         ||= get_filename(file)
+    persistence_file ||= get_persistence_file(filename, prefix, options)
+
+    if persistence_update or not File.exists? persistence_file
+      Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
+
+      res = yield file, options, filename, persistence_file
+      serializer = tsv_serializer res
+
+      if File.exists? persistence_file
+        Log.debug "Erasing old #{ persistence_file }. Prefix = #{prefix}"
+        FileUtils.rm persistence_file
+      end
+
+      per = Persistence::TSV.get persistence_file, true, serializer
+
+      per.write
+      per.merge! res
+
+      Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
+        if res.respond_to?(key.to_sym)  and per.respond_to?("#{key}=".to_sym)
+          per.send "#{key}=".to_sym, res.send(key.to_sym) 
         end
-        ddd data.filename
-
-        case persistence_type.to_sym
-        when :tsv 
-          if Hash === data or Object::TSV === data
-            Log.debug "Creating #{Persistence::TSV} for #{ persistence_file }"
-            per = Persistence::TSV.get persistence_file
-            per.write
-            data.each{|k,v| per[k.to_s] = v}
-            %w(case_insensitive fields key_field type filename). each do |key| 
-              if data.respond_to? key
-                per.send "#{key}=".to_sym, data.send(key.to_sym) 
-              else
-                per.send "#{key}=".to_sym, extra[key.to_sym]
-              end
-            end
-            per.read
-
-            data = per
-          end
-        when :string
-          Open.write(persistence_file, data.to_s)
-        when :marshal
-          Open.write(persistence_file, Marshal.dump(data))
-        when :yaml
-          Open.write(persistence_file, YAML.dump(data))
+      end
+
+      per.read
+
+      tsv = Object::TSV.new per
+
+      tsv
+    else
+      Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
+
+      per = Persistence::TSV.get persistence_file, true, serializer
+      tsv = Object::TSV.new per
+      Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
+        if tsv.respond_to?(key.to_sym)  and per.respond_to?(key.to_sym)
+          tsv.send "#{key}=".to_sym, per.send(key.to_sym) 
         end
-        
-        return [data, extra]
+      end
 
-      #{{{ LOAD
-      else
-        Log.low "Opening Persistence #{ persistence_file } for #{ filename }"
-        case persistence_type.to_sym
-        when :tsv
-          data        = Persistence::TSV.get persistence_file
-          
-          extra = {}
-          %W(case_insensitive fields key_field type filename).each{|key| extra[key.to_sym] = data.send key.to_sym}
-
-          return [data, extra]
-        when :string
-          return [Open.read(persistence_type), nil]
-        when :marshal
-          return [File.open(persistence_file){|f| Marshal.load(f)}, nil]
-        when :yaml
-          return [File.open(persistence_file){|f| YAML.load(f)}, nil]
+      tsv
+    end
+  end
+
+  def self.persist_tsv_extra(file, prefix = "", options = {})
+    options = 
+      Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
+    persistence_update, persistence_file, filename =
+      Misc.process_options options, :persistence_update, :persistence_file, :filename
+
+    filename         ||= get_filename(file)
+    persistence_file ||= get_persistence_file(filename, prefix, options)
+
+    if persistence_update or not File.exists? persistence_file
+      Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
+      res, extra = yield file, options, filename, persistence_file
+      serializer = tsv_serializer res
+
+      per = Persistence::TSV.get persistence_file, true, serializer
+
+      per.write
+      per.merge! res
+      Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
+        if extra.include?(key.to_sym)  and per.respond_to?(key.to_sym)
+          per.send "#{key}=".to_sym, extra[key.to_sym]
         end
+      end
+      per.read
 
+     [ per, extra ]
+    else
+      Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
+      per = Persistence::TSV.get persistence_file, true, serializer
+
+      extra = {}
+      Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
+        if per.respond_to?(key.to_sym)
+          extra[key] = per.send(key.to_sym)
+        end
       end
+
+     [ per, extra ]
+    end
+  end
+
+  def self.persist_fwt(file, prefix = "", options = {})
+    options = 
+      Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
+    persistence_update, persistence_file, filename =
+      Misc.process_options options, :persistence_update, :persistence_file, :filename
+
+    filename         ||= get_filename(file)
+    persistence_file ||= get_persistence_file(filename, prefix, options)
+
+    if persistence_update or not File.exists? persistence_file
+      Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
+
+      range = options[:range]
+
+      res = yield file, options, filename, persistence_file
+
+      if File.exists? persistence_file
+        Log.debug "Erasing old #{ persistence_file }. Prefix = #{prefix}"
+        FileUtils.rm persistence_file
+      end
+
+
+      max_length = res.collect{|k,v| k.length}.max
+
+      if range
+        begin
+          fwt = FixWidthTable.new persistence_file, max_length, true
+          fwt.add_range res
+        rescue
+          FileUtils.rm persistence_file
+          raise $!
+        end
+      else
+        begin
+          fwt = FixWidthTable.new persistence_file, max_length, false
+          fwt.add_point res
+        rescue
+          FileUtils.rm persistence_file
+          raise $!
+        end
+      end
+
+      fwt.read
+
+      fwt
     else
-      yield file, options
+      Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
+
+      fwt = FixWidthTable.new persistence_file, nil, nil
+
+      fwt
+    end
+  end
+
+  def self.persist(file, prefix = "", persistence_type = :string, options = {}, &block)
+    options = Misc.add_defaults options, :persistence => true
+    persistence =
+      Misc.process_options options, :persistence
+
+    filename = get_filename(file)
+
+    if not persistence
+      Log.low "Non Persistent Loading for #{filename}. Prefix: #{prefix}"
+      yield file, options, filename
+    else
+      Log.low "Persistent Loading for #{filename}. Prefix: #{prefix}. Type #{persistence_type.to_s}"
+
+      case persistence_type.to_sym
+      when :string
+        persist_string(file, prefix, options, &block)
+      when :marshal
+        persist_marshal(file, prefix, options, &block)
+      when :yaml
+        persist_yaml(file, prefix, options, &block)
+      when :tsv
+        persist_tsv(file, prefix, options, &block)
+      when :tsv_string
+        persist_tsv_string(file, prefix, options, &block)
+      when :tsv_extra
+        persist_tsv_extra(file, prefix, options, &block)
+      when :fwt
+        persist_fwt(file, prefix, options, &block)
+      end
     end
   end
 end