rbbt-util 1.1.0 → 1.2.1

data/lib/rbbt/util/tc_hash.rb

@@ -6,7 +6,13 @@ class TCHash < TokyoCabinet::HDB
 
   Serializer = Marshal
 
-  FIELD_INFO_ENTRIES = {:fields => '__tokyocabinet_hash_fields', :key_field => '__tokyocabinet_hash_native_field'}
+  FIELD_INFO_ENTRIES = {
+    :fields    => '__tokyocabinet_hash_fields', 
+    :key_field => '__tokyocabinet_hash_key_field',
+    :filename  => '__tokyocabinet_hash_filename',
+    :type      => '__tokyocabinet_hash_type',
+    :case_insensitive      => '__tokyocabinet_hash_case_insensitive'
+  }
   CONNECTIONS = {}
 
   FIELD_INFO_ENTRIES.each do |entry, key|

data/lib/rbbt/util/tsv.rb

@@ -3,17 +3,10 @@ require 'rbbt/util/open'
 require 'rbbt/util/tc_hash'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/log'
+require 'rbbt/util/persistence'
 require 'digest'
 require 'fileutils'
 
-def add_defaults(options, defaults = {})
-  new_options = options.dup
-  defaults.each do |key, value|
-    new_options[key] = value if new_options[key].nil?
-  end
-  new_options
-end
-
 class TSV
   class FieldNotFoundError < StandardError;end
 
@@ -26,8 +19,6 @@ class TSV
 
   #{{{ Persistence
 
-  PersistenceHash = TCHash
-
   CACHEDIR="/tmp/tsv_persistent_cache"
   FileUtils.mkdir CACHEDIR unless File.exist? CACHEDIR
 
@@ -40,10 +31,7 @@ class TSV
     CACHEDIR
   end
 
-  def self.get_persistence_file(file, prefix, options = {})
-    File.join(CACHEDIR, prefix.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, options].inspect))
-  end
-
+  
   #{{{ Headers and Field Stuff
 
   def self.headers(file, options = {})
@@ -130,7 +118,7 @@ class TSV
       end
 
       each do |key, values|
-        if list
+        if type == :double
           tmp_values = values + [[key]]
         else
           tmp_values = values + [key]
@@ -144,11 +132,15 @@ class TSV
           new_values = tmp_values.values_at(*new_field_positions)
         end
 
-        tmp_values[new_key_position].each do |new_key|
-          if new_field_names
-            yield new_key, NamedArray.name(new_values, new_field_names)
-          else
-            yield new_key, new_values
+        if not Array === tmp_values[new_key_position]
+          yield tmp_values[new_key_position], NamedArray.name(new_values, new_field_names)  
+        else
+          tmp_values[new_key_position].each do |new_key|
+            if new_field_names
+              yield new_key, NamedArray.name(new_values, new_field_names)
+            else
+              yield new_key, new_values
+            end
           end
         end
       end
@@ -165,7 +157,7 @@ class TSV
 
   def reorder(new_key_field, new_fields = nil, options = {})
     options = Misc.add_defaults options
-    return TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => case_insensitive)  if options[:persistence_file] and File.exists?(options[:persistence_file])
+    return TSV.new(Persistence::TSV.get(options[:persistence_file], false), :case_insensitive => case_insensitive)  if options[:persistence_file] and File.exists?(options[:persistence_file])
 
     new = {}
     new_key_field, new_fields = through new_key_field, new_fields do |key, values|
@@ -181,7 +173,7 @@ class TSV
     end
 
     if options[:persistence_file]
-      reordered = TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => case_insensitive)
+      reordered = TSV.new(Persistence::TSV.get(options[:persistence_file], false), :case_insensitive => case_insensitive)
       reordered.merge! new
     else
       reordered = TSV.new(new, :case_insensitive => case_insensitive)
@@ -199,12 +191,12 @@ class TSV
 
   def add_field(name = nil)
     each do |key, values|
-      self[key] = values << yield(key, values)
+      self[key] = values + [yield(key, values)]
     end
 
-    fields << name if list
-    if PersistenceHash === @data
-      @data.fields = fields
+    if fields != nil
+      new_fields = fields + [name]
+      self.fields = new_fields
     end
   end
 
@@ -212,6 +204,9 @@ class TSV
     new = TSV.new({})
     new.key_field = key_field
     new.fields    = fields.dup
+    new.type      = type
+    new.filename  = filename + "#Select: #{method.inspect}"
+    new.case_insensitive  = case_insensitive
     
     case
     when Array === method
@@ -222,6 +217,10 @@ class TSV
       through do |key, values|
         new[key] = values if [key,values].flatten.select{|v| v =~ method}.any?
       end
+    when String === method
+      through do |key, values|
+        new[key] = values if [key,values].flatten.select{|v| v == method}.any?
+      end
     when Hash === method
       key  = method.keys.first
       method = method.values.first
@@ -230,89 +229,87 @@ class TSV
         method.each{|item| if values = self[item]; then  new[item] = values; end}
       when Array === method
         through :main, key do |key, values|
-          new[key] = values if (values.flatten & method).any?
+          new[key] = self[key] if (values.flatten & method).any?
         end
       when Regexp === method
         through :main, key do |key, values|
-          new[key] = values if values.flatten.select{|v| v =~ method}.any?
+          new[key] = self[key] if values.flatten.select{|v| v =~ method}.any?
+        end
+      when String === method
+        through :main, key do |key, values|
+          new[key] = self[key] if values.flatten.select{|v| v == method}.any?
         end
       end
     end
 
+
     new
   end
 
   def index(options = {})
-    options = Misc.add_defaults options, :order => false
+    options = Misc.add_defaults options, :order => false, :persistence => false
 
-    if options[:persistence] and ! options[:persistence_file]
-      options[:persistence_file] = TSV.get_persistence_file(filename, "index:#{ filename }_#{options[:field]}:", options)
-    end
+    new, extra = Persistence.persist(filename, :Index, :tsv, options) do |filename, options|
+      new = {}
+      if options[:order]
+        new_key_field, new_fields = through options[:target], options[:others] do |key, values|
 
-    if options[:persistence_file] and File.exists?(options[:persistence_file])
-      return TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => options[:case_insensitive]) 
-    end
+          values.each_with_index do |list, i|
+            next if list.nil? or list.empty?
 
-    new = {}
-    if options[:order]
-      new_key_field, new_fields = through options[:field], options[:others] do |key, values|
-
-        values.each_with_index do |list, i|
-          next if list.nil? or list.empty?
-
-          list = [list] unless Array === list
+            list = [list] unless Array === list
 
-          list.each do |value|
-            next if value.nil? or value.empty?
-            value = value.downcase if options[:case_insensitive]
-            new[value]    ||= []
-            new[value][i + 1] ||= []
-            new[value][i + 1] << key
-          end
+            list.each do |value|
+              next if value.nil? or value.empty?
+              value = value.downcase if options[:case_insensitive]
+              new[value]    ||= []
+              new[value][i + 1] ||= []
+              new[value][i + 1] << key
+            end
           new[key]    ||= []
           new[key][0] = key
-        end
+          end
 
-      end
+        end
 
-      new.each do |key, values| 
-        values.flatten!
-        values.compact!
-      end
+        new.each do |key, values| 
+          values.flatten!
+          values.compact!
+        end
 
-    else
-      new_key_field, new_fields = through options[:field], options[:others] do |key, values|
-        new[key] ||= []
-        new[key] << key 
-        values.each do |list|
-          next if list.nil? 
-          if Array === list
-            list.each do |value|
+      else
+        new_key_field, new_fields = through options[:target], options[:others] do |key, values|
+          new[key] ||= []
+          new[key] << key 
+          values.each do |list|
+            next if list.nil? 
+            if Array === list
+              list.each do |value|
+                value = value.downcase if options[:case_insensitive]
+                new[value] ||= []
+                new[value] << key
+              end
+            else
+              next if list.empty?
+              value = list
               value = value.downcase if options[:case_insensitive]
               new[value] ||= []
               new[value] << key
             end
-          else
-            next if list.empty?
-            value = list
-            value = value.downcase if options[:case_insensitive]
-            new[value] ||= []
-            new[value] << key
           end
         end
       end
-    end
 
-    if options[:persistence_file]
-      index = TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => options[:case_insensitive])
-      index.merge! new
-    else
-      index = TSV.new(new, :case_insensitive => options[:case_insensitive])
+      [new, {:key_field => new_key_field, :fields => new_fields, :type => :double, :case_insensitive => options[:case_insensitive]}]
     end
 
-    index.key_field = new_key_field
-    index.fields    = new_fields
-    index
+    new = TSV.new(new)
+    new.filename = "Index: " + filename + options.inspect
+    new.fields = extra[:fields]
+    new.key_field = extra[:key_field]
+    new.case_insensitive = extra[:case_insensitive]
+    new.type = extra[:type]
+    new
   end
 
   def smart_merge(other, match = nil, new_fields = nil)
@@ -413,7 +410,7 @@ class TSV
 
         if nofieldinfo
           next if other[other_code].nil?
-          if list
+          if type == :double
             other_values = [[other_code]] + other[other_code]
           else
             other_values = [other_code] + other[other_code]
@@ -427,13 +424,13 @@ class TSV
           new_values = values + other_values 
         else
           if other[other_code].nil?
-            if list
+            if type == :double
               other_values = [[]] * other.fields.length
             else
               other_values = [] * other.fields.length
             end
           else
-            if list
+            if type == :double
               other_values = other[other_code] + [[other_code]]
             else
               other_values = other[other_code] + [other_code]
@@ -443,11 +440,11 @@ class TSV
 
           new_values = values.dup
 
-          if list
+          if type == :double
             this_common_field_positions.zip(other_common_field_positions).each do |tpos, opos|
               new_values_tops = new_values[tpos]
 
-              if other.list
+              if other.type == :double
                 new_values_tops += other_values[opos]
               else
                 new_values_tops += [other_values[opos]]
@@ -466,10 +463,58 @@ class TSV
 
     self.fields = self.fields + new_fields unless nofieldinfo
   end
+  
+
+  def self.field_matches(tsv, values)
+    if values.flatten.sort[0..9].compact.collect{|n| n.to_i} == (1..10).to_a
+      return {}
+    end
+
+    key_field = tsv.key_field
+    fields    = tsv.fields
+
+    field_values = {} 
+    fields.each{|field| 
+      field_values[field] = []
+    }
+
+    tsv.through do |key,entry_values|
+      fields.zip(entry_values).each do |field,entry_field_values|
+        field_values[field].concat entry_field_values
+      end
+    end
+
+    field_values.each do |field,field_value_list|
+      field_value_list.replace(values & field_value_list.flatten.uniq)
+    end
+
+    field_values[key_field] = values & tsv.keys 
+
+    field_values
+  end
+
+  def field_matches(values)
+    TSV.field_matches(self, values)
+  end
+
+
 
   #{{{ Helpers
 
   def self.index(file, options = {})
+    options = Misc.add_defaults options, :data_persistence => true, :persistence => true
+    persistence, persistence_file = Misc.process_options options, :persistence, :persistence_file
+    options[:persistence], options[:persistence_file] =  options.values_at :data_persistence, :data_persistence_file
+    options.delete :data_persistence
+    options.delete :data_persistence_file
+
+    index, extra = Persistence.persist(file, :Index, :tsv, options) do |file, options, filename|
+      TSV.new(file, :double, options).index
+    end
+    index
+  end
+
+  def self.index2(file, options = {})
     opt_data = options.dup
     opt_index = options.dup
     opt_data.delete  :field
@@ -482,7 +527,7 @@ class TSV
 
     if ! opt_index[:persistence_file].nil? && File.exists?(opt_index[:persistence_file])
       Log.low "Reloading persistent index for #{ file }: #{opt_index[:persistence_file]}"
-      TSV.new(PersistenceHash.get(opt_index[:persistence_file], false), opt_index)
+      TSV.new(Persistence::TSV.get(opt_index[:persistence_file], false), opt_index)
     else
       Log.low "Creating index for #{ file }: #{opt_index[:persistence_file]}"
       data = TSV.new(file, opt_data)
@@ -501,6 +546,23 @@ class TSV
   end
 
   #{{{ Accesor Methods
+  attr_accessor :filename, :type, :case_insensitive, :key_field, :fields, :data
+
+  def fields
+    return nil if @fields.nil?
+    fields = @fields
+    fields.each do |f| f.extend Field end if Array === fields
+    fields
+  end
+
+  def fields=(new_fields)
+    @fields = new_fields
+    if Persistence::TSV === @data
+      @data.fields = new_fields
+    end
+  end
+
+
 
   def keys
     @data.keys
@@ -531,6 +593,7 @@ class TSV
   # Read
 
   def follow(value)
+    return nil if value.nil?
     if String === value && value =~ /__Ref:(.*)/
       return self[$1]
     else
@@ -546,7 +609,7 @@ class TSV
       return nil
     end
 
-    key = key.downcase if @case_insensitive
+    key = key.downcase if @case_insensitive and key !~ /^__Ref:/
     follow @data[key]
   end
 
@@ -587,23 +650,37 @@ class TSV
     collect.sort_by &block
   end
 
-  def to_s
+  def values_to_s(values)
+      case
+      when (values.nil? and fields.nil?)
+        "\n"
+      when (values.nil? and not fields.nil?)
+        "\t" << ([""] * fields.length) * "\t" << "\n"
+      when (not Array === values)
+        "\t" << values.to_s << "\n"
+      when Array === values.first
+        "\t" << values.collect{|list| (list || []) * "|"} * "\t" << "\n"
+      else
+        "\t" << values * "\t" << "\n"
+      end
+  end
+
+  def to_s(keys = nil)
     str = ""
 
     if fields
       str << "#" << key_field << "\t" << fields * "\t" << "\n"
     end
 
-    each do |key, values|
-      case
-      when values.nil?
-        str << key.dup << "\n"
-      when (not Array === values)
-        str << key.dup << "\t" << values.to_s << "\n"
-      when Array === values.first
-        str << key.dup <<  "\t" << values.collect{|list| (list || []) * "|"} * "\t" << "\n"
-      else
-        str << key.dup <<  "\t" << values * "\t" << "\n"
+    if keys.nil?
+      each do |key, values|
+        key = key.to_s if Symbol === key
+        str << key.dup << values_to_s(values)
+      end
+    else
+      keys.zip(values_at(*keys)).each do |key, values|
+        key = key.to_s if Symbol === key
+        str << key.dup << values_to_s(values)
       end
     end
 
@@ -625,11 +702,301 @@ class TSV
     zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields 
     zipped 
   end
-  
-  def self.parse(data, file, options = {})
 
+  def self.key_order(file, options = {})
     # Prepare options
     options = add_defaults options, 
+      :sep              => "\t",
+      :sep2             => "|",
+      :native           => 0,
+      :fix              => nil,
+      :exclude          => nil,
+      :select           => nil,
+      :grep             => nil,
+      :case_insensitive => false,
+      :header_hash      => '#'
+
+    options[:extra]   = [options[:extra]] if options[:extra] != nil && ! (Array === options[:extra])
+
+    if String === file and File.exists? file
+      file = File.open(file)
+    end
+
+    #{{{ Process first line
+
+    line = file.gets
+    raise "Empty content" if line.nil?
+    line.chomp!
+
+    if line =~ /^#{options[:header_hash]}/
+      header_fields    = parse_fields(line, options[:sep])
+      header_fields[0] = header_fields[0][(0 + options[:header_hash].length)..-1] # Remove initial hash character
+      line = file.gets
+    else
+      header_fields = nil
+    end
+    
+    id_pos = Misc.field_position(header_fields, options[:native])
+
+    if options[:extra].nil?
+      extra_pos = nil
+      max_cols = 0
+    else
+      extra_pos = options[:extra].collect{|pos| Misc.field_position(header_fields, pos) }
+    end
+
+    ids = []
+    #{{{ Process rest
+    while line do
+      line.chomp!
+
+      line = options[:fix].call line if options[:fix]
+      break if not line
+
+      # Select and fix lines
+      if line.empty?                                               or
+         (options[:exclude] and     options[:exclude].call(line))  or
+         (options[:select]  and not options[:select].call(line))
+
+         line = file.gets
+         next
+      end
+
+      ### Process line
+
+      # Chunk fields
+      parts = parse_fields(line, options[:sep])
+
+      # Get next line
+      line = file.gets
+
+      # Get id field
+      next if parts[id_pos].nil? || parts[id_pos].empty?
+      ids << parts[id_pos]
+    end
+
+    ids
+  end
+
+  def self.parse_header(stream, sep, header_hash)
+    fields, key_field = nil
+    options = {}
+    
+    line = stream.gets
+
+    if line and line =~ /^#{header_hash}: (.*)/
+      options = Misc.string2hash $1
+      line = stream.gets
+    end
+
+    sep = options[:sep] if options[:sep]
+
+    if line and line =~ /^#{header_hash}/
+      line.chomp!
+      fields = parse_fields(line, sep)
+      key_field = fields.shift
+      key_field = key_field[(0 + header_hash.length)..-1] # Remove initial hash character
+      line = stream.gets
+    end
+
+    raise "Empty content" if line.nil?
+    return key_field, fields, options, line
+  end
+
+  def self.parse(stream, options = {})
+    # Prepare options
+    options = Misc.add_defaults options, 
+      :case_insensitive => false,
+      :type             => :double,
+
+      :merge            => false,
+      :keep_empty       => true,
+      :cast             => nil,
+
+      :sep              => "\t",
+      :sep2             => "|",
+      :header_hash      => '#',
+
+      :key              => 0,
+      :fields           => nil,
+
+      :fix              => nil,
+      :exclude          => nil,
+      :select           => nil,
+      :grep             => nil
+
+
+    sep, header_hash =
+      Misc.process_options options, :sep,  :header_hash
+
+    key_field, other_fields, more_options, line = TSV.parse_header(stream, sep, header_hash)
+
+    sep     = more_options[:sep] if more_options[:sep]
+    options = Misc.add_defaults options, more_options
+    sep2    = Misc.process_options options, :sep2
+
+    key, others =
+      Misc.process_options options, :key, :others
+
+    if key_field.nil?
+      key_pos      = key
+      key_field, fields = nil
+    else
+      all_fields = [key_field].concat other_fields
+
+      key_pos   = Misc.field_position(all_fields, key)
+
+      if String === others or Symbol === others
+        others = [others]
+      end
+
+      if others.nil?
+        other_pos    = (0..(all_fields.length - 1)).to_a
+        other_pos.delete key_pos
+      else
+        other_pos = Misc.field_position(all_fields, *others)
+      end
+
+      key_field = all_fields[key_pos]
+      fields    = all_fields.values_at *other_pos
+    end
+
+    case_insensitive, type, merge, keep_empty, cast = 
+      Misc.process_options options, :case_insensitive, :type, :merge, :keep_empty, :cast
+    fix, exclude, select, grep = 
+      Misc.process_options options, :fix, :exclude, :select, :grep 
+    
+    #{{{ Process rest
+    data = {}
+    single = type.to_sym != :double
+    max_cols = 0
+    while line do
+      line.chomp!
+
+      line = fix.call line if fix
+      break if not line
+
+      if header_hash and line =~ /^#{header_hash}/
+        line = stream.gets
+        next
+      end
+
+      if line.empty?                           or
+         (exclude and     exclude.call(line))  or
+         (select  and not select.call(line))
+
+         line = stream.gets
+         next
+      end
+
+      # Chunk fields
+      parts = parse_fields(line, sep)
+
+      # Get next line
+      line = stream.gets
+
+      # Get id field
+      next if parts[key_pos].nil? || parts[key_pos].empty?
+     
+      if single
+        ids = parse_fields(parts[key_pos], sep2)
+        ids.collect!{|id| id.downcase} if case_insensitive
+        
+        id = ids.shift
+        ids.each do |id2| data[id2] = "__Ref:#{id}"  end
+
+        if key_field.nil?
+          other_pos    = (0..(parts.length - 1)).to_a
+          other_pos.delete key_pos
+        end
+
+        extra = parts.values_at(*other_pos).collect{|f| parse_fields(f, sep2).first}
+        extra.collect! do |elem| 
+          case
+          when String === cast
+            elem.send(cast)
+          when Proc === cast
+            cast.call elem
+          end
+        end if cast
+
+        max_cols = extra.size if extra.size > (max_cols || 0)
+        case type
+        when :list
+          data[id] = extra unless data.include? id
+        when :flat
+          data[id] = extra.flatten unless data.include? id  
+        when :single
+          data[id] = extra.flatten.first unless data.include? id  
+        end
+ 
+      else
+        ids = parse_fields(parts[key_pos], sep2)
+        ids.collect!{|id| id.downcase} if case_insensitive
+
+        id = ids.shift
+        ids.each do |id2| data[id2] = "__Ref:#{id}"  end
+
+        if key_field.nil?
+          other_pos    = (0..(parts.length - 1)).to_a
+          other_pos.delete key_pos
+        end
+
+        extra = parts.values_at(*other_pos).collect{|f| parse_fields(f, sep2)}
+        extra.collect! do |list| 
+          case
+          when String === cast
+            list.collect{|elem| elem.send(cast)}
+          when Proc === cast
+            list.collect{|elem| cast.call elem}
+          end
+        end if cast
+
+        max_cols = extra.size if extra.size > (max_cols || 0)
+        if merge
+          data[id] = extra unless data.include? id
+        else
+          if not data.include? id
+            data[id] = extra
+          else
+            entry = data[id]
+            while entry =~ /__Ref:(.*)/ do entry = data[$1] end
+            extra.each_with_index do |f, i|
+              if f.empty?
+                next unless keep_empty
+                f= [""]
+              end
+              entry[i] ||= []
+              entry[i] = entry[i].concat f
+            end
+            data[id] = entry
+          end
+        end
+      end
+    end
+
+    if keep_empty and max_cols > 0
+      data.each do |key, values| 
+        next if values =~ /__Ref:/
+        new_values = values
+        max_cols.times do |i|
+          if type == :double
+            new_values[i] = [""] if new_values[i].nil? or new_values[i].empty?
+          else
+            new_values[i] = "" if new_values[i].nil?
+          end
+        end
+        data[key] = new_values
+      end
+    end
+
+    [data, {:key_field => key_field, :fields => fields, :type => type, :case_insensitive => case_insensitive}]
+  end
+ 
+  def self.parse2(data, file, options = {})
+
+    # Prepare options
+    options = Misc.add_defaults options, 
       :sep              => "\t",
       :sep2             => "|",
       :native           => 0,
@@ -640,18 +1007,19 @@ class TSV
       :grep             => nil,
       :single           => false,
       :unique           => false,
+      :merge            => false,
       :flatten          => false,
-      :overwrite        => false,
       :keep_empty       => true,
       :case_insensitive => false,
       :header_hash      => '#' ,
+      :cast             => nil,
       :persistence_file => nil
+       
 
+    options[:unique]  = options[:uniq] if options[:unique].nil?
     options[:extra]   = [options[:extra]] if options[:extra] != nil && ! (Array === options[:extra])
     options[:flatten] = true if options[:single]
 
-
-
     #{{{ Process first line
 
     line = file.gets
@@ -680,10 +1048,18 @@ class TSV
       line.chomp!
 
       line = options[:fix].call line if options[:fix]
+      break if not line
+
+      if options[:header_hash] && line =~ /^#{options[:header_hash]}/
+        line = file.gets
+        next
+      end
 
       # Select and fix lines
-      if (options[:exclude] and   options[:exclude].call(line)) or
+      if line.empty?                                               or
+         (options[:exclude] and     options[:exclude].call(line))  or
          (options[:select]  and not options[:select].call(line))
+
          line = file.gets
          next
       end
@@ -721,35 +1097,48 @@ class TSV
       extra.flatten!                              if options[:flatten]
       extra = extra.first                         if options[:single]
 
-      if options[:overwrite]
-        main_entry = ids.shift
-        ids.each do |id|
-          data[id] = "__Ref:#{main_entry}"  
+      if options[:cast]
+        if Array === extra[0]
+          e = extra
+        else
+          e = [extra]
         end
 
-        data[main_entry] = extra
-      else
-        main_entry = ids.shift
-        ids.each do |id|
-          data[id] = "__Ref:#{main_entry}"
+        e.each do |list|
+          case
+          when String === options[:cast]
+            list.collect!{|elem| elem.send(options[:cast])}
+          when Proc === options[:cast]
+            list.collect!{|elem| options[:cast].call elem}
+          end
         end
+      end
 
-        case
-        when (options[:single] or options[:unique])
-          data[main_entry] ||= extra
-        when options[:flatten]
-          if PersistenceHash === data
-            data[main_entry] = (data[main_entry] || []).concat extra
+      main_entry = ids.shift
+      ids.each do |id| data[id] = "__Ref:#{main_entry}"  end
+
+      case
+      when (options[:single] or options[:unique] or not options[:merge])
+        data[main_entry] = extra unless data.include? main_entry
+      when options[:flatten]
+        entry = data[main_entry]
+
+        if entry.nil?
+          data[main_entry] = extra
+        else
+          while entry =~ /__Ref:(.*)/ do entry = data[$1] end
+          if Persistence::TSV === data
+            data[main_entry] = entry.concat extra
           else
-            data[main_entry] ||= []
             data[main_entry].concat extra
           end
+        end
+      else
+        entry = data[main_entry]
+        if entry.nil?
+          data[main_entry] = extra
         else
-          entry = data[main_entry] || []
-          while entry =~ /__Ref:(.*)/ do
-            entry = data[$1]
-          end
-
+          while entry =~ /__Ref:(.*)/ do entry = data[$1] end
           extra.each_with_index do |fields, i|
             if fields.empty?
               next unless options[:keep_empty]
@@ -758,7 +1147,6 @@ class TSV
             entry[i] ||= []
             entry[i] = entry[i].concat fields
           end
-
           data[main_entry] = entry
         end
       end
@@ -774,7 +1162,6 @@ class TSV
       end
     end
 
-
     # Save header information
     key_field = nil
     fields   = nil
@@ -788,19 +1175,61 @@ class TSV
       end
     end
 
-    data.read if PersistenceHash === data
+    data.read if Persistence::TSV === data
 
     [key_field, fields]
   end
+  def initialize(file = {}, type = :double, options = {})
+    if Hash === type
+      options = type 
+      type    = :double 
+    end
 
-  attr_accessor :data, :key_field, :fields, :list, :case_insensitive, :filename
-  def fields
-    fields = @fields
-    fields.each do |f| f.extend Field end if Array === fields
-    fields
+    if String === file and file =~/(.*?)#(.*)/ and File.exists? $1
+      options = Misc.add_defaults options, Misc.string2hash($2) 
+      file = $1
+    end
+
+    options = Misc.add_defaults options, :persistence => false, :case_insensitive => false, :type => type
+
+    @filename = Misc.process_options options, :filename
+    @filename ||= case
+                  when (String === file and File.exists? file)
+                    File.expand_path file
+                  when File === file
+                    File.expand_path file.path
+                  else
+                    Digest::MD5.hexdigest(file.inspect)
+                  end
+
+    if block_given?
+      @data, extra = Persistence.persist(@filename, :TSV, :tsv, options) do |filename, options| yield filename, options end
+    else
+      @data, extra = Persistence.persist(@filename, :TSV, :tsv, options) do |filename, options|
+        data, extra = nil
+        case
+        when String === file
+          File.open(file) do |f|
+            data, extra = TSV.parse(f, options)
+          end
+        when File === file
+          data, extra = TSV.parse(file, options)
+        when Hash === file
+          data = file
+          extra = {:case_insensitive => options[:case_insensitive], :type => type}
+        end
+
+        [data, extra]
+      end
+    end
+
+    @type             = extra[:type]
+    @key_field        = extra[:key_field]
+    @fields           = extra[:fields]
+    @case_insensitive = extra[:case_insensitive]
   end
 
-  def initialize(file = {}, options = {})
+  def initialize2(file = {}, options = {})
     options = Misc.add_defaults options
     options[:persistence] = true if options[:persistence_file]
 
@@ -817,7 +1246,7 @@ class TSV
       Log.low "Copying TSV"
       @filename         = file.filename
 
-      if options[:persistence] and not PersistenceHash === file.data
+      if options[:persistence] and not Persistence::TSV === file.data
         persistence_file = options.delete(:persistence_file) || TSV.get_persistence_file(@filename, "file:#{ @filename }:", options)
         Log.low "Making persistance #{ persistence_file }"
         @data = TCHash.get(persistence_file)
@@ -834,7 +1263,7 @@ class TSV
       @list             = file.list
       return self
     when Hash === file
-      Log.low "Encapsulating Hash"
+      Log.low "Encapsulating Hash in TSV object"
       @filename = "Hash:" + Digest::MD5.hexdigest(file.inspect)
       if options[:persistence] 
         persistence_file = options.delete(:persistence_file) || TSV.get_persistence_file(@filename, "file:#{ @filename }:", options)
@@ -845,9 +1274,9 @@ class TSV
         @data = file
       end
       return self
-    when PersistenceHash === file
-      Log.low "Encapsulating PersistenceHash"
-      @filename = "PersistenceHash:" + Digest::MD5.hexdigest(file.inspect)
+    when Persistence::TSV === file
+      Log.low "Encapsulating Persistence::TSV"
+      @filename = "Persistence::TSV:" + Digest::MD5.hexdigest(file.inspect)
       @data             = file
       @key_field        = file.key_field
       @fields           = file.fields
@@ -860,7 +1289,7 @@ class TSV
     when StringIO
     else 
       raise "File #{file} not found"
-   end
+    end
 
     if options[:persistence]
       options.delete :persistence
@@ -868,11 +1297,11 @@ class TSV
 
       if File.exists? persistence_file
         Log.low "Loading Persistence for #{ @filename } in #{persistence_file}"
-        @data      = PersistenceHash.get(persistence_file, false)
+        @data      = Persistence::TSV.get(persistence_file, false)
         @key_field = @data.key_field
         @fields    = @data.fields
       else
-        @data = PersistenceHash.get(persistence_file, true)
+        @data = Persistence::TSV.get(persistence_file, true)
         file = Open.grep(file, options[:grep]) if options[:grep]
 
         Log.low "Persistent Parsing for #{ @filename } in #{persistence_file}"
@@ -897,17 +1326,17 @@ end
 #{{{ CacheHelper
 require 'rbbt/util/cachehelper'
 module CacheHelper
-   def self.tsv_cache(name, key = [])
-     cache_file = CacheHelper.build_filename name, key
-
-     if File.exists? cache_file
-       Log.debug "TSV cache file '#{cache_file}' found"
-       hash = TCHash.get(cache_file)
-       TSV.new(hash)
-     else
-       Log.debug "Producing TSV cache file '#{cache_file}'"
-       data = yield
-       TSV.new(data, :persistence_file => cache_file)
-     end
+  def self.tsv_cache(name, key = [])
+    cache_file = CacheHelper.build_filename name, key
+
+    if File.exists? cache_file
+      Log.debug "TSV cache file '#{cache_file}' found"
+      hash = TCHash.get(cache_file)
+      TSV.new(hash)
+    else
+      Log.debug "Producing TSV cache file '#{cache_file}'"
+      data = yield
+      TSV.new(data,  :persistence_file => cache_file)
+    end
   end
 end