rbbt-util 1.0.1 → 1.1.0

data/lib/rbbt/util/cmd.rb

@@ -3,20 +3,27 @@ require 'rbbt/util/log'
 require 'stringio'
 
 module CMD
-  class CMDError < StandardError;end
+  class CMDError < RBBTError;end
 
   module SmartIO 
-    def self.tie(io, pid = nil, post = nil)
+    def self.tie(io, pid = nil, cmd = "",  post = nil)
       io.instance_eval{
         @pid  = pid
+        @cmd  = cmd
         @post = post
         alias original_close close
         def close
           begin
-            Process.waitpid(@pid, Process::WNOHANG) if @pid
+            Process.waitpid(@pid) if @pid
           rescue
           end
 
+          if $? and not $?.success?
+            Log.debug "Raising exception"
+            exception      = CMDError.new "Command [#{@pid}] #{@cmd} failed with error status #{$?.exitstatus}"
+            raise exception
+          end
+
           @post.call if @post
           original_close
         end
@@ -107,6 +114,8 @@ module CMD
     sout.last.close
     serr.last.close
 
+    Log.debug "CMD: [#{pid}] #{cmd}"
+
     case 
     when String === in_content
       sin.last.write in_content
@@ -120,20 +129,39 @@ module CMD
       end
     end
 
-    Thread.new do
-      while l = serr.first.gets
-        Log.log l, stderr if Integer === stderr
+    if pipe
+      Thread.new do
+        while l = serr.first.gets
+          Log.log l, stderr if Integer === stderr
+        end
+        serr.first.close
       end
-      serr.first.close
-    end
 
-    if pipe
-      SmartIO.tie sout.first, pid, post
+      SmartIO.tie sout.first, pid, cmd, post
       sout.first
+
     else
+      err = ""
+      Thread.new do
+        while l = serr.first.gets
+          err << l if Integer === stderr
+        end
+        serr.first.close
+      end
+
       out = StringIO.new sout.first.read
-      SmartIO.tie out
+      SmartIO.tie out, pid, cmd, post
+
       Process.waitpid pid
+
+      if not $?.success?
+        exception      = CMDError.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}"
+        exception.info = err if Integer === stderr and stderr >= Log.severity
+        raise exception
+      else
+        Log.log err, stderr if Integer === stderr
+      end
+
       out
     end
   end

data/lib/rbbt/util/data_module.rb

@@ -38,6 +38,7 @@ module DataModule
       pkg_module.add_datafiles filename => ['', self.to_s, sharedir]
     rescue 
       Log.debug $!.message
+      Log.debug $!.backtrace * "\n"
       old_method_missing name, *args, &block
     end
 

data/lib/rbbt/util/excel2tsv.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/tsv'
+require 'rbbt/util/open'
 require 'spreadsheet'
 
 class TSV
@@ -8,7 +9,7 @@ class TSV
     header = true unless header == false
     sheet ||= 0
     TmpFile.with_file do |filename|
-      workbook = Spreadsheet.open File.open(file)
+      workbook = Spreadsheet.open Open.open(file)
       sheet    = workbook.worksheet sheet
 
       rows = []

data/lib/rbbt/util/log.rb

@@ -5,11 +5,11 @@ module Log
   MEDIUM = 2
   HIGH   = 3
 
-  def severity=(severity)
+  def self.severity=(severity)
     @@severity = severity
   end
 
-  def severity
+  def self.severity
     @@severity
   end
 

data/lib/rbbt/util/misc.rb

@@ -1,7 +1,25 @@
 require 'iconv'
+
+class RBBTError < StandardError
+  attr_accessor :info
+
+  alias old_to_s to_s
+  def to_s
+    str = old_to_s
+    if info
+      str << "\n" << "Additional Info:\n---\n" << info << "---"
+    end
+    str
+  end
+end
+
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  def self.this_dir
+    File.expand_path(File.dirname(caller[0]))
+  end
+
   def self.env_add(var, value, sep = ":", prepend = true)
     ENV[var] ||= ""
     return if ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
@@ -113,6 +131,14 @@ module Misc
   end
 end
 
+module PDF2Text
+  def self.pdf2text(filename)
+    TmpFile.with_file(Open.read(filename)) do |pdf|
+      CMD.cmd("pdftotext #{pdf} -", :pipe => false, :stderr => true)
+    end
+  end
+end
+
 class NamedArray < Array
   attr_accessor :fields
 

data/lib/rbbt/util/open.rb

@@ -34,6 +34,7 @@ module Open
   end
 
   def self.wget(url, options = {})
+    Log.low "WGET:\n -URL: #{ url }\n -OPTIONS: #{options.inspect}"
     options = Misc.add_defaults options, "--user-agent=" => 'firefox', :pipe => true
 
     wait(options[:nice], options[:nice_key]) if options[:nice]
@@ -42,7 +43,18 @@ module Open
 
     pipe  = options.delete(:pipe)
     quiet = options.delete(:quiet)
-    options["--quiet"] = quiet if options["--quiet"].nil?
+    post  = options.delete(:post)
+    cookies = options.delete(:cookies)
+
+    options["--quiet"]     = quiet if options["--quiet"].nil?
+    options["--post-data="] ||= post if post
+
+    if cookies
+      options["--save-cookies"] = cookies
+      options["--load-cookies"] = cookies
+      options["--keep-session-cookies"] = true
+    end
+
 
     stderr = case
              when options['stderr']
@@ -52,6 +64,7 @@ module Open
              else
                nil
              end
+
     begin
       CMD.cmd("wget '#{ url }'", options.merge(
         '-O' => '-', 
@@ -141,6 +154,9 @@ module Open
     wget_options = options[:wget_options] || {}
     wget_options[:nice] = options.delete(:nice)
     wget_options[:nice_key] = options.delete(:nice_key)
+    wget_options[:quiet] = options.delete(:quiet)
+    wget_options[:post] = options.delete(:post)
+    wget_options[:cookies] = options.delete(:cookies)
 
     io = case
          when (not remote?(url))
@@ -155,8 +171,8 @@ module Open
            io.close
            file_open(in_cache(url), options[:grep])
          end
-    io = unzip(io)  if zip?  url and not options[:noz]
-    io = gunzip(io) if gzip? url and not options[:noz]
+    io = unzip(io)  if (zip?  url and not options[:noz]) or options[:zip]
+    io = gunzip(io) if (gzip? url and not options[:noz]) or options[:gzip]
 
     io
   end

data/lib/rbbt/util/pkg_software.rb

@@ -70,6 +70,7 @@ module PKGSoftware
       if not File.exists?(path)
         sharedir ||= PKGSoftware.get_caller_sharedir
         get_pkg(pkg.to_s, path, get, sharedir)
+        setup_env(software_dir)
       end
 
       SOFTWARE[pkg.to_s] = path
@@ -80,7 +81,6 @@ module PKGSoftware
     SOFTWARE[pkg.to_s]
   end
 
-
   def setup_env(software_dir)
     Misc.env_add 'PATH', bin_dir
 
@@ -127,4 +127,5 @@ module PKGSoftware
 
     CMD.cmd(File.join(opt_dir, '.post_install'))
   end
+
 end

data/lib/rbbt/util/tc_hash.rb

@@ -39,7 +39,7 @@ class TCHash < TokyoCabinet::HDB
   alias original_keys keys
   def keys
     list = self.original_keys
-    indexes = FIELD_INFO_ENTRIES.values.collect do |field| list.index(field) end.compact
+    indexes = FIELD_INFO_ENTRIES.values.collect do |field| list.index(field) end.compact.sort.reverse
     indexes.each do |index| list.delete_at index end
     list
   end
@@ -48,19 +48,12 @@ class TCHash < TokyoCabinet::HDB
   def values
     values = self.original_values
     keys   = self.original_keys
-    indexes = FIELD_INFO_ENTRIES.values.collect do |field| keys.index(field) end.compact
+    indexes = FIELD_INFO_ENTRIES.values.collect do |field| keys.index(field) end.compact.sort.reverse
     indexes.each do |index| values.delete_at index end
 
     values.collect{|v| Serializer.load(v)}
   end
 
-  def merge!(data)
-    new_data = {}
-    data.each do |key, values|
-      self[key] = values
-    end
-  end
-
   # This version of each fixes a problem in ruby 1.9. It also
   # removes the special entries
   def each19(&block)
@@ -77,10 +70,17 @@ class TCHash < TokyoCabinet::HDB
   
   def collect
     res = []
-    self.each{|k, v| res << [k,v]}
+    self.each{|k, v| res << yield(k,v)}
     res
   end
 
+  def merge!(data)
+    new_data = {}
+    data.each do |key, values|
+      self[key] = values
+    end
+  end
+
   alias original_open open
   def open(write = false)
     flags = write ? TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT : TokyoCabinet::BDB::OREADER

data/lib/rbbt/util/tsv.rb

@@ -2,6 +2,7 @@ require 'rbbt/util/misc'
 require 'rbbt/util/open'
 require 'rbbt/util/tc_hash'
 require 'rbbt/util/tmpfile'
+require 'rbbt/util/log'
 require 'digest'
 require 'fileutils'
 
@@ -16,6 +17,13 @@ end
 class TSV
   class FieldNotFoundError < StandardError;end
 
+  module Field
+    def ==(string)
+      return false unless String === string
+      self.sub(/#.*/,'').casecmp(string.sub(/#.*/,'')) == 0
+    end
+  end
+
   #{{{ Persistence
 
   PersistenceHash = TCHash
@@ -36,14 +44,7 @@ class TSV
     File.join(CACHEDIR, prefix.gsub(/\s/,'_').gsub(/\//,'>') + Digest::MD5.hexdigest([file, options].inspect))
   end
 
-  @debug = ENV['TSV_DEBUG'] == "true"
-  def self.log(message)
-    STDERR.puts message if @debug == true
-  end
-
-  def self.debug=(value)
-    @debug = value
-  end
+  #{{{ Headers and Field Stuff
 
   def self.headers(file, options = {})
     if file =~ /(.*)#(.*)/ and File.exists? $1
@@ -63,742 +64,850 @@ class TSV
     end
   end
 
-  #{{{ Accesor Methods
-
-  def keys
-    @data.keys
+  def self.fields_include(key_field, fields, field)
+    return true if key_field == field or fields.include? field
+    return false
   end
 
-  def values
-    @data.values
+  def self.field_positions(key_field, fields, *selected)
+    selected.collect do |sel|
+      case
+      when (sel.nil? or sel == :main or sel == key_field)
+        -1
+      when Integer === sel
+        sel
+      else
+        Misc.field_position fields, sel
+      end
+    end
   end
 
-  def size
-    @data.size
+  def fields_include(field)
+    return TSV.fields_include key_field, fields, field
   end
 
-  # Write
+  def field_positions(*selected)
+    return nil if selected.nil? or selected == [nil]
+    TSV.field_positions(key_field, fields, *selected)
+  end
 
-  def []=(key, value)
-    key = key.downcase if @case_insensitive
-    @data[key] = value
+  def fields_at(*positions)
+    return nil if fields.nil?
+    return nil if positions.nil? or positions == [nil]
+    (fields + [key_field]).values_at(*positions)
   end
 
+  #{{{ Iteration, Merging, etc
+  def through(new_key_field = nil, new_fields = nil, &block)
+    new_key_position = (field_positions(new_key_field) || [-1]).first
+    new_fields = [new_fields] if String === new_fields
 
-  def merge!(new_data)
-    new_data.each do |key, value|
-      self[key] = value
-    end
-  end
+    if new_key_position == -1
 
-  # Read
+      if new_fields.nil? or new_fields == fields
+        each &block 
+        return [key_field, fields]
+      else
+        new_field_positions = field_positions(*new_fields)
+        each do |key, values|
+          if values.nil?
+            yield key, nil
+          else
+            yield key, values.values_at(*new_field_positions)
+          end
+        end
+        return [key_field, fields_at(*new_field_positions)]
+      end
 
-  def follow(value)
-    if String === value && value =~ /__Ref:(.*)/
-      return self[$1]
     else
-      value = NamedArray.name value, fields if Array === value and fields 
-      value
+      new_field_positions = field_positions(*new_fields)
+
+      new_field_names = fields_at(*new_field_positions)
+      if new_field_names.nil? and fields
+        new_field_names = fields.dup
+        new_field_names.delete_at new_key_position
+        new_field_names.unshift key_field
+      end
+
+      each do |key, values|
+        if list
+          tmp_values = values + [[key]]
+        else
+          tmp_values = values + [key]
+        end
+
+        if new_field_positions.nil?
+          new_values = values.dup
+          new_values.delete_at new_key_position
+          new_values.unshift [key] 
+        else
+          new_values = tmp_values.values_at(*new_field_positions)
+        end
+
+        tmp_values[new_key_position].each do |new_key|
+          if new_field_names
+            yield new_key, NamedArray.name(new_values, new_field_names)
+          else
+            yield new_key, new_values
+          end
+        end
+      end
+      return [(fields_at(new_key_position) || [nil]).first, new_field_names]
     end
   end
-  def [](key)
-    if Array === key
-      return @data[key] if @data[key] != nil
-      key.each{|k| v = self[k]; return v unless v.nil?}
-      return nil
+  
+  def process(field)
+    through do |key, values|
+      values[field].replace yield(values[field], key, values) unless values[field].nil? 
     end
-
-    key = key.downcase if @case_insensitive
-    follow @data[key]
   end
 
-  def values_at(*keys)
-    keys.collect{|k|
-      self[k]
-    }
-  end
 
-  def each(&block)
-    @data.each do |key, value|
-      block.call(key, follow(value))
-    end
-  end
+  def reorder(new_key_field, new_fields = nil, options = {})
+    options = Misc.add_defaults options
+    return TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => case_insensitive)  if options[:persistence_file] and File.exists?(options[:persistence_file])
 
-  def collect
-    if block_given?
-      @data.collect do |key, value|
-        value = follow(value)
-        key, values = yield key, value
-      end
-    else
-      @data.collect do |key, value|
-        [key, follow(value)]
+    new = {}
+    new_key_field, new_fields = through new_key_field, new_fields do |key, values|
+      if new[key].nil?
+        new[key] = values
+      else
+        new[key] = new[key].zip(values)
       end
     end
-  end
 
-  def sort(&block)
-    collect.sort(&block).collect{|p|
-      key, value = p
-      value = NamedArray.name value, fields if fields
-      [key, value]
-    }
-  end
+    new.each do |key,values| 
+      values.each{|list| list.flatten! if Array === list}
+    end
 
-  def sort_by(&block)
-    collect.sort_by &block
-  end
+    if options[:persistence_file]
+      reordered = TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => case_insensitive)
+      reordered.merge! new
+    else
+      reordered = TSV.new(new, :case_insensitive => case_insensitive)
+    end
 
-  #{{{ Parsing
-  
-  def self.parse_fields(io, delimiter = "\t")
-    return [] if io.nil?
-    fields = io.split(delimiter, -1)
-    fields
-  end
+    reordered.key_field = new_key_field
+    reordered.fields    = new_fields
 
-  def self.zip_fields(list, fields = nil)
-    return [] if list.nil? || list.empty?
-    fields ||= list.fields if list.respond_to? :fields
-    zipped = list[0].zip(*list[1..-1])
-    zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields 
-    zipped 
+    reordered
   end
-  
-  def self.parse(data, file, options = {})
 
-    # Prepare options
-    options = add_defaults options, 
-      :sep              => "\t",
-      :sep2             => "|",
-      :native           => 0,
-      :extra            => nil,
-      :fix              => nil,
-      :exclude          => nil,
-      :select           => nil,
-      :grep             => nil,
-      :single           => false,
-      :unique           => false,
-      :flatten          => false,
-      :overwrite        => false,
-      :keep_empty       => true,
-      :case_insensitive => false,
-      :header_hash      => '#' ,
-      :persistence_file => nil
+  def slice(new_fields, options = {})
+    reorder(:main, new_fields)
+  end
 
-    options[:extra]   = [options[:extra]] if options[:extra] != nil && ! (Array === options[:extra])
-    options[:flatten] = true if options[:single]
+  def add_field(name = nil)
+    each do |key, values|
+      self[key] = values << yield(key, values)
+    end
 
+    fields << name if list
+    if PersistenceHash === @data
+      @data.fields = fields
+    end
+  end
 
+  def select(method)
+    new = TSV.new({})
+    new.key_field = key_field
+    new.fields    = fields.dup
+    
+    case
+    when Array === method
+      through do |key, values|
+        new[key] = values if ([key,values].flatten & method).any?
+      end
+    when Regexp === method
+      through do |key, values|
+        new[key] = values if [key,values].flatten.select{|v| v =~ method}.any?
+      end
+    when Hash === method
+      key  = method.keys.first
+      method = method.values.first
+      case
+      when (Array === method and (:main == key or key_field == key))
+        method.each{|item| if values = self[item]; then  new[item] = values; end}
+      when Array === method
+        through :main, key do |key, values|
+          new[key] = values if (values.flatten & method).any?
+        end
+      when Regexp === method
+        through :main, key do |key, values|
+          new[key] = values if values.flatten.select{|v| v =~ method}.any?
+        end
+      end
+    end
 
-    #{{{ Process first line
+    new
+  end
 
-    line = file.gets
-    raise "Empty content" if line.nil?
-    line.chomp!
+  def index(options = {})
+    options = Misc.add_defaults options, :order => false
 
-    if line =~ /^#{options[:header_hash]}/
-      header_fields    = parse_fields(line, options[:sep])
-      header_fields[0] = header_fields[0][(0 + options[:header_hash].length)..-1] # Remove initial hash character
-      line = file.gets
-    else
-      header_fields = nil
+    if options[:persistence] and ! options[:persistence_file]
+      options[:persistence_file] = TSV.get_persistence_file(filename, "index:#{ filename }_#{options[:field]}:", options)
     end
-    
-    id_pos = Misc.field_position(header_fields, options[:native])
 
-    if options[:extra].nil?
-      extra_pos = nil
-      max_cols = 0
-    else
-      extra_pos = options[:extra].collect{|pos| Misc.field_position(header_fields, pos) }
+    if options[:persistence_file] and File.exists?(options[:persistence_file])
+      return TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => options[:case_insensitive]) 
     end
 
-    #{{{ Process rest
-    while line do
-      line.chomp!
-
-      line = options[:fix].call line if options[:fix]
+    new = {}
+    if options[:order]
+      new_key_field, new_fields = through options[:field], options[:others] do |key, values|
 
-      # Select and fix lines
-      if (options[:exclude] and   options[:exclude].call(line)) or
-         (options[:select]  and not options[:select].call(line))
-         line = file.gets
-         next
-      end
+        values.each_with_index do |list, i|
+          next if list.nil? or list.empty?
 
-      ### Process line
+          list = [list] unless Array === list
 
-      # Chunk fields
-      parts = parse_fields(line, options[:sep])
-
-      # Get next line
-      line = file.gets
-
-      # Get id field
-      next if parts[id_pos].nil? || parts[id_pos].empty?
-      ids = parse_fields(parts[id_pos], options[:sep2])
-      ids.collect!{|id| id.downcase } if options[:case_insensitive]
-
-      # Get extra fields
-
-      if options[:extra].nil? and not (options[:flatten] or options[:single])
-        extra = parts 
-        extra.delete_at(id_pos)
-        max_cols = extra.size if extra.size > (max_cols || 0)
-      else
-        if extra_pos.nil?
-          extra = parts
-          extra.delete_at id_pos
-        else
-          extra = parts.values_at(*extra_pos)
-        end
-      end
-
-      extra.collect!{|value| parse_fields(value, options[:sep2])}  
-      extra.collect!{|values| values.first}       if options[:unique]
-      extra.flatten!                              if options[:flatten]
-      extra = extra.first                         if options[:single]
-
-      if options[:overwrite]
-        main_entry = ids.shift
-        ids.each do |id|
-          data[id] = "__Ref:#{main_entry}"  
+          list.each do |value|
+            next if value.nil? or value.empty?
+            value = value.downcase if options[:case_insensitive]
+            new[value]    ||= []
+            new[value][i + 1] ||= []
+            new[value][i + 1] << key
+          end
+          new[key]    ||= []
+          new[key][0] = key
         end
 
-        data[main_entry] = extra
-      else
-        main_entry = ids.shift
-        ids.each do |id|
-          data[id] = "__Ref:#{main_entry}"
-        end
+      end
 
-        case
-        when (options[:single] or options[:unique])
-          data[main_entry] ||= extra
-        when options[:flatten]
-          if PersistenceHash === data
-            data[main_entry] = (data[main_entry] || []).concat extra
-          else
-            data[main_entry] ||= []
-            data[main_entry].concat extra
-          end
-        else
-          entry = data[main_entry] || []
-          while entry =~ /__Ref:(.*)/ do
-            entry = data[$1]
-          end
+      new.each do |key, values| 
+        values.flatten!
+        values.compact!
+      end
 
-          extra.each_with_index do |fields, i|
-            if fields.empty?
-              next unless options[:keep_empty]
-              fields = [""]
+    else
+      new_key_field, new_fields = through options[:field], options[:others] do |key, values|
+        new[key] ||= []
+        new[key] << key 
+        values.each do |list|
+          next if list.nil? 
+          if Array === list
+            list.each do |value|
+              value = value.downcase if options[:case_insensitive]
+              new[value] ||= []
+              new[value] << key
             end
-            entry[i] ||= []
-            entry[i] = entry[i].concat fields
+          else
+            next if list.empty?
+            value = list
+            value = value.downcase if options[:case_insensitive]
+            new[value] ||= []
+            new[value] << key
           end
-
-          data[main_entry] = entry
-        end
-      end
-    end
-
-    if options[:keep_empty] and not max_cols.nil?
-      data.each do |key,values| 
-        new_values = values
-        max_cols.times do |i|
-          new_values[i] ||= [""]
         end
-        data[key] = new_values
       end
     end
 
-
-    # Save header information
-    key_field = nil
-    fields   = nil
-    if header_fields && header_fields.any?
-      key_field = header_fields[id_pos] 
-      if extra_pos.nil?
-        fields = header_fields
-        fields.delete_at(id_pos) 
-      else
-        fields = header_fields.values_at(*extra_pos) 
-      end
+    if options[:persistence_file]
+      index = TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => options[:case_insensitive])
+      index.merge! new
+    else
+      index = TSV.new(new, :case_insensitive => options[:case_insensitive])
     end
 
-    data.read if PersistenceHash === data
-
-    [key_field, fields]
+    index.key_field = new_key_field
+    index.fields    = new_fields
+    index
   end
 
-  attr_accessor :data, :key_field, :fields, :list, :case_insensitive, :filename
-  def initialize(file = {}, options = {})
-    @case_insensitive = options[:case_insensitive] == true
-    @list = ! (options[:flatten] == true || options[:single] == true || options[:unique] == true)
-
-    case
-    when TSV === file
-      @filename         = file.filename
-      @data             = file.data
-      @key_field        = file.key_field
-      @fields           = file.fields
-      @case_insensitive = file.case_insensitive
-      @list             = file.is_list
-      return self
-    when (Hash === file or PersistenceHash === file)
-      @filename = "Hash:" + Digest::MD5.hexdigest(file.inspect)
-      @data = file
-      return self
-    when File === file
-      @filename = File.expand_path file.path
-    when String === file && File.exists?(file)
-      @filename = File.expand_path file
-      file = Open.open(file)
-    when StringIO
-    else 
-      raise "File #{file} not found"
-    end
+  def smart_merge(other, match = nil, new_fields = nil)
 
-    if options[:persistence]
-      options.delete :persistence
-      persistence_file = TSV.get_persistence_file @filename, "file:#{ @filename }:", options
+    new_fields = [new_fields] if String === new_fields
+    if self.fields and other.fields 
+      common_fields = ([self.key_field] + self.fields)   & ([other.key_field] + other.fields)
+      new_fields    ||= ([other.key_field] + other.fields) - ([self.key_field] + self.fields)
 
-      if File.exists? persistence_file
-        TSV.log "Loading Persistence for #{ @filename } in #{persistence_file}"
-        @data      = PersistenceHash.get(persistence_file, false)
-        @key_field = @data.key_field
-        @fields    = @data.fields
-      else
-        @data = PersistenceHash.get(persistence_file, true)
-        file = Open.grep(file, options[:grep]) if options[:grep]
+      common_fields.delete match if String === match
+      common_fields.delete_at match if Integer === match
 
-        TSV.log "Persistent Parsing for #{ @filename } in #{persistence_file}"
-        @key_field, @fields = TSV.parse(@data, file, options.merge(:persistence_file => persistence_file))
-        @data.key_field            = @key_field
-        @data.fields               = @fields
-        @data.read
-      end
+      this_common_field_positions   = self.field_positions *common_fields 
+      other_common_field_positions  = other.field_positions *common_fields 
+      other_new_field_positions     = other.field_positions *new_fields
     else
-      TSV.log "Non-persistent parsing for #{ @filename }"
-      @data = {}
-      file = Open.grep(file, options[:grep]) if options[:grep]
-      @key_field, @fields = TSV.parse(@data, file, options)
+      nofieldinfo = true
     end
 
-    file.close
-    @case_insensitive = options[:case_insensitive] == true
-  end
+    case
+    when TSV === match
+      match_index = match
+      matching_code_position = nil
 
+    when Array === match
+      match_index = match.first
+      matching_code_position = field_positions(match.last).first
 
-  def to_s
-    str = ""
+    when match =~ /^through:(.*)/
+      through = $1
+      if through =~ /(.*)#using:(.*)/
+        through = $1
+        matching_code_position = field_positions($2).first
+      else
+        matching_code_position = nil
+      end
+      index_fields = TSV.headers(through)
+      target_field = index_fields.select{|field| other.fields_include field}.first
+      Log.debug "Target Field: #{ target_field }"
+      match_index = TSV.open_file(through).index(:field => target_field)
 
-    if fields
-      str << "#" << key_field << "\t" << fields * "\t" << "\n"
+    when field_positions(match).first
+      matching_code_position = field_positions(match).first
+      match_index = nil
     end
 
-    each do |key, values|
-      case
-      when values.nil?
-        str << key.dup << "\n"
-      when (not Array === values)
-        str << key.dup << "\t" << values.to_s << "\n"
-      when Array === values.first
-        str << key.dup <<  "\t" << values.collect{|list| (list || []) * "|"} * "\t" << "\n"
-      else
-        str << key.dup <<  "\t" << values * "\t" << "\n"
+    if matching_code_position.nil? and match_index.fields
+      match_index.fields.each do |field| 
+        if matching_code_position = field_positions(field).first
+          break
+        end
       end
     end
 
-    str
-  end
-
-  #{{{ New
-
-  def self.fields_include(key_field, fields, field)
-    return true if field == key_field or fields.include? field
-    return false
-  end
+    if match_index and match_index.key_field == other.key_field
+      other_index = nil
+    else
+      other_index = (match === String and other.fields_include(match)) ? 
+        other.index(:other => match, :order => true) : other.index(:order => true)
+    end
 
-  def self.field_positions(key_field, fields, *selected)
-    selected.collect do |sel|
-      case
-      when (sel.nil? or sel == :main or sel == key_field)
-        -1
-      when Integer === sel
-        sel
+    each do |key,values|
+      Log.debug "Key: #{ key }. Values: #{values * ", "}"
+      if matching_code_position.nil? or matching_code_position == -1
+        matching_codes = [key] 
       else
-        Misc.field_position fields, sel
+        matching_codes = values[matching_code_position]
+        matching_codes = [matching_codes] unless  matching_codes.nil? or Array === matching_codes
       end
-    end
-  end
-
-  def fields_include(field)
-    return TSV.fields_include key_field, fields, field
-  end
-
-  def field_positions(*selected)
-    return nil if selected.nil? or selected == [nil]
-    TSV.field_positions(key_field, fields, *selected)
-  end
+      Log.debug "Matching codes: #{matching_codes}"
 
-  def fields_at(*positions)
-    return nil if fields.nil?
-    return nil if positions.nil? or positions == [nil]
-    (fields + [key_field]).values_at(*positions)
-  end
+      next if matching_codes.nil?
 
-  def through(new_key_field = nil, new_fields = nil, &block)
-    new_key_position = (field_positions(new_key_field) || [-1]).first
+      matching_codes.each do |matching_code|
+        if match_index
+          if match_index[matching_code]
+            matching_code_fix = match_index[matching_code].first
+          else
+            matching_code_fix = nil
+          end
+        else
+          matching_code_fix = matching_code
+        end
 
-    if new_key_position == -1
+        Log.debug "Matching code (fix): #{matching_code_fix}"
+        next if matching_code_fix.nil?
 
-      if new_fields.nil? or new_fields == fields
-        each &block 
-        return [key_field, fields]
-      else
-        new_field_positions = field_positions(*new_fields)
-        each do |key, values|
-          yield key, values.values_at(*new_field_positions)
+        if other_index
+          Log.debug "Using other_index"
+          other_codes = other_index[matching_code_fix]
+        else
+          other_codes = matching_code_fix
         end
-        return [key_field, fields_at(*new_field_positions)]
-      end
+        Log.debug "Other codes: #{other_codes}"
 
-    else
-      new_field_positions = field_positions(*new_fields)
+        next if other_codes.nil? or other_codes.empty?
+        other_code = other_codes.first
 
-      new_field_names = fields_at(*new_field_positions)
-      if new_field_names.nil? and fields
-        new_field_names = fields.dup
-        new_field_names.delete_at new_key_position
-        new_field_names.unshift key_field
-      end
+        if nofieldinfo
+          next if other[other_code].nil?
+          if list
+            other_values = [[other_code]] + other[other_code]
+          else
+            other_values = [other_code] + other[other_code]
+          end
+          other_values.delete_if do |list| 
+            list = [list] unless Array === list
+            list.collect{|e| case_insensitive ? e.downcase : e }.
+                 select{|e| case_insensitive ? e == matching_code.downcase : e == matching_code }.any?
+          end
 
-      each do |key, values|
-        if list
-          tmp_values = values + [[key]]
+          new_values = values + other_values 
         else
-          tmp_values = values + [key]
-        end
+          if other[other_code].nil?
+            if list
+              other_values = [[]] * other.fields.length
+            else
+              other_values = [] * other.fields.length
+            end
+          else
+            if list
+              other_values = other[other_code] + [[other_code]]
+            else
+              other_values = other[other_code] + [other_code]
+            end
+          end
+  
 
-        if new_field_positions.nil?
           new_values = values.dup
-          new_values.delete_at new_key_position
-          new_values.unshift [key] 
-        else
-          new_values = tmp_values.values_at(*new_field_positions)
-        end
 
-        tmp_values[new_key_position].each do |new_key|
-          if new_field_names
-            yield new_key, NamedArray.name(new_values, new_field_names)
-          else
-            yield new_key, new_values
+          if list
+            this_common_field_positions.zip(other_common_field_positions).each do |tpos, opos|
+              new_values_tops = new_values[tpos]
+
+              if other.list
+                new_values_tops += other_values[opos]
+              else
+                new_values_tops += [other_values[opos]]
+              end
+
+              new_values[tpos] = new_values_tops.uniq
+            end
           end
+
+          new_values.concat other_values.values_at *other_new_field_positions
         end
+
+        self[key] = new_values
       end
-      return [(fields_at(new_key_position) || [nil]).first, new_field_names]
     end
+
+    self.fields = self.fields + new_fields unless nofieldinfo
+  end
+
+  #{{{ Helpers
+
+  def self.index(file, options = {})
+    opt_data = options.dup
+    opt_index = options.dup
+    opt_data.delete  :field
+    opt_data.delete  :persistence
+    opt_index.delete :persistence
+
+    opt_data[:persistence] = true if options[:data_persistence]
+
+    opt_index.merge! :persistence_file => get_persistence_file(file, "index:#{ file }_#{options[:field]}:", opt_index) if options[:persistence]
+
+    if ! opt_index[:persistence_file].nil? && File.exists?(opt_index[:persistence_file])
+      Log.low "Reloading persistent index for #{ file }: #{opt_index[:persistence_file]}"
+      TSV.new(PersistenceHash.get(opt_index[:persistence_file], false), opt_index)
+    else
+      Log.low "Creating index for #{ file }: #{opt_index[:persistence_file]}"
+      data = TSV.new(file, opt_data)
+      data.index(opt_index)
+    end
+  end
+
+  def self.open_file(file)
+    if file =~ /(.*?)#(.*)/
+      file, options = $1, Misc.string2hash($2.to_s)
+    else
+      options = {}
+    end
+
+    TSV.new(file, options)
   end
-  
-  def process(field)
-    through do |key, values|
-      values[field].replace yield(values[field], key, values) unless values[field].nil? 
-    end
+
+  #{{{ Accesor Methods
+
+  def keys
+    @data.keys
   end
 
+  def values
+    @data.values
+  end
 
-  def reorder(new_key_field, new_fields = nil, options = {})
-    options = Misc.add_defaults options
-    return TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => case_insensitive)  if options[:persistence_file] and File.exists?(options[:persistence_file])
+  def size
+    @data.size
+  end
 
-    new = {}
-    new_key_field, new_fields = through new_key_field, new_fields do |key, values|
-      if new[key].nil?
-        new[key] = values
-      else
-        new[key] = new[key].zip(values)
-      end
-    end
+  # Write
 
-    new.each do |key,values| 
-      values.each{|list| list.flatten! if Array === list}
+  def []=(key, value)
+    key = key.downcase if @case_insensitive
+    @data[key] = value
+  end
+
+
+  def merge!(new_data)
+    new_data.each do |key, value|
+      self[key] = value
     end
+  end
 
-    if options[:persistence_file]
-      reordered = TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => case_insensitive)
-      reordered.merge! new
+  # Read
+
+  def follow(value)
+    if String === value && value =~ /__Ref:(.*)/
+      return self[$1]
     else
-      reordered = TSV.new(new, :case_insensitive => case_insensitive)
+      value = NamedArray.name value, fields if Array === value and fields 
+      value
     end
+  end
 
-    reordered.key_field = new_key_field
-    reordered.fields    = new_fields
+  def [](key)
+    if Array === key
+      return @data[key] if @data[key] != nil
+      key.each{|k| v = self[k]; return v unless v.nil?}
+      return nil
+    end
 
-    reordered
+    key = key.downcase if @case_insensitive
+    follow @data[key]
   end
 
-  def slice(new_fields, options = {})
-    reorder(:main, new_fields)
+  def values_at(*keys)
+    keys.collect{|k|
+      self[k]
+    }
   end
 
-  def index(options = {})
-    options = Misc.add_defaults options, :order => false
-
-    if options[:persistence] and ! options[:persistence_file]
-      options[:persistence_file] = TSV.get_persistence_file(filename, "index:#{ filename }_#{options[:field]}:", options)
+  def each(&block)
+    @data.each do |key, value|
+      block.call(key, follow(value))
     end
+  end
 
-    if options[:persistence_file] and File.exists?(options[:persistence_file])
-      return TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => options[:case_insensitive]) 
+  def collect
+    if block_given?
+      @data.collect do |key, value|
+        value = follow(value)
+        key, values = yield key, value
+      end
+    else
+      @data.collect do |key, value|
+        [key, follow(value)]
+      end
     end
+  end
 
-    new = {}
-    if options[:order]
-      new_key_field, new_fields = through options[:field], options[:others] do |key, values|
-
-        values.each_with_index do |list, i|
-          next if list.nil? or list.empty?
-
-          list = [list] unless Array === list
-
-          list.each do |value|
-            next if value.nil? or value.empty?
-            value = value.downcase if options[:case_insensitive]
-            new[value]    ||= []
-            new[value][i + 1] ||= []
-            new[value][i + 1] << key
-          end
-          new[key]    ||= []
-          new[key][0] = key
-        end
+  def sort(&block)
+    collect.sort(&block).collect{|p|
+      key, value = p
+      value = NamedArray.name value, fields if fields
+      [key, value]
+    }
+  end
 
-      end
+  def sort_by(&block)
+    collect.sort_by &block
+  end
 
-      new.each do |key, values| 
-        values.flatten!
-        values.compact!
-      end
+  def to_s
+    str = ""
 
-    else
-      new_key_field, new_fields = through options[:field], options[:others] do |key, values|
-        new[key] ||= []
-        new[key] << key 
-        values.each do |list|
-          next if list.nil? 
-          if Array === list
-            list.each do |value|
-              value = value.downcase if options[:case_insensitive]
-              new[value] ||= []
-              new[value] << key
-            end
-          else
-            next if list.empty?
-            value = list
-            value = value.downcase if options[:case_insensitive]
-            new[value] ||= []
-            new[value] << key
-          end
-        end
-      end
+    if fields
+      str << "#" << key_field << "\t" << fields * "\t" << "\n"
     end
 
-    if options[:persistence_file]
-      index = TSV.new(PersistenceHash.get(options[:persistence_file], false), :case_insensitive => options[:case_insensitive])
-      index.merge! new
-    else
-      index = TSV.new(new, :case_insensitive => options[:case_insensitive])
+    each do |key, values|
+      case
+      when values.nil?
+        str << key.dup << "\n"
+      when (not Array === values)
+        str << key.dup << "\t" << values.to_s << "\n"
+      when Array === values.first
+        str << key.dup <<  "\t" << values.collect{|list| (list || []) * "|"} * "\t" << "\n"
+      else
+        str << key.dup <<  "\t" << values * "\t" << "\n"
+      end
     end
 
-    index.key_field = new_key_field
-    index.fields    = new_fields
-    index
+    str
   end
 
-  def smart_merge(other, match = nil, new_fields = nil)
-
-    new_fields = [new_fields] if String === new_fields
-    if self.fields and other.fields 
-      common_fields = ([self.key_field] + self.fields)   & ([other.key_field] + other.fields)
-      new_fields    ||= ([other.key_field] + other.fields) - ([self.key_field] + self.fields)
+  #{{{ Parsing
+  
+  def self.parse_fields(io, delimiter = "\t")
+    return [] if io.nil?
+    fields = io.split(delimiter, -1)
+    fields
+  end
 
-      common_fields.delete match if String === match
-      common_fields.delete_at match if Integer === match
+  def self.zip_fields(list, fields = nil)
+    return [] if list.nil? || list.empty?
+    fields ||= list.fields if list.respond_to? :fields
+    zipped = list[0].zip(*list[1..-1])
+    zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields 
+    zipped 
+  end
+  
+  def self.parse(data, file, options = {})
 
-      this_common_field_positions   = self.field_positions *common_fields 
-      other_common_field_positions  = other.field_positions *common_fields 
-      other_new_field_positions     = other.field_positions *new_fields
-    else
-      nofieldinfo = true
-    end
+    # Prepare options
+    options = add_defaults options, 
+      :sep              => "\t",
+      :sep2             => "|",
+      :native           => 0,
+      :extra            => nil,
+      :fix              => nil,
+      :exclude          => nil,
+      :select           => nil,
+      :grep             => nil,
+      :single           => false,
+      :unique           => false,
+      :flatten          => false,
+      :overwrite        => false,
+      :keep_empty       => true,
+      :case_insensitive => false,
+      :header_hash      => '#' ,
+      :persistence_file => nil
 
-    case
-    when TSV === match
-      match_index = match
-      matching_code_position = nil
+    options[:extra]   = [options[:extra]] if options[:extra] != nil && ! (Array === options[:extra])
+    options[:flatten] = true if options[:single]
 
-    when Array === match
-      match_index = match.first
-      matching_code_position = field_positions(match.last).first
 
-    when match =~ /^through:(.*)/
-      through = $1
-      if through =~ /(.*)#using:(.*)/
-        through = $1
-        matching_code_position = field_positions($2).first
-      else
-        matching_code_position = nil
-      end
-      index_fields = TSV.headers(through)
-      target_field = index_fields.select{|field| other.fields_include field}.first
-      Log.debug "Target Field: #{ target_field }"
-      match_index = TSV.open_file(through).index(:field => target_field)
 
-    when field_positions(match).first
-      matching_code_position = field_positions(match).first
-      match_index = nil
-    end
+    #{{{ Process first line
 
-    if matching_code_position.nil? and match_index.fields
-      match_index.fields.each do |field| 
-        if matching_code_position = field_positions(field).first
-          break
-        end
-      end
+    line = file.gets
+    raise "Empty content" if line.nil?
+    line.chomp!
+
+    if line =~ /^#{options[:header_hash]}/
+      header_fields    = parse_fields(line, options[:sep])
+      header_fields[0] = header_fields[0][(0 + options[:header_hash].length)..-1] # Remove initial hash character
+      line = file.gets
+    else
+      header_fields = nil
     end
+    
+    id_pos = Misc.field_position(header_fields, options[:native])
 
-    if match_index and match_index.key_field == other.key_field
-      other_index = nil
+    if options[:extra].nil?
+      extra_pos = nil
+      max_cols = 0
     else
-      other_index = (match === String and other.fields_include(match)) ? 
-        other.index(:other => match, :order => true) : other.index(:order => true)
+      extra_pos = options[:extra].collect{|pos| Misc.field_position(header_fields, pos) }
     end
 
-    each do |key,values|
-      Log.debug "Key: #{ key }. Values: #{values * ", "}"
-      if matching_code_position.nil? or matching_code_position == -1
-        matching_codes = [key] 
-      else
-        matching_codes = values[matching_code_position]
-        matching_codes = [matching_codes] unless  matching_codes.nil? or Array === matching_codes
+    #{{{ Process rest
+    while line do
+      line.chomp!
+
+      line = options[:fix].call line if options[:fix]
+
+      # Select and fix lines
+      if (options[:exclude] and   options[:exclude].call(line)) or
+         (options[:select]  and not options[:select].call(line))
+         line = file.gets
+         next
       end
-      Log.debug "Matching codes: #{matching_codes}"
 
-      next if matching_codes.nil?
+      ### Process line
 
-      matching_codes.each do |matching_code|
-        if match_index
-          if match_index[matching_code]
-            matching_code_fix = match_index[matching_code].first
-          else
-            matching_code_fix = nil
-          end
+      # Chunk fields
+      parts = parse_fields(line, options[:sep])
+
+      # Get next line
+      line = file.gets
+
+      # Get id field
+      next if parts[id_pos].nil? || parts[id_pos].empty?
+      ids = parse_fields(parts[id_pos], options[:sep2])
+      ids.collect!{|id| id.downcase } if options[:case_insensitive]
+
+      # Get extra fields
+
+      if options[:extra].nil? and not (options[:flatten] or options[:single])
+        extra = parts 
+        extra.delete_at(id_pos)
+        max_cols = extra.size if extra.size > (max_cols || 0)
+      else
+        if extra_pos.nil?
+          extra = parts
+          extra.delete_at id_pos
         else
-          matching_code_fix = matching_code
+          extra = parts.values_at(*extra_pos)
         end
+      end
 
-        Log.debug "Matching code (fix): #{matching_code_fix}"
-        next if matching_code_fix.nil?
+      extra.collect!{|value| parse_fields(value, options[:sep2])}  
+      extra.collect!{|values| values.first}       if options[:unique]
+      extra.flatten!                              if options[:flatten]
+      extra = extra.first                         if options[:single]
 
-        if other_index
-          Log.debug "Using other_index"
-          other_codes = other_index[matching_code_fix]
-        else
-          other_codes = matching_code_fix
+      if options[:overwrite]
+        main_entry = ids.shift
+        ids.each do |id|
+          data[id] = "__Ref:#{main_entry}"  
         end
-        Log.debug "Other codes: #{other_codes}"
 
-        next if other_codes.nil? or other_codes.empty?
-        other_code = other_codes.first
+        data[main_entry] = extra
+      else
+        main_entry = ids.shift
+        ids.each do |id|
+          data[id] = "__Ref:#{main_entry}"
+        end
 
-        if nofieldinfo
-          next if other[other_code].nil?
-          if list
-            other_values = [[other_code]] + other[other_code]
+        case
+        when (options[:single] or options[:unique])
+          data[main_entry] ||= extra
+        when options[:flatten]
+          if PersistenceHash === data
+            data[main_entry] = (data[main_entry] || []).concat extra
           else
-            other_values = [other_code] + other[other_code]
-          end
-          other_values.delete_if do |list| 
-            list = [list] unless Array === list
-            list.collect{|e| case_insensitive ? e.downcase : e }.
-                 select{|e| case_insensitive ? e == matching_code.downcase : e == matching_code }.any?
+            data[main_entry] ||= []
+            data[main_entry].concat extra
           end
-
-          new_values = values + other_values 
         else
-          if other[other_code].nil?
-            if list
-              other_values = [[]] * other.fields.length
-            else
-              other_values = [] * other.fields.length
-            end
-          else
-            if list
-              other_values = other[other_code] + [[other_code]]
-            else
-              other_values = other[other_code] + [other_code]
-            end
+          entry = data[main_entry] || []
+          while entry =~ /__Ref:(.*)/ do
+            entry = data[$1]
           end
-  
-
-          new_values = values.dup
-
-          if list
-            this_common_field_positions.zip(other_common_field_positions).each do |tpos, opos|
-              new_values_tops = new_values[tpos]
-
-              if other.list
-                new_values_tops += other_values[opos]
-              else
-                new_values_tops += [other_values[opos]]
-              end
 
-              new_values[tpos] = new_values_tops.uniq
+          extra.each_with_index do |fields, i|
+            if fields.empty?
+              next unless options[:keep_empty]
+              fields = [""]
             end
+            entry[i] ||= []
+            entry[i] = entry[i].concat fields
           end
 
-          new_values.concat other_values.values_at *other_new_field_positions
+          data[main_entry] = entry
         end
+      end
+    end
 
-        self[key] = new_values
+    if options[:keep_empty] and not max_cols.nil?
+      data.each do |key,values| 
+        new_values = values
+        max_cols.times do |i|
+          new_values[i] ||= [""]
+        end
+        data[key] = new_values
       end
     end
 
-    self.fields = self.fields + new_fields unless nofieldinfo
-  end
 
-   #{{{ Helpers
+    # Save header information
+    key_field = nil
+    fields   = nil
+    if header_fields && header_fields.any?
+      key_field = header_fields[id_pos] 
+      if extra_pos.nil?
+        fields = header_fields
+        fields.delete_at(id_pos) 
+      else
+        fields = header_fields.values_at(*extra_pos) 
+      end
+    end
 
-  def self.index(file, options = {})
-    opt_data = options.dup
-    opt_index = options.dup
-    opt_data.delete  :field
-    opt_data.delete  :persistence
-    opt_index.delete :persistence
+    data.read if PersistenceHash === data
 
-    opt_data[:persistence] = true if options[:data_persistence]
+    [key_field, fields]
+  end
 
-    opt_index.merge! :persistence_file => get_persistence_file(file, "index:#{ file }_#{options[:field]}:", opt_index) if options[:persistence]
+  attr_accessor :data, :key_field, :fields, :list, :case_insensitive, :filename
+  def fields
+    fields = @fields
+    fields.each do |f| f.extend Field end if Array === fields
+    fields
+  end
 
-    if ! opt_index[:persistence_file].nil? && File.exists?(opt_index[:persistence_file])
-      TSV.log "Reloading persistent index for #{ file }: #{opt_index[:persistence_file]}"
-      TSV.new(PersistenceHash.get(opt_index[:persistence_file], false), opt_index)
-    else
-      TSV.log "Creating index for #{ file }: #{opt_index[:persistence_file]}"
-      data = TSV.new(file, opt_data)
-      data.index(opt_index)
+  def initialize(file = {}, options = {})
+    options = Misc.add_defaults options
+    options[:persistence] = true if options[:persistence_file]
+
+    if String === file && file =~ /(.*?)#(.*)/
+      file, file_options = $1, $2
+      options = Misc.add_defaults file_options, options
     end
-  end
 
-  def self.open_file(file)
-    if file =~ /(.*?)#(.*)/
-      file, options = $1, Misc.string2hash($2.to_s)
+    @case_insensitive = options[:case_insensitive] == true
+    @list = ! (options[:flatten] == true || options[:single] == true || options[:unique] == true)
+
+    case
+    when TSV === file
+      Log.low "Copying TSV"
+      @filename         = file.filename
+
+      if options[:persistence] and not PersistenceHash === file.data
+        persistence_file = options.delete(:persistence_file) || TSV.get_persistence_file(@filename, "file:#{ @filename }:", options)
+        Log.low "Making persistance #{ persistence_file }"
+        @data = TCHash.get(persistence_file)
+        @data.merge! file
+        @data.key_field        = file.key_field
+        @data.fields           = file.fields
+      else
+        @data             = file.data
+      end
+
+      @key_field        = file.key_field
+      @fields           = file.fields
+      @case_insensitive = file.case_insensitive
+      @list             = file.list
+      return self
+    when Hash === file
+      Log.low "Encapsulating Hash"
+      @filename = "Hash:" + Digest::MD5.hexdigest(file.inspect)
+      if options[:persistence] 
+        persistence_file = options.delete(:persistence_file) || TSV.get_persistence_file(@filename, "file:#{ @filename }:", options)
+        Log.low "Making persistance #{ persistence_file }"
+        @data = TCHash.get(persistence_file)
+        @data.merge! file
+      else
+        @data = file
+      end
+      return self
+    when PersistenceHash === file
+      Log.low "Encapsulating PersistenceHash"
+      @filename = "PersistenceHash:" + Digest::MD5.hexdigest(file.inspect)
+      @data             = file
+      @key_field        = file.key_field
+      @fields           = file.fields
+      return self
+    when File === file
+      @filename = File.expand_path file.path
+    when String === file && File.exists?(file)
+      @filename = File.expand_path file
+      file = Open.open(file)
+    when StringIO
+    else 
+      raise "File #{file} not found"
+   end
+
+    if options[:persistence]
+      options.delete :persistence
+      persistence_file = options.delete(:persistence_file) || TSV.get_persistence_file(@filename, "file:#{ @filename }:", options)
+
+      if File.exists? persistence_file
+        Log.low "Loading Persistence for #{ @filename } in #{persistence_file}"
+        @data      = PersistenceHash.get(persistence_file, false)
+        @key_field = @data.key_field
+        @fields    = @data.fields
+      else
+        @data = PersistenceHash.get(persistence_file, true)
+        file = Open.grep(file, options[:grep]) if options[:grep]
+
+        Log.low "Persistent Parsing for #{ @filename } in #{persistence_file}"
+        @key_field, @fields = TSV.parse(@data, file, options.merge(:persistence_file => persistence_file))
+        @data.key_field            = @key_field
+        @data.fields               = @fields
+        @data.read
+      end
     else
-      options = {}
+      Log.low "Non-persistent parsing for #{ @filename }"
+      @data = {}
+      file = Open.grep(file, options[:grep]) if options[:grep]
+      @key_field, @fields = TSV.parse(@data, file, options)
     end
 
-    TSV.new(file, options)
+    file.close
+    @case_insensitive = options[:case_insensitive] == true
   end
 
 end
+
+#{{{ CacheHelper
+require 'rbbt/util/cachehelper'
+module CacheHelper
+   def self.tsv_cache(name, key = [])
+     cache_file = CacheHelper.build_filename name, key
+
+     if File.exists? cache_file
+       Log.debug "TSV cache file '#{cache_file}' found"
+       hash = TCHash.get(cache_file)
+       TSV.new(hash)
+     else
+       Log.debug "Producing TSV cache file '#{cache_file}'"
+       data = yield
+       TSV.new(data, :persistence_file => cache_file)
+     end
+  end
+end