rbbt-util 3.0.3 → 3.1.0

data/lib/rbbt/util/tsv.rb

@@ -41,9 +41,6 @@ class TSV
     end
   end
 
-  def self.encapsulate_persistence(file, options)
-  end
-
   def initialize(file = {}, type = nil, options = {})
     # Process Options
     
@@ -58,7 +55,7 @@ class TSV
       file = $1
     end
 
-    options = Misc.add_defaults options, :persistence => false, :type => type
+    options = Misc.add_defaults options, :persistence => false, :type => type, :in_situ_persistence => true
 
     # Extract Filename
 
@@ -114,50 +111,79 @@ class TSV
           end
         end
       else
-        @data, extra = Persistence.persist(file, :TSV, :tsv_extra, options) do |file, options, filename|
+        in_situ_persistence = Misc.process_options(options, :in_situ_persistence)
+        @data, extra = Persistence.persist(file, :TSV, :tsv_extra, options) do |file, options, filename, persistence_file|
           data, extra = nil
 
-          case
-            ## Parse source
-          when Resource::Path === file #(String === file and file.respond_to? :open)
-            data, extra = TSV.parse(file.open(:grep => options[:grep]) , options)
-            extra[:namespace] ||= file.namespace
-            extra[:datadir]   ||= file.datadir
-          when StringIO === file
-            data, extra = TSV.parse(file, options)
-          when Open.can_open?(file)
-            Open.open(file, :grep => options[:grep]) do |f|
-              data, extra = TSV.parse(f, options)
-            end
-          when File === file
-            path = file.path
-            file = Open.grep(file, options[:grep]) if options[:grep]
-            data, extra = TSV.parse(file, options)
-          when IO === file
-            file = Open.grep(file, options[:grep]) if options[:grep]
-            data, extra = TSV.parse(file, options)
-          when block_given?
-            data 
-          else
-            raise "Unknown input in TSV.new #{file.inspect}"
+          if in_situ_persistence and persistence_file
+
+            cast = options[:cast]
+            type = options[:type]
+            serializer = case
+                         when ((cast == "to_i" or cast == :to_i) and type == :single)
+                           :integer 
+                         when ((cast == "to_i" or cast == :to_i) and (type == :flat or type == :list))
+                           :integer_array 
+                         when type == :double
+                           :double
+                         when type == :list
+                           :list
+                         when type == :single
+                           :single
+                         else
+                           :marshal
+                         end
+
+            options.merge! :persistence_data => Persistence::TSV.get(persistence_file, true, serializer)
           end
 
-          extra[:filename] = filename
+          begin
+            case
+              ## Parse source
+            when Resource::Path === file #(String === file and file.respond_to? :open)
+              data, extra = TSV.parse(file.open(:grep => options[:grep]) , options)
+              extra[:namespace] ||= file.namespace
+              extra[:datadir]   ||= file.datadir
+            when StringIO === file
+              data, extra = TSV.parse(file, options)
+            when Open.can_open?(file)
+              Open.open(file, :grep => options[:grep]) do |f|
+                data, extra = TSV.parse(f, options)
+              end
+            when File === file
+              path = file.path
+              file = Open.grep(file, options[:grep]) if options[:grep]
+              data, extra = TSV.parse(file, options)
+            when IO === file
+              file = Open.grep(file, options[:grep]) if options[:grep]
+              data, extra = TSV.parse(file, options)
+            when block_given?
+              data 
+            else
+              raise "Unknown input in TSV.new #{file.inspect}"
+            end
+
+            extra[:filename] = filename
+          rescue Exception
+            FileUtils.rm persistence_file if persistence_file and File.exists?(persistence_file)
+            raise $!
+          end
 
           [data, extra]
         end
       end
     end
 
-    if not extra.nil?
+    if not extra.nil? 
       %w(case_insensitive namespace identifiers datadir fields key_field type filename cast).each do |key| 
         if extra.include? key.to_sym
           self.send("#{key}=".to_sym, extra[key.to_sym])
-          if @data.respond_to? "#{key}=".to_sym
-            @data.send("#{key}=".to_sym, extra[key.to_sym])
-          end
+          #if @data.respond_to? "#{key}=".to_sym
+          #  @data.send("#{key}=".to_sym, extra[key.to_sym])
+          #end
         end
       end 
+      @data.read if Persistence::TSV === @data
     end
   end
 

data/lib/rbbt/util/tsv/accessor.rb

@@ -85,6 +85,13 @@ class TSV
     fields.select{|f| f.namespace.nil? or f.namespace == namespace}
   end
 
+  def key_field
+    return nil if @key_field.nil?
+    k = @key_field.dup
+    k.extend Field
+    k
+  end
+
   def fields
     return nil if @fields.nil?
     fds = @fields
@@ -113,7 +120,11 @@ class TSV
 
   def self.identify_field(key, fields, field)
     return field if Integer === field
-    return :key if field.nil? or field == 0 or field.to_sym == :key or key == field 
+    if String === field
+      field = field.dup
+      field.extend Field
+    end
+    return :key if field.nil? or field == 0 or field.to_sym == :key or field == key
     return nil if fields.nil?
     return fields.collect{|f| f.to_s}.index field if fields.collect{|f| f.to_s}.index field
     return fields.index field 
@@ -136,7 +147,7 @@ class TSV
       end
     end if Array === new_fields
     @fields = new_fields
-    @data.fields = new_fields if @data.respond_to? :fields=
+    @data.fields = new_fields if @data.respond_to? :fields= and @data.write?
   end
 
   def old_fields=(new_fields)
@@ -196,6 +207,10 @@ class TSV
     follow @data[key]
   end
 
+  def delete(key)
+    @data.delete(key)
+  end
+
   def values_at(*keys)
     keys.collect{|k|
       self[k]
@@ -258,6 +273,10 @@ class TSV
       keys = nil
     end
 
+    if keys == :sort
+      keys = self.keys.sort
+    end
+
     str = ""
 
     str << "#: " << Misc.hash2string(EXTRA_ACCESSORS.collect{|key| [key, self.send(key)]}) << "\n" unless no_options
@@ -279,4 +298,10 @@ class TSV
 
     str
   end
+
+  def value_peek
+    peek = {}
+    keys[0..10].zip(values[0..10]).each do |k,v| peek[k] = v end
+    peek
+  end
 end

data/lib/rbbt/util/tsv/attach.rb

@@ -47,6 +47,8 @@ class TSV
       file1 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file1 } | grep -v '^#{sep}' ", :pipe => true)
     when (String === file1 or StringIO === file1)
       file1 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file1, :pipe => true)
+    when TSV === file1
+      file1 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file1.to_s(:sort, true), :pipe => true)
     end
 
     case
@@ -54,6 +56,8 @@ class TSV
       file2 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file2 } | grep -v '^#{sep}' ", :pipe => true)
     when (String === file2 or StringIO === file2)
       file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2, :pipe => true)
+    when TSV === file2
+      file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2.to_s(:sort, true), :pipe => true)
     end
 
     output = File.open(output, 'w') if String === output
@@ -115,110 +119,6 @@ class TSV
 
     output.close
   end
-
-  def self.paste(file1, file2, output, sep = "\t")
-    case
-    when (String === file1 and not file1.index("\n") and file1.length < 250 and File.exists?(file1))
-      file1 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file1 } ", :pipe => true)
-    when String === file1
-      file1 = CMD.cmd("sort -k1,1 -t'#{sep}'", :in => file1, :pipe => true)
-    end
-
-    case
-    when (String === file2 and not file2.index("\n") and file2.length < 250 and File.exists?(file2))
-      file2 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file2 } ", :pipe => true)
-    when String === file2
-      file2 = CMD.cmd("sort -k1,1 -t'#{sep}'", :in => file2, :pipe => true)
-    end
-
-    output = File.open(output, 'w') if String === output
-
-    cols1 = nil
-    cols2 = nil
-
-    done1 = false
-    done2 = false
-
-    while (line1 = file1.gets) =~ /#/; end
-    line1.strip!
-    parts1 = line1.split(sep)
-    key1 = parts1.shift
-    cols1 = parts1.length
-
-    while (line2 = file2.gets) =~ /#/; end
-    line2.strip!
-    parts2 = line2.split(sep)
-    key2 = parts2.shift
-    cols2 = parts2.length
-    while not (done1 or done2)
-      case
-      when key1 < key2
-        output.puts [key1, parts1,  [""] * cols2] * sep
-        if file1.eof?
-          done1 = true
-        else
-          line1 = file1.gets
-          line1.strip!
-          parts1 = line1.split(sep)
-          key1 = parts1.shift
-        end
-      when key2 < key1
-        output.puts [key2,  [""] * cols1, parts2] * sep
-        if file2.eof?
-          done2 = true
-        else
-          line2 = file2.gets
-          line2.strip!
-          parts2 = line2.split(sep)
-          key2 = parts2.shift
-        end
-      when key1 == key2
-        output.puts [key1, parts1,  parts2] * sep
-        if file1.eof?
-          done1 = true
-        else
-          line1 = file1.gets
-          line1.strip!
-          parts1 = line1.split(sep)
-          key1 = parts1.shift
-        end
-        if file2.eof?
-          done2 = true
-        else
-          line2 = file2.gets
-          line2.strip!
-          parts2 = line2.split(sep)
-          key2 = parts2.shift
-        end
-      end
-    end
-
-    while not done1
-      output.puts [key1, parts1,  [""] * cols2] * sep
-      if file1.eof?
-        done1 = true
-      else
-        line1 = file1.gets
-        line1.strip!
-        parts1 = line1.split(sep)
-        key1 = parts1.shift
-      end
-    end
-
-    while not done2
-      output.puts [key2,  [""] * cols1, parts2] * sep
-      if file2.eof?
-        done2 = true
-      else
-        line2 = file2.gets
-        line2.strip!
-        parts2 = line2.split(sep)
-        key2 = parts2.shift
-      end
-    end
-
-    output.close
-  end
   #{{{ Attach Methods
   
   def attach_same_key(other, fields = nil)
@@ -296,6 +196,8 @@ class TSV
     field_positions = fields.collect{|field| other.identify_field field}
     field_names     = field_positions.collect{|pos| pos == :key ? other.key_field : other.fields[pos] }
 
+     
+    length = self.fields.length
     through do |key, values|
       source_keys = index[key]
       if source_keys.nil? or source_keys.empty?
@@ -315,7 +217,7 @@ class TSV
               other[source_key][pos]
             end
           end
-          new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
+          new_values.collect!{|v| v.nil? ? [[]] : [v]}    if     type == :double and not other.type == :double
           new_values.collect!{|v| v.nil? ? nil : v.first} if not type == :double and     other.type == :double
           all_new_values << new_values
         end
@@ -323,17 +225,28 @@ class TSV
 
       if all_new_values.empty?
         if type == :double
-          self[key] = self[key].concat [[]] * field_positions.length
+          all_new_values = [[[]] * field_positions.length]
         else
-          self[key] = self[key].concat [""] * field_positions.length
+          all_new_values =  [[""] * field_positions.length]
         end
+      end
+
+      current = self[key]
+
+      if current.length > length
+        all_new_values << current.slice!(length..current.length - 1)
+      end
+
+      if type == :double
+        all_new_values = TSV.zip_fields(all_new_values).collect{|l| l.flatten}
       else
-        if type == :double
-          self[key] = self[key].concat TSV.zip_fields(all_new_values).collect{|l| l.flatten}
-        else
-          self[key] = self[key].concat all_new_values.first
-        end
+        all_new_values = all_new_values.first
       end
+     
+      current += all_new_values
+
+      self[key] = current
+
     end
 
     self.fields = self.fields.concat field_names
@@ -385,13 +298,13 @@ class TSV
     Log.medium "Found Traversal: #{traversal_ids * " => "}"
     
     data_key, data_file = path.shift
-    if data_key == data_file.key_field
-      Log.debug "Data index not required '#{data_file.key_field}' => '#{data_key}'"
-      data_index = nil
-    else
-      Log.debug "Data index required"
-      data_index = data_file.index :target => data_key, :fields => data_file.key_field, :persistence => false
-    end
+    data_index = if data_key == data_file.key_field
+                   Log.debug "Data index not required '#{data_file.key_field}' => '#{data_key}'"
+                   nil
+                 else
+                   Log.debug "Data index required"
+                   data_file.index :target => data_key, :fields => data_file.key_field, :persistence => false
+                 end
 
     current_index = data_index
     current_key   = data_key
@@ -402,7 +315,7 @@ class TSV
         current_index = next_file.index :target => next_key, :fields => current_key, :persistence => persist_input
       else
         next_index = next_file.index :target => next_key, :fields => current_key, :persistence => persist_input
-        current_index.process current_index.fields.first do |key, values, values|
+        current_index.process current_index.fields.first do |values|
           if values.nil?
             nil
           else
@@ -411,6 +324,7 @@ class TSV
         end
         current_index.fields = [next_key]
       end
+      current_key = next_key
     end
 
     current_index
@@ -470,6 +384,8 @@ class TSV
       attach_index other, index, fields
     end
     Log.medium("Attachment of fields:#{fields.inspect} from #{other.filename.inspect} finished.")
+
+    self
   end
 
   def detach(file)
@@ -489,10 +405,21 @@ class TSV
     if self.fields and other.fields
       new.fields = self.fields + other.fields
     end
-    
+
     FileUtils.rm tmpfile if File.exists? tmpfile
 
     new
   end
 
+
+  def paste(other, options = {})
+    TmpFile.with_file do |output|
+      TSV.paste_merge(self, other, output, options[:sep] || "\t")
+      TSV.new output, options
+    end
+  end
+
+  def self.fast_paste(files, delim = "$")
+    CMD.cmd("paste #{ files.collect{|f| "'#{f}'"} * " "} -d'#{delim}' |sed 's/#{delim}[^\\t]*//g'", :pipe => true)
+  end
 end

data/lib/rbbt/util/tsv/index.rb

@@ -34,6 +34,7 @@ class TSV
             list = [list] unless Array === list
             i += 1 if fields.nil?
             list.each do |elem|
+              next if elem.empty?
               elem.downcase if case_insensitive
               new[elem] ||= []
               new[elem][i] ||= []
@@ -73,6 +74,7 @@ class TSV
           end
           list.collect!{|e| e.downcase} if case_insensitive
           list.each do |elem|
+            next if elem.empty?
             new[elem] ||= []
             if double_keys
               new[elem].concat key 
@@ -233,6 +235,8 @@ class TSV
 
   def self.field_matches(tsv, values)
     values = [values] if not Array === values
+    Log.debug "Matcing #{values.length} values to #{tsv.filename}"
+
     if values.flatten.sort[0..9].compact.collect{|n| n.to_i} == (1..10).to_a
       return {}
     end