rbbt-util 3.1.0 → 3.2.0

data/lib/rbbt/util/tsv/accessor.rb

@@ -89,6 +89,7 @@ class TSV
     return nil if @key_field.nil?
     k = @key_field.dup
     k.extend Field
+    k.namespace = namespace unless namespace.nil?
     k
   end
 
@@ -134,6 +135,11 @@ class TSV
     TSV.identify_field(key_field, fields, field)
   end
 
+  def key_field=(new_key_field)
+    @key_field = new_key_field
+    @data.key_field = new_key_field if @data.respond_to? :key_field= and @data.write?
+  end
+
   def fields=(new_fields)
     new_fields.collect! do |field| 
       if Field === field
@@ -150,11 +156,6 @@ class TSV
     @data.fields = new_fields if @data.respond_to? :fields= and @data.write?
   end
 
-  def old_fields=(new_fields)
-    @fields = new_fields
-    @data.fields = new_fields if @data.respond_to? :fields=
-  end
-
   def keys
     @data.keys
   end
@@ -183,13 +184,14 @@ class TSV
 
   # Read
 
+  attr_accessor :unnamed
   def follow(value)
     return nil if value.nil?
     if String === value && value =~ /__Ref:(.*)/
       return self[$1]
     else
 
-      if Array === value and fields 
+      if Array === value and not unnamed
         value = NamedArray.name value, fields 
       end
       value

data/lib/rbbt/util/tsv/attach.rb

@@ -128,7 +128,7 @@ class TSV
       if other.include? key
         new_values = other[key].values_at *fields
         new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
-        new_values.collect!{|v| v.nil? ? nil : v.first} if not type == :double and     other.type == :double
+        new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
         self[key] = self[key].concat new_values
       else
         if type == :double
@@ -151,6 +151,7 @@ class TSV
 
     through do |key, values|
       source_keys = values[source]
+      source_keys = [source_keys] unless Array === source_keys
       if source_keys.nil? or source_keys.empty?
         all_new_values = []
       else
@@ -165,8 +166,8 @@ class TSV
             end
           end
 
-          new_values.collect!{|v| [v]}                    if     type == :double and not other.type == :double
-          new_values.collect!{|v| v.nil? ? nil : v.first} if not type == :double and     other.type == :double
+          new_values.collect!{|v| [v]}                                                  if     type == :double and not other.type == :double
+          new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
           all_new_values << new_values
         end
       end
@@ -191,12 +192,12 @@ class TSV
 
   def attach_index(other, index, fields = nil)
     fields = other.fields - [key_field].concat(self.fields) if fields.nil?
+    fields = [fields] unless Array === fields
 
     other = other.tsv unless TSV === other
     field_positions = fields.collect{|field| other.identify_field field}
     field_names     = field_positions.collect{|pos| pos == :key ? other.key_field : other.fields[pos] }
 
-     
     length = self.fields.length
     through do |key, values|
       source_keys = index[key]
@@ -218,7 +219,7 @@ class TSV
             end
           end
           new_values.collect!{|v| v.nil? ? [[]] : [v]}    if     type == :double and not other.type == :double
-          new_values.collect!{|v| v.nil? ? nil : v.first} if not type == :double and     other.type == :double
+          new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
           all_new_values << new_values
         end
       end
@@ -227,7 +228,7 @@ class TSV
         if type == :double
           all_new_values = [[[]] * field_positions.length]
         else
-          all_new_values =  [[""] * field_positions.length]
+          all_new_values = [[""] * field_positions.length]
         end
       end
 
@@ -260,7 +261,11 @@ class TSV
     in_namespace = options[:in_namespace]
 
     if in_namespace
-      ids = [files.first.all_namespace_fields(in_namespace)]
+      if files.first.all_fields.include? in_namespace
+        ids = [[in_namespace]]
+      else
+        ids = [files.first.all_namespace_fields(in_namespace)]
+      end
       ids += files[1..-1].collect{|f| f.all_fields}
     else
       ids = files.collect{|f| f.all_fields}
@@ -276,7 +281,7 @@ class TSV
       id_list << match.first
     end
     
-    if id_list.last.first != files.last.all_fields.first
+    if id_list.last != files.last.all_fields.first
       id_list << files.last.all_fields.first
       id_list.zip(files)
     else
@@ -312,14 +317,14 @@ class TSV
       next_key, next_file = path.shift
 
       if current_index.nil?
-        current_index = next_file.index :target => next_key, :fields => current_key, :persistence => persist_input
+        current_index = next_file.index :target => next_key, :fields => current_key, :persistence => (persist_input and path.empty?)
       else
         next_index = next_file.index :target => next_key, :fields => current_key, :persistence => persist_input
         current_index.process current_index.fields.first do |values|
           if values.nil?
             nil
           else
-            next_index.values_at(*values).flatten.collect
+            next_index.values_at(*values).flatten.collect.to_a
           end
         end
         current_index.fields = [next_key]
@@ -395,27 +400,13 @@ class TSV
     reorder :key, detached_fields
   end
 
-  def paste(other, options = {})
-    tmpfile = TmpFile.tmp_file
-    TSV.paste(self.to_s, other.to_s, tmpfile)
-
-    new = TSV.new(tmpfile, options)
-
-    new.key_field = self.key_field unless self.key_field.nil?
-    if self.fields and other.fields
-      new.fields = self.fields + other.fields
-    end
-
-    FileUtils.rm tmpfile if File.exists? tmpfile
-
-    new
-  end
-
-
   def paste(other, options = {})
     TmpFile.with_file do |output|
       TSV.paste_merge(self, other, output, options[:sep] || "\t")
-      TSV.new output, options
+      tsv = TSV.new output, options
+      tsv.key_field = self.key_field unless self.key_field.nil?
+      tsv.fields = self.fields + other.fields unless self.fields.nil? or other.fields.nil?
+      tsv
     end
   end
 

data/lib/rbbt/util/tsv/filters.rb

@@ -0,0 +1,193 @@
+require 'rbbt/util/misc'
+module Filtered
+
+  class Filter
+    attr_accessor :data, :match, :fieldnum, :value, :list, :unsaved
+    attr_accessor :persistence
+    def initialize(data, match, value, persistence = nil)
+      @data = data
+      @value = value
+      @unsaved = []
+
+      case 
+      when Hash === persistence
+        @persistence = persistence
+      when String === persistence
+        @persistence = TSV.new TCHash.get(persistence)
+        @persistence.read
+      end
+
+      @list = nil
+      case
+      when match.match(/field:(.*)/)
+        field_num = data.identify_field $1
+        Misc.add_method(self, :match) do |entry|
+          entry[field_num] == value
+        end
+      end
+    end
+
+    def key
+      case 
+      when String === value
+        value
+      else
+        Marshal.dump(value)
+      end
+    end
+
+    def save(ids)
+      if persistence
+        persistence.write
+        persistence[self.key] = ids
+        persistence.read
+      else
+        if list.nil?
+          @list = ids
+        else
+          @list.replace ids
+        end
+      end
+    end
+
+    def update
+      ids = []
+      data.unfiltered_each do |key, entry|
+        ids << key if match(entry)
+      end
+      save(ids.sort)
+    end
+
+    def saved
+      if persistence.nil?
+        return nil if list.nil?
+        list
+      else
+        return nil if not persistence.include?(self.key)
+        persistence[self.key]
+      end
+    end
+
+    def add_unsaved
+      save(Misc.merge_sorted_arrays(unsaved.sort, saved || [])) if unsaved.any?
+      unsaved.clear
+    end
+
+    def ids
+      add_unsaved
+
+      list = saved
+      if list.nil?
+        update
+        list = saved
+      end
+      list
+    end
+
+    def add(id)
+      unsaved.push id
+    end
+
+    def reset
+      if persistence
+        persistence.clear
+      else
+        @list = nil
+      end
+    end
+  end
+
+  def self.extended(base)
+    class << base
+      attr_accessor :filter_dir, :filters
+    end
+
+    Misc.redefine_method base, :[]=, :unfiltered_set do |key,value|
+      if filters.empty?
+        self.send(:unfiltered_set, key, value)
+      else
+        filters.each do |filter| 
+          filter.add key if filter.match value
+        end
+        self.send(:unfiltered_set, key, value)
+      end
+    end
+
+    Misc.redefine_method base, :keys, :unfiltered_keys do 
+    if filters.empty?
+      self.send(:unfiltered_keys)
+    else
+      filters.inject(nil){|list,filter| list.nil? ? filter.ids : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+    end
+    end
+
+    Misc.redefine_method base, :values, :unfiltered_values do
+      if filters.empty?
+        self.send(:unfiltered_values)
+      else
+        ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+        self.send :values_at, *ids
+    end
+    end
+
+    Misc.redefine_method base, :each, :unfiltered_each do |&block|
+    if filters.empty?
+      self.send(:unfiltered_each, &block)
+    else
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+      new = self.dup
+      new.data = {}
+
+      ids.zip(self.send(:values_at, *ids)).each do |id, values|
+        new[id] = values
+      end
+
+      new.send :each, &block
+    end
+    end
+
+    Misc.redefine_method base, :collect, :unfiltered_collect do |&block|
+    if filters.empty?
+      self.send(:unfiltered_collect, &block)
+    else
+      ids = filters.inject(nil){|list,filter| list = (list.nil? ? filter.ids : Misc.intersect_sorted_arrays(list, filter.ids))}
+      new = self.dup
+      new.data = {}
+      ids.zip(self.send(:values_at, *ids)).each do |id, values|
+        new[id] = values
+      end
+      new.send :collect, &block
+    end
+    end
+  end
+
+  def filter_name(match, value)
+    @filename + "&F[#{match}=#{value}]"
+  end
+
+  def add_filter(match, value, persistence = nil)
+    if persistence.nil? and filter_dir
+      persistence = File.join(filter_dir, match.to_s)
+    end
+
+    @filename = filter_name(match, value)  if @filename
+
+    filters.push Filter.new self, match, value, persistence
+  end
+
+  def pop_filter
+    @filename = @filename.sub(/&F\[[^\]]*\]$/, '') if @filename
+    filters.pop
+  end
+
+end
+
+class TSV
+  def filter(filter_dir = nil)
+    self.extend Filtered
+    self.filter_dir = filter_dir
+    self.filters = []
+    self
+  end
+end
+

data/lib/rbbt/util/tsv/index.rb

@@ -13,7 +13,7 @@ class TSV
                "Index[:key]"
              end
 
-    new = Persistence.persist(self, prefix, :tsv, options) do |tsv, options, filename|
+    Persistence.persist(self, prefix, :tsv, options) do |tsv, options, filename|
       order, target, fields, case_insensitive = Misc.process_options options, :order, :target, :fields, :case_insensitive
 
       new = {}
@@ -34,7 +34,7 @@ class TSV
             list = [list] unless Array === list
             i += 1 if fields.nil?
             list.each do |elem|
-              next if elem.empty?
+              next if elem.nil? or elem.empty?
               elem.downcase if case_insensitive
               new[elem] ||= []
               new[elem][i] ||= []
@@ -56,10 +56,9 @@ class TSV
         # flatten
 
         new.each do |key, values| 
-          values.flatten!
-          values.compact!
+          new[key] = values.flatten.compact
         end
-      
+
       ## Not ordered
       else
         double_keys = true unless type != :double or identify_field(target) == :key
@@ -74,7 +73,7 @@ class TSV
           end
           list.collect!{|e| e.downcase} if case_insensitive
           list.each do |elem|
-            next if elem.empty?
+            next if elem.nil? or elem.empty?
             new[elem] ||= []
             if double_keys
               new[elem].concat key 
@@ -103,7 +102,9 @@ class TSV
                    else
                      [new_key_field]
                    end
+
       new = TSV.new([new, {:namespace => namespace, :key_field => key_field, :fields => fields, :type => :flat, :filename => (filename.nil? ? nil : "Index:" + filename), :case_insensitive => case_insensitive}])
+
       new
     end
   end
@@ -293,7 +294,7 @@ class TSV
 
     range = ! pos_end.nil?
 
-    index = Persistence.persist(filename, "SortedIndex[#{range ? pos_start + ":" + pos_end: pos_start}]", :fwt, :start => pos_start, :end => pos_end, :range => range) do |filename, options|
+    index = Persistence.persist(filename, "SortedIndex[#{range ? pos_start + ":" + pos_end : pos_start}]", :fwt, :start => pos_start, :end => pos_end, :range => range) do |filename, options|
       pos_start, pos_end, range = Misc.process_options options, :start, :end, :range
       data = case
              when (type == :double and range)
@@ -319,6 +320,77 @@ class TSV
     index
   end
 
+  def pos_index(pos_field, file = nil, update = false)
+    value_size = 0
+    index_data = []
 
-end
+    file ||= filename + "-PosIndex[#{ pos_field }]" if filename
+    
+    through :key, pos_field do |key, values|
+      value_size = key.length if key.length > value_size
+
+      pos = values.first
+      if Array === pos
+        pos.each do |p|
+          index_data << [key, p.to_i]
+        end
+      else
+        index_data << [key, pos.to_i]
+      end
+    end
+   
+    pos_index = case
+                when file == :memory
+                  index = FixWidthTable.new(file, value_size, false)
+                  index.add_point index_data
+                  index
+                when (update or not File.exists? file)
+                  index = FixWidthTable.new(file, value_size, false, true)
+                  index.add_point index_data
+                  index
+                else
+                  FixWidthTable.new(file, value_size, false)
+                end
+
+    pos_index
+  end
 
+  def range_index(start_field, end_field, file = nil, update = false)
+    value_size = 0
+    index_data = []
+
+    file ||= filename + "-PosIndex[#{ start_field }-#{end_field}]" if filename
+    
+    through :key, [start_field, end_field] do |key, values|
+      value_size = key.length if key.length > value_size
+
+      start_pos, end_pos = values
+    
+      if Array === start_pos
+        start_pos.zip(end_pos).each do |s,e|
+          index_data << [key, [s.to_i, e.to_i]]
+        end
+      else
+        index_data << [key, [start_pos.to_i, end_pos.to_i]]
+      end
+    end
+    
+    pos_index = case
+                when file == :memory
+                  index = FixWidthTable.get(file, value_size, true)
+                  index.add_range index_data
+                  index.read
+                  index
+                when (update or not File.exists?(file))
+                  index = FixWidthTable.get(file, value_size, true, true)
+                  index.add_range index_data
+                  index.read
+                  index
+                else
+                  FixWidthTable.get(file, value_size, true)
+                end
+
+    pos_index
+  end
+
+end