rbbt-util 3.2.0 → 3.2.1

data/lib/rbbt/util/tsv.rb

@@ -119,23 +119,34 @@ class TSV
         @data = Hash[file.collect{|v| 
           [v,[]]
         }]
-        @data.key_field = key_field if key_field
-        @data.fields = fields if fields
+        self.key_field = options[:key]
+        self.fields = options[:fields]
+        self.type = options[:type] || :double
       when Hash === file 
         @data = file
-        @data.key_field = key_field if key_field
-        @data.fields = fields if fields
+        self.key_field = options[:key]
+        self.fields = options[:fields]
+        self.type = options[:type] || :double
       when TSV === file
         @data = file.data
-        @data.key_field = key_field if key_field
-        @data.fields = fields if fields
+        self.key_field = options[:key] || file.key_field
+        self.fields = options[:fields] || file.fields
+        self.type = options[:type] || file.type
       when Persistence::TSV === file
+        Log.debug("Reopening persistence file #{ file.path_to_db }")
         @data = file
         %w(case_insensitive namespace identifiers datadir fields key_field type filename cast).each do |key|
           if @data.respond_to?(key.to_sym)  and self.respond_to?("#{key}=".to_sym)
             self.send "#{key}=".to_sym, @data.send(key.to_sym) 
           end
         end
+        self.key_field = options[:key] || file.key_field
+        self.fields    = options[:fields] || file.fields
+        self.type      = options[:type] || file.type
+
+        file.key_field = self.key_field
+        file.fields    = self.fields
+        file.type      = self.type
       else
         in_situ_persistence = Misc.process_options(options, :in_situ_persistence)
         @data, extra = Persistence.persist(file, :TSV, :tsv_extra, options) do |file, options, filename, persistence_file|
@@ -178,7 +189,7 @@ class TSV
           end
 
           if Persistence::TSV === data
-            %w(case_insensitive namespace identifiers datadir fields key_field type filename cast).each do |key| 
+            %w(case_insensitive namespace identifiers fields key_field type filename cast).each do |key| 
               if extra.include? key.to_sym
                 if data.respond_to? "#{key}=".to_sym
                   data.send("#{key}=".to_sym, extra[key.to_sym])
@@ -194,7 +205,7 @@ class TSV
     end
 
     if not extra.nil? 
-      %w(case_insensitive namespace identifiers datadir fields key_field type filename cast).each do |key| 
+      %w(case_insensitive namespace identifiers fields key_field type filename cast).each do |key| 
         if extra.include? key.to_sym
           self.send("#{key}=".to_sym, extra[key.to_sym])
           #if @data.respond_to? "#{key}=".to_sym
@@ -217,5 +228,9 @@ class TSV
     @data.write? if @data.respond_to? :write
   end
 
+  def clear
+    @data.clear
+  end
+
 
 end

data/lib/rbbt/util/tsv/accessor.rb

@@ -3,8 +3,8 @@ require 'rbbt/util/misc'
 
 class TSV
   ## Make sure we overwrite the methods declared by attr_accessor
-  MAIN_ACCESSORS = :data,  :key_field, :fields, :cast
-  EXTRA_ACCESSORS = :filename, :identifiers, :namespace, :datadir, :type, :case_insensitive
+  MAIN_ACCESSORS = :data, :key_field, :fields, :cast
+  EXTRA_ACCESSORS = :filename, :identifiers, :namespace, :type, :case_insensitive
   attr_accessor *(MAIN_ACCESSORS + EXTRA_ACCESSORS)
 
   def self.zip_fields(list, fields = nil)
@@ -33,7 +33,7 @@ class TSV
 
     def fullname
       return self if self =~ /:/ or namespace.nil?
-      namespace + ":" << self
+      namespace.to_s + ":" << self
     end
 
     def ==(string)
@@ -143,8 +143,8 @@ class TSV
   def fields=(new_fields)
     new_fields.collect! do |field| 
       if Field === field
-        if field !~ /:/ and field.namespace != nil and field.namespace != namespace
-          field.namespace + ":" + field.to_s
+        if field !~ /:/ and field.namespace != nil and field.namespace.to_s != namespace.to_s
+          field.namespace.to_s + ":" + field.to_s
         else
           field
         end
@@ -286,6 +286,8 @@ class TSV
       str << "#" << key_field << "\t" << fields * "\t" << "\n"
     end
 
+    saved_unnamed = unnamed
+    unnamed = false
     if keys.nil?
       each do |key, values|
         key = key.to_s if Symbol === key
@@ -298,6 +300,7 @@ class TSV
       end
     end
 
+    unnamed = saved_unnamed
     str
   end
 

data/lib/rbbt/util/tsv/filters.rb

@@ -1,11 +1,27 @@
 require 'rbbt/util/misc'
 module Filtered
 
+  class FilterArray
+    attr_accessor :filters
+
+    def ids
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.merge_sorted_arrays(list, filter.ids.dup)}
+    end
+
+    def method_missing(name, *args)
+      filters.each do |filter|
+        filter.send(name, *args)
+      end
+    end
+  end
+
   class Filter
     attr_accessor :data, :match, :fieldnum, :value, :list, :unsaved
     attr_accessor :persistence
+
     def initialize(data, match, value, persistence = nil)
       @data = data
+      @match = match
       @value = value
       @unsaved = []
 
@@ -19,11 +35,15 @@ module Filtered
 
       @list = nil
       case
-      when match.match(/field:(.*)/)
-        field_num = data.identify_field $1
-        Misc.add_method(self, :match) do |entry|
-          entry[field_num] == value
-        end
+      when @match.match(/field:(.*)/)
+        @fieldnum = data.identify_field $1
+        class << self
+          self
+        end.class_eval <<-EOC
+          def match_entry(entry)
+            entry[@fieldnum] == @value
+          end
+        EOC
       end
     end
 
@@ -53,7 +73,7 @@ module Filtered
     def update
       ids = []
       data.unfiltered_each do |key, entry|
-        ids << key if match(entry)
+        ids << key if match_entry(entry)
       end
       save(ids.sort)
     end
@@ -88,7 +108,20 @@ module Filtered
       unsaved.push id
     end
 
+    def clean
+      add_unsaved
+      if persistence and persistence.include? self.key
+        restore = ! persistence.write?
+        persistence.write unless persistence.write?
+        persistence.delete self.key
+        persistence.read if restore
+      else
+        @list = nil
+      end
+    end
+
     def reset
+      add_unsaved
       if persistence
         persistence.clear
       else
@@ -107,7 +140,7 @@ module Filtered
         self.send(:unfiltered_set, key, value)
       else
         filters.each do |filter| 
-          filter.add key if filter.match value
+          filter.add key if filter.match_entry value
         end
         self.send(:unfiltered_set, key, value)
       end
@@ -117,7 +150,7 @@ module Filtered
     if filters.empty?
       self.send(:unfiltered_keys)
     else
-      filters.inject(nil){|list,filter| list.nil? ? filter.ids : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+      filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
     end
     end
 
@@ -125,7 +158,7 @@ module Filtered
       if filters.empty?
         self.send(:unfiltered_values)
       else
-        ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+        ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
         self.send :values_at, *ids
     end
     end
@@ -134,7 +167,7 @@ module Filtered
     if filters.empty?
       self.send(:unfiltered_each, &block)
     else
-      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
       new = self.dup
       new.data = {}
 
@@ -150,7 +183,8 @@ module Filtered
     if filters.empty?
       self.send(:unfiltered_collect, &block)
     else
-      ids = filters.inject(nil){|list,filter| list = (list.nil? ? filter.ids : Misc.intersect_sorted_arrays(list, filter.ids))}
+      ids = filters.inject(nil){|list,filter| list = (list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup))}
+
       new = self.dup
       new.data = {}
       ids.zip(self.send(:values_at, *ids)).each do |id, values|
@@ -159,25 +193,28 @@ module Filtered
       new.send :collect, &block
     end
     end
-  end
 
-  def filter_name(match, value)
-    @filename + "&F[#{match}=#{value}]"
+    Misc.redefine_method base, :delete, :unfiltered_delete do |key|
+    if filters.empty?
+      self.send(:unfiltered_delete, key)
+    else
+      reset_filters
+      self.send :unfiltered_delete, key
+    end
+    end
   end
 
   def add_filter(match, value, persistence = nil)
     if persistence.nil? and filter_dir
-      persistence = File.join(filter_dir, match.to_s)
+      persistence = File.join(filter_dir, match.to_s + '.filter')
     end
 
-    @filename = filter_name(match, value)  if @filename
-
-    filters.push Filter.new self, match, value, persistence
+    filter = Filter.new self, match, value, persistence
+    filters.push filter
   end
 
   def pop_filter
-    @filename = @filename.sub(/&F\[[^\]]*\]$/, '') if @filename
-    filters.pop
+    filters.pop.add_unsaved
   end
 
 end
@@ -189,5 +226,16 @@ class TSV
     self.filters = []
     self
   end
+
+  def reset_filters
+    if filter_dir.nil? or filter_dir.empty?
+      filters.each do |filter| filter.reset end
+      return
+    end
+
+    Dir.glob(File.join(filter_dir, '*.filter')).each do |f|
+      FileUtils.rm f
+    end
+  end
 end
 

data/lib/rbbt/util/tsv/index.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/tsv/manipulate'
+require 'rbbt/util/tsv/filters'
 require 'rbbt/util/fix_width_table'
 
 class TSV
@@ -281,116 +282,138 @@ class TSV
     field_matches(values).sort_by{|field, matches| matches.uniq.length}.last
   end
 
-  def sorted_index(pos_start = nil, pos_end = nil)
-    raise "Please specify indexing fields" if (pos_start.nil? and fields.length > 2)
+  def pos_index(pos_field = nil, options = {})
+    pos_field ||= "Position"
 
-    case
-    when (pos_start.nil? and pos_end.nil? and fields.length == 2)
-      pos_start = fields.first
-      pos_end   = fields.last
-    when (pos_start.nil? and pos_end.nil? and fields.length == 1)
-      pos_start = fields.first
-    end
+    options = Misc.add_defaults options,
+      :persistence => true, :persistence_file => nil, :persistence_update => false 
 
-    range = ! pos_end.nil?
-
-    index = Persistence.persist(filename, "SortedIndex[#{range ? pos_start + ":" + pos_end : pos_start}]", :fwt, :start => pos_start, :end => pos_end, :range => range) do |filename, options|
-      pos_start, pos_end, range = Misc.process_options options, :start, :end, :range
-      data = case
-             when (type == :double and range)
-               collect do |key, values|
-                 p_start, p_end = values.values_at pos_start, pos_end
-                 next if p_start.nil? or p_end.nil? or p_start.empty? or p_end.empty?
-                 [[p_start.first, p_end.first], key]
-               end
-             when (type == :double and not range)
-               collect do |key, values|
-                 p_start = values.values_at pos_start
-                 next if p_start.nil? or p_start.empty? 
-                 [p_start.first, key]
-               end
-             when range
-               slice [pos_start, pos_end]
-             else
-               slice pos_start
-             end
-      data
+    prefix = "Pos[#{pos_field}]"
+
+    Persistence.persist(filename, prefix, :fwt, options.merge({
+      :pos_field => pos_field,
+      :filters => (self.respond_to?(:filters)? filters.collect{|f| [f.match, f.value]} : [])
+    })) do |file, options, filename|
+      pos_field = options[:pos_field]
+      value_size = 0
+      index_data = []
+
+      through :key, pos_field do |key, values|
+        value_size = key.length if key.length > value_size
+
+        pos = values.first
+        if Array === pos
+          pos.each do |p|
+            index_data << [key, p.to_i]
+          end
+        else
+          index_data << [key, pos.to_i]
+        end
+      end
+
+      index = FixWidthTable.get(:memory, value_size, false)
+      index.add_point index_data
+      index.read
+      index
     end
+  end
+
+  def self.pos_index(file, pos_field = nil, options = {})
+    options = Misc.add_defaults options,
+      :persistence => true, :persistence_file => nil, :persistence_update => false, :persistence_source => file, :tsv_serializer => :list,
+      :data_persistence => false, :data_persistence_file => nil, :data_persistence_update => false, :data_persistence_source => file
+
+    options_data = {
+      :persistence        => Misc.process_options(options, :data_persistence),
+      :persistence_file   => Misc.process_options(options, :data_persistence_file),
+      :persistence_update => Misc.process_options(options, :data_persistence_update),
+      :persistence_source => Misc.process_options(options, :data_persistence_source),
+    }
+
+
+    prefix = "Pos[#{pos_field}]"
+
+    new = Persistence.persist(file, prefix, :fwt, options.merge({:pos_field => pos_field})) do |file, options, filename|
+      tsv = TSV.new(file, :list, options_data)
+
+      if options.include?(:filters) and Array === options[:filters] and not options[:filters].empty?
+        tsv.filter
+        options[:filters].each do |match, value, persistence|
+          tsv.add_filter(match, value, persistence)
+        end
+      end
 
-    index
+      tsv.pos_index options[:pos_field], options.merge(:persistence => false, :persistence_file => nil)
+    end
   end
 
-  def pos_index(pos_field, file = nil, update = false)
-    value_size = 0
-    index_data = []
+  def range_index(start_field = nil, end_field = nil, options = {})
+    start_field ||= "Start"
+    end_field ||= "End"
+    options = Misc.add_defaults options,
+      :persistence => true, :persistence_file => nil, :persistence_update => false 
+
+    prefix = "Range[#{start_field}-#{end_field}]"
 
-    file ||= filename + "-PosIndex[#{ pos_field }]" if filename
-    
-    through :key, pos_field do |key, values|
-      value_size = key.length if key.length > value_size
+    Persistence.persist(filename, prefix, :fwt, options.merge({
+      :start_field => start_field, :end_field => end_field,
+      :filters => (self.respond_to?(:filters)? filters.collect{|f| [f.match, f.value]} : [])
+    })) do |file, options, filename|
+      start_field, end_field = options.values_at :start_field, :end_field
 
-      pos = values.first
-      if Array === pos
-        pos.each do |p|
-          index_data << [key, p.to_i]
+      value_size = 0
+      index_data = []
+
+      through :key, [start_field, end_field] do |key, values|
+        value_size = key.length if key.length > value_size
+
+        start_pos, end_pos = values
+
+        if Array === start_pos
+          start_pos.zip(end_pos).each do |s,e|
+            index_data << [key, [s.to_i, e.to_i]]
+          end
+        else
+          index_data << [key, [start_pos.to_i, end_pos.to_i]]
         end
-      else
-        index_data << [key, pos.to_i]
       end
+
+      index = FixWidthTable.get(:memory, value_size, true)
+      index.add_range index_data
+      index.read
+      index
     end
-   
-    pos_index = case
-                when file == :memory
-                  index = FixWidthTable.new(file, value_size, false)
-                  index.add_point index_data
-                  index
-                when (update or not File.exists? file)
-                  index = FixWidthTable.new(file, value_size, false, true)
-                  index.add_point index_data
-                  index
-                else
-                  FixWidthTable.new(file, value_size, false)
-                end
-
-    pos_index
   end
 
-  def range_index(start_field, end_field, file = nil, update = false)
-    value_size = 0
-    index_data = []
+  def self.range_index(file, start_field = nil, end_field = nil, options = {})
+    options = Misc.add_defaults options,
+      :persistence => true, :persistence_file => nil, :persistence_update => false, :persistence_source => file, :tsv_serializer => :list,
+      :data_persistence => false, :data_persistence_file => nil, :data_persistence_update => false, :data_persistence_source => file
 
-    file ||= filename + "-PosIndex[#{ start_field }-#{end_field}]" if filename
-    
-    through :key, [start_field, end_field] do |key, values|
-      value_size = key.length if key.length > value_size
+    options_data = {
+      :persistence        => Misc.process_options(options, :data_persistence),
+      :persistence_file   => Misc.process_options(options, :data_persistence_file),
+      :persistence_update => Misc.process_options(options, :data_persistence_update),
+      :persistence_source => Misc.process_options(options, :data_persistence_source),
+    }
 
-      start_pos, end_pos = values
-    
-      if Array === start_pos
-        start_pos.zip(end_pos).each do |s,e|
-          index_data << [key, [s.to_i, e.to_i]]
+    prefix = "Range[#{start_field}-#{end_field}]"
+
+    options_data[:type] = :flat if options[:order] == false
+
+    Persistence.persist(file, prefix, :fwt, options.merge({:start_field => start_field, :end_field => end_field})) do |file, options, filename|
+      tsv = TSV.new(file, :list, options_data)
+
+      if options.include?(:filters) and Array === options[:filters] and not options[:filters].empty?
+        tsv.filter
+        options[:filters].each do |match, value, persistence|
+          tsv.add_filter(match, value, persistence)
         end
-      else
-        index_data << [key, [start_pos.to_i, end_pos.to_i]]
       end
+
+      tsv.range_index options[:start_field], options[:end_field], options.merge(:persistence => false, :persistence_file => nil)
     end
-    
-    pos_index = case
-                when file == :memory
-                  index = FixWidthTable.get(file, value_size, true)
-                  index.add_range index_data
-                  index.read
-                  index
-                when (update or not File.exists?(file))
-                  index = FixWidthTable.get(file, value_size, true, true)
-                  index.add_range index_data
-                  index.read
-                  index
-                else
-                  FixWidthTable.get(file, value_size, true)
-                end
-
-    pos_index
   end
 
 end
+