rsi 0.4

data/lib/rsi/compressed_serializers.rb

@@ -0,0 +1,60 @@
+#
+# Defines a serializer which uses zlib compression in an attempt to 
+# reduce on-disk index size.
+#
+require 'zlib'
+require 'rsi/serializers'
+
+module RSI
+
+  # Serializer which performs gzip/libz compression on the output
+  # of another base serializer.
+  # By default, this sits on top of an RSI::NativeSerializer.
+  #
+  # (unconfirmed) This may be particularly sensitive to when the 
+  # underlying stream passed to #dump and #load is closed.
+  class CompressedSerializer
+    attr_accessor :base
+
+    # By default, this serializer is based on NativeSerializer.
+    # If you prefer another underlying serializer, you may 
+    # pass it as an argument to new().
+    def initialize( base_serializer=RSI::NativeSerializer.new() )
+      @base=base_serializer
+    end
+
+    def dump( obj, stream )
+      w = ZLib::GZipWriter.new( stream )
+      @base.dump( obj, w )
+    end
+
+    def load( stream )
+      r = ZLib.GZipReader.new( stream )
+      return @base.load( r )
+    end
+  end
+
+  # Serializer which performs bzip (de)compression on the 
+  # output of another base serializer.
+  # By default, this sits on a NativeSerializer.
+  # This requires BZ2: http://raa.ruby-lang.org/project/bz2/ .
+  class BZip2Serializer
+    attr_accessor :base
+    # Pass another Serializer as an argument, if don't want the default.
+    def initialize( base_serializer=NativeSerializer.new() )
+      unless defined? BZ2
+        raise "The BZ2 module is not loaded (it must be installed and `require`d before this module is used)"
+      end
+      @base = base_serializer
+    end
+    def dump( obj, stream )
+      w = BZ2::Writer.new( stream )
+      @base.dump( obj, w )
+    end
+    def load( stream )
+      r = BZ2::Reader.new( stream )
+      return @base.load( r )
+    end
+  end
+
+end

data/lib/rsi/dictionary.rb

@@ -0,0 +1,232 @@
+#
+# Dictionary maintenence for text indexes
+#
+require 'yaml'
+require 'rsi/logmanager'
+
+module RSI
+
+  # An occurrence of a term in a document.
+  # [huh.  freq and pos_list don't seem to add much to the dict size]
+  # freq is redundant.  ==pos_list.length
+  class TermEntry
+    attr_accessor :docid, :freq, :pos_list
+    def initialize( docid )
+      @docid = docid
+      @freq = 0
+      @pos_list = []
+    end
+    def to_s
+      YAML.dump(self)
+    end
+  end
+
+  class Dictionary
+    include Loggable
+
+    attr_accessor :terms, :serializer
+    attr_reader :root
+
+    @@termgroup_loading = 50
+
+    META_FILE = "meta.yaml"
+    TERMS_FILE = "terms.list"
+    TERMGROUP_FILE = "termgroup.list"
+
+    def initialize( root )
+      @root = root
+      @serializer = NativeSerializer.new()
+      @terms_root = File.join( @root, "terms" )
+      @terms = {} # term => id
+      @entries = {} # termid => [TermEntry...]
+      @pending_entries = {} # of termid=>1
+      @termgroups = {} # termid -> termgroupid
+      @meta = { 
+        :next_termid => 0,
+        :next_termgroup_id => 0,
+        :next_termgroup_count => 0,
+      }
+    end
+
+    def open()
+      Dir.mkdir( @root ) unless FileTest.exists?( @root ) 
+      logger.info( "Opening dictionary" )
+      begin
+        self.reload()
+      rescue
+        logger.debug( $! )
+        logger.info( "DB does not exist (#{$!}), creating..." )
+        self.create_store()
+      end
+      @opened = true
+    end
+
+    # Serialize the current state of the dictionary.
+    # (Currently requires time proportional (at least) to the
+    # full size of the dictionary.  This is a bug.)
+    def store()
+      logger.info( "Storing at #{@root}" )
+      # meta info, stored as yaml
+      @meta[ :serializer ] = @serializer;
+      File.open( File.join( @root, META_FILE ), "w" ) do |meta_f|
+        YAML.dump( @meta, meta_f )
+      end
+      
+      # store terms
+      term_fn = File.join( @root, TERMS_FILE )
+      logger.debug( "Storing terms to #{term_fn}" )
+      File.open( term_fn, "w" ) do |term_f|
+        logger.debug( "terms=#{@terms}" )
+        @serializer.dump( @terms, term_f )
+      end
+
+      File.open( File.join( @root, TERMGROUP_FILE), "w" ) do |termgroups_f|
+        @serializer.dump( @termgroups, termgroups_f )
+      end
+
+      store_term_entries()
+    end
+
+    def has_term?( term )
+      return @terms.has_key?( term )
+    end
+
+    # Get the termid for the given (tokenized) term.  If create is
+    # true (the default), the given term has not been previously added
+    # to the dictionary, a new id will be created and returned.
+    def get_termid_for( term, create=false )
+      unless @terms.has_key?( term )
+        return nil unless create
+        t = self.next_termid()
+        @terms[term] = t
+      end
+      return @terms[term]
+    end
+
+    def add_term_entries( docid, termid, pos_list=[0] )
+      e = TermEntry.new( docid )
+      e.pos_list = pos_list
+      e.freq = pos_list.length()
+      add_entry( termid, e )
+    end
+  
+    # Get a list of entries for the given termid.
+    # Creates the entry list, if it doesn't already exist.
+    # Returns a list of TermEntries
+    def get_entry_list( termid )
+      logger.debug( "[termid #{termid}]" )
+      unless @entries.has_key?( termid )
+        logger.debug( "  No entry[#{termid}]" )
+        unless @termgroups.has_key?( termid )
+          logger.debug( "  No termgroups[#{termid}]"  )
+          @termgroups[ termid ] = next_termgroup_id()
+        end
+        id = @termgroups[ termid ]
+        logger.debug( "  Termgroup id=#{id}" )
+        tg_fn = File.join( @terms_root, "#{id}.tg" )
+        logger.debug( "            fn=#{tg_fn}" )
+        if FileTest.exists?( tg_fn )
+          logger.debug( "  Reloading termgroup record #{tg_fn}" )
+          tg_f = File.open( tg_fn, "r" ) 
+          tg = @serializer.load( tg_f )
+          tg_f.close()
+          tg.each do |tid, term_entries|
+            @entries[tid] = term_entries
+          end
+        end
+        unless @entries.has_key?( termid )
+          logger.debug( "  Creating termgroup record" )
+          @entries[termid] = []
+        end
+      end
+      logger.debug( "[returning #{@entries[termid]}]" )
+      return @entries[termid]
+    end
+
+    protected
+
+    # Create a new storage location.
+    def create_store() 
+      logger.info( "Creating store at #{@root}" )
+      Dir.mkdir( @root ) unless FileTest.exists?( @root )
+      Dir.mkdir( @terms_root ) unless FileTest.exists?( @terms_root )
+    end
+
+    # Load the dictionary from storage.
+    def reload()
+      logger.info( "Reloading from #{@root}" )
+      # meta file is dumped/loaded as yaml, always
+      File.open( File.join( @root, META_FILE ), "r" ) do |meta_f|
+        @meta = YAML.load( meta_f )
+        @serializer = @meta[ :serializer ]
+        logger.debug( "Loaded meta from #{META_FILE}" )
+      end
+      term_fn = File.join( @root, TERMS_FILE )
+      File.open( term_fn, "r" ) do |term_f|
+        @terms = @serializer.load( term_f )
+        logger.debug( "Loaded terms from #{term_fn}" )
+      end
+      File.open( File.join( @root, TERMGROUP_FILE), "r" ) do |termgroups_f|
+        @termgroups = @serializer.load( termgroups_f )
+        logger.debug( "Loaded termgroup map from #{TERMGROUP_FILE}" )
+      end
+      # entries are loaded lazily... use get_entry_list
+    end
+
+    # Return the next sequential document id.
+    def next_docid  #not threadsafe
+      @meta[:next_docid] += 1
+      return @meta[:next_docid]
+    end
+
+    # Return the next sequential term id.
+    def next_termid
+      @meta[:next_termid] += 1
+      return @meta[:next_termid]
+    end
+
+    def next_termgroup_id
+      # totally not threadsafe
+      @meta[:next_termgroup_count] += 1
+      if @meta[:next_termgroup_count] > @@termgroup_loading
+        @meta[:next_termgroup_id] += 1
+        @meta[:next_termgroup_count] = 0
+      end
+      return @meta[:next_termgroup_id]
+    end
+
+    def store_term_entries()
+      logger.info( "Storing term entries" )
+      @pending_entries.each do |tg_id, termids|
+        tg_fn = File.join( @terms_root, "#{tg_id}.tg" )
+        tg = nil
+        if FileTest.exists?( tg_fn )
+          File.open( tg_fn, "r" ) {|tg_f| tg = @serializer.load( tg_f )}
+        else
+          tg = {}
+        end
+        termids.each do |termid|
+          tg[ termid ] = @entries[ termid ] # update to internal state
+        end
+        logger.debug( "Writing #{tg_fn}" )
+        File.open( tg_fn, "w" ) {|f| @serializer.dump( tg, f )}
+      end
+      @pending_entries = {} # clear pending set
+    end
+
+
+    # Add an entry for the given termid.
+    def add_entry( termid, entry )
+      # load entrylist and add entry to it, for internal state
+      get_entry_list( termid ) << entry
+      # track entry for later store()
+      tg_id = @termgroups[ termid ]
+      unless @pending_entries.has_key?( tg_id )
+        @pending_entries[ tg_id ] = []
+      end
+      @pending_entries[ tg_id ] << termid
+    end
+
+  end
+
+end

data/lib/rsi/index.rb

@@ -0,0 +1,245 @@
+
+
+require 'rsi/porter'
+require 'rsi/logmanager'
+
+# mixin Stemmable.stem (from porter.rb) into String
+class String
+  include Stemmable
+end
+
+#
+# Classes for building and querying indexes.
+#
+module RSI
+
+  class IndexException < RuntimeError; end
+
+  # Document index.  Interface for adding documents to index, and
+  # for querying an index.
+  class Indexer
+    include RSI::Loggable
+
+    # Dictionary of terms.
+    attr_reader :root
+    # Analyzer to use for document and query tokenization.
+    attr_accessor :analyzer, :query_analyzer, :serializer, :dicts
+
+    META_FILE = "meta.yaml"
+    DOCS_FILE = "docs.list"
+
+    def initialize( root )
+      @root = root
+      @docs = {}
+      @meta = { :next_docid => 0 }
+      @serializer = RSI::NativeSerializer.new()
+      @analyzer = RSI::DefaultTextAnalyzer.new()
+      @query_analyzer = RSI::DefaultTextAnalyzer.new()
+      @dicts = {}
+      @opened = false
+    end
+
+    def open()
+      Dir.mkdir( @root ) unless FileTest.exists?( @root )
+      log_fh = File.open( File.join( @root, "index.log" ), 
+                          File::WRONLY|File::APPEND|File::CREAT )
+      log_fh.sync = true
+      logger.info( "Trying to reload index..." )
+      begin
+        reload()
+      rescue
+        logger.info( "Reload failed (#{$!}), creating new index" )
+        # nothing to do
+      end
+      # Query the analyzer, getting the fields it tokenizes.
+      # Initialize and open a dictionary for each field.
+      logger.info( "Assigning dictionaries..." )
+      @analyzer.get_field_types().each do |field, type|
+        field_root = File.join( @root, field )
+        klass = map_field_type( type )
+        logger.debug( "Field: #{field} at #{field_root} is #{klass}" )
+        @dicts[field] = klass.new( field_root )
+        @dicts[field].serializer = @serializer
+      end
+      logger.info( "Opening dictionaries" )
+      @dicts.each do |name, dict|
+        logger.debug( "Dictionary: #{name}" )
+        dict.open()
+      end
+      @opened = true
+    end
+    
+    # Gets a dictionary instance for the given field type
+    def map_field_type( type )
+      case type
+        when RSI::FIELD_TYPE_TEXT
+          return RSI::Dictionary
+        when RSI::FIELD_TYPE_DATE
+          raise "implement me! XXX"
+      end
+    end
+
+    # Add a document to the index.
+    def add_document( doc_uri, content )
+      open() unless @opened
+      logger.info("Adding document #{doc_uri}")
+      if @docs.has_value?( doc_uri )
+        raise IndexException, "Cannot do updates yet"
+      else
+        docid = next_docid()
+        @docs[ docid ] = doc_uri
+        pos = 0
+        term_entries = {}
+        logger.debug("Tokenizing")
+        @analyzer.tokenize( content ).each do |field, termlist|
+          termlist.each do |term|
+            termid = @dicts[field].get_termid_for(term, true)
+            raise "POO" if termid==nil
+            unless term_entries.has_key?( termid )
+              term_entries[termid] = []
+            end
+            term_entries[termid] << pos
+            pos += 1
+          end
+          logger.debug("Adding term entries to #{field}")
+          term_entries.each do |termid, pos_list|
+            @dicts[field].add_term_entries(docid, termid, term_entries[termid])
+          end
+        end
+      end
+    end
+
+    # Remove a document from the index (slow!).
+    def delete_document( doc_uri )
+      open() unless @opened
+      raise "This is too hard for me, yet"
+    end
+
+    # Stop adding documents to the index, and serialize to storage.
+    def flush()
+      open() unless @opened
+      logger.info("Finishing")
+      store_metadata()
+      store_doclist()
+      @dicts.each do |field, dict|
+        dict.store()
+      end
+    end
+
+    # Return a list of document ids which contain any of the given
+    # search termsn (OR query).  The terms will be tokenized by the
+    # current Analyzer.
+    def find_any( terms_str )
+      open() unless @opened
+      raise "unimplemented"
+    end
+
+    def get_dict_for_field( field )
+      return @dicts[field]
+    end
+
+    # Return a list of document ids which contain any of the given
+    # search terms (AND query).  The terms will be tokenized by the
+    # current Analyzer.
+    #
+    def find_all( terms_str )
+      q = @query_analyzer.tokenize_query( terms_str )
+      logger.debug( "Query=#{q.to_s}" )
+      docids = q.evaluate( self )
+      docids.uniq!
+      return docids.collect {|id| @docs[id]}
+    end
+
+    def OLD_find_all( terms_str )
+      open() unless @opened
+    
+      # this querying logic is too fragile
+      logger.info { "Query: #{terms_str}" }
+      t_set = @query_analyzer.tokenize_query( terms_str )
+      logger.debug { "Tokenized: #{t_set}" }
+      # build map of docid => term-match-count
+      finds = {}
+      t_set.each do |field, term_list|
+        term_list.each do |term|
+          logger.debug { "field='#{field}', term='#{term}'" }
+          # lookup termid in dict for field
+          unless @dicts[field].has_term?( term )
+            logger.info { "No term #{term} in dictionary #{field}" }
+            next
+          end
+          termid = @dicts[field].get_termid_for( term )
+          logger.debug { "termid=#{termid}" }
+          # get list of entries for termid
+          e_list = @dicts[field].get_entry_list( termid )
+          # get list of docids
+          e_list.each do |e|
+            logger.debug { "  docid=#{e.docid}" }
+            finds[ e.docid ] = finds[ e.docid ].to_i + 1
+          end
+        end
+      end
+      total_terms = 0
+      t_set.each_value {|vl| total_terms += vl.size() }
+      logger.debug { "Total terms: #{total_terms}" }
+      # foreach docid in map: match if term-match-count == terms-count
+      d_return = []
+      finds.each do |docid, count|
+        if count == total_terms
+          # return docid
+          uri = @docs[ docid ]
+          d_return << uri
+        end
+      end
+      return d_return
+    end
+
+    protected
+
+    # needs synchro
+    def next_docid()
+      @meta[ :next_docid ] += 1
+      return @meta[ :next_docid ]
+    end
+
+    def reload()
+      logger.info("Reloading from #{@root}")
+      load_metadata()
+      load_doclist()
+    end
+
+    def store_metadata()
+      @meta[ :serializer ] = @serializer
+      @meta[ :analyzer ] = @analyzer.class.name
+      @meta[ :query_analyzer ] = @query_analyzer.class.name
+      fn = File.join( @root, META_FILE )
+      logger.info( "Storing metadata to #{fn}" )
+      File.open( fn, "w" ) {|f| YAML.dump( @meta, f ) }
+    end
+
+    def load_metadata()
+      fn = File.join( @root, META_FILE )
+      logger.info( "Loading metadata from #{fn}" )
+      File.open( fn, "r" ) {|f| @meta = YAML.load( f ) }
+      @serializer = @meta[ :serializer ]
+      @analyzer = eval "#{@meta[ :analyzer ]}.new()"
+      @query_analyzer = eval "#{@meta[ :query_analyzer ]}.new()"
+    end
+
+    def store_doclist()
+      fn = File.join( @root, DOCS_FILE )
+      logger.info( "Storing doc list to #{fn}" )
+      File.open( fn, "w" ) {|f| @serializer.dump( @docs, f ) }
+    end
+
+    def load_doclist()
+      fn = File.join( @root, DOCS_FILE )
+      logger.info( "Loading doc list from #{fn}" )
+      File.open( fn, "r" ) {|f| @docs = @serializer.load( f ) }
+    end
+
+
+    
+  end
+
+end
+