xapian-fu 0.2 → 1.0.1

data/lib/xapian_fu/xapian_doc.rb

@@ -1,20 +1,72 @@
-module XapianFu
+class Time #:nodoc:
+  def to_xapian_fu_string
+    utc.strftime("%Y%m%d%H%M%S")
+  end
+end
+
+class Date #:nodoc:
+  def to_xapian_fu_string
+    strftime("%Y%m%d")
+  end
+end
+
+require 'date'
+
+class DateTime #:nodoc:
+  def to_xapian_fu_string
+    strftime("%Y%m%d%H%M%S")
+  end
+end
+
+module XapianFu #:nodoc:
+  require 'xapian_doc_value_accessor'
   
+  # Raised whenever a XapianDb is needed but has not been provided,
+  # such as when retrieving the terms list for a document
   class XapianDbNotSet < XapianFuError ; end
-  class XapianDocNotSet < XapianFuError ; end
+  # Raised if a given value cannot be stored in the database (anything
+  # without a to_s method)
   class XapianTypeError < XapianFuError ; end
-  
+
+  # A XapianDoc represents a document in a XapianDb.  Searches return
+  # XapianDoc objects and they are used internally when adding new
+  # documents to the database.  You usually don't need to instantiate
+  # them yourself unless you're doing something a bit advanced.
   class XapianDoc
-    attr_reader :fields, :data, :weight, :match
-    attr_reader :xapian_document
-    attr_accessor :id, :db
+    
+    # A hash of the fields given to this object on initialize
+    attr_reader :fields
+    
+    # An abitrary blob of data stored alongside the document in the
+    # Xapian database.
+    attr_reader :data
+    
+    # The search score of this document when returned as part of a
+    # search result
+    attr_reader :weight
+    
+    # The Xapian::Match object for this document when returned as part
+    # of a search result.
+    attr_reader :match
+    
+    # The unsigned integer "primary key" for this document in the
+    # Xapian database.
+    attr_accessor :id
+    
+    # The XapianDb object that this document was retrieved from, or
+    # should be stored in.
+    attr_accessor :db
 
     # Expects a Xapian::Document, a Hash-like object, or anything that
     # with a to_s method.  Anything else raises a XapianTypeError.
-    # Options can be <tt>:weight</tt> to set the search weight or
-    # <tt>:data</tt> to set some additional data to be stored with the
-    # record in the database.
+    # The <tt>:weight</tt> option sets the search weight when setting
+    # up search results.  The <tt>:data</tt> option sets some
+    # additional data to be stored with the document in the database.
+    # The <tt>:xapian_db</tt> option sets the XapianDb to allow saves
+    # and term enumeration.
     def initialize(doc, options = {})
+      @options = options
+      
       @fields = {}
       if doc.is_a? Xapian::Match
         match = doc
@@ -28,19 +80,8 @@ module XapianFu
       if doc.is_a?(Xapian::Document)
         @xapian_document = doc
         @id = doc.docid
-        begin
-          xdoc_data = Marshal::load(doc.data) unless doc.data.empty?
-        rescue ArgumentError
-          @data = nil
-        end
-        if xdoc_data.is_a? Hash
-          @data = xdoc_data.delete(:__data)
-          @fields = xdoc_data
-        else
-          @data = xdoc_data
-        end
       # Handle initialisation from a hash-like object
-      elsif doc.respond_to?("[]") and doc.respond_to?(:has_key?)
+      elsif doc.respond_to?(:has_key?) and doc.respond_to?("[]")
         @fields = doc
         @id = doc[:id] if doc.has_key?(:id)
       # Handle initialisation from anything else that can be coerced
@@ -52,18 +93,21 @@ module XapianFu
       end
       @weight = options[:weight] if options[:weight]
       @data = options[:data] if options[:data]
+      @db = options[:xapian_db] if options[:xapian_db]
     end
 
-    # Retrieve the given Xapianvalue from the XapianDb.  <tt>vkey</tt>
-    # can be a symbol or string, in which case it's hashed to get an
-    # integer value number. Or you can give the integer value number
-    # if you know it.
-    def get_value(vkey)
-      raise XapianDocNotSet unless @xapian_document
-      vkey = vkey.to_s.hash unless vkey.is_a? Integer
-      @xapian_document.value(vkey)
+    # The arbitrary data stored in the Xapian database with this
+    # document.  Returns an empty string if none available.
+    def data
+      @data ||= xapian_document.data
     end
 
+    # The XapianFu::XapianDocValueAccessor for accessing the values in
+    # this document.
+    def values
+      @value_accessor ||= XapianDocValueAccessor.new(self)
+    end
+    
     # Return a list of terms that the db has for this document.
     def terms
       raise XapianFu::XapianDbNotSet unless db      
@@ -74,17 +118,25 @@ module XapianFu
     # database. Requires that the db attribute has been set up.
     def to_xapian_document
       raise XapianFu::XapianDbNotSet unless db
-      xdoc = Xapian::Document.new
-      add_stored_fields_to_xapian_doc(xdoc)
-      add_stored_values_to_xapian_doc(xdoc)
-      xdoc
+      xapian_document.data = data
+      # Clear and add values
+      xapian_document.clear_values
+      add_values_to_xapian_document
+      # Clear and add terms
+      xapian_document.clear_terms      
+      generate_terms
+      xapian_document
     end
 
-    # Return text for indexing from the fields
-    def text
-      fields.keys.collect { |key| fields[key].to_s }.join(' ')
+    # The Xapian::Document for this XapianFu::Document.  If this
+    # document was retrieved from a XapianDb then this will have been
+    # initialized by Xapian, otherwise a new Xapian::Document.new is
+    # allocated.
+    def xapian_document
+      @xapian_document ||= Xapian::Document.new
     end
     
+    # Compare IDs with another XapianDoc
     def ==(b)
       if b.is_a?(XapianDoc)
         id == b.id
@@ -94,24 +146,154 @@ module XapianFu
     end
     
     def inspect
-      "<#{self.class.to_s} id=#{id}>"
+      s = ["<#{self.class.to_s} id=#{id}"]
+      s << "weight=%.5f" % weight if weight
+      s << "db=#{db.nil? ? 'nil' : db}"
+      s.join(' ') + ">"
+    end
+
+    # Add this document to the Xapian Database, or replace it if it
+    # already has an id.
+    def save
+      id ? update : create
+    end
+
+    # Add this document to the Xapian Database
+    def create
+      self.id = db.rw.add_document(to_xapian_document)
+    end
+
+    # Update this document in the Xapian Database
+    def update
+      db.rw.replace_document(id, to_xapian_document)
+    end
+    
+    # Set the stemmer to use for this document.  Accepts any string
+    # that the Xapian::Stem class accepts (Either the English name for
+    # the language or the two letter ISO639 code). Can also be an
+    # existing Xapian::Stem object.
+    def stemmer=(s)
+      @stemmer = StemFactory.stemmer_for(s)
+    end
+
+    # Return the stemmer for this document.  If not set on initialize
+    # by the :stemmer or :language option, it will try the database's
+    # stemmer and otherwise defaults to an English stemmer.
+    def stemmer
+      if @stemmer
+        @stemmer
+      else
+        @stemmer = 
+          if ! @options[:stemmer].nil?
+            @options[:stemmer]
+          elsif @options[:language]
+            @options[:language]
+          elsif db
+            db.stemmer
+          else
+            :english
+          end
+        @stemmer = StemFactory.stemmer_for(@stemmer)
+      end
+    end
+    
+    # Return the stopper for this document.  If not set on initialize
+    # by the :stopper or :language option, it will try the database's
+    # stopper and otherwise default to an English stopper..
+    def stopper
+      if @stopper
+        @stopper
+      else
+        @stopper =
+          if ! @options[:stopper].nil?
+            @options[:stopper]
+          elsif @options[:language]
+            @options[:language]
+          elsif db
+            db.stopper
+          else
+            :english
+          end
+        @stopper = StopperFactory.stopper_for(@stopper)
+      end
+    end
+
+    # Return this document's language which is set on initialize, inherited 
+    # from the database or defaults to :english
+    def language
+      if @language
+        @language
+      else
+        @language =
+          if ! @options[:language].nil?
+            @options[:language]
+          elsif db and db.language
+            db.language
+          else
+            :english
+          end
+      end
     end
     
     private
     
-    def add_stored_fields_to_xapian_doc(xdoc)
-      stored_fields = fields.reject { |k,v| ! db.store_fields.include? k }
-      stored_fields[:__data] = data if data
-      xdoc.data = Marshal.dump(stored_fields) unless stored_fields.empty?
-      xdoc
+    # Array of field names not to run through the TermGenerator
+    def unindexed_fields
+      db ? db.unindexed_fields : []
+    end
+    
+    # Add all the fields to be stored as XapianDb values
+    def add_values_to_xapian_document
+      db.store_values.collect do |key|
+        values[key] = fields[key]
+        key
+      end
+    end
+
+    # Run the Xapian term generator against this documents text
+    def generate_terms
+      tg = Xapian::TermGenerator.new
+      tg.database = db.rw
+      tg.document = xapian_document
+      tg.stopper = stopper
+      tg.stemmer = stemmer      
+      index_method = db.index_positions ? :index_text : :index_text_without_positions
+      fields.each do |k,v|
+        next if unindexed_fields.include?(k)
+        if v.respond_to?(:to_xapian_fu_string)
+          v = v.to_xapian_fu_string
+        else
+          v = v.to_s
+        end
+        # add value with field name
+        tg.send(index_method, v, 1, 'X' + k.to_s.upcase)
+        # add value without field name
+        tg.send(index_method, v)
+      end
+      xapian_document
     end
     
-    def add_stored_values_to_xapian_doc(xdoc)
-      stored_values = fields.reject { |k,v| ! db.store_values.include? k }
-      stored_values.each do |k,v|
-        xdoc.add_value(k.to_s.hash, v.to_s)
+  end
+  
+  
+  class StemFactory
+    # Return a Xapian::Stem object for the given option. Accepts any
+    # string that the Xapian::Stem class accepts (Either the English
+    # name for the language or the two letter ISO639 code).
+    #
+    # If given false or nil, will return a "none" stemmer.
+    #
+    # It will also accept and return an existing Xapian::Stem object.
+    #
+    def self.stemmer_for(stemmer)
+      if stemmer.is_a? Xapian::Stem
+        stemmer
+      elsif stemmer.is_a?(String) or stemmer.is_a?(Symbol)
+        Xapian::Stem.new(stemmer.to_s)
+      else
+        Xapian::Stem.new("none")
       end
-      xdoc
     end
   end
+  
 end

data/lib/xapian_fu/xapian_doc_value_accessor.rb

@@ -0,0 +1,125 @@
+class Integer #:nodoc:
+  def to_xapian_fu_storage_value
+    [self].pack("l")
+  end
+
+  def self.from_xapian_fu_storage_value(value)
+    value.unpack("l").first
+  end
+end
+
+class Bignum #:nodoc:
+  def to_xapian_fu_storage_value
+    [self].pack("G")
+  end
+
+  def self.from_xapian_fu_storage_value(value)
+    value.unpack("G").first
+  end
+end
+
+class Float #:nodoc:
+  def to_xapian_fu_storage_value
+    [self].pack("G")
+  end
+
+  def self.from_xapian_fu_storage_value(value)
+    value.unpack("G").first
+  end
+end
+
+class Time #:nodoc:
+  def to_xapian_fu_storage_value
+    [self.utc.to_f].pack("G")
+  end
+
+  def self.from_xapian_fu_storage_value(value)
+    Time.at(value.unpack("G").first)
+  end
+end
+
+class Date #:nodoc:
+  def to_xapian_fu_storage_value
+    to_s
+  end
+
+  def self.from_xapian_fu_storage_value(value)
+    self.parse(value)
+  end
+end
+
+module XapianFu #:nodoc:
+  
+  # A XapianDocValueAccessor is used to provide the XapianDoc#values
+  # interface to read and write field values to a XapianDb.  It is
+  # usually set up by a XapianDoc so you shouldn't need to set up your
+  # own.
+  class XapianDocValueAccessor
+    def initialize(xapian_doc)
+      @doc = xapian_doc
+    end
+    
+    # Add the given <tt>value</tt> with the given <tt>key</tt> to the
+    # XapianDoc.  If the value has a
+    # <tt>to_xapian_fu_storage_value</tt> method then it is used to
+    # generate the final value to be stored, otherwise <tt>to_s</tt>
+    # is used.  This is usually paired with a
+    # <tt>from_xapian_fu_storage_value</tt> class method on retrieval.
+    def store(key, value, type = nil)
+      type = @doc.db.fields[key] if type.nil? and @doc.db
+      if type and value.is_a?(type) and value.respond_to?(:to_xapian_fu_storage_value)
+        converted_value = value.to_xapian_fu_storage_value
+      else
+        converted_value = value.to_s
+      end
+      @doc.xapian_document.add_value(value_key(key), converted_value)
+      value
+    end
+    alias_method "[]=", :store
+    
+    # Retrieve the value with the given <tt>key</tt> from the
+    # XapianDoc. <tt>key</tt> can be a symbol or string, in which case
+    # it's hashed to get an integer value number. Or you can give the
+    # integer value number if you know it.
+    #
+    # If the class specified in the database fields for this key (or
+    # as the optional argument) has a
+    # <tt>from_xapian_fu_storage_value</tt> method then it is used to
+    # instaniate the object from the stored value.  This is usually
+    # paired with a <tt>to_xapian_fu_storage_value</tt> instance
+    # method.
+    #
+    # Due to the design of Xapian, if the value does not exist then an
+    # empty string is returned.
+    def fetch(key, type = nil)
+      value = @doc.xapian_document.value(value_key(key))
+      type = @doc.db.fields[key] if type.nil? and @doc.db
+      if type and type.respond_to?(:from_xapian_fu_storage_value)
+        type.from_xapian_fu_storage_value(value)
+      else
+        value
+      end
+    end
+    alias_method "[]", :fetch
+    
+    # Count the values stored in the XapianDoc
+    def size
+      @doc.xapian_document.values_count
+    end
+    
+    # Remove the value with the given key from the XapianDoc and return it
+    def delete(key)
+      value = fetch(key)
+      @doc.xapian_document.remove_value(value_key(key))
+      value
+    end
+
+    private
+
+    # Convert the given key to an integer that can be used as a Xapian
+    # value number
+    def value_key(key)
+      key.is_a?(Integer) ? key : key.to_s.hash
+    end    
+  end
+end

data/lib/xapian_fu/xapian_documents_accessor.rb

@@ -0,0 +1,82 @@
+module XapianFu
+  # A XapianDocumentsAccessor is used to provide the
+  # XapianDb#documents interface.  It is usually set up by a XapianDb
+  # so you shouldn't need to set up your own.
+  class XapianDocumentsAccessor
+    def initialize(xdb) #:nodoc:
+      @xdb = xdb
+    end
+
+    # Build a new XapianDoc for this database
+    def new(doc = nil, options = { })
+      options = options.merge({ :xapian_db => @xdb })
+      XapianDoc.new(doc, options)
+    end
+
+    # Add a document to the index. A document can be just a hash, the
+    # keys representing field names and their values the data to be
+    # indexed.  Or it can be a XapianDoc, or any object with a to_s method.
+    #
+    # If the document has an :id field, it is used as the primary key
+    # in the Xapian database.
+    #
+    # If the document object reponds to the method :data, whatever it
+    # returns is marshalled and stored in the  Xapian database.  Any
+    # arbitrary data up to Xmeg can be stored here.
+    #
+    # Currently, all fields are stored in the database. This will
+    # change to store only those fields requested to be stored.
+    def add(doc)
+      doc = XapianDoc.new(doc) unless doc.is_a? XapianDoc
+      doc.db = @xdb
+      doc.save
+      doc
+    end
+    alias_method "<<", :add
+
+    # Return the document with the given id from the
+    # database. Raises a XapianFu::DocNotFoundError exception
+    # if it doesn't exist.
+    def find(doc_id)
+      xdoc = @xdb.ro.document(doc_id)
+      XapianDoc.new(xdoc, :xapian_db => @xdb)
+    rescue RuntimeError => e
+      raise e.to_s =~ /^DocNotFoundError/ ? XapianFu::DocNotFound : e
+    end
+
+    # Return the document with the given id from the database or nil
+    # if it doesn't exist
+    def [](doc_id)
+      find(doc_id)
+    rescue XapianFu::DocNotFound
+      nil
+    end
+
+    # Delete the given document from the database and return the
+    # document id, or nil if it doesn't exist
+    def delete(doc)
+      if doc.respond_to?(:to_i)
+        @xdb.rw.delete_document(doc.to_i)
+        doc.to_i
+      end
+    rescue RuntimeError => e
+      raise e unless e.to_s =~ /^DocNotFoundError/
+    end
+
+    # Return the document with the highest value in the specified field or nil if it doesn't exist
+    def max(key = :id)
+      if key == :id
+        # for :id we can use lastdocid
+        find(@xdb.ro.lastdocid) rescue nil
+      else
+        # for other values, we do a search ordered by that key in descening order
+        query = Xapian::Query.new(Xapian::Query::OP_VALUE_GE, key.to_s.hash, "0")
+        e = Xapian::Enquire.new(@xdb.ro)
+        e.query = query
+        e.sort_by_value!(key.to_s.hash)
+        r = e.mset(0, 1).matches.first
+        find(r.docid) rescue nil
+      end
+    end
+  end
+end

data/lib/xapian_fu.rb

@@ -1,3 +1,4 @@
 $:.unshift File.join(File.dirname(__FILE__), 'xapian_fu')
 require 'xapian_db'
 require 'xapian_doc'
+require 'stopper_factory'

data/spec/query_parser_spec.rb

@@ -0,0 +1,43 @@
+require 'xapian'
+require 'lib/xapian_fu.rb'
+include XapianFu
+
+describe QueryParser do
+  describe "parse_query" do
+    it "should use the database's stopper" do
+      xdb = XapianDb.new(:stopper => :french)
+      qp = QueryParser.new(:database => xdb)
+      terms = qp.parse_query("avec and").terms.collect { |t| t.term }
+      terms.should_not include "Zavec"
+      terms.should include "Zand"
+    end
+
+    it "should use the database's stemmer" do
+      xdb = XapianDb.new(:stemmer => :french)
+      qp = QueryParser.new(:database => xdb)
+      terms = qp.parse_query("contournait fishing").terms.collect { |t| t.term }
+      terms.should include "Zcontourn"
+      terms.should_not include "Zfish"
+    end
+
+    it "should use the :fields option to set field names" do
+      qp = QueryParser.new(:fields => [:name, :age])
+      terms = qp.parse_query("name:john age:30").terms.collect { |t| t.term }
+      terms.should include "XNAMEjohn"
+      terms.should_not include "john"
+      terms.should include "XAGE30"
+      terms.should_not include "30"
+    end
+
+    it "should use the database's field names as prefixes" do
+      xdb = XapianDb.new(:fields => [:name], :stemmer => :none)
+      qp = QueryParser.new(:database => xdb)
+      terms = qp.parse_query("name:john").terms.collect { |t| t.term }
+      terms.should include "XNAMEjohn"
+      terms.should_not include "john"
+    end
+
+  end
+
+end
+

data/spec/stopper_factory_spec.rb

@@ -0,0 +1,57 @@
+require 'xapian'
+require 'lib/xapian_fu.rb'
+include XapianFu
+require 'fileutils'
+
+describe StopperFactory do
+  describe "stopper_for" do
+    it "should return a SimpleStopper loaded with the given languages stop words" do
+      stopper = StopperFactory.stopper_for(:english)
+      stopper.should be_a_kind_of Xapian::SimpleStopper
+      stopper.call("and").should be_true
+      stopper.call("theremin").should_not be_true
+    end
+
+    it "should return the given stopper unmodified if given a Xapian::Stopper object" do
+      stopper = Xapian::SimpleStopper.new
+      StopperFactory.stopper_for(stopper).should === stopper
+    end
+  end
+
+  describe "stop_words_for" do
+
+    it "should return an array of words for the given language" do
+      words = StopperFactory.stop_words_for(:english)
+      words.should be_a_kind_of Array
+      words.should_not be_empty
+      words.should include 'and'
+      words.should include "they're"
+    end
+    
+    %w(danish dutch english finnish french german hungarian italian norwegian portuguese russian spanish swedish).each do |lang|
+      describe lang do
+        it "should return an array of words" do
+          words = StopperFactory.stop_words_for(lang.to_sym)
+          words.should_not be_empty
+        end
+
+        it "should return an array with no empty strings, nils or pipes" do
+          StopperFactory.stop_words_for(lang.to_sym).should_not include ''
+          StopperFactory.stop_words_for(lang.to_sym).should_not include nil
+          StopperFactory.stop_words_for(lang.to_sym).should_not include '|'
+        end
+      end
+    end
+
+  
+    it "should raise a UnsupportedStopperLanguage error if there is no data for the given language" do
+      Proc.new { StopperFactory.stop_words_for(:no_existy) }.should raise_error UnsupportedStopperLanguage
+    end
+
+    it "should return characters in utf8" do
+      words = StopperFactory.stop_words_for(:russian)
+      words.should include "человек"
+    end
+
+  end
+end