xapian_db 0.3.2 → 0.3.3

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+##0.3.3 (December 13th, 2010)
+
+Features:
+
+  - Support for multi language stop words. The implementation was inspired by John Leachs xapian-fu gem
+  - Support for query spelling correction (similar to Google's 'did you mean...'). This feature is only
+    available for persistent databases (due to a limitation of Xapian)
+
+Changes:
+
+  - Languages must be configured by the iso language code (:en, :de, ...). No more support for the english
+    language names (:english, :german, ...)
+  - Reduced the memory footprint when reindexing large tables
+
 ##0.3.2 (December 10th, 2010)
 
 Features:

data/README.rdoc

@@ -114,8 +114,7 @@ you can configure the blueprint to use the language of the object when indexing:
     blueprint.language_method :language
   end
 
-Don't worry if you have languages in your database that are not supported by Xapian. If the language is not supported, XapianDb
-will fall back to the global language configuration or none, if you haven't configured one.
+The method must return the iso code for the language (:en, :de, ...) as a symbol or a string. Don't worry if you have languages in your database that are not supported by Xapian. If the language is not supported, XapianDb will fall back to the global language configuration or none, if you haven't configured one.
 
 You can place this configuration anywhere, e.g. in an initializer.
 
@@ -145,7 +144,13 @@ You can query attributes:
 
 <code>XapianDb.search</code> returns a resultset object. You can access the number of hits directly:
 
-  result.size # Very fast, does not load the resulting documents
+  results.size # Very fast, does not load the resulting documents
+
+If you use a persistent database, the resultset may contain a spelling correction:
+
+  # Assuming you have at least one document containing "mouse"
+  results = XapianDb.search("moose")
+  results.corrected_query # "mouse"
 
 To access the found documents, get a page from the resultset:
 
@@ -162,7 +167,6 @@ Now you can access the documents:
 
 == What to expect from future releases
 
-* multi language support (spelling correction, stop words)
 * facet support
 * will_paginate support
 * asynchronous index writer based on {resque}[https://github.com/defunkt/resque] for production environments

data/lib/xapian_db/config.rb

@@ -30,7 +30,7 @@ module XapianDb
       end
 
       # Install delegates for the config instance variables
-      [:database, :adapter, :writer, :stemmer].each do |attr|
+      [:database, :adapter, :writer, :stemmer, :stopper].each do |attr|
         define_method attr do
           @config.nil? ? nil : @config.instance_variable_get("@_#{attr}")
         end
@@ -42,7 +42,7 @@ module XapianDb
     # ---------------------------------------------------------------------------------
 
     #
-    attr_reader :_database, :_adapter, :_writer, :_stemmer
+    attr_reader :_database, :_adapter, :_writer, :_stemmer, :_stopper
 
     # Set the global database to use
     # @param [String] path The path to the database. Either apply a file sytem path or :memory
@@ -87,20 +87,17 @@ module XapianDb
       @_writer = XapianDb::IndexWriters.const_get("#{camelize(type.to_s)}Writer")
     end
 
-    # Set the language
-    # @param [Symbol] lang The language; either apply the english name of the language
-    #   or the two letter IS639 code
-    # @example Use the english name of the language
-    #   XapianDb::Config.setup do |config|
-    #     config.language :german
-    #   end
-    # @example Use the iso code of the language
+    # Set the language.
+    # @param [Symbol] lang The language; apply the two letter ISO639 code for the language
+    # @example
     #   XapianDb::Config.setup do |config|
     #     config.language :de
     #   end
-    # see http://xapian.org/docs/apidoc/html/classXapian_1_1Stem.html for supported languages
+    # see {LANGUAGE_MAP} for supported languages
     def language(lang)
-      @_stemmer = Xapian::Stem.new(lang.to_s)
+      key = lang.to_sym
+      @_stemmer = XapianDb::Repositories::Stemmer.stemmer_for key
+      key == :none ? @_stopper = nil : @_stopper = XapianDb::Repositories::Stopper.stopper_for(key)
     end
 
     private

data/lib/xapian_db/database.rb

@@ -59,6 +59,7 @@ module XapianDb
       query         = @query_parser.parse(expression)
       enquiry       = Xapian::Enquire.new(reader)
       enquiry.query = query
+      opts[:corrected_query] = @query_parser.corrected_query
       Resultset.new(enquiry, opts)
     end
 

data/lib/xapian_db/document_blueprint.rb

@@ -30,7 +30,6 @@ module XapianDb
       def setup(klass, &block)
         @blueprints ||= {}
         blueprint = DocumentBlueprint.new
-        blueprint.indexer = Indexer.new(blueprint)
         yield blueprint if block_given? # configure the blueprint through the block
         @blueprints[klass] = blueprint
         @adapter = blueprint.adapter || XapianDb::Config.adapter || Adapters::GenericAdapter
@@ -62,10 +61,6 @@ module XapianDb
     # Instance methods
     # ---------------------------------------------------------------------------------
 
-    # Set / get the indexer
-    # @return [XapianDb::Indexer]
-    attr_accessor :indexer
-
     # Return an array of all configured text methods in this blueprint
     # @return [Array<String>] All searchable prefixes
     def searchable_prefixes
@@ -102,7 +97,7 @@ module XapianDb
     # Blueprint DSL methods
     # ---------------------------------------------------------------------------------
 
-    # The name of the method that returns a Xapian compliant language code. The
+    # The name of the method that returns an iso language code. The
     # configured class must implement this method.
     attr_reader :lang_method
 
@@ -127,8 +122,8 @@ module XapianDb
     end
 
     # Set the name of the method to get the language for an indexed object
-    # @param [Symbol] lang The method name. The method must return a language supported
-    #   by Xapian (see http://xapian.org/docs/apidoc/html/classXapian_1_1Stem.html for supported languages)
+    # @param [Symbol] lang The method name. The method must return an iso language code (:en, :de, ...)
+    #   see LANGUAGE_MAP for the supported lanugaes
     def language_method(lang)
       @lang_method = lang
     end

data/lib/xapian_db/index_writers/direct_writer.rb

@@ -18,7 +18,8 @@ module XapianDb
         # @param [Object] obj An instance of a class with a blueprint configuration
         def index(obj)
           blueprint = XapianDb::DocumentBlueprint.blueprint_for(obj.class)
-          doc = blueprint.indexer.build_document_for(obj)
+          indexer   = XapianDb::Indexer.new(XapianDb.database, blueprint)
+          doc       = indexer.build_document_for(obj)
           XapianDb.database.store_doc(doc)
           XapianDb.database.commit
         end
@@ -39,19 +40,25 @@ module XapianDb
           # First, delete all docs of this class
           XapianDb.database.delete_docs_of_class(klass)
           blueprint = XapianDb::DocumentBlueprint.blueprint_for(klass)
+          indexer   = XapianDb::Indexer.new(XapianDb.database, blueprint)
           show_progressbar = false
+          obj_count = klass.count
           if opts[:verbose]
             if defined?(ProgressBar)
               show_progressbar = true
             end
-            obj_count = klass.count
             puts "Reindexing #{obj_count} objects..."
             pbar = ProgressBar.new("Status", obj_count) if show_progressbar
           end
-          klass.all.each do |obj|
-            doc = blueprint.indexer.build_document_for(obj)
-            XapianDb.database.store_doc(doc)
-            pbar.inc if show_progressbar
+
+          # Process the objects in batches to reduce the memory footprint
+          nr_of_batches = (obj_count / 1000) + 1
+          nr_of_batches.times do |batch|
+            klass.all(:offset => batch * 1000, :limit => 1000) .each do |obj|
+              doc = indexer.build_document_for(obj)
+              XapianDb.database.store_doc(doc)
+              pbar.inc if show_progressbar
+            end
           end
           XapianDb.database.commit
         end

data/lib/xapian_db/indexer.rb

@@ -7,28 +7,11 @@ module XapianDb
   # @author Gernot Kogler
   class Indexer
 
-    # Supported languages and mapping to the stemmer to use
-    LANGUAGE_MAP = {:da => :danish,
-                    :nl => :dutch,
-                    :en => :english,
-                    :fi => :finnish,
-                    :fr => :french,
-                    :de => :german2, # Normalises umlauts and ß
-                    :hu => :hungarian,
-                    :it => :italian,
-                    :nb => :norwegian,
-                    :nn => :norwegian,
-                    :no => :norwegian,
-                    :pt => :portuguese,
-                    :ro => :romanian,
-                    :ru => :russian,
-                    :es => :spanish,
-                    :sv => :swedish,
-                    :tr => :turkish}
     # Constructor
+    # @param [XapianDb::Database] database The database to use (needed to build a spelling index)
     # @param [XapianDb::DocumentBlueprint] document_blueprint The blueprint to use
-    def initialize(document_blueprint)
-      @document_blueprint = document_blueprint
+    def initialize(database, document_blueprint)
+      @database, @document_blueprint = database, document_blueprint
     end
 
     # Build the document for an object. The object must respond to 'xapian_id'.
@@ -62,12 +45,18 @@ module XapianDb
 
     # Index all configured text methods
     def index_text
+      setup_language_helpers
       term_generator = Xapian::TermGenerator.new
+      term_generator.database = @database.writer
       term_generator.document = @xapian_doc
-      term_generator.stemmer  = get_stemmer
-      # TODO: Configure and enable these features
-      # tg.stopper = stopper if stopper
-      # tg.set_flags Xapian::TermGenerator::FLAG_SPELLING if db.spelling
+      if @stemmer
+        term_generator.stemmer  = @stemmer
+        term_generator.stopper  = @stopper unless @stopper.nil?
+        # Enable the creation of a spelling index if the database is not in memory
+        if @database.is_a? XapianDb::PersistentDatabase
+          term_generator.set_flags Xapian::TermGenerator::FLAG_SPELLING if @database.is_a? XapianDb::PersistentDatabase
+        end
+      end
 
       # Always index the class and the primary key
       @xapian_doc.add_term("C#{@obj.class}")
@@ -89,16 +78,22 @@ module XapianDb
 
     private
 
-    # Configure the stemmer to use
-    def get_stemmer
+    # Configure the stemmer and stopper to use
+    def setup_language_helpers
       # Do we have a language config on the blueprint?
       if @blueprint.lang_method
         lang = @obj.send(@blueprint.lang_method)
-        return Xapian::Stem.new(LANGUAGE_MAP[lang.to_sym].to_s) if lang && LANGUAGE_MAP.has_key?(lang.to_sym)
+        if lang && LANGUAGE_MAP.has_key?(lang.to_sym)
+          @stemmer = XapianDb::Repositories::Stemmer.stemmer_for lang.to_sym
+          @stopper = XapianDb::Repositories::Stopper.stopper_for lang.to_sym
+          return
+        end
       end
-      # Do we have a global stemmer?
-      return XapianDb::Config.stemmer if XapianDb::Config.stemmer
-      return Xapian::Stem.new("none") # No language config
+
+      # Use the global config
+      @stemmer = XapianDb::Config.stemmer
+      @stopper = XapianDb::Config.stopper
+
     end
 
   end

data/lib/xapian_db/query_parser.rb

@@ -6,6 +6,10 @@ module XapianDb
   # @author Gernot Kogler
   class QueryParser
 
+    # The spelling corrected query (if a language is configured)
+    # @return [String]
+    attr_reader :corrected_query
+
     # Constructor
     # @param [XapianDb::Database] database The database to query
     def initialize(database)
@@ -13,9 +17,10 @@ module XapianDb
 
       # Set the parser options
       @query_flags = 0
-      @query_flags |= Xapian::QueryParser::FLAG_WILDCARD         # enable wildcards
-      @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN          # enable boolean operators
-      @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE # enable case insensitive boolean operators
+      @query_flags |= Xapian::QueryParser::FLAG_WILDCARD            # enable wildcards
+      @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN             # enable boolean operators
+      @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE    # enable case insensitive boolean operators
+      @query_flags |= Xapian::QueryParser::FLAG_SPELLING_CORRECTION # enable spelling corrections
     end
 
     # Parse an expression
@@ -24,12 +29,18 @@ module XapianDb
       parser            = Xapian::QueryParser.new
       parser.database   = @db.reader
       parser.default_op = Xapian::Query::OP_AND # Could be made configurable
-      # TODO: Setup stopper, stemmer, defaults and fields
+      if XapianDb::Config.stemmer
+        parser.stemmer           = XapianDb::Config.stemmer
+        parser.stemming_strategy = Xapian::QueryParser::STEM_SOME
+        parser.stopper           = XapianDb::Config.stopper
+      end
 
       # Add the searchable prefixes to allow searches by field
       # (like "name:Kogler")
       XapianDb::DocumentBlueprint.searchable_prefixes.each{|prefix| parser.add_prefix(prefix.to_s.downcase, "X#{prefix.to_s.upcase}") }
-      parser.parse_query(expression, @query_flags)
+      query = parser.parse_query(expression, @query_flags)
+      @corrected_query = parser.get_corrected_query_string
+      query
     end
 
   end

data/lib/xapian_db/repositories/stemmer.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+module XapianDb
+  module Repositories
+
+    # The stemmer is a repository that manages stemmers for the supported
+    # languges
+    # @author Gernot Kogler
+    class Stemmer
+
+      class << self
+
+        # Get or build the stemmer for a language
+        # @param [Symbol, String] iso_cd The iso code for the language (:en, :de ...)
+        # @return [Xapian::Stem] The Stemmer for this lanugage
+        def stemmer_for(iso_cd)
+          @stemmers ||= {}
+          return nil if iso_cd.nil?
+          key = iso_cd.to_sym
+
+          # Do we already have a stemmer for this language?
+          return @stemmers[key] unless @stemmers[key].nil?
+
+          # Do we support this language?
+          unless (LANGUAGE_MAP.keys + [:none]).include?(key)
+            raise ArgumentError.new "Language #{iso_cd} is not supported by XapianDb (remember to use the language iso codes)"
+          end
+
+          # Let's build the stemmer
+          @stemmers[key] = Xapian::Stem.new(key.to_s)
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/xapian_db/repositories/stopper.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+module XapianDb
+  module Repositories
+
+    # The stopper is a repository that manages stoppers for the supported
+    # languges
+    # @author Gernot Kogler
+    class Stopper
+
+      class << self
+
+        # Get or build the stopper for a language
+        # @param [Symbol, String] iso_cd The iso code for the language (:en, :de ...)
+        # @return [Xapian::SimpleStopper] The Stopper for this lanugage
+        def stopper_for(iso_cd)
+          @stoppers ||= {}
+          return nil if iso_cd.nil?
+          key = iso_cd.to_sym
+
+          # Do we already have a stopper for this language?
+          return @stoppers[key] unless @stoppers[key].nil?
+
+          # Do we support this language?
+          unless (LANGUAGE_MAP.keys + [:none]).include?(key)
+            raise ArgumentError.new "Language #{iso_cd} is not supported by XapianDb (remember to use the language iso codes)"
+          end
+
+          # build the stopper
+          stopper = Xapian::SimpleStopper.new
+          stopwords_file = File.join(File.dirname(__FILE__), '../stopwords', "#{iso_cd}.txt")
+          open(stopwords_file, "r") do |file|
+            file.each do |word|
+              stopper.add word.chomp
+            end
+          end
+          @stoppers[key] = stopper
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/xapian_db/resultset.rb

@@ -15,15 +15,21 @@ module XapianDb
     # @return [Integer]
     attr_reader :size
 
+    # The spelling corrected query (if a language is configured)
+    # @return [String]
+    attr_reader :corrected_query
+
     # Constructor
     # @param [Xapian::Enquire] enquiry a Xapian query result (see http://xapian.org/docs/apidoc/html/classXapian_1_1Enquire.html)
     # @param [Hash] options
     # @option options [Integer] :per_page (10) How many docs per page?
+    # @option options [String] :corrected_query (nil) The spelling corrected query (if a language is configured)
     def initialize(enquiry, options)
       @enquiry = enquiry
       # By passing 0 as the max parameter to the mset method,
       # we only get statistics about the query, no results
-      @size     = enquiry.mset(0, 0).matches_estimated
+      @size            = enquiry.mset(0, 0).matches_estimated
+      @corrected_query = options[:corrected_query]
       @per_page = options[:per_page]
     end
 

data/lib/xapian_db/stopwords/README

@@ -0,0 +1,5 @@
+These stopword lists are from the Snowball library which is covered by the BSD License, with Copyright (c) 2001, Dr Martin Porter, and (for the Java developments) Copyright (c) 2002, Richard Boulton.
+
+http://snowball.tartarus.org/
+
+run ruby update.rb from this directory to update the stop word files.

data/lib/xapian_db/stopwords/da.txt

@@ -0,0 +1,94 @@
+og
+i
+jeg
+det
+at
+en
+den
+til
+er
+som
+på
+de
+med
+han
+af
+for
+ikke
+der
+var
+mig
+sig
+men
+et
+har
+om
+vi
+min
+havde
+ham
+hun
+nu
+over
+da
+fra
+du
+ud
+sin
+dem
+os
+op
+man
+hans
+hvor
+eller
+hvad
+skal
+selv
+her
+alle
+vil
+blev
+kunne
+ind
+når
+være
+dog
+noget
+ville
+jo
+deres
+efter
+ned
+skulle
+denne
+end
+dette
+mit
+også
+under
+have
+dig
+anden
+hende
+mine
+alt
+meget
+sit
+sine
+vor
+mod
+disse
+hvis
+din
+nogle
+hos
+blive
+mange
+ad
+bliver
+hendes
+været
+thi
+jer
+sådan