xapian_db 0.3.1 → 0.3.2

data/lib/xapian_db/database.rb

@@ -1,50 +1,69 @@
 # encoding: utf-8
 
-# Singleton class representing a Xapian database.
 # @author Gernot Kogler
 
 module XapianDb
 
-  # Base class for a Xapian database.    
+  # Base class for a Xapian database
   class Database
-    attr_reader :reader    
-    
+
+    # A readable xapian database (see http://xapian.org/docs/apidoc/html/classXapian_1_1Database.html)
+    attr_reader :reader
+
     # Size of the database (number of docs)
+    # @return [Integer] The number of docs in the database
     def size
       reader.doccount
     end
-    
+
     # Store a Xapian document
+    # @param [Xapian::Document] doc A Xapian document (see http://xapian.org/docs/sourcedoc/html/classXapian_1_1Document.html).
+    #   While you can pass any valid xapian document, you might want to use the {XapianDb::Indexer} to build a xapian doc
     def store_doc(doc)
       # We always replace; Xapian adds the document automatically if
       # it is not found
       writer.replace_document("Q#{doc.data}", doc)
     end
 
-    # Delete a document by a unique term; this method is used by the
+    # Delete a document identified by a unique term; this method is used by the
     # orm adapters
+    # @param [String] term A term that uniquely identifies a document
     def delete_doc_with_unique_term(term)
       writer.delete_document("Q#{term}")
       true
     end
 
-    # Delete all docs of a specific class 
+    # Delete all docs of a specific class
+    # @param [Class] klass A class that has a {XapianDb::DocumentBlueprint} configuration
     def delete_docs_of_class(klass)
       writer.delete_document("C#{klass}")
       true
     end
-       
+
     # Perform a search
-    def search(expression)
+    # @param [String] expression A valid search expression.
+    # @param [Hash] options
+    # @option options [Integer] :per_page (10) How many docs per page?
+    # @example Simple Query
+    #   resultset = db.search("foo")
+    # @example Wildcard Query
+    #   resultset = db.search("fo*")
+    # @example Boolean Query
+    #   resultset = db.search("foo or baz")
+    # @example Field Query
+    #   resultset = db.search("name:foo")
+    # @return [XapianDb::Resultset] The resultset
+    def search(expression, options={})
+      opts          = {:per_page => 10}.merge(options)
       @query_parser ||= QueryParser.new(self)
-      query = @query_parser.parse(expression)
-      enquiry = Xapian::Enquire.new(reader)
+      query         = @query_parser.parse(expression)
+      enquiry       = Xapian::Enquire.new(reader)
       enquiry.query = query
-      Resultset.new(enquiry)
+      Resultset.new(enquiry, opts)
     end
-           
+
   end
-  
+
   # In Memory database
   class InMemoryDatabase < Database
 
@@ -52,51 +71,65 @@ module XapianDb
       @writer ||= Xapian::inmemory_open
       @reader = @writer
     end
-    
+
+    # Get the writer to write to the database
+    # @return [Xapian::WritableDatabase] A xapian database that is writable (see http://xapian.org/docs/apidoc/html/classXapian_1_1WritableDatabase.html)
     def writer
       @writer
     end
 
-    # Commit all pending changes 
+    # Commit all pending changes
     def commit
       # Nothing to do for an in memory database
     end
-    
+
   end
 
   # Persistent database on disk
   class PersistentDatabase < Database
-        
+
+    # Constructor
+    # @param [Hash] options Options for the persistent database
+    # @option options [String] :path A path to the file system
+    # @option options [Boolean] :create Should the database be created? <b>Will overwrite an existing database if true!</b>
+    # @example Force the creation of a database. Will overwrite an existing database
+    #   db = XapianDb::PersistentDatabase.new(:path => "/tmp/mydb", :create => true)
+    # @example Open an existing database. The database must exist
+    #   db = XapianDb::PersistentDatabase.new(:path => "/tmp/mydb", :create => false)
     def initialize(options)
       @path    = options[:path]
       @db_flag = options[:create] ? Xapian::DB_CREATE_OR_OVERWRITE : Xapian::DB_OPEN
       if options[:create]
-        # make sure the path exists; Xapian will not create the necessary directories 
+        # make sure the path exists; Xapian will not create the necessary directories
         FileUtils.makedirs @path
         @writer = Xapian::WritableDatabase.new(@path, @db_flag)
       end
       @reader = Xapian::Database.new(@path)
     end
-    
-    # Get the readable instance of the database
+
+    # Get the readable instance of the database. On each access this method reopens the readable database
+    # to make sure you get the latest changes to the index
+    # @return [Xapian::Database] A readable xapian database (see http://xapian.org/docs/apidoc/html/classXapian_1_1Database.html)
     def reader
       # Always reopen the readable database so we get live index data
       # TODO: make this configurable
       @reader.reopen
       @reader
     end
-    
-    # The writer is instantiated layzily to avoid a permanent write lock on the database
+
+    # The writer is instantiated layzily to avoid a permanent write lock on the database. Please note that
+    # you will get locking exceptions if you open the same database multiple times and access the writer
+    # in more than one instance!
+    # @return [Xapian::WritableDatabase] A xapian database that is writable (see http://xapian.org/docs/apidoc/html/classXapian_1_1WritableDatabase.html)
     def writer
       @writer ||= Xapian::WritableDatabase.new(@path, @db_flag)
     end
-   
-    # Commit all pending changes 
+
+    # Commit all pending changes
     def commit
       writer.commit
-      reader.reopen
     end
-    
+
   end
-  
+
 end

data/lib/xapian_db/document_blueprint.rb

@@ -1,19 +1,32 @@
 # encoding: utf-8
 
-# A document blueprint describes the mapping of an object to a Xapian document
-# for a given class.
-# @author Gernot Kogler
-
 module XapianDb
-    
+
+  # A document blueprint describes the mapping of an object to a Xapian document
+  # for a given class.
+  # @example A simple document blueprint configuration for the class Person
+  #   XapianDb::DocumentBlueprint.setup(Person) do |blueprint|
+  #     # Our Person class has a method lang_cd. We use this method to
+  #     # index each person with its language
+  #     blueprint.language_method :lang_cd
+  #     blueprint.attribute       :name, :weight => 10
+  #     blueprint.attribute       :first_name
+  #     blueprint.index           :remarks
+  #   end
+  # @author Gernot Kogler
   class DocumentBlueprint
 
-    # ---------------------------------------------------------------------------------   
+    # ---------------------------------------------------------------------------------
     # Singleton methods
-    # ---------------------------------------------------------------------------------   
+    # ---------------------------------------------------------------------------------
     class << self
 
-      # Configure the blueprint for a class
+      # Configure the blueprint for a class.
+      # Available options:
+      # - language_method (see {#language_method} for details)
+      # - adapter (see {#adapter} for details)
+      # - attribute (see {#attribute} for details)
+      # - index (see {#index} for details)
       def setup(klass, &block)
         @blueprints ||= {}
         blueprint = DocumentBlueprint.new
@@ -24,46 +37,54 @@ module XapianDb
         @adapter.add_class_helper_methods_to klass
         @searchable_prefixes = nil # force rebuild of the searchable prefixes
       end
-      
+
       # Get the blueprint for a class
+      # @return [DocumentBlueprint]
       def blueprint_for(klass)
         @blueprints[klass] if @blueprints
       end
 
       # Return an array of all configured text methods in any blueprint
+      # @return [Array<String>] All searchable prefixes
       def searchable_prefixes
         return [] unless @blueprints
         return @searchable_prefixes unless @searchable_prefixes.nil?
         prefixes = []
-        @blueprints.each do |klass, blueprint|
+        @blueprints.values.each do |blueprint|
           prefixes << blueprint.searchable_prefixes
         end
         @searchable_prefixes = prefixes.flatten.compact.uniq
       end
-            
+
     end
 
-    # ---------------------------------------------------------------------------------   
+    # ---------------------------------------------------------------------------------
     # Instance methods
-    # ---------------------------------------------------------------------------------       
+    # ---------------------------------------------------------------------------------
+
+    # Set / get the indexer
+    # @return [XapianDb::Indexer]
     attr_accessor :indexer
-    
+
     # Return an array of all configured text methods in this blueprint
+    # @return [Array<String>] All searchable prefixes
     def searchable_prefixes
-      @prefixes ||= indexed_methods.map{|method_name, options| method_name}
+      @prefixes ||= indexed_methods.keys
     end
-    
+
     # Lazily build and return a module that implements accessors for each field
+    # @return [Module] A module containing all accessor methods
     def accessors_module
       return @accessors_module unless @accessors_module.nil?
       @accessors_module = Module.new
-      
+
+      # Add the accessor for the indexed class
       @accessors_module.instance_eval do
         define_method :domain_class do
           self.values[0].value
         end
       end
-      
+
       @attributes.each_with_index do |field, index|
         @accessors_module.instance_eval do
           define_method field do
@@ -72,50 +93,81 @@ module XapianDb
         end
       end
       # Let the adapter add its document helper methods (if any)
-      adapter = XapianDb::Config.adapter || XapianDb::Adapters::GenericAdapter
+      adapter = @adapter || XapianDb::Config.adapter || XapianDb::Adapters::GenericAdapter
       adapter.add_doc_helper_methods_to(@accessors_module)
       @accessors_module
     end
-          
-    # ---------------------------------------------------------------------------------   
+
+    # ---------------------------------------------------------------------------------
     # Blueprint DSL methods
-    # ---------------------------------------------------------------------------------   
-    attr_reader :adapter, :attributes, :indexed_methods
-        
+    # ---------------------------------------------------------------------------------
+
+    # The name of the method that returns a Xapian compliant language code. The
+    # configured class must implement this method.
+    attr_reader :lang_method
+
+    # Collection of the configured attribute methods
+    # @return [Array<Symbol>] The names of the configured attribute methods
+    attr_reader :attributes
+
+    # Collection of the configured index methods
+    # @return [Hash<Symbol, IndexOptions>] A hashtable containing all index methods as
+    #   keys and IndexOptions as values
+    attr_reader :indexed_methods
+
+    # Set / read a custom adapter.
+    # Use this configuration option if you need a specific adapter for an indexed class.
+    # If set, it overrides the globally configured adapter (see also {Config#adapter})
+    attr_accessor :adapter
+
     # Construct the blueprint
     def initialize
       @attributes = []
       @indexed_methods = {}
     end
-    
-    # Set a custom adapter for this blueprint
-    def adapter=(adapter)
-      @adapter = adapter
+
+    # Set the name of the method to get the language for an indexed object
+    # @param [Symbol] lang The method name. The method must return a language supported
+    #   by Xapian (see http://xapian.org/docs/apidoc/html/classXapian_1_1Stem.html for supported languages)
+    def language_method(lang)
+      @lang_method = lang
     end
-    
-    # Add an attribute to the list
-    # TODO: Make sure the name does not collide with a method name of Xapian::Document since
-    # we generate methods in the documents for all defined fields
+
+    # Add an attribute to the blueprint. Attributes will be stored in the xapian documents an can be
+    # accessed from a search result.
+    # @param [String] name The name of the method that delivers the value for the attribute
+    # @param [Hash] options
+    # @option options [Integer] :weight (1) The weight for this attribute.
+    # @option options [Boolean] :index (true) Should the attribute be indexed?
+    # @todo Make sure the name does not collide with a method name of Xapian::Document since
     def attribute(name, options={})
       opts = {:index => true}.merge(options)
       @attributes << name
       self.index(name, opts) if opts[:index]
     end
 
-    # Add an indexed value to the list
+    # Add an indexed value to the blueprint. Indexed values are not accessible from a search result.
+    # @param [String] name The name of the method that delivers the value for the index
+    # @param [Hash] options
+    # @option options [Integer] :weight (1) The weight for this indexed value
     def index(name, options={})
       @indexed_methods[name] = IndexOptions.new(options)
     end
 
-    # Options for an indexed text
-    class IndexOptions      
+    # Options for an indexed method
+    class IndexOptions
+
+      # The weight for the indexed value
       attr_accessor :weight
-      
+
+      # Constructor
+      # @param [Hash] options
+      # @option options [Integer] :weight (1) The weight for the indexed value
       def initialize(options)
         @weight = options[:weight] || 1
       end
     end
-          
+
   end
-  
+
 end

data/lib/xapian_db/index_writers/direct_writer.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-# This writer writes changes directly to the open database. 
+# This writer writes changes directly to the open database.
 # Use the direct writer only for single process environments
 # (one single rails app server, e.g. one mongrel).
 # For multi process environemnts you should use a writer that
@@ -9,12 +9,13 @@
 
 module XapianDb
   module IndexWriters
-    
+
     class DirectWriter
-      
+
       class << self
-        
+
         # Update an object in the index
+        # @param [Object] obj An instance of a class with a blueprint configuration
         def index(obj)
           blueprint = XapianDb::DocumentBlueprint.blueprint_for(obj.class)
           doc = blueprint.indexer.build_document_for(obj)
@@ -23,30 +24,41 @@ module XapianDb
         end
 
         # Remove an object from the index
+        # @param [Object] obj An instance of a class with a blueprint configuration
         def unindex(obj)
           XapianDb.database.delete_doc_with_unique_term(obj.xapian_id)
           XapianDb.database.commit
         end
 
         # Reindex all objects of a given class
-        def reindex_class(klass)
+        # @param [Class] klass The class to reindex
+        # @param [Hash] options Options for reindexing
+        # @option options [Boolean] :verbose (false) Should the reindexing give status informations?
+        def reindex_class(klass, options={})
+          opts = {:verbose => false}.merge(options)
           # First, delete all docs of this class
           XapianDb.database.delete_docs_of_class(klass)
           blueprint = XapianDb::DocumentBlueprint.blueprint_for(klass)
-          obj_count = klass.count
-          puts "Reindexing #{obj_count} objects..."
-          pbar = ProgressBar.new("Status", obj_count)
-          klass.all.each do |obj| 
+          show_progressbar = false
+          if opts[:verbose]
+            if defined?(ProgressBar)
+              show_progressbar = true
+            end
+            obj_count = klass.count
+            puts "Reindexing #{obj_count} objects..."
+            pbar = ProgressBar.new("Status", obj_count) if show_progressbar
+          end
+          klass.all.each do |obj|
             doc = blueprint.indexer.build_document_for(obj)
             XapianDb.database.store_doc(doc)
-            pbar.inc
+            pbar.inc if show_progressbar
           end
           XapianDb.database.commit
         end
-        
+
       end
-      
-    end 
-    
+
+    end
+
   end
 end

data/lib/xapian_db/indexer.rb

@@ -1,18 +1,39 @@
 # encoding: utf-8
 
-# The indexer creates a Xapian::Document from a configured object
-# @author Gernot Kogler
-
 module XapianDb
-  
+
+  # The indexer creates a Xapian::Document from an object. They object must be an instance
+  # of a class that has a blueprint configuration.
+  # @author Gernot Kogler
   class Indexer
-    
+
+    # Supported languages and mapping to the stemmer to use
+    LANGUAGE_MAP = {:da => :danish,
+                    :nl => :dutch,
+                    :en => :english,
+                    :fi => :finnish,
+                    :fr => :french,
+                    :de => :german2, # Normalises umlauts and ß
+                    :hu => :hungarian,
+                    :it => :italian,
+                    :nb => :norwegian,
+                    :nn => :norwegian,
+                    :no => :norwegian,
+                    :pt => :portuguese,
+                    :ro => :romanian,
+                    :ru => :russian,
+                    :es => :spanish,
+                    :sv => :swedish,
+                    :tr => :turkish}
+    # Constructor
+    # @param [XapianDb::DocumentBlueprint] document_blueprint The blueprint to use
     def initialize(document_blueprint)
       @document_blueprint = document_blueprint
     end
-    
-    # Build the doc for an object. The object must respond to 'xapian_id'.
+
+    # Build the document for an object. The object must respond to 'xapian_id'.
     # The configured adapter should implement this method.
+    # @return [Xapian::Document] The xapian document (see http://xapian.org/docs/sourcedoc/html/classXapian_1_1Document.html)
     def build_document_for(obj)
       @obj = obj
       @blueprint = DocumentBlueprint.blueprint_for(@obj.class)
@@ -22,15 +43,15 @@ module XapianDb
       index_text
       @xapian_doc
     end
-    
+
     private
-    
+
     # Store all configured fields
     def store_fields
 
       # We store the class name of the object at position 0
       @xapian_doc.add_value(0, @obj.class.name)
-      
+
       pos = 1
       @blueprint.attributes.each do |attribute, options|
         value = @obj.send(attribute)
@@ -38,24 +59,20 @@ module XapianDb
         pos += 1
       end
     end
-    
+
     # Index all configured text methods
     def index_text
-      term_generator = Xapian::TermGenerator.new()
+      term_generator = Xapian::TermGenerator.new
       term_generator.document = @xapian_doc
-      # TODO: make this configurable globally and per document 
-      # (retrieve the language from the object, if configured)
-      stemmer = Xapian::Stem.new("english")
-      term_generator.stemmer = stemmer
+      term_generator.stemmer  = get_stemmer
       # TODO: Configure and enable these features
       # tg.stopper = stopper if stopper
-      # tg.stemmer = stemmer
       # tg.set_flags Xapian::TermGenerator::FLAG_SPELLING if db.spelling
 
       # Always index the class and the primary key
       @xapian_doc.add_term("C#{@obj.class}")
       @xapian_doc.add_term("Q#{@obj.xapian_id}")
-      
+
       @blueprint.indexed_methods.each do |method, options|
         value = @obj.send(method)
         unless value.nil?
@@ -69,7 +86,21 @@ module XapianDb
         end
       end
     end
-    
+
+    private
+
+    # Configure the stemmer to use
+    def get_stemmer
+      # Do we have a language config on the blueprint?
+      if @blueprint.lang_method
+        lang = @obj.send(@blueprint.lang_method)
+        return Xapian::Stem.new(LANGUAGE_MAP[lang.to_sym].to_s) if lang && LANGUAGE_MAP.has_key?(lang.to_sym)
+      end
+      # Do we have a global stemmer?
+      return XapianDb::Config.stemmer if XapianDb::Config.stemmer
+      return Xapian::Stem.new("none") # No language config
+    end
+
   end
-  
+
 end

data/lib/xapian_db/query_parser.rb

@@ -1,34 +1,37 @@
 # encoding: utf-8
 
-# Parse a query expression and convert it to Xapian Query arguments
-# @author Gernot Kogler
-
 module XapianDb
-  
+
+  # Parse a query expression and create a xapian query object
+  # @author Gernot Kogler
   class QueryParser
-    
+
+    # Constructor
+    # @param [XapianDb::Database] database The database to query
     def initialize(database)
       @db = database
-      
+
       # Set the parser options
       @query_flags = 0
-      @query_flags |= Xapian::QueryParser::FLAG_WILDCARD # enable wildcards
-      @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN
-      @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE
+      @query_flags |= Xapian::QueryParser::FLAG_WILDCARD         # enable wildcards
+      @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN          # enable boolean operators
+      @query_flags |= Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE # enable case insensitive boolean operators
     end
-    
+
+    # Parse an expression
+    # @return [Xapian::Query] The query object (see http://xapian.org/docs/apidoc/html/classXapian_1_1Query.html)
     def parse(expression)
       parser            = Xapian::QueryParser.new
       parser.database   = @db.reader
       parser.default_op = Xapian::Query::OP_AND # Could be made configurable
       # TODO: Setup stopper, stemmer, defaults and fields
-      
-      # Add the searchable prefixes to allow searches by field 
+
+      # Add the searchable prefixes to allow searches by field
       # (like "name:Kogler")
       XapianDb::DocumentBlueprint.searchable_prefixes.each{|prefix| parser.add_prefix(prefix.to_s.downcase, "X#{prefix.to_s.upcase}") }
       parser.parse_query(expression, @query_flags)
     end
-    
+
   end
-  
+
 end

data/lib/xapian_db/railtie.rb

@@ -1,12 +1,12 @@
 # encoding: utf-8
 
-# Configuration for a rails app
-# @author Gernot Kogler
-
 require 'xapian_db'
 require 'rails'
 
 module XapianDb
+
+  # Configuration for a rails app
+  # @author Gernot Kogler
   class Railtie < ::Rails::Railtie
 
     config.before_configuration do
@@ -25,7 +25,7 @@ module XapianDb
         adapter = :active_record
         writer  = :direct
       end
-      
+
       # Do the configuration
       XapianDb::Config.setup do |config|
         if database_path == ":memory:"
@@ -33,11 +33,12 @@ module XapianDb
         else
           config.database database_path
         end
-        config.adapter adapter.to_sym  
+        config.adapter adapter.to_sym
         config.writer writer.to_sym
+        config.language(env_config["language"]) if env_config["language"]
       end
-      
+
     end
-    
+
   end
 end