xapian_db 0.3.3 → 0.3.4

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+##0.3.4 (December 14th, 2010)
+
+Features:
+
+  - perform searches on indexed classes to scope the search to objects of a specific class
+  - specify multiple blueprint attributes and index methods in one statement (without specifying options)
+  - use blocks for complex attribute or index specifications
+  - changed the implementation of Resultset.size to get more accurate estimations
+  - changed the indexing of active_record or datamapper models when declared as attributes or indexes
+    in a blueprint (indexes now all attributes of the object instead of using to_s)
+
 ##0.3.3 (December 13th, 2010)
 
 Features:
@@ -15,6 +26,7 @@ Changes:
 ##0.3.2 (December 10th, 2010)
 
 Features:
+
   - Moved the per_page option from Resultset.paginate to Database.search
   - Added support for language settings (global and dynamic per object)
   - Added support for xapian stemmers
@@ -30,6 +42,7 @@ Bugfixes:
 ##0.3.0 (December 4th, 2010)
 
 Features:
+
   - Rails integration with configuration file (config/xapian_db.yml) and automatic setup
 
 ##0.2.0 (December 1st, 2010)

data/README.rdoc

@@ -116,6 +116,27 @@ you can configure the blueprint to use the language of the object when indexing:
 
 The method must return the iso code for the language (:en, :de, ...) as a symbol or a string. Don't worry if you have languages in your database that are not supported by Xapian. If the language is not supported, XapianDb will fall back to the global language configuration or none, if you haven't configured one.
 
+If you want to declare multiple attributes or indexes with default options, you can do this in one statement:
+
+  XapianDb::DocumentBlueprint.setup(Person) do |blueprint|
+    blueprint.attributes :name, :first_name, :profession
+    blueprint.index      :notes, :remarks, :cv
+  end
+
+Note that you cannot add options using this mass declaration syntax (e.g. <code>blueprint.attributes :name, :weight => 10, :first_name</code> is not valid).
+
+Use blocks for complex evaluations of attributes or indexed values:
+
+  XapianDb::DocumentBlueprint.setup(IndexedObject) do |blueprint|
+    blueprint.attribute :complex do
+      if @id == 1
+        "One"
+      else
+        "Not one"
+      end
+    end
+  end
+
 You can place this configuration anywhere, e.g. in an initializer.
 
 === Update the index
@@ -126,6 +147,12 @@ use the method <code>rebuild_xapian_index</code>:
 
   Person.rebuild_xapian_index
 
+To get info about the reindex process, use the verbose option:
+
+  Person.rebuild_xapian_index :verbose => true
+
+In verbose mode, XapianDb will use the progressbar gem if available.
+
 === Query the index
 
 A simple query looks like this:
@@ -134,12 +161,20 @@ A simple query looks like this:
 
 You can use wildcards and boolean operators:
 
-  results = XapianDb.search("Fo*" OR "Baz")
+  results = XapianDb.search("fo* or baz")
 
 You can query attributes:
 
   results = XapianDb.search("name:Foo")
 
+You can query objects of a specific class:
+
+  results = Person.search("name:Foo")
+
+If you want to override the default of 10 docs per page, pass the :per_page argument:
+
+  results = Person.search("name:Foo", :per_page => 20)
+
 === Process the results
 
 <code>XapianDb.search</code> returns a resultset object. You can access the number of hits directly:
@@ -150,17 +185,17 @@ If you use a persistent database, the resultset may contain a spelling correctio
 
   # Assuming you have at least one document containing "mouse"
   results = XapianDb.search("moose")
-  results.corrected_query # "mouse"
+  results.spelling_suggestion # "mouse"
 
 To access the found documents, get a page from the resultset:
 
-  page = result.paginate                              # Get the first page with 10 documents
-  page = result.paginate(:page => 2, :per_page => 20) # Get the second page page with documents 21-40
+  page = result.paginate            # Get the first page
+  page = result.paginate :page => 2 # Get the second page
 
 Now you can access the documents:
 
   doc = page.first
-  puts doc.domain_class       # Get the type of the indexed object, e.g. "Person"
+  puts doc.indexed_class      # Get the type of the indexed object as a string, e.g. "Person"
   puts doc.name               # We can access the configured attributes
   person = doc.indexed_object # Access the object behind this doc (lazy loaded)
 

data/lib/xapian_db/adapters/active_record_adapter.rb

@@ -16,7 +16,7 @@ module XapianDb
     #   in every found xapian document
     # @author Gernot Kogler
 
-     class ActiveRecordAdapter
+     class ActiveRecordAdapter < BaseAdapter
 
        class << self
 
@@ -24,6 +24,9 @@ module XapianDb
          # @param [Class] klass The class to add the helper methods to
          def add_class_helper_methods_to(klass)
 
+           # Add the helpers from the base class
+           super klass
+
            klass.instance_eval do
              # define the method to retrieve a unique key
              define_method(:xapian_id) do

data/lib/xapian_db/adapters/base_adapter.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+module XapianDb
+  module Adapters
+
+    # base class for all adapters.
+    # This adapter does the following:
+    # - adds the class method <code>search(expression)</code> to an indexed class
+    # @author Gernot Kogler
+
+     class BaseAdapter
+
+       class << self
+
+         # Implement the class helper methods
+         # @param [Class] klass The class to add the helper methods to
+         def add_class_helper_methods_to(klass)
+
+           klass.class_eval do
+
+             # Add a method to search models of this class
+             define_singleton_method(:search) do |expression|
+               XapianDb.database.search "indexed_class:#{klass.name.downcase} and (#{expression})"
+             end
+
+           end
+         end
+       end
+     end
+   end
+ end

data/lib/xapian_db/adapters/datamapper_adapter.rb

@@ -15,7 +15,7 @@ module XapianDb
     # - adds the instance method <code>indexed_object</code> to the module that will be included
     #   in every found xapian document
     # @author Gernot Kogler
-     class DatamapperAdapter
+     class DatamapperAdapter < BaseAdapter
 
        class << self
 
@@ -23,6 +23,9 @@ module XapianDb
          # @param [Class] klass The class to add the helper methods to
          def add_class_helper_methods_to(klass)
 
+           # Add the helpers from the base class
+           super klass
+
            klass.instance_eval do
              # define the method to retrieve a unique key
              define_method(:xapian_id) do

data/lib/xapian_db/adapters/generic_adapter.rb

@@ -10,7 +10,7 @@ module XapianDb
     # This adapter does the following:
     # - adds the instance method <code>xapian_id</code> to an indexed class
     # @author Gernot Kogler
-    class GenericAdapter
+    class GenericAdapter < BaseAdapter
 
       class << self
 
@@ -27,6 +27,10 @@ module XapianDb
         # @param [Class] klass The class to add the helper methods to
         def add_class_helper_methods_to(klass)
           raise "Unique key is not configured for generic adapter!" if @unique_key_block.nil?
+
+          # Add the helpers from the base class
+          super klass
+
           expression = @unique_key_block
           klass.instance_eval do
             define_method(:xapian_id) do

data/lib/xapian_db/database.rb

@@ -59,7 +59,8 @@ module XapianDb
       query         = @query_parser.parse(expression)
       enquiry       = Xapian::Enquire.new(reader)
       enquiry.query = query
-      opts[:corrected_query] = @query_parser.corrected_query
+      opts[:spelling_suggestion] = @query_parser.spelling_suggestion
+      opts[:db_size]             = self.size
       Resultset.new(enquiry, opts)
     end
 

data/lib/xapian_db/document_blueprint.rb

@@ -13,6 +13,15 @@ module XapianDb
   #     blueprint.attribute       :first_name
   #     blueprint.index           :remarks
   #   end
+  # @example A document blueprint configuration with a complex attribute for the class Person
+  #   XapianDb::DocumentBlueprint.setup(Person) do |blueprint|
+  #     # Our Person class has a method lang_cd. We use this method to
+  #     # index each person with its language
+  #     blueprint.language_method :lang_cd
+  #     blueprint.attribute       :complex, :weight => 10 do
+  #       # add some logic here to evaluate the value of 'complex'
+  #     end
+  #   end
   # @author Gernot Kogler
   class DocumentBlueprint
 
@@ -53,6 +62,8 @@ module XapianDb
           prefixes << blueprint.searchable_prefixes
         end
         @searchable_prefixes = prefixes.flatten.compact.uniq
+        # We can always do a field search on the name of the indexed class
+        @searchable_prefixes << "indexed_class"
       end
 
     end
@@ -64,7 +75,7 @@ module XapianDb
     # Return an array of all configured text methods in this blueprint
     # @return [Array<String>] All searchable prefixes
     def searchable_prefixes
-      @prefixes ||= indexed_methods.keys
+      @prefixes ||= indexed_methods_hash.keys
     end
 
     # Lazily build and return a module that implements accessors for each field
@@ -75,12 +86,12 @@ module XapianDb
 
       # Add the accessor for the indexed class
       @accessors_module.instance_eval do
-        define_method :domain_class do
+        define_method :indexed_class do
           self.values[0].value
         end
       end
 
-      @attributes.each_with_index do |field, index|
+      @attributes_hash.keys.each_with_index do |field, index|
         @accessors_module.instance_eval do
           define_method field do
             YAML::load(self.values[index+1].value)
@@ -103,12 +114,12 @@ module XapianDb
 
     # Collection of the configured attribute methods
     # @return [Array<Symbol>] The names of the configured attribute methods
-    attr_reader :attributes
+    attr_reader :attributes_hash
 
     # Collection of the configured index methods
     # @return [Hash<Symbol, IndexOptions>] A hashtable containing all index methods as
     #   keys and IndexOptions as values
-    attr_reader :indexed_methods
+    attr_reader :indexed_methods_hash
 
     # Set / read a custom adapter.
     # Use this configuration option if you need a specific adapter for an indexed class.
@@ -117,8 +128,8 @@ module XapianDb
 
     # Construct the blueprint
     def initialize
-      @attributes = []
-      @indexed_methods = {}
+      @attributes_hash = {}
+      @indexed_methods_hash = {}
     end
 
     # Set the name of the method to get the language for an indexed object
@@ -134,32 +145,91 @@ module XapianDb
     # @param [Hash] options
     # @option options [Integer] :weight (1) The weight for this attribute.
     # @option options [Boolean] :index (true) Should the attribute be indexed?
-    # @todo Make sure the name does not collide with a method name of Xapian::Document since
-    def attribute(name, options={})
+    # @example For complex attribute configurations you may pass a block:
+    #   XapianDb::DocumentBlueprint.setup(IndexedObject) do |blueprint|
+    #     blueprint.attribute :complex do
+    #       if @id == 1
+    #         "One"
+    #       else
+    #         "Not one"
+    #       end
+    #     end
+    #   end
+    # @todo Make sure the name does not collide with a method name of Xapian::Document
+    def attribute(name, options={}, &block)
       opts = {:index => true}.merge(options)
-      @attributes << name
-      self.index(name, opts) if opts[:index]
+      if block_given?
+        @attributes_hash[name] = block
+      else
+        @attributes_hash[name] = nil
+      end
+      self.index(name, opts, &block) if opts[:index]
+    end
+
+    # Add list of attributes to the blueprint. Attributes will be stored in the xapian documents an can be
+    # accessed from a search result.
+    # @param [Array] attributes An array of method names that deliver the values for the attributes
+    # @todo Make sure the name does not collide with a method name of Xapian::Document
+    def attributes(*attributes)
+      attributes.each do |attr|
+        @attributes_hash[attr] = nil
+        self.index attr
+      end
     end
 
     # Add an indexed value to the blueprint. Indexed values are not accessible from a search result.
-    # @param [String] name The name of the method that delivers the value for the index
-    # @param [Hash] options
-    # @option options [Integer] :weight (1) The weight for this indexed value
-    def index(name, options={})
-      @indexed_methods[name] = IndexOptions.new(options)
+    # @param [Array] args An array of arguments; you can pass a method name, an array of method names
+    #   or a method name and an options hash.
+    # @param [Block] &block An optional block for complex configurations
+    # Avaliable options:
+    # - :weight (default: 1) The weight for this indexed value
+    # @example Simple index declaration
+    #   blueprint.index :name
+    # @example Index declaration with options
+    #   blueprint.index :name, :weight => 10
+    # @example Mass index declaration
+    #   blueprint.index :name, :first_name, :profession
+    # @example Index declaration with a block
+    #   blueprint.index :complex, :weight => 10 do
+    #     # add some logic here to calculate the value for 'complex'
+    #   end
+    def index(*args, &block)
+      case args.size
+        when 1
+          @indexed_methods_hash[args.first] = IndexOptions.new(:weight => 1, :block => block)
+        when 2
+          # Is it a method name with options?
+          if args.last.is_a? Hash
+            @indexed_methods_hash[args.first] = IndexOptions.new(args.last.merge(:block => block))
+          else
+            add_indexes_from args
+          end
+        else # multiple arguments
+          add_indexes_from args
+      end
     end
 
     # Options for an indexed method
     class IndexOptions
 
       # The weight for the indexed value
-      attr_accessor :weight
+      attr_accessor :weight, :block
 
       # Constructor
       # @param [Hash] options
       # @option options [Integer] :weight (1) The weight for the indexed value
       def initialize(options)
         @weight = options[:weight] || 1
+        @block  = options[:block]
+      end
+    end
+
+    private
+
+    # Add index configurations from an array
+    def add_indexes_from(array)
+      array.each do |arg|
+        @indexed_methods_hash[arg] = IndexOptions.new(:weight => 1)
       end
     end
 

data/lib/xapian_db/indexer.rb

@@ -36,8 +36,12 @@ module XapianDb
       @xapian_doc.add_value(0, @obj.class.name)
 
       pos = 1
-      @blueprint.attributes.each do |attribute, options|
-        value = @obj.send(attribute)
+      @blueprint.attributes_hash.each do |attribute, block|
+        if block
+          value = @obj.instance_eval(&block)
+        else
+          value = @obj.send(attribute)
+        end
         @xapian_doc.add_value(pos, value.to_yaml)
         pos += 1
       end
@@ -52,20 +56,26 @@ module XapianDb
       if @stemmer
         term_generator.stemmer  = @stemmer
         term_generator.stopper  = @stopper unless @stopper.nil?
-        # Enable the creation of a spelling index if the database is not in memory
-        if @database.is_a? XapianDb::PersistentDatabase
-          term_generator.set_flags Xapian::TermGenerator::FLAG_SPELLING if @database.is_a? XapianDb::PersistentDatabase
-        end
+        # Enable the creation of a spelling dictionary if the database is not in memory
+        term_generator.set_flags Xapian::TermGenerator::FLAG_SPELLING if @database.is_a? XapianDb::PersistentDatabase
       end
 
-      # Always index the class and the primary key
-      @xapian_doc.add_term("C#{@obj.class}")
+      # Index the primary key as a unique term
       @xapian_doc.add_term("Q#{@obj.xapian_id}")
 
-      @blueprint.indexed_methods.each do |method, options|
-        value = @obj.send(method)
-        unless value.nil?
-          values = value.is_a?(Array) ? value : [value]
+      # Index the class with the field name
+      term_generator.index_text("#{@obj.class}".downcase, 1, "XINDEXED_CLASS")
+      @xapian_doc.add_term("C#{@obj.class}")
+
+
+      @blueprint.indexed_methods_hash.each do |method, options|
+        if options.block
+          obj = @obj.instance_eval(&options.block)
+        else
+          obj = @obj.send(method)
+        end
+        unless obj.nil?
+          values = get_values_to_index_from obj
           values.each do |value|
             # Add value with field name
             term_generator.index_text(value.to_s.downcase, options.weight, "X#{method.upcase}")
@@ -96,6 +106,20 @@ module XapianDb
 
     end
 
+    # Get the values to index from an object
+    def get_values_to_index_from(obj)
+
+      # if it's an array, that's fine
+      return obj if obj.is_a? Array
+
+      # if the object responds to attributes and attributes is a hash,
+      # we use the attributes values (works well for active_record and datamapper objects)
+      return obj.attributes.values if obj.respond_to?(:attributes) && obj.attributes.is_a?(Hash)
+
+      # The object is unkown and will be indexed by its to_s method
+      return [obj]
+    end
+
   end
 
 end

data/lib/xapian_db/query_parser.rb

@@ -8,7 +8,7 @@ module XapianDb
 
     # The spelling corrected query (if a language is configured)
     # @return [String]
-    attr_reader :corrected_query
+    attr_reader :spelling_suggestion
 
     # Constructor
     # @param [XapianDb::Database] database The database to query
@@ -39,7 +39,7 @@ module XapianDb
       # (like "name:Kogler")
       XapianDb::DocumentBlueprint.searchable_prefixes.each{|prefix| parser.add_prefix(prefix.to_s.downcase, "X#{prefix.to_s.upcase}") }
       query = parser.parse_query(expression, @query_flags)
-      @corrected_query = parser.get_corrected_query_string
+      @spelling_suggestion = parser.get_corrected_query_string
       query
     end
 

data/lib/xapian_db/resultset.rb

@@ -17,19 +17,18 @@ module XapianDb
 
     # The spelling corrected query (if a language is configured)
     # @return [String]
-    attr_reader :corrected_query
+    attr_reader :spelling_suggestion
 
     # Constructor
     # @param [Xapian::Enquire] enquiry a Xapian query result (see http://xapian.org/docs/apidoc/html/classXapian_1_1Enquire.html)
     # @param [Hash] options
     # @option options [Integer] :per_page (10) How many docs per page?
-    # @option options [String] :corrected_query (nil) The spelling corrected query (if a language is configured)
+    # @option options [String] :spelling_suggestion (nil) The spelling corrected query (if a language is configured)
     def initialize(enquiry, options)
       @enquiry = enquiry
-      # By passing 0 as the max parameter to the mset method,
-      # we only get statistics about the query, no results
-      @size            = enquiry.mset(0, 0).matches_estimated
-      @corrected_query = options[:corrected_query]
+      # To get more accurate results, we pass the doc count to the mset method
+      @size                = enquiry.mset(0, options[:db_size]).matches_estimated
+      @spelling_suggestion = options[:spelling_suggestion]
       @per_page = options[:per_page]
     end
 

data/lib/xapian_db.rb

@@ -79,8 +79,10 @@ module XapianDb
 
 end
 
-do_not_require = %w(update_stopwords.rb railtie.rb)
+do_not_require = %w(update_stopwords.rb railtie.rb base_adapter.rb)
 files = Dir.glob("#{File.dirname(__FILE__)}/**/*.rb").reject{|path| do_not_require.include?(File.basename(path))}
+# Require the base adapter first
+require "#{File.dirname(__FILE__)}/xapian_db/adapters/base_adapter"
 files.each {|file| require file}
 
 # Configure XapianDB if we are in a Rails app

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 3
-  - 3
-  version: 0.3.3
+  - 4
+  version: 0.3.4
 platform: ruby
 authors: 
 - Gernot Kogler
@@ -57,6 +57,7 @@ extra_rdoc_files: []
 
 files: 
 - lib/xapian_db/adapters/active_record_adapter.rb
+- lib/xapian_db/adapters/base_adapter.rb
 - lib/xapian_db/adapters/datamapper_adapter.rb
 - lib/xapian_db/adapters/generic_adapter.rb
 - lib/xapian_db/config.rb