xapian_db 1.1 → 1.1.1

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+##1.1.1 (September 9th, 2011)
+
+Fixes:
+
+  - fixed a bug in XapianDb.rebuild_xapian_index that did not index all records of a klass; it ist highly recommended to run Xapian.rebuild_xapian_index after installing this version
+
+Features:
+
+  - base query option for blueprints; may speed up reindexing signifiantly if you index associations
+
 ##1.1 (September 7th, 2011)
 
 Fixes:

data/README.rdoc

@@ -131,10 +131,17 @@ You can add a type information to an attribute. As of now the special types :str
 You can override the global adapter configuration in a specific blueprint. Let's say you use ActiveRecord, but you have
 one more class that is not stored in the database, but you want it to be indexed:
 
-XapianDb::DocumentBlueprint.setup(SpecialClass) do |blueprint|
-  blueprint.adapter :generic
-  blueprint.index   :some_stuff
-end
+  XapianDb::DocumentBlueprint.setup(SpecialClass) do |blueprint|
+    blueprint.adapter :generic
+    blueprint.index   :some_stuff
+  end
+
+If you use associations in your blueprints, it might be a good idea to specify a base query to speed up rebuild_xapian_index calls (avoiding 1+n queries):
+
+XapianDb::DocumentBlueprint.setup(Person) do |blueprint|
+    blueprint.index :addresses
+    blueprint.base_query Person.includes(:addresses)
+  end
 
 place these configurations either into the corresponding class or - I prefer to have the index configurations outside
 the models - into the file config/xapian_blueprints.rb.

data/lib/xapian_db/adapters/active_record_adapter.rb

@@ -20,6 +20,13 @@ module XapianDb
 
        class << self
 
+         # return the name of the primary key column of a class
+         # @param [Class] klass the class
+         # @return [Symbol] the name of the primary key column
+         def primary_key_for(klass)
+          klass.primary_key
+         end
+
          # Implement the class helper methods
          # @param [Class] klass The class to add the helper methods to
          def add_class_helper_methods_to(klass)

data/lib/xapian_db/adapters/datamapper_adapter.rb

@@ -19,6 +19,13 @@ module XapianDb
 
        class << self
 
+         # return the name of the primary key column of a class
+         # @param [Class] klass the class
+         # @return [Symbol] the name of the primary key column
+         def primary_key_for(klass)
+           klass.serial.name
+         end
+
          # Implement the class helper methods
          # @param [Class] klass The class to add the helper methods to
          def add_class_helper_methods_to(klass)

data/lib/xapian_db/document_blueprint.rb

@@ -205,8 +205,8 @@ module XapianDb
     # Blueprint DSL methods
     # ---------------------------------------------------------------------------------
 
-    # An optional custom adapter
     attr_accessor :_adapter
+    attr_reader :_base_query
 
     # Construct the blueprint
     def initialize
@@ -306,6 +306,16 @@ module XapianDb
       @ignore_expression = block
     end
 
+    # Define a base query to select one or all objects of the indexed class. The reason for a
+    # base query is to optimize the query avoiding th 1+n problematic. The base query should only
+    # include joins(...) and includes(...) calls.
+    # @param [expression] a base query expression
+    # @example Include the adresses
+    #   blueprint.base_query Person.includes(:addresses)
+    def base_query(expression)
+      @_base_query = expression
+    end
+
     # Options for an indexed method
     class IndexOptions
 

data/lib/xapian_db/index_writers/direct_writer.rb

@@ -12,6 +12,7 @@ module XapianDb
 
     class DirectWriter
 
+      BATCH_SIZE = 200
       class << self
 
         # Update an object in the index
@@ -37,30 +38,25 @@ module XapianDb
         # @option options [Boolean] :verbose (false) Should the reindexing give status informations?
         def reindex_class(klass, options={})
           opts = {:verbose => false}.merge(options)
-          # First, delete all docs of this class
           XapianDb.database.delete_docs_of_class(klass)
-          blueprint = XapianDb::DocumentBlueprint.blueprint_for(klass)
-          indexer   = XapianDb::Indexer.new(XapianDb.database, blueprint)
+          blueprint  = XapianDb::DocumentBlueprint.blueprint_for(klass)
+          indexer    = XapianDb::Indexer.new(XapianDb.database, blueprint)
+          base_query = blueprint._base_query || klass
           show_progressbar = false
-          obj_count = klass.count
+          obj_count = base_query.count
           if opts[:verbose]
-            if defined?(ProgressBar)
-              show_progressbar = true
-            end
+            show_progressbar = defined?(ProgressBar)
             puts "reindexing #{obj_count} objects of #{klass}..."
             pbar = ProgressBar.new("Status", obj_count) if show_progressbar
           end
 
           # Process the objects in batches to reduce the memory footprint
-          nr_of_batches = (obj_count / 1000) + 1
+          nr_of_batches = (obj_count / BATCH_SIZE) + 1
+          order_expression = "#{klass.name.tableize}.#{options[:primary_key]}"
           nr_of_batches.times do |batch|
-            klass.all(:offset => batch * 1000, :limit => 1000, :order => options[:primary_key]).each do |obj|
-              if blueprint.should_index? obj
-                doc = indexer.build_document_for(obj)
-                XapianDb.database.store_doc(doc)
-              else
-                XapianDb.database.delete_doc_with_unique_term(obj.xapian_id)
-              end
+            # raise "PK: #{options[:primary_key]}"
+            base_query.all(:offset => batch * BATCH_SIZE, :limit => BATCH_SIZE, :order => order_expression).each do |obj|
+              XapianDb.reindex obj
               pbar.inc if show_progressbar
             end
           end
@@ -69,8 +65,6 @@ module XapianDb
         end
 
       end
-
     end
-
   end
 end

data/lib/xapian_db.rb

@@ -149,7 +149,12 @@ module XapianDb
     configured_classes = XapianDb::DocumentBlueprint.configured_classes
     return false unless configured_classes.size > 0
     configured_classes.each do |klass|
-      XapianDb::Config.writer.reindex_class(klass, options) if klass.respond_to?(:rebuild_xapian_index)
+      if klass.respond_to?(:rebuild_xapian_index)
+        blueprint = XapianDb::DocumentBlueprint.blueprint_for klass
+        adapter = blueprint._adapter || XapianDb::Config.adapter || Adapters::GenericAdapter
+        options[:primary_key] = adapter.primary_key_for(klass)
+        XapianDb::Config.writer.reindex_class(klass, options)
+      end
     end
     true
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: xapian_db
 version: !ruby/object:Gem::Version
-  version: '1.1'
+  version: 1.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-07 00:00:00.000000000 +02:00
+date: 2011-09-09 00:00:00.000000000 +02:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &70318028242380 !ruby/object:Gem::Requirement
+  requirement: &70154206238860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +22,10 @@ dependencies:
         version: 1.0.10
   type: :runtime
   prerelease: false
-  version_requirements: *70318028242380
+  version_requirements: *70154206238860
 - !ruby/object:Gem::Dependency
   name: xapian-ruby
-  requirement: &70318028241920 !ruby/object:Gem::Requirement
+  requirement: &70154206238400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +33,10 @@ dependencies:
         version: 1.2.6
   type: :runtime
   prerelease: false
-  version_requirements: *70318028241920
+  version_requirements: *70154206238400
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70318028241460 !ruby/object:Gem::Requirement
+  requirement: &70154206237940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +44,10 @@ dependencies:
         version: 2.3.1
   type: :development
   prerelease: false
-  version_requirements: *70318028241460
+  version_requirements: *70154206237940
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70318028241000 !ruby/object:Gem::Requirement
+  requirement: &70154206237480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +55,10 @@ dependencies:
         version: 0.3.7
   type: :development
   prerelease: false
-  version_requirements: *70318028241000
+  version_requirements: *70154206237480
 - !ruby/object:Gem::Dependency
   name: beanstalk-client
-  requirement: &70318028240520 !ruby/object:Gem::Requirement
+  requirement: &70154206237020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,7 +66,7 @@ dependencies:
         version: 1.1.0
   type: :development
   prerelease: false
-  version_requirements: *70318028240520
+  version_requirements: *70154206237020
 description: XapianDb is a ruby gem that combines features of nosql databases and
   fulltext indexing. It is based on Xapian, an efficient and powerful indexing library
 email: gernot.kogler (at) garaio (dot) com