xapian_db 1.1.1 → 1.1.2

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+##1.1.2 (September 10th, 2011)
+
+Fixes:
+
+  - beanstalk_work flushes log messages (reinstall the worker script: rails generate xapian_db:install)
+
+Features:
+
+  - massive performance optimization in rebuild_xapian_index
+
 ##1.1.1 (September 9th, 2011)
 
 Fixes:

data/README.rdoc

@@ -138,7 +138,7 @@ one more class that is not stored in the database, but you want it to be indexed
 
 If you use associations in your blueprints, it might be a good idea to specify a base query to speed up rebuild_xapian_index calls (avoiding 1+n queries):
 
-XapianDb::DocumentBlueprint.setup(Person) do |blueprint|
+  XapianDb::DocumentBlueprint.setup(Person) do |blueprint|
     blueprint.index :addresses
     blueprint.base_query Person.includes(:addresses)
   end

data/lib/generators/templates/beanstalk_worker

@@ -16,21 +16,33 @@ module XapianDb
     def process_requests
 
       Rails.logger.info "XapianDb beanstalk worker: initializing..."
+      Rails.logger.flush
 
-      url = XapianDb::Config.beanstalk_daemon_url
-      beanstalk = Beanstalk::Pool.new([url])
-      worker    = XapianDb::IndexWriters::BeanstalkWorker.new
-      Rails.logger.info "XapianDb beanstalk worker: ready"
+      begin
+        url = XapianDb::Config.beanstalk_daemon_url
+        beanstalk = Beanstalk::Pool.new([url])
+        worker    = XapianDb::IndexWriters::BeanstalkWorker.new
+        Rails.logger.info "XapianDb beanstalk worker: ready"
+        Rails.logger.flush
+      rescue Exception => ex
+        Rails.logger.error "cannot connect to beanstalk daemon: (#{ex}), terminating"
+        Rails.logger.flush
+        return
+      end
 
       loop do
         job = beanstalk.reserve
         begin
           params = YAML::load job.body
           Rails.logger.info "XapianDb beanstalk worker: executing task #{params}"
+          Rails.logger.flush
           task = params.delete :task
           worker.send task, params
+          Rails.logger.info "XapianDb beanstalk worker: done"
+          Rails.logger.flush
         rescue Exception => ex
           Rails.logger.error "XapianDb beanstalk worker: could not process #{job.body} (#{ex})"
+          Rails.logger.flush
         end
         job.delete
       end

data/lib/xapian_db/index_writers/beanstalk_writer.rb

@@ -17,13 +17,13 @@ module XapianDb
 
         # Update an object in the index
         # @param [Object] obj An instance of a class with a blueprint configuration
-        def index(obj)
+        def index(obj, commit=true)
           beanstalk.put( {:task => "index_task", :class => obj.class.name, :id => obj.id }.to_yaml )
         end
 
         # Remove an object from the index
         # @param [String] xapian_id The document id
-        def delete_doc_with(xapian_id)
+        def delete_doc_with(xapian_id, commit=true)
           beanstalk.put( { :task => "delete_doc_task", :xapian_id => xapian_id }.to_yaml )
         end
 

data/lib/xapian_db/index_writers/direct_writer.rb

@@ -12,24 +12,36 @@ module XapianDb
 
     class DirectWriter
 
-      BATCH_SIZE = 200
+      BATCH_SIZE = 500
+
       class << self
 
         # Update an object in the index
         # @param [Object] obj An instance of a class with a blueprint configuration
-        def index(obj)
+        def index(obj, commit=true)
           blueprint = XapianDb::DocumentBlueprint.blueprint_for(obj.class)
           indexer   = XapianDb::Indexer.new(XapianDb.database, blueprint)
           doc       = indexer.build_document_for(obj)
           XapianDb.database.store_doc(doc)
-          XapianDb.database.commit
+          XapianDb.database.commit if commit
         end
 
         # Remove an object from the index
         # @param [String] xapian_id The document id of an object
-        def delete_doc_with(xapian_id)
+        def delete_doc_with(xapian_id, commit=true)
           XapianDb.database.delete_doc_with_unique_term xapian_id
-          XapianDb.database.commit
+          XapianDb.database.commit if commit
+        end
+
+        # Update or delete a xapian document belonging to an object depending on the ignore_if logic(if present)
+        # @param [Object] object An instance of a class with a blueprint configuration
+        def reindex(object, commit=true)
+          blueprint = XapianDb::DocumentBlueprint.blueprint_for object.class
+          if blueprint.should_index?(object)
+            index object, commit
+          else
+            delete_doc_with object.xapian_id, commit
+          end
         end
 
         # Reindex all objects of a given class
@@ -38,6 +50,9 @@ module XapianDb
         # @option options [Boolean] :verbose (false) Should the reindexing give status informations?
         def reindex_class(klass, options={})
           opts = {:verbose => false}.merge(options)
+          blueprint = XapianDb::DocumentBlueprint.blueprint_for klass
+          adapter = blueprint._adapter || XapianDb::Config.adapter || Adapters::GenericAdapter
+          primary_key = adapter.primary_key_for(klass)
           XapianDb.database.delete_docs_of_class(klass)
           blueprint  = XapianDb::DocumentBlueprint.blueprint_for(klass)
           indexer    = XapianDb::Indexer.new(XapianDb.database, blueprint)
@@ -52,11 +67,11 @@ module XapianDb
 
           # Process the objects in batches to reduce the memory footprint
           nr_of_batches = (obj_count / BATCH_SIZE) + 1
-          order_expression = "#{klass.name.tableize}.#{options[:primary_key]}"
+          order_expression = "#{klass.name.tableize}.#{primary_key}"
+          # raise "vor loop"
           nr_of_batches.times do |batch|
-            # raise "PK: #{options[:primary_key]}"
             base_query.all(:offset => batch * BATCH_SIZE, :limit => BATCH_SIZE, :order => order_expression).each do |obj|
-              XapianDb.reindex obj
+              reindex obj, false
               pbar.inc if show_progressbar
             end
           end

data/lib/xapian_db/index_writers/no_op_writer.rb

@@ -10,11 +10,11 @@ module XapianDb
 
       # Update an object in the index
       # @param [Object] obj An instance of a class with a blueprint configuration
-      def index(obj); end
+      def index(obj, commit=true); end
 
       # Remove an object from the index
       # @param [String] xapian_id The document id
-      def delete_doc_with(xapian_id); end
+      def delete_doc_with(xapian_id, commit=true); end
 
       # Reindex all objects of a given class
       # @param [Class] klass The class to reindex

data/lib/xapian_db/index_writers/transactional_writer.rb

@@ -20,13 +20,13 @@ module XapianDb
 
       # Update an object in the index
       # @param [Object] obj An instance of a class with a blueprint configuration
-      def index(obj)
+      def index(obj, commit=false)
         @index_requests << obj
       end
 
       # Remove a document from the index
       # @param [String] xapian_id The document id
-      def delete_doc_with(xapian_id)
+      def delete_doc_with(xapian_id, commit=false)
         @delete_requests << xapian_id
       end
 
@@ -41,8 +41,9 @@ module XapianDb
       # Commit all pending changes to the database
       # @param [DirectWriter, BeanstalkWriter] writer The writer to use
       def commit_using(writer)
-        @index_requests.each { |obj| writer.index obj }
-        @delete_requests.each { |xapian_id| writer.delete_doc_with xapian_id }
+        @index_requests.each { |obj| writer.index obj, false }
+        @delete_requests.each { |xapian_id| writer.delete_doc_with xapian_id, false }
+        XapianDb.database.commit
       end
 
     end

data/lib/xapian_db.rb

@@ -110,26 +110,27 @@ module XapianDb
 
   # Update an object in the index
   # @param [Object] obj An instance of a class with a blueprint configuration
-  def self.index(obj)
+  def self.index(obj, commit=true)
     writer = @block_writer || XapianDb::Config.writer
-    writer.index obj
+    writer.index obj, commit
   end
 
   # Remove a document from the index
   # @param [String] xapian_id The document id
-  def self.delete_doc_with(xapian_id)
+  def self.delete_doc_with(xapian_id, commit=true)
     writer = @block_writer || XapianDb::Config.writer
-    writer.delete_doc_with xapian_id
+    writer.delete_doc_with xapian_id, commit
   end
 
   # Update or delete a xapian document belonging to an object depending on the ignore_if logic(if present)
   # @param [Object] object An instance of a class with a blueprint configuration
-  def self.reindex(object)
+  def self.reindex(object, commit=true)
+    writer = @block_writer || XapianDb::Config.writer
     blueprint = XapianDb::DocumentBlueprint.blueprint_for object.class
     if blueprint.should_index?(object)
-      XapianDb.index object
+      writer.index object, commit
     else
-      XapianDb.delete_doc_with object.xapian_id
+      writer.delete_doc_with object.xapian_id, commit
     end
   end
 
@@ -150,9 +151,6 @@ module XapianDb
     return false unless configured_classes.size > 0
     configured_classes.each do |klass|
       if klass.respond_to?(:rebuild_xapian_index)
-        blueprint = XapianDb::DocumentBlueprint.blueprint_for klass
-        adapter = blueprint._adapter || XapianDb::Config.adapter || Adapters::GenericAdapter
-        options[:primary_key] = adapter.primary_key_for(klass)
         XapianDb::Config.writer.reindex_class(klass, options)
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: xapian_db
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-09 00:00:00.000000000 +02:00
+date: 2011-09-10 00:00:00.000000000 +02:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &70154206238860 !ruby/object:Gem::Requirement
+  requirement: &70182803109440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +22,10 @@ dependencies:
         version: 1.0.10
   type: :runtime
   prerelease: false
-  version_requirements: *70154206238860
+  version_requirements: *70182803109440
 - !ruby/object:Gem::Dependency
   name: xapian-ruby
-  requirement: &70154206238400 !ruby/object:Gem::Requirement
+  requirement: &70182803108960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +33,10 @@ dependencies:
         version: 1.2.6
   type: :runtime
   prerelease: false
-  version_requirements: *70154206238400
+  version_requirements: *70182803108960
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70154206237940 !ruby/object:Gem::Requirement
+  requirement: &70182803108480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +44,10 @@ dependencies:
         version: 2.3.1
   type: :development
   prerelease: false
-  version_requirements: *70154206237940
+  version_requirements: *70182803108480
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70154206237480 !ruby/object:Gem::Requirement
+  requirement: &70182803108020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +55,10 @@ dependencies:
         version: 0.3.7
   type: :development
   prerelease: false
-  version_requirements: *70154206237480
+  version_requirements: *70182803108020
 - !ruby/object:Gem::Dependency
   name: beanstalk-client
-  requirement: &70154206237020 !ruby/object:Gem::Requirement
+  requirement: &70182803107560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,7 +66,7 @@ dependencies:
         version: 1.1.0
   type: :development
   prerelease: false
-  version_requirements: *70154206237020
+  version_requirements: *70182803107560
 description: XapianDb is a ruby gem that combines features of nosql databases and
   fulltext indexing. It is based on Xapian, an efficient and powerful indexing library
 email: gernot.kogler (at) garaio (dot) com