xapian_db 0.5.9 → 0.5.10

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+##0.5.10 (April 6th, 2011)
+
+Features:
+
+  - the beanstalk worker is now implemented as a daemon script (execute 'rails generate xapian_db:install' to install it)
+  - execute a block with auto indexing disabled (see 'Bulk inserts / updates / deletes' in teh README)
+  - updated the xapian source to version 1.2.5
+
 ##0.5.9 (March 25th, 2011)
 
 Fixes:

data/README.rdoc

@@ -244,6 +244,19 @@ only if the block does not raise an exception.
     object2.save
   end
 
+=== Bulk inserts / updates / deletes
+
+When you change a lot of models, it is not very efficient to update the xapian index on each insert / update / delete. Instead,
+you can use the auto_indexing_disabled method with a block and rebuild the whole index afterwards:
+
+  XapianDb.auto_indexing_disabled do
+    Person.each do |person|
+      # change person
+      person.save
+    end
+  end
+  Person.rebuild_xapian_index
+
 == Production setup
 
 Since Xapian allows only one database instance to write to the index, the default setup of XapianDb will not work
@@ -282,21 +295,26 @@ The easiest way is to use macports or homebrew:
   gem 'beanstalk-client' # Add this to your Gemfile
   bundle install
 
-=== 3. Configure your production environment in config/xapian_db.yml
+=== 3. Install the beanstalk worker script
+
+  rails generate xapian:db install
+
+=== 4. Configure your production environment in config/xapian_db.yml
 
   production:
     database: db/xapian_db/production
     writer:   beanstalk
     beanstalk_daemon: localhost:11300
 
-=== 4. start the beanstalk daemon
+=== 5. start the beanstalk daemon
 
   beanstalkd -d
 
-=== 5. start the beanstalk worker from within your Rails app root directory
-
-  rake RAILS_ENV=production xapian_db:beanstalk_worker
+=== 6. start the beanstalk worker from within your Rails app root directory
 
-<b>Important: Do not start multiple instances of this worker task!</b>
+  RAILS_ENV=production script/beanstalk_worker start
 
+If everything is fine, you should find a file namend beanstalk_worker.pid in tmp/pids. If something
+goes wrong, you'll find beanstalk_worker.log instead showing the stack trace.
 
+<b>Important: Do not start multiple instances of this daemon!</b>

data/Rakefile

@@ -9,7 +9,7 @@ def system!(cmd)
 	system(cmd) or raise
 end
 
-ver = '1.2.4'
+ver = '1.2.5'
 source_dir = 'xapian_source'
 core = "xapian-core-#{ver}"
 bindings = "xapian-bindings-#{ver}"

data/lib/generators/install_generator.rb

@@ -0,0 +1,17 @@
+require 'rails/generators'
+
+module XapianDb
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("../templates", __FILE__)
+
+      desc "Creates a beanstalk worker script."
+
+      def copy_script
+        copy_file "beanstalk_worker", "script/beanstalk_worker"
+        chmod "script/beanstalk_worker", 0755
+      end
+
+    end
+  end
+end

data/lib/generators/templates/beanstalk_worker

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require "bundler/setup"
+require 'rails'
+require 'daemons'
+require "xapian_db"
+require 'yaml'
+require 'rails/railtie' # execute xapian_db railtie
+
+# daemon script to manage the beanstalk worker daemon
+# @author Gernot Kogler
+module XapianDb
+  class BeanstalkWorker
+
+    def process_requests
+
+      Rails.logger.info "XapianDb beanstalk worker: initializing..."
+
+      url = XapianDb::Config.beanstalk_daemon_url
+      beanstalk = Beanstalk::Pool.new([url])
+      worker    = XapianDb::IndexWriters::BeanstalkWorker.new
+      Rails.logger.info "XapianDb beanstalk worker: ready"
+
+      loop do
+        begin
+          job = beanstalk.reserve
+          params = YAML::load job.body
+          Rails.logger.info "XapianDb beanstalk worker: executing task #{params}"
+          task = params.delete :task
+          worker.send task, params
+        rescue Exception => ex
+          Rails.logger.error "XapianDb beanstalk worker: could not process #{job.body} (#{ex})"
+        end
+
+        begin
+          job.delete
+        rescue Exception => ex
+          Rails.logger.error "XapianDb beanstalk worker: could delete job #{job.body} (#{ex})"
+        end
+
+      end
+    end
+  end
+end
+
+# =========================================================================================
+# daemon setup
+# =========================================================================================
+dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+pids_dir = File.join(dir, 'tmp', 'pids')
+FileUtils.makedirs pids_dir unless File.exists?(pids_dir)
+
+daemon_options = {
+  :multiple   => false,
+  :dir_mode   => :normal,
+  :dir        => pids_dir,
+  :backtrace  => true
+}
+
+Daemons.run_proc('beanstalk_worker', daemon_options) do
+  if ARGV.include?('--')
+    ARGV.slice! 0..ARGV.index('--')
+  else
+    ARGV.clear
+  end
+
+  Dir.chdir dir
+  require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
+  XapianDb::BeanstalkWorker.new.process_requests
+end

data/lib/xapian_db/index_writers/no_op_writer.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+# This writer is a dummy writer that simply does nothing.
+# @author Gernot Kogler
+
+module XapianDb
+  module IndexWriters
+
+    class NoOpWriter
+
+      # Update an object in the index
+      # @param [Object] obj An instance of a class with a blueprint configuration
+      def index(obj); end
+
+      # Remove an object from the index
+      # @param [Object] obj An instance of a class with a blueprint configuration
+      def unindex(obj); end
+
+      # Reindex all objects of a given class
+      # @param [Class] klass The class to reindex
+      # @param [Hash] options Options for reindexing
+      # @option options [Boolean] :verbose (false) Should the reindexing give status informations?
+      def reindex_class(klass, options={})
+        raise "rebuild_xapian_index is not supported inside a block with auto indexing disabled"
+      end
+
+    end
+  end
+end

data/lib/xapian_db/railtie.rb

@@ -9,11 +9,16 @@ module XapianDb
   # @author Gernot Kogler
   class Railtie < ::Rails::Railtie
 
-    # Require our rake tasks
+    # require our rake tasks
     rake_tasks do
       load "#{File.dirname(__FILE__)}/../../tasks/beanstalk_worker.rake"
     end
 
+    # require our generators
+    generators do
+      require "#{File.dirname(__FILE__)}/../generators/install_generator.rb"
+    end
+
     config.before_configuration do
 
       # Read the database configuration file if there is one
@@ -49,24 +54,24 @@ module XapianDb
       load blueprints_file_path if File.exist?(blueprints_file_path)
     end
 
-  private
+    private
 
-  # use the config options from the config file
-  def self.configure_from(env_config)
-    @database_path    = env_config["database"] || ":memory:"
-    @adapter          = env_config["adapter"]  || :active_record
-    @writer           = env_config["writer"]   || :direct
-    @beanstalk_daemon = env_config["beanstalk_daemon"]
-    @language         = env_config["language"]
-  end
+    # use the config options from the config file
+    def self.configure_from(env_config)
+      @database_path    = env_config["database"] || ":memory:"
+      @adapter          = env_config["adapter"]  || :active_record
+      @writer           = env_config["writer"]   || :direct
+      @beanstalk_daemon = env_config["beanstalk_daemon"]
+      @language         = env_config["language"]
+    end
 
-  # set default config options
-  def self.configure_defaults
-    Rails.env == "test" ? @database_path = ":memory:" : @database_path = "db/xapian_db/#{Rails.env}"
-    @adapter          = :active_record
-    @writer           = :direct
-    @beanstalk_daemon = nil
-  end
+    # set default config options
+    def self.configure_defaults
+      Rails.env == "test" ? @database_path = ":memory:" : @database_path = "db/xapian_db/#{Rails.env}"
+      @adapter          = :active_record
+      @writer           = :direct
+      @beanstalk_daemon = nil
+    end
 
   end
 end

data/lib/xapian_db.rb

@@ -88,14 +88,14 @@ module XapianDb
   # Update an object in the index
   # @param [Object] obj An instance of a class with a blueprint configuration
   def self.index(obj)
-    writer = @transactional_writer || XapianDb::Config.writer
+    writer = @block_writer || XapianDb::Config.writer
     writer.index obj
   end
 
   # Remove an object from the index
   # @param [Object] obj An instance of a class with a blueprint configuration
   def self.unindex(obj)
-    writer = @transactional_writer || XapianDb::Config.writer
+    writer = @block_writer || XapianDb::Config.writer
     writer.unindex obj
   end
 
@@ -122,27 +122,47 @@ module XapianDb
 
   # Execute a block as a transaction
   def self.transaction(&block)
-    # Temporarily use the transactional writer
-    @transactional_writer = XapianDb::IndexWriters::TransactionalWriter.new
+    writer = XapianDb::IndexWriters::TransactionalWriter.new
+    execute_block :writer => writer, :error_message => "error in XapianDb transaction block, transaction aborted" do
+      block.call
+      writer.commit_using XapianDb::Config.writer
+    end
+  end
+
+  # Execute a block and do not update the index
+  def self.auto_indexing_disabled(&block)
+    execute_block :writer => XapianDb::IndexWriters::NoOpWriter.new do
+      block.call
+    end
+
+  end
+
+  # execute a block of code with a given writer and handle errors
+  # @param [Hash] opts Options
+  # @option opts [Object] :writer An index writer
+  # @option opts [String] :error_message the error message to log if an error occurs
+  def self.execute_block(opts, &block)
+    @block_writer = opts[:writer]
     begin
       block.call
-      @transactional_writer.commit_using XapianDb::Config.writer
     rescue Exception => ex
-      msg = "error in XapianDb transaction block: #{ex}, transaction aborted"
-      if defined?(Rails)
-        Rails.logger.error msg
-      else
-        puts msg
+      if opts[:error_message]
+        if defined?(Rails)
+          Rails.logger.error opts[:error_message]
+        else
+          puts opts[:error_message]
+        end
       end
+      raise
     ensure
-      # release the transactional writer
-      @transactional_writer = nil
+      # release the block writer
+      @block_writer = nil
     end
   end
 
 end
 
-do_not_require = %w(update_stopwords.rb railtie.rb base_adapter.rb beanstalk_writer.rb utilities.rb)
+do_not_require = %w(update_stopwords.rb railtie.rb base_adapter.rb beanstalk_writer.rb utilities.rb install_generator.rb)
 files = Dir.glob("#{File.dirname(__FILE__)}/**/*.rb").reject{|path| do_not_require.include?(File.basename(path))}
 # Require these first
 require "#{File.dirname(__FILE__)}/xapian_db/utilities"

data/tasks/beanstalk_worker.rake

@@ -11,16 +11,18 @@ require 'yaml'
 
 namespace :xapian_db do
 
-  desc "Run the beanstalk worker process to update the xapian index"
+  DEPRECATION_WARNING = "beanstalk_worker.rake is deprecated, use script/beanstalk_worker instead"
+  desc DEPRECATION_WARNING
   task :beanstalk_worker do
 
     url = XapianDb::Config.beanstalk_daemon_url
     beanstalk = Beanstalk::Pool.new([url])
     worker    = XapianDb::IndexWriters::BeanstalkWorker.new
+    puts DEPRECATION_WARNING
     puts "XapianDb beanstalk worker is serving on #{url}..."
     loop do
-      job = beanstalk.reserve
       begin
+        job = beanstalk.reserve
         params = YAML::load job.body
         Rails.logger.info "XapianDb beanstalk worker: executing task #{params}"
         task = params.delete :task
@@ -28,7 +30,13 @@ namespace :xapian_db do
       rescue Exception => ex
         Rails.logger.error "XapianDb beanstalk worker: could not process #{job.body} (#{ex})"
       end
-      job.delete
+
+      begin
+        job.delete
+      rescue Exception => ex
+        Rails.logger.error "XapianDb beanstalk worker: could delete job #{job.body} (#{ex})"
+      end
+
     end
 
   end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 5
-  - 9
-  version: 0.5.9
+  - 10
+  version: 0.5.10
 platform: ruby
 authors: 
 - Gernot Kogler
@@ -14,13 +14,28 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-25 00:00:00 +01:00
+date: 2011-04-06 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: daemons
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 0
+        - 10
+        version: 1.0.10
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -31,11 +46,11 @@ dependencies:
         - 1
         version: 2.3.1
   type: :development
-  version_requirements: *id001
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: simplecov
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -46,11 +61,11 @@ dependencies:
         - 7
         version: 0.3.7
   type: :development
-  version_requirements: *id002
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: beanstalk-client
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -61,7 +76,7 @@ dependencies:
         - 0
         version: 1.1.0
   type: :development
-  version_requirements: *id003
+  version_requirements: *id004
 description: XapianDb is a ruby gem that combines features of nosql databases and fulltext indexing. It is based on Xapian, an efficient and powerful indexing library
 email: gernot.kogler (at) garaio (dot) com
 executables: []
@@ -71,6 +86,8 @@ extensions:
 extra_rdoc_files: []
 
 files: 
+- lib/generators/install_generator.rb
+- lib/generators/templates/beanstalk_worker
 - lib/xapian_db/adapters/active_record_adapter.rb
 - lib/xapian_db/adapters/base_adapter.rb
 - lib/xapian_db/adapters/datamapper_adapter.rb
@@ -81,6 +98,7 @@ files:
 - lib/xapian_db/index_writers/beanstalk_worker.rb
 - lib/xapian_db/index_writers/beanstalk_writer.rb
 - lib/xapian_db/index_writers/direct_writer.rb
+- lib/xapian_db/index_writers/no_op_writer.rb
 - lib/xapian_db/index_writers/transactional_writer.rb
 - lib/xapian_db/indexer.rb
 - lib/xapian_db/query_parser.rb
@@ -106,8 +124,8 @@ files:
 - lib/xapian_db/utilities.rb
 - lib/xapian_db.rb
 - tasks/beanstalk_worker.rake
-- xapian_source/xapian-bindings-1.2.4.tar.gz
-- xapian_source/xapian-core-1.2.4.tar.gz
+- xapian_source/xapian-bindings-1.2.5.tar.gz
+- xapian_source/xapian-core-1.2.5.tar.gz
 - LICENSE
 - README.rdoc
 - CHANGELOG.md