xapian_db 1.1.4 → 1.2.0

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+##1.2.0 (November 8th, 2011)
+
+Fixes:
+
+  - find_similar_to supports the :limit option
+
+Features:
+
+  - new index worker / writer for resque (thanks, Michael)
+  - support for namespaced models (thanks, Albert)
+
 ##1.1.4 (October 25th, 2011)
 
 Fixes:

data/README.rdoc

@@ -331,7 +331,9 @@ However, dates need special handling to support date range queries. To support d
 
 Since Xapian allows only one database instance to write to the index, the default setup of XapianDb will not work
 with multiple app instances trying to write to the same database (you will get lock errors).
-Therefore, XapianDb provides a solution based on beanstalk to overcome this.
+Therefore, XapianDb provides two solutions based on queueing systems to overcome this. The first solution uses beanstalk and the second one uses resque.
+
+== Installation with beanstalk
 
 === 1. Install beanstalkd
 
@@ -389,3 +391,32 @@ If everything is fine, you should find a file namend beanstalk_worker.pid in tmp
 goes wrong, you'll find beanstalk_worker.log instead showing the stack trace.
 
 <b>Important: Do not start multiple instances of this daemon!</b>
+
+== Installation with Resque
+
+=== 1. Install and start redis
+
+Install and start redis as described on the {resque github page}[https://github.com/defunkt/resque].
+
+=== 2. Add the resque gem to your config
+
+  gem 'resque'
+  bundle install
+
+=== 3. Configure XapianDb to use resque in production
+
+  production:
+    database:     db/xapian_db/production
+    writer:       resque
+    resque_queue: my_queue
+
+If you don't specify a queue name XapianDb will use 'xapian_db' by default.
+
+=== 4. Start the resque worker
+
+   RAILS_ENV=production QUEUE=my_queue rake resque:work
+
+Be sure to specify the correct queue name when starting the worker.
+
+<b>If you don't provide a queue name, it WON'T take 'xapian_db' by default! Do not start multiple
+instances of this worker!</b>

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/lib/xapian_db/config.rb

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# -*- coding: utf-8 -*-
 
 module XapianDb
 
@@ -44,13 +44,17 @@ module XapianDb
         return default_url if @config.nil?
         @config.instance_variable_get("@_beanstalk_daemon_url") || default_url
       end
+
+      def resque_queue
+        @config.instance_variable_get("@_resque_queue") || 'xapian_db'
+      end
     end
 
     # ---------------------------------------------------------------------------------
     # DSL methods
     # ---------------------------------------------------------------------------------
 
-    attr_reader :_database, :_adapter, :_writer, :_beanstalk_daemon, :_stemmer, :_stopper
+    attr_reader :_database, :_adapter, :_writer, :_beanstalk_daemon, :_resque_queue, :_stemmer, :_stopper
 
     # Set the global database to use
     # @param [String] path The path to the database. Either apply a file sytem path or :memory
@@ -100,6 +104,12 @@ module XapianDb
       @_beanstalk_daemon_url = url
     end
 
+    # Set the name of the resque queue
+    # @param [String] name The name of the resque queue
+    def resque_queue(name)
+      @_resque_queue = name
+    end
+
     # Set the language.
     # @param [Symbol] lang The language; apply the two letter ISO639 code for the language
     # @example
@@ -115,4 +125,4 @@ module XapianDb
 
   end
 
-end
+end

data/lib/xapian_db/database.rb

@@ -116,7 +116,7 @@ module XapianDb
       end
       enquiry       = Xapian::Enquire.new(reader)
       enquiry.query = final_query
-      Resultset.new(enquiry, :db_size => self.size)
+      Resultset.new(enquiry, :db_size => self.size, :limit => options[:limit])
     end
 
     # A very simple implementation of facets using Xapian collapse key.

data/lib/xapian_db/index_writers/direct_writer.rb

@@ -67,7 +67,7 @@ module XapianDb
 
           # Process the objects in batches to reduce the memory footprint
           nr_of_batches = (obj_count / BATCH_SIZE) + 1
-          order_expression = "#{klass.name.tableize}.#{primary_key}"
+          order_expression = "#{klass.name.parameterize.tableize}.#{primary_key}"
           # raise "vor loop"
           nr_of_batches.times do |batch|
             base_query.all(:offset => batch * BATCH_SIZE, :limit => BATCH_SIZE, :order => order_expression).each do |obj|
@@ -82,4 +82,4 @@ module XapianDb
       end
     end
   end
-end
+end

data/lib/xapian_db/index_writers/no_op_writer.rb

@@ -8,22 +8,24 @@ module XapianDb
 
     class NoOpWriter
 
-      # Update an object in the index
-      # @param [Object] obj An instance of a class with a blueprint configuration
-      def index(obj, commit=true); end
+      class << self
 
-      # Remove an object from the index
-      # @param [String] xapian_id The document id
-      def delete_doc_with(xapian_id, commit=true); end
+        # Update an object in the index
+        # @param [Object] obj An instance of a class with a blueprint configuration
+        def index(obj, commit=true); end
 
-      # Reindex all objects of a given class
-      # @param [Class] klass The class to reindex
-      # @param [Hash] options Options for reindexing
-      # @option options [Boolean] :verbose (false) Should the reindexing give status informations?
-      def reindex_class(klass, options={})
-        raise "rebuild_xapian_index is not supported inside a block with auto indexing disabled"
-      end
+        # Remove an object from the index
+        # @param [String] xapian_id The document id
+        def delete_doc_with(xapian_id, commit=true); end
 
+        # Reindex all objects of a given class
+        # @param [Class] klass The class to reindex
+        # @param [Hash] options Options for reindexing
+        # @option options [Boolean] :verbose (false) Should the reindexing give status informations?
+        def reindex_class(klass, options={})
+          raise "rebuild_xapian_index is not supported inside a block with auto indexing disabled"
+        end
+      end
     end
   end
 end

data/lib/xapian_db/index_writers/resque_worker.rb

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+module XapianDb
+  module IndexWriters
+    # Worker to update the Xapian index; the worker will be called by resque
+    # and uses the DirectWriter to do the real work
+    # @author Michael Stämpfli
+    class ResqueWorker
+
+      extend XapianDb::Utilities
+
+      APPROVED_TASKS = [:index, :delete_doc, :reindex_class]
+
+      class << self
+        def queue
+          XapianDb::Config.resque_queue
+        end
+
+        def perform(task, options)
+          send(task, options) if APPROVED_TASKS.include?(task.to_sym)
+        end
+
+        def index(options)
+          klass = constantize options['class']
+          obj   = klass.respond_to?('get') ? klass.get(options['id']) : klass.find(options['id'])
+          DirectWriter.index obj
+        end
+
+        def delete_doc(options)
+          DirectWriter.delete_doc_with options['xapian_id']
+        end
+
+        def reindex_class(options)
+          klass = constantize options['class']
+          DirectWriter.reindex_class klass, :verbose => false
+        end
+      end
+
+    end
+  end
+end

data/lib/xapian_db/index_writers/resque_writer.rb

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# This writer uses resque to enqueue index jobs
+# @author Michael Stämpfli
+
+require 'resque'
+
+module XapianDb
+  module IndexWriters
+    class ResqueWriter
+
+      class << self
+
+        # Update an object in the index
+        # @param [Object] obj An instance of a class with a blueprint configuration
+        def index(obj, commit=true)
+          Resque.enqueue worker_class, :index, :class => obj.class.name, :id => obj.id
+        end
+
+        # Remove an object from the index
+        # @param [String] xapian_id The document id
+        def delete_doc_with(xapian_id, commit=true)
+          Resque.enqueue worker_class, :delete_doc, :xapian_id => xapian_id
+        end
+
+        # Reindex all objects of a given class
+        # @param [Class] klass The class to reindex
+        def reindex_class(klass, options = {})
+          Resque.enqueue worker_class, :reindex_class, :class => klass.name
+        end
+
+        def worker_class
+          ResqueWorker
+        end
+        private :worker_class
+      end
+
+    end
+  end
+end

data/lib/xapian_db/railtie.rb

@@ -37,6 +37,7 @@ module XapianDb
         config.adapter @adapter.try(:to_sym)
         config.writer @writer.try(:to_sym)
         config.beanstalk_daemon_url @beanstalk_daemon
+        config.resque_queue @resque_queue
         config.language @language.try(:to_sym)
       end
 
@@ -57,6 +58,7 @@ module XapianDb
       @adapter          = env_config["adapter"]  || :active_record
       @writer           = env_config["writer"]   || :direct
       @beanstalk_daemon = env_config["beanstalk_daemon"]
+      @resque_queue     = env_config["resque_queue"]
       @language         = env_config["language"]
     end
 
@@ -69,4 +71,4 @@ module XapianDb
     end
 
   end
-end
+end

data/lib/xapian_db.rb

@@ -168,7 +168,7 @@ module XapianDb
 
   # Execute a block and do not update the index
   def self.auto_indexing_disabled(&block)
-    execute_block :writer => XapianDb::IndexWriters::NoOpWriter.new do
+    execute_block :writer => XapianDb::IndexWriters::NoOpWriter do
       block.call
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: xapian_db
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-25 00:00:00.000000000 +02:00
+date: 2011-11-07 00:00:00.000000000 +01:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &70215097800680 !ruby/object:Gem::Requirement
+  requirement: &70141547189340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +22,10 @@ dependencies:
         version: 1.0.10
   type: :runtime
   prerelease: false
-  version_requirements: *70215097800680
+  version_requirements: *70141547189340
 - !ruby/object:Gem::Dependency
   name: xapian-ruby
-  requirement: &70215097800160 !ruby/object:Gem::Requirement
+  requirement: &70141547188660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +33,10 @@ dependencies:
         version: 1.2.6
   type: :runtime
   prerelease: false
-  version_requirements: *70215097800160
+  version_requirements: *70141547188660
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70215097799660 !ruby/object:Gem::Requirement
+  requirement: &70141547187880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +44,10 @@ dependencies:
         version: 2.3.1
   type: :development
   prerelease: false
-  version_requirements: *70215097799660
+  version_requirements: *70141547187880
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70215097799180 !ruby/object:Gem::Requirement
+  requirement: &70141547187160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +55,10 @@ dependencies:
         version: 0.3.7
   type: :development
   prerelease: false
-  version_requirements: *70215097799180
+  version_requirements: *70141547187160
 - !ruby/object:Gem::Dependency
   name: beanstalk-client
-  requirement: &70215097798700 !ruby/object:Gem::Requirement
+  requirement: &70141547186400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,7 +66,40 @@ dependencies:
         version: 1.1.0
   type: :development
   prerelease: false
-  version_requirements: *70215097798700
+  version_requirements: *70141547186400
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70141547185540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70141547185540
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: &70141547200900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70141547200900
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: &70141547200060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.19.0
+  type: :development
+  prerelease: false
+  version_requirements: *70141547200060
 description: XapianDb is a ruby gem that combines features of nosql databases and
   fulltext indexing. It is based on Xapian, an efficient and powerful indexing library
 email: gernot.kogler (at) garaio (dot) com
@@ -88,6 +121,8 @@ files:
 - lib/xapian_db/index_writers/beanstalk_writer.rb
 - lib/xapian_db/index_writers/direct_writer.rb
 - lib/xapian_db/index_writers/no_op_writer.rb
+- lib/xapian_db/index_writers/resque_worker.rb
+- lib/xapian_db/index_writers/resque_writer.rb
 - lib/xapian_db/index_writers/transactional_writer.rb
 - lib/xapian_db/indexer.rb
 - lib/xapian_db/query_parser.rb