flying-sphinx 0.2.6 → 0.3.0

data/VERSION

@@ -1 +1 @@
-0.2.6
+0.3.0

data/lib/flying_sphinx.rb

@@ -8,6 +8,8 @@ require 'riddle/0.9.9'
 
 require 'flying_sphinx/api'
 require 'flying_sphinx/configuration'
+require 'flying_sphinx/delayed_delta'
+require 'flying_sphinx/flag_as_deleted_job'
 require 'flying_sphinx/heroku_shared_adapter'
 require 'flying_sphinx/index_request'
 require 'flying_sphinx/tunnel'

data/lib/flying_sphinx/delayed_delta.rb

@@ -0,0 +1,85 @@
+class FlyingSphinx::DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
+  # Adds a job to the queue, if it doesn't already exist. This is to ensure
+  # multiple indexing requests for the same delta index don't get added, as the
+  # index only needs to be processed once.
+  # 
+  # Because indexing jobs are all the same object, they all get serialised to
+  # the same YAML value.
+  # 
+  # @param [Object] object The job, which must respond to the #perform method.
+  # @param [Integer] priority (0)
+  # 
+  def self.enqueue(object, priority = 0)
+    return if duplicates_exist? object
+    
+    ::Delayed::Job.enqueue(object, :priority => priority)
+  end
+  
+  # Checks whether a given job already exists in the queue.
+  # 
+  # @param [Object] object The job
+  # @return [Boolean] True if a duplicate of the job already exists in the queue
+  # 
+  def self.duplicates_exist?(object)
+    ::Delayed::Job.count(
+      :conditions => {
+        :handler    => object.to_yaml,
+        :locked_at  => nil
+      }
+    ) > 0
+  end
+  
+  # Adds a job to the queue for processing the given model's delta index. A job
+  # for hiding the instance in the core index is also created, if an instance is
+  # provided.
+  # 
+  # Neither job will be queued if updates or deltas are disabled, or if the 
+  # instance (when given) is not toggled to be in the delta index. The first two
+  # options are controlled via ThinkingSphinx.updates_enabled? and
+  # ThinkingSphinx.deltas_enabled?.
+  # 
+  # @param [Class] model the ActiveRecord model to index.
+  # @param [ActiveRecord::Base] instance the instance of the given model that
+  #   has changed. Optional.
+  # @return [Boolean] true
+  #
+  def index(model, instance = nil)
+    return true if skip? instance
+    
+    self.class.enqueue(
+      FlyingSphinx::IndexRequest.new(model.delta_index_names),
+      delayed_job_priority
+    )
+    
+    Delayed::Job.enqueue(
+      FlyingSphinx::FlagAsDeletedJob.new(
+        model.core_index_names, instance.sphinx_document_id
+      ),
+      :priority => delayed_job_priority
+    ) if instance
+    
+    true
+  end
+  
+  private
+  
+  def config
+    @config ||= ThinkingSphinx::Configuration.instance
+  end
+  
+  def delayed_job_priority
+    config.delayed_job_priority
+  end
+  
+  # Checks whether jobs should be enqueued. Only true if updates and deltas are
+  # enabled, and the instance (if there is one) is toggled.
+  # 
+  # @param [ActiveRecord::Base, NilClass] instance
+  # @return [Boolean]
+  # 
+  def skip?(instance)
+    !ThinkingSphinx.updates_enabled? ||
+    !ThinkingSphinx.deltas_enabled?  ||
+    (instance && !toggled(instance))
+  end
+end

data/lib/flying_sphinx/flag_as_deleted_job.rb

@@ -0,0 +1,45 @@
+# A simple job for flagging a specified Sphinx document in a given index as
+# 'deleted'.
+#  
+class FlyingSphinx::FlagAsDeletedJob
+  attr_accessor :indices, :document_id
+  
+  # Initialises the object with an index name and document id. Please note that
+  # the document id is Sphinx's unique identifier, and will almost certainly not
+  # be the model instance's primary key value.
+  # 
+  # @param [String] index The index name
+  # @param [Integer] document_id The document id
+  # 
+  def initialize(indices, document_id)
+    @indices, @document_id = indices, document_id
+  end
+  
+  # Updates the sphinx_deleted attribute for the given document, setting the
+  # value to 1 (true). This is not a special attribute in Sphinx, but is used
+  # by Thinking Sphinx to ignore deleted values between full re-indexing. It's
+  # particularly useful in this situation to avoid old values in the core index
+  # and just use the new values in the delta index as a reference point.
+  # 
+  # @return [Boolean] true
+  # 
+  def perform
+    indices.each do |index|
+      client.update(
+        index,
+        ['sphinx_deleted'],
+        {@document_id => [1]}
+      ) if ThinkingSphinx.search_for_id(@document_id, index)
+    end
+  rescue Riddle::ConnectionError
+    # If it fails here, so be it.
+  ensure
+    true
+  end
+  
+  private
+  
+  def client
+    @client ||= ThinkingSphinx::Configuration.instance.client
+  end
+end

data/lib/flying_sphinx/index_request.rb

@@ -1,26 +1,60 @@
 class FlyingSphinx::IndexRequest
-  attr_reader :configuration, :index_id
+  attr_reader :index_id, :indices
   
-  def initialize(configuration)
-    @configuration = configuration
+  # Remove all Delta jobs from the queue. If the
+  # delayed_jobs table does not exist, this method will do nothing.
+  # 
+  def self.cancel_jobs
+    return unless ::Delayed::Job.table_exists?
     
+    ::Delayed::Job.delete_all "handler LIKE '--- !ruby/object:FlyingSphinx::%'"
+  end
+  
+  def initialize(indices = [])
+    @indices = indices
+  end
+  
+  # Shows index name in Delayed::Job#name.
+  # 
+  def display_name
+    "#{self.class.name} for #{indices.join(', ')}"
+  end
+  
+  def update_and_index
     update_sphinx_configuration
-    
-    FlyingSphinx::Tunnel.connect(configuration) do
-      begin_request unless request_begun?
-      
-      !request_complete?
-    end
+    index
+  end
+  
+  # Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx is
+  # running.
+  # 
+  # @return [Boolean] true
+  # 
+  def perform
+    index
+    true
   end
   
   private
   
+  def configuration
+    @configuration ||= FlyingSphinx::Configuration.new
+  end
+  
   def update_sphinx_configuration
     api.put '/app', :configuration => configuration.sphinx_configuration
   end
   
+  def index
+    FlyingSphinx::Tunnel.connect(configuration) do
+      begin_request unless request_begun?
+      
+      !request_complete?
+    end
+  end
+  
   def begin_request
-    @index_id      = api.post('/app/indices')
+    @index_id      = api.post('/app/indices', :indices => indices.join(','))
     @request_begun = true
   end
   

data/lib/flying_sphinx/tasks.rb

@@ -1,7 +1,8 @@
 namespace :fs do
   task :index => :environment do
     puts "Starting Index Request"
-    FlyingSphinx::IndexRequest.new FlyingSphinx::Configuration.new
+    FlyingSphinx::IndexRequest.cancel_jobs
+    FlyingSphinx::IndexRequest.new.update_and_index
     puts "Index Request has completed"
   end
   

data/spec/flying_sphinx/delayed_delta_spec.rb

@@ -0,0 +1,146 @@
+require 'spec_helper'
+
+describe FlyingSphinx::DelayedDelta do
+  describe '.enqueue' do
+    before :each do
+      Delayed::Job.stub!(:count => 0)
+    end
+    
+    it "should enqueue if there's no existing jobs for the same index" do
+      Delayed::Job.should_receive(:enqueue)
+      
+      FlyingSphinx::DelayedDelta.enqueue(stub('object'))
+    end
+    
+    it "should not enqueue the job if there's an existing job already" do
+      Delayed::Job.stub!(:count => 1)
+      Delayed::Job.should_not_receive(:enqueue)
+      
+      FlyingSphinx::DelayedDelta.enqueue(stub('object'))
+    end
+  end
+  
+  describe '#index' do
+    let(:config)        { ThinkingSphinx::Configuration.instance }
+    let(:delayed_delta) { FlyingSphinx::DelayedDelta.new stub('instance'), {} }
+    let(:model)         {
+      stub 'foo',
+        :name              => 'foo',
+        :core_index_names  => ['foo_core'],
+        :delta_index_names => ['foo_delta']
+    }
+    let(:instance) { stub('instance', :sphinx_document_id => 42) }
+    
+    before :each do
+      ThinkingSphinx.updates_enabled = true
+      ThinkingSphinx.deltas_enabled  = true
+      
+      config.delayed_job_priority = 2
+      
+      FlyingSphinx::DelayedDelta.stub!(:enqueue => true)
+      Delayed::Job.stub!(:enqueue => true, :inspect => 'Delayed::Job')
+      
+      delayed_delta.stub!(:toggled => true)
+    end
+    
+    context 'updates disabled' do
+      before :each do
+        ThinkingSphinx.updates_enabled = false
+      end
+      
+      it "should not enqueue a delta job" do
+        FlyingSphinx::DelayedDelta.should_not_receive(:enqueue)
+        
+        delayed_delta.index model
+      end
+      
+      it "should not enqueue a flag as deleted job" do
+        Delayed::Job.should_not_receive(:enqueue)
+        
+        delayed_delta.index model
+      end
+    end
+    
+    context 'deltas disabled' do
+      before :each do
+        ThinkingSphinx.deltas_enabled = false
+      end
+      
+      it "should not enqueue a delta job" do
+        FlyingSphinx::DelayedDelta.should_not_receive(:enqueue)
+        
+        delayed_delta.index model
+      end
+      
+      it "should not enqueue a flag as deleted job" do
+        Delayed::Job.should_not_receive(:enqueue)
+        
+        delayed_delta.index model
+      end
+    end
+    
+    context "instance isn't toggled" do
+      before :each do
+        delayed_delta.stub!(:toggled => false)
+      end
+      
+      it "should not enqueue a delta job" do
+        FlyingSphinx::DelayedDelta.should_not_receive(:enqueue)
+        
+        delayed_delta.index model, instance
+      end
+      
+      it "should not enqueue a flag as deleted job" do
+        Delayed::Job.should_not_receive(:enqueue)
+        
+        delayed_delta.index model, instance
+      end
+    end
+    
+    it "should enqueue a delta job for the appropriate indexes" do
+      FlyingSphinx::DelayedDelta.should_receive(:enqueue) do |job, priority|
+        job.indices.should == ['foo_delta']
+      end
+      
+      delayed_delta.index model
+    end
+    
+    it "should use the defined priority for the delta job" do
+      FlyingSphinx::DelayedDelta.should_receive(:enqueue) do |job, priority|
+        priority.should == 2
+      end
+      
+      delayed_delta.index model
+    end
+    
+    it "should enqueue a flag-as-deleted job for the appropriate indexes" do
+      Delayed::Job.should_receive(:enqueue) do |job, options|
+        job.indices.should == ['foo_core']
+      end
+      
+      delayed_delta.index model, instance
+    end
+    
+    it "should enqueue a flag-as-deleted job for the appropriate id" do
+      Delayed::Job.should_receive(:enqueue) do |job, options|
+        job.document_id.should == 42
+      end
+      
+      delayed_delta.index model, instance
+    end
+    
+    it "should use the defined priority for the flag-as-deleted job" do
+      Delayed::Job.should_receive(:enqueue) do |job, options|
+        options[:priority].should == 2
+      end
+      
+      delayed_delta.index model, instance
+    end
+    
+    it "should not enqueue a flag-as-deleted job if no instance is provided" do
+      Delayed::Job.should_not_receive(:enqueue)
+      
+      delayed_delta.index model
+    end
+  end
+end

data/spec/flying_sphinx/flag_as_deleted_job_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+
+describe FlyingSphinx::FlagAsDeletedJob do
+  describe '#perform' do
+    let(:config) { ThinkingSphinx::Configuration.instance }
+    let(:client) { stub('client', :update => true) }
+    let(:job)    { FlyingSphinx::FlagAsDeletedJob.new(['foo_core'], 12) }
+    
+    before :each do
+      config.stub!(:client => client)
+      ThinkingSphinx.stub!(:search_for_id => true)
+    end
+    
+    it "should not update if the document isn't in the index" do
+      ThinkingSphinx.stub!(:search_for_id => false)
+      client.should_not_receive(:update)
+      
+      job.perform
+    end
+    
+    it "should update the specified index" do
+      client.should_receive(:update) do |index, attributes, values|
+        index.should == 'foo_core'
+      end
+      
+      job.perform
+    end
+    
+    it "should update all specified indexes" do
+      job.indices = ['foo_core', 'bar_core']
+      client.should_receive(:update).with('foo_core', anything, anything)
+      client.should_receive(:update).with('bar_core', anything, anything)
+      
+      job.perform
+    end
+    
+    it "should update the sphinx_deleted attribute" do
+      client.should_receive(:update) do |index, attributes, values|
+        attributes.should == ['sphinx_deleted']
+      end
+      
+      job.perform
+    end
+    
+    it "should set sphinx_deleted for the given document to true" do
+      client.should_receive(:update) do |index, attributes, values|
+        values[12].should == [1]
+      end
+      
+      job.perform
+    end
+    
+    it "should check for the existence of the document in the specified index" do
+      ThinkingSphinx.should_receive(:search_for_id) do |id, index|
+        index.should == 'foo_core'
+      end
+      
+      job.perform
+    end
+    
+    it "should check for the existence of the given document id" do
+      ThinkingSphinx.should_receive(:search_for_id) do |id, index|
+        id.should == 12
+      end
+      
+      job.perform
+    end
+  end
+end

data/spec/flying_sphinx/index_request_spec.rb

@@ -2,28 +2,106 @@ require 'spec_helper'
 
 describe FlyingSphinx::IndexRequest do
   let(:api)           { FlyingSphinx::API.new 'foo', 'bar' }
-  let(:configuration) { stub(:configuration, :api => api) }
+  let(:configuration) {
+    stub(:configuration, :api => api, :sphinx_configuration => 'foo {}')
+  }
   
   before :each do
+    FlyingSphinx::Configuration.stub!(:new => configuration)
     FlyingSphinx::Tunnel.stub(:connect) { |config, block| block.call }
   end
   
-  describe '#initialize' do
+  describe '.cancel_jobs' do
+    before :each do
+      Delayed::Job.stub!(:delete_all => true)
+    end
+    
+    it "should not delete any rows if the delayed_jobs table does not exist" do
+      Delayed::Job.stub!(:table_exists? => false)
+      Delayed::Job.should_not_receive(:delete_all)
+      
+      FlyingSphinx::IndexRequest.cancel_jobs
+    end
+    
+    it "should delete rows if the delayed_jobs table does exist" do
+      Delayed::Job.stub!(:table_exists? => true)
+      Delayed::Job.should_receive(:delete_all)
+      
+      FlyingSphinx::IndexRequest.cancel_jobs
+    end
+    
+    it "should delete only Thinking Sphinx jobs" do
+      Delayed::Job.stub!(:table_exists? => true)
+      Delayed::Job.should_receive(:delete_all) do |sql|
+        sql.should match(/handler LIKE '--- !ruby\/object:FlyingSphinx::\%'/)
+      end
+      
+      FlyingSphinx::IndexRequest.cancel_jobs
+    end
+  end
+  
+  describe '#update_and_index' do
+    let(:index_request) { FlyingSphinx::IndexRequest.new }
+    let(:conf_params)   { {:configuration => 'foo {}'} }
+    let(:index_params)  { {:indices => ''} }
+    
     it "makes a new request" do
-      api.should_receive(:post).with('/app/indices').and_return(42)
+      api.should_receive(:put).with('/app', conf_params).and_return('ok')
+      api.should_receive(:post).
+        with('/app/indices', index_params).and_return(42)
       api.should_receive(:get).with('/app/indices/42').and_return('PENDING')
       
       begin
-        Timeout::timeout(0.2) { FlyingSphinx::IndexRequest.new(configuration) }
+        Timeout::timeout(0.2) {
+          index_request.update_and_index
+        }
       rescue Timeout::Error
       end
     end
     
     it "should finish when the index request has been completed" do
-      api.should_receive(:post).with('/app/indices').and_return(42)
+      api.should_receive(:put).with('/app', conf_params).and_return('ok')
+      api.should_receive(:post).
+        with('/app/indices', index_params).and_return(42)
       api.should_receive(:get).with('/app/indices/42').and_return('FINISHED')
       
-      FlyingSphinx::IndexRequest.new(configuration)
+      index_request.update_and_index
+    end
+  end
+  
+  describe '#perform' do
+    let(:index_request) { FlyingSphinx::IndexRequest.new ['foo_delta'] }
+    let(:index_params)  { {:indices => 'foo_delta'} }
+    
+    it "makes a new request" do
+      api.should_receive(:post).
+        with('/app/indices', index_params).and_return(42)
+      api.should_receive(:get).with('/app/indices/42').and_return('PENDING')
+      
+      begin
+        Timeout::timeout(0.2) {
+          index_request.perform
+        }
+      rescue Timeout::Error
+      end
+    end
+    
+    it "should finish when the index request has been completed" do
+      api.should_receive(:post).
+        with('/app/indices', index_params).and_return(42)
+      api.should_receive(:get).with('/app/indices/42').and_return('FINISHED')
+      
+      index_request.perform
+    end
+  end
+  
+  describe "#display_name" do
+    let(:index_request) {
+      FlyingSphinx::IndexRequest.new ['foo_core', 'bar_core']
+    }
+    
+    it "should display class name with all indexes" do
+      index_request.display_name.should == "FlyingSphinx::IndexRequest for foo_core, bar_core"
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: flying-sphinx
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 19
   prerelease: false
   segments: 
   - 0
-  - 2
-  - 6
-  version: 0.2.6
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Pat Allan
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-01 00:00:00 +11:00
+date: 2011-01-02 00:00:00 +11:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -160,6 +160,22 @@ dependencies:
         - 2
         version: 1.2.2
   requirement: *id009
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  name: delayed_job
+  version_requirements: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 1
+        - 2
+        version: 2.1.2
+  requirement: *id010
 description: Hooks Thinking Sphinx into the Flying Sphinx service
 email: pat@freelancing-gods.com
 executables: []
@@ -177,6 +193,8 @@ files:
 - lib/flying_sphinx.rb
 - lib/flying_sphinx/api.rb
 - lib/flying_sphinx/configuration.rb
+- lib/flying_sphinx/delayed_delta.rb
+- lib/flying_sphinx/flag_as_deleted_job.rb
 - lib/flying_sphinx/heroku_shared_adapter.rb
 - lib/flying_sphinx/index_request.rb
 - lib/flying_sphinx/rails.rb
@@ -184,6 +202,8 @@ files:
 - lib/flying_sphinx/tasks.rb
 - lib/flying_sphinx/tunnel.rb
 - spec/flying_sphinx/configuration_spec.rb
+- spec/flying_sphinx/delayed_delta_spec.rb
+- spec/flying_sphinx/flag_as_deleted_job_spec.rb
 - spec/flying_sphinx/index_request_spec.rb
 has_rdoc: true
 homepage: http://flying-sphinx.com
@@ -221,4 +241,6 @@ specification_version: 3
 summary: Sphinx in the Cloud
 test_files: 
 - spec/flying_sphinx/configuration_spec.rb
+- spec/flying_sphinx/delayed_delta_spec.rb
+- spec/flying_sphinx/flag_as_deleted_job_spec.rb
 - spec/flying_sphinx/index_request_spec.rb