ts-delayed-delta 1.1.3 → 2.0.0

data/.gitignore

@@ -18,6 +18,7 @@ coverage
 rdoc
 pkg
 Gemfile.lock
+gemfiles/*.gemfile.lock
 
 ## PROJECT::SPECIFIC
 features/thinking_sphinx/database.yml

data/.travis.yml

@@ -1,16 +1,18 @@
+language: ruby
 rvm:
-  # - 1.8.6
-  - 1.8.7
   - 1.9.2
   - 1.9.3
-  - rbx
-  - rbx-2.0
-  - ree
-#   - jruby
-#   - ruby-head
-# env:
-#   - SPHINX_BIN=/usr/local/sphinx-0.9.9/bin SPHINX_VERSION=0.9.9
-#   - SPHINX_BIN=/usr/local/sphinx-1.10/bin SPHINX_VERSION=1.10
-#   - SPHINX_BIN=/usr/local/sphinx-2.0.1/bin SPHINX_VERSION=2.0.1
+before_install:
+  - gem update --system
+  - gem install bundler --no-ri --no-rdoc
+  - curl -O http://fs-packages.s3.amazonaws.com/fs-sphinx-2.0.6_x86_64_12.04.deb
+  - sudo dpkg -i fs-sphinx-2.0.6_x86_64_12.04.deb
+  - bundle install
 before_script:
-  - "mysql -e 'create database thinking_sphinx;' > /dev/null"
+  - "mysql -e 'create database thinking_sphinx;' > /dev/null"
+  - "psql -c 'create database thinking_sphinx;' -U postgres > /dev/null"
+  - "bundle exec rake appraisal:install"
+env:
+  - SPHINX_BIN=/usr/local/sphinx-2.0.6/bin/ SPHINX_VERSION=2.0.6 DATABASE=mysql2
+  - SPHINX_BIN=/usr/local/sphinx-2.0.6/bin/ SPHINX_VERSION=2.0.6 DATABASE=postgresql
+script: "bundle exec rake appraisal spec"

data/Appraisals

@@ -0,0 +1,11 @@
+appraise "binary" do
+  gem 'thinking-sphinx',
+    :git => 'git://github.com/pat/thinking-sphinx.git',
+    :ref => '94851c0e6b'
+end
+
+appraise "sphinxql" do
+  gem 'thinking-sphinx',
+    :git => 'git://github.com/pat/thinking-sphinx.git',
+    :ref => '86ac3edfbe'
+end

data/Gemfile

@@ -1,5 +1,3 @@
-source :rubygems
+source 'https://rubygems.org'
 
 gemspec
-
-gem 'rcov', '>= 0.9.8', :platform => :ruby_18

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Pat Allan
+Copyright (c) 2009-2012 Pat Allan
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.textile

@@ -2,48 +2,49 @@ h1. Delayed Deltas for Thinking Sphinx
 
 h2. Installation
 
-You'll need Thinking Sphinx 1.3.0 or later, and Delayed Job as well. The latter is flagged as a dependency.
+This version requires Thinking Sphinx 3.x (currently the edge branch) and Delayed Job 3.x.
 
-<pre><code>gem install ts-delayed-delta --source http://gemcutter.org</code></pre>
+In your @Gemfile@, it'll likely look something like this:
 
-In your @environment.rb@ file, with the rest of your gem dependencies:
+<pre><code>gem 'ts-delayed-delta', '~> 2.0.0'</code></pre>
 
-<pre><code>config.gem 'ts-delayed-delta',
-  :lib     => 'thinking_sphinx/deltas/delayed_delta',
-  :version => '>= 1.0.0',
-  :source  => 'http://gemcutter.org'</code></pre>
+Or, if you're referring directly to the git repository:
 
-And add the following line to the bottom of your @Rakefile@ if you're not using Rails 3 or newer:
+<pre><code>gem 'ts-delayed-delta',
+  :git    => 'git://github.com/pat/ts-delayed-delta.git',
+  :branch => 'edge',
+  :ref    => 'use-latest-commit-here'</code></pre>
 
-<pre><code>require 'thinking_sphinx/deltas/delayed_delta/tasks'</code></pre>
+And given this gem's built for Delayed Job, we're presuming you've got that set up (with the database table), and you know how to run it. This gem no longer provides a custom task for that purpose - there's no need.
 
-If this is your first time running Delayed Job, then you're going to need the jobs table migration as well:
-
-<pre><code>script/generate delayed_job</code></pre>
+h2. Usage
 
-For the indices you want to use this delta approach, make sure you set that up in their @define_index@ blocks.
+In your index definitions, make sure you set the @:delta@ setting to @ThinkingSphinx::Deltas::DelayedDelta@:
 
-<pre><code>define_index do
+<pre><code>ThinkingSphinx::Index.define :book, :with => :active_record, :delta => ThinkingSphinx::Deltas::DelayedDelta do
   # ...
-
-  set_property :delta => :delayed
 end</code></pre>
 
-If you've never used delta indices before, you'll want to add the boolean column named delta to each model that is using the approach.
+If you've never used delta indices before, you'll want to add the boolean column named delta to each model that is using the approach. It probably doesn't hurt to add an index to that column as well.
 
-<pre><code>def self.up
-  add_column :articles, :delta, :boolean, :default => true, :null => false
-end</code></pre>
+<pre><code>add_column :books, :delta, :boolean, :default => true, :null => false
+add_index  :books, :delta</code></pre>
 
-h2. Usage
+And that's pretty much it. Make sure Delayed Job is running, and it'll take care of the jobs to keep your Sphinx indices up to date.
+
+Please keep in mind that you need to run the Delayed Job workers on the same machine as Sphinx, as direct access to Sphinx files (and the Sphinx command-line tools) is essential.
+
+h2. Limitations
 
-Once you've got it all set up, all you need to do is make sure that the delayed job process is running - either by Delayed Job's built-in approach, or Thinking Sphinx's custom rake task:
+This version of the gem is built for Ruby 1.9 and Rails 3.1 or newer, just like Thinking Sphinx 3.
 
-<pre><code>rake thinking_sphinx:delayed_delta</code></pre>
+h2. Contributing
 
-There's also a short name for the same task, to save your fingers some effort:
+Contributions are very much welcome - but keep in mind the following:
 
-<pre><code>rake ts:dd</code></pre>
+* Keep patches in a separate branch
+* Don't mess with the version number. I'll take care of that when the patch is merged in.
+* Please write tests - currently, there's only acceptance tests in place, because the underlying code is simple enough. If you get stuck into anything complex, unit tests are a smart move.
 
 h2. Contributors
 
@@ -56,7 +57,9 @@ h2. Contributors
 * "Reinier de Lange":http://www.nedforce.nl/ (Fix for table name reference)
 * "Enrico Brunetta":http://github.com/enrico (Adding Railtie for Rails 3)
 * "Jonathan Viney":https://github.com/jviney and "James Healy":http://yob.id.au/ (Rails 3.2 deprecation fixes)
+* "Georg Ledermann":http://www.georg-ledermann.de (DJ 2.0.x backwards compatibility)
+* "Jonathan Lim":http://snowblink.co.uk (Cleaning up version references)
 
 h2. Copyright
 
-Copyright (c) 2009-2010 Pat Allan, and released under an MIT Licence.
+Copyright (c) 2009-2012, Thinking Sphinx Delayed Deltas (ts-delayed-delta) is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/ts-delayed-delta/contributors.

data/Rakefile

@@ -1,23 +1,10 @@
-require 'rubygems'
-require 'bundler'
+require 'bundler/setup'
 
 Bundler::GemHelper.install_tasks
-Bundler.require :default, :development
 
+require 'appraisal'
 require 'rspec/core/rake_task'
-require 'cucumber/rake/task'
 
 RSpec::Core::RakeTask.new
 
-RSpec::Core::RakeTask.new(:rcov) do |spec|
-  spec.rcov_opts = ['--exclude', 'spec', '--exclude', 'gems']
-  spec.rcov      = true
-end
-
-Cucumber::Rake::Task.new do |task|
-  task.cucumber_opts = '--exclude features/thinking_sphinx'
-end
-
-YARD::Rake::YardocTask.new
-
-task :default => [:spec, :cucumber]
+task :default => :spec

data/gemfiles/binary.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "thinking-sphinx", :git=>"git://github.com/pat/thinking-sphinx.git", :ref=>"94851c0e6b"
+
+gemspec :path=>"../"

data/gemfiles/sphinxql.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "thinking-sphinx", :git=>"git://github.com/pat/thinking-sphinx.git", :ref=>"86ac3edfbe"
+
+gemspec :path=>"../"

data/lib/thinking_sphinx/deltas/delayed_delta.rb

@@ -1,10 +1,5 @@
 require 'delayed_job'
-
 require 'thinking_sphinx'
-require 'thinking_sphinx/deltas/delayed_delta/delta_job'
-require 'thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job'
-require 'thinking_sphinx/deltas/delayed_delta/job'
-require 'thinking_sphinx/deltas/delayed_delta/version'
 
 # Delayed Deltas for Thinking Sphinx, using Delayed Job.
 #
@@ -16,52 +11,127 @@ require 'thinking_sphinx/deltas/delayed_delta/version'
 # @author Patrick Allan
 # @see http://ts.freelancing-gods.com Thinking Sphinx
 #
-class ThinkingSphinx::Deltas::DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
-
-  # Adds a job to the queue for processing the given model's delta index. A job
-  # for hiding the instance in the core index is also created, if an instance is
-  # provided.
-  #
-  # Neither job will be queued if updates or deltas are disabled, or if the
-  # instance (when given) is not toggled to be in the delta index. The first two
-  # options are controlled via ThinkingSphinx.updates_enabled? and
-  # ThinkingSphinx.deltas_enabled?.
-  #
-  # @param [Class] model the ActiveRecord model to index.
-  # @param [ActiveRecord::Base] instance the instance of the given model that
-  #   has changed. Optional.
-  # @return [Boolean] true
-  #
-  def index(model, instance = nil)
-    return true if skip? instance
-    return true if instance && !toggled(instance)
-
-    ThinkingSphinx::Deltas::Job.enqueue(
-      ThinkingSphinx::Deltas::DeltaJob.new(model.delta_index_names),
-      ThinkingSphinx::Configuration.instance.delayed_job_priority
+class ThinkingSphinx::Deltas::DelayedDelta <
+  ThinkingSphinx::Deltas::DefaultDelta
+
+  def self.cancel_jobs
+    Delayed::Job.delete_all(
+      "handler LIKE '--- !ruby/object:ThinkingSphinx::Deltas::%'"
     )
+  end
+
+  def self.enqueue_unless_duplicates(object)
+    if Delayed::Job.respond_to?(:where)
+      return if Delayed::Job.where(
+        :handler => object.to_yaml, :locked_at => nil
+      ).count > 0
+    else
+      return if Delayed::Job.count(
+        :conditions => {:handler => object.to_yaml, :locked_at => nil}
+      ) > 0
+    end
 
-    Delayed::Job.enqueue(
-      ThinkingSphinx::Deltas::FlagAsDeletedJob.new(
-        model.core_index_names, instance.sphinx_document_id
-      ),
-      :priority => ThinkingSphinx::Configuration.instance.delayed_job_priority
-    ) if instance
+    Delayed::Job.enqueue object, priority_option
+  end
 
-    true
+  def self.priority_option
+    if Gem.loaded_specs['delayed_job'].version.to_s.match(/^2\.0\./)
+      # Fallback for compatibility with old release 2.0.x of DJ
+      priority
+    else
+      {:priority => priority}
+    end
   end
 
-  private
-
-  # Checks whether jobs should be enqueued. Only true if updates and deltas are
-  # enabled, and the instance (if there is one) is toggled.
-  #
-  # @param [ActiveRecord::Base, NilClass] instance
-  # @return [Boolean]
-  #
-  def skip?(instance)
-    !ThinkingSphinx.updates_enabled? ||
-    !ThinkingSphinx.deltas_enabled?  ||
-    (instance && !toggled(instance))
+  def self.priority
+    configuration = ThinkingSphinx::Configuration.instance
+    if configuration.respond_to? :delayed_job_priority
+      configuration.delayed_job_priority
+    else
+      configuration.settings['delayed_job_priority'] || 0
+    end
+  end
+
+  module Binary
+    # Adds a job to the queue for processing the given model's delta index. A job
+    # for hiding the instance in the core index is also created, if an instance is
+    # provided.
+    #
+    # Neither job will be queued if updates or deltas are disabled, or if the
+    # instance (when given) is not toggled to be in the delta index. The first two
+    # options are controlled via ThinkingSphinx.updates_enabled? and
+    # ThinkingSphinx.deltas_enabled?.
+    #
+    # @param [Class] model the ActiveRecord model to index.
+    # @param [ActiveRecord::Base] instance the instance of the given model that
+    #   has changed. Optional.
+    # @return [Boolean] true
+    #
+    def index(model, instance = nil)
+      return true if skip? instance
+
+      self.class.enqueue_unless_duplicates(
+        ThinkingSphinx::Deltas::DelayedDelta::DeltaJob.new(
+          model.delta_index_names
+        )
+      )
+
+      Delayed::Job.enqueue(
+        ThinkingSphinx::Deltas::DelayedDelta::FlagAsDeletedJob.new(
+          model.core_index_names, instance.sphinx_document_id
+        ), self.class.priority_option
+      ) if instance
+
+      true
+    end
+
+    private
+
+    # Checks whether jobs should be enqueued. Only true if updates and deltas are
+    # enabled, and the instance (if there is one) is toggled.
+    #
+    # @param [ActiveRecord::Base, NilClass] instance
+    # @return [Boolean]
+    #
+    def skip?(instance)
+      !ThinkingSphinx.updates_enabled? ||
+      !ThinkingSphinx.deltas_enabled?  ||
+      (instance && !toggled(instance))
+    end
+  end
+
+  module SphinxQL
+    def delete(index, instance)
+      Delayed::Job.enqueue(
+        ThinkingSphinx::Deltas::DelayedDelta::FlagAsDeletedJob.new(
+          index.name, index.document_id_for_key(instance.id)
+        ), self.class.priority_option
+      )
+    end
+
+    # Adds a job to the queue for processing the given index.
+    #
+    # @param [Class] index the Thinking Sphinx index object.
+    #
+    def index(index)
+      self.class.enqueue_unless_duplicates(
+        ThinkingSphinx::Deltas::DelayedDelta::DeltaJob.new(index.name)
+      )
+    end
+  end
+
+  if [:delayed_job_priority, 'delayed_job_priority'].any? { |method|
+    ThinkingSphinx::Configuration.instance_methods.include?(method)
+  }
+    include Binary
+  else
+    include SphinxQL
   end
 end
+
+require 'thinking_sphinx/deltas/delayed_delta/delta_job'
+require 'thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job'
+
+ThinkingSphinx.before_index_hooks << Proc.new {
+  ThinkingSphinx::Deltas::DelayedDelta.cancel_jobs
+}

data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb

@@ -1,33 +1,24 @@
 # A simple job class that processes a given index.
 #
-class ThinkingSphinx::Deltas::DeltaJob
-  attr_accessor :indices
-
+class ThinkingSphinx::Deltas::DelayedDelta::DeltaJob
   # Initialises the object with an index name.
   #
   # @param [String] index the name of the Sphinx index
   #
-  def initialize(indices)
-    @indices = indices
+  def initialize(index)
+    @index = index
   end
 
   # Shows index name in Delayed::Job#name.
   #
   def display_name
-    "#{self.class.name} for #{indices.join(', ')}"
+    "Thinking Sphinx: Process #{@index}"
   end
 
-  # Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx is
-  # running.
-  #
-  # @return [Boolean] true
+  # Processes just the given index. Output is hidden only if the quiet_deltas
+  # setting is true.
   #
   def perform
-    config = ThinkingSphinx::Configuration.instance
-
-    output = `#{config.bin_path}#{config.indexer_binary_name} --config "#{config.config_file}" --rotate #{indices.join(' ')}`
-    puts output unless ThinkingSphinx.suppress_delta_output?
-
-    true
+    ThinkingSphinx::Deltas::IndexJob.new(@index).perform
   end
 end

data/lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb

@@ -1,9 +1,7 @@
 # A simple job for flagging a specified Sphinx document in a given index as
 # 'deleted'.
 #
-class ThinkingSphinx::Deltas::FlagAsDeletedJob
-  attr_accessor :indices, :document_id
-
+class ThinkingSphinx::Deltas::DelayedDelta::FlagAsDeletedJob
   # Initialises the object with an index name and document id. Please note that
   # the document id is Sphinx's unique identifier, and will almost certainly not
   # be the model instance's primary key value.
@@ -11,8 +9,12 @@ class ThinkingSphinx::Deltas::FlagAsDeletedJob
   # @param [String] index The index name
   # @param [Integer] document_id The document id
   #
-  def initialize(indices, document_id)
-    @indices, @document_id = indices, document_id
+  def initialize(index, document_id)
+    @index, @document_id = index, document_id
+  end
+
+  def display_name
+    "Thinking Sphinx: Mark #{@document_id} in #{@index} as deleted"
   end
 
   # Updates the sphinx_deleted attribute for the given document, setting the
@@ -21,20 +23,7 @@ class ThinkingSphinx::Deltas::FlagAsDeletedJob
   # particularly useful in this situation to avoid old values in the core index
   # and just use the new values in the delta index as a reference point.
   #
-  # @return [Boolean] true
-  #
   def perform
-    config = ThinkingSphinx::Configuration.instance
-
-    indices.each do |index|
-      config.client.update(
-        index,
-        ['sphinx_deleted'],
-        {@document_id => [1]}
-      ) if ThinkingSphinx.sphinx_running? &&
-        ThinkingSphinx.search_for_id(@document_id, index)
-    end
-
-    true
+    ThinkingSphinx::Deltas::DeleteJob.new(@index, @document_id).perform
   end
 end