ts-sidekiq-delta 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 69d263abd6456994f89536fed3e8ab516e91d029
+  data.tar.gz: d8068a4e447382b7b8c95f08702ba8b7988a13f3
+SHA512:
+  metadata.gz: 7ad79d13705125cdbbc5b7d171c7de7a3de5411189e36845adf9b5db7a4c9a9187c8eb0d91fcf32db56d6e5df377a8762541def04441c43b4c4f490a916653d5
+  data.tar.gz: 31e37ee5b4ae26931e4b0d18eeb275a5ea251c4dac16e6ee9deaebc0332250e3e73fa79431f4a1e254010c44e659341e7c6a8fb5e8341c3db3b9610e624aa5fe

data/.gitignore

@@ -0,0 +1,16 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+*~
+.DS_Store
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp
+coverage
+rdoc
+features/thinking_sphinx/database.yml
+tmp

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,10 @@
+source :rubygems
+
+gemspec
+
+gem 'thinking-sphinx',
+  :git    => 'git://github.com/pat/thinking-sphinx.git',
+  :branch => 'edge'
+gem 'combustion',
+  :git => 'git://github.com/pat/combustion.git',
+  :ref => '45f50e64c3'

data/Guardfile

@@ -0,0 +1,17 @@
+guard 'bundler' do
+  watch('Gemfile')
+  watch(/^.+\.gemspec/)
+end
+
+guard 'rspec', :version => 2, :cli => "-c --format progress", :all_on_start => false do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+
+guard 'cucumber', :all_on_start => false do
+  watch(%r{^features/.+\.feature$})
+  watch(%r{^features/support/.+$})          { 'features' }
+  watch(%r{^features/step_definitions/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'features' }
+end
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Pat Allan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,36 @@
+# Delayed Deltas for Thinking Sphinx with Sidekiq
+
+This code was heavily based on Aaron Gibralter's [ts-resque-delta](https://github.com/agibralter/ts-resque-delta), and was initially adapted for Sidekiq by [Danny Hawkins](https://github.com/danhawkins). This release is maintained by [Pat Allan](https://github.com/pat).
+
+This version of `ts-sidekiq-delta` works only with [Thinking Sphinx](https://github.com/pat/thinking-sphinx) v3 or newer. v1/v2 releases are not supported, and almost certainly will never be. It does work with the Flying Sphinx service, provided you're using 1.0.0 or newer of the `flying-sphinx` gem.
+
+## Installation
+
+Get it into your Gemfile - and don't forget the version constraint!
+
+    gem 'ts-sidekiq-delta', '~> 0.1.0'
+
+## Usage
+
+In your index definitions, you'll want to include the delta setting as an initial option:
+
+    ThinkingSphinx::Index.define(:article,
+      :with  => :active_record,
+      :delta => ThinkingSphinx::Deltas::SidekiqDelta
+    ) do
+      # fields and attributes and such
+    end
+
+If you've never used delta indexes before, you'll want to add the boolean
+column named `:delta` to each model's table and a corresponding database index:
+
+    def change
+      add_column :articles, :delta, :boolean, :default => true, :null => false
+      add_index  :articles, :delta
+    end
+
+From here on in, just use Thinking Sphinx and Sidekiq as you normally would, and you'll find your Sphinx indices are updated quite promptly by Sidekiq.
+
+## Licence
+
+Copyright (c) 2013, ts-sidekiq-delta was originally developed by Danny Hawkins, is currently maintained by Pat Allan, and is released under the open MIT Licence.

data/Rakefile

@@ -0,0 +1,16 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+require 'cucumber'
+require 'cucumber/rake/task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ["-c", "--format progress"]
+end
+
+Cucumber::Rake::Task.new(:features) do |t|
+end
+
+task :all_tests => [:spec, :features]
+
+task :default => :all_tests

data/config/redis-cucumber.conf

@@ -0,0 +1,13 @@
+daemonize yes
+pidfile tmp/redis-cucumber.pid
+port 6398
+timeout 300
+save 900 1
+save 300 10
+save 60 10000
+dbfilename tmp/redis-cucumber-dump.rdb
+dir .
+loglevel debug
+logfile stdout
+databases 16
+glueoutputbuf yes

data/cucumber.yml

@@ -0,0 +1,2 @@
+---
+default: --color --format progress --strict

data/features/sidekiq_deltas.feature

@@ -0,0 +1,62 @@
+Feature: Resque Delta Indexing
+  In order to have delta indexing on frequently-updated sites
+  Developers
+  Should be able to use Resque to handle delta indices to lower system load
+
+  Background:
+    Given Sphinx is running
+    And I am searching on delayed betas
+    And I have data and it has been indexed
+
+  Scenario: Delta Index should not fire automatically
+    When I search for one
+    Then I should get 1 result
+
+    When I change the name of delayed beta one to eleven
+    And I wait for Sphinx to catch up
+    And I search for one
+    Then I should get 1 result
+
+    When I search for eleven
+    Then I should get 0 results
+
+  Scenario: Delta Index should fire when jobs are run
+    When I search for one
+    Then I should get 1 result
+
+    When I change the name of delayed beta two to twelve
+    And I wait for Sphinx to catch up
+    And I search for twelve
+    Then I should get 0 results
+
+    When I run the delayed jobs
+    And I wait for Sphinx to catch up
+    And I search for twelve
+    Then I should get 1 result
+
+    When I search for two
+    Then I should get 0 results
+
+  Scenario: ensuring that duplicate jobs are deleted
+    When I change the name of delayed beta two to fifty
+    And I change the name of delayed beta five to twelve
+    And I change the name of delayed beta one to fifteen
+    And I change the name of delayed beta six to twenty
+    And I run one delayed job
+    Then there should be no more DeltaJobs on the Resque queue
+
+    When I run the delayed jobs
+    And I wait for Sphinx to catch up
+    And I search for fifty
+    Then I should get 1 result
+
+    When I search for two
+    Then I should get 0 results
+
+  Scenario: canceling jobs
+    When I change the name of delayed beta two to fifty
+    And I cancel the jobs
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+    And I search for fifty
+    Then I should get 0 results

data/features/smart_indexing.feature

@@ -0,0 +1,43 @@
+Feature: Smart Indexing
+  In order to have core indexing that works well with resque delta indexing
+  Developers
+  Should be able to use smart index to update core indices
+
+  Background:
+    Given Sphinx is running
+    And I am searching on delayed betas
+    And I have data
+
+  Scenario: Smart indexing should update core indices
+    When I run the smart indexer
+    And I wait for Sphinx to catch up
+    And I search for one
+    Then I should get 1 result
+
+  Scenario: Smart indexing should reset the delta index
+    Given I have indexed
+    When I change the name of delayed beta one to eleven
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I change the name of delayed beta eleven to one
+    And I run the smart indexer
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I search for eleven
+    Then I should get 0 results
+
+  Scenario: Delta Index running after smart indexing should not hide records
+    When I run the smart indexer
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I search for two
+    Then I should get 1 result
+
+  Scenario: Smart index should remove existing delta jobs
+    When I run the smart indexer
+    And I run one delayed job
+    And I wait for Sphinx to catch up
+    Then there should be no more DeltaJobs on the Resque queue

data/features/step_definitions/common_steps.rb

@@ -0,0 +1,76 @@
+Before do
+  $queries_executed = []
+  ThinkingSphinx::Deltas::SidekiqDelta.clear!
+  @model      = nil
+  @method     = :search
+  @query      = ""
+  @conditions = {}
+  @with       = {}
+  @without    = {}
+  @with_all   = {}
+  @options    = {}
+  @results    = nil
+end
+
+Given "Sphinx is running" do
+  ThinkingSphinx::Configuration.instance.controller.should be_running
+end
+
+Given /^I am searching on (.+)$/ do |model|
+  @model = model.gsub(/\s/, '_').singularize.camelize.constantize
+end
+
+Given "I have data" do
+  DelayedBeta.create(:name => "one")
+  DelayedBeta.create(:name => "two")
+  DelayedBeta.create(:name => "three")
+  DelayedBeta.create(:name => "four")
+  DelayedBeta.create(:name => "five")
+  DelayedBeta.create(:name => "six")
+  DelayedBeta.create(:name => "seven")
+  DelayedBeta.create(:name => "eight")
+  DelayedBeta.create(:name => "nine")
+  DelayedBeta.create(:name => "ten")
+end
+
+Given "I have indexed" do
+  ThinkingSphinx::Deltas::SidekiqDelta.clear!
+  ThinkingSphinx::Configuration.instance.controller.index
+  sleep(1.5)
+end
+
+Given "I have data and it has been indexed" do
+  step "I have data"
+  step "I have indexed"
+end
+
+When "I wait for Sphinx to catch up" do
+  sleep(0.5)
+end
+
+When /^I search for (\w+)$/ do |query|
+  @results = nil
+  @query = query
+end
+
+Then /^I should get (\d+) results?$/ do |count|
+  results.length.should == count.to_i
+end
+
+Then /^I debug$/ do
+  debugger
+  0
+end
+
+def results
+  @results ||= (@model || ThinkingSphinx).send(
+    @method,
+    @query,
+    @options.merge(
+      :conditions => @conditions,
+      :with       => @with,
+      :without    => @without,
+      :with_all   => @with_all
+    )
+  )
+end

data/features/step_definitions/sidekiq_delta_steps.rb

@@ -0,0 +1,33 @@
+When /^I run the delayed jobs$/ do
+  unless @resque_worker
+    @resque_worker = Resque::Worker.new("ts_delta")
+    @resque_worker.register_worker
+  end
+  while job = @resque_worker.reserve
+    @resque_worker.perform(job)
+  end
+end
+
+When /^I run one delayed job$/ do
+  unless @resque_worker
+    @resque_worker = Resque::Worker.new("ts_delta")
+    @resque_worker.register_worker
+  end
+  job = @resque_worker.reserve
+  @resque_worker.perform(job)
+end
+
+When /^I cancel the jobs$/ do
+  ThinkingSphinx::Deltas::SidekiqDelta.clear!
+end
+
+When /^I change the name of delayed beta (\w+) to (\w+)$/ do |current, replacement|
+  DelayedBeta.find_by_name(current).update_attributes(:name => replacement)
+end
+
+Then /^there should be no more DeltaJobs on the Resque queue$/ do
+  job_classes = Resque.redis.lrange("queue:ts_delta", 0, -1).collect do |j|
+    Resque.decode(j)["class"]
+  end
+  job_classes.should_not include("ThinkingSphinx::Deltas::SidekiqDelta::DeltaJob")
+end

data/features/step_definitions/smart_indexing_steps.rb

@@ -0,0 +1,3 @@
+When /^I run the smart indexer$/ do
+  ThinkingSphinx::Deltas::SidekiqDelta::CoreIndex.new.smart_index(:verbose => false)
+end

data/features/support/env.rb

@@ -0,0 +1,31 @@
+require 'cucumber'
+require 'rspec/expectations'
+require 'fileutils'
+require 'active_record'
+require 'mock_redis'
+
+PROJECT_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+
+$:.unshift(File.join(PROJECT_ROOT, 'lib'))
+$:.unshift(File.dirname(__FILE__))
+
+require 'cucumber/thinking_sphinx/internal_world'
+
+ActiveRecord::Base.default_timezone = :utc
+
+world = Cucumber::ThinkingSphinx::InternalWorld.new
+world.configure_database
+
+require 'thinking_sphinx'
+require 'thinking_sphinx/deltas/resque_delta'
+
+world.setup
+
+Resque.redis = MockRedis.new
+Before do
+  Sidekiq.redis{|r| r.flushall}
+end
+
+require 'database_cleaner'
+require 'database_cleaner/cucumber'
+DatabaseCleaner.strategy = :truncation

data/features/thinking_sphinx/database.example.yml

@@ -0,0 +1,3 @@
+username: root
+host: localhost
+password: 

data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb

@@ -0,0 +1,4 @@
+ActiveRecord::Base.connection.create_table :delayed_betas, :force => true do |t|
+  t.column  :name, :string,  :null => false
+  t.column :delta, :boolean, :null => false, :default => false
+end

data/features/thinking_sphinx/models/delayed_beta.rb

@@ -0,0 +1,6 @@
+class DelayedBeta < ActiveRecord::Base
+  define_index do
+    indexes :name, :sortable => true
+    set_property :delta => ThinkingSphinx::Deltas::SidekiqDelta
+  end
+end

data/lib/thinking_sphinx/deltas/sidekiq_delta.rb

@@ -0,0 +1,64 @@
+require 'sidekiq'
+require 'thinking_sphinx'
+
+class ThinkingSphinx::Deltas::SidekiqDelta < ThinkingSphinx::Deltas::DefaultDelta
+  JOB_TYPES  = []
+  JOB_PREFIX = 'ts-delta'
+
+  # LTRIM + LPOP deletes all items from the Resque queue without loading it
+  # into client memory (unlike Resque.dequeue).
+  # WARNING: This will clear ALL jobs in any queue used by a ResqueDelta job.
+  # If you're sharing a queue with other jobs they'll be deleted!
+  def self.clear_thinking_sphinx_queues
+    JOB_TYPES.collect { |job|
+      job.sidekiq_options['queue']
+    }.uniq.each do |queue|
+      Sidekiq.redis { |redis| redis.srem "queues", queue }
+      Sidekiq.redis { |redis| redis.del  "queue:#{queue}" }
+    end
+  end
+
+  # Clear both the resque queues and any other state maintained in redis
+  def self.clear!
+    self.clear_thinking_sphinx_queues
+
+    FlagAsDeletedSet.clear_all!
+  end
+
+  # Use simplistic locking.  We're assuming that the user won't run more than one
+  # `rake ts:si` or `rake ts:in` task at a time.
+  def self.lock(index_name)
+    Sidekiq.redis {|redis|
+      redis.set("#{JOB_PREFIX}:index:#{index_name}:locked", 'true')
+    }
+  end
+
+  def self.unlock(index_name)
+    Sidekiq.redis {|redis|
+      redis.del("#{JOB_PREFIX}:index:#{index_name}:locked")
+    }
+  end
+
+  def self.locked?(index_name)
+    Sidekiq.redis {|redis|
+      redis.get("#{JOB_PREFIX}:index:#{index_name}:locked") == 'true'
+    }
+  end
+
+  def delete(index, instance)
+    return if self.class.locked?(index.reference)
+
+    ThinkingSphinx::Deltas::SidekiqDelta::FlagAsDeletedJob.perform_async(
+      index.name, index.document_id_for_key(instance.id)
+    )
+  end
+
+  def index(index)
+    return if self.class.locked?(index.reference)
+
+    ThinkingSphinx::Deltas::SidekiqDelta::DeltaJob.perform_async(index.name)
+  end
+end
+
+require 'thinking_sphinx/deltas/sidekiq_delta/delta_job'
+require 'thinking_sphinx/deltas/sidekiq_delta/flag_as_deleted_job'

data/lib/thinking_sphinx/deltas/sidekiq_delta/delta_job.rb

@@ -0,0 +1,51 @@
+# A simple job class that processes a given index.
+#
+class ThinkingSphinx::Deltas::SidekiqDelta::DeltaJob
+  include Sidekiq::Worker
+
+  sidekiq_options unique: true, retry: true, queue: 'ts_delta'
+
+  # Runs Sphinx's indexer tool to process the index.
+  #
+  # @param [String] index the name of the Sphinx index
+  #
+  def perform(index)
+    config = ThinkingSphinx::Configuration.instance
+    config.controller.index index, :verbose => !config.settings['quiet_deltas']
+  end
+
+  # Try again later if lock is in use.
+  def self.lock_failed(*arguments)
+    self.class.perform_async(*arguments)
+  end
+
+  # This allows us to have a concurrency safe version of ts-delayed-delta's
+  # duplicates_exist:
+  #
+  # http://github.com/freelancing-god/ts-delayed-delta/blob/master/lib/thinkin
+  # g_sphinx/deltas/delayed_delta/job.rb#L47
+  #
+  # The name of this method ensures that it runs within around_perform_lock.
+  #
+  # We've leveraged resque-lock-timeout to ensure that only one DeltaJob is
+  # running at a time. Now, this around filter essentially ensures that only
+  # one DeltaJob of each index type can sit at the queue at once. If the queue
+  # has more than one, lrem will clear the rest off.
+  #
+  def self.around_perform_lock1(*arguments)
+    # Remove all other instances of this job (with the same args) from the
+    # queue. Uses LREM (http://code.google.com/p/redis/wiki/LremCommand) which
+    # takes the form: "LREM key count value" and if count == 0 removes all
+    # instances of value from the list.
+    redis_job_value = {:class => self.to_s, :args => arguments}.to_json
+
+    Sidekiq.redis {|redis|
+      redis.lrem("queue:#{@queue}", 0, redis_job_value)
+    }
+
+    yield
+  end
+end
+
+ThinkingSphinx::Deltas::SidekiqDelta::JOB_TYPES <<
+  ThinkingSphinx::Deltas::SidekiqDelta::DeltaJob

data/lib/thinking_sphinx/deltas/sidekiq_delta/flag_as_deleted_job.rb

@@ -0,0 +1,20 @@
+class ThinkingSphinx::Deltas::SidekiqDelta::FlagAsDeletedJob
+  include Sidekiq::Worker
+
+  # Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx
+  # is running.
+  sidekiq_options unique: true, retry: true, queue: 'ts_delta'
+
+  def perform(index, document_id)
+    ThinkingSphinx::Connection.pool.take do |connection|
+      connection.query(
+        Riddle::Query.update(index, document_id, :sphinx_deleted => true)
+      )
+    end
+  rescue Mysql2::Error => error
+    # This isn't vital, so don't raise the error
+  end
+end
+
+ThinkingSphinx::Deltas::SidekiqDelta::JOB_TYPES <<
+  ThinkingSphinx::Deltas::SidekiqDelta::FlagAsDeletedJob

data/lib/thinking_sphinx/deltas/sidekiq_delta/railtie.rb

@@ -0,0 +1,5 @@
+class ThinkingSphinx::Deltas::SidekiqDelta::Railtie < Rails::Railtie
+  rake_tasks do
+    load File.expand_path('../tasks.rb', __FILE__)
+  end
+end

data/lib/thinking_sphinx/deltas/sidekiq_delta/tasks.rb

@@ -0,0 +1,38 @@
+require 'thinking_sphinx/deltas/sidekiq_delta'
+
+namespace :thinking_sphinx do
+  desc 'Lock all delta indices (Resque will not run indexer or place new jobs on the :ts_delta queue).'
+  task :lock_deltas do
+    ThinkingSphinx::Deltas::SidekiqDelta::CoreIndex.new.lock_deltas
+  end
+
+  desc 'Unlock all delta indices.'
+  task :unlock_deltas do
+    ThinkingSphinx::Deltas::SidekiqDelta::CoreIndex.new.unlock_deltas
+  end
+
+  desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
+  task :smart_index => :app_env do
+    ret = ThinkingSphinx::Deltas::SidekiqDelta::CoreIndex.new.smart_index
+
+    abort("Indexing failed.") if ret != true
+  end
+end
+
+namespace :ts do
+  desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
+  task :si => 'thinking_sphinx:smart_index'
+end
+
+unless Rake::Task.task_defined?('ts:index')
+  require 'thinking_sphinx/tasks'
+end
+
+# Ensure that indexing does not conflict with ts-resque-delta delta jobs.
+# Rake::Task['ts:index'].enhance ['thinking_sphinx:lock_deltas'] do
+#   Rake::Task['thinking_sphinx:unlock_deltas'].invoke
+# end
+
+# Rake::Task['ts:reindex'].enhance ['thinking_sphinx:lock_deltas'] do
+#   Rake::Task['thinking_sphinx:unlock_deltas'].invoke
+# end

data/lib/ts-sidekiq-delta.rb

@@ -0,0 +1,2 @@
+require 'thinking_sphinx/deltas/sidekiq_delta'
+require 'thinking_sphinx/deltas/sidekiq_delta/railtie' if defined?(Rails::Railtie)

data/spec/acceptance/sidekiq_deltas_spec.rb

@@ -0,0 +1,50 @@
+require 'acceptance/spec_helper'
+
+describe 'SQL delta indexing', :live => true do
+  before :each do
+    Sidekiq.redis = { url: "resque://localhost:6379/", namespace: 'test' }
+  end
+
+  it "automatically indexes new records" do
+    guards = Book.create(
+      :title => 'Guards! Guards!', :author => 'Terry Pratchett'
+    )
+    index
+
+    Book.search('Terry Pratchett').to_a.should == [guards]
+
+    men = Book.create(
+      :title => 'Men At Arms', :author => 'Terry Pratchett'
+    )
+    work
+    sleep 0.25
+
+    Book.search('Terry Pratchett').to_a.should == [guards, men]
+  end
+
+  it "automatically indexes updated records" do
+    book = Book.create :title => 'Night Watch', :author => 'Harry Pritchett'
+    index
+
+    Book.search('Harry').to_a.should == [book]
+
+    book.reload.update_attributes(:author => 'Terry Pratchett')
+    work
+    sleep 0.25
+
+    Book.search('Terry').to_a.should == [book]
+  end
+
+  it "does not match on old values" do
+    book = Book.create :title => 'Night Watch', :author => 'Harry Pritchett'
+    index
+
+    Book.search('Harry').to_a.should == [book]
+
+    book.reload.update_attributes(:author => 'Terry Pratchett')
+    work
+    sleep 0.25
+
+    Book.search('Harry').should be_empty
+  end
+end

data/spec/acceptance/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+require 'sidekiq/processor'
+
+root = File.expand_path File.dirname(__FILE__)
+Dir["#{root}/support/**/*.rb"].each { |file| require file }

data/spec/acceptance/support/database_cleaner.rb

@@ -0,0 +1,11 @@
+RSpec.configure do |config|
+  config.before(:suite) do
+    DatabaseCleaner.strategy = :truncation
+  end
+
+  config.after(:each) do
+    if example.example_group_instance.class.metadata[:live]
+      DatabaseCleaner.clean
+    end
+  end
+end

data/spec/acceptance/support/sphinx_controller.rb

@@ -0,0 +1,39 @@
+class SphinxController
+  def initialize
+    config.searchd.mysql41 = 9307
+  end
+
+  def setup
+    FileUtils.mkdir_p config.indices_location
+    config.render_to_file && index
+
+    ThinkingSphinx::Configuration.reset
+    ActiveSupport::Dependencies.clear
+
+    config.index_paths.each do |path|
+      Dir["#{path}/**/*.rb"].each { |file| $LOADED_FEATURES.delete file }
+    end
+
+    config.searchd.mysql41 = 9307
+    config.settings['quiet_deltas']      = true
+    config.settings['attribute_updates'] = true
+  end
+
+  def start
+    config.controller.start
+  end
+
+  def stop
+    config.controller.stop
+  end
+
+  def index(*indices)
+    config.controller.index *indices
+  end
+
+  private
+
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+end

data/spec/acceptance/support/sphinx_helpers.rb

@@ -0,0 +1,39 @@
+module SphinxHelpers
+  include Sidekiq::Util
+
+  def sphinx
+    @sphinx ||= SphinxController.new
+  end
+
+  def index(*indices)
+    yield if block_given?
+
+    ThinkingSphinx::Deltas::SidekiqDelta.clear_thinking_sphinx_queues
+    sphinx.index *indices
+    sleep 0.25
+  end
+
+  def work
+    client    = Redis.connect(:url => "resque://localhost:6379/")
+    namespace = Redis::Namespace.new('test', :redis => client)
+
+    Sidekiq::Client.registered_queues.each do |queue_name|
+      while message = namespace.lpop("queue:#{queue_name}")
+        message = JSON.parse(message)
+        message['class'].constantize.new.perform(*message['args'])
+      end
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include SphinxHelpers
+
+  config.before :all do |group|
+    sphinx.setup && sphinx.start if group.class.metadata[:live]
+  end
+
+  config.after :all do |group|
+    sphinx.stop if group.class.metadata[:live]
+  end
+end

data/spec/internal/.gitignore

@@ -0,0 +1,2 @@
+config/*.sphinx.conf
+db/sphinx/*

data/spec/internal/app/indices/book_index.rb

@@ -0,0 +1,3 @@
+ThinkingSphinx::Index.define :book, :with => :active_record, :delta => ThinkingSphinx::Deltas::SidekiqDelta do
+  indexes title, author
+end

data/spec/internal/app/models/book.rb

@@ -0,0 +1,3 @@
+class Book < ActiveRecord::Base
+  #
+end

data/spec/internal/config/database.yml

@@ -0,0 +1,5 @@
+test:
+  adapter:      <%= ENV['DATABASE'] || 'mysql2' %>
+  database:     thinking_sphinx
+  username:     <%= ENV['DATABASE'] == 'postgresql' ? ENV['USER'] : 'root' %>
+  min_messages: warning

data/spec/internal/db/schema.rb

@@ -0,0 +1,24 @@
+ActiveRecord::Schema.define do
+  create_table(:books, :force => true) do |t|
+    t.string  :title
+    t.string  :author
+    t.integer :year
+    t.string  :blurb_file
+    t.boolean :delta, :default => true, :null => false
+    t.timestamps
+  end
+
+  create_table(:delayed_jobs, :force => true) do |t|
+    t.column :priority, :integer, :default => 0
+    t.column :attempts, :integer, :default => 0
+    t.column :handler, :text
+    t.column :last_error, :text
+    t.column :run_at, :datetime
+    t.column :locked_at, :datetime
+    t.column :failed_at, :datetime
+    t.column :locked_by, :string
+    t.column :queue, :string
+    t.column :created_at, :datetime
+    t.column :updated_at, :datetime
+  end
+end

data/spec/internal/log/.gitignore

@@ -0,0 +1 @@
+*.log

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'bundler'
+
+Bundler.require :default, :development
+
+require 'thinking_sphinx/railtie'
+
+Combustion.initialize! :active_record
+
+root = File.expand_path File.dirname(__FILE__)
+Dir["#{root}/support/**/*.rb"].each { |file| require file }
+
+RSpec.configure do |config|
+  # enable filtering for examples
+  config.filter_run :wip => nil
+  config.run_all_when_everything_filtered = true
+end

data/tasks/rails.rake

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), '/../lib/thinking_sphinx/deltas/resque_delta/tasks')

data/ts-sidekiq-delta.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |s|
+  s.name        = 'ts-sidekiq-delta'
+  s.version     = '0.1.0'
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Pat Allan', 'Aaron Gibralter', 'Danny Hawkins']
+  s.email       = ['pat@freelancing-gods.com', 'danny.hawkins@gmail.com']
+  s.homepage    = 'https://github.com/pat/ts-sidekiq-delta'
+  s.summary     = %q{Thinking Sphinx - Sidekiq Deltas}
+  s.description = %q{Manage delta indexes via Sidekiq for Thinking Sphinx}
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'thinking-sphinx', '>= 3.0.0'
+  s.add_dependency 'sidekiq',         '>= 2.5.4'
+
+  s.add_development_dependency 'activerecord',     '>= 3.1.0'
+  s.add_development_dependency 'database_cleaner', '>= 0.5.2'
+  s.add_development_dependency 'mysql2',           '>= 0.3.12b4'
+  s.add_development_dependency 'rake',             '>= 0.8.7'
+  s.add_development_dependency 'rspec',            '>= 2.11.0'
+end

metadata

@@ -0,0 +1,204 @@
+--- !ruby/object:Gem::Specification
+name: ts-sidekiq-delta
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Pat Allan
+- Aaron Gibralter
+- Danny Hawkins
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thinking-sphinx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.5.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.5.4
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.2
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.12b4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.12b4
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+description: Manage delta indexes via Sidekiq for Thinking Sphinx
+email:
+- pat@freelancing-gods.com
+- danny.hawkins@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Guardfile
+- LICENSE
+- README.markdown
+- Rakefile
+- config/redis-cucumber.conf
+- cucumber.yml
+- features/sidekiq_deltas.feature
+- features/smart_indexing.feature
+- features/step_definitions/common_steps.rb
+- features/step_definitions/sidekiq_delta_steps.rb
+- features/step_definitions/smart_indexing_steps.rb
+- features/support/env.rb
+- features/thinking_sphinx/database.example.yml
+- features/thinking_sphinx/db/migrations/create_delayed_betas.rb
+- features/thinking_sphinx/models/delayed_beta.rb
+- lib/thinking_sphinx/deltas/sidekiq_delta.rb
+- lib/thinking_sphinx/deltas/sidekiq_delta/delta_job.rb
+- lib/thinking_sphinx/deltas/sidekiq_delta/flag_as_deleted_job.rb
+- lib/thinking_sphinx/deltas/sidekiq_delta/railtie.rb
+- lib/thinking_sphinx/deltas/sidekiq_delta/tasks.rb
+- lib/ts-sidekiq-delta.rb
+- spec/acceptance/sidekiq_deltas_spec.rb
+- spec/acceptance/spec_helper.rb
+- spec/acceptance/support/database_cleaner.rb
+- spec/acceptance/support/sphinx_controller.rb
+- spec/acceptance/support/sphinx_helpers.rb
+- spec/internal/.gitignore
+- spec/internal/app/indices/book_index.rb
+- spec/internal/app/models/book.rb
+- spec/internal/config/database.yml
+- spec/internal/db/schema.rb
+- spec/internal/log/.gitignore
+- spec/spec_helper.rb
+- tasks/rails.rake
+- ts-sidekiq-delta.gemspec
+homepage: https://github.com/pat/ts-sidekiq-delta
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Thinking Sphinx - Sidekiq Deltas
+test_files:
+- features/sidekiq_deltas.feature
+- features/smart_indexing.feature
+- features/step_definitions/common_steps.rb
+- features/step_definitions/sidekiq_delta_steps.rb
+- features/step_definitions/smart_indexing_steps.rb
+- features/support/env.rb
+- features/thinking_sphinx/database.example.yml
+- features/thinking_sphinx/db/migrations/create_delayed_betas.rb
+- features/thinking_sphinx/models/delayed_beta.rb
+- spec/acceptance/sidekiq_deltas_spec.rb
+- spec/acceptance/spec_helper.rb
+- spec/acceptance/support/database_cleaner.rb
+- spec/acceptance/support/sphinx_controller.rb
+- spec/acceptance/support/sphinx_helpers.rb
+- spec/internal/.gitignore
+- spec/internal/app/indices/book_index.rb
+- spec/internal/app/models/book.rb
+- spec/internal/config/database.yml
+- spec/internal/db/schema.rb
+- spec/internal/log/.gitignore
+- spec/spec_helper.rb