ryansch-ts-resque-delta 1.1.5.1

data/.gitignore

@@ -0,0 +1,16 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+*~
+.DS_Store
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp
+coverage
+rdoc
+features/thinking_sphinx/database.yml
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--color
+-d
+--format progress

data/Gemfile

@@ -0,0 +1,25 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in ts-resque-delta.gemspec
+gemspec
+
+group :development do
+  if RUBY_PLATFORM =~ /darwin/i
+    gem 'rb-fsevent'
+    gem 'ruby_gntp'
+  else
+    gem 'rb-fsevent', :require => false
+    gem 'growl', :require => false
+  end
+  if RUBY_PLATFORM =~ /linux/i
+    gem 'rb-inotify'
+    gem 'libnotify'
+  else
+    gem 'rb-inotify', :require => false
+    gem 'libnotify', :require => false
+  end
+end
+
+group :test do
+  gem 'ruby-debug'
+end

data/Guardfile

@@ -0,0 +1,16 @@
+guard 'bundler' do
+  watch('Gemfile')
+  watch(/^.+\.gemspec/)
+end
+
+guard 'cucumber', :all_on_start => false do
+  watch(%r{^features/.+\.feature$})
+  watch(%r{^features/support/.+$})          { 'features' }
+  watch(%r{^features/step_definitions/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'features' }
+end
+
+guard 'rspec', :version => 2, :cli => "-c -d --format progress", :all_on_start => false do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Pat Allan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,82 @@
+Delayed Deltas for Thinking Sphinx (with Resque)
+================================================
+**This code is HEAVILY borrowed from
+[ts-delayed-delta](https://github.com/freelancing-god/ts-delayed-delta).**
+
+Installation
+------------
+This gem depends on the following gems: _thinking-sphinx_, _resque_, and
+_resque-lock-timeout_.
+
+    gem install ts-resque-delta
+
+Add _ts-resque-delta_ to your **Gemfile** file, with the rest of your gem
+dependencies:
+
+    gem 'ts-resque-delta', '1.1.1'
+
+If you're using Rails 3, the rake tasks will automatically be loaded by Rails.
+If you're using Rails 2, add the following line to your **Rakefile**:
+
+    require 'thinking_sphinx/deltas/resque_delta/tasks'
+
+Add the delta property to each `define_index` block:
+
+    define_index do
+      # ...
+      set_property :delta => ThinkingSphinx::Deltas::ResqueDelta
+    end
+
+If you've never used delta indexes before, you'll want to add the boolean
+column named `:delta` to each model's table (note, you must set the `:default`
+value to `true`):
+
+    def self.up
+      add_column :foos, :delta, :boolean, :default => true, :null => false
+    end
+
+Also, I highly recommend adding a MySQL index to the table of any model using
+delta indexes. The Sphinx indexer uses `WHERE table.delta = 1` whenever the
+delta indexer runs and `... = 0` whenever the normal indexer runs. Having the
+MySQL index on the delta column will generally be a win:
+
+    def self.up
+      # ...
+      add_index :foos, :delta
+    end
+
+Usage
+-----
+Once you've got it all set up, all you need to do is make sure that the Resque
+worker is running. You can do this by specifying the `:ts_delta` queue when
+running Resque:
+
+    QUEUE=ts_delta,other_queues rake resque:work
+
+Additionally, ts-resque-delta will wrap thinking-sphinx's
+`thinking_sphinx:index` and `thinking_sphinx:reindex` tasks with
+`thinking_sphinx:lock_deltas` and `thinking_sphinx:unlock_deltas`. This will
+prevent the delta indexer from running at the same time as the main indexer.
+
+Finally, ts-resque-delta also provides a rake task called
+`thinking_sphinx:smart_index` (or `ts:si` for short). This task, instead of
+locking all the delta indexes at once while the main indexer runs, will lock
+each delta index independently and sequentially. Thay way, your delta indexer
+can run while the main indexer is processing large core indexes.
+
+Contributors (for ts-delayed-delta)
+-----------------------------------
+* [Aaron Gibralter](https://github.com/agibralter)
+* [Ryan Schlesinger](https://github.com/ryansch) (Locking/`smart_index`)
+
+Original Contributors (for ts-delayed-delta)
+--------------------------------------------
+* [Pat Allan](https://github.com/freelancing-god)
+* [Ryan Schlesinger](https://github.com/ryansch) (Allowing installs as a plugin)
+* [Maximilian Schulz](https://max.jungeelite.de) (Ensuring compatibility with Bundler)
+* [Edgars Beigarts](https://github.com/ebeigarts) (Adding intelligent description for tasks)
+* [Alexander Simonov](https://simonov.me/) (Explicit table definition)
+
+Copyright
+---------
+Copyright (c) 2011 Aaron Gibralter, and released under an MIT Licence.

data/Rakefile

@@ -0,0 +1,10 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  # Define options in .rspec so they run with guard as well
+#  t.rspec_opts = ["-c"]
+end
+
+task :default => :spec

data/config/redis-cucumber.conf

@@ -0,0 +1,13 @@
+daemonize yes
+pidfile tmp/redis-cucumber.pid
+port 6398
+timeout 300
+save 900 1
+save 300 10
+save 60 10000
+dbfilename tmp/redis-cucumber-dump.rdb
+dir .
+loglevel debug
+logfile stdout
+databases 16
+glueoutputbuf yes

data/features/resque_deltas.feature

@@ -0,0 +1,62 @@
+Feature: Resque Delta Indexing
+  In order to have delta indexing on frequently-updated sites
+  Developers
+  Should be able to use Resque to handle delta indices to lower system load
+
+  Background:
+    Given Sphinx is running
+    And I am searching on delayed betas
+    And I have data and it has been indexed
+
+  Scenario: Delta Index should not fire automatically
+    When I search for one
+    Then I should get 1 result
+
+    When I change the name of delayed beta one to eleven
+    And I wait for Sphinx to catch up
+    And I search for one
+    Then I should get 1 result
+
+    When I search for eleven
+    Then I should get 0 results
+
+  Scenario: Delta Index should fire when jobs are run
+    When I search for one
+    Then I should get 1 result
+
+    When I change the name of delayed beta two to twelve
+    And I wait for Sphinx to catch up
+    And I search for twelve
+    Then I should get 0 results
+
+    When I run the delayed jobs
+    And I wait for Sphinx to catch up
+    And I search for twelve
+    Then I should get 1 result
+
+    When I search for two
+    Then I should get 0 results
+
+  Scenario: ensuring that duplicate jobs are deleted
+    When I change the name of delayed beta two to fifty
+    And I change the name of delayed beta five to twelve
+    And I change the name of delayed beta one to fifteen
+    And I change the name of delayed beta six to twenty
+    And I run one delayed job
+    Then there should be no more DeltaJobs on the Resque queue
+
+    When I run the delayed jobs
+    And I wait for Sphinx to catch up
+    And I search for fifty
+    Then I should get 1 result
+
+    When I search for two
+    Then I should get 0 results
+
+  Scenario: canceling jobs
+    When I change the name of delayed beta two to fifty
+    And I cancel the jobs
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+    And I search for fifty
+    Then I should get 0 results

data/features/smart_indexing.feature

@@ -0,0 +1,42 @@
+Feature: Smart Indexing
+  In order to have core indexing that works well with resque delta indexing
+  Developers
+  Should be able to use smart index to update core indices
+
+  Background:
+    Given Sphinx is running
+    And I am searching on delayed betas
+    And I have data
+
+  Scenario: Smart indexing should update core indices
+    When I run the smart indexer
+    And I search for one
+    Then I should get 1 result
+
+  Scenario: Smart indexing should reset the delta index
+    Given I have indexed
+    When I change the name of delayed beta one to eleven
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I change the name of delayed beta eleven to one
+    And I run the smart indexer
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I search for eleven
+    Then I should get 0 results
+
+  Scenario: Delta Index running after smart indexing should not hide records
+    When I run the smart indexer
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I search for two
+    Then I should get 1 result
+
+  Scenario: Smart index should remove existing delta jobs
+    When I run the smart indexer
+    And I run one delayed job
+    Then there should be no more DeltaJobs on the Resque queue
+

data/features/step_definitions/common_steps.rb

@@ -0,0 +1,76 @@
+Before do
+  $queries_executed = []
+  ThinkingSphinx::Deltas::ResqueDelta.clear!
+  @model      = nil
+  @method     = :search
+  @query      = ""
+  @conditions = {}
+  @with       = {}
+  @without    = {}
+  @with_all   = {}
+  @options    = {}
+  @results    = nil
+end
+
+Given "Sphinx is running" do
+  ThinkingSphinx::Configuration.instance.controller.should be_running
+end
+
+Given /^I am searching on (.+)$/ do |model|
+  @model = model.gsub(/\s/, '_').singularize.camelize.constantize
+end
+
+Given "I have data" do
+  DelayedBeta.create(:name => "one")
+  DelayedBeta.create(:name => "two")
+  DelayedBeta.create(:name => "three")
+  DelayedBeta.create(:name => "four")
+  DelayedBeta.create(:name => "five")
+  DelayedBeta.create(:name => "six")
+  DelayedBeta.create(:name => "seven")
+  DelayedBeta.create(:name => "eight")
+  DelayedBeta.create(:name => "nine")
+  DelayedBeta.create(:name => "ten")
+end
+
+Given "I have indexed" do
+  ThinkingSphinx::Deltas::ResqueDelta.clear!
+  ThinkingSphinx::Configuration.instance.controller.index
+  sleep(1.5)
+end
+
+Given "I have data and it has been indexed" do
+  step "I have data"
+  step "I have indexed"
+end
+
+When "I wait for Sphinx to catch up" do
+  sleep(0.5)
+end
+
+When /^I search for (\w+)$/ do |query|
+  @results = nil
+  @query = query
+end
+
+Then /^I should get (\d+) results?$/ do |count|
+  results.length.should == count.to_i
+end
+
+Then /^I debug$/ do
+  debugger
+  0
+end
+
+def results
+  @results ||= (@model || ThinkingSphinx).send(
+    @method,
+    @query,
+    @options.merge(
+      :conditions => @conditions,
+      :with       => @with,
+      :without    => @without,
+      :with_all   => @with_all
+    )
+  )
+end

data/features/step_definitions/resque_delta_steps.rb

@@ -0,0 +1,33 @@
+When /^I run the delayed jobs$/ do
+  unless @resque_worker
+    @resque_worker = Resque::Worker.new("ts_delta")
+    @resque_worker.register_worker
+  end
+  while job = @resque_worker.reserve
+    @resque_worker.perform(job)
+  end
+end
+
+When /^I run one delayed job$/ do
+  unless @resque_worker
+    @resque_worker = Resque::Worker.new("ts_delta")
+    @resque_worker.register_worker
+  end
+  job = @resque_worker.reserve
+  @resque_worker.perform(job)
+end
+
+When /^I cancel the jobs$/ do
+  ThinkingSphinx::Deltas::ResqueDelta.clear!
+end
+
+When /^I change the name of delayed beta (\w+) to (\w+)$/ do |current, replacement|
+  DelayedBeta.find_by_name(current).update_attributes(:name => replacement)
+end
+
+Then /^there should be no more DeltaJobs on the Resque queue$/ do
+  job_classes = Resque.redis.lrange("queue:ts_delta", 0, -1).collect do |j|
+    Resque.decode(j)["class"]
+  end
+  job_classes.should_not include("ThinkingSphinx::Deltas::ResqueDelta::DeltaJob")
+end

data/features/step_definitions/smart_indexing_steps.rb

@@ -0,0 +1,3 @@
+When /^I run the smart indexer$/ do
+  ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.smart_index
+end

data/features/support/env.rb

@@ -0,0 +1,32 @@
+require 'cucumber'
+require 'rspec/expectations'
+require 'fileutils'
+require 'active_record'
+require 'mock_redis'
+require 'ruby-debug'
+
+PROJECT_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+
+$:.unshift(File.join(PROJECT_ROOT, 'lib'))
+$:.unshift(File.dirname(__FILE__))
+
+require 'cucumber/thinking_sphinx/internal_world'
+
+ActiveRecord::Base.default_timezone = :utc
+
+world = Cucumber::ThinkingSphinx::InternalWorld.new
+world.configure_database
+
+require 'thinking_sphinx'
+require 'thinking_sphinx/deltas/resque_delta'
+
+world.setup
+
+Resque.redis = MockRedis.new
+Before do
+  Resque.redis.flushall
+end
+
+require 'database_cleaner'
+require 'database_cleaner/cucumber'
+DatabaseCleaner.strategy = :truncation