ryansch-ts-resque-delta 1.1.5.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +16 -0
- data/.rspec +3 -0
- data/Gemfile +25 -0
- data/Guardfile +16 -0
- data/LICENSE +20 -0
- data/README.markdown +82 -0
- data/Rakefile +10 -0
- data/config/redis-cucumber.conf +13 -0
- data/features/resque_deltas.feature +62 -0
- data/features/smart_indexing.feature +42 -0
- data/features/step_definitions/common_steps.rb +76 -0
- data/features/step_definitions/resque_delta_steps.rb +33 -0
- data/features/step_definitions/smart_indexing_steps.rb +3 -0
- data/features/support/env.rb +32 -0
- data/features/thinking_sphinx/database.example.yml +3 -0
- data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +4 -0
- data/features/thinking_sphinx/models/delayed_beta.rb +6 -0
- data/lib/flying_sphinx/resque_delta.rb +38 -0
- data/lib/flying_sphinx/resque_delta/delta_job.rb +14 -0
- data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb +7 -0
- data/lib/thinking_sphinx/deltas/resque_delta.rb +118 -0
- data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +98 -0
- data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +90 -0
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +56 -0
- data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +47 -0
- data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb +8 -0
- data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +38 -0
- data/lib/thinking_sphinx/deltas/resque_delta/version.rb +7 -0
- data/lib/ts-resque-delta.rb +2 -0
- data/spec/flying_sphinx/resque_delta/delta_job_spec.rb +32 -0
- data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb +23 -0
- data/spec/flying_sphinx/resque_delta_spec.rb +131 -0
- data/spec/spec_helper.rb +13 -0
- data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +208 -0
- data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +172 -0
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +126 -0
- data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +67 -0
- data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +191 -0
- data/tasks/rails.rake +1 -0
- data/ts-resque-delta.gemspec +40 -0
- metadata +393 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in ts-resque-delta.gemspec
|
4
|
+
gemspec
|
5
|
+
|
6
|
+
group :development do
|
7
|
+
if RUBY_PLATFORM =~ /darwin/i
|
8
|
+
gem 'rb-fsevent'
|
9
|
+
gem 'ruby_gntp'
|
10
|
+
else
|
11
|
+
gem 'rb-fsevent', :require => false
|
12
|
+
gem 'growl', :require => false
|
13
|
+
end
|
14
|
+
if RUBY_PLATFORM =~ /linux/i
|
15
|
+
gem 'rb-inotify'
|
16
|
+
gem 'libnotify'
|
17
|
+
else
|
18
|
+
gem 'rb-inotify', :require => false
|
19
|
+
gem 'libnotify', :require => false
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
group :test do
|
24
|
+
gem 'ruby-debug'
|
25
|
+
end
|
data/Guardfile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
guard 'bundler' do
|
2
|
+
watch('Gemfile')
|
3
|
+
watch(/^.+\.gemspec/)
|
4
|
+
end
|
5
|
+
|
6
|
+
guard 'cucumber', :all_on_start => false do
|
7
|
+
watch(%r{^features/.+\.feature$})
|
8
|
+
watch(%r{^features/support/.+$}) { 'features' }
|
9
|
+
watch(%r{^features/step_definitions/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'features' }
|
10
|
+
end
|
11
|
+
|
12
|
+
guard 'rspec', :version => 2, :cli => "-c -d --format progress", :all_on_start => false do
|
13
|
+
watch(%r{^spec/.+_spec\.rb$})
|
14
|
+
watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
|
15
|
+
watch('spec/spec_helper.rb') { "spec" }
|
16
|
+
end
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Pat Allan
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.markdown
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
Delayed Deltas for Thinking Sphinx (with Resque)
|
2
|
+
================================================
|
3
|
+
**This code is HEAVILY borrowed from
|
4
|
+
[ts-delayed-delta](https://github.com/freelancing-god/ts-delayed-delta).**
|
5
|
+
|
6
|
+
Installation
|
7
|
+
------------
|
8
|
+
This gem depends on the following gems: _thinking-sphinx_, _resque_, and
|
9
|
+
_resque-lock-timeout_.
|
10
|
+
|
11
|
+
gem install ts-resque-delta
|
12
|
+
|
13
|
+
Add _ts-resque-delta_ to your **Gemfile** file, with the rest of your gem
|
14
|
+
dependencies:
|
15
|
+
|
16
|
+
gem 'ts-resque-delta', '1.1.1'
|
17
|
+
|
18
|
+
If you're using Rails 3, the rake tasks will automatically be loaded by Rails.
|
19
|
+
If you're using Rails 2, add the following line to your **Rakefile**:
|
20
|
+
|
21
|
+
require 'thinking_sphinx/deltas/resque_delta/tasks'
|
22
|
+
|
23
|
+
Add the delta property to each `define_index` block:
|
24
|
+
|
25
|
+
define_index do
|
26
|
+
# ...
|
27
|
+
set_property :delta => ThinkingSphinx::Deltas::ResqueDelta
|
28
|
+
end
|
29
|
+
|
30
|
+
If you've never used delta indexes before, you'll want to add the boolean
|
31
|
+
column named `:delta` to each model's table (note, you must set the `:default`
|
32
|
+
value to `true`):
|
33
|
+
|
34
|
+
def self.up
|
35
|
+
add_column :foos, :delta, :boolean, :default => true, :null => false
|
36
|
+
end
|
37
|
+
|
38
|
+
Also, I highly recommend adding a MySQL index to the table of any model using
|
39
|
+
delta indexes. The Sphinx indexer uses `WHERE table.delta = 1` whenever the
|
40
|
+
delta indexer runs and `... = 0` whenever the normal indexer runs. Having the
|
41
|
+
MySQL index on the delta column will generally be a win:
|
42
|
+
|
43
|
+
def self.up
|
44
|
+
# ...
|
45
|
+
add_index :foos, :delta
|
46
|
+
end
|
47
|
+
|
48
|
+
Usage
|
49
|
+
-----
|
50
|
+
Once you've got it all set up, all you need to do is make sure that the Resque
|
51
|
+
worker is running. You can do this by specifying the `:ts_delta` queue when
|
52
|
+
running Resque:
|
53
|
+
|
54
|
+
QUEUE=ts_delta,other_queues rake resque:work
|
55
|
+
|
56
|
+
Additionally, ts-resque-delta will wrap thinking-sphinx's
|
57
|
+
`thinking_sphinx:index` and `thinking_sphinx:reindex` tasks with
|
58
|
+
`thinking_sphinx:lock_deltas` and `thinking_sphinx:unlock_deltas`. This will
|
59
|
+
prevent the delta indexer from running at the same time as the main indexer.
|
60
|
+
|
61
|
+
Finally, ts-resque-delta also provides a rake task called
|
62
|
+
`thinking_sphinx:smart_index` (or `ts:si` for short). This task, instead of
|
63
|
+
locking all the delta indexes at once while the main indexer runs, will lock
|
64
|
+
each delta index independently and sequentially. Thay way, your delta indexer
|
65
|
+
can run while the main indexer is processing large core indexes.
|
66
|
+
|
67
|
+
Contributors (for ts-delayed-delta)
|
68
|
+
-----------------------------------
|
69
|
+
* [Aaron Gibralter](https://github.com/agibralter)
|
70
|
+
* [Ryan Schlesinger](https://github.com/ryansch) (Locking/`smart_index`)
|
71
|
+
|
72
|
+
Original Contributors (for ts-delayed-delta)
|
73
|
+
--------------------------------------------
|
74
|
+
* [Pat Allan](https://github.com/freelancing-god)
|
75
|
+
* [Ryan Schlesinger](https://github.com/ryansch) (Allowing installs as a plugin)
|
76
|
+
* [Maximilian Schulz](https://max.jungeelite.de) (Ensuring compatibility with Bundler)
|
77
|
+
* [Edgars Beigarts](https://github.com/ebeigarts) (Adding intelligent description for tasks)
|
78
|
+
* [Alexander Simonov](https://simonov.me/) (Explicit table definition)
|
79
|
+
|
80
|
+
Copyright
|
81
|
+
---------
|
82
|
+
Copyright (c) 2011 Aaron Gibralter, and released under an MIT Licence.
|
data/Rakefile
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
Feature: Resque Delta Indexing
|
2
|
+
In order to have delta indexing on frequently-updated sites
|
3
|
+
Developers
|
4
|
+
Should be able to use Resque to handle delta indices to lower system load
|
5
|
+
|
6
|
+
Background:
|
7
|
+
Given Sphinx is running
|
8
|
+
And I am searching on delayed betas
|
9
|
+
And I have data and it has been indexed
|
10
|
+
|
11
|
+
Scenario: Delta Index should not fire automatically
|
12
|
+
When I search for one
|
13
|
+
Then I should get 1 result
|
14
|
+
|
15
|
+
When I change the name of delayed beta one to eleven
|
16
|
+
And I wait for Sphinx to catch up
|
17
|
+
And I search for one
|
18
|
+
Then I should get 1 result
|
19
|
+
|
20
|
+
When I search for eleven
|
21
|
+
Then I should get 0 results
|
22
|
+
|
23
|
+
Scenario: Delta Index should fire when jobs are run
|
24
|
+
When I search for one
|
25
|
+
Then I should get 1 result
|
26
|
+
|
27
|
+
When I change the name of delayed beta two to twelve
|
28
|
+
And I wait for Sphinx to catch up
|
29
|
+
And I search for twelve
|
30
|
+
Then I should get 0 results
|
31
|
+
|
32
|
+
When I run the delayed jobs
|
33
|
+
And I wait for Sphinx to catch up
|
34
|
+
And I search for twelve
|
35
|
+
Then I should get 1 result
|
36
|
+
|
37
|
+
When I search for two
|
38
|
+
Then I should get 0 results
|
39
|
+
|
40
|
+
Scenario: ensuring that duplicate jobs are deleted
|
41
|
+
When I change the name of delayed beta two to fifty
|
42
|
+
And I change the name of delayed beta five to twelve
|
43
|
+
And I change the name of delayed beta one to fifteen
|
44
|
+
And I change the name of delayed beta six to twenty
|
45
|
+
And I run one delayed job
|
46
|
+
Then there should be no more DeltaJobs on the Resque queue
|
47
|
+
|
48
|
+
When I run the delayed jobs
|
49
|
+
And I wait for Sphinx to catch up
|
50
|
+
And I search for fifty
|
51
|
+
Then I should get 1 result
|
52
|
+
|
53
|
+
When I search for two
|
54
|
+
Then I should get 0 results
|
55
|
+
|
56
|
+
Scenario: canceling jobs
|
57
|
+
When I change the name of delayed beta two to fifty
|
58
|
+
And I cancel the jobs
|
59
|
+
And I run the delayed jobs
|
60
|
+
And I wait for Sphinx to catch up
|
61
|
+
And I search for fifty
|
62
|
+
Then I should get 0 results
|
@@ -0,0 +1,42 @@
|
|
1
|
+
Feature: Smart Indexing
|
2
|
+
In order to have core indexing that works well with resque delta indexing
|
3
|
+
Developers
|
4
|
+
Should be able to use smart index to update core indices
|
5
|
+
|
6
|
+
Background:
|
7
|
+
Given Sphinx is running
|
8
|
+
And I am searching on delayed betas
|
9
|
+
And I have data
|
10
|
+
|
11
|
+
Scenario: Smart indexing should update core indices
|
12
|
+
When I run the smart indexer
|
13
|
+
And I search for one
|
14
|
+
Then I should get 1 result
|
15
|
+
|
16
|
+
Scenario: Smart indexing should reset the delta index
|
17
|
+
Given I have indexed
|
18
|
+
When I change the name of delayed beta one to eleven
|
19
|
+
And I run the delayed jobs
|
20
|
+
And I wait for Sphinx to catch up
|
21
|
+
|
22
|
+
When I change the name of delayed beta eleven to one
|
23
|
+
And I run the smart indexer
|
24
|
+
And I run the delayed jobs
|
25
|
+
And I wait for Sphinx to catch up
|
26
|
+
|
27
|
+
When I search for eleven
|
28
|
+
Then I should get 0 results
|
29
|
+
|
30
|
+
Scenario: Delta Index running after smart indexing should not hide records
|
31
|
+
When I run the smart indexer
|
32
|
+
And I run the delayed jobs
|
33
|
+
And I wait for Sphinx to catch up
|
34
|
+
|
35
|
+
When I search for two
|
36
|
+
Then I should get 1 result
|
37
|
+
|
38
|
+
Scenario: Smart index should remove existing delta jobs
|
39
|
+
When I run the smart indexer
|
40
|
+
And I run one delayed job
|
41
|
+
Then there should be no more DeltaJobs on the Resque queue
|
42
|
+
|
@@ -0,0 +1,76 @@
|
|
1
|
+
Before do
|
2
|
+
$queries_executed = []
|
3
|
+
ThinkingSphinx::Deltas::ResqueDelta.clear!
|
4
|
+
@model = nil
|
5
|
+
@method = :search
|
6
|
+
@query = ""
|
7
|
+
@conditions = {}
|
8
|
+
@with = {}
|
9
|
+
@without = {}
|
10
|
+
@with_all = {}
|
11
|
+
@options = {}
|
12
|
+
@results = nil
|
13
|
+
end
|
14
|
+
|
15
|
+
Given "Sphinx is running" do
|
16
|
+
ThinkingSphinx::Configuration.instance.controller.should be_running
|
17
|
+
end
|
18
|
+
|
19
|
+
Given /^I am searching on (.+)$/ do |model|
|
20
|
+
@model = model.gsub(/\s/, '_').singularize.camelize.constantize
|
21
|
+
end
|
22
|
+
|
23
|
+
Given "I have data" do
|
24
|
+
DelayedBeta.create(:name => "one")
|
25
|
+
DelayedBeta.create(:name => "two")
|
26
|
+
DelayedBeta.create(:name => "three")
|
27
|
+
DelayedBeta.create(:name => "four")
|
28
|
+
DelayedBeta.create(:name => "five")
|
29
|
+
DelayedBeta.create(:name => "six")
|
30
|
+
DelayedBeta.create(:name => "seven")
|
31
|
+
DelayedBeta.create(:name => "eight")
|
32
|
+
DelayedBeta.create(:name => "nine")
|
33
|
+
DelayedBeta.create(:name => "ten")
|
34
|
+
end
|
35
|
+
|
36
|
+
Given "I have indexed" do
|
37
|
+
ThinkingSphinx::Deltas::ResqueDelta.clear!
|
38
|
+
ThinkingSphinx::Configuration.instance.controller.index
|
39
|
+
sleep(1.5)
|
40
|
+
end
|
41
|
+
|
42
|
+
Given "I have data and it has been indexed" do
|
43
|
+
step "I have data"
|
44
|
+
step "I have indexed"
|
45
|
+
end
|
46
|
+
|
47
|
+
When "I wait for Sphinx to catch up" do
|
48
|
+
sleep(0.5)
|
49
|
+
end
|
50
|
+
|
51
|
+
When /^I search for (\w+)$/ do |query|
|
52
|
+
@results = nil
|
53
|
+
@query = query
|
54
|
+
end
|
55
|
+
|
56
|
+
Then /^I should get (\d+) results?$/ do |count|
|
57
|
+
results.length.should == count.to_i
|
58
|
+
end
|
59
|
+
|
60
|
+
Then /^I debug$/ do
|
61
|
+
debugger
|
62
|
+
0
|
63
|
+
end
|
64
|
+
|
65
|
+
def results
|
66
|
+
@results ||= (@model || ThinkingSphinx).send(
|
67
|
+
@method,
|
68
|
+
@query,
|
69
|
+
@options.merge(
|
70
|
+
:conditions => @conditions,
|
71
|
+
:with => @with,
|
72
|
+
:without => @without,
|
73
|
+
:with_all => @with_all
|
74
|
+
)
|
75
|
+
)
|
76
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
When /^I run the delayed jobs$/ do
|
2
|
+
unless @resque_worker
|
3
|
+
@resque_worker = Resque::Worker.new("ts_delta")
|
4
|
+
@resque_worker.register_worker
|
5
|
+
end
|
6
|
+
while job = @resque_worker.reserve
|
7
|
+
@resque_worker.perform(job)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
When /^I run one delayed job$/ do
|
12
|
+
unless @resque_worker
|
13
|
+
@resque_worker = Resque::Worker.new("ts_delta")
|
14
|
+
@resque_worker.register_worker
|
15
|
+
end
|
16
|
+
job = @resque_worker.reserve
|
17
|
+
@resque_worker.perform(job)
|
18
|
+
end
|
19
|
+
|
20
|
+
When /^I cancel the jobs$/ do
|
21
|
+
ThinkingSphinx::Deltas::ResqueDelta.clear!
|
22
|
+
end
|
23
|
+
|
24
|
+
When /^I change the name of delayed beta (\w+) to (\w+)$/ do |current, replacement|
|
25
|
+
DelayedBeta.find_by_name(current).update_attributes(:name => replacement)
|
26
|
+
end
|
27
|
+
|
28
|
+
Then /^there should be no more DeltaJobs on the Resque queue$/ do
|
29
|
+
job_classes = Resque.redis.lrange("queue:ts_delta", 0, -1).collect do |j|
|
30
|
+
Resque.decode(j)["class"]
|
31
|
+
end
|
32
|
+
job_classes.should_not include("ThinkingSphinx::Deltas::ResqueDelta::DeltaJob")
|
33
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'cucumber'
|
2
|
+
require 'rspec/expectations'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'active_record'
|
5
|
+
require 'mock_redis'
|
6
|
+
require 'ruby-debug'
|
7
|
+
|
8
|
+
PROJECT_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
|
9
|
+
|
10
|
+
$:.unshift(File.join(PROJECT_ROOT, 'lib'))
|
11
|
+
$:.unshift(File.dirname(__FILE__))
|
12
|
+
|
13
|
+
require 'cucumber/thinking_sphinx/internal_world'
|
14
|
+
|
15
|
+
ActiveRecord::Base.default_timezone = :utc
|
16
|
+
|
17
|
+
world = Cucumber::ThinkingSphinx::InternalWorld.new
|
18
|
+
world.configure_database
|
19
|
+
|
20
|
+
require 'thinking_sphinx'
|
21
|
+
require 'thinking_sphinx/deltas/resque_delta'
|
22
|
+
|
23
|
+
world.setup
|
24
|
+
|
25
|
+
Resque.redis = MockRedis.new
|
26
|
+
Before do
|
27
|
+
Resque.redis.flushall
|
28
|
+
end
|
29
|
+
|
30
|
+
require 'database_cleaner'
|
31
|
+
require 'database_cleaner/cucumber'
|
32
|
+
DatabaseCleaner.strategy = :truncation
|