ts-delayed-delta 1.1.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/.travis.yml +14 -12
- data/Appraisals +11 -0
- data/Gemfile +1 -3
- data/LICENSE +1 -1
- data/README.textile +29 -26
- data/Rakefile +3 -16
- data/gemfiles/binary.gemfile +7 -0
- data/gemfiles/sphinxql.gemfile +7 -0
- data/lib/thinking_sphinx/deltas/delayed_delta.rb +117 -47
- data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb +7 -16
- data/lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb +8 -19
- data/lib/ts-delayed-delta.rb +0 -3
- data/spec/acceptance/delayed_job_deltas_spec.rb +46 -0
- data/spec/acceptance/spec_helper.rb +4 -0
- data/spec/acceptance/support/database_cleaner.rb +11 -0
- data/spec/acceptance/support/sphinx_controller.rb +61 -0
- data/spec/acceptance/support/sphinx_helpers.rb +31 -0
- data/spec/internal/.gitignore +2 -0
- data/spec/internal/app/indices/book_index.rb +3 -0
- data/spec/internal/app/models/book.rb +7 -0
- data/spec/internal/config/database.yml +5 -0
- data/spec/internal/db/schema.rb +24 -0
- data/spec/internal/log/.gitignore +1 -0
- data/spec/spec_helper.rb +9 -7
- data/ts-delayed-delta.gemspec +15 -11
- metadata +129 -86
- data/VERSION +0 -1
- data/features/delayed_deltas.feature +0 -37
- data/features/step_definitions/common_steps.rb +0 -48
- data/features/step_definitions/delayed_delta_steps.rb +0 -11
- data/features/support/env.rb +0 -33
- data/features/thinking_sphinx/database.example.yml +0 -3
- data/features/thinking_sphinx/db/fixtures/delayed_betas.rb +0 -10
- data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +0 -17
- data/features/thinking_sphinx/models/delayed_beta.rb +0 -7
- data/lib/thinking_sphinx/deltas/delayed_delta/job.rb +0 -65
- data/lib/thinking_sphinx/deltas/delayed_delta/railtie.rb +0 -5
- data/lib/thinking_sphinx/deltas/delayed_delta/tasks.rb +0 -23
- data/lib/thinking_sphinx/deltas/delayed_delta/version.rb +0 -5
- data/spec/thinking_sphinx/deltas/delayed_delta/delta_job_spec.rb +0 -53
- data/spec/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job_spec.rb +0 -78
- data/spec/thinking_sphinx/deltas/delayed_delta/job_spec.rb +0 -52
- data/spec/thinking_sphinx/deltas/delayed_delta_spec.rb +0 -128
- data/tasks/rails.rake +0 -1
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
1.1.2
|
@@ -1,37 +0,0 @@
|
|
1
|
-
Feature: Delayed Delta Indexing
|
2
|
-
In order to have delta indexing on frequently-updated sites
|
3
|
-
Developers
|
4
|
-
Should be able to use delayed_job to handle delta indexes to lower system load
|
5
|
-
|
6
|
-
Scenario: Delta Index should not fire automatically
|
7
|
-
Given Sphinx is running
|
8
|
-
And I am searching on delayed betas
|
9
|
-
When I search for one
|
10
|
-
Then I should get 1 result
|
11
|
-
|
12
|
-
When I change the name of delayed beta one to eleven
|
13
|
-
And I wait for Sphinx to catch up
|
14
|
-
And I search for one
|
15
|
-
Then I should get 1 result
|
16
|
-
|
17
|
-
When I search for eleven
|
18
|
-
Then I should get 0 results
|
19
|
-
|
20
|
-
Scenario: Delta Index should fire when jobs are run
|
21
|
-
Given Sphinx is running
|
22
|
-
And I am searching on delayed betas
|
23
|
-
When I search for one
|
24
|
-
Then I should get 1 result
|
25
|
-
|
26
|
-
When I change the name of delayed beta two to twelve
|
27
|
-
And I wait for Sphinx to catch up
|
28
|
-
And I search for twelve
|
29
|
-
Then I should get 0 results
|
30
|
-
|
31
|
-
When I run the delayed jobs
|
32
|
-
And I wait for Sphinx to catch up
|
33
|
-
And I search for twelve
|
34
|
-
Then I should get 1 result
|
35
|
-
|
36
|
-
When I search for two
|
37
|
-
Then I should get 0 results
|
@@ -1,48 +0,0 @@
|
|
1
|
-
Before do
|
2
|
-
$queries_executed = []
|
3
|
-
ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
|
4
|
-
|
5
|
-
@model = nil
|
6
|
-
@method = :search
|
7
|
-
@query = ""
|
8
|
-
@conditions = {}
|
9
|
-
@with = {}
|
10
|
-
@without = {}
|
11
|
-
@with_all = {}
|
12
|
-
@options = {}
|
13
|
-
@results = nil
|
14
|
-
end
|
15
|
-
|
16
|
-
Given "Sphinx is running" do
|
17
|
-
ThinkingSphinx::Configuration.instance.controller.should be_running
|
18
|
-
end
|
19
|
-
|
20
|
-
Given /^I am searching on (.+)$/ do |model|
|
21
|
-
@model = model.gsub(/\s/, '_').singularize.camelize.constantize
|
22
|
-
end
|
23
|
-
|
24
|
-
When "I wait for Sphinx to catch up" do
|
25
|
-
sleep(0.25)
|
26
|
-
end
|
27
|
-
|
28
|
-
When /^I search for (\w+)$/ do |query|
|
29
|
-
@results = nil
|
30
|
-
@query = query
|
31
|
-
end
|
32
|
-
|
33
|
-
Then /^I should get (\d+) results?$/ do |count|
|
34
|
-
results.length.should == count.to_i
|
35
|
-
end
|
36
|
-
|
37
|
-
def results
|
38
|
-
@results ||= (@model || ThinkingSphinx).send(
|
39
|
-
@method,
|
40
|
-
@query,
|
41
|
-
@options.merge(
|
42
|
-
:conditions => @conditions,
|
43
|
-
:with => @with,
|
44
|
-
:without => @without,
|
45
|
-
:with_all => @with_all
|
46
|
-
)
|
47
|
-
)
|
48
|
-
end
|
@@ -1,11 +0,0 @@
|
|
1
|
-
When /^I run the delayed jobs$/ do
|
2
|
-
if Delayed::Worker.respond_to? :backend
|
3
|
-
Delayed::Worker.new(:quiet => true).work_off
|
4
|
-
else
|
5
|
-
Delayed::Job.work_off
|
6
|
-
end
|
7
|
-
end
|
8
|
-
|
9
|
-
When /^I change the name of delayed beta (\w+) to (\w+)$/ do |current, replacement|
|
10
|
-
DelayedBeta.find_by_name(current).update_attributes(:name => replacement)
|
11
|
-
end
|
data/features/support/env.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
require 'fileutils'
|
3
|
-
require 'bundler'
|
4
|
-
|
5
|
-
Bundler.require :default, :development
|
6
|
-
|
7
|
-
require 'active_record'
|
8
|
-
require 'thinking_sphinx'
|
9
|
-
require 'delayed_job'
|
10
|
-
|
11
|
-
ActiveRecord::Base.send(:include, ThinkingSphinx::ActiveRecord)
|
12
|
-
Delayed::Worker.backend = :active_record
|
13
|
-
|
14
|
-
ActiveSupport::Inflector.inflections do |inflect|
|
15
|
-
inflect.plural /^(.*)beta$/i, '\1betas'
|
16
|
-
inflect.singular /^(.*)betas$/i, '\1beta'
|
17
|
-
end
|
18
|
-
|
19
|
-
$:.unshift File.dirname(__FILE__) + '/../../lib'
|
20
|
-
|
21
|
-
require 'cucumber/thinking_sphinx/internal_world'
|
22
|
-
|
23
|
-
# Time.zone_default = Time.__send__(:get_zone, 'Melbourne')
|
24
|
-
# ActiveRecord::Base.time_zone_aware_attributes = true
|
25
|
-
# ActiveRecord::Base.default_timezone = :utc
|
26
|
-
#
|
27
|
-
world = Cucumber::ThinkingSphinx::InternalWorld.new
|
28
|
-
world.configure_database
|
29
|
-
|
30
|
-
require "thinking_sphinx"
|
31
|
-
require 'thinking_sphinx/deltas/delayed_delta'
|
32
|
-
|
33
|
-
world.setup
|
@@ -1,10 +0,0 @@
|
|
1
|
-
DelayedBeta.create :name => "one"
|
2
|
-
DelayedBeta.create :name => "two"
|
3
|
-
DelayedBeta.create :name => "three"
|
4
|
-
DelayedBeta.create :name => "four"
|
5
|
-
DelayedBeta.create :name => "five"
|
6
|
-
DelayedBeta.create :name => "six"
|
7
|
-
DelayedBeta.create :name => "seven"
|
8
|
-
DelayedBeta.create :name => "eight"
|
9
|
-
DelayedBeta.create :name => "nine"
|
10
|
-
DelayedBeta.create :name => "ten"
|
@@ -1,17 +0,0 @@
|
|
1
|
-
ActiveRecord::Base.connection.create_table :delayed_betas, :force => true do |t|
|
2
|
-
t.column :name, :string, :null => false
|
3
|
-
t.column :delta, :boolean, :null => false, :default => false
|
4
|
-
end
|
5
|
-
|
6
|
-
ActiveRecord::Base.connection.create_table :delayed_jobs, :force => true do |t|
|
7
|
-
t.column :priority, :integer, :default => 0
|
8
|
-
t.column :attempts, :integer, :default => 0
|
9
|
-
t.column :handler, :text
|
10
|
-
t.column :last_error, :string
|
11
|
-
t.column :run_at, :datetime
|
12
|
-
t.column :locked_at, :datetime
|
13
|
-
t.column :failed_at, :datetime
|
14
|
-
t.column :locked_by, :string
|
15
|
-
t.column :created_at, :datetime
|
16
|
-
t.column :updated_at, :datetime
|
17
|
-
end
|
@@ -1,65 +0,0 @@
|
|
1
|
-
module Delayed
|
2
|
-
module Backend
|
3
|
-
module ActiveRecord
|
4
|
-
class Job < ::ActiveRecord::Base
|
5
|
-
self.table_name = "delayed_jobs"
|
6
|
-
end
|
7
|
-
end
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
# A custom job model, subclassed from Delayed::Job. The two things it does
|
12
|
-
# differently is that it checks for duplicate tasks before enqueuing them, and
|
13
|
-
# provides the option to remove all Delayed Delta jobs from the queue.
|
14
|
-
#
|
15
|
-
# As such, this class should not be used for any other tasks.
|
16
|
-
#
|
17
|
-
class ThinkingSphinx::Deltas::Job < Delayed::Backend::ActiveRecord::Job
|
18
|
-
self.table_name = "delayed_jobs"
|
19
|
-
# Adds a job to the queue, if it doesn't already exist. This is to ensure
|
20
|
-
# multiple indexing requests for the same delta index don't get added, as the
|
21
|
-
# index only needs to be processed once.
|
22
|
-
#
|
23
|
-
# Because indexing jobs are all the same object, with a single instance
|
24
|
-
# variable (the index name), they all get serialised to the same YAML value.
|
25
|
-
#
|
26
|
-
# @param [Object] object The job, which must respond to the #perform method.
|
27
|
-
# @param [Integer] priority (0)
|
28
|
-
#
|
29
|
-
def self.enqueue(object, priority = 0)
|
30
|
-
::Delayed::Job.enqueue(object, :priority => priority) unless duplicates_exist(object)
|
31
|
-
end
|
32
|
-
|
33
|
-
# Remove all Thinking Sphinx/Delayed Delta jobs from the queue. If the
|
34
|
-
# delayed_jobs table does not exist, this method will do nothing.
|
35
|
-
#
|
36
|
-
def self.cancel_thinking_sphinx_jobs
|
37
|
-
if connection.tables.include?("delayed_jobs")
|
38
|
-
delete_all("handler LIKE '--- !ruby/object:ThinkingSphinx::Deltas::%'")
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
# This is to stop ActiveRecord complaining about a missing database when
|
43
|
-
# running specs (otherwise printing failure messages raises confusing stack
|
44
|
-
# traces).
|
45
|
-
#
|
46
|
-
def self.inspect
|
47
|
-
"Job"
|
48
|
-
end
|
49
|
-
|
50
|
-
private
|
51
|
-
|
52
|
-
# Checks whether a given job already exists in the queue.
|
53
|
-
#
|
54
|
-
# @param [Object] object The job
|
55
|
-
# @return [Boolean] True if a duplicate of the job already exists in the queue
|
56
|
-
#
|
57
|
-
def self.duplicates_exist(object)
|
58
|
-
count(
|
59
|
-
:conditions => {
|
60
|
-
:handler => object.to_yaml,
|
61
|
-
:locked_at => nil
|
62
|
-
}
|
63
|
-
) > 0
|
64
|
-
end
|
65
|
-
end
|
@@ -1,23 +0,0 @@
|
|
1
|
-
namespace :thinking_sphinx do
|
2
|
-
task :index do
|
3
|
-
require 'thinking_sphinx/deltas/delayed_delta'
|
4
|
-
ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
|
5
|
-
end
|
6
|
-
|
7
|
-
desc "Process stored delta index requests"
|
8
|
-
task :delayed_delta => :app_env do
|
9
|
-
require 'delayed_job'
|
10
|
-
require 'delayed/worker'
|
11
|
-
require 'thinking_sphinx/deltas/delayed_delta'
|
12
|
-
|
13
|
-
Delayed::Worker.new(
|
14
|
-
:min_priority => ENV['MIN_PRIORITY'],
|
15
|
-
:max_priority => ENV['MAX_PRIORITY']
|
16
|
-
).start
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
namespace :ts do
|
21
|
-
desc "Process stored delta index requests"
|
22
|
-
task :dd => "thinking_sphinx:delayed_delta"
|
23
|
-
end
|
@@ -1,53 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::DeltaJob do
|
4
|
-
describe '#perform' do
|
5
|
-
before :each do
|
6
|
-
ThinkingSphinx.suppress_delta_output = false
|
7
|
-
|
8
|
-
@delta_job = ThinkingSphinx::Deltas::DeltaJob.new(['foo_core'])
|
9
|
-
@delta_job.stub! :` => true
|
10
|
-
@delta_job.stub! :puts => nil
|
11
|
-
end
|
12
|
-
|
13
|
-
it "should output the delta indexing by default" do
|
14
|
-
@delta_job.should_receive(:puts)
|
15
|
-
|
16
|
-
@delta_job.perform
|
17
|
-
end
|
18
|
-
|
19
|
-
it "should not output the delta indexing if requested" do
|
20
|
-
ThinkingSphinx.suppress_delta_output = true
|
21
|
-
@delta_job.should_not_receive(:puts)
|
22
|
-
|
23
|
-
@delta_job.perform
|
24
|
-
end
|
25
|
-
|
26
|
-
it "should process just the requested indices" do
|
27
|
-
@delta_job.should_receive(:`) do |command|
|
28
|
-
command.should match(/foo_core/)
|
29
|
-
command.should_not match(/--all/)
|
30
|
-
end
|
31
|
-
|
32
|
-
@delta_job.perform
|
33
|
-
end
|
34
|
-
|
35
|
-
context 'multiple indices' do
|
36
|
-
it "should process all requested indices" do
|
37
|
-
@delta_job.indices = ['foo_core', 'bar_core']
|
38
|
-
@delta_job.should_receive(:`) do |command|
|
39
|
-
command.should match(/foo_core bar_core/)
|
40
|
-
end
|
41
|
-
|
42
|
-
@delta_job.perform
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
describe "#display_name" do
|
48
|
-
it "should display class name with all indices" do
|
49
|
-
@delta_job = ThinkingSphinx::Deltas::DeltaJob.new(['foo_core', 'bar_core'])
|
50
|
-
@delta_job.display_name.should == "ThinkingSphinx::Deltas::DeltaJob for foo_core, bar_core"
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
@@ -1,78 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::FlagAsDeletedJob do
|
4
|
-
describe '#perform' do
|
5
|
-
before :each do
|
6
|
-
ThinkingSphinx.updates_enabled = true
|
7
|
-
@client = stub('client', :update => true)
|
8
|
-
|
9
|
-
ThinkingSphinx::Configuration.instance.stub!(:client => @client)
|
10
|
-
ThinkingSphinx.stub!(:search_for_id => true)
|
11
|
-
ThinkingSphinx.stub!(:sphinx_running? => true)
|
12
|
-
|
13
|
-
@job = ThinkingSphinx::Deltas::FlagAsDeletedJob.new(['foo_core'], 12)
|
14
|
-
end
|
15
|
-
|
16
|
-
it "should not update if Sphinx isn't running" do
|
17
|
-
ThinkingSphinx.stub!(:sphinx_running? => false)
|
18
|
-
@client.should_not_receive(:update)
|
19
|
-
|
20
|
-
@job.perform
|
21
|
-
end
|
22
|
-
|
23
|
-
it "should not update if the document isn't in the index" do
|
24
|
-
ThinkingSphinx.stub!(:search_for_id => false)
|
25
|
-
@client.should_not_receive(:update)
|
26
|
-
|
27
|
-
@job.perform
|
28
|
-
end
|
29
|
-
|
30
|
-
it "should update the specified index" do
|
31
|
-
@client.should_receive(:update) do |index, attributes, values|
|
32
|
-
index.should == 'foo_core'
|
33
|
-
end
|
34
|
-
|
35
|
-
@job.perform
|
36
|
-
end
|
37
|
-
|
38
|
-
it "should update all specified indices" do
|
39
|
-
@job.indices = ['foo_core', 'bar_core']
|
40
|
-
@client.should_receive(:update).with('foo_core', anything, anything)
|
41
|
-
@client.should_receive(:update).with('bar_core', anything, anything)
|
42
|
-
|
43
|
-
@job.perform
|
44
|
-
end
|
45
|
-
|
46
|
-
it "should update the sphinx_deleted attribute" do
|
47
|
-
@client.should_receive(:update) do |index, attributes, values|
|
48
|
-
attributes.should == ['sphinx_deleted']
|
49
|
-
end
|
50
|
-
|
51
|
-
@job.perform
|
52
|
-
end
|
53
|
-
|
54
|
-
it "should set sphinx_deleted for the given document to true" do
|
55
|
-
@client.should_receive(:update) do |index, attributes, values|
|
56
|
-
values[12].should == [1]
|
57
|
-
end
|
58
|
-
|
59
|
-
@job.perform
|
60
|
-
end
|
61
|
-
|
62
|
-
it "should check for the existence of the document in the specified index" do
|
63
|
-
ThinkingSphinx.should_receive(:search_for_id) do |id, index|
|
64
|
-
index.should == 'foo_core'
|
65
|
-
end
|
66
|
-
|
67
|
-
@job.perform
|
68
|
-
end
|
69
|
-
|
70
|
-
it "should check for the existence of the given document id" do
|
71
|
-
ThinkingSphinx.should_receive(:search_for_id) do |id, index|
|
72
|
-
id.should == 12
|
73
|
-
end
|
74
|
-
|
75
|
-
@job.perform
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
@@ -1,52 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::Job do
|
4
|
-
describe '.enqueue' do
|
5
|
-
before :each do
|
6
|
-
ThinkingSphinx::Deltas::Job.stub!(:count => 0)
|
7
|
-
end
|
8
|
-
|
9
|
-
it "should enqueue if there's no existing jobs for the same index" do
|
10
|
-
Delayed::Job.should_receive(:enqueue)
|
11
|
-
|
12
|
-
ThinkingSphinx::Deltas::Job.enqueue(stub('object'))
|
13
|
-
end
|
14
|
-
|
15
|
-
it "should not enqueue the job if there's an existing job already" do
|
16
|
-
ThinkingSphinx::Deltas::Job.stub!(:count => 1)
|
17
|
-
Delayed::Job.should_not_receive(:enqueue)
|
18
|
-
|
19
|
-
ThinkingSphinx::Deltas::Job.enqueue(stub('object'))
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
describe '.cancel_thinking_sphinx_jobs' do
|
24
|
-
before :each do
|
25
|
-
ThinkingSphinx::Deltas::Job.stub!(:connection => double('connection'))
|
26
|
-
ThinkingSphinx::Deltas::Job.stub!(:delete_all => true)
|
27
|
-
end
|
28
|
-
|
29
|
-
it "should not delete any rows if the delayed_jobs table does not exist" do
|
30
|
-
ThinkingSphinx::Deltas::Job.connection.stub!(:tables => [])
|
31
|
-
ThinkingSphinx::Deltas::Job.should_not_receive(:delete_all)
|
32
|
-
|
33
|
-
ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
|
34
|
-
end
|
35
|
-
|
36
|
-
it "should delete rows if the delayed_jobs table does exist" do
|
37
|
-
ThinkingSphinx::Deltas::Job.connection.stub!(:tables => ['delayed_jobs'])
|
38
|
-
ThinkingSphinx::Deltas::Job.should_receive(:delete_all)
|
39
|
-
|
40
|
-
ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
|
41
|
-
end
|
42
|
-
|
43
|
-
it "should delete only Thinking Sphinx jobs" do
|
44
|
-
ThinkingSphinx::Deltas::Job.connection.stub!(:tables => ['delayed_jobs'])
|
45
|
-
ThinkingSphinx::Deltas::Job.should_receive(:delete_all) do |sql|
|
46
|
-
sql.should match(/handler LIKE '--- !ruby\/object:ThinkingSphinx::Deltas::\%'/)
|
47
|
-
end
|
48
|
-
|
49
|
-
ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|