ts-resque-delta 1.1.5 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +24 -0
- data/Guardfile +17 -0
- data/README.markdown +1 -0
- data/Rakefile +16 -4
- data/cucumber.yml +2 -0
- data/features/smart_indexing.feature +43 -0
- data/features/step_definitions/common_steps.rb +16 -3
- data/features/step_definitions/resque_delta_steps.rb +1 -1
- data/features/step_definitions/smart_indexing_steps.rb +3 -0
- data/features/support/env.rb +3 -4
- data/lib/thinking_sphinx/deltas/resque_delta.rb +32 -10
- data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +101 -0
- data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +72 -10
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +56 -0
- data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +47 -0
- data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +4 -46
- data/lib/thinking_sphinx/deltas/resque_delta/version.rb +1 -1
- data/spec/spec_helper.rb +9 -5
- data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +210 -0
- data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +138 -35
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +126 -0
- data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +67 -0
- data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +126 -53
- data/ts-resque-delta.gemspec +8 -2
- metadata +185 -180
- data/features/support/redis_test_setup.rb +0 -23
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +0 -30
- data/spec/spec.opts +0 -1
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job_spec.rb +0 -66
- data/tasks/testing.rb +0 -20
@@ -0,0 +1,56 @@
|
|
1
|
+
class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
|
2
|
+
module FlagAsDeletedSet
|
3
|
+
extend self
|
4
|
+
|
5
|
+
def set_name(core_name)
|
6
|
+
"#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:set"
|
7
|
+
end
|
8
|
+
|
9
|
+
def temp_name(core_name)
|
10
|
+
"#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:temp"
|
11
|
+
end
|
12
|
+
|
13
|
+
def processing_name(core_name)
|
14
|
+
"#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:processing"
|
15
|
+
end
|
16
|
+
|
17
|
+
def add(core_name, document_id)
|
18
|
+
Resque.redis.sadd(set_name(core_name), document_id)
|
19
|
+
end
|
20
|
+
|
21
|
+
def clear!(core_name)
|
22
|
+
Resque.redis.del(set_name(core_name))
|
23
|
+
|
24
|
+
#Clear processing set as well
|
25
|
+
delta_name = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_to_delta(core_name)
|
26
|
+
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.around_perform_lock(delta_name) do
|
27
|
+
Resque.redis.del(processing_name(core_name))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def clear_all!
|
32
|
+
ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_indices.each do |core_index|
|
33
|
+
clear!(core_index)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def get_subset_for_processing(core_name)
|
38
|
+
# Copy set to temp
|
39
|
+
Resque.redis.sunionstore temp_name(core_name), set_name(core_name)
|
40
|
+
# Store (set - temp) into set. This removes all items we copied into temp from set.
|
41
|
+
Resque.redis.sdiffstore set_name(core_name), set_name(core_name), temp_name(core_name)
|
42
|
+
# Merge processing and temp together and store into processing.
|
43
|
+
Resque.redis.sunionstore processing_name(core_name), processing_name(core_name), temp_name(core_name)
|
44
|
+
|
45
|
+
Resque.redis.del temp_name(core_name)
|
46
|
+
end
|
47
|
+
|
48
|
+
def processing_members(core_name)
|
49
|
+
Resque.redis.smembers(processing_name(core_name)).collect(&:to_i)
|
50
|
+
end
|
51
|
+
|
52
|
+
def clear_processing(core_name)
|
53
|
+
Resque.redis.del(processing_name(core_name))
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module ThinkingSphinx::Deltas::ResqueDelta::IndexUtils
|
2
|
+
extend self
|
3
|
+
|
4
|
+
# Public: Return a list of index prefixes (i.e. without "_core"/"_delta").
|
5
|
+
#
|
6
|
+
# Examples
|
7
|
+
#
|
8
|
+
# sphinx_indices
|
9
|
+
# # => ['foo', 'bar']
|
10
|
+
#
|
11
|
+
# Returns an Array of index prefixes.
|
12
|
+
def index_prefixes
|
13
|
+
@prefixes ||= indices.reject { |i| i =~ /_(core|delta)$/ }
|
14
|
+
end
|
15
|
+
|
16
|
+
def core_indices
|
17
|
+
@core_indices ||= indices.select { |i| i =~ /_core$/ }
|
18
|
+
end
|
19
|
+
|
20
|
+
def delta_indices
|
21
|
+
@delta_indices ||= indices.select { |i| i =~ /_delta$/ }
|
22
|
+
end
|
23
|
+
|
24
|
+
def reload!
|
25
|
+
@ts_config = @indices = @prefixes = @core_indices = @delta_indices = nil
|
26
|
+
end
|
27
|
+
|
28
|
+
def delta_to_core(delta_name)
|
29
|
+
delta_name.sub(/_delta$/, '_core')
|
30
|
+
end
|
31
|
+
|
32
|
+
def core_to_delta(core_name)
|
33
|
+
core_name.sub(/_core$/, '_delta')
|
34
|
+
end
|
35
|
+
|
36
|
+
def ts_config
|
37
|
+
@ts_config ||= ThinkingSphinx::Configuration.instance.tap do |config|
|
38
|
+
config.generate
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
def indices
|
44
|
+
@indices ||= ts_config.configuration.indices.collect { |i| i.name }
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
@@ -1,67 +1,25 @@
|
|
1
1
|
require 'thinking_sphinx/deltas/resque_delta'
|
2
2
|
|
3
3
|
namespace :thinking_sphinx do
|
4
|
-
|
5
|
-
# Return a list of index prefixes (i.e. without "_core"/"_delta").
|
6
|
-
def sphinx_indices
|
7
|
-
unless @sphinx_indices
|
8
|
-
@ts_config ||= ThinkingSphinx::Configuration.instance
|
9
|
-
@ts_config.generate
|
10
|
-
@sphinx_indices = @ts_config.configuration.indices.collect { |i| i.name }
|
11
|
-
# The collected indices look like:
|
12
|
-
# ["foo_core", "foo_delta", "foo", "bar_core", "bar_delta", "bar"]
|
13
|
-
@sphinx_indices.reject! { |i| i =~ /_(core|delta)$/}
|
14
|
-
# Now we have:
|
15
|
-
# ["foo", "bar"]
|
16
|
-
end
|
17
|
-
@sphinx_indices
|
18
|
-
end
|
19
|
-
|
20
|
-
def lock_delta(index_name)
|
21
|
-
ThinkingSphinx::Deltas::ResqueDelta.lock("#{index_name}_delta")
|
22
|
-
end
|
23
|
-
|
24
|
-
def unlock_delta(index_name)
|
25
|
-
ThinkingSphinx::Deltas::ResqueDelta.unlock("#{index_name}_delta")
|
26
|
-
end
|
27
|
-
|
28
4
|
desc 'Lock all delta indices (Resque will not run indexer or place new jobs on the :ts_delta queue).'
|
29
5
|
task :lock_deltas do
|
30
|
-
|
6
|
+
ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.lock_deltas
|
31
7
|
end
|
32
8
|
|
33
9
|
desc 'Unlock all delta indices.'
|
34
10
|
task :unlock_deltas do
|
35
|
-
|
11
|
+
ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.unlock_deltas
|
36
12
|
end
|
37
13
|
|
38
14
|
desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
|
39
15
|
task :smart_index => :app_env do
|
40
|
-
|
41
|
-
@ts_config = ThinkingSphinx::Configuration.instance
|
42
|
-
unless ENV['INDEX_ONLY'] == 'true'
|
43
|
-
puts "Generating Configuration to #{@ts_config.config_file}"
|
44
|
-
@ts_config.build
|
45
|
-
end
|
46
|
-
FileUtils.mkdir_p(@ts_config.searchd_file_path)
|
16
|
+
ret = ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.smart_index
|
47
17
|
|
48
|
-
|
49
|
-
sphinx_indices.each do |index_name|
|
50
|
-
lock_delta(index_name)
|
51
|
-
@ts_config.controller.index("#{index_name}_core", :verbose => true)
|
52
|
-
ret = $?
|
53
|
-
unlock_delta(index_name)
|
54
|
-
exit(-1) if ret.to_i != 0
|
55
|
-
Resque.enqueue(
|
56
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
|
57
|
-
["#{index_name}_delta"]
|
58
|
-
)
|
59
|
-
end
|
18
|
+
abort("Indexing failed.") if ret != true
|
60
19
|
end
|
61
20
|
end
|
62
21
|
|
63
22
|
namespace :ts do
|
64
|
-
|
65
23
|
desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
|
66
24
|
task :si => 'thinking_sphinx:smart_index'
|
67
25
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,9 +1,13 @@
|
|
1
|
-
$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
|
2
|
-
|
3
|
-
require 'spec'
|
4
|
-
require 'spec/autorun'
|
5
|
-
|
6
1
|
require 'thinking_sphinx'
|
7
2
|
require 'thinking_sphinx/deltas/resque_delta'
|
8
3
|
require 'flying_sphinx'
|
9
4
|
require 'flying_sphinx/resque_delta'
|
5
|
+
|
6
|
+
require 'mock_redis'
|
7
|
+
require 'fakefs/spec_helpers'
|
8
|
+
|
9
|
+
RSpec.configure do |c|
|
10
|
+
c.filter_run :focus => true
|
11
|
+
c.run_all_when_everything_filtered = true
|
12
|
+
c.treat_symbols_as_metadata_keys_with_true_values = true
|
13
|
+
end
|
@@ -0,0 +1,210 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe ThinkingSphinx::Deltas::ResqueDelta::CoreIndex do
|
4
|
+
let(:indices) { %w[foo bar] }
|
5
|
+
let(:config) { double('config') }
|
6
|
+
|
7
|
+
describe '#lock_delta' do
|
8
|
+
it 'should lock the delta' do
|
9
|
+
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock)
|
10
|
+
|
11
|
+
subject.lock_delta('foo')
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'should lock the delta for the given index' do
|
15
|
+
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock).with('foo_delta')
|
16
|
+
|
17
|
+
subject.lock_delta('foo')
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe '#unlock_delta' do
|
22
|
+
it 'should unlock the delta' do
|
23
|
+
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock)
|
24
|
+
|
25
|
+
subject.unlock_delta('foo')
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should unlock the delta for the given index' do
|
29
|
+
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock).with('foo_delta')
|
30
|
+
|
31
|
+
subject.unlock_delta('foo')
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
describe '#lock_deltas' do
|
36
|
+
it 'should lock all delta indices' do
|
37
|
+
subject.stub(:sphinx_indices => indices)
|
38
|
+
|
39
|
+
indices.each {|index| subject.should_receive(:lock_delta).once.with(index) }
|
40
|
+
|
41
|
+
subject.lock_deltas
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
describe '#unlock_deltas' do
|
46
|
+
it 'should unlock all delta indices' do
|
47
|
+
subject.stub(:sphinx_indices => indices)
|
48
|
+
|
49
|
+
indices.each {|index| subject.should_receive(:unlock_delta).once.with(index) }
|
50
|
+
|
51
|
+
subject.unlock_deltas
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe '#with_delta_index_lock' do
|
56
|
+
before :each do
|
57
|
+
subject.stub(:lock_delta)
|
58
|
+
subject.stub(:unlock_delta)
|
59
|
+
|
60
|
+
subject.stub(:block_called)
|
61
|
+
|
62
|
+
@block_called = false
|
63
|
+
@block = lambda { @block_called = true; subject.block_called }
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'should yield' do
|
67
|
+
subject.with_delta_index_lock('foo', &@block)
|
68
|
+
@block_called.should be_true
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'should lock before yielding' do
|
72
|
+
subject.should_receive(:lock_delta).with('foo').ordered
|
73
|
+
subject.should_receive(:block_called).ordered
|
74
|
+
|
75
|
+
subject.with_delta_index_lock('foo', &@block)
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'should unlock after yielding' do
|
79
|
+
subject.should_receive(:block_called).ordered
|
80
|
+
subject.should_receive(:unlock_delta).with('foo').ordered
|
81
|
+
|
82
|
+
subject.with_delta_index_lock('foo', &@block)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
describe '#smart_index' do
|
87
|
+
include FakeFS::SpecHelpers
|
88
|
+
|
89
|
+
let(:test_path) { '/tmp/ts-resque-delta/foo' }
|
90
|
+
|
91
|
+
before :each do
|
92
|
+
subject.stub(:ts_config => config)
|
93
|
+
config.stub(:config_file => 'foo_config')
|
94
|
+
config.stub(:build)
|
95
|
+
config.stub(:searchd_file_path => test_path)
|
96
|
+
config.stub_chain(:controller, :index) do
|
97
|
+
# Set $? to 0
|
98
|
+
`/usr/bin/true`
|
99
|
+
end
|
100
|
+
|
101
|
+
# Silence Generating config message
|
102
|
+
subject.stub(:puts)
|
103
|
+
|
104
|
+
subject.stub(:index_prefixes => indices)
|
105
|
+
subject.stub(:lock_delta)
|
106
|
+
subject.stub(:unlock_delta)
|
107
|
+
|
108
|
+
ThinkingSphinx::Deltas::ResqueDelta.stub(:prepare_for_core_index)
|
109
|
+
Resque.stub(:enqueue)
|
110
|
+
end
|
111
|
+
|
112
|
+
it 'should not generate sphinx configuration if INDEX_ONLY is true' do
|
113
|
+
ENV.stub(:[]).with('INDEX_ONLY').and_return('true')
|
114
|
+
ENV.stub(:[]).with('SILENT').and_return(nil)
|
115
|
+
config.should_not_receive(:build)
|
116
|
+
|
117
|
+
subject.smart_index
|
118
|
+
end
|
119
|
+
|
120
|
+
it 'should generate sphinx configuration if INDEX_ONLY is not true' do
|
121
|
+
ENV.stub(:[]).with('INDEX_ONLY').and_return(nil)
|
122
|
+
ENV.stub(:[]).with('SILENT').and_return(nil)
|
123
|
+
config.should_receive(:build).once
|
124
|
+
|
125
|
+
subject.smart_index
|
126
|
+
end
|
127
|
+
|
128
|
+
it 'should create the sphinx file directory' do
|
129
|
+
subject.smart_index
|
130
|
+
|
131
|
+
File.directory?(test_path).should be_true
|
132
|
+
end
|
133
|
+
|
134
|
+
it 'should index all core indices' do
|
135
|
+
indices.each do |index|
|
136
|
+
config.controller.should_receive(:index).with("#{index}_core", anything)
|
137
|
+
end
|
138
|
+
|
139
|
+
subject.smart_index
|
140
|
+
end
|
141
|
+
|
142
|
+
context "with delta lock" do
|
143
|
+
before :each do
|
144
|
+
subject.stub(:with_delta_index_lock).and_yield
|
145
|
+
end
|
146
|
+
|
147
|
+
it 'should index' do
|
148
|
+
config.controller.should_receive(:index)
|
149
|
+
subject.smart_index
|
150
|
+
end
|
151
|
+
|
152
|
+
it 'should signal resque delta to prepare for the core index' do
|
153
|
+
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:prepare_for_core_index)
|
154
|
+
subject.smart_index
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
context "without delta lock" do
|
159
|
+
before :each do
|
160
|
+
subject.stub(:with_delta_index_lock)
|
161
|
+
end
|
162
|
+
|
163
|
+
it 'should not index without the delta lock' do
|
164
|
+
config.controller.should_not_receive(:index)
|
165
|
+
subject.smart_index
|
166
|
+
end
|
167
|
+
|
168
|
+
it 'should not signal resque delta to prepare for the core index' do
|
169
|
+
ThinkingSphinx::Deltas::ResqueDelta.should_not_receive(:prepare_for_core_index)
|
170
|
+
subject.smart_index
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
it 'should create a delta job after the delta is unlocked' do
|
175
|
+
# Create a dummy method on subject that's called when Resque.enqueue is called so we can enforce order.
|
176
|
+
subject.stub(:resque_called)
|
177
|
+
Resque.stub(:enqueue) { subject.resque_called }
|
178
|
+
|
179
|
+
Resque.should_receive(:enqueue)
|
180
|
+
|
181
|
+
subject.should_receive(:with_delta_index_lock).ordered.exactly(indices.size)
|
182
|
+
subject.should_receive(:resque_called).ordered.exactly(indices.size)
|
183
|
+
|
184
|
+
subject.smart_index
|
185
|
+
end
|
186
|
+
|
187
|
+
context 'with an error' do
|
188
|
+
before :each do
|
189
|
+
config.stub_chain(:controller, :index) do
|
190
|
+
# Set $? to 1
|
191
|
+
`/usr/bin/false`
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
it 'should stop processing indexes after an error' do
|
196
|
+
config.controller.should_receive(:index).once
|
197
|
+
|
198
|
+
subject.smart_index
|
199
|
+
end
|
200
|
+
|
201
|
+
it 'should return false on failure' do
|
202
|
+
subject.smart_index.should be_false
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
it 'should return true on success' do
|
207
|
+
subject.smart_index.should be_true
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
@@ -1,48 +1,37 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
|
4
|
+
subject do
|
5
|
+
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.tap do |s|
|
6
|
+
s.stub(:` => true)
|
7
|
+
s.stub(:puts => nil)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
4
11
|
describe '.perform' do
|
5
12
|
before :each do
|
6
13
|
ThinkingSphinx.suppress_delta_output = false
|
7
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:` => true)
|
8
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:puts => nil)
|
9
14
|
ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?).and_return(false)
|
15
|
+
ThinkingSphinx.stub(:sphinx_running? => false)
|
10
16
|
end
|
11
17
|
|
12
18
|
it "should output the delta indexing by default" do
|
13
|
-
|
14
|
-
|
15
|
-
['foo_delta']
|
16
|
-
)
|
19
|
+
subject.should_receive(:puts)
|
20
|
+
subject.perform('foo_delta')
|
17
21
|
end
|
18
22
|
|
19
23
|
it "should not output the delta indexing if requested" do
|
20
24
|
ThinkingSphinx.suppress_delta_output = true
|
21
|
-
|
22
|
-
|
23
|
-
['foo_delta']
|
24
|
-
)
|
25
|
+
subject.should_not_receive(:puts)
|
26
|
+
subject.perform('foo_delta')
|
25
27
|
end
|
26
28
|
|
27
|
-
it "should process just the requested
|
28
|
-
|
29
|
+
it "should process just the requested index" do
|
30
|
+
subject.should_receive(:`) do |c|
|
29
31
|
c.should match(/foo_delta/)
|
30
32
|
c.should_not match(/--all/)
|
31
33
|
end
|
32
|
-
|
33
|
-
['foo_delta']
|
34
|
-
)
|
35
|
-
end
|
36
|
-
|
37
|
-
context 'multiple indices' do
|
38
|
-
it "should process all requested indices" do
|
39
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:`) do |c|
|
40
|
-
c.should match(/foo_delta bar_delta/)
|
41
|
-
end
|
42
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
|
43
|
-
['foo_delta', 'bar_delta']
|
44
|
-
)
|
45
|
-
end
|
34
|
+
subject.perform('foo_delta')
|
46
35
|
end
|
47
36
|
|
48
37
|
context 'when an index is locked' do
|
@@ -53,18 +42,132 @@ describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
|
|
53
42
|
end
|
54
43
|
|
55
44
|
it "should not start the indexer" do
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
45
|
+
subject.should_not_receive(:`)
|
46
|
+
subject.perform('foo_delta')
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should start the indexer for unlocked indexes" do
|
50
|
+
subject.should_receive(:`)
|
51
|
+
subject.perform('bar_delta')
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'with flag as deleted document ids' do
|
56
|
+
let(:client) { stub('client', :update => true) }
|
57
|
+
let(:document_ids) { [1, 2, 3] }
|
58
|
+
|
59
|
+
before :each do
|
60
|
+
ThinkingSphinx.updates_enabled = true
|
61
|
+
|
62
|
+
ThinkingSphinx::Configuration.instance.stub(:client => client)
|
63
|
+
ThinkingSphinx.stub(:sphinx_running? => true)
|
64
|
+
|
65
|
+
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:processing_members => document_ids)
|
66
|
+
subject.stub(:filter_flag_as_deleted_ids => document_ids)
|
60
67
|
end
|
61
68
|
|
62
|
-
it
|
63
|
-
ThinkingSphinx::Deltas::ResqueDelta::
|
64
|
-
|
65
|
-
['bar_delta', 'foo_delta']
|
66
|
-
)
|
69
|
+
it 'should get the processing set of flag as deleted document ids' do
|
70
|
+
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:processing_members).with('foo_core')
|
71
|
+
subject.perform('foo_delta')
|
67
72
|
end
|
73
|
+
|
74
|
+
it "should not update if Sphinx isn't running" do
|
75
|
+
ThinkingSphinx.stub(:sphinx_running? => false)
|
76
|
+
client.should_not_receive(:update)
|
77
|
+
subject.perform('foo_delta')
|
78
|
+
end
|
79
|
+
|
80
|
+
it "should validate the document ids with sphinx" do
|
81
|
+
subject.should_receive(:filter_flag_as_deleted_ids).with(document_ids, 'foo_core')
|
82
|
+
|
83
|
+
subject.perform('foo_delta')
|
84
|
+
end
|
85
|
+
|
86
|
+
context "with invalid ids" do
|
87
|
+
before :each do
|
88
|
+
subject.stub(:filter_flag_as_deleted_ids => document_ids.reject {|x| x == 2} )
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should not update documents that aren't in the index" do
|
92
|
+
client.should_receive(:update) do |index, attributes, values|
|
93
|
+
values.should_not include(2)
|
94
|
+
end
|
95
|
+
subject.perform('foo_delta')
|
96
|
+
end
|
97
|
+
|
98
|
+
it "should update documents that are in the index" do
|
99
|
+
client.should_receive(:update) do |index, attributes, values|
|
100
|
+
values.keys.should eql(document_ids.reject{|x| x == 2})
|
101
|
+
end
|
102
|
+
subject.perform('foo_delta')
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
it "should update the specified index" do
|
107
|
+
client.should_receive(:update) do |index, attributes, values|
|
108
|
+
index.should == 'foo_core'
|
109
|
+
end
|
110
|
+
subject.perform('foo_delta')
|
111
|
+
end
|
112
|
+
|
113
|
+
it "should update the sphinx_deleted attribute" do
|
114
|
+
client.should_receive(:update) do |index, attributes, values|
|
115
|
+
attributes.should == ['sphinx_deleted']
|
116
|
+
end
|
117
|
+
subject.perform('foo_delta')
|
118
|
+
end
|
119
|
+
|
120
|
+
it "should set sphinx_deleted for valid documents to true" do
|
121
|
+
client.should_receive(:update) do |index, attributes, values|
|
122
|
+
document_ids.each {|id| values[id].should == [1] }
|
123
|
+
end
|
124
|
+
subject.perform('foo_delta')
|
125
|
+
end
|
126
|
+
|
127
|
+
context "without any ids" do
|
128
|
+
let(:document_ids) { [] }
|
129
|
+
|
130
|
+
it "should not validate the ids with sphinx" do
|
131
|
+
subject.should_not_receive(:filter_flag_as_deleted_ids)
|
132
|
+
subject.perform('foo_delta')
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
describe '.around_perform_lock1' do
|
139
|
+
before :each do
|
140
|
+
Resque.stub(:encode => 'DeltaJobsAreAwesome')
|
141
|
+
Resque.stub_chain(:redis, :lrem)
|
142
|
+
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:get_subset_for_processing)
|
143
|
+
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:clear_processing)
|
144
|
+
end
|
145
|
+
|
146
|
+
it 'should clear all other delta jobs' do
|
147
|
+
Resque.redis.should_receive(:lrem).with("queue:#{subject.instance_variable_get(:@queue)}", 0, 'DeltaJobsAreAwesome')
|
148
|
+
|
149
|
+
subject.around_perform_lock1('foo_delta') {}
|
150
|
+
end
|
151
|
+
|
152
|
+
it 'should set up the processing set of document ids' do
|
153
|
+
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:get_subset_for_processing).with('foo_core')
|
154
|
+
|
155
|
+
subject.around_perform_lock1('foo_delta') {}
|
156
|
+
end
|
157
|
+
|
158
|
+
it 'should clear the processing set when finished' do
|
159
|
+
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:clear_processing).with('foo_core')
|
160
|
+
|
161
|
+
subject.around_perform_lock1('foo_delta') {}
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
describe '.lock_failed' do
|
166
|
+
it 'should enqueue the delta job again' do
|
167
|
+
Resque.stub(:enqueue => true)
|
168
|
+
Resque.should_receive(:enqueue)
|
169
|
+
|
170
|
+
subject.lock_failed('foo_delta')
|
68
171
|
end
|
69
172
|
end
|
70
173
|
end
|