ts-resque-delta 1.2.4 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +1 -1
- data/.travis.yml +14 -14
- data/Appraisals +4 -6
- data/Gemfile +1 -1
- data/README.markdown +31 -39
- data/Rakefile +6 -13
- data/gemfiles/binary.gemfile +7 -0
- data/gemfiles/sphinxql.gemfile +7 -0
- data/lib/thinking_sphinx/deltas/resque_delta.rb +66 -76
- data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +1 -113
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +7 -0
- data/lib/ts-resque-delta.rb +0 -2
- data/spec/acceptance/resque_deltas_spec.rb +50 -0
- data/spec/acceptance/spec_helper.rb +4 -0
- data/spec/acceptance/support/database_cleaner.rb +11 -0
- data/spec/acceptance/support/sphinx_controller.rb +61 -0
- data/spec/acceptance/support/sphinx_helpers.rb +36 -0
- data/spec/internal/.gitignore +2 -0
- data/spec/internal/app/indices/book_index.rb +3 -0
- data/spec/internal/app/models/book.rb +7 -0
- data/spec/internal/config/database.yml +5 -0
- data/spec/internal/db/schema.rb +10 -0
- data/spec/internal/log/.gitignore +1 -0
- data/spec/spec_helper.rb +16 -12
- data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +11 -155
- data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +3 -166
- data/ts-resque-delta.gemspec +19 -28
- metadata +90 -230
- data/Guardfile +0 -17
- data/features/resque_deltas.feature +0 -62
- data/features/smart_indexing.feature +0 -43
- data/features/step_definitions/common_steps.rb +0 -76
- data/features/step_definitions/resque_delta_steps.rb +0 -33
- data/features/step_definitions/smart_indexing_steps.rb +0 -3
- data/features/support/env.rb +0 -41
- data/features/thinking_sphinx/database.example.yml +0 -4
- data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +0 -4
- data/features/thinking_sphinx/models/delayed_beta.rb +0 -6
- data/gemfiles/activerecord2.gemfile +0 -8
- data/gemfiles/activerecord2.gemfile.lock +0 -114
- data/gemfiles/activerecord3.gemfile +0 -8
- data/gemfiles/activerecord3.gemfile.lock +0 -123
- data/lib/flying_sphinx/resque_delta.rb +0 -35
- data/lib/flying_sphinx/resque_delta/delta_job.rb +0 -14
- data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb +0 -7
- data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +0 -116
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +0 -59
- data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +0 -47
- data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb +0 -8
- data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +0 -38
- data/lib/thinking_sphinx/deltas/resque_delta/version.rb +0 -7
- data/spec/flying_sphinx/resque_delta/delta_job_spec.rb +0 -32
- data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb +0 -23
- data/spec/flying_sphinx/resque_delta_spec.rb +0 -130
- data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +0 -210
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +0 -126
- data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +0 -67
@@ -1,210 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::ResqueDelta::CoreIndex do
|
4
|
-
let(:indices) { %w[foo bar] }
|
5
|
-
let(:config) { double('config') }
|
6
|
-
|
7
|
-
describe '#lock_delta' do
|
8
|
-
it 'should lock the delta' do
|
9
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock)
|
10
|
-
|
11
|
-
subject.lock_delta('foo')
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'should lock the delta for the given index' do
|
15
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock).with('foo_delta')
|
16
|
-
|
17
|
-
subject.lock_delta('foo')
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
describe '#unlock_delta' do
|
22
|
-
it 'should unlock the delta' do
|
23
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock)
|
24
|
-
|
25
|
-
subject.unlock_delta('foo')
|
26
|
-
end
|
27
|
-
|
28
|
-
it 'should unlock the delta for the given index' do
|
29
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock).with('foo_delta')
|
30
|
-
|
31
|
-
subject.unlock_delta('foo')
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
describe '#lock_deltas' do
|
36
|
-
it 'should lock all delta indices' do
|
37
|
-
subject.stub(:sphinx_indices => indices)
|
38
|
-
|
39
|
-
indices.each {|index| subject.should_receive(:lock_delta).once.with(index) }
|
40
|
-
|
41
|
-
subject.lock_deltas
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
describe '#unlock_deltas' do
|
46
|
-
it 'should unlock all delta indices' do
|
47
|
-
subject.stub(:sphinx_indices => indices)
|
48
|
-
|
49
|
-
indices.each {|index| subject.should_receive(:unlock_delta).once.with(index) }
|
50
|
-
|
51
|
-
subject.unlock_deltas
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe '#with_delta_index_lock' do
|
56
|
-
before :each do
|
57
|
-
subject.stub(:lock_delta)
|
58
|
-
subject.stub(:unlock_delta)
|
59
|
-
|
60
|
-
subject.stub(:block_called)
|
61
|
-
|
62
|
-
@block_called = false
|
63
|
-
@block = lambda { @block_called = true; subject.block_called }
|
64
|
-
end
|
65
|
-
|
66
|
-
it 'should yield' do
|
67
|
-
subject.with_delta_index_lock('foo', &@block)
|
68
|
-
@block_called.should be_true
|
69
|
-
end
|
70
|
-
|
71
|
-
it 'should lock before yielding' do
|
72
|
-
subject.should_receive(:lock_delta).with('foo').ordered
|
73
|
-
subject.should_receive(:block_called).ordered
|
74
|
-
|
75
|
-
subject.with_delta_index_lock('foo', &@block)
|
76
|
-
end
|
77
|
-
|
78
|
-
it 'should unlock after yielding' do
|
79
|
-
subject.should_receive(:block_called).ordered
|
80
|
-
subject.should_receive(:unlock_delta).with('foo').ordered
|
81
|
-
|
82
|
-
subject.with_delta_index_lock('foo', &@block)
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
describe '#smart_index' do
|
87
|
-
include FakeFS::SpecHelpers
|
88
|
-
|
89
|
-
let(:test_path) { '/tmp/ts-resque-delta/foo' }
|
90
|
-
|
91
|
-
before :each do
|
92
|
-
subject.stub(:ts_config => config)
|
93
|
-
config.stub(:config_file => 'foo_config')
|
94
|
-
config.stub(:build)
|
95
|
-
config.stub(:searchd_file_path => test_path)
|
96
|
-
config.stub_chain(:controller, :index) do
|
97
|
-
# Set $? to 0
|
98
|
-
`#{File.join(SPEC_BIN_PATH, 'true.rb')}`
|
99
|
-
end
|
100
|
-
|
101
|
-
# Silence Generating config message
|
102
|
-
subject.stub(:puts)
|
103
|
-
|
104
|
-
subject.stub(:index_prefixes => indices)
|
105
|
-
subject.stub(:lock_delta)
|
106
|
-
subject.stub(:unlock_delta)
|
107
|
-
|
108
|
-
ThinkingSphinx::Deltas::ResqueDelta.stub(:prepare_for_core_index)
|
109
|
-
Resque.stub(:enqueue)
|
110
|
-
end
|
111
|
-
|
112
|
-
it 'should not generate sphinx configuration if INDEX_ONLY is true' do
|
113
|
-
ENV.stub(:[]).with('INDEX_ONLY').and_return('true')
|
114
|
-
ENV.stub(:[]).with('SILENT').and_return(nil)
|
115
|
-
config.should_not_receive(:build)
|
116
|
-
|
117
|
-
subject.smart_index
|
118
|
-
end
|
119
|
-
|
120
|
-
it 'should generate sphinx configuration if INDEX_ONLY is not true' do
|
121
|
-
ENV.stub(:[]).with('INDEX_ONLY').and_return(nil)
|
122
|
-
ENV.stub(:[]).with('SILENT').and_return(nil)
|
123
|
-
config.should_receive(:build).once
|
124
|
-
|
125
|
-
subject.smart_index
|
126
|
-
end
|
127
|
-
|
128
|
-
it 'should create the sphinx file directory' do
|
129
|
-
subject.smart_index
|
130
|
-
|
131
|
-
File.directory?(test_path).should be_true
|
132
|
-
end
|
133
|
-
|
134
|
-
it 'should index all core indices' do
|
135
|
-
indices.each do |index|
|
136
|
-
config.controller.should_receive(:index).with("#{index}_core", anything)
|
137
|
-
end
|
138
|
-
|
139
|
-
subject.smart_index
|
140
|
-
end
|
141
|
-
|
142
|
-
context "with delta lock" do
|
143
|
-
before :each do
|
144
|
-
subject.stub(:with_delta_index_lock).and_yield
|
145
|
-
end
|
146
|
-
|
147
|
-
it 'should index' do
|
148
|
-
config.controller.should_receive(:index)
|
149
|
-
subject.smart_index
|
150
|
-
end
|
151
|
-
|
152
|
-
it 'should signal resque delta to prepare for the core index' do
|
153
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:prepare_for_core_index)
|
154
|
-
subject.smart_index
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
context "without delta lock" do
|
159
|
-
before :each do
|
160
|
-
subject.stub(:with_delta_index_lock)
|
161
|
-
end
|
162
|
-
|
163
|
-
it 'should not index without the delta lock' do
|
164
|
-
config.controller.should_not_receive(:index)
|
165
|
-
subject.smart_index
|
166
|
-
end
|
167
|
-
|
168
|
-
it 'should not signal resque delta to prepare for the core index' do
|
169
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_not_receive(:prepare_for_core_index)
|
170
|
-
subject.smart_index
|
171
|
-
end
|
172
|
-
end
|
173
|
-
|
174
|
-
it 'should create a delta job after the delta is unlocked' do
|
175
|
-
# Create a dummy method on subject that's called when Resque.enqueue is called so we can enforce order.
|
176
|
-
subject.stub(:resque_called)
|
177
|
-
Resque.stub(:enqueue) { subject.resque_called }
|
178
|
-
|
179
|
-
Resque.should_receive(:enqueue)
|
180
|
-
|
181
|
-
subject.should_receive(:with_delta_index_lock).ordered.exactly(indices.size)
|
182
|
-
subject.should_receive(:resque_called).ordered.exactly(indices.size)
|
183
|
-
|
184
|
-
subject.smart_index
|
185
|
-
end
|
186
|
-
|
187
|
-
context 'with an error' do
|
188
|
-
before :each do
|
189
|
-
config.stub_chain(:controller, :index) do
|
190
|
-
# Set $? to 1
|
191
|
-
`#{File.join(SPEC_BIN_PATH, 'false.rb')}`
|
192
|
-
end
|
193
|
-
end
|
194
|
-
|
195
|
-
it 'should stop processing indexes after an error' do
|
196
|
-
config.controller.should_receive(:index).once
|
197
|
-
|
198
|
-
subject.smart_index
|
199
|
-
end
|
200
|
-
|
201
|
-
it 'should return false on failure' do
|
202
|
-
subject.smart_index.should be_false
|
203
|
-
end
|
204
|
-
end
|
205
|
-
|
206
|
-
it 'should return true on success' do
|
207
|
-
subject.smart_index.should be_true
|
208
|
-
end
|
209
|
-
end
|
210
|
-
end
|
@@ -1,126 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet do
|
4
|
-
describe '.add' do
|
5
|
-
before :each do
|
6
|
-
Resque.stub_chain(:redis, :sadd => true)
|
7
|
-
end
|
8
|
-
|
9
|
-
it 'should add the document id to the correct set' do
|
10
|
-
Resque.redis.should_receive(:sadd).once.with(subject.set_name('foo_core'), 42)
|
11
|
-
subject.add('foo_core', 42)
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
describe '.clear!' do
|
16
|
-
before :each do
|
17
|
-
Resque.stub_chain(:redis, :del)
|
18
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock)
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'should delete all items in the set' do
|
22
|
-
Resque.redis.should_receive(:del).once.with(subject.set_name('foo_core'))
|
23
|
-
subject.clear!('foo_core')
|
24
|
-
end
|
25
|
-
|
26
|
-
context "with DeltaJob integration" do
|
27
|
-
before :each do
|
28
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock).and_yield
|
29
|
-
end
|
30
|
-
|
31
|
-
it 'should acquire the DeltaJob lock' do
|
32
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:around_perform_lock).once.with('foo_delta')
|
33
|
-
subject.clear!('foo_core')
|
34
|
-
end
|
35
|
-
|
36
|
-
it 'should delete all items in the processing set' do
|
37
|
-
Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
|
38
|
-
subject.clear!('foo_core')
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
describe '.clear_all!' do
|
44
|
-
let(:core_indices) { %w[foo_core bar_core] }
|
45
|
-
|
46
|
-
it 'should clear each index' do
|
47
|
-
ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.stub_chain(:core_indices, :each).tap do |s|
|
48
|
-
core_indices.inject(s) do |s, index|
|
49
|
-
s.and_yield(index)
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
core_indices.each do |index|
|
54
|
-
subject.should_receive(:clear!).with(index)
|
55
|
-
end
|
56
|
-
|
57
|
-
subject.clear_all!
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
describe '.get_subset_for_processing' do
|
62
|
-
let(:mock_redis) do
|
63
|
-
Resque.redis = mr = MockRedis.new
|
64
|
-
subject.add 'foo_core', 42
|
65
|
-
subject.add 'foo_core', 52
|
66
|
-
subject.add 'foo_core', 100
|
67
|
-
mr
|
68
|
-
end
|
69
|
-
|
70
|
-
before :each do
|
71
|
-
Resque.redis = mock_redis.clone
|
72
|
-
end
|
73
|
-
|
74
|
-
it 'should move all members from the flag as deleted set to the processing set' do
|
75
|
-
subject.get_subset_for_processing('foo_core')
|
76
|
-
|
77
|
-
Resque.redis.scard(subject.set_name('foo_core')).should eql(0)
|
78
|
-
Resque.redis.scard(subject.processing_name('foo_core')).should eql(3)
|
79
|
-
end
|
80
|
-
|
81
|
-
it 'should remove the temp set' do
|
82
|
-
subject.get_subset_for_processing('foo_core')
|
83
|
-
|
84
|
-
Resque.redis.scard(subject.temp_name('foo_core')).should eql(0)
|
85
|
-
end
|
86
|
-
|
87
|
-
it 'should preserve existing members of the processing set' do
|
88
|
-
Resque.redis.sadd(subject.processing_name('foo_core'), 1)
|
89
|
-
|
90
|
-
subject.get_subset_for_processing('foo_core')
|
91
|
-
|
92
|
-
Resque.redis.smembers(subject.processing_name('foo_core')).should =~ %w[1 42 52 100]
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
describe '.processing_members' do
|
97
|
-
let(:document_ids) { %w[1, 2, 3] }
|
98
|
-
|
99
|
-
before :each do
|
100
|
-
Resque.stub_chain(:redis, :smembers => document_ids)
|
101
|
-
end
|
102
|
-
|
103
|
-
it 'should get the members of the correct set' do
|
104
|
-
Resque.redis.should_receive(:smembers).once.with(subject.processing_name('foo_core'))
|
105
|
-
subject.processing_members('foo_core')
|
106
|
-
end
|
107
|
-
|
108
|
-
it 'should return a list of integers' do
|
109
|
-
subject.processing_members('foo_core').each do |id|
|
110
|
-
id.class.should == Fixnum
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
describe '.clear_processing' do
|
116
|
-
before :each do
|
117
|
-
Resque.stub_chain(:redis, :del)
|
118
|
-
end
|
119
|
-
|
120
|
-
it 'should delete the processing set' do
|
121
|
-
Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
|
122
|
-
|
123
|
-
subject.clear_processing('foo_core')
|
124
|
-
end
|
125
|
-
end
|
126
|
-
end
|
@@ -1,67 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::ResqueDelta::IndexUtils do
|
4
|
-
let(:indices) { %w[foo_core foo_delta foo bar_core bar_delta bar] }
|
5
|
-
let(:config) { double('config') }
|
6
|
-
|
7
|
-
before :each do
|
8
|
-
ThinkingSphinx::Configuration.stub(:instance => config)
|
9
|
-
config.stub(:generate)
|
10
|
-
config.stub_chain(:configuration, :indices, :collect => indices)
|
11
|
-
|
12
|
-
subject.reload!
|
13
|
-
end
|
14
|
-
|
15
|
-
describe '.index_prefixes' do
|
16
|
-
it 'should use a cached value if one exists' do
|
17
|
-
indices = []
|
18
|
-
subject.instance_variable_set(:@prefixes, indices)
|
19
|
-
|
20
|
-
subject.index_prefixes.should be(indices)
|
21
|
-
end
|
22
|
-
|
23
|
-
it 'should return a list of only index prefixes' do
|
24
|
-
subject.index_prefixes.should =~ %w[foo bar]
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
describe '.core_indices' do
|
29
|
-
it 'should use a cached value if one exists' do
|
30
|
-
indices = []
|
31
|
-
subject.instance_variable_set(:@core_indices, indices)
|
32
|
-
|
33
|
-
subject.core_indices.should be(indices)
|
34
|
-
end
|
35
|
-
|
36
|
-
it 'should return a list of only core indices' do
|
37
|
-
subject.core_indices.should =~ %w[foo_core bar_core]
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
describe '.delta_indices' do
|
42
|
-
it 'should use a cached value if one exists' do
|
43
|
-
indices = []
|
44
|
-
subject.instance_variable_set(:@delta_indices, indices)
|
45
|
-
|
46
|
-
subject.delta_indices.should be(indices)
|
47
|
-
end
|
48
|
-
|
49
|
-
it 'should return a list of only delta indices' do
|
50
|
-
subject.delta_indices.should =~ %w[foo_delta bar_delta]
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
describe '.ts_config' do
|
55
|
-
it 'should use a cached value if one exists' do
|
56
|
-
subject.instance_variable_set(:@ts_config, config)
|
57
|
-
|
58
|
-
subject.ts_config.should be(config)
|
59
|
-
end
|
60
|
-
|
61
|
-
it 'should generate the config when fetching the Configuration instance' do
|
62
|
-
config.should_receive(:generate)
|
63
|
-
|
64
|
-
subject.ts_config
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|