ts-resque-delta 1.2.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +1 -1
- data/.travis.yml +14 -14
- data/Appraisals +4 -6
- data/Gemfile +1 -1
- data/README.markdown +31 -39
- data/Rakefile +6 -13
- data/gemfiles/binary.gemfile +7 -0
- data/gemfiles/sphinxql.gemfile +7 -0
- data/lib/thinking_sphinx/deltas/resque_delta.rb +66 -76
- data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +1 -113
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +7 -0
- data/lib/ts-resque-delta.rb +0 -2
- data/spec/acceptance/resque_deltas_spec.rb +50 -0
- data/spec/acceptance/spec_helper.rb +4 -0
- data/spec/acceptance/support/database_cleaner.rb +11 -0
- data/spec/acceptance/support/sphinx_controller.rb +61 -0
- data/spec/acceptance/support/sphinx_helpers.rb +36 -0
- data/spec/internal/.gitignore +2 -0
- data/spec/internal/app/indices/book_index.rb +3 -0
- data/spec/internal/app/models/book.rb +7 -0
- data/spec/internal/config/database.yml +5 -0
- data/spec/internal/db/schema.rb +10 -0
- data/spec/internal/log/.gitignore +1 -0
- data/spec/spec_helper.rb +16 -12
- data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +11 -155
- data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +3 -166
- data/ts-resque-delta.gemspec +19 -28
- metadata +90 -230
- data/Guardfile +0 -17
- data/features/resque_deltas.feature +0 -62
- data/features/smart_indexing.feature +0 -43
- data/features/step_definitions/common_steps.rb +0 -76
- data/features/step_definitions/resque_delta_steps.rb +0 -33
- data/features/step_definitions/smart_indexing_steps.rb +0 -3
- data/features/support/env.rb +0 -41
- data/features/thinking_sphinx/database.example.yml +0 -4
- data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +0 -4
- data/features/thinking_sphinx/models/delayed_beta.rb +0 -6
- data/gemfiles/activerecord2.gemfile +0 -8
- data/gemfiles/activerecord2.gemfile.lock +0 -114
- data/gemfiles/activerecord3.gemfile +0 -8
- data/gemfiles/activerecord3.gemfile.lock +0 -123
- data/lib/flying_sphinx/resque_delta.rb +0 -35
- data/lib/flying_sphinx/resque_delta/delta_job.rb +0 -14
- data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb +0 -7
- data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +0 -116
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +0 -59
- data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +0 -47
- data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb +0 -8
- data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +0 -38
- data/lib/thinking_sphinx/deltas/resque_delta/version.rb +0 -7
- data/spec/flying_sphinx/resque_delta/delta_job_spec.rb +0 -32
- data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb +0 -23
- data/spec/flying_sphinx/resque_delta_spec.rb +0 -130
- data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +0 -210
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +0 -126
- data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +0 -67
@@ -1,210 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::ResqueDelta::CoreIndex do
|
4
|
-
let(:indices) { %w[foo bar] }
|
5
|
-
let(:config) { double('config') }
|
6
|
-
|
7
|
-
describe '#lock_delta' do
|
8
|
-
it 'should lock the delta' do
|
9
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock)
|
10
|
-
|
11
|
-
subject.lock_delta('foo')
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'should lock the delta for the given index' do
|
15
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock).with('foo_delta')
|
16
|
-
|
17
|
-
subject.lock_delta('foo')
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
describe '#unlock_delta' do
|
22
|
-
it 'should unlock the delta' do
|
23
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock)
|
24
|
-
|
25
|
-
subject.unlock_delta('foo')
|
26
|
-
end
|
27
|
-
|
28
|
-
it 'should unlock the delta for the given index' do
|
29
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock).with('foo_delta')
|
30
|
-
|
31
|
-
subject.unlock_delta('foo')
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
describe '#lock_deltas' do
|
36
|
-
it 'should lock all delta indices' do
|
37
|
-
subject.stub(:sphinx_indices => indices)
|
38
|
-
|
39
|
-
indices.each {|index| subject.should_receive(:lock_delta).once.with(index) }
|
40
|
-
|
41
|
-
subject.lock_deltas
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
describe '#unlock_deltas' do
|
46
|
-
it 'should unlock all delta indices' do
|
47
|
-
subject.stub(:sphinx_indices => indices)
|
48
|
-
|
49
|
-
indices.each {|index| subject.should_receive(:unlock_delta).once.with(index) }
|
50
|
-
|
51
|
-
subject.unlock_deltas
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe '#with_delta_index_lock' do
|
56
|
-
before :each do
|
57
|
-
subject.stub(:lock_delta)
|
58
|
-
subject.stub(:unlock_delta)
|
59
|
-
|
60
|
-
subject.stub(:block_called)
|
61
|
-
|
62
|
-
@block_called = false
|
63
|
-
@block = lambda { @block_called = true; subject.block_called }
|
64
|
-
end
|
65
|
-
|
66
|
-
it 'should yield' do
|
67
|
-
subject.with_delta_index_lock('foo', &@block)
|
68
|
-
@block_called.should be_true
|
69
|
-
end
|
70
|
-
|
71
|
-
it 'should lock before yielding' do
|
72
|
-
subject.should_receive(:lock_delta).with('foo').ordered
|
73
|
-
subject.should_receive(:block_called).ordered
|
74
|
-
|
75
|
-
subject.with_delta_index_lock('foo', &@block)
|
76
|
-
end
|
77
|
-
|
78
|
-
it 'should unlock after yielding' do
|
79
|
-
subject.should_receive(:block_called).ordered
|
80
|
-
subject.should_receive(:unlock_delta).with('foo').ordered
|
81
|
-
|
82
|
-
subject.with_delta_index_lock('foo', &@block)
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
describe '#smart_index' do
|
87
|
-
include FakeFS::SpecHelpers
|
88
|
-
|
89
|
-
let(:test_path) { '/tmp/ts-resque-delta/foo' }
|
90
|
-
|
91
|
-
before :each do
|
92
|
-
subject.stub(:ts_config => config)
|
93
|
-
config.stub(:config_file => 'foo_config')
|
94
|
-
config.stub(:build)
|
95
|
-
config.stub(:searchd_file_path => test_path)
|
96
|
-
config.stub_chain(:controller, :index) do
|
97
|
-
# Set $? to 0
|
98
|
-
`#{File.join(SPEC_BIN_PATH, 'true.rb')}`
|
99
|
-
end
|
100
|
-
|
101
|
-
# Silence Generating config message
|
102
|
-
subject.stub(:puts)
|
103
|
-
|
104
|
-
subject.stub(:index_prefixes => indices)
|
105
|
-
subject.stub(:lock_delta)
|
106
|
-
subject.stub(:unlock_delta)
|
107
|
-
|
108
|
-
ThinkingSphinx::Deltas::ResqueDelta.stub(:prepare_for_core_index)
|
109
|
-
Resque.stub(:enqueue)
|
110
|
-
end
|
111
|
-
|
112
|
-
it 'should not generate sphinx configuration if INDEX_ONLY is true' do
|
113
|
-
ENV.stub(:[]).with('INDEX_ONLY').and_return('true')
|
114
|
-
ENV.stub(:[]).with('SILENT').and_return(nil)
|
115
|
-
config.should_not_receive(:build)
|
116
|
-
|
117
|
-
subject.smart_index
|
118
|
-
end
|
119
|
-
|
120
|
-
it 'should generate sphinx configuration if INDEX_ONLY is not true' do
|
121
|
-
ENV.stub(:[]).with('INDEX_ONLY').and_return(nil)
|
122
|
-
ENV.stub(:[]).with('SILENT').and_return(nil)
|
123
|
-
config.should_receive(:build).once
|
124
|
-
|
125
|
-
subject.smart_index
|
126
|
-
end
|
127
|
-
|
128
|
-
it 'should create the sphinx file directory' do
|
129
|
-
subject.smart_index
|
130
|
-
|
131
|
-
File.directory?(test_path).should be_true
|
132
|
-
end
|
133
|
-
|
134
|
-
it 'should index all core indices' do
|
135
|
-
indices.each do |index|
|
136
|
-
config.controller.should_receive(:index).with("#{index}_core", anything)
|
137
|
-
end
|
138
|
-
|
139
|
-
subject.smart_index
|
140
|
-
end
|
141
|
-
|
142
|
-
context "with delta lock" do
|
143
|
-
before :each do
|
144
|
-
subject.stub(:with_delta_index_lock).and_yield
|
145
|
-
end
|
146
|
-
|
147
|
-
it 'should index' do
|
148
|
-
config.controller.should_receive(:index)
|
149
|
-
subject.smart_index
|
150
|
-
end
|
151
|
-
|
152
|
-
it 'should signal resque delta to prepare for the core index' do
|
153
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_receive(:prepare_for_core_index)
|
154
|
-
subject.smart_index
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
context "without delta lock" do
|
159
|
-
before :each do
|
160
|
-
subject.stub(:with_delta_index_lock)
|
161
|
-
end
|
162
|
-
|
163
|
-
it 'should not index without the delta lock' do
|
164
|
-
config.controller.should_not_receive(:index)
|
165
|
-
subject.smart_index
|
166
|
-
end
|
167
|
-
|
168
|
-
it 'should not signal resque delta to prepare for the core index' do
|
169
|
-
ThinkingSphinx::Deltas::ResqueDelta.should_not_receive(:prepare_for_core_index)
|
170
|
-
subject.smart_index
|
171
|
-
end
|
172
|
-
end
|
173
|
-
|
174
|
-
it 'should create a delta job after the delta is unlocked' do
|
175
|
-
# Create a dummy method on subject that's called when Resque.enqueue is called so we can enforce order.
|
176
|
-
subject.stub(:resque_called)
|
177
|
-
Resque.stub(:enqueue) { subject.resque_called }
|
178
|
-
|
179
|
-
Resque.should_receive(:enqueue)
|
180
|
-
|
181
|
-
subject.should_receive(:with_delta_index_lock).ordered.exactly(indices.size)
|
182
|
-
subject.should_receive(:resque_called).ordered.exactly(indices.size)
|
183
|
-
|
184
|
-
subject.smart_index
|
185
|
-
end
|
186
|
-
|
187
|
-
context 'with an error' do
|
188
|
-
before :each do
|
189
|
-
config.stub_chain(:controller, :index) do
|
190
|
-
# Set $? to 1
|
191
|
-
`#{File.join(SPEC_BIN_PATH, 'false.rb')}`
|
192
|
-
end
|
193
|
-
end
|
194
|
-
|
195
|
-
it 'should stop processing indexes after an error' do
|
196
|
-
config.controller.should_receive(:index).once
|
197
|
-
|
198
|
-
subject.smart_index
|
199
|
-
end
|
200
|
-
|
201
|
-
it 'should return false on failure' do
|
202
|
-
subject.smart_index.should be_false
|
203
|
-
end
|
204
|
-
end
|
205
|
-
|
206
|
-
it 'should return true on success' do
|
207
|
-
subject.smart_index.should be_true
|
208
|
-
end
|
209
|
-
end
|
210
|
-
end
|
@@ -1,126 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet do
|
4
|
-
describe '.add' do
|
5
|
-
before :each do
|
6
|
-
Resque.stub_chain(:redis, :sadd => true)
|
7
|
-
end
|
8
|
-
|
9
|
-
it 'should add the document id to the correct set' do
|
10
|
-
Resque.redis.should_receive(:sadd).once.with(subject.set_name('foo_core'), 42)
|
11
|
-
subject.add('foo_core', 42)
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
describe '.clear!' do
|
16
|
-
before :each do
|
17
|
-
Resque.stub_chain(:redis, :del)
|
18
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock)
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'should delete all items in the set' do
|
22
|
-
Resque.redis.should_receive(:del).once.with(subject.set_name('foo_core'))
|
23
|
-
subject.clear!('foo_core')
|
24
|
-
end
|
25
|
-
|
26
|
-
context "with DeltaJob integration" do
|
27
|
-
before :each do
|
28
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock).and_yield
|
29
|
-
end
|
30
|
-
|
31
|
-
it 'should acquire the DeltaJob lock' do
|
32
|
-
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:around_perform_lock).once.with('foo_delta')
|
33
|
-
subject.clear!('foo_core')
|
34
|
-
end
|
35
|
-
|
36
|
-
it 'should delete all items in the processing set' do
|
37
|
-
Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
|
38
|
-
subject.clear!('foo_core')
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
describe '.clear_all!' do
|
44
|
-
let(:core_indices) { %w[foo_core bar_core] }
|
45
|
-
|
46
|
-
it 'should clear each index' do
|
47
|
-
ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.stub_chain(:core_indices, :each).tap do |s|
|
48
|
-
core_indices.inject(s) do |s, index|
|
49
|
-
s.and_yield(index)
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
core_indices.each do |index|
|
54
|
-
subject.should_receive(:clear!).with(index)
|
55
|
-
end
|
56
|
-
|
57
|
-
subject.clear_all!
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
describe '.get_subset_for_processing' do
|
62
|
-
let(:mock_redis) do
|
63
|
-
Resque.redis = mr = MockRedis.new
|
64
|
-
subject.add 'foo_core', 42
|
65
|
-
subject.add 'foo_core', 52
|
66
|
-
subject.add 'foo_core', 100
|
67
|
-
mr
|
68
|
-
end
|
69
|
-
|
70
|
-
before :each do
|
71
|
-
Resque.redis = mock_redis.clone
|
72
|
-
end
|
73
|
-
|
74
|
-
it 'should move all members from the flag as deleted set to the processing set' do
|
75
|
-
subject.get_subset_for_processing('foo_core')
|
76
|
-
|
77
|
-
Resque.redis.scard(subject.set_name('foo_core')).should eql(0)
|
78
|
-
Resque.redis.scard(subject.processing_name('foo_core')).should eql(3)
|
79
|
-
end
|
80
|
-
|
81
|
-
it 'should remove the temp set' do
|
82
|
-
subject.get_subset_for_processing('foo_core')
|
83
|
-
|
84
|
-
Resque.redis.scard(subject.temp_name('foo_core')).should eql(0)
|
85
|
-
end
|
86
|
-
|
87
|
-
it 'should preserve existing members of the processing set' do
|
88
|
-
Resque.redis.sadd(subject.processing_name('foo_core'), 1)
|
89
|
-
|
90
|
-
subject.get_subset_for_processing('foo_core')
|
91
|
-
|
92
|
-
Resque.redis.smembers(subject.processing_name('foo_core')).should =~ %w[1 42 52 100]
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
describe '.processing_members' do
|
97
|
-
let(:document_ids) { %w[1, 2, 3] }
|
98
|
-
|
99
|
-
before :each do
|
100
|
-
Resque.stub_chain(:redis, :smembers => document_ids)
|
101
|
-
end
|
102
|
-
|
103
|
-
it 'should get the members of the correct set' do
|
104
|
-
Resque.redis.should_receive(:smembers).once.with(subject.processing_name('foo_core'))
|
105
|
-
subject.processing_members('foo_core')
|
106
|
-
end
|
107
|
-
|
108
|
-
it 'should return a list of integers' do
|
109
|
-
subject.processing_members('foo_core').each do |id|
|
110
|
-
id.class.should == Fixnum
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
describe '.clear_processing' do
|
116
|
-
before :each do
|
117
|
-
Resque.stub_chain(:redis, :del)
|
118
|
-
end
|
119
|
-
|
120
|
-
it 'should delete the processing set' do
|
121
|
-
Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
|
122
|
-
|
123
|
-
subject.clear_processing('foo_core')
|
124
|
-
end
|
125
|
-
end
|
126
|
-
end
|
@@ -1,67 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe ThinkingSphinx::Deltas::ResqueDelta::IndexUtils do
|
4
|
-
let(:indices) { %w[foo_core foo_delta foo bar_core bar_delta bar] }
|
5
|
-
let(:config) { double('config') }
|
6
|
-
|
7
|
-
before :each do
|
8
|
-
ThinkingSphinx::Configuration.stub(:instance => config)
|
9
|
-
config.stub(:generate)
|
10
|
-
config.stub_chain(:configuration, :indices, :collect => indices)
|
11
|
-
|
12
|
-
subject.reload!
|
13
|
-
end
|
14
|
-
|
15
|
-
describe '.index_prefixes' do
|
16
|
-
it 'should use a cached value if one exists' do
|
17
|
-
indices = []
|
18
|
-
subject.instance_variable_set(:@prefixes, indices)
|
19
|
-
|
20
|
-
subject.index_prefixes.should be(indices)
|
21
|
-
end
|
22
|
-
|
23
|
-
it 'should return a list of only index prefixes' do
|
24
|
-
subject.index_prefixes.should =~ %w[foo bar]
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
describe '.core_indices' do
|
29
|
-
it 'should use a cached value if one exists' do
|
30
|
-
indices = []
|
31
|
-
subject.instance_variable_set(:@core_indices, indices)
|
32
|
-
|
33
|
-
subject.core_indices.should be(indices)
|
34
|
-
end
|
35
|
-
|
36
|
-
it 'should return a list of only core indices' do
|
37
|
-
subject.core_indices.should =~ %w[foo_core bar_core]
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
describe '.delta_indices' do
|
42
|
-
it 'should use a cached value if one exists' do
|
43
|
-
indices = []
|
44
|
-
subject.instance_variable_set(:@delta_indices, indices)
|
45
|
-
|
46
|
-
subject.delta_indices.should be(indices)
|
47
|
-
end
|
48
|
-
|
49
|
-
it 'should return a list of only delta indices' do
|
50
|
-
subject.delta_indices.should =~ %w[foo_delta bar_delta]
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
describe '.ts_config' do
|
55
|
-
it 'should use a cached value if one exists' do
|
56
|
-
subject.instance_variable_set(:@ts_config, config)
|
57
|
-
|
58
|
-
subject.ts_config.should be(config)
|
59
|
-
end
|
60
|
-
|
61
|
-
it 'should generate the config when fetching the Configuration instance' do
|
62
|
-
config.should_receive(:generate)
|
63
|
-
|
64
|
-
subject.ts_config
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|