ts-resque-delta 1.1.5 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +24 -0
- data/Guardfile +17 -0
- data/README.markdown +1 -0
- data/Rakefile +16 -4
- data/cucumber.yml +2 -0
- data/features/smart_indexing.feature +43 -0
- data/features/step_definitions/common_steps.rb +16 -3
- data/features/step_definitions/resque_delta_steps.rb +1 -1
- data/features/step_definitions/smart_indexing_steps.rb +3 -0
- data/features/support/env.rb +3 -4
- data/lib/thinking_sphinx/deltas/resque_delta.rb +32 -10
- data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +101 -0
- data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +72 -10
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +56 -0
- data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +47 -0
- data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +4 -46
- data/lib/thinking_sphinx/deltas/resque_delta/version.rb +1 -1
- data/spec/spec_helper.rb +9 -5
- data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +210 -0
- data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +138 -35
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +126 -0
- data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +67 -0
- data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +126 -53
- data/ts-resque-delta.gemspec +8 -2
- metadata +185 -180
- data/features/support/redis_test_setup.rb +0 -23
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +0 -30
- data/spec/spec.opts +0 -1
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job_spec.rb +0 -66
- data/tasks/testing.rb +0 -20
@@ -0,0 +1,126 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet do
|
4
|
+
describe '.add' do
|
5
|
+
before :each do
|
6
|
+
Resque.stub_chain(:redis, :sadd => true)
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should add the document id to the correct set' do
|
10
|
+
Resque.redis.should_receive(:sadd).once.with(subject.set_name('foo_core'), 42)
|
11
|
+
subject.add('foo_core', 42)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
describe '.clear!' do
|
16
|
+
before :each do
|
17
|
+
Resque.stub_chain(:redis, :del)
|
18
|
+
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock)
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'should delete all items in the set' do
|
22
|
+
Resque.redis.should_receive(:del).once.with(subject.set_name('foo_core'))
|
23
|
+
subject.clear!('foo_core')
|
24
|
+
end
|
25
|
+
|
26
|
+
context "with DeltaJob integration" do
|
27
|
+
before :each do
|
28
|
+
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock).and_yield
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'should acquire the DeltaJob lock' do
|
32
|
+
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:around_perform_lock).once.with('foo_delta')
|
33
|
+
subject.clear!('foo_core')
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should delete all items in the processing set' do
|
37
|
+
Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
|
38
|
+
subject.clear!('foo_core')
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
describe '.clear_all!' do
|
44
|
+
let(:core_indices) { %w[foo_core bar_core] }
|
45
|
+
|
46
|
+
it 'should clear each index' do
|
47
|
+
ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.stub_chain(:core_indices, :each).tap do |s|
|
48
|
+
core_indices.inject(s) do |s, index|
|
49
|
+
s.and_yield(index)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
core_indices.each do |index|
|
54
|
+
subject.should_receive(:clear!).with(index)
|
55
|
+
end
|
56
|
+
|
57
|
+
subject.clear_all!
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
describe '.get_subset_for_processing' do
|
62
|
+
let(:mock_redis) do
|
63
|
+
Resque.redis = mr = MockRedis.new
|
64
|
+
subject.add 'foo_core', 42
|
65
|
+
subject.add 'foo_core', 52
|
66
|
+
subject.add 'foo_core', 100
|
67
|
+
mr
|
68
|
+
end
|
69
|
+
|
70
|
+
before :each do
|
71
|
+
Resque.redis = mock_redis.clone
|
72
|
+
end
|
73
|
+
|
74
|
+
it 'should move all members from the flag as deleted set to the processing set' do
|
75
|
+
subject.get_subset_for_processing('foo_core')
|
76
|
+
|
77
|
+
Resque.redis.scard(subject.set_name('foo_core')).should eql(0)
|
78
|
+
Resque.redis.scard(subject.processing_name('foo_core')).should eql(3)
|
79
|
+
end
|
80
|
+
|
81
|
+
it 'should remove the temp set' do
|
82
|
+
subject.get_subset_for_processing('foo_core')
|
83
|
+
|
84
|
+
Resque.redis.scard(subject.temp_name('foo_core')).should eql(0)
|
85
|
+
end
|
86
|
+
|
87
|
+
it 'should preserve existing members of the processing set' do
|
88
|
+
Resque.redis.sadd(subject.processing_name('foo_core'), 1)
|
89
|
+
|
90
|
+
subject.get_subset_for_processing('foo_core')
|
91
|
+
|
92
|
+
Resque.redis.smembers(subject.processing_name('foo_core')).should =~ %w[1 42 52 100]
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
describe '.processing_members' do
|
97
|
+
let(:document_ids) { %w[1, 2, 3] }
|
98
|
+
|
99
|
+
before :each do
|
100
|
+
Resque.stub_chain(:redis, :smembers => document_ids)
|
101
|
+
end
|
102
|
+
|
103
|
+
it 'should get the members of the correct set' do
|
104
|
+
Resque.redis.should_receive(:smembers).once.with(subject.processing_name('foo_core'))
|
105
|
+
subject.processing_members('foo_core')
|
106
|
+
end
|
107
|
+
|
108
|
+
it 'should return a list of integers' do
|
109
|
+
subject.processing_members('foo_core').each do |id|
|
110
|
+
id.class.should == Fixnum
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
describe '.clear_processing' do
|
116
|
+
before :each do
|
117
|
+
Resque.stub_chain(:redis, :del)
|
118
|
+
end
|
119
|
+
|
120
|
+
it 'should delete the processing set' do
|
121
|
+
Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
|
122
|
+
|
123
|
+
subject.clear_processing('foo_core')
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe ThinkingSphinx::Deltas::ResqueDelta::IndexUtils do
|
4
|
+
let(:indices) { %w[foo_core foo_delta foo bar_core bar_delta bar] }
|
5
|
+
let(:config) { double('config') }
|
6
|
+
|
7
|
+
before :each do
|
8
|
+
ThinkingSphinx::Configuration.stub(:instance => config)
|
9
|
+
config.stub(:generate)
|
10
|
+
config.stub_chain(:configuration, :indices, :collect => indices)
|
11
|
+
|
12
|
+
subject.reload!
|
13
|
+
end
|
14
|
+
|
15
|
+
describe '.index_prefixes' do
|
16
|
+
it 'should use a cached value if one exists' do
|
17
|
+
indices = []
|
18
|
+
subject.instance_variable_set(:@prefixes, indices)
|
19
|
+
|
20
|
+
subject.index_prefixes.should be(indices)
|
21
|
+
end
|
22
|
+
|
23
|
+
it 'should return a list of only index prefixes' do
|
24
|
+
subject.index_prefixes.should =~ %w[foo bar]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
describe '.core_indices' do
|
29
|
+
it 'should use a cached value if one exists' do
|
30
|
+
indices = []
|
31
|
+
subject.instance_variable_set(:@core_indices, indices)
|
32
|
+
|
33
|
+
subject.core_indices.should be(indices)
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should return a list of only core indices' do
|
37
|
+
subject.core_indices.should =~ %w[foo_core bar_core]
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
describe '.delta_indices' do
|
42
|
+
it 'should use a cached value if one exists' do
|
43
|
+
indices = []
|
44
|
+
subject.instance_variable_set(:@delta_indices, indices)
|
45
|
+
|
46
|
+
subject.delta_indices.should be(indices)
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'should return a list of only delta indices' do
|
50
|
+
subject.delta_indices.should =~ %w[foo_delta bar_delta]
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe '.ts_config' do
|
55
|
+
it 'should use a cached value if one exists' do
|
56
|
+
subject.instance_variable_set(:@ts_config, config)
|
57
|
+
|
58
|
+
subject.ts_config.should be(config)
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should generate the config when fetching the Configuration instance' do
|
62
|
+
config.should_receive(:generate)
|
63
|
+
|
64
|
+
subject.ts_config
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -1,30 +1,46 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe ThinkingSphinx::Deltas::ResqueDelta do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
ThinkingSphinx.deltas_enabled = true
|
4
|
+
before :each do
|
5
|
+
ThinkingSphinx.updates_enabled = true
|
6
|
+
ThinkingSphinx.deltas_enabled = true
|
8
7
|
|
9
|
-
|
8
|
+
Resque.redis = MockRedis.new
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#index' do
|
12
|
+
def flag_as_deleted_document_in_set?
|
13
|
+
Resque.redis.sismember(ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.set_name('foo_core'), 42)
|
14
|
+
end
|
10
15
|
|
11
|
-
|
16
|
+
subject do
|
17
|
+
ThinkingSphinx::Deltas::ResqueDelta.new(
|
12
18
|
stub('instance'), {}
|
13
|
-
)
|
14
|
-
|
19
|
+
).tap do |s|
|
20
|
+
s.stub(:toggled).and_return(true)
|
21
|
+
s.stub(:lock)
|
22
|
+
s.stub(:unlock)
|
23
|
+
s.stub(:locked?).and_return(false)
|
24
|
+
end
|
25
|
+
end
|
15
26
|
|
16
|
-
|
17
|
-
|
18
|
-
|
27
|
+
let(:model) do
|
28
|
+
stub('foo').tap do |m|
|
29
|
+
m.stub(:name => 'foo')
|
30
|
+
m.stub(:source_of_sphinx_index => m)
|
31
|
+
m.stub(:core_index_names => ['foo_core'])
|
32
|
+
m.stub(:delta_index_names => ['foo_delta'])
|
33
|
+
end
|
34
|
+
end
|
19
35
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
36
|
+
let(:instance) do
|
37
|
+
stub('instance').tap do |i|
|
38
|
+
i.stub(:sphinx_document_id => 42)
|
39
|
+
end
|
40
|
+
end
|
25
41
|
|
26
|
-
|
27
|
-
|
42
|
+
before :each do
|
43
|
+
Resque.stub(:enqueue => true)
|
28
44
|
end
|
29
45
|
|
30
46
|
context 'updates disabled' do
|
@@ -34,12 +50,12 @@ describe ThinkingSphinx::Deltas::ResqueDelta do
|
|
34
50
|
|
35
51
|
it "should not enqueue a delta job" do
|
36
52
|
Resque.should_not_receive(:enqueue)
|
37
|
-
|
53
|
+
subject.index(model)
|
38
54
|
end
|
39
55
|
|
40
|
-
it "should not
|
41
|
-
|
42
|
-
|
56
|
+
it "should not add a flag as deleted document to the set" do
|
57
|
+
subject.index(model, instance)
|
58
|
+
flag_as_deleted_document_in_set?.should be_false
|
43
59
|
end
|
44
60
|
end
|
45
61
|
|
@@ -50,46 +66,42 @@ describe ThinkingSphinx::Deltas::ResqueDelta do
|
|
50
66
|
|
51
67
|
it "should not enqueue a delta job" do
|
52
68
|
Resque.should_not_receive(:enqueue)
|
53
|
-
|
69
|
+
subject.index(model)
|
54
70
|
end
|
55
71
|
|
56
|
-
it "should not
|
57
|
-
|
58
|
-
|
72
|
+
it "should not add a flag as deleted document to the set" do
|
73
|
+
subject.index(model, instance)
|
74
|
+
flag_as_deleted_document_in_set?.should be_false
|
59
75
|
end
|
60
76
|
end
|
61
77
|
|
62
78
|
context "instance isn't toggled" do
|
63
79
|
before :each do
|
64
|
-
|
80
|
+
subject.stub(:toggled => false)
|
65
81
|
end
|
66
82
|
|
67
83
|
it "should not enqueue a delta job" do
|
68
84
|
Resque.should_not_receive(:enqueue)
|
69
|
-
|
85
|
+
subject.index(model, instance)
|
70
86
|
end
|
71
87
|
|
72
|
-
it "should not
|
73
|
-
|
74
|
-
|
88
|
+
it "should not add a flag as deleted document to the set" do
|
89
|
+
subject.index(model, instance)
|
90
|
+
flag_as_deleted_document_in_set?.should be_false
|
75
91
|
end
|
76
92
|
end
|
77
93
|
|
78
94
|
it "should enqueue a delta job" do
|
79
|
-
Resque.should_receive(:enqueue).
|
95
|
+
Resque.should_receive(:enqueue).once.with(
|
80
96
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
|
81
|
-
|
97
|
+
'foo_delta'
|
82
98
|
)
|
83
|
-
|
99
|
+
subject.index(model)
|
84
100
|
end
|
85
101
|
|
86
|
-
it "should
|
87
|
-
|
88
|
-
|
89
|
-
['foo_core'],
|
90
|
-
42
|
91
|
-
)
|
92
|
-
@delayed_delta.index(@model, @instance)
|
102
|
+
it "should add the flag as deleted document id to the set" do
|
103
|
+
subject.index(model, instance)
|
104
|
+
flag_as_deleted_document_in_set?.should be_true
|
93
105
|
end
|
94
106
|
|
95
107
|
context "delta index is locked" do
|
@@ -98,21 +110,82 @@ describe ThinkingSphinx::Deltas::ResqueDelta do
|
|
98
110
|
end
|
99
111
|
|
100
112
|
it "should not enqueue a delta job" do
|
101
|
-
Resque.should_not_receive(:enqueue)
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
113
|
+
Resque.should_not_receive(:enqueue)
|
114
|
+
subject.index(model, instance)
|
115
|
+
end
|
116
|
+
|
117
|
+
it "should add the flag as deleted document id to the set" do
|
118
|
+
subject.index(model, instance)
|
119
|
+
flag_as_deleted_document_in_set?.should be_true
|
106
120
|
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
describe '.clear_thinking_sphinx_queues' do
|
125
|
+
subject { ThinkingSphinx::Deltas::ResqueDelta.clear_thinking_sphinx_queues }
|
107
126
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
['foo_core'],
|
112
|
-
42
|
113
|
-
)
|
114
|
-
@delayed_delta.index(@model, @instance)
|
127
|
+
before :all do
|
128
|
+
class RandomJob
|
129
|
+
@queue = 'ts_delta'
|
115
130
|
end
|
116
131
|
end
|
132
|
+
|
133
|
+
before :each do
|
134
|
+
Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'foo_delta')
|
135
|
+
Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'bar_delta')
|
136
|
+
Resque.enqueue(RandomJob, '1234')
|
137
|
+
end
|
138
|
+
|
139
|
+
it 'should remove all jobs' do
|
140
|
+
subject
|
141
|
+
Resque.size('ts_delta').should eq(0)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
describe '.lock' do
|
146
|
+
it 'should set the lock key in redis' do
|
147
|
+
ThinkingSphinx::Deltas::ResqueDelta.lock('foo')
|
148
|
+
Resque.redis.get("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked").should eql('true')
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
describe '.unlock' do
|
153
|
+
it 'should unset the lock key in redis' do
|
154
|
+
Resque.redis.set("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked", 'true')
|
155
|
+
ThinkingSphinx::Deltas::ResqueDelta.unlock('foo')
|
156
|
+
Resque.redis.get("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked").should be_nil
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
describe '.locked?' do
|
161
|
+
subject { ThinkingSphinx::Deltas::ResqueDelta.locked?('foo') }
|
162
|
+
|
163
|
+
context "when lock key in redis is true" do
|
164
|
+
before { Resque.redis.set("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked", 'true') }
|
165
|
+
it { should be_true }
|
166
|
+
end
|
167
|
+
|
168
|
+
context "when lock key in redis is nil" do
|
169
|
+
it { should be_false }
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
describe '.prepare_for_core_index' do
|
174
|
+
subject { ThinkingSphinx::Deltas::ResqueDelta.prepare_for_core_index('foo') }
|
175
|
+
|
176
|
+
before :each do
|
177
|
+
Resque.stub(:dequeue)
|
178
|
+
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:clear!)
|
179
|
+
end
|
180
|
+
|
181
|
+
it "should call FlagAsDeletedSet.clear!" do
|
182
|
+
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:clear!).with('foo_core')
|
183
|
+
subject
|
184
|
+
end
|
185
|
+
|
186
|
+
it "should clear delta jobs" do
|
187
|
+
Resque.should_receive(:dequeue).with(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'foo_delta')
|
188
|
+
subject
|
189
|
+
end
|
117
190
|
end
|
118
191
|
end
|
data/ts-resque-delta.gemspec
CHANGED
@@ -24,11 +24,17 @@ Gem::Specification.new do |s|
|
|
24
24
|
s.add_dependency "resque", "~> 1.10"
|
25
25
|
s.add_dependency "resque-lock-timeout", "~> 0.3.1"
|
26
26
|
|
27
|
-
s.add_development_dependency "rspec", "~>
|
27
|
+
s.add_development_dependency "rspec", "~> 2.7.0"
|
28
28
|
s.add_development_dependency "cucumber", ">= 0"
|
29
29
|
s.add_development_dependency "database_cleaner", ">= 0.5.2"
|
30
30
|
s.add_development_dependency "mysql2", "~> 0.2.7"
|
31
|
-
s.add_development_dependency "rake", "0.8.7"
|
31
|
+
s.add_development_dependency "rake", ">= 0.8.7"
|
32
32
|
s.add_development_dependency "activerecord", "~> 2.3.11"
|
33
33
|
s.add_development_dependency "flying-sphinx", ">= 0.5.1"
|
34
|
+
s.add_development_dependency "ryansch-mock_redis", "~> 0.3.0"
|
35
|
+
s.add_development_dependency "guard", "~> 0.8.8"
|
36
|
+
s.add_development_dependency "guard-rspec", "~> 0.5.8"
|
37
|
+
s.add_development_dependency "guard-bundler", "~> 0.1.3"
|
38
|
+
s.add_development_dependency "guard-cucumber", "~> 0.7.4"
|
39
|
+
s.add_development_dependency "fakefs", "~> 0.4.0"
|
34
40
|
end
|