ts-resque-delta 1.1.5 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. data/Gemfile +24 -0
  2. data/Guardfile +17 -0
  3. data/README.markdown +1 -0
  4. data/Rakefile +16 -4
  5. data/cucumber.yml +2 -0
  6. data/features/smart_indexing.feature +43 -0
  7. data/features/step_definitions/common_steps.rb +16 -3
  8. data/features/step_definitions/resque_delta_steps.rb +1 -1
  9. data/features/step_definitions/smart_indexing_steps.rb +3 -0
  10. data/features/support/env.rb +3 -4
  11. data/lib/thinking_sphinx/deltas/resque_delta.rb +32 -10
  12. data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +101 -0
  13. data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +72 -10
  14. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +56 -0
  15. data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +47 -0
  16. data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +4 -46
  17. data/lib/thinking_sphinx/deltas/resque_delta/version.rb +1 -1
  18. data/spec/spec_helper.rb +9 -5
  19. data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +210 -0
  20. data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +138 -35
  21. data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +126 -0
  22. data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +67 -0
  23. data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +126 -53
  24. data/ts-resque-delta.gemspec +8 -2
  25. metadata +185 -180
  26. data/features/support/redis_test_setup.rb +0 -23
  27. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +0 -30
  28. data/spec/spec.opts +0 -1
  29. data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job_spec.rb +0 -66
  30. data/tasks/testing.rb +0 -20
@@ -0,0 +1,126 @@
1
+ require 'spec_helper'
2
+
3
+ describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet do
4
+ describe '.add' do
5
+ before :each do
6
+ Resque.stub_chain(:redis, :sadd => true)
7
+ end
8
+
9
+ it 'should add the document id to the correct set' do
10
+ Resque.redis.should_receive(:sadd).once.with(subject.set_name('foo_core'), 42)
11
+ subject.add('foo_core', 42)
12
+ end
13
+ end
14
+
15
+ describe '.clear!' do
16
+ before :each do
17
+ Resque.stub_chain(:redis, :del)
18
+ ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock)
19
+ end
20
+
21
+ it 'should delete all items in the set' do
22
+ Resque.redis.should_receive(:del).once.with(subject.set_name('foo_core'))
23
+ subject.clear!('foo_core')
24
+ end
25
+
26
+ context "with DeltaJob integration" do
27
+ before :each do
28
+ ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock).and_yield
29
+ end
30
+
31
+ it 'should acquire the DeltaJob lock' do
32
+ ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:around_perform_lock).once.with('foo_delta')
33
+ subject.clear!('foo_core')
34
+ end
35
+
36
+ it 'should delete all items in the processing set' do
37
+ Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
38
+ subject.clear!('foo_core')
39
+ end
40
+ end
41
+ end
42
+
43
+ describe '.clear_all!' do
44
+ let(:core_indices) { %w[foo_core bar_core] }
45
+
46
+ it 'should clear each index' do
47
+ ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.stub_chain(:core_indices, :each).tap do |s|
48
+ core_indices.inject(s) do |s, index|
49
+ s.and_yield(index)
50
+ end
51
+ end
52
+
53
+ core_indices.each do |index|
54
+ subject.should_receive(:clear!).with(index)
55
+ end
56
+
57
+ subject.clear_all!
58
+ end
59
+ end
60
+
61
+ describe '.get_subset_for_processing' do
62
+ let(:mock_redis) do
63
+ Resque.redis = mr = MockRedis.new
64
+ subject.add 'foo_core', 42
65
+ subject.add 'foo_core', 52
66
+ subject.add 'foo_core', 100
67
+ mr
68
+ end
69
+
70
+ before :each do
71
+ Resque.redis = mock_redis.clone
72
+ end
73
+
74
+ it 'should move all members from the flag as deleted set to the processing set' do
75
+ subject.get_subset_for_processing('foo_core')
76
+
77
+ Resque.redis.scard(subject.set_name('foo_core')).should eql(0)
78
+ Resque.redis.scard(subject.processing_name('foo_core')).should eql(3)
79
+ end
80
+
81
+ it 'should remove the temp set' do
82
+ subject.get_subset_for_processing('foo_core')
83
+
84
+ Resque.redis.scard(subject.temp_name('foo_core')).should eql(0)
85
+ end
86
+
87
+ it 'should preserve existing members of the processing set' do
88
+ Resque.redis.sadd(subject.processing_name('foo_core'), 1)
89
+
90
+ subject.get_subset_for_processing('foo_core')
91
+
92
+ Resque.redis.smembers(subject.processing_name('foo_core')).should =~ %w[1 42 52 100]
93
+ end
94
+ end
95
+
96
+ describe '.processing_members' do
97
+ let(:document_ids) { %w[1, 2, 3] }
98
+
99
+ before :each do
100
+ Resque.stub_chain(:redis, :smembers => document_ids)
101
+ end
102
+
103
+ it 'should get the members of the correct set' do
104
+ Resque.redis.should_receive(:smembers).once.with(subject.processing_name('foo_core'))
105
+ subject.processing_members('foo_core')
106
+ end
107
+
108
+ it 'should return a list of integers' do
109
+ subject.processing_members('foo_core').each do |id|
110
+ id.class.should == Fixnum
111
+ end
112
+ end
113
+ end
114
+
115
+ describe '.clear_processing' do
116
+ before :each do
117
+ Resque.stub_chain(:redis, :del)
118
+ end
119
+
120
+ it 'should delete the processing set' do
121
+ Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
122
+
123
+ subject.clear_processing('foo_core')
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,67 @@
1
+ require 'spec_helper'
2
+
3
+ describe ThinkingSphinx::Deltas::ResqueDelta::IndexUtils do
4
+ let(:indices) { %w[foo_core foo_delta foo bar_core bar_delta bar] }
5
+ let(:config) { double('config') }
6
+
7
+ before :each do
8
+ ThinkingSphinx::Configuration.stub(:instance => config)
9
+ config.stub(:generate)
10
+ config.stub_chain(:configuration, :indices, :collect => indices)
11
+
12
+ subject.reload!
13
+ end
14
+
15
+ describe '.index_prefixes' do
16
+ it 'should use a cached value if one exists' do
17
+ indices = []
18
+ subject.instance_variable_set(:@prefixes, indices)
19
+
20
+ subject.index_prefixes.should be(indices)
21
+ end
22
+
23
+ it 'should return a list of only index prefixes' do
24
+ subject.index_prefixes.should =~ %w[foo bar]
25
+ end
26
+ end
27
+
28
+ describe '.core_indices' do
29
+ it 'should use a cached value if one exists' do
30
+ indices = []
31
+ subject.instance_variable_set(:@core_indices, indices)
32
+
33
+ subject.core_indices.should be(indices)
34
+ end
35
+
36
+ it 'should return a list of only core indices' do
37
+ subject.core_indices.should =~ %w[foo_core bar_core]
38
+ end
39
+ end
40
+
41
+ describe '.delta_indices' do
42
+ it 'should use a cached value if one exists' do
43
+ indices = []
44
+ subject.instance_variable_set(:@delta_indices, indices)
45
+
46
+ subject.delta_indices.should be(indices)
47
+ end
48
+
49
+ it 'should return a list of only delta indices' do
50
+ subject.delta_indices.should =~ %w[foo_delta bar_delta]
51
+ end
52
+ end
53
+
54
+ describe '.ts_config' do
55
+ it 'should use a cached value if one exists' do
56
+ subject.instance_variable_set(:@ts_config, config)
57
+
58
+ subject.ts_config.should be(config)
59
+ end
60
+
61
+ it 'should generate the config when fetching the Configuration instance' do
62
+ config.should_receive(:generate)
63
+
64
+ subject.ts_config
65
+ end
66
+ end
67
+ end
@@ -1,30 +1,46 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe ThinkingSphinx::Deltas::ResqueDelta do
4
- describe '#index' do
5
- before :each do
6
- ThinkingSphinx.updates_enabled = true
7
- ThinkingSphinx.deltas_enabled = true
4
+ before :each do
5
+ ThinkingSphinx.updates_enabled = true
6
+ ThinkingSphinx.deltas_enabled = true
8
7
 
9
- Resque.stub(:enqueue => true)
8
+ Resque.redis = MockRedis.new
9
+ end
10
+
11
+ describe '#index' do
12
+ def flag_as_deleted_document_in_set?
13
+ Resque.redis.sismember(ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.set_name('foo_core'), 42)
14
+ end
10
15
 
11
- @delayed_delta = ThinkingSphinx::Deltas::ResqueDelta.new(
16
+ subject do
17
+ ThinkingSphinx::Deltas::ResqueDelta.new(
12
18
  stub('instance'), {}
13
- )
14
- @delayed_delta.stub(:toggled).and_return(true)
19
+ ).tap do |s|
20
+ s.stub(:toggled).and_return(true)
21
+ s.stub(:lock)
22
+ s.stub(:unlock)
23
+ s.stub(:locked?).and_return(false)
24
+ end
25
+ end
15
26
 
16
- ThinkingSphinx::Deltas::ResqueDelta.stub(:lock)
17
- ThinkingSphinx::Deltas::ResqueDelta.stub(:unlock)
18
- ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?).and_return(false)
27
+ let(:model) do
28
+ stub('foo').tap do |m|
29
+ m.stub(:name => 'foo')
30
+ m.stub(:source_of_sphinx_index => m)
31
+ m.stub(:core_index_names => ['foo_core'])
32
+ m.stub(:delta_index_names => ['foo_delta'])
33
+ end
34
+ end
19
35
 
20
- @model = stub('foo')
21
- @model.stub(:name => 'foo')
22
- @model.stub(:source_of_sphinx_index => @model)
23
- @model.stub(:core_index_names => ['foo_core'])
24
- @model.stub(:delta_index_names => ['foo_delta'])
36
+ let(:instance) do
37
+ stub('instance').tap do |i|
38
+ i.stub(:sphinx_document_id => 42)
39
+ end
40
+ end
25
41
 
26
- @instance = stub('instance')
27
- @instance.stub(:sphinx_document_id => 42)
42
+ before :each do
43
+ Resque.stub(:enqueue => true)
28
44
  end
29
45
 
30
46
  context 'updates disabled' do
@@ -34,12 +50,12 @@ describe ThinkingSphinx::Deltas::ResqueDelta do
34
50
 
35
51
  it "should not enqueue a delta job" do
36
52
  Resque.should_not_receive(:enqueue)
37
- @delayed_delta.index(@model)
53
+ subject.index(model)
38
54
  end
39
55
 
40
- it "should not enqueue a flag as deleted job" do
41
- Resque.should_not_receive(:enqueue)
42
- @delayed_delta.index(@model)
56
+ it "should not add a flag as deleted document to the set" do
57
+ subject.index(model, instance)
58
+ flag_as_deleted_document_in_set?.should be_false
43
59
  end
44
60
  end
45
61
 
@@ -50,46 +66,42 @@ describe ThinkingSphinx::Deltas::ResqueDelta do
50
66
 
51
67
  it "should not enqueue a delta job" do
52
68
  Resque.should_not_receive(:enqueue)
53
- @delayed_delta.index(@model)
69
+ subject.index(model)
54
70
  end
55
71
 
56
- it "should not enqueue a flag as deleted job" do
57
- Resque.should_not_receive(:enqueue)
58
- @delayed_delta.index(@model)
72
+ it "should not add a flag as deleted document to the set" do
73
+ subject.index(model, instance)
74
+ flag_as_deleted_document_in_set?.should be_false
59
75
  end
60
76
  end
61
77
 
62
78
  context "instance isn't toggled" do
63
79
  before :each do
64
- @delayed_delta.stub(:toggled => false)
80
+ subject.stub(:toggled => false)
65
81
  end
66
82
 
67
83
  it "should not enqueue a delta job" do
68
84
  Resque.should_not_receive(:enqueue)
69
- @delayed_delta.index(@model, @instance)
85
+ subject.index(model, instance)
70
86
  end
71
87
 
72
- it "should not enqueue a flag as deleted job" do
73
- Resque.should_not_receive(:enqueue)
74
- @delayed_delta.index(@model, @instance)
88
+ it "should not add a flag as deleted document to the set" do
89
+ subject.index(model, instance)
90
+ flag_as_deleted_document_in_set?.should be_false
75
91
  end
76
92
  end
77
93
 
78
94
  it "should enqueue a delta job" do
79
- Resque.should_receive(:enqueue).at_least(:once).with(
95
+ Resque.should_receive(:enqueue).once.with(
80
96
  ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
81
- ['foo_delta']
97
+ 'foo_delta'
82
98
  )
83
- @delayed_delta.index(@model)
99
+ subject.index(model)
84
100
  end
85
101
 
86
- it "should enqueue a flag-as-deleted job" do
87
- Resque.should_receive(:enqueue).at_least(:once).with(
88
- ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob,
89
- ['foo_core'],
90
- 42
91
- )
92
- @delayed_delta.index(@model, @instance)
102
+ it "should add the flag as deleted document id to the set" do
103
+ subject.index(model, instance)
104
+ flag_as_deleted_document_in_set?.should be_true
93
105
  end
94
106
 
95
107
  context "delta index is locked" do
@@ -98,21 +110,82 @@ describe ThinkingSphinx::Deltas::ResqueDelta do
98
110
  end
99
111
 
100
112
  it "should not enqueue a delta job" do
101
- Resque.should_not_receive(:enqueue).with(
102
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
103
- ['foo_delta']
104
- )
105
- @delayed_delta.index(@model, @instance)
113
+ Resque.should_not_receive(:enqueue)
114
+ subject.index(model, instance)
115
+ end
116
+
117
+ it "should add the flag as deleted document id to the set" do
118
+ subject.index(model, instance)
119
+ flag_as_deleted_document_in_set?.should be_true
106
120
  end
121
+ end
122
+ end
123
+
124
+ describe '.clear_thinking_sphinx_queues' do
125
+ subject { ThinkingSphinx::Deltas::ResqueDelta.clear_thinking_sphinx_queues }
107
126
 
108
- it "should enqueue a flag-as-deleted job" do
109
- Resque.should_receive(:enqueue).at_least(:once).with(
110
- ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob,
111
- ['foo_core'],
112
- 42
113
- )
114
- @delayed_delta.index(@model, @instance)
127
+ before :all do
128
+ class RandomJob
129
+ @queue = 'ts_delta'
115
130
  end
116
131
  end
132
+
133
+ before :each do
134
+ Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'foo_delta')
135
+ Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'bar_delta')
136
+ Resque.enqueue(RandomJob, '1234')
137
+ end
138
+
139
+ it 'should remove all jobs' do
140
+ subject
141
+ Resque.size('ts_delta').should eq(0)
142
+ end
143
+ end
144
+
145
+ describe '.lock' do
146
+ it 'should set the lock key in redis' do
147
+ ThinkingSphinx::Deltas::ResqueDelta.lock('foo')
148
+ Resque.redis.get("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked").should eql('true')
149
+ end
150
+ end
151
+
152
+ describe '.unlock' do
153
+ it 'should unset the lock key in redis' do
154
+ Resque.redis.set("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked", 'true')
155
+ ThinkingSphinx::Deltas::ResqueDelta.unlock('foo')
156
+ Resque.redis.get("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked").should be_nil
157
+ end
158
+ end
159
+
160
+ describe '.locked?' do
161
+ subject { ThinkingSphinx::Deltas::ResqueDelta.locked?('foo') }
162
+
163
+ context "when lock key in redis is true" do
164
+ before { Resque.redis.set("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked", 'true') }
165
+ it { should be_true }
166
+ end
167
+
168
+ context "when lock key in redis is nil" do
169
+ it { should be_false }
170
+ end
171
+ end
172
+
173
+ describe '.prepare_for_core_index' do
174
+ subject { ThinkingSphinx::Deltas::ResqueDelta.prepare_for_core_index('foo') }
175
+
176
+ before :each do
177
+ Resque.stub(:dequeue)
178
+ ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:clear!)
179
+ end
180
+
181
+ it "should call FlagAsDeletedSet.clear!" do
182
+ ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:clear!).with('foo_core')
183
+ subject
184
+ end
185
+
186
+ it "should clear delta jobs" do
187
+ Resque.should_receive(:dequeue).with(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'foo_delta')
188
+ subject
189
+ end
117
190
  end
118
191
  end
@@ -24,11 +24,17 @@ Gem::Specification.new do |s|
24
24
  s.add_dependency "resque", "~> 1.10"
25
25
  s.add_dependency "resque-lock-timeout", "~> 0.3.1"
26
26
 
27
- s.add_development_dependency "rspec", "~> 1.0"
27
+ s.add_development_dependency "rspec", "~> 2.7.0"
28
28
  s.add_development_dependency "cucumber", ">= 0"
29
29
  s.add_development_dependency "database_cleaner", ">= 0.5.2"
30
30
  s.add_development_dependency "mysql2", "~> 0.2.7"
31
- s.add_development_dependency "rake", "0.8.7"
31
+ s.add_development_dependency "rake", ">= 0.8.7"
32
32
  s.add_development_dependency "activerecord", "~> 2.3.11"
33
33
  s.add_development_dependency "flying-sphinx", ">= 0.5.1"
34
+ s.add_development_dependency "ryansch-mock_redis", "~> 0.3.0"
35
+ s.add_development_dependency "guard", "~> 0.8.8"
36
+ s.add_development_dependency "guard-rspec", "~> 0.5.8"
37
+ s.add_development_dependency "guard-bundler", "~> 0.1.3"
38
+ s.add_development_dependency "guard-cucumber", "~> 0.7.4"
39
+ s.add_development_dependency "fakefs", "~> 0.4.0"
34
40
  end