ts-resque-delta 1.2.4 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -1
  3. data/.travis.yml +14 -14
  4. data/Appraisals +4 -6
  5. data/Gemfile +1 -1
  6. data/README.markdown +31 -39
  7. data/Rakefile +6 -13
  8. data/gemfiles/binary.gemfile +7 -0
  9. data/gemfiles/sphinxql.gemfile +7 -0
  10. data/lib/thinking_sphinx/deltas/resque_delta.rb +66 -76
  11. data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +1 -113
  12. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +7 -0
  13. data/lib/ts-resque-delta.rb +0 -2
  14. data/spec/acceptance/resque_deltas_spec.rb +50 -0
  15. data/spec/acceptance/spec_helper.rb +4 -0
  16. data/spec/acceptance/support/database_cleaner.rb +11 -0
  17. data/spec/acceptance/support/sphinx_controller.rb +61 -0
  18. data/spec/acceptance/support/sphinx_helpers.rb +36 -0
  19. data/spec/internal/.gitignore +2 -0
  20. data/spec/internal/app/indices/book_index.rb +3 -0
  21. data/spec/internal/app/models/book.rb +7 -0
  22. data/spec/internal/config/database.yml +5 -0
  23. data/spec/internal/db/schema.rb +10 -0
  24. data/spec/internal/log/.gitignore +1 -0
  25. data/spec/spec_helper.rb +16 -12
  26. data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +11 -155
  27. data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +3 -166
  28. data/ts-resque-delta.gemspec +19 -28
  29. metadata +90 -230
  30. data/Guardfile +0 -17
  31. data/features/resque_deltas.feature +0 -62
  32. data/features/smart_indexing.feature +0 -43
  33. data/features/step_definitions/common_steps.rb +0 -76
  34. data/features/step_definitions/resque_delta_steps.rb +0 -33
  35. data/features/step_definitions/smart_indexing_steps.rb +0 -3
  36. data/features/support/env.rb +0 -41
  37. data/features/thinking_sphinx/database.example.yml +0 -4
  38. data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +0 -4
  39. data/features/thinking_sphinx/models/delayed_beta.rb +0 -6
  40. data/gemfiles/activerecord2.gemfile +0 -8
  41. data/gemfiles/activerecord2.gemfile.lock +0 -114
  42. data/gemfiles/activerecord3.gemfile +0 -8
  43. data/gemfiles/activerecord3.gemfile.lock +0 -123
  44. data/lib/flying_sphinx/resque_delta.rb +0 -35
  45. data/lib/flying_sphinx/resque_delta/delta_job.rb +0 -14
  46. data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb +0 -7
  47. data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +0 -116
  48. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +0 -59
  49. data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +0 -47
  50. data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb +0 -8
  51. data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +0 -38
  52. data/lib/thinking_sphinx/deltas/resque_delta/version.rb +0 -7
  53. data/spec/flying_sphinx/resque_delta/delta_job_spec.rb +0 -32
  54. data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb +0 -23
  55. data/spec/flying_sphinx/resque_delta_spec.rb +0 -130
  56. data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +0 -210
  57. data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +0 -126
  58. data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +0 -67
@@ -1,210 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::ResqueDelta::CoreIndex do
4
- let(:indices) { %w[foo bar] }
5
- let(:config) { double('config') }
6
-
7
- describe '#lock_delta' do
8
- it 'should lock the delta' do
9
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock)
10
-
11
- subject.lock_delta('foo')
12
- end
13
-
14
- it 'should lock the delta for the given index' do
15
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock).with('foo_delta')
16
-
17
- subject.lock_delta('foo')
18
- end
19
- end
20
-
21
- describe '#unlock_delta' do
22
- it 'should unlock the delta' do
23
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock)
24
-
25
- subject.unlock_delta('foo')
26
- end
27
-
28
- it 'should unlock the delta for the given index' do
29
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock).with('foo_delta')
30
-
31
- subject.unlock_delta('foo')
32
- end
33
- end
34
-
35
- describe '#lock_deltas' do
36
- it 'should lock all delta indices' do
37
- subject.stub(:sphinx_indices => indices)
38
-
39
- indices.each {|index| subject.should_receive(:lock_delta).once.with(index) }
40
-
41
- subject.lock_deltas
42
- end
43
- end
44
-
45
- describe '#unlock_deltas' do
46
- it 'should unlock all delta indices' do
47
- subject.stub(:sphinx_indices => indices)
48
-
49
- indices.each {|index| subject.should_receive(:unlock_delta).once.with(index) }
50
-
51
- subject.unlock_deltas
52
- end
53
- end
54
-
55
- describe '#with_delta_index_lock' do
56
- before :each do
57
- subject.stub(:lock_delta)
58
- subject.stub(:unlock_delta)
59
-
60
- subject.stub(:block_called)
61
-
62
- @block_called = false
63
- @block = lambda { @block_called = true; subject.block_called }
64
- end
65
-
66
- it 'should yield' do
67
- subject.with_delta_index_lock('foo', &@block)
68
- @block_called.should be_true
69
- end
70
-
71
- it 'should lock before yielding' do
72
- subject.should_receive(:lock_delta).with('foo').ordered
73
- subject.should_receive(:block_called).ordered
74
-
75
- subject.with_delta_index_lock('foo', &@block)
76
- end
77
-
78
- it 'should unlock after yielding' do
79
- subject.should_receive(:block_called).ordered
80
- subject.should_receive(:unlock_delta).with('foo').ordered
81
-
82
- subject.with_delta_index_lock('foo', &@block)
83
- end
84
- end
85
-
86
- describe '#smart_index' do
87
- include FakeFS::SpecHelpers
88
-
89
- let(:test_path) { '/tmp/ts-resque-delta/foo' }
90
-
91
- before :each do
92
- subject.stub(:ts_config => config)
93
- config.stub(:config_file => 'foo_config')
94
- config.stub(:build)
95
- config.stub(:searchd_file_path => test_path)
96
- config.stub_chain(:controller, :index) do
97
- # Set $? to 0
98
- `#{File.join(SPEC_BIN_PATH, 'true.rb')}`
99
- end
100
-
101
- # Silence Generating config message
102
- subject.stub(:puts)
103
-
104
- subject.stub(:index_prefixes => indices)
105
- subject.stub(:lock_delta)
106
- subject.stub(:unlock_delta)
107
-
108
- ThinkingSphinx::Deltas::ResqueDelta.stub(:prepare_for_core_index)
109
- Resque.stub(:enqueue)
110
- end
111
-
112
- it 'should not generate sphinx configuration if INDEX_ONLY is true' do
113
- ENV.stub(:[]).with('INDEX_ONLY').and_return('true')
114
- ENV.stub(:[]).with('SILENT').and_return(nil)
115
- config.should_not_receive(:build)
116
-
117
- subject.smart_index
118
- end
119
-
120
- it 'should generate sphinx configuration if INDEX_ONLY is not true' do
121
- ENV.stub(:[]).with('INDEX_ONLY').and_return(nil)
122
- ENV.stub(:[]).with('SILENT').and_return(nil)
123
- config.should_receive(:build).once
124
-
125
- subject.smart_index
126
- end
127
-
128
- it 'should create the sphinx file directory' do
129
- subject.smart_index
130
-
131
- File.directory?(test_path).should be_true
132
- end
133
-
134
- it 'should index all core indices' do
135
- indices.each do |index|
136
- config.controller.should_receive(:index).with("#{index}_core", anything)
137
- end
138
-
139
- subject.smart_index
140
- end
141
-
142
- context "with delta lock" do
143
- before :each do
144
- subject.stub(:with_delta_index_lock).and_yield
145
- end
146
-
147
- it 'should index' do
148
- config.controller.should_receive(:index)
149
- subject.smart_index
150
- end
151
-
152
- it 'should signal resque delta to prepare for the core index' do
153
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:prepare_for_core_index)
154
- subject.smart_index
155
- end
156
- end
157
-
158
- context "without delta lock" do
159
- before :each do
160
- subject.stub(:with_delta_index_lock)
161
- end
162
-
163
- it 'should not index without the delta lock' do
164
- config.controller.should_not_receive(:index)
165
- subject.smart_index
166
- end
167
-
168
- it 'should not signal resque delta to prepare for the core index' do
169
- ThinkingSphinx::Deltas::ResqueDelta.should_not_receive(:prepare_for_core_index)
170
- subject.smart_index
171
- end
172
- end
173
-
174
- it 'should create a delta job after the delta is unlocked' do
175
- # Create a dummy method on subject that's called when Resque.enqueue is called so we can enforce order.
176
- subject.stub(:resque_called)
177
- Resque.stub(:enqueue) { subject.resque_called }
178
-
179
- Resque.should_receive(:enqueue)
180
-
181
- subject.should_receive(:with_delta_index_lock).ordered.exactly(indices.size)
182
- subject.should_receive(:resque_called).ordered.exactly(indices.size)
183
-
184
- subject.smart_index
185
- end
186
-
187
- context 'with an error' do
188
- before :each do
189
- config.stub_chain(:controller, :index) do
190
- # Set $? to 1
191
- `#{File.join(SPEC_BIN_PATH, 'false.rb')}`
192
- end
193
- end
194
-
195
- it 'should stop processing indexes after an error' do
196
- config.controller.should_receive(:index).once
197
-
198
- subject.smart_index
199
- end
200
-
201
- it 'should return false on failure' do
202
- subject.smart_index.should be_false
203
- end
204
- end
205
-
206
- it 'should return true on success' do
207
- subject.smart_index.should be_true
208
- end
209
- end
210
- end
@@ -1,126 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet do
4
- describe '.add' do
5
- before :each do
6
- Resque.stub_chain(:redis, :sadd => true)
7
- end
8
-
9
- it 'should add the document id to the correct set' do
10
- Resque.redis.should_receive(:sadd).once.with(subject.set_name('foo_core'), 42)
11
- subject.add('foo_core', 42)
12
- end
13
- end
14
-
15
- describe '.clear!' do
16
- before :each do
17
- Resque.stub_chain(:redis, :del)
18
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock)
19
- end
20
-
21
- it 'should delete all items in the set' do
22
- Resque.redis.should_receive(:del).once.with(subject.set_name('foo_core'))
23
- subject.clear!('foo_core')
24
- end
25
-
26
- context "with DeltaJob integration" do
27
- before :each do
28
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock).and_yield
29
- end
30
-
31
- it 'should acquire the DeltaJob lock' do
32
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:around_perform_lock).once.with('foo_delta')
33
- subject.clear!('foo_core')
34
- end
35
-
36
- it 'should delete all items in the processing set' do
37
- Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
38
- subject.clear!('foo_core')
39
- end
40
- end
41
- end
42
-
43
- describe '.clear_all!' do
44
- let(:core_indices) { %w[foo_core bar_core] }
45
-
46
- it 'should clear each index' do
47
- ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.stub_chain(:core_indices, :each).tap do |s|
48
- core_indices.inject(s) do |s, index|
49
- s.and_yield(index)
50
- end
51
- end
52
-
53
- core_indices.each do |index|
54
- subject.should_receive(:clear!).with(index)
55
- end
56
-
57
- subject.clear_all!
58
- end
59
- end
60
-
61
- describe '.get_subset_for_processing' do
62
- let(:mock_redis) do
63
- Resque.redis = mr = MockRedis.new
64
- subject.add 'foo_core', 42
65
- subject.add 'foo_core', 52
66
- subject.add 'foo_core', 100
67
- mr
68
- end
69
-
70
- before :each do
71
- Resque.redis = mock_redis.clone
72
- end
73
-
74
- it 'should move all members from the flag as deleted set to the processing set' do
75
- subject.get_subset_for_processing('foo_core')
76
-
77
- Resque.redis.scard(subject.set_name('foo_core')).should eql(0)
78
- Resque.redis.scard(subject.processing_name('foo_core')).should eql(3)
79
- end
80
-
81
- it 'should remove the temp set' do
82
- subject.get_subset_for_processing('foo_core')
83
-
84
- Resque.redis.scard(subject.temp_name('foo_core')).should eql(0)
85
- end
86
-
87
- it 'should preserve existing members of the processing set' do
88
- Resque.redis.sadd(subject.processing_name('foo_core'), 1)
89
-
90
- subject.get_subset_for_processing('foo_core')
91
-
92
- Resque.redis.smembers(subject.processing_name('foo_core')).should =~ %w[1 42 52 100]
93
- end
94
- end
95
-
96
- describe '.processing_members' do
97
- let(:document_ids) { %w[1, 2, 3] }
98
-
99
- before :each do
100
- Resque.stub_chain(:redis, :smembers => document_ids)
101
- end
102
-
103
- it 'should get the members of the correct set' do
104
- Resque.redis.should_receive(:smembers).once.with(subject.processing_name('foo_core'))
105
- subject.processing_members('foo_core')
106
- end
107
-
108
- it 'should return a list of integers' do
109
- subject.processing_members('foo_core').each do |id|
110
- id.class.should == Fixnum
111
- end
112
- end
113
- end
114
-
115
- describe '.clear_processing' do
116
- before :each do
117
- Resque.stub_chain(:redis, :del)
118
- end
119
-
120
- it 'should delete the processing set' do
121
- Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
122
-
123
- subject.clear_processing('foo_core')
124
- end
125
- end
126
- end
@@ -1,67 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::ResqueDelta::IndexUtils do
4
- let(:indices) { %w[foo_core foo_delta foo bar_core bar_delta bar] }
5
- let(:config) { double('config') }
6
-
7
- before :each do
8
- ThinkingSphinx::Configuration.stub(:instance => config)
9
- config.stub(:generate)
10
- config.stub_chain(:configuration, :indices, :collect => indices)
11
-
12
- subject.reload!
13
- end
14
-
15
- describe '.index_prefixes' do
16
- it 'should use a cached value if one exists' do
17
- indices = []
18
- subject.instance_variable_set(:@prefixes, indices)
19
-
20
- subject.index_prefixes.should be(indices)
21
- end
22
-
23
- it 'should return a list of only index prefixes' do
24
- subject.index_prefixes.should =~ %w[foo bar]
25
- end
26
- end
27
-
28
- describe '.core_indices' do
29
- it 'should use a cached value if one exists' do
30
- indices = []
31
- subject.instance_variable_set(:@core_indices, indices)
32
-
33
- subject.core_indices.should be(indices)
34
- end
35
-
36
- it 'should return a list of only core indices' do
37
- subject.core_indices.should =~ %w[foo_core bar_core]
38
- end
39
- end
40
-
41
- describe '.delta_indices' do
42
- it 'should use a cached value if one exists' do
43
- indices = []
44
- subject.instance_variable_set(:@delta_indices, indices)
45
-
46
- subject.delta_indices.should be(indices)
47
- end
48
-
49
- it 'should return a list of only delta indices' do
50
- subject.delta_indices.should =~ %w[foo_delta bar_delta]
51
- end
52
- end
53
-
54
- describe '.ts_config' do
55
- it 'should use a cached value if one exists' do
56
- subject.instance_variable_set(:@ts_config, config)
57
-
58
- subject.ts_config.should be(config)
59
- end
60
-
61
- it 'should generate the config when fetching the Configuration instance' do
62
- config.should_receive(:generate)
63
-
64
- subject.ts_config
65
- end
66
- end
67
- end