ts-resque-delta 1.2.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -1
  3. data/.travis.yml +14 -14
  4. data/Appraisals +4 -6
  5. data/Gemfile +1 -1
  6. data/README.markdown +31 -39
  7. data/Rakefile +6 -13
  8. data/gemfiles/binary.gemfile +7 -0
  9. data/gemfiles/sphinxql.gemfile +7 -0
  10. data/lib/thinking_sphinx/deltas/resque_delta.rb +66 -76
  11. data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +1 -113
  12. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +7 -0
  13. data/lib/ts-resque-delta.rb +0 -2
  14. data/spec/acceptance/resque_deltas_spec.rb +50 -0
  15. data/spec/acceptance/spec_helper.rb +4 -0
  16. data/spec/acceptance/support/database_cleaner.rb +11 -0
  17. data/spec/acceptance/support/sphinx_controller.rb +61 -0
  18. data/spec/acceptance/support/sphinx_helpers.rb +36 -0
  19. data/spec/internal/.gitignore +2 -0
  20. data/spec/internal/app/indices/book_index.rb +3 -0
  21. data/spec/internal/app/models/book.rb +7 -0
  22. data/spec/internal/config/database.yml +5 -0
  23. data/spec/internal/db/schema.rb +10 -0
  24. data/spec/internal/log/.gitignore +1 -0
  25. data/spec/spec_helper.rb +16 -12
  26. data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +11 -155
  27. data/spec/thinking_sphinx/deltas/resque_delta_spec.rb +3 -166
  28. data/ts-resque-delta.gemspec +19 -28
  29. metadata +90 -230
  30. data/Guardfile +0 -17
  31. data/features/resque_deltas.feature +0 -62
  32. data/features/smart_indexing.feature +0 -43
  33. data/features/step_definitions/common_steps.rb +0 -76
  34. data/features/step_definitions/resque_delta_steps.rb +0 -33
  35. data/features/step_definitions/smart_indexing_steps.rb +0 -3
  36. data/features/support/env.rb +0 -41
  37. data/features/thinking_sphinx/database.example.yml +0 -4
  38. data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +0 -4
  39. data/features/thinking_sphinx/models/delayed_beta.rb +0 -6
  40. data/gemfiles/activerecord2.gemfile +0 -8
  41. data/gemfiles/activerecord2.gemfile.lock +0 -114
  42. data/gemfiles/activerecord3.gemfile +0 -8
  43. data/gemfiles/activerecord3.gemfile.lock +0 -123
  44. data/lib/flying_sphinx/resque_delta.rb +0 -35
  45. data/lib/flying_sphinx/resque_delta/delta_job.rb +0 -14
  46. data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb +0 -7
  47. data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb +0 -116
  48. data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb +0 -59
  49. data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb +0 -47
  50. data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb +0 -8
  51. data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +0 -38
  52. data/lib/thinking_sphinx/deltas/resque_delta/version.rb +0 -7
  53. data/spec/flying_sphinx/resque_delta/delta_job_spec.rb +0 -32
  54. data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb +0 -23
  55. data/spec/flying_sphinx/resque_delta_spec.rb +0 -130
  56. data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb +0 -210
  57. data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb +0 -126
  58. data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb +0 -67
@@ -1,210 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::ResqueDelta::CoreIndex do
4
- let(:indices) { %w[foo bar] }
5
- let(:config) { double('config') }
6
-
7
- describe '#lock_delta' do
8
- it 'should lock the delta' do
9
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock)
10
-
11
- subject.lock_delta('foo')
12
- end
13
-
14
- it 'should lock the delta for the given index' do
15
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock).with('foo_delta')
16
-
17
- subject.lock_delta('foo')
18
- end
19
- end
20
-
21
- describe '#unlock_delta' do
22
- it 'should unlock the delta' do
23
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock)
24
-
25
- subject.unlock_delta('foo')
26
- end
27
-
28
- it 'should unlock the delta for the given index' do
29
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock).with('foo_delta')
30
-
31
- subject.unlock_delta('foo')
32
- end
33
- end
34
-
35
- describe '#lock_deltas' do
36
- it 'should lock all delta indices' do
37
- subject.stub(:sphinx_indices => indices)
38
-
39
- indices.each {|index| subject.should_receive(:lock_delta).once.with(index) }
40
-
41
- subject.lock_deltas
42
- end
43
- end
44
-
45
- describe '#unlock_deltas' do
46
- it 'should unlock all delta indices' do
47
- subject.stub(:sphinx_indices => indices)
48
-
49
- indices.each {|index| subject.should_receive(:unlock_delta).once.with(index) }
50
-
51
- subject.unlock_deltas
52
- end
53
- end
54
-
55
- describe '#with_delta_index_lock' do
56
- before :each do
57
- subject.stub(:lock_delta)
58
- subject.stub(:unlock_delta)
59
-
60
- subject.stub(:block_called)
61
-
62
- @block_called = false
63
- @block = lambda { @block_called = true; subject.block_called }
64
- end
65
-
66
- it 'should yield' do
67
- subject.with_delta_index_lock('foo', &@block)
68
- @block_called.should be_true
69
- end
70
-
71
- it 'should lock before yielding' do
72
- subject.should_receive(:lock_delta).with('foo').ordered
73
- subject.should_receive(:block_called).ordered
74
-
75
- subject.with_delta_index_lock('foo', &@block)
76
- end
77
-
78
- it 'should unlock after yielding' do
79
- subject.should_receive(:block_called).ordered
80
- subject.should_receive(:unlock_delta).with('foo').ordered
81
-
82
- subject.with_delta_index_lock('foo', &@block)
83
- end
84
- end
85
-
86
- describe '#smart_index' do
87
- include FakeFS::SpecHelpers
88
-
89
- let(:test_path) { '/tmp/ts-resque-delta/foo' }
90
-
91
- before :each do
92
- subject.stub(:ts_config => config)
93
- config.stub(:config_file => 'foo_config')
94
- config.stub(:build)
95
- config.stub(:searchd_file_path => test_path)
96
- config.stub_chain(:controller, :index) do
97
- # Set $? to 0
98
- `#{File.join(SPEC_BIN_PATH, 'true.rb')}`
99
- end
100
-
101
- # Silence Generating config message
102
- subject.stub(:puts)
103
-
104
- subject.stub(:index_prefixes => indices)
105
- subject.stub(:lock_delta)
106
- subject.stub(:unlock_delta)
107
-
108
- ThinkingSphinx::Deltas::ResqueDelta.stub(:prepare_for_core_index)
109
- Resque.stub(:enqueue)
110
- end
111
-
112
- it 'should not generate sphinx configuration if INDEX_ONLY is true' do
113
- ENV.stub(:[]).with('INDEX_ONLY').and_return('true')
114
- ENV.stub(:[]).with('SILENT').and_return(nil)
115
- config.should_not_receive(:build)
116
-
117
- subject.smart_index
118
- end
119
-
120
- it 'should generate sphinx configuration if INDEX_ONLY is not true' do
121
- ENV.stub(:[]).with('INDEX_ONLY').and_return(nil)
122
- ENV.stub(:[]).with('SILENT').and_return(nil)
123
- config.should_receive(:build).once
124
-
125
- subject.smart_index
126
- end
127
-
128
- it 'should create the sphinx file directory' do
129
- subject.smart_index
130
-
131
- File.directory?(test_path).should be_true
132
- end
133
-
134
- it 'should index all core indices' do
135
- indices.each do |index|
136
- config.controller.should_receive(:index).with("#{index}_core", anything)
137
- end
138
-
139
- subject.smart_index
140
- end
141
-
142
- context "with delta lock" do
143
- before :each do
144
- subject.stub(:with_delta_index_lock).and_yield
145
- end
146
-
147
- it 'should index' do
148
- config.controller.should_receive(:index)
149
- subject.smart_index
150
- end
151
-
152
- it 'should signal resque delta to prepare for the core index' do
153
- ThinkingSphinx::Deltas::ResqueDelta.should_receive(:prepare_for_core_index)
154
- subject.smart_index
155
- end
156
- end
157
-
158
- context "without delta lock" do
159
- before :each do
160
- subject.stub(:with_delta_index_lock)
161
- end
162
-
163
- it 'should not index without the delta lock' do
164
- config.controller.should_not_receive(:index)
165
- subject.smart_index
166
- end
167
-
168
- it 'should not signal resque delta to prepare for the core index' do
169
- ThinkingSphinx::Deltas::ResqueDelta.should_not_receive(:prepare_for_core_index)
170
- subject.smart_index
171
- end
172
- end
173
-
174
- it 'should create a delta job after the delta is unlocked' do
175
- # Create a dummy method on subject that's called when Resque.enqueue is called so we can enforce order.
176
- subject.stub(:resque_called)
177
- Resque.stub(:enqueue) { subject.resque_called }
178
-
179
- Resque.should_receive(:enqueue)
180
-
181
- subject.should_receive(:with_delta_index_lock).ordered.exactly(indices.size)
182
- subject.should_receive(:resque_called).ordered.exactly(indices.size)
183
-
184
- subject.smart_index
185
- end
186
-
187
- context 'with an error' do
188
- before :each do
189
- config.stub_chain(:controller, :index) do
190
- # Set $? to 1
191
- `#{File.join(SPEC_BIN_PATH, 'false.rb')}`
192
- end
193
- end
194
-
195
- it 'should stop processing indexes after an error' do
196
- config.controller.should_receive(:index).once
197
-
198
- subject.smart_index
199
- end
200
-
201
- it 'should return false on failure' do
202
- subject.smart_index.should be_false
203
- end
204
- end
205
-
206
- it 'should return true on success' do
207
- subject.smart_index.should be_true
208
- end
209
- end
210
- end
@@ -1,126 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet do
4
- describe '.add' do
5
- before :each do
6
- Resque.stub_chain(:redis, :sadd => true)
7
- end
8
-
9
- it 'should add the document id to the correct set' do
10
- Resque.redis.should_receive(:sadd).once.with(subject.set_name('foo_core'), 42)
11
- subject.add('foo_core', 42)
12
- end
13
- end
14
-
15
- describe '.clear!' do
16
- before :each do
17
- Resque.stub_chain(:redis, :del)
18
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock)
19
- end
20
-
21
- it 'should delete all items in the set' do
22
- Resque.redis.should_receive(:del).once.with(subject.set_name('foo_core'))
23
- subject.clear!('foo_core')
24
- end
25
-
26
- context "with DeltaJob integration" do
27
- before :each do
28
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock).and_yield
29
- end
30
-
31
- it 'should acquire the DeltaJob lock' do
32
- ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:around_perform_lock).once.with('foo_delta')
33
- subject.clear!('foo_core')
34
- end
35
-
36
- it 'should delete all items in the processing set' do
37
- Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
38
- subject.clear!('foo_core')
39
- end
40
- end
41
- end
42
-
43
- describe '.clear_all!' do
44
- let(:core_indices) { %w[foo_core bar_core] }
45
-
46
- it 'should clear each index' do
47
- ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.stub_chain(:core_indices, :each).tap do |s|
48
- core_indices.inject(s) do |s, index|
49
- s.and_yield(index)
50
- end
51
- end
52
-
53
- core_indices.each do |index|
54
- subject.should_receive(:clear!).with(index)
55
- end
56
-
57
- subject.clear_all!
58
- end
59
- end
60
-
61
- describe '.get_subset_for_processing' do
62
- let(:mock_redis) do
63
- Resque.redis = mr = MockRedis.new
64
- subject.add 'foo_core', 42
65
- subject.add 'foo_core', 52
66
- subject.add 'foo_core', 100
67
- mr
68
- end
69
-
70
- before :each do
71
- Resque.redis = mock_redis.clone
72
- end
73
-
74
- it 'should move all members from the flag as deleted set to the processing set' do
75
- subject.get_subset_for_processing('foo_core')
76
-
77
- Resque.redis.scard(subject.set_name('foo_core')).should eql(0)
78
- Resque.redis.scard(subject.processing_name('foo_core')).should eql(3)
79
- end
80
-
81
- it 'should remove the temp set' do
82
- subject.get_subset_for_processing('foo_core')
83
-
84
- Resque.redis.scard(subject.temp_name('foo_core')).should eql(0)
85
- end
86
-
87
- it 'should preserve existing members of the processing set' do
88
- Resque.redis.sadd(subject.processing_name('foo_core'), 1)
89
-
90
- subject.get_subset_for_processing('foo_core')
91
-
92
- Resque.redis.smembers(subject.processing_name('foo_core')).should =~ %w[1 42 52 100]
93
- end
94
- end
95
-
96
- describe '.processing_members' do
97
- let(:document_ids) { %w[1, 2, 3] }
98
-
99
- before :each do
100
- Resque.stub_chain(:redis, :smembers => document_ids)
101
- end
102
-
103
- it 'should get the members of the correct set' do
104
- Resque.redis.should_receive(:smembers).once.with(subject.processing_name('foo_core'))
105
- subject.processing_members('foo_core')
106
- end
107
-
108
- it 'should return a list of integers' do
109
- subject.processing_members('foo_core').each do |id|
110
- id.class.should == Fixnum
111
- end
112
- end
113
- end
114
-
115
- describe '.clear_processing' do
116
- before :each do
117
- Resque.stub_chain(:redis, :del)
118
- end
119
-
120
- it 'should delete the processing set' do
121
- Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
122
-
123
- subject.clear_processing('foo_core')
124
- end
125
- end
126
- end
@@ -1,67 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ThinkingSphinx::Deltas::ResqueDelta::IndexUtils do
4
- let(:indices) { %w[foo_core foo_delta foo bar_core bar_delta bar] }
5
- let(:config) { double('config') }
6
-
7
- before :each do
8
- ThinkingSphinx::Configuration.stub(:instance => config)
9
- config.stub(:generate)
10
- config.stub_chain(:configuration, :indices, :collect => indices)
11
-
12
- subject.reload!
13
- end
14
-
15
- describe '.index_prefixes' do
16
- it 'should use a cached value if one exists' do
17
- indices = []
18
- subject.instance_variable_set(:@prefixes, indices)
19
-
20
- subject.index_prefixes.should be(indices)
21
- end
22
-
23
- it 'should return a list of only index prefixes' do
24
- subject.index_prefixes.should =~ %w[foo bar]
25
- end
26
- end
27
-
28
- describe '.core_indices' do
29
- it 'should use a cached value if one exists' do
30
- indices = []
31
- subject.instance_variable_set(:@core_indices, indices)
32
-
33
- subject.core_indices.should be(indices)
34
- end
35
-
36
- it 'should return a list of only core indices' do
37
- subject.core_indices.should =~ %w[foo_core bar_core]
38
- end
39
- end
40
-
41
- describe '.delta_indices' do
42
- it 'should use a cached value if one exists' do
43
- indices = []
44
- subject.instance_variable_set(:@delta_indices, indices)
45
-
46
- subject.delta_indices.should be(indices)
47
- end
48
-
49
- it 'should return a list of only delta indices' do
50
- subject.delta_indices.should =~ %w[foo_delta bar_delta]
51
- end
52
- end
53
-
54
- describe '.ts_config' do
55
- it 'should use a cached value if one exists' do
56
- subject.instance_variable_set(:@ts_config, config)
57
-
58
- subject.ts_config.should be(config)
59
- end
60
-
61
- it 'should generate the config when fetching the Configuration instance' do
62
- config.should_receive(:generate)
63
-
64
- subject.ts_config
65
- end
66
- end
67
- end