ts-resque-delta 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/flying_sphinx/resque_delta.rb +38 -0
- data/lib/flying_sphinx/resque_delta/delta_job.rb +14 -0
- data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb +7 -0
- data/lib/thinking_sphinx/deltas/resque_delta.rb +15 -8
- data/lib/thinking_sphinx/deltas/resque_delta/version.rb +1 -1
- data/spec/flying_sphinx/resque_delta/delta_job_spec.rb +32 -0
- data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb +23 -0
- data/spec/flying_sphinx/resque_delta_spec.rb +131 -0
- data/spec/spec_helper.rb +2 -0
- data/ts-resque-delta.gemspec +3 -2
- metadata +39 -13
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'thinking_sphinx/deltas/resque_delta'
|
2
|
+
|
3
|
+
class FlyingSphinx::ResqueDelta < ThinkingSphinx::Deltas::ResqueDelta
|
4
|
+
def self.job_types
|
5
|
+
[
|
6
|
+
FlyingSphinx::ResqueDelta::DeltaJob,
|
7
|
+
FlyingSphinx::ResqueDelta::FlagAsDeletedJob
|
8
|
+
]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.job_prefix
|
12
|
+
'fs-delta'
|
13
|
+
end
|
14
|
+
|
15
|
+
def index(model, instance = nil)
|
16
|
+
return true if skip?(instance)
|
17
|
+
|
18
|
+
model.delta_index_names.each do |delta|
|
19
|
+
next if self.class.locked?(delta)
|
20
|
+
|
21
|
+
Resque.enqueue(
|
22
|
+
FlyingSphinx::ResqueDelta::DeltaJob,
|
23
|
+
[delta]
|
24
|
+
)
|
25
|
+
end
|
26
|
+
|
27
|
+
Resque.enqueue(
|
28
|
+
FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
|
29
|
+
model.core_index_names,
|
30
|
+
instance.sphinx_document_id
|
31
|
+
) if instance
|
32
|
+
|
33
|
+
true
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
require 'flying_sphinx/resque_delta/delta_job'
|
38
|
+
require 'flying_sphinx/resque_delta/flag_as_deleted_job'
|
@@ -0,0 +1,14 @@
|
|
1
|
+
class FlyingSphinx::ResqueDelta::DeltaJob < ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
|
2
|
+
@queue = :fs_delta
|
3
|
+
|
4
|
+
# Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx
|
5
|
+
# is running.
|
6
|
+
#
|
7
|
+
# @param [String] index the name of the Sphinx index
|
8
|
+
#
|
9
|
+
def self.perform(indices)
|
10
|
+
return if skip?(indices)
|
11
|
+
|
12
|
+
FlyingSphinx::IndexRequest.new(indices).perform
|
13
|
+
end
|
14
|
+
end
|
@@ -12,29 +12,36 @@ require 'thinking_sphinx'
|
|
12
12
|
# @see http://ts.freelancing-gods.com Thinking Sphinx
|
13
13
|
#
|
14
14
|
class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
|
15
|
-
|
16
|
-
# LTRIM + LPOP deletes all items from the Resque queue without loading it
|
17
|
-
# into client memory (unlike Resque.dequeue).
|
18
|
-
def self.cancel_thinking_sphinx_jobs
|
15
|
+
def self.job_types
|
19
16
|
[
|
20
17
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
|
21
18
|
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob
|
22
|
-
]
|
19
|
+
]
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.job_prefix
|
23
|
+
'ts-delta'
|
24
|
+
end
|
25
|
+
|
26
|
+
# LTRIM + LPOP deletes all items from the Resque queue without loading it
|
27
|
+
# into client memory (unlike Resque.dequeue).
|
28
|
+
def self.cancel_thinking_sphinx_jobs
|
29
|
+
job_types.collect { |c| c.instance_variable_get(:@queue) }.uniq.each do |q|
|
23
30
|
Resque.redis.ltrim("queue:#{q}", 0, 0)
|
24
31
|
Resque.redis.lpop("queue:#{q}")
|
25
32
|
end
|
26
33
|
end
|
27
34
|
|
28
35
|
def self.lock(index_name)
|
29
|
-
Resque.redis.set("
|
36
|
+
Resque.redis.set("#{job_prefix}:index:#{index_name}:locked", 'true')
|
30
37
|
end
|
31
38
|
|
32
39
|
def self.unlock(index_name)
|
33
|
-
Resque.redis.del("
|
40
|
+
Resque.redis.del("#{job_prefix}:index:#{index_name}:locked")
|
34
41
|
end
|
35
42
|
|
36
43
|
def self.locked?(index_name)
|
37
|
-
Resque.redis.get("
|
44
|
+
Resque.redis.get("#{job_prefix}:index:#{index_name}:locked") == 'true'
|
38
45
|
end
|
39
46
|
|
40
47
|
# Adds a job to the queue for processing the given model's delta index. A job
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe FlyingSphinx::ResqueDelta::DeltaJob do
|
4
|
+
describe '@queue' do
|
5
|
+
it "uses the fs_delta queue" do
|
6
|
+
FlyingSphinx::ResqueDelta::DeltaJob.instance_variable_get(:@queue).
|
7
|
+
should == :fs_delta
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '.perform' do
|
12
|
+
it "doesn't create an index request when skipping" do
|
13
|
+
FlyingSphinx::ResqueDelta::DeltaJob.stub!(:skip? => true)
|
14
|
+
|
15
|
+
FlyingSphinx::IndexRequest.should_not_receive(:new)
|
16
|
+
|
17
|
+
FlyingSphinx::ResqueDelta::DeltaJob.perform ['foo_delta']
|
18
|
+
end
|
19
|
+
|
20
|
+
it "performs an index request when not skipping" do
|
21
|
+
request = double('index request', :perform => true)
|
22
|
+
FlyingSphinx::ResqueDelta::DeltaJob.stub!(:skip? => false)
|
23
|
+
|
24
|
+
FlyingSphinx::IndexRequest.should_receive(:new).
|
25
|
+
with(['foo_delta']).
|
26
|
+
and_return(request)
|
27
|
+
request.should_receive(:perform)
|
28
|
+
|
29
|
+
FlyingSphinx::ResqueDelta::DeltaJob.perform ['foo_delta']
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe FlyingSphinx::ResqueDelta::FlagAsDeletedJob do
|
4
|
+
describe '@queue' do
|
5
|
+
it "uses the fs_delta queue" do
|
6
|
+
FlyingSphinx::ResqueDelta::FlagAsDeletedJob.
|
7
|
+
instance_variable_get(:@queue).should == :fs_delta
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '.perform' do
|
12
|
+
it "performs a flag-as-deleted job" do
|
13
|
+
job = double('flag as deleted job', :perform => true)
|
14
|
+
|
15
|
+
FlyingSphinx::FlagAsDeletedJob.should_receive(:new).
|
16
|
+
with(['foo_core'], 5).
|
17
|
+
and_return(job)
|
18
|
+
job.should_receive(:perform)
|
19
|
+
|
20
|
+
FlyingSphinx::ResqueDelta::FlagAsDeletedJob.perform ['foo_core'], 5
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe FlyingSphinx::ResqueDelta do
|
4
|
+
describe '.job_types' do
|
5
|
+
it "contains just the Flying Sphinx delta and delete jobs" do
|
6
|
+
FlyingSphinx::ResqueDelta.job_types.should == [
|
7
|
+
FlyingSphinx::ResqueDelta::DeltaJob,
|
8
|
+
FlyingSphinx::ResqueDelta::FlagAsDeletedJob
|
9
|
+
]
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '.job_prefix' do
|
14
|
+
it "is fs-delta" do
|
15
|
+
FlyingSphinx::ResqueDelta.job_prefix.should == 'fs-delta'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
describe '#index' do
|
20
|
+
before :each do
|
21
|
+
ThinkingSphinx.updates_enabled = true
|
22
|
+
ThinkingSphinx.deltas_enabled = true
|
23
|
+
|
24
|
+
Resque.stub(:enqueue => true)
|
25
|
+
|
26
|
+
@delayed_delta = FlyingSphinx::ResqueDelta.new(
|
27
|
+
stub('instance'), {}
|
28
|
+
)
|
29
|
+
@delayed_delta.stub(:toggled).and_return(true)
|
30
|
+
|
31
|
+
FlyingSphinx::ResqueDelta.stub(:lock)
|
32
|
+
FlyingSphinx::ResqueDelta.stub(:unlock)
|
33
|
+
FlyingSphinx::ResqueDelta.stub(:locked?).and_return(false)
|
34
|
+
|
35
|
+
@model = stub('foo')
|
36
|
+
@model.stub(:core_index_names => ['foo_core'])
|
37
|
+
@model.stub(:delta_index_names => ['foo_delta'])
|
38
|
+
|
39
|
+
@instance = stub('instance')
|
40
|
+
@instance.stub(:sphinx_document_id => 42)
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'updates disabled' do
|
44
|
+
before :each do
|
45
|
+
ThinkingSphinx.updates_enabled = false
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should not enqueue a delta job" do
|
49
|
+
Resque.should_not_receive(:enqueue)
|
50
|
+
@delayed_delta.index(@model)
|
51
|
+
end
|
52
|
+
|
53
|
+
it "should not enqueue a flag as deleted job" do
|
54
|
+
Resque.should_not_receive(:enqueue)
|
55
|
+
@delayed_delta.index(@model)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
context 'deltas disabled' do
|
60
|
+
before :each do
|
61
|
+
ThinkingSphinx.deltas_enabled = false
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should not enqueue a delta job" do
|
65
|
+
Resque.should_not_receive(:enqueue)
|
66
|
+
@delayed_delta.index(@model)
|
67
|
+
end
|
68
|
+
|
69
|
+
it "should not enqueue a flag as deleted job" do
|
70
|
+
Resque.should_not_receive(:enqueue)
|
71
|
+
@delayed_delta.index(@model)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
context "instance isn't toggled" do
|
76
|
+
before :each do
|
77
|
+
@delayed_delta.stub(:toggled => false)
|
78
|
+
end
|
79
|
+
|
80
|
+
it "should not enqueue a delta job" do
|
81
|
+
Resque.should_not_receive(:enqueue)
|
82
|
+
@delayed_delta.index(@model, @instance)
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should not enqueue a flag as deleted job" do
|
86
|
+
Resque.should_not_receive(:enqueue)
|
87
|
+
@delayed_delta.index(@model, @instance)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should enqueue a delta job" do
|
92
|
+
Resque.should_receive(:enqueue).at_least(:once).with(
|
93
|
+
FlyingSphinx::ResqueDelta::DeltaJob,
|
94
|
+
['foo_delta']
|
95
|
+
)
|
96
|
+
@delayed_delta.index(@model)
|
97
|
+
end
|
98
|
+
|
99
|
+
it "should enqueue a flag-as-deleted job" do
|
100
|
+
Resque.should_receive(:enqueue).at_least(:once).with(
|
101
|
+
FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
|
102
|
+
['foo_core'],
|
103
|
+
42
|
104
|
+
)
|
105
|
+
@delayed_delta.index(@model, @instance)
|
106
|
+
end
|
107
|
+
|
108
|
+
context "delta index is locked" do
|
109
|
+
before :each do
|
110
|
+
FlyingSphinx::ResqueDelta.stub(:locked?).and_return(true)
|
111
|
+
end
|
112
|
+
|
113
|
+
it "should not enqueue a delta job" do
|
114
|
+
Resque.should_not_receive(:enqueue).with(
|
115
|
+
FlyingSphinx::ResqueDelta::DeltaJob,
|
116
|
+
['foo_delta']
|
117
|
+
)
|
118
|
+
@delayed_delta.index(@model, @instance)
|
119
|
+
end
|
120
|
+
|
121
|
+
it "should enqueue a flag-as-deleted job" do
|
122
|
+
Resque.should_receive(:enqueue).at_least(:once).with(
|
123
|
+
FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
|
124
|
+
['foo_core'],
|
125
|
+
42
|
126
|
+
)
|
127
|
+
@delayed_delta.index(@model, @instance)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
data/spec/spec_helper.rb
CHANGED
data/ts-resque-delta.gemspec
CHANGED
@@ -21,14 +21,15 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
s.add_dependency "thinking-sphinx", ">= 1.3.6"
|
23
23
|
s.add_dependency "resque", "~> 1.10"
|
24
|
-
s.add_dependency "resque-lock-timeout", "~> 0.
|
24
|
+
s.add_dependency "resque-lock-timeout", "~> 0.3.1"
|
25
25
|
|
26
26
|
s.add_development_dependency "rspec", "~> 1"
|
27
27
|
s.add_development_dependency "cucumber", ">= 0"
|
28
28
|
s.add_development_dependency "database_cleaner", ">= 0.5.2"
|
29
|
-
s.add_development_dependency "
|
29
|
+
s.add_development_dependency "mysql2", "~> 0.2.7"
|
30
30
|
s.add_development_dependency "SystemTimer"
|
31
31
|
s.add_development_dependency "rake", "0.8.7"
|
32
32
|
s.add_development_dependency "ruby-debug"
|
33
33
|
s.add_development_dependency "activerecord", "~> 2.3.11"
|
34
|
+
s.add_development_dependency "flying-sphinx", ">= 0.5.1"
|
34
35
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ts-resque-delta
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 19
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 1
|
8
|
+
- 1
|
8
9
|
- 0
|
9
|
-
|
10
|
-
version: 1.0.0
|
10
|
+
version: 1.1.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Aaron Gibralter
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-08-29 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: thinking-sphinx
|
@@ -56,12 +56,12 @@ dependencies:
|
|
56
56
|
requirements:
|
57
57
|
- - ~>
|
58
58
|
- !ruby/object:Gem::Version
|
59
|
-
hash:
|
59
|
+
hash: 17
|
60
60
|
segments:
|
61
61
|
- 0
|
62
|
-
-
|
62
|
+
- 3
|
63
63
|
- 1
|
64
|
-
version: 0.
|
64
|
+
version: 0.3.1
|
65
65
|
type: :runtime
|
66
66
|
version_requirements: *id003
|
67
67
|
- !ruby/object:Gem::Dependency
|
@@ -109,17 +109,19 @@ dependencies:
|
|
109
109
|
type: :development
|
110
110
|
version_requirements: *id006
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
112
|
+
name: mysql2
|
113
113
|
prerelease: false
|
114
114
|
requirement: &id007 !ruby/object:Gem::Requirement
|
115
115
|
none: false
|
116
116
|
requirements:
|
117
|
-
- -
|
117
|
+
- - ~>
|
118
118
|
- !ruby/object:Gem::Version
|
119
|
-
hash:
|
119
|
+
hash: 25
|
120
120
|
segments:
|
121
121
|
- 0
|
122
|
-
|
122
|
+
- 2
|
123
|
+
- 7
|
124
|
+
version: 0.2.7
|
123
125
|
type: :development
|
124
126
|
version_requirements: *id007
|
125
127
|
- !ruby/object:Gem::Dependency
|
@@ -182,6 +184,22 @@ dependencies:
|
|
182
184
|
version: 2.3.11
|
183
185
|
type: :development
|
184
186
|
version_requirements: *id011
|
187
|
+
- !ruby/object:Gem::Dependency
|
188
|
+
name: flying-sphinx
|
189
|
+
prerelease: false
|
190
|
+
requirement: &id012 !ruby/object:Gem::Requirement
|
191
|
+
none: false
|
192
|
+
requirements:
|
193
|
+
- - ">="
|
194
|
+
- !ruby/object:Gem::Version
|
195
|
+
hash: 9
|
196
|
+
segments:
|
197
|
+
- 0
|
198
|
+
- 5
|
199
|
+
- 1
|
200
|
+
version: 0.5.1
|
201
|
+
type: :development
|
202
|
+
version_requirements: *id012
|
185
203
|
description: Manage delta indexes via Resque for Thinking Sphinx
|
186
204
|
email:
|
187
205
|
- aaron.gibralter@gmail.com
|
@@ -206,11 +224,17 @@ files:
|
|
206
224
|
- features/thinking_sphinx/database.example.yml
|
207
225
|
- features/thinking_sphinx/db/migrations/create_delayed_betas.rb
|
208
226
|
- features/thinking_sphinx/models/delayed_beta.rb
|
227
|
+
- lib/flying_sphinx/resque_delta.rb
|
228
|
+
- lib/flying_sphinx/resque_delta/delta_job.rb
|
229
|
+
- lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb
|
209
230
|
- lib/thinking_sphinx/deltas/resque_delta.rb
|
210
231
|
- lib/thinking_sphinx/deltas/resque_delta/delta_job.rb
|
211
232
|
- lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb
|
212
233
|
- lib/thinking_sphinx/deltas/resque_delta/tasks.rb
|
213
234
|
- lib/thinking_sphinx/deltas/resque_delta/version.rb
|
235
|
+
- spec/flying_sphinx/resque_delta/delta_job_spec.rb
|
236
|
+
- spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb
|
237
|
+
- spec/flying_sphinx/resque_delta_spec.rb
|
214
238
|
- spec/spec.opts
|
215
239
|
- spec/spec_helper.rb
|
216
240
|
- spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb
|
@@ -248,7 +272,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
248
272
|
requirements: []
|
249
273
|
|
250
274
|
rubyforge_project: ts-resque-delta
|
251
|
-
rubygems_version: 1.
|
275
|
+
rubygems_version: 1.8.9
|
252
276
|
signing_key:
|
253
277
|
specification_version: 3
|
254
278
|
summary: Thinking Sphinx - Resque Deltas
|
@@ -261,9 +285,11 @@ test_files:
|
|
261
285
|
- features/thinking_sphinx/database.example.yml
|
262
286
|
- features/thinking_sphinx/db/migrations/create_delayed_betas.rb
|
263
287
|
- features/thinking_sphinx/models/delayed_beta.rb
|
288
|
+
- spec/flying_sphinx/resque_delta/delta_job_spec.rb
|
289
|
+
- spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb
|
290
|
+
- spec/flying_sphinx/resque_delta_spec.rb
|
264
291
|
- spec/spec.opts
|
265
292
|
- spec/spec_helper.rb
|
266
293
|
- spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb
|
267
294
|
- spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job_spec.rb
|
268
295
|
- spec/thinking_sphinx/deltas/resque_delta_spec.rb
|
269
|
-
has_rdoc:
|