ts-resque-delta 1.1.4 → 1.1.5
Sign up to get free protection for your applications and to get access to all the features.
- data/features/resque_deltas.feature +1 -1
- data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +1 -1
- data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +6 -6
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +3 -3
- data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +11 -11
- data/lib/thinking_sphinx/deltas/resque_delta/version.rb +1 -1
- data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +4 -4
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job_spec.rb +1 -1
- data/ts-resque-delta.gemspec +1 -0
- metadata +58 -43
@@ -1,7 +1,7 @@
|
|
1
1
|
Feature: Resque Delta Indexing
|
2
2
|
In order to have delta indexing on frequently-updated sites
|
3
3
|
Developers
|
4
|
-
Should be able to use Resque to handle delta
|
4
|
+
Should be able to use Resque to handle delta indices to lower system load
|
5
5
|
|
6
6
|
Background:
|
7
7
|
Given Sphinx is running
|
@@ -1,4 +1,4 @@
|
|
1
|
-
ActiveRecord::Base.connection.create_table :
|
1
|
+
ActiveRecord::Base.connection.create_table :delayed_betas, :force => true do |t|
|
2
2
|
t.column :name, :string, :null => false
|
3
3
|
t.column :delta, :boolean, :null => false, :default => false
|
4
4
|
end
|
@@ -13,11 +13,11 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
|
|
13
13
|
#
|
14
14
|
# @param [String] index the name of the Sphinx index
|
15
15
|
#
|
16
|
-
def self.perform(
|
17
|
-
return if skip?(
|
16
|
+
def self.perform(indices)
|
17
|
+
return if skip?(indices)
|
18
18
|
|
19
19
|
config = ThinkingSphinx::Configuration.instance
|
20
|
-
output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} --rotate #{
|
20
|
+
output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} --rotate #{indices.join(' ')}`
|
21
21
|
puts output unless ThinkingSphinx.suppress_delta_output?
|
22
22
|
end
|
23
23
|
|
@@ -26,7 +26,7 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
|
|
26
26
|
Resque.enqueue(self, *args)
|
27
27
|
end
|
28
28
|
|
29
|
-
# Run only one DeltaJob at a time regardless of
|
29
|
+
# Run only one DeltaJob at a time regardless of indices.
|
30
30
|
def self.identifier(*args)
|
31
31
|
nil
|
32
32
|
end
|
@@ -56,8 +56,8 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
|
|
56
56
|
|
57
57
|
protected
|
58
58
|
|
59
|
-
def self.skip?(
|
60
|
-
|
59
|
+
def self.skip?(indices)
|
60
|
+
indices.any? do |index|
|
61
61
|
ThinkingSphinx::Deltas::ResqueDelta.locked?(index)
|
62
62
|
end
|
63
63
|
end
|
@@ -14,14 +14,14 @@ class ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob
|
|
14
14
|
# situation to avoid old values in the core index and just use the new
|
15
15
|
# values in the delta index as a reference point.
|
16
16
|
#
|
17
|
-
# @param [Array]
|
17
|
+
# @param [Array] indices An array of index names
|
18
18
|
# @param [Integer] document_id The document id
|
19
19
|
#
|
20
20
|
# @return [Boolean] true
|
21
21
|
#
|
22
|
-
def self.perform(
|
22
|
+
def self.perform(indices, document_id)
|
23
23
|
config = ThinkingSphinx::Configuration.instance
|
24
|
-
|
24
|
+
indices.each do |index|
|
25
25
|
if ThinkingSphinx.sphinx_running? && ThinkingSphinx.search_for_id(document_id, index)
|
26
26
|
config.client.update(index, ['sphinx_deleted'], {document_id => [1]})
|
27
27
|
end
|
@@ -3,18 +3,18 @@ require 'thinking_sphinx/deltas/resque_delta'
|
|
3
3
|
namespace :thinking_sphinx do
|
4
4
|
|
5
5
|
# Return a list of index prefixes (i.e. without "_core"/"_delta").
|
6
|
-
def
|
7
|
-
unless @
|
6
|
+
def sphinx_indices
|
7
|
+
unless @sphinx_indices
|
8
8
|
@ts_config ||= ThinkingSphinx::Configuration.instance
|
9
9
|
@ts_config.generate
|
10
|
-
@
|
11
|
-
# The collected
|
10
|
+
@sphinx_indices = @ts_config.configuration.indices.collect { |i| i.name }
|
11
|
+
# The collected indices look like:
|
12
12
|
# ["foo_core", "foo_delta", "foo", "bar_core", "bar_delta", "bar"]
|
13
|
-
@
|
13
|
+
@sphinx_indices.reject! { |i| i =~ /_(core|delta)$/}
|
14
14
|
# Now we have:
|
15
15
|
# ["foo", "bar"]
|
16
16
|
end
|
17
|
-
@
|
17
|
+
@sphinx_indices
|
18
18
|
end
|
19
19
|
|
20
20
|
def lock_delta(index_name)
|
@@ -25,14 +25,14 @@ namespace :thinking_sphinx do
|
|
25
25
|
ThinkingSphinx::Deltas::ResqueDelta.unlock("#{index_name}_delta")
|
26
26
|
end
|
27
27
|
|
28
|
-
desc 'Lock all delta
|
28
|
+
desc 'Lock all delta indices (Resque will not run indexer or place new jobs on the :ts_delta queue).'
|
29
29
|
task :lock_deltas do
|
30
|
-
|
30
|
+
sphinx_indices.each { |index_name| lock_delta(index_name) }
|
31
31
|
end
|
32
32
|
|
33
|
-
desc 'Unlock all delta
|
33
|
+
desc 'Unlock all delta indices.'
|
34
34
|
task :unlock_deltas do
|
35
|
-
|
35
|
+
sphinx_indices.each { |index_name| unlock_delta(index_name) }
|
36
36
|
end
|
37
37
|
|
38
38
|
desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
|
@@ -46,7 +46,7 @@ namespace :thinking_sphinx do
|
|
46
46
|
FileUtils.mkdir_p(@ts_config.searchd_file_path)
|
47
47
|
|
48
48
|
# Index each core, one at a time. Wrap with delta locking logic.
|
49
|
-
|
49
|
+
sphinx_indices.each do |index_name|
|
50
50
|
lock_delta(index_name)
|
51
51
|
@ts_config.controller.index("#{index_name}_core", :verbose => true)
|
52
52
|
ret = $?
|
@@ -24,7 +24,7 @@ describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
|
|
24
24
|
)
|
25
25
|
end
|
26
26
|
|
27
|
-
it "should process just the requested
|
27
|
+
it "should process just the requested indices" do
|
28
28
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:`) do |c|
|
29
29
|
c.should match(/foo_delta/)
|
30
30
|
c.should_not match(/--all/)
|
@@ -34,8 +34,8 @@ describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
|
|
34
34
|
)
|
35
35
|
end
|
36
36
|
|
37
|
-
context 'multiple
|
38
|
-
it "should process all requested
|
37
|
+
context 'multiple indices' do
|
38
|
+
it "should process all requested indices" do
|
39
39
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:`) do |c|
|
40
40
|
c.should match(/foo_delta bar_delta/)
|
41
41
|
end
|
@@ -59,7 +59,7 @@ describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
|
|
59
59
|
)
|
60
60
|
end
|
61
61
|
|
62
|
-
it "should not start the indexer for multiple
|
62
|
+
it "should not start the indexer for multiple indices" do
|
63
63
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_not_receive(:`)
|
64
64
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
|
65
65
|
['bar_delta', 'foo_delta']
|
@@ -29,7 +29,7 @@ describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob do
|
|
29
29
|
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob.perform(['foo_core'], 12)
|
30
30
|
end
|
31
31
|
|
32
|
-
it "should update all specified
|
32
|
+
it "should update all specified indices" do
|
33
33
|
@client.should_receive(:update).with('foo_core', anything, anything)
|
34
34
|
@client.should_receive(:update).with('bar_core', anything, anything)
|
35
35
|
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob.perform(['foo_core', 'bar_core'], 12)
|
data/ts-resque-delta.gemspec
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ts-resque-delta
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 25
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 1
|
8
8
|
- 1
|
9
|
-
-
|
10
|
-
version: 1.1.
|
9
|
+
- 5
|
10
|
+
version: 1.1.5
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Aaron Gibralter
|
@@ -15,12 +15,10 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-11-09 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
|
-
|
22
|
-
prerelease: false
|
23
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
21
|
+
version_requirements: &id001 !ruby/object:Gem::Requirement
|
24
22
|
none: false
|
25
23
|
requirements:
|
26
24
|
- - ">="
|
@@ -31,12 +29,27 @@ dependencies:
|
|
31
29
|
- 4
|
32
30
|
- 1
|
33
31
|
version: 1.4.1
|
32
|
+
requirement: *id001
|
33
|
+
prerelease: false
|
34
|
+
name: thinking-sphinx
|
34
35
|
type: :runtime
|
35
|
-
version_requirements: *id001
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
|
-
|
37
|
+
version_requirements: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
hash: 5
|
43
|
+
segments:
|
44
|
+
- 1
|
45
|
+
- 5
|
46
|
+
version: "1.5"
|
47
|
+
requirement: *id002
|
38
48
|
prerelease: false
|
39
|
-
|
49
|
+
name: riddle
|
50
|
+
type: :runtime
|
51
|
+
- !ruby/object:Gem::Dependency
|
52
|
+
version_requirements: &id003 !ruby/object:Gem::Requirement
|
40
53
|
none: false
|
41
54
|
requirements:
|
42
55
|
- - ~>
|
@@ -46,12 +59,12 @@ dependencies:
|
|
46
59
|
- 1
|
47
60
|
- 10
|
48
61
|
version: "1.10"
|
62
|
+
requirement: *id003
|
63
|
+
prerelease: false
|
64
|
+
name: resque
|
49
65
|
type: :runtime
|
50
|
-
version_requirements: *id002
|
51
66
|
- !ruby/object:Gem::Dependency
|
52
|
-
|
53
|
-
prerelease: false
|
54
|
-
requirement: &id003 !ruby/object:Gem::Requirement
|
67
|
+
version_requirements: &id004 !ruby/object:Gem::Requirement
|
55
68
|
none: false
|
56
69
|
requirements:
|
57
70
|
- - ~>
|
@@ -62,12 +75,12 @@ dependencies:
|
|
62
75
|
- 3
|
63
76
|
- 1
|
64
77
|
version: 0.3.1
|
78
|
+
requirement: *id004
|
79
|
+
prerelease: false
|
80
|
+
name: resque-lock-timeout
|
65
81
|
type: :runtime
|
66
|
-
version_requirements: *id003
|
67
82
|
- !ruby/object:Gem::Dependency
|
68
|
-
|
69
|
-
prerelease: false
|
70
|
-
requirement: &id004 !ruby/object:Gem::Requirement
|
83
|
+
version_requirements: &id005 !ruby/object:Gem::Requirement
|
71
84
|
none: false
|
72
85
|
requirements:
|
73
86
|
- - ~>
|
@@ -77,12 +90,12 @@ dependencies:
|
|
77
90
|
- 1
|
78
91
|
- 0
|
79
92
|
version: "1.0"
|
93
|
+
requirement: *id005
|
94
|
+
prerelease: false
|
95
|
+
name: rspec
|
80
96
|
type: :development
|
81
|
-
version_requirements: *id004
|
82
97
|
- !ruby/object:Gem::Dependency
|
83
|
-
|
84
|
-
prerelease: false
|
85
|
-
requirement: &id005 !ruby/object:Gem::Requirement
|
98
|
+
version_requirements: &id006 !ruby/object:Gem::Requirement
|
86
99
|
none: false
|
87
100
|
requirements:
|
88
101
|
- - ">="
|
@@ -91,12 +104,12 @@ dependencies:
|
|
91
104
|
segments:
|
92
105
|
- 0
|
93
106
|
version: "0"
|
107
|
+
requirement: *id006
|
108
|
+
prerelease: false
|
109
|
+
name: cucumber
|
94
110
|
type: :development
|
95
|
-
version_requirements: *id005
|
96
111
|
- !ruby/object:Gem::Dependency
|
97
|
-
|
98
|
-
prerelease: false
|
99
|
-
requirement: &id006 !ruby/object:Gem::Requirement
|
112
|
+
version_requirements: &id007 !ruby/object:Gem::Requirement
|
100
113
|
none: false
|
101
114
|
requirements:
|
102
115
|
- - ">="
|
@@ -107,12 +120,12 @@ dependencies:
|
|
107
120
|
- 5
|
108
121
|
- 2
|
109
122
|
version: 0.5.2
|
123
|
+
requirement: *id007
|
124
|
+
prerelease: false
|
125
|
+
name: database_cleaner
|
110
126
|
type: :development
|
111
|
-
version_requirements: *id006
|
112
127
|
- !ruby/object:Gem::Dependency
|
113
|
-
|
114
|
-
prerelease: false
|
115
|
-
requirement: &id007 !ruby/object:Gem::Requirement
|
128
|
+
version_requirements: &id008 !ruby/object:Gem::Requirement
|
116
129
|
none: false
|
117
130
|
requirements:
|
118
131
|
- - ~>
|
@@ -123,12 +136,12 @@ dependencies:
|
|
123
136
|
- 2
|
124
137
|
- 7
|
125
138
|
version: 0.2.7
|
139
|
+
requirement: *id008
|
140
|
+
prerelease: false
|
141
|
+
name: mysql2
|
126
142
|
type: :development
|
127
|
-
version_requirements: *id007
|
128
143
|
- !ruby/object:Gem::Dependency
|
129
|
-
|
130
|
-
prerelease: false
|
131
|
-
requirement: &id008 !ruby/object:Gem::Requirement
|
144
|
+
version_requirements: &id009 !ruby/object:Gem::Requirement
|
132
145
|
none: false
|
133
146
|
requirements:
|
134
147
|
- - "="
|
@@ -139,12 +152,12 @@ dependencies:
|
|
139
152
|
- 8
|
140
153
|
- 7
|
141
154
|
version: 0.8.7
|
155
|
+
requirement: *id009
|
156
|
+
prerelease: false
|
157
|
+
name: rake
|
142
158
|
type: :development
|
143
|
-
version_requirements: *id008
|
144
159
|
- !ruby/object:Gem::Dependency
|
145
|
-
|
146
|
-
prerelease: false
|
147
|
-
requirement: &id009 !ruby/object:Gem::Requirement
|
160
|
+
version_requirements: &id010 !ruby/object:Gem::Requirement
|
148
161
|
none: false
|
149
162
|
requirements:
|
150
163
|
- - ~>
|
@@ -155,12 +168,12 @@ dependencies:
|
|
155
168
|
- 3
|
156
169
|
- 11
|
157
170
|
version: 2.3.11
|
171
|
+
requirement: *id010
|
172
|
+
prerelease: false
|
173
|
+
name: activerecord
|
158
174
|
type: :development
|
159
|
-
version_requirements: *id009
|
160
175
|
- !ruby/object:Gem::Dependency
|
161
|
-
|
162
|
-
prerelease: false
|
163
|
-
requirement: &id010 !ruby/object:Gem::Requirement
|
176
|
+
version_requirements: &id011 !ruby/object:Gem::Requirement
|
164
177
|
none: false
|
165
178
|
requirements:
|
166
179
|
- - ">="
|
@@ -171,8 +184,10 @@ dependencies:
|
|
171
184
|
- 5
|
172
185
|
- 1
|
173
186
|
version: 0.5.1
|
187
|
+
requirement: *id011
|
188
|
+
prerelease: false
|
189
|
+
name: flying-sphinx
|
174
190
|
type: :development
|
175
|
-
version_requirements: *id010
|
176
191
|
description: Manage delta indexes via Resque for Thinking Sphinx
|
177
192
|
email:
|
178
193
|
- aaron.gibralter@gmail.com
|