ts-resque-delta 1.1.4 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/features/resque_deltas.feature +1 -1
- data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb +1 -1
- data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb +6 -6
- data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb +3 -3
- data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb +11 -11
- data/lib/thinking_sphinx/deltas/resque_delta/version.rb +1 -1
- data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb +4 -4
- data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job_spec.rb +1 -1
- data/ts-resque-delta.gemspec +1 -0
- metadata +58 -43
@@ -1,7 +1,7 @@
|
|
1
1
|
Feature: Resque Delta Indexing
|
2
2
|
In order to have delta indexing on frequently-updated sites
|
3
3
|
Developers
|
4
|
-
Should be able to use Resque to handle delta
|
4
|
+
Should be able to use Resque to handle delta indices to lower system load
|
5
5
|
|
6
6
|
Background:
|
7
7
|
Given Sphinx is running
|
@@ -1,4 +1,4 @@
|
|
1
|
-
ActiveRecord::Base.connection.create_table :
|
1
|
+
ActiveRecord::Base.connection.create_table :delayed_betas, :force => true do |t|
|
2
2
|
t.column :name, :string, :null => false
|
3
3
|
t.column :delta, :boolean, :null => false, :default => false
|
4
4
|
end
|
@@ -13,11 +13,11 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
|
|
13
13
|
#
|
14
14
|
# @param [String] index the name of the Sphinx index
|
15
15
|
#
|
16
|
-
def self.perform(
|
17
|
-
return if skip?(
|
16
|
+
def self.perform(indices)
|
17
|
+
return if skip?(indices)
|
18
18
|
|
19
19
|
config = ThinkingSphinx::Configuration.instance
|
20
|
-
output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} --rotate #{
|
20
|
+
output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} --rotate #{indices.join(' ')}`
|
21
21
|
puts output unless ThinkingSphinx.suppress_delta_output?
|
22
22
|
end
|
23
23
|
|
@@ -26,7 +26,7 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
|
|
26
26
|
Resque.enqueue(self, *args)
|
27
27
|
end
|
28
28
|
|
29
|
-
# Run only one DeltaJob at a time regardless of
|
29
|
+
# Run only one DeltaJob at a time regardless of indices.
|
30
30
|
def self.identifier(*args)
|
31
31
|
nil
|
32
32
|
end
|
@@ -56,8 +56,8 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
|
|
56
56
|
|
57
57
|
protected
|
58
58
|
|
59
|
-
def self.skip?(
|
60
|
-
|
59
|
+
def self.skip?(indices)
|
60
|
+
indices.any? do |index|
|
61
61
|
ThinkingSphinx::Deltas::ResqueDelta.locked?(index)
|
62
62
|
end
|
63
63
|
end
|
@@ -14,14 +14,14 @@ class ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob
|
|
14
14
|
# situation to avoid old values in the core index and just use the new
|
15
15
|
# values in the delta index as a reference point.
|
16
16
|
#
|
17
|
-
# @param [Array]
|
17
|
+
# @param [Array] indices An array of index names
|
18
18
|
# @param [Integer] document_id The document id
|
19
19
|
#
|
20
20
|
# @return [Boolean] true
|
21
21
|
#
|
22
|
-
def self.perform(
|
22
|
+
def self.perform(indices, document_id)
|
23
23
|
config = ThinkingSphinx::Configuration.instance
|
24
|
-
|
24
|
+
indices.each do |index|
|
25
25
|
if ThinkingSphinx.sphinx_running? && ThinkingSphinx.search_for_id(document_id, index)
|
26
26
|
config.client.update(index, ['sphinx_deleted'], {document_id => [1]})
|
27
27
|
end
|
@@ -3,18 +3,18 @@ require 'thinking_sphinx/deltas/resque_delta'
|
|
3
3
|
namespace :thinking_sphinx do
|
4
4
|
|
5
5
|
# Return a list of index prefixes (i.e. without "_core"/"_delta").
|
6
|
-
def
|
7
|
-
unless @
|
6
|
+
def sphinx_indices
|
7
|
+
unless @sphinx_indices
|
8
8
|
@ts_config ||= ThinkingSphinx::Configuration.instance
|
9
9
|
@ts_config.generate
|
10
|
-
@
|
11
|
-
# The collected
|
10
|
+
@sphinx_indices = @ts_config.configuration.indices.collect { |i| i.name }
|
11
|
+
# The collected indices look like:
|
12
12
|
# ["foo_core", "foo_delta", "foo", "bar_core", "bar_delta", "bar"]
|
13
|
-
@
|
13
|
+
@sphinx_indices.reject! { |i| i =~ /_(core|delta)$/}
|
14
14
|
# Now we have:
|
15
15
|
# ["foo", "bar"]
|
16
16
|
end
|
17
|
-
@
|
17
|
+
@sphinx_indices
|
18
18
|
end
|
19
19
|
|
20
20
|
def lock_delta(index_name)
|
@@ -25,14 +25,14 @@ namespace :thinking_sphinx do
|
|
25
25
|
ThinkingSphinx::Deltas::ResqueDelta.unlock("#{index_name}_delta")
|
26
26
|
end
|
27
27
|
|
28
|
-
desc 'Lock all delta
|
28
|
+
desc 'Lock all delta indices (Resque will not run indexer or place new jobs on the :ts_delta queue).'
|
29
29
|
task :lock_deltas do
|
30
|
-
|
30
|
+
sphinx_indices.each { |index_name| lock_delta(index_name) }
|
31
31
|
end
|
32
32
|
|
33
|
-
desc 'Unlock all delta
|
33
|
+
desc 'Unlock all delta indices.'
|
34
34
|
task :unlock_deltas do
|
35
|
-
|
35
|
+
sphinx_indices.each { |index_name| unlock_delta(index_name) }
|
36
36
|
end
|
37
37
|
|
38
38
|
desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
|
@@ -46,7 +46,7 @@ namespace :thinking_sphinx do
|
|
46
46
|
FileUtils.mkdir_p(@ts_config.searchd_file_path)
|
47
47
|
|
48
48
|
# Index each core, one at a time. Wrap with delta locking logic.
|
49
|
-
|
49
|
+
sphinx_indices.each do |index_name|
|
50
50
|
lock_delta(index_name)
|
51
51
|
@ts_config.controller.index("#{index_name}_core", :verbose => true)
|
52
52
|
ret = $?
|
@@ -24,7 +24,7 @@ describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
|
|
24
24
|
)
|
25
25
|
end
|
26
26
|
|
27
|
-
it "should process just the requested
|
27
|
+
it "should process just the requested indices" do
|
28
28
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:`) do |c|
|
29
29
|
c.should match(/foo_delta/)
|
30
30
|
c.should_not match(/--all/)
|
@@ -34,8 +34,8 @@ describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
|
|
34
34
|
)
|
35
35
|
end
|
36
36
|
|
37
|
-
context 'multiple
|
38
|
-
it "should process all requested
|
37
|
+
context 'multiple indices' do
|
38
|
+
it "should process all requested indices" do
|
39
39
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:`) do |c|
|
40
40
|
c.should match(/foo_delta bar_delta/)
|
41
41
|
end
|
@@ -59,7 +59,7 @@ describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
|
|
59
59
|
)
|
60
60
|
end
|
61
61
|
|
62
|
-
it "should not start the indexer for multiple
|
62
|
+
it "should not start the indexer for multiple indices" do
|
63
63
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_not_receive(:`)
|
64
64
|
ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
|
65
65
|
['bar_delta', 'foo_delta']
|
@@ -29,7 +29,7 @@ describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob do
|
|
29
29
|
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob.perform(['foo_core'], 12)
|
30
30
|
end
|
31
31
|
|
32
|
-
it "should update all specified
|
32
|
+
it "should update all specified indices" do
|
33
33
|
@client.should_receive(:update).with('foo_core', anything, anything)
|
34
34
|
@client.should_receive(:update).with('bar_core', anything, anything)
|
35
35
|
ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob.perform(['foo_core', 'bar_core'], 12)
|
data/ts-resque-delta.gemspec
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ts-resque-delta
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 25
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 1
|
8
8
|
- 1
|
9
|
-
-
|
10
|
-
version: 1.1.
|
9
|
+
- 5
|
10
|
+
version: 1.1.5
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Aaron Gibralter
|
@@ -15,12 +15,10 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-11-09 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
|
-
|
22
|
-
prerelease: false
|
23
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
21
|
+
version_requirements: &id001 !ruby/object:Gem::Requirement
|
24
22
|
none: false
|
25
23
|
requirements:
|
26
24
|
- - ">="
|
@@ -31,12 +29,27 @@ dependencies:
|
|
31
29
|
- 4
|
32
30
|
- 1
|
33
31
|
version: 1.4.1
|
32
|
+
requirement: *id001
|
33
|
+
prerelease: false
|
34
|
+
name: thinking-sphinx
|
34
35
|
type: :runtime
|
35
|
-
version_requirements: *id001
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
|
-
|
37
|
+
version_requirements: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
hash: 5
|
43
|
+
segments:
|
44
|
+
- 1
|
45
|
+
- 5
|
46
|
+
version: "1.5"
|
47
|
+
requirement: *id002
|
38
48
|
prerelease: false
|
39
|
-
|
49
|
+
name: riddle
|
50
|
+
type: :runtime
|
51
|
+
- !ruby/object:Gem::Dependency
|
52
|
+
version_requirements: &id003 !ruby/object:Gem::Requirement
|
40
53
|
none: false
|
41
54
|
requirements:
|
42
55
|
- - ~>
|
@@ -46,12 +59,12 @@ dependencies:
|
|
46
59
|
- 1
|
47
60
|
- 10
|
48
61
|
version: "1.10"
|
62
|
+
requirement: *id003
|
63
|
+
prerelease: false
|
64
|
+
name: resque
|
49
65
|
type: :runtime
|
50
|
-
version_requirements: *id002
|
51
66
|
- !ruby/object:Gem::Dependency
|
52
|
-
|
53
|
-
prerelease: false
|
54
|
-
requirement: &id003 !ruby/object:Gem::Requirement
|
67
|
+
version_requirements: &id004 !ruby/object:Gem::Requirement
|
55
68
|
none: false
|
56
69
|
requirements:
|
57
70
|
- - ~>
|
@@ -62,12 +75,12 @@ dependencies:
|
|
62
75
|
- 3
|
63
76
|
- 1
|
64
77
|
version: 0.3.1
|
78
|
+
requirement: *id004
|
79
|
+
prerelease: false
|
80
|
+
name: resque-lock-timeout
|
65
81
|
type: :runtime
|
66
|
-
version_requirements: *id003
|
67
82
|
- !ruby/object:Gem::Dependency
|
68
|
-
|
69
|
-
prerelease: false
|
70
|
-
requirement: &id004 !ruby/object:Gem::Requirement
|
83
|
+
version_requirements: &id005 !ruby/object:Gem::Requirement
|
71
84
|
none: false
|
72
85
|
requirements:
|
73
86
|
- - ~>
|
@@ -77,12 +90,12 @@ dependencies:
|
|
77
90
|
- 1
|
78
91
|
- 0
|
79
92
|
version: "1.0"
|
93
|
+
requirement: *id005
|
94
|
+
prerelease: false
|
95
|
+
name: rspec
|
80
96
|
type: :development
|
81
|
-
version_requirements: *id004
|
82
97
|
- !ruby/object:Gem::Dependency
|
83
|
-
|
84
|
-
prerelease: false
|
85
|
-
requirement: &id005 !ruby/object:Gem::Requirement
|
98
|
+
version_requirements: &id006 !ruby/object:Gem::Requirement
|
86
99
|
none: false
|
87
100
|
requirements:
|
88
101
|
- - ">="
|
@@ -91,12 +104,12 @@ dependencies:
|
|
91
104
|
segments:
|
92
105
|
- 0
|
93
106
|
version: "0"
|
107
|
+
requirement: *id006
|
108
|
+
prerelease: false
|
109
|
+
name: cucumber
|
94
110
|
type: :development
|
95
|
-
version_requirements: *id005
|
96
111
|
- !ruby/object:Gem::Dependency
|
97
|
-
|
98
|
-
prerelease: false
|
99
|
-
requirement: &id006 !ruby/object:Gem::Requirement
|
112
|
+
version_requirements: &id007 !ruby/object:Gem::Requirement
|
100
113
|
none: false
|
101
114
|
requirements:
|
102
115
|
- - ">="
|
@@ -107,12 +120,12 @@ dependencies:
|
|
107
120
|
- 5
|
108
121
|
- 2
|
109
122
|
version: 0.5.2
|
123
|
+
requirement: *id007
|
124
|
+
prerelease: false
|
125
|
+
name: database_cleaner
|
110
126
|
type: :development
|
111
|
-
version_requirements: *id006
|
112
127
|
- !ruby/object:Gem::Dependency
|
113
|
-
|
114
|
-
prerelease: false
|
115
|
-
requirement: &id007 !ruby/object:Gem::Requirement
|
128
|
+
version_requirements: &id008 !ruby/object:Gem::Requirement
|
116
129
|
none: false
|
117
130
|
requirements:
|
118
131
|
- - ~>
|
@@ -123,12 +136,12 @@ dependencies:
|
|
123
136
|
- 2
|
124
137
|
- 7
|
125
138
|
version: 0.2.7
|
139
|
+
requirement: *id008
|
140
|
+
prerelease: false
|
141
|
+
name: mysql2
|
126
142
|
type: :development
|
127
|
-
version_requirements: *id007
|
128
143
|
- !ruby/object:Gem::Dependency
|
129
|
-
|
130
|
-
prerelease: false
|
131
|
-
requirement: &id008 !ruby/object:Gem::Requirement
|
144
|
+
version_requirements: &id009 !ruby/object:Gem::Requirement
|
132
145
|
none: false
|
133
146
|
requirements:
|
134
147
|
- - "="
|
@@ -139,12 +152,12 @@ dependencies:
|
|
139
152
|
- 8
|
140
153
|
- 7
|
141
154
|
version: 0.8.7
|
155
|
+
requirement: *id009
|
156
|
+
prerelease: false
|
157
|
+
name: rake
|
142
158
|
type: :development
|
143
|
-
version_requirements: *id008
|
144
159
|
- !ruby/object:Gem::Dependency
|
145
|
-
|
146
|
-
prerelease: false
|
147
|
-
requirement: &id009 !ruby/object:Gem::Requirement
|
160
|
+
version_requirements: &id010 !ruby/object:Gem::Requirement
|
148
161
|
none: false
|
149
162
|
requirements:
|
150
163
|
- - ~>
|
@@ -155,12 +168,12 @@ dependencies:
|
|
155
168
|
- 3
|
156
169
|
- 11
|
157
170
|
version: 2.3.11
|
171
|
+
requirement: *id010
|
172
|
+
prerelease: false
|
173
|
+
name: activerecord
|
158
174
|
type: :development
|
159
|
-
version_requirements: *id009
|
160
175
|
- !ruby/object:Gem::Dependency
|
161
|
-
|
162
|
-
prerelease: false
|
163
|
-
requirement: &id010 !ruby/object:Gem::Requirement
|
176
|
+
version_requirements: &id011 !ruby/object:Gem::Requirement
|
164
177
|
none: false
|
165
178
|
requirements:
|
166
179
|
- - ">="
|
@@ -171,8 +184,10 @@ dependencies:
|
|
171
184
|
- 5
|
172
185
|
- 1
|
173
186
|
version: 0.5.1
|
187
|
+
requirement: *id011
|
188
|
+
prerelease: false
|
189
|
+
name: flying-sphinx
|
174
190
|
type: :development
|
175
|
-
version_requirements: *id010
|
176
191
|
description: Manage delta indexes via Resque for Thinking Sphinx
|
177
192
|
email:
|
178
193
|
- aaron.gibralter@gmail.com
|