thinking-sphinx 2.0.14 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/HISTORY +17 -1
- data/features/attribute_updates.feature +15 -13
- data/features/deleting_instances.feature +16 -13
- data/features/handling_edits.feature +20 -17
- data/features/searching_by_index.feature +6 -5
- data/features/step_definitions/common_steps.rb +4 -0
- data/features/support/env.rb +0 -3
- data/lib/thinking_sphinx.rb +8 -1
- data/lib/thinking_sphinx/active_record.rb +3 -3
- data/lib/thinking_sphinx/active_record/attribute_updates.rb +5 -4
- data/lib/thinking_sphinx/adapters/abstract_adapter.rb +7 -0
- data/lib/thinking_sphinx/auto_version.rb +1 -1
- data/lib/thinking_sphinx/bundled_search.rb +6 -10
- data/lib/thinking_sphinx/configuration.rb +19 -33
- data/lib/thinking_sphinx/connection.rb +71 -0
- data/lib/thinking_sphinx/deltas.rb +2 -0
- data/lib/thinking_sphinx/deltas/default_delta.rb +14 -18
- data/lib/thinking_sphinx/deltas/delete_job.rb +16 -0
- data/lib/thinking_sphinx/deltas/index_job.rb +17 -0
- data/lib/thinking_sphinx/search.rb +26 -15
- data/lib/thinking_sphinx/tasks.rb +1 -5
- data/spec/spec_helper.rb +0 -3
- data/spec/thinking_sphinx/active_record/delta_spec.rb +6 -5
- data/spec/thinking_sphinx/active_record/scopes_spec.rb +2 -1
- data/spec/thinking_sphinx/active_record_spec.rb +2 -2
- data/spec/thinking_sphinx/adapters/abstract_adapter_spec.rb +18 -0
- data/spec/thinking_sphinx/configuration_spec.rb +0 -68
- data/spec/thinking_sphinx/connection_spec.rb +77 -0
- data/spec/thinking_sphinx/facet_search_spec.rb +25 -25
- data/spec/thinking_sphinx/search_methods_spec.rb +34 -34
- data/spec/thinking_sphinx/search_spec.rb +4 -16
- metadata +197 -186
- data/lib/thinking_sphinx/version.rb +0 -3
@@ -1,19 +1,15 @@
|
|
1
1
|
module ThinkingSphinx
|
2
2
|
class BundledSearch
|
3
|
-
attr_reader :client
|
4
|
-
|
5
3
|
def initialize
|
6
4
|
@searches = []
|
7
5
|
end
|
8
6
|
|
9
7
|
def search(*args)
|
10
8
|
@searches << ThinkingSphinx.search(*args)
|
11
|
-
@searches.last.append_to client
|
12
9
|
end
|
13
10
|
|
14
11
|
def search_for_ids(*args)
|
15
12
|
@searches << ThinkingSphinx.search_for_ids(*args)
|
16
|
-
@searches.last.append_to client
|
17
13
|
end
|
18
14
|
|
19
15
|
def searches
|
@@ -23,10 +19,6 @@ module ThinkingSphinx
|
|
23
19
|
|
24
20
|
private
|
25
21
|
|
26
|
-
def client
|
27
|
-
@client ||= ThinkingSphinx::Configuration.instance.client
|
28
|
-
end
|
29
|
-
|
30
22
|
def populated?
|
31
23
|
@populated
|
32
24
|
end
|
@@ -36,8 +28,12 @@ module ThinkingSphinx
|
|
36
28
|
|
37
29
|
@populated = true
|
38
30
|
|
39
|
-
|
40
|
-
searches
|
31
|
+
ThinkingSphinx::Connection.take do |client|
|
32
|
+
@searches.each { |search| search.append_to client }
|
33
|
+
|
34
|
+
client.run.each_with_index do |results, index|
|
35
|
+
searches[index].populate_from_queue results
|
36
|
+
end
|
41
37
|
end
|
42
38
|
end
|
43
39
|
end
|
@@ -69,8 +69,8 @@ module ThinkingSphinx
|
|
69
69
|
:hard_retry_count
|
70
70
|
|
71
71
|
attr_accessor :source_options, :index_options
|
72
|
-
|
73
|
-
|
72
|
+
attr_reader :configuration
|
73
|
+
attr_writer :controller
|
74
74
|
|
75
75
|
@@environment = nil
|
76
76
|
|
@@ -96,14 +96,12 @@ module ThinkingSphinx
|
|
96
96
|
self.app_root ||= app_root
|
97
97
|
end
|
98
98
|
|
99
|
+
@controller = nil
|
99
100
|
@configuration = Riddle::Configuration.new
|
100
101
|
@configuration.searchd.pid_file = "#{self.app_root}/log/searchd.#{environment}.pid"
|
101
102
|
@configuration.searchd.log = "#{self.app_root}/log/searchd.log"
|
102
103
|
@configuration.searchd.query_log = "#{self.app_root}/log/searchd.query.log"
|
103
104
|
|
104
|
-
@controller = Riddle::Controller.new @configuration,
|
105
|
-
"#{self.app_root}/config/#{environment}.sphinx.conf"
|
106
|
-
|
107
105
|
self.address = "127.0.0.1"
|
108
106
|
self.port = 9312
|
109
107
|
self.searchd_file_path = "#{self.app_root}/db/sphinx/#{environment}"
|
@@ -122,7 +120,9 @@ module ThinkingSphinx
|
|
122
120
|
|
123
121
|
self.version = nil
|
124
122
|
parse_config
|
125
|
-
|
123
|
+
if controller.respond_to?(:sphinx_version)
|
124
|
+
self.version ||= controller.sphinx_version
|
125
|
+
end
|
126
126
|
|
127
127
|
ThinkingSphinx::Attribute::SphinxTypeMappings.merge!(
|
128
128
|
:string => :sql_attr_string
|
@@ -153,6 +153,11 @@ module ThinkingSphinx
|
|
153
153
|
self.class.environment
|
154
154
|
end
|
155
155
|
|
156
|
+
def controller
|
157
|
+
@controller ||= Riddle::Controller.new @configuration,
|
158
|
+
"#{self.app_root}/config/#{environment}.sphinx.conf"
|
159
|
+
end
|
160
|
+
|
156
161
|
def generate
|
157
162
|
@configuration.indices.clear
|
158
163
|
|
@@ -230,47 +235,39 @@ module ThinkingSphinx
|
|
230
235
|
end
|
231
236
|
|
232
237
|
def config_file
|
233
|
-
|
238
|
+
controller.path
|
234
239
|
end
|
235
240
|
|
236
241
|
def config_file=(file)
|
237
|
-
|
242
|
+
controller.path = file
|
238
243
|
end
|
239
244
|
|
240
245
|
def bin_path
|
241
|
-
|
246
|
+
controller.bin_path
|
242
247
|
end
|
243
248
|
|
244
249
|
def bin_path=(path)
|
245
|
-
|
250
|
+
controller.bin_path = path
|
246
251
|
end
|
247
252
|
|
248
253
|
def searchd_binary_name
|
249
|
-
|
254
|
+
controller.searchd_binary_name
|
250
255
|
end
|
251
256
|
|
252
257
|
def searchd_binary_name=(name)
|
253
|
-
|
258
|
+
controller.searchd_binary_name = name
|
254
259
|
end
|
255
260
|
|
256
261
|
def indexer_binary_name
|
257
|
-
|
262
|
+
controller.indexer_binary_name
|
258
263
|
end
|
259
264
|
|
260
265
|
def indexer_binary_name=(name)
|
261
|
-
|
266
|
+
controller.indexer_binary_name = name
|
262
267
|
end
|
263
268
|
|
264
269
|
attr_accessor :timeout
|
265
270
|
|
266
|
-
def client
|
267
|
-
client = Riddle::Client.new shuffled_addresses, port,
|
268
|
-
configuration.searchd.client_key
|
269
|
-
client.max_matches = configuration.searchd.max_matches || 1000
|
270
|
-
client.timeout = timeout || 0
|
271
|
-
client
|
272
|
-
end
|
273
|
-
|
274
271
|
def models_by_crc
|
275
272
|
@models_by_crc ||= begin
|
276
273
|
ThinkingSphinx.context.indexed_models.inject({}) do |hash, model|
|
@@ -347,17 +344,6 @@ module ThinkingSphinx
|
|
347
344
|
}
|
348
345
|
end
|
349
346
|
|
350
|
-
def shuffled_addresses
|
351
|
-
return address unless shuffle
|
352
|
-
|
353
|
-
addresses = Array(address)
|
354
|
-
if addresses.respond_to?(:shuffle)
|
355
|
-
addresses.shuffle
|
356
|
-
else
|
357
|
-
address.sort_by { rand }
|
358
|
-
end
|
359
|
-
end
|
360
|
-
|
361
347
|
def initial_model_directories
|
362
348
|
directories = ["#{app_root}/app/models/"] +
|
363
349
|
Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
|
@@ -0,0 +1,71 @@
|
|
1
|
+
class ThinkingSphinx::Connection
|
2
|
+
def self.pool
|
3
|
+
@pool ||= Innertube::Pool.new(
|
4
|
+
Proc.new { ThinkingSphinx::Connection.new },
|
5
|
+
Proc.new { |connection| connection.close }
|
6
|
+
)
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.take
|
10
|
+
retries = 0
|
11
|
+
original = nil
|
12
|
+
begin
|
13
|
+
pool.take do |connection|
|
14
|
+
connection.reset
|
15
|
+
begin
|
16
|
+
yield connection
|
17
|
+
rescue Riddle::ConnectionError, Riddle::ResponseError, SystemCallError => error
|
18
|
+
original = error
|
19
|
+
raise Innertube::Pool::BadResource
|
20
|
+
end
|
21
|
+
end
|
22
|
+
rescue Innertube::Pool::BadResource
|
23
|
+
retries += 1
|
24
|
+
retry if retries < 3
|
25
|
+
raise original
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def initialize
|
30
|
+
client.open
|
31
|
+
end
|
32
|
+
|
33
|
+
def client
|
34
|
+
@client ||= begin
|
35
|
+
client = Riddle::Client.new shuffled_addresses, configuration.port,
|
36
|
+
client_key
|
37
|
+
client.max_matches = _max_matches
|
38
|
+
client.timeout = configuration.timeout || 0
|
39
|
+
client
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def client_key
|
46
|
+
configuration.configuration.searchd.client_key
|
47
|
+
end
|
48
|
+
|
49
|
+
def configuration
|
50
|
+
ThinkingSphinx::Configuration.instance
|
51
|
+
end
|
52
|
+
|
53
|
+
def _max_matches
|
54
|
+
configuration.configuration.searchd.max_matches || 1000
|
55
|
+
end
|
56
|
+
|
57
|
+
def method_missing(method, *arguments, &block)
|
58
|
+
client.send method, *arguments, &block
|
59
|
+
end
|
60
|
+
|
61
|
+
def shuffled_addresses
|
62
|
+
return configuration.address unless configuration.shuffle
|
63
|
+
|
64
|
+
addresses = Array(configuration.address)
|
65
|
+
if addresses.respond_to?(:shuffle)
|
66
|
+
addresses.shuffle
|
67
|
+
else
|
68
|
+
address.sort_by { rand }
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -2,23 +2,23 @@ module ThinkingSphinx
|
|
2
2
|
module Deltas
|
3
3
|
class DefaultDelta
|
4
4
|
attr_accessor :column
|
5
|
-
|
5
|
+
|
6
6
|
def initialize(index, options)
|
7
7
|
@index = index
|
8
8
|
@column = options.delete(:delta_column) || :delta
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
11
|
def index(model, instance = nil)
|
12
12
|
return true unless ThinkingSphinx.updates_enabled? &&
|
13
13
|
ThinkingSphinx.deltas_enabled?
|
14
14
|
return true if instance && !toggled(instance)
|
15
|
-
|
15
|
+
|
16
16
|
update_delta_indexes model
|
17
17
|
delete_from_core model, instance if instance
|
18
|
-
|
18
|
+
|
19
19
|
true
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
def toggle(instance)
|
23
23
|
instance.send "#{@column}=", true
|
24
24
|
end
|
@@ -32,28 +32,24 @@ module ThinkingSphinx
|
|
32
32
|
"#{model.connection.quote_column_name(@column.to_s)} = #{adapter.boolean(false)} " +
|
33
33
|
"WHERE #{model.connection.quote_column_name(@column.to_s)} = #{adapter.boolean(true)}"
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
def clause(model, toggled)
|
37
37
|
"#{model.quoted_table_name}.#{model.connection.quote_column_name(@column.to_s)}" +
|
38
38
|
" = #{adapter.boolean(toggled)}"
|
39
39
|
end
|
40
|
-
|
40
|
+
|
41
41
|
private
|
42
|
-
|
42
|
+
|
43
43
|
def update_delta_indexes(model)
|
44
|
-
|
45
|
-
rotate = ThinkingSphinx.sphinx_running? ? "--rotate" : ""
|
46
|
-
|
47
|
-
output = `#{config.bin_path}#{config.indexer_binary_name} --config "#{config.config_file}" #{rotate} #{model.delta_index_names.join(' ')}`
|
48
|
-
puts(output) unless ThinkingSphinx.suppress_delta_output?
|
44
|
+
ThinkingSphinx::Deltas::IndexJob.new(model.delta_index_names).perform
|
49
45
|
end
|
50
|
-
|
46
|
+
|
51
47
|
def delete_from_core(model, instance)
|
52
|
-
|
53
|
-
model.
|
54
|
-
|
48
|
+
ThinkingSphinx::Deltas::DeleteJob.new(
|
49
|
+
model.core_index_names, instance.sphinx_document_id
|
50
|
+
).perform
|
55
51
|
end
|
56
|
-
|
52
|
+
|
57
53
|
def adapter
|
58
54
|
@adapter = @index.model.sphinx_database_adapter
|
59
55
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
class ThinkingSphinx::Deltas::DeleteJob
|
2
|
+
def initialize(indices, document_id)
|
3
|
+
@indices, @document_id = indices, document_id
|
4
|
+
end
|
5
|
+
|
6
|
+
def perform
|
7
|
+
ThinkingSphinx::Connection.take do |client|
|
8
|
+
@indices.each do |index|
|
9
|
+
client.update(index, ['sphinx_deleted'], {@document_id => [1]})
|
10
|
+
end
|
11
|
+
end
|
12
|
+
rescue Riddle::ConnectionError, Riddle::ResponseError,
|
13
|
+
ThinkingSphinx::SphinxError, Errno::ETIMEDOUT, Timeout::Error
|
14
|
+
# Not the end of the world if Sphinx isn't running.
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class ThinkingSphinx::Deltas::IndexJob
|
2
|
+
def initialize(indices)
|
3
|
+
@indices = indices
|
4
|
+
@indices << {:verbose => !ThinkingSphinx.suppress_delta_output?}
|
5
|
+
end
|
6
|
+
|
7
|
+
def perform
|
8
|
+
ThinkingSphinx::Configuration.instance.controller.index *@indices
|
9
|
+
ThinkingSphinx::Connection.pool.clear
|
10
|
+
|
11
|
+
true
|
12
|
+
end
|
13
|
+
|
14
|
+
def configuration
|
15
|
+
ThinkingSphinx::Configuration.instance
|
16
|
+
end
|
17
|
+
end
|
@@ -359,13 +359,15 @@ module ThinkingSphinx
|
|
359
359
|
populate
|
360
360
|
|
361
361
|
index = options[:index] || "#{model.core_index_names.first}"
|
362
|
-
client
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
362
|
+
take_client do |client|
|
363
|
+
client.excerpts(
|
364
|
+
{
|
365
|
+
:docs => [string.to_s],
|
366
|
+
:words => query,
|
367
|
+
:index => index.split(',').first.strip
|
368
|
+
}.merge(options[:excerpt_options] || {})
|
369
|
+
).first
|
370
|
+
end
|
369
371
|
end
|
370
372
|
|
371
373
|
def search(*args)
|
@@ -391,10 +393,16 @@ module ThinkingSphinx
|
|
391
393
|
ThinkingSphinx::FacetSearch.new(*args)
|
392
394
|
end
|
393
395
|
|
394
|
-
def
|
395
|
-
|
396
|
-
|
397
|
-
|
396
|
+
def take_client
|
397
|
+
if options[:client]
|
398
|
+
prepare options[:client]
|
399
|
+
yield options[:client]
|
400
|
+
else
|
401
|
+
ThinkingSphinx::Connection.take do |client|
|
402
|
+
prepare client
|
403
|
+
yield client
|
404
|
+
end
|
405
|
+
end
|
398
406
|
end
|
399
407
|
|
400
408
|
def append_to(client)
|
@@ -425,12 +433,14 @@ module ThinkingSphinx
|
|
425
433
|
begin
|
426
434
|
retry_on_stale_index do
|
427
435
|
begin
|
436
|
+
@results = nil
|
428
437
|
log query do
|
429
|
-
|
438
|
+
take_client do |client|
|
439
|
+
@results = client.query query, indexes, comment
|
440
|
+
end
|
430
441
|
end
|
431
442
|
total = @results[:total_found].to_i
|
432
443
|
log "Found #{total} result#{'s' unless total == 1}"
|
433
|
-
|
434
444
|
log "Sphinx Daemon returned warning: #{warning}" if warning?
|
435
445
|
|
436
446
|
if error?
|
@@ -468,7 +478,7 @@ module ThinkingSphinx
|
|
468
478
|
replace instances_from_matches
|
469
479
|
add_excerpter
|
470
480
|
add_sphinx_attributes
|
471
|
-
add_matching_fields if
|
481
|
+
add_matching_fields if options[:rank_mode] == :fieldmask
|
472
482
|
end
|
473
483
|
end
|
474
484
|
|
@@ -643,7 +653,8 @@ module ThinkingSphinx
|
|
643
653
|
return '' if @options[:conditions].blank?
|
644
654
|
|
645
655
|
' ' + @options[:conditions].keys.collect { |key|
|
646
|
-
"
|
656
|
+
search_key = key.is_a?(::Array) ? "(#{key.join(',')})" : key
|
657
|
+
"@#{search_key} #{options[:conditions][key]}"
|
647
658
|
}.join(' ')
|
648
659
|
end
|
649
660
|
|
@@ -13,11 +13,6 @@ namespace :thinking_sphinx do
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
-
desc "Output the current Thinking Sphinx version"
|
17
|
-
task :version => :app_env do
|
18
|
-
puts "Thinking Sphinx v" + ThinkingSphinx::Version
|
19
|
-
end
|
20
|
-
|
21
16
|
desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
|
22
17
|
task :running_start => :app_env do
|
23
18
|
Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
|
@@ -89,6 +84,7 @@ namespace :thinking_sphinx do
|
|
89
84
|
end
|
90
85
|
|
91
86
|
FileUtils.mkdir_p config.searchd_file_path
|
87
|
+
ThinkingSphinx.before_index_hooks.each { |hook| hook.call }
|
92
88
|
config.controller.index :verbose => true
|
93
89
|
end
|
94
90
|
|