thinking-sphinx 1.4.14 → 1.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (34) hide show
  1. data/features/attribute_updates.feature +15 -13
  2. data/features/deleting_instances.feature +16 -13
  3. data/features/handling_edits.feature +20 -17
  4. data/features/searching_by_index.feature +6 -5
  5. data/features/step_definitions/common_steps.rb +8 -4
  6. data/features/support/env.rb +0 -3
  7. data/lib/thinking_sphinx.rb +8 -1
  8. data/lib/thinking_sphinx/active_record.rb +3 -3
  9. data/lib/thinking_sphinx/active_record/attribute_updates.rb +5 -4
  10. data/lib/thinking_sphinx/adapters/abstract_adapter.rb +7 -0
  11. data/lib/thinking_sphinx/auto_version.rb +1 -1
  12. data/lib/thinking_sphinx/bundled_search.rb +6 -10
  13. data/lib/thinking_sphinx/configuration.rb +19 -33
  14. data/lib/thinking_sphinx/connection.rb +71 -0
  15. data/lib/thinking_sphinx/deltas.rb +2 -0
  16. data/lib/thinking_sphinx/deltas/default_delta.rb +14 -18
  17. data/lib/thinking_sphinx/deltas/delete_job.rb +16 -0
  18. data/lib/thinking_sphinx/deltas/index_job.rb +17 -0
  19. data/lib/thinking_sphinx/search.rb +26 -14
  20. data/lib/thinking_sphinx/tasks.rb +1 -5
  21. data/spec/spec_helper.rb +0 -3
  22. data/spec/thinking_sphinx/active_record/delta_spec.rb +6 -5
  23. data/spec/thinking_sphinx/active_record/scopes_spec.rb +2 -1
  24. data/spec/thinking_sphinx/active_record_spec.rb +2 -2
  25. data/spec/thinking_sphinx/adapters/abstract_adapter_spec.rb +18 -0
  26. data/spec/thinking_sphinx/configuration_spec.rb +0 -68
  27. data/spec/thinking_sphinx/connection_spec.rb +77 -0
  28. data/spec/thinking_sphinx/facet_search_spec.rb +25 -25
  29. data/spec/thinking_sphinx/search_methods_spec.rb +34 -34
  30. data/spec/thinking_sphinx/search_spec.rb +4 -16
  31. metadata +48 -39
  32. data/lib/thinking_sphinx/version.rb +0 -3
  33. data/rails/init.rb +0 -16
  34. data/tasks/rails.rake +0 -1
@@ -14,7 +14,7 @@ module ThinkingSphinx
14
14
  else
15
15
  documentation_link = %Q{
16
16
  For more information, read the documentation:
17
- http://pat.github.com/ts/en/advanced_config.html
17
+ http://pat.github.io/thinking-sphinx/advanced_config.html
18
18
  }
19
19
 
20
20
  if version.nil? || version.empty?
@@ -1,19 +1,15 @@
1
1
  module ThinkingSphinx
2
2
  class BundledSearch
3
- attr_reader :client
4
-
5
3
  def initialize
6
4
  @searches = []
7
5
  end
8
6
 
9
7
  def search(*args)
10
8
  @searches << ThinkingSphinx.search(*args)
11
- @searches.last.append_to client
12
9
  end
13
10
 
14
11
  def search_for_ids(*args)
15
12
  @searches << ThinkingSphinx.search_for_ids(*args)
16
- @searches.last.append_to client
17
13
  end
18
14
 
19
15
  def searches
@@ -23,10 +19,6 @@ module ThinkingSphinx
23
19
 
24
20
  private
25
21
 
26
- def client
27
- @client ||= ThinkingSphinx::Configuration.instance.client
28
- end
29
-
30
22
  def populated?
31
23
  @populated
32
24
  end
@@ -36,8 +28,12 @@ module ThinkingSphinx
36
28
 
37
29
  @populated = true
38
30
 
39
- client.run.each_with_index do |results, index|
40
- searches[index].populate_from_queue results
31
+ ThinkingSphinx::Connection.take do |client|
32
+ @searches.each { |search| search.append_to client }
33
+
34
+ client.run.each_with_index do |results, index|
35
+ searches[index].populate_from_queue results
36
+ end
41
37
  end
42
38
  end
43
39
  end
@@ -69,8 +69,8 @@ module ThinkingSphinx
69
69
  :hard_retry_count
70
70
 
71
71
  attr_accessor :source_options, :index_options
72
-
73
- attr_reader :configuration, :controller
72
+ attr_reader :configuration
73
+ attr_writer :controller
74
74
 
75
75
  @@environment = nil
76
76
 
@@ -96,14 +96,12 @@ module ThinkingSphinx
96
96
  self.app_root ||= app_root
97
97
  end
98
98
 
99
+ @controller = nil
99
100
  @configuration = Riddle::Configuration.new
100
101
  @configuration.searchd.pid_file = "#{self.app_root}/log/searchd.#{environment}.pid"
101
102
  @configuration.searchd.log = "#{self.app_root}/log/searchd.log"
102
103
  @configuration.searchd.query_log = "#{self.app_root}/log/searchd.query.log"
103
104
 
104
- @controller = Riddle::Controller.new @configuration,
105
- "#{self.app_root}/config/#{environment}.sphinx.conf"
106
-
107
105
  self.address = "127.0.0.1"
108
106
  self.port = 9312
109
107
  self.searchd_file_path = "#{self.app_root}/db/sphinx/#{environment}"
@@ -123,7 +121,9 @@ module ThinkingSphinx
123
121
 
124
122
  self.version = nil
125
123
  parse_config
126
- self.version ||= @controller.sphinx_version
124
+ if controller.respond_to?(:sphinx_version)
125
+ self.version ||= controller.sphinx_version
126
+ end
127
127
 
128
128
  ThinkingSphinx::Attribute::SphinxTypeMappings.merge!(
129
129
  :string => :sql_attr_string
@@ -154,6 +154,11 @@ module ThinkingSphinx
154
154
  self.class.environment
155
155
  end
156
156
 
157
+ def controller
158
+ @controller ||= Riddle::Controller.new @configuration,
159
+ "#{self.app_root}/config/#{environment}.sphinx.conf"
160
+ end
161
+
157
162
  def generate
158
163
  @configuration.indices.clear
159
164
 
@@ -231,47 +236,39 @@ module ThinkingSphinx
231
236
  end
232
237
 
233
238
  def config_file
234
- @controller.path
239
+ controller.path
235
240
  end
236
241
 
237
242
  def config_file=(file)
238
- @controller.path = file
243
+ controller.path = file
239
244
  end
240
245
 
241
246
  def bin_path
242
- @controller.bin_path
247
+ controller.bin_path
243
248
  end
244
249
 
245
250
  def bin_path=(path)
246
- @controller.bin_path = path
251
+ controller.bin_path = path
247
252
  end
248
253
 
249
254
  def searchd_binary_name
250
- @controller.searchd_binary_name
255
+ controller.searchd_binary_name
251
256
  end
252
257
 
253
258
  def searchd_binary_name=(name)
254
- @controller.searchd_binary_name = name
259
+ controller.searchd_binary_name = name
255
260
  end
256
261
 
257
262
  def indexer_binary_name
258
- @controller.indexer_binary_name
263
+ controller.indexer_binary_name
259
264
  end
260
265
 
261
266
  def indexer_binary_name=(name)
262
- @controller.indexer_binary_name = name
267
+ controller.indexer_binary_name = name
263
268
  end
264
269
 
265
270
  attr_accessor :timeout
266
271
 
267
- def client
268
- client = Riddle::Client.new shuffled_addresses, port,
269
- configuration.searchd.client_key
270
- client.max_matches = configuration.searchd.max_matches || 1000
271
- client.timeout = timeout || 0
272
- client
273
- end
274
-
275
272
  def models_by_crc
276
273
  @models_by_crc ||= begin
277
274
  ThinkingSphinx.context.indexed_models.inject({}) do |hash, model|
@@ -347,16 +344,5 @@ module ThinkingSphinx
347
344
  }
348
345
  }
349
346
  end
350
-
351
- def shuffled_addresses
352
- return address unless shuffle
353
-
354
- addresses = Array(address)
355
- if addresses.respond_to?(:shuffle)
356
- addresses.shuffle
357
- else
358
- address.sort_by { rand }
359
- end
360
- end
361
347
  end
362
348
  end
@@ -0,0 +1,71 @@
1
+ class ThinkingSphinx::Connection
2
+ def self.pool
3
+ @pool ||= Innertube::Pool.new(
4
+ Proc.new { ThinkingSphinx::Connection.new },
5
+ Proc.new { |connection| connection.close }
6
+ )
7
+ end
8
+
9
+ def self.take
10
+ retries = 0
11
+ original = nil
12
+ begin
13
+ pool.take do |connection|
14
+ connection.reset
15
+ begin
16
+ yield connection
17
+ rescue Riddle::ConnectionError, Riddle::ResponseError, SystemCallError => error
18
+ original = error
19
+ raise Innertube::Pool::BadResource
20
+ end
21
+ end
22
+ rescue Innertube::Pool::BadResource
23
+ retries += 1
24
+ retry if retries < 3
25
+ raise original
26
+ end
27
+ end
28
+
29
+ def initialize
30
+ client.open
31
+ end
32
+
33
+ def client
34
+ @client ||= begin
35
+ client = Riddle::Client.new shuffled_addresses, configuration.port,
36
+ client_key
37
+ client.max_matches = _max_matches
38
+ client.timeout = configuration.timeout || 0
39
+ client
40
+ end
41
+ end
42
+
43
+ private
44
+
45
+ def client_key
46
+ configuration.configuration.searchd.client_key
47
+ end
48
+
49
+ def configuration
50
+ ThinkingSphinx::Configuration.instance
51
+ end
52
+
53
+ def _max_matches
54
+ configuration.configuration.searchd.max_matches || 1000
55
+ end
56
+
57
+ def method_missing(method, *arguments, &block)
58
+ client.send method, *arguments, &block
59
+ end
60
+
61
+ def shuffled_addresses
62
+ return configuration.address unless configuration.shuffle
63
+
64
+ addresses = Array(configuration.address)
65
+ if addresses.respond_to?(:shuffle)
66
+ addresses.shuffle
67
+ else
68
+ address.sort_by { rand }
69
+ end
70
+ end
71
+ end
@@ -1,4 +1,6 @@
1
1
  require 'thinking_sphinx/deltas/default_delta'
2
+ require 'thinking_sphinx/deltas/delete_job'
3
+ require 'thinking_sphinx/deltas/index_job'
2
4
 
3
5
  module ThinkingSphinx
4
6
  module Deltas
@@ -2,23 +2,23 @@ module ThinkingSphinx
2
2
  module Deltas
3
3
  class DefaultDelta
4
4
  attr_accessor :column
5
-
5
+
6
6
  def initialize(index, options)
7
7
  @index = index
8
8
  @column = options.delete(:delta_column) || :delta
9
9
  end
10
-
10
+
11
11
  def index(model, instance = nil)
12
12
  return true unless ThinkingSphinx.updates_enabled? &&
13
13
  ThinkingSphinx.deltas_enabled?
14
14
  return true if instance && !toggled(instance)
15
-
15
+
16
16
  update_delta_indexes model
17
17
  delete_from_core model, instance if instance
18
-
18
+
19
19
  true
20
20
  end
21
-
21
+
22
22
  def toggle(instance)
23
23
  instance.send "#{@column}=", true
24
24
  end
@@ -32,28 +32,24 @@ module ThinkingSphinx
32
32
  "#{model.connection.quote_column_name(@column.to_s)} = #{adapter.boolean(false)} " +
33
33
  "WHERE #{model.connection.quote_column_name(@column.to_s)} = #{adapter.boolean(true)}"
34
34
  end
35
-
35
+
36
36
  def clause(model, toggled)
37
37
  "#{model.quoted_table_name}.#{model.connection.quote_column_name(@column.to_s)}" +
38
38
  " = #{adapter.boolean(toggled)}"
39
39
  end
40
-
40
+
41
41
  private
42
-
42
+
43
43
  def update_delta_indexes(model)
44
- config = ThinkingSphinx::Configuration.instance
45
- rotate = ThinkingSphinx.sphinx_running? ? "--rotate" : ""
46
-
47
- output = `#{config.bin_path}#{config.indexer_binary_name} --config "#{config.config_file}" #{rotate} #{model.delta_index_names.join(' ')}`
48
- puts(output) unless ThinkingSphinx.suppress_delta_output?
44
+ ThinkingSphinx::Deltas::IndexJob.new(model.delta_index_names).perform
49
45
  end
50
-
46
+
51
47
  def delete_from_core(model, instance)
52
- model.core_index_names.each do |index_name|
53
- model.delete_in_index index_name, instance.sphinx_document_id
54
- end
48
+ ThinkingSphinx::Deltas::DeleteJob.new(
49
+ model.core_index_names, instance.sphinx_document_id
50
+ ).perform
55
51
  end
56
-
52
+
57
53
  def adapter
58
54
  @adapter = @index.model.sphinx_database_adapter
59
55
  end
@@ -0,0 +1,16 @@
1
+ class ThinkingSphinx::Deltas::DeleteJob
2
+ def initialize(indices, document_id)
3
+ @indices, @document_id = indices, document_id
4
+ end
5
+
6
+ def perform
7
+ ThinkingSphinx::Connection.take do |client|
8
+ @indices.each do |index|
9
+ client.update(index, ['sphinx_deleted'], {@document_id => [1]})
10
+ end
11
+ end
12
+ rescue Riddle::ConnectionError, Riddle::ResponseError,
13
+ ThinkingSphinx::SphinxError, Errno::ETIMEDOUT, Timeout::Error
14
+ # Not the end of the world if Sphinx isn't running.
15
+ end
16
+ end
@@ -0,0 +1,17 @@
1
+ class ThinkingSphinx::Deltas::IndexJob
2
+ def initialize(indices)
3
+ @indices = indices
4
+ @indices << {:verbose => !ThinkingSphinx.suppress_delta_output?}
5
+ end
6
+
7
+ def perform
8
+ ThinkingSphinx::Configuration.instance.controller.index *@indices
9
+ ThinkingSphinx::Connection.pool.clear
10
+
11
+ true
12
+ end
13
+
14
+ def configuration
15
+ ThinkingSphinx::Configuration.instance
16
+ end
17
+ end
@@ -344,13 +344,15 @@ module ThinkingSphinx
344
344
  populate
345
345
 
346
346
  index = options[:index] || "#{model.core_index_names.first}"
347
- client.excerpts(
348
- {
349
- :docs => [string.to_s],
350
- :words => query,
351
- :index => index.split(',').first.strip
352
- }.merge(options[:excerpt_options] || {})
353
- ).first
347
+ take_client do |client|
348
+ client.excerpts(
349
+ {
350
+ :docs => [string.to_s],
351
+ :words => query,
352
+ :index => index.split(',').first.strip
353
+ }.merge(options[:excerpt_options] || {})
354
+ ).first
355
+ end
354
356
  end
355
357
 
356
358
  def search(*args)
@@ -376,10 +378,16 @@ module ThinkingSphinx
376
378
  ThinkingSphinx::FacetSearch.new(*args)
377
379
  end
378
380
 
379
- def client
380
- client = options[:client] || config.client
381
-
382
- prepare client
381
+ def take_client
382
+ if options[:client]
383
+ prepare options[:client]
384
+ yield options[:client]
385
+ else
386
+ ThinkingSphinx::Connection.take do |client|
387
+ prepare client
388
+ yield client
389
+ end
390
+ end
383
391
  end
384
392
 
385
393
  def append_to(client)
@@ -412,7 +420,10 @@ module ThinkingSphinx
412
420
  begin
413
421
  log "Querying: '#{query}'"
414
422
  runtime = Benchmark.realtime {
415
- @results = client.query query, indexes, comment
423
+ @results = nil
424
+ take_client do |client|
425
+ @results = client.query query, indexes, comment
426
+ end
416
427
  }
417
428
  log "Found #{@results[:total_found]} results", :debug,
418
429
  "Sphinx (#{sprintf("%f", runtime)}s)"
@@ -454,7 +465,7 @@ module ThinkingSphinx
454
465
  replace instances_from_matches
455
466
  add_excerpter
456
467
  add_sphinx_attributes
457
- add_matching_fields if client.rank_mode == :fieldmask
468
+ add_matching_fields if options[:rank_mode] == :fieldmask
458
469
  end
459
470
  end
460
471
 
@@ -629,7 +640,8 @@ module ThinkingSphinx
629
640
  return '' if @options[:conditions].blank?
630
641
 
631
642
  ' ' + @options[:conditions].keys.collect { |key|
632
- "@#{key} #{options[:conditions][key]}"
643
+ search_key = key.is_a?(::Array) ? "(#{key.join(',')})" : key
644
+ "@#{search_key} #{options[:conditions][key]}"
633
645
  }.join(' ')
634
646
  end
635
647
 
@@ -17,11 +17,6 @@ namespace :thinking_sphinx do
17
17
  end
18
18
  end
19
19
 
20
- desc "Output the current Thinking Sphinx version"
21
- task :version => :app_env do
22
- puts "Thinking Sphinx v" + ThinkingSphinx::Version
23
- end
24
-
25
20
  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
26
21
  task :running_start => :app_env do
27
22
  Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
@@ -93,6 +88,7 @@ namespace :thinking_sphinx do
93
88
  end
94
89
 
95
90
  FileUtils.mkdir_p config.searchd_file_path
91
+ ThinkingSphinx.before_index_hooks.each { |hook| hook.call }
96
92
  config.controller.index :verbose => true
97
93
  end
98
94