thinking-sphinx 3.0.3 → 3.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +2 -1
  3. data/HISTORY +25 -0
  4. data/lib/thinking_sphinx.rb +1 -0
  5. data/lib/thinking_sphinx/active_record/association_proxy.rb +13 -55
  6. data/lib/thinking_sphinx/active_record/association_proxy/attribute_finder.rb +47 -0
  7. data/lib/thinking_sphinx/active_record/base.rb +16 -15
  8. data/lib/thinking_sphinx/active_record/callbacks/delete_callbacks.rb +1 -12
  9. data/lib/thinking_sphinx/active_record/database_adapters.rb +41 -42
  10. data/lib/thinking_sphinx/active_record/polymorpher.rb +7 -2
  11. data/lib/thinking_sphinx/active_record/property_query.rb +23 -19
  12. data/lib/thinking_sphinx/active_record/sql_builder.rb +108 -129
  13. data/lib/thinking_sphinx/active_record/sql_builder/clause_builder.rb +28 -0
  14. data/lib/thinking_sphinx/active_record/sql_builder/query.rb +43 -0
  15. data/lib/thinking_sphinx/active_record/sql_builder/statement.rb +110 -0
  16. data/lib/thinking_sphinx/active_record/sql_source.rb +143 -138
  17. data/lib/thinking_sphinx/capistrano.rb +11 -8
  18. data/lib/thinking_sphinx/configuration.rb +57 -35
  19. data/lib/thinking_sphinx/connection.rb +15 -6
  20. data/lib/thinking_sphinx/core.rb +1 -0
  21. data/lib/thinking_sphinx/core/index.rb +18 -10
  22. data/lib/thinking_sphinx/core/settings.rb +9 -0
  23. data/lib/thinking_sphinx/deletion.rb +48 -0
  24. data/lib/thinking_sphinx/errors.rb +7 -0
  25. data/lib/thinking_sphinx/excerpter.rb +1 -0
  26. data/lib/thinking_sphinx/facet_search.rb +42 -19
  27. data/lib/thinking_sphinx/masks/scopes_mask.rb +7 -0
  28. data/lib/thinking_sphinx/middlewares.rb +27 -33
  29. data/lib/thinking_sphinx/middlewares/active_record_translator.rb +18 -18
  30. data/lib/thinking_sphinx/middlewares/geographer.rb +49 -16
  31. data/lib/thinking_sphinx/middlewares/sphinxql.rb +128 -58
  32. data/lib/thinking_sphinx/panes/excerpts_pane.rb +7 -3
  33. data/lib/thinking_sphinx/rake_interface.rb +10 -0
  34. data/lib/thinking_sphinx/real_time.rb +7 -1
  35. data/lib/thinking_sphinx/real_time/attribute.rb +4 -0
  36. data/lib/thinking_sphinx/real_time/callbacks/real_time_callbacks.rb +14 -10
  37. data/lib/thinking_sphinx/real_time/index.rb +20 -12
  38. data/lib/thinking_sphinx/search/glaze.rb +5 -0
  39. data/lib/thinking_sphinx/search/query.rb +4 -8
  40. data/lib/thinking_sphinx/tasks.rb +8 -0
  41. data/spec/acceptance/excerpts_spec.rb +22 -0
  42. data/spec/acceptance/remove_deleted_records_spec.rb +10 -0
  43. data/spec/acceptance/searching_across_models_spec.rb +10 -0
  44. data/spec/acceptance/searching_with_filters_spec.rb +15 -0
  45. data/spec/acceptance/specifying_sql_spec.rb +3 -3
  46. data/spec/acceptance/sphinx_scopes_spec.rb +11 -0
  47. data/spec/internal/app/indices/product_index.rb +2 -0
  48. data/spec/internal/app/models/categorisation.rb +6 -0
  49. data/spec/internal/app/models/category.rb +3 -0
  50. data/spec/internal/app/models/product.rb +4 -1
  51. data/spec/internal/db/schema.rb +10 -0
  52. data/spec/thinking_sphinx/active_record/base_spec.rb +33 -0
  53. data/spec/thinking_sphinx/active_record/callbacks/delete_callbacks_spec.rb +4 -35
  54. data/spec/thinking_sphinx/active_record/sql_builder_spec.rb +5 -10
  55. data/spec/thinking_sphinx/active_record/sql_source_spec.rb +4 -3
  56. data/spec/thinking_sphinx/configuration_spec.rb +26 -6
  57. data/spec/thinking_sphinx/connection_spec.rb +4 -1
  58. data/spec/thinking_sphinx/deletion_spec.rb +76 -0
  59. data/spec/thinking_sphinx/facet_search_spec.rb +54 -5
  60. data/spec/thinking_sphinx/panes/excerpts_pane_spec.rb +4 -6
  61. data/spec/thinking_sphinx/rake_interface_spec.rb +35 -0
  62. data/spec/thinking_sphinx/real_time/callbacks/real_time_callbacks_spec.rb +68 -28
  63. data/spec/thinking_sphinx/search/glaze_spec.rb +19 -0
  64. data/spec/thinking_sphinx/search/query_spec.rb +39 -2
  65. data/thinking-sphinx.gemspec +2 -2
  66. metadata +31 -45
@@ -35,7 +35,7 @@ module ThinkingSphinx::Connection
35
35
  pool.take do |connection|
36
36
  begin
37
37
  yield connection
38
- rescue Mysql2::Error => error
38
+ rescue ThinkingSphinx::QueryExecutionError, Mysql2::Error => error
39
39
  original = ThinkingSphinx::SphinxError.new_from_mysql error
40
40
  raise original if original.is_a?(ThinkingSphinx::QueryError)
41
41
  raise Innertube::Pool::BadResource
@@ -43,8 +43,12 @@ module ThinkingSphinx::Connection
43
43
  end
44
44
  rescue Innertube::Pool::BadResource
45
45
  retries += 1
46
- retry if retries < 3
47
- raise original
46
+ raise original unless retries < 3
47
+
48
+ ActiveSupport::Notifications.instrument(
49
+ "message.thinking_sphinx", :message => "Retrying query \"#{original.statement}\" after error: #{original.message}"
50
+ )
51
+ retry
48
52
  end
49
53
  end
50
54
 
@@ -65,9 +69,10 @@ module ThinkingSphinx::Connection
65
69
 
66
70
  def execute(statement)
67
71
  client.query statement
68
- rescue
69
- puts "Error with statement: #{statement}"
70
- raise
72
+ rescue => error
73
+ wrapper = ThinkingSphinx::QueryExecutionError.new error.message
74
+ wrapper.statement = statement
75
+ raise wrapper
71
76
  end
72
77
 
73
78
  def query(statement)
@@ -78,6 +83,10 @@ module ThinkingSphinx::Connection
78
83
  results = [client.query(statements.join('; '))]
79
84
  results << client.store_result while client.next_result
80
85
  results
86
+ rescue => error
87
+ wrapper = ThinkingSphinx::QueryExecutionError.new error.message
88
+ wrapper.statement = statements.join('; ')
89
+ raise wrapper
81
90
  end
82
91
  end
83
92
 
@@ -2,6 +2,7 @@ module ThinkingSphinx::Core
2
2
  #
3
3
  end
4
4
 
5
+ require 'thinking_sphinx/core/settings'
5
6
  require 'thinking_sphinx/core/field'
6
7
  require 'thinking_sphinx/core/index'
7
8
  require 'thinking_sphinx/core/interpreter'
@@ -1,5 +1,6 @@
1
1
  module ThinkingSphinx::Core::Index
2
2
  extend ActiveSupport::Concern
3
+ include ThinkingSphinx::Core::Settings
3
4
 
4
5
  included do
5
6
  attr_reader :reference, :offset, :options
@@ -12,6 +13,7 @@ module ThinkingSphinx::Core::Index
12
13
  @charset_type = 'utf-8'
13
14
  @options = options
14
15
  @offset = config.next_offset(reference)
16
+ @type = 'plain'
15
17
 
16
18
  super "#{options[:name] || reference.to_s.gsub('/', '_')}_#{name_suffix}"
17
19
  end
@@ -27,10 +29,7 @@ module ThinkingSphinx::Core::Index
27
29
  def interpret_definition!
28
30
  return if @interpreted_definition
29
31
 
30
- self.class.settings.each do |setting|
31
- value = config.settings[setting.to_s]
32
- send("#{setting}=", value) unless value.nil?
33
- end
32
+ apply_defaults!
34
33
 
35
34
  @interpreted_definition = true
36
35
  interpreter.translate! self, @definition_block if @definition_block
@@ -42,19 +41,24 @@ module ThinkingSphinx::Core::Index
42
41
 
43
42
  def render
44
43
  pre_render
44
+ set_path
45
45
 
46
- @path ||= File.join config.indices_location, name
47
-
48
- if respond_to?(:infix_fields)
49
- self.infix_fields = fields.select(&:infixing?).collect(&:name)
50
- self.prefix_fields = fields.select(&:prefixing?).collect(&:name)
51
- end
46
+ assign_infix_fields
47
+ assign_prefix_fields
52
48
 
53
49
  super
54
50
  end
55
51
 
56
52
  private
57
53
 
54
+ def assign_infix_fields
55
+ self.infix_fields = fields.select(&:infixing?).collect(&:name)
56
+ end
57
+
58
+ def assign_prefix_fields
59
+ self.prefix_fields = fields.select(&:prefixing?).collect(&:name)
60
+ end
61
+
58
62
  def config
59
63
  ThinkingSphinx::Configuration.instance
60
64
  end
@@ -66,4 +70,8 @@ module ThinkingSphinx::Core::Index
66
70
  def pre_render
67
71
  interpret_definition!
68
72
  end
73
+
74
+ def set_path
75
+ @path ||= File.join config.indices_location, name
76
+ end
69
77
  end
@@ -0,0 +1,9 @@
1
+ module ThinkingSphinx::Core::Settings
2
+ private
3
+ def apply_defaults!(defaults = self.class.settings)
4
+ defaults.each do |setting|
5
+ value = config.settings[setting.to_s]
6
+ send("#{setting}=", value) unless value.nil?
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,48 @@
1
+ class ThinkingSphinx::Deletion
2
+ delegate :name, :to => :index
3
+
4
+ def self.perform(index, instance)
5
+ {
6
+ 'plain' => PlainDeletion,
7
+ 'rt' => RealtimeDeletion
8
+ }[index.type].new(index, instance).perform
9
+ rescue Mysql2::Error => error
10
+ # This isn't vital, so don't raise the error.
11
+ end
12
+
13
+ def initialize(index, instance)
14
+ @index, @instance = index, instance
15
+ end
16
+
17
+ private
18
+
19
+ attr_reader :index, :instance
20
+
21
+ def connection
22
+ @connection ||= ThinkingSphinx::Connection.new
23
+ end
24
+
25
+ def document_id_for_key
26
+ index.document_id_for_key instance.id
27
+ end
28
+
29
+ def execute(statement)
30
+ ThinkingSphinx::Connection.pool.take do |connection|
31
+ connection.execute statement
32
+ end
33
+ end
34
+
35
+ class RealtimeDeletion < ThinkingSphinx::Deletion
36
+ def perform
37
+ execute Riddle::Query::Delete.new(name, document_id_for_key).to_sql
38
+ end
39
+ end
40
+
41
+ class PlainDeletion < ThinkingSphinx::Deletion
42
+ def perform
43
+ execute Riddle::Query.update(
44
+ name, document_id_for_key, :sphinx_deleted => true
45
+ )
46
+ end
47
+ end
48
+ end
@@ -1,4 +1,6 @@
1
1
  class ThinkingSphinx::SphinxError < StandardError
2
+ attr_accessor :statement
3
+
2
4
  def self.new_from_mysql(error)
3
5
  case error.message
4
6
  when /parse error/
@@ -12,6 +14,7 @@ class ThinkingSphinx::SphinxError < StandardError
12
14
  end
13
15
 
14
16
  replacement.set_backtrace error.backtrace
17
+ replacement.statement = error.statement if error.respond_to?(:statement)
15
18
  replacement
16
19
  end
17
20
  end
@@ -25,5 +28,9 @@ end
25
28
  class ThinkingSphinx::ParseError < ThinkingSphinx::QueryError
26
29
  end
27
30
 
31
+ class ThinkingSphinx::QueryExecutionError < StandardError
32
+ attr_accessor :statement
33
+ end
34
+
28
35
  class ThinkingSphinx::MixedScopesError < StandardError
29
36
  end
@@ -10,6 +10,7 @@ class ThinkingSphinx::Excerpter
10
10
  def initialize(index, words, options = {})
11
11
  @index, @words = index, words
12
12
  @options = DefaultOptions.merge(options)
13
+ @words = @options.delete(:words) if @options[:words]
13
14
  end
14
15
 
15
16
  def excerpt!(text)
@@ -1,7 +1,8 @@
1
1
  class ThinkingSphinx::FacetSearch
2
2
  include Enumerable
3
3
 
4
- attr_reader :query, :options
4
+ attr_reader :options
5
+ attr_accessor :query
5
6
 
6
7
  def initialize(query = nil, options = {})
7
8
  query, options = nil, query if query.is_a?(Hash)
@@ -36,12 +37,7 @@ class ThinkingSphinx::FacetSearch
36
37
 
37
38
  batch = ThinkingSphinx::BatchedSearch.new
38
39
  facets.each do |facet|
39
- search = ThinkingSphinx::Search.new query, options.merge(
40
- :select => '*, @groupby, @count',
41
- :group_by => facet.name,
42
- :indices => index_names_for(facet)
43
- )
44
- batch.searches << search
40
+ batch.searches << ThinkingSphinx::Search.new(query, options_for(facet))
45
41
  end
46
42
 
47
43
  batch.populate ThinkingSphinx::Middlewares::RAW_ONLY
@@ -64,26 +60,53 @@ class ThinkingSphinx::FacetSearch
64
60
  private
65
61
 
66
62
  def facets
67
- @facets ||= begin
68
- properties = indices.collect(&:facets).flatten
69
- properties.group_by(&:name).collect { |name, matches|
70
- ThinkingSphinx::Facet.new name, matches
63
+ @facets ||= properties.group_by(&:name).collect { |name, matches|
64
+ ThinkingSphinx::Facet.new name, matches
65
+ }
66
+ end
67
+
68
+ def properties
69
+ properties = indices.collect(&:facets).flatten
70
+ if options[:facets].present?
71
+ properties = properties.select { |property|
72
+ options[:facets].include? property.name.to_sym
71
73
  }
72
74
  end
75
+ properties
73
76
  end
74
77
 
75
78
  def index_names_for(*facets)
76
- indices.select { |index|
77
- facet_names = index.facets.collect(&:name)
78
- facets.all? { |facet|
79
- facet_names.include?(facet.name)
80
- }
81
- }.collect &:name
79
+ facet_names(
80
+ indices.select do |index|
81
+ facets.all? { |facet| facet_names(index.facets).include?(facet.name) }
82
+ end
83
+ )
84
+ end
85
+
86
+ def facet_names(facets)
87
+ facets.collect(&:name)
82
88
  end
83
89
 
84
90
  def indices
85
- @indices ||= ThinkingSphinx::IndexSet.new options[:classes],
86
- options[:indices]
91
+ @indices ||= ThinkingSphinx::IndexSet.new options[:classes], options[:indices]
92
+ end
93
+
94
+ def max_matches
95
+ ThinkingSphinx::Configuration.instance.settings['max_matches'] || 1000
96
+ end
97
+
98
+ def limit
99
+ limit = options[:limit] || options[:per_page] || max_matches
100
+ end
101
+
102
+ def options_for(facet)
103
+ options.merge(
104
+ :select => '*, @groupby, @count',
105
+ :group_by => facet.name,
106
+ :indices => index_names_for(facet),
107
+ :max_matches => limit,
108
+ :limit => limit
109
+ )
87
110
  end
88
111
 
89
112
  class Filter
@@ -7,6 +7,13 @@ class ThinkingSphinx::Masks::ScopesMask
7
7
  public_methods(false).include?(method) || can_apply_scope?(method)
8
8
  end
9
9
 
10
+ def facets(query = nil, options = {})
11
+ search = ThinkingSphinx.facets query, options
12
+ ThinkingSphinx::Search::Merger.new(search).merge!(
13
+ @search.query, @search.options
14
+ )
15
+ end
16
+
10
17
  def search(query = nil, options = {})
11
18
  query, options = nil, query if query.is_a?(Hash)
12
19
  ThinkingSphinx::Search::Merger.new(@search).merge! query, options
@@ -1,39 +1,33 @@
1
- module ThinkingSphinx::Middlewares
2
- #
1
+ module ThinkingSphinx::Middlewares; end
2
+
3
+ %w[middleware active_record_translator geographer glazier ids_only inquirer
4
+ sphinxql stale_id_checker stale_id_filter utf8].each do |middleware|
5
+ require "thinking_sphinx/middlewares/#{middleware}"
3
6
  end
4
7
 
5
- require 'thinking_sphinx/middlewares/middleware'
6
- require 'thinking_sphinx/middlewares/active_record_translator'
7
- require 'thinking_sphinx/middlewares/geographer'
8
- require 'thinking_sphinx/middlewares/glazier'
9
- require 'thinking_sphinx/middlewares/ids_only'
10
- require 'thinking_sphinx/middlewares/inquirer'
11
- require 'thinking_sphinx/middlewares/sphinxql'
12
- require 'thinking_sphinx/middlewares/stale_id_checker'
13
- require 'thinking_sphinx/middlewares/stale_id_filter'
14
- require 'thinking_sphinx/middlewares/utf8'
8
+ module ThinkingSphinx::Middlewares
9
+ def self.use(builder, middlewares)
10
+ middlewares.each { |m| builder.use m }
11
+ end
12
+
13
+ BASE_MIDDLEWARES = [SphinxQL, Geographer, Inquirer]
15
14
 
16
- ThinkingSphinx::Middlewares::DEFAULT = ::Middleware::Builder.new do
17
- use ThinkingSphinx::Middlewares::StaleIdFilter
18
- use ThinkingSphinx::Middlewares::SphinxQL
19
- use ThinkingSphinx::Middlewares::Geographer
20
- use ThinkingSphinx::Middlewares::Inquirer
21
- use ThinkingSphinx::Middlewares::UTF8
22
- use ThinkingSphinx::Middlewares::ActiveRecordTranslator
23
- use ThinkingSphinx::Middlewares::StaleIdChecker
24
- use ThinkingSphinx::Middlewares::Glazier
25
- end
15
+ DEFAULT = ::Middleware::Builder.new do
16
+ use StaleIdFilter
17
+ ThinkingSphinx::Middlewares.use self, BASE_MIDDLEWARES
18
+ use UTF8
19
+ use ActiveRecordTranslator
20
+ use StaleIdChecker
21
+ use Glazier
22
+ end
26
23
 
27
- ThinkingSphinx::Middlewares::RAW_ONLY = ::Middleware::Builder.new do
28
- use ThinkingSphinx::Middlewares::SphinxQL
29
- use ThinkingSphinx::Middlewares::Geographer
30
- use ThinkingSphinx::Middlewares::Inquirer
31
- use ThinkingSphinx::Middlewares::UTF8
32
- end
24
+ RAW_ONLY = ::Middleware::Builder.new do
25
+ ThinkingSphinx::Middlewares.use self, BASE_MIDDLEWARES
26
+ use UTF8
27
+ end
33
28
 
34
- ThinkingSphinx::Middlewares::IDS_ONLY = ::Middleware::Builder.new do
35
- use ThinkingSphinx::Middlewares::SphinxQL
36
- use ThinkingSphinx::Middlewares::Geographer
37
- use ThinkingSphinx::Middlewares::Inquirer
38
- use ThinkingSphinx::Middlewares::IdsOnly
29
+ IDS_ONLY = ::Middleware::Builder.new do
30
+ ThinkingSphinx::Middlewares.use self, BASE_MIDDLEWARES
31
+ use IdsOnly
32
+ end
39
33
  end
@@ -18,7 +18,7 @@ class ThinkingSphinx::Middlewares::ActiveRecordTranslator <
18
18
 
19
19
  def call
20
20
  results_for_models # load now to avoid segfaults
21
- context[:results] = context[:results].collect { |row| result_for row }
21
+ context[:results] = context[:results].collect { |row| result_for(row) }
22
22
  end
23
23
 
24
24
  private
@@ -26,11 +26,9 @@ class ThinkingSphinx::Middlewares::ActiveRecordTranslator <
26
26
  attr_reader :context
27
27
 
28
28
  def ids_for_model(model_name)
29
- context[:results].select { |row|
30
- row['sphinx_internal_class'] == model_name
31
- }.collect { |row|
32
- row['sphinx_internal_id']
33
- }
29
+ context[:results].collect { |row|
30
+ row['sphinx_internal_id'] if row['sphinx_internal_class'] == model_name
31
+ }.compact
34
32
  end
35
33
 
36
34
  def model_names
@@ -51,20 +49,22 @@ class ThinkingSphinx::Middlewares::ActiveRecordTranslator <
51
49
  end
52
50
 
53
51
  def results_for_models
54
- @results_for_models ||= model_names.inject({}) { |hash, name|
55
- ids = ids_for_model(name)
56
- model = name.constantize
57
- relation = model.unscoped
58
-
59
- relation = relation.includes sql_options[:include] if sql_options[:include]
60
- relation = relation.joins sql_options[:joins] if sql_options[:joins]
61
- relation = relation.order sql_options[:order] if sql_options[:order]
62
- relation = relation.select sql_options[:select] if sql_options[:select]
63
-
64
- hash[name] = relation.where(model.primary_key => ids)
52
+ @results_for_models ||= model_names.inject({}) do |hash, name|
53
+ model = name.constantize
54
+ hash[name] = model_relation_with_sql_options(model.unscoped).where(
55
+ model.primary_key => ids_for_model(name)
56
+ )
65
57
 
66
58
  hash
67
- }
59
+ end
60
+ end
61
+
62
+ def model_relation_with_sql_options(relation)
63
+ relation = relation.includes sql_options[:include] if sql_options[:include]
64
+ relation = relation.joins sql_options[:joins] if sql_options[:joins]
65
+ relation = relation.order sql_options[:order] if sql_options[:order]
66
+ relation = relation.select sql_options[:select] if sql_options[:select]
67
+ relation
68
68
  end
69
69
 
70
70
  def sql_options