thinking-sphinx 4.3.2 → 5.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +152 -0
  3. data/.travis.yml +16 -21
  4. data/Appraisals +11 -20
  5. data/CHANGELOG.markdown +79 -0
  6. data/README.textile +15 -17
  7. data/bin/loadsphinx +30 -7
  8. data/lib/thinking_sphinx.rb +5 -4
  9. data/lib/thinking_sphinx/active_record.rb +1 -0
  10. data/lib/thinking_sphinx/active_record/association_proxy.rb +1 -2
  11. data/lib/thinking_sphinx/active_record/association_proxy/attribute_finder.rb +1 -1
  12. data/lib/thinking_sphinx/active_record/attribute/sphinx_presenter.rb +1 -1
  13. data/lib/thinking_sphinx/active_record/base.rb +17 -6
  14. data/lib/thinking_sphinx/active_record/callbacks/association_delta_callbacks.rb +21 -0
  15. data/lib/thinking_sphinx/active_record/callbacks/delete_callbacks.rb +1 -1
  16. data/lib/thinking_sphinx/active_record/callbacks/delta_callbacks.rb +3 -2
  17. data/lib/thinking_sphinx/active_record/callbacks/update_callbacks.rb +1 -1
  18. data/lib/thinking_sphinx/active_record/interpreter.rb +4 -4
  19. data/lib/thinking_sphinx/active_record/sql_source.rb +12 -0
  20. data/lib/thinking_sphinx/active_record/sql_source/template.rb +2 -2
  21. data/lib/thinking_sphinx/callbacks.rb +9 -0
  22. data/lib/thinking_sphinx/callbacks/appender.rb +59 -0
  23. data/lib/thinking_sphinx/commands/index_real_time.rb +1 -3
  24. data/lib/thinking_sphinx/connection.rb +4 -0
  25. data/lib/thinking_sphinx/connection/client.rb +6 -1
  26. data/lib/thinking_sphinx/core/index.rb +5 -2
  27. data/lib/thinking_sphinx/deletion.rb +18 -17
  28. data/lib/thinking_sphinx/errors.rb +1 -1
  29. data/lib/thinking_sphinx/index_set.rb +7 -3
  30. data/lib/thinking_sphinx/middlewares/sphinxql.rb +1 -1
  31. data/lib/thinking_sphinx/railtie.rb +14 -1
  32. data/lib/thinking_sphinx/real_time.rb +17 -0
  33. data/lib/thinking_sphinx/real_time/index.rb +5 -3
  34. data/lib/thinking_sphinx/real_time/index/template.rb +12 -0
  35. data/lib/thinking_sphinx/real_time/interpreter.rb +8 -6
  36. data/lib/thinking_sphinx/real_time/populator.rb +5 -2
  37. data/lib/thinking_sphinx/real_time/processor.rb +36 -0
  38. data/lib/thinking_sphinx/real_time/transcriber.rb +41 -19
  39. data/lib/thinking_sphinx/real_time/translator.rb +1 -0
  40. data/lib/thinking_sphinx/settings.rb +13 -9
  41. data/lib/thinking_sphinx/subscribers/populator_subscriber.rb +0 -4
  42. data/spec/acceptance/big_integers_spec.rb +1 -1
  43. data/spec/acceptance/geosearching_spec.rb +13 -3
  44. data/spec/acceptance/merging_spec.rb +1 -1
  45. data/spec/acceptance/real_time_updates_spec.rb +2 -2
  46. data/spec/acceptance/sql_deltas_spec.rb +15 -3
  47. data/spec/acceptance/support/sphinx_helpers.rb +4 -4
  48. data/spec/acceptance/suspended_deltas_spec.rb +3 -3
  49. data/spec/internal/app/indices/article_index.rb +0 -1
  50. data/spec/internal/app/indices/colour_index.rb +7 -0
  51. data/spec/internal/app/models/admin/person.rb +3 -1
  52. data/spec/internal/app/models/album.rb +3 -1
  53. data/spec/internal/app/models/animal.rb +1 -0
  54. data/spec/internal/app/models/article.rb +2 -0
  55. data/spec/internal/app/models/bird.rb +1 -0
  56. data/spec/internal/app/models/book.rb +2 -0
  57. data/spec/internal/app/models/car.rb +1 -1
  58. data/spec/internal/app/models/city.rb +2 -0
  59. data/spec/internal/app/models/colour.rb +2 -0
  60. data/spec/internal/app/models/product.rb +1 -1
  61. data/spec/internal/app/models/tee.rb +5 -0
  62. data/spec/internal/app/models/user.rb +2 -0
  63. data/spec/internal/config/database.yml +6 -1
  64. data/spec/internal/db/schema.rb +1 -0
  65. data/spec/thinking_sphinx/active_record/callbacks/update_callbacks_spec.rb +2 -1
  66. data/spec/thinking_sphinx/active_record/index_spec.rb +0 -12
  67. data/spec/thinking_sphinx/active_record/interpreter_spec.rb +15 -14
  68. data/spec/thinking_sphinx/active_record/sql_source_spec.rb +41 -3
  69. data/spec/thinking_sphinx/connection/mri_spec.rb +49 -0
  70. data/spec/thinking_sphinx/deletion_spec.rb +5 -4
  71. data/spec/thinking_sphinx/index_set_spec.rb +28 -12
  72. data/spec/thinking_sphinx/middlewares/sphinxql_spec.rb +2 -1
  73. data/spec/thinking_sphinx/real_time/index_spec.rb +51 -13
  74. data/spec/thinking_sphinx/real_time/interpreter_spec.rb +14 -14
  75. data/spec/thinking_sphinx/real_time/transcriber_spec.rb +9 -1
  76. data/thinking-sphinx.gemspec +3 -5
  77. metadata +17 -11
  78. data/spec/acceptance/connection_spec.rb +0 -25
@@ -17,8 +17,6 @@ require 'active_support/core_ext/module/delegation'
17
17
  require 'active_support/core_ext/module/attribute_accessors'
18
18
 
19
19
  module ThinkingSphinx
20
- MAXIMUM_STATEMENT_LENGTH = (2 ** 23) - 5
21
-
22
20
  def self.count(query = '', options = {})
23
21
  search_for_ids(query, options).total_entries
24
22
  end
@@ -98,12 +96,15 @@ require 'thinking_sphinx/test'
98
96
  require 'thinking_sphinx/utf8'
99
97
  require 'thinking_sphinx/wildcard'
100
98
  # Extended
101
- require 'thinking_sphinx/active_record'
102
99
  require 'thinking_sphinx/deltas'
103
100
  require 'thinking_sphinx/distributed'
104
101
  require 'thinking_sphinx/logger'
105
102
  require 'thinking_sphinx/real_time'
106
103
 
107
- require 'thinking_sphinx/railtie' if defined?(Rails::Railtie)
104
+ if defined?(Rails::Railtie)
105
+ require 'thinking_sphinx/railtie'
106
+ else
107
+ require 'thinking_sphinx/active_record'
108
+ end
108
109
 
109
110
  ThinkingSphinx.before_index_hooks << ThinkingSphinx::Hooks::GuardPresence
@@ -29,6 +29,7 @@ require 'thinking_sphinx/active_record/source_joins'
29
29
  require 'thinking_sphinx/active_record/sql_builder'
30
30
  require 'thinking_sphinx/active_record/sql_source'
31
31
 
32
+ require 'thinking_sphinx/active_record/callbacks/association_delta_callbacks'
32
33
  require 'thinking_sphinx/active_record/callbacks/delete_callbacks'
33
34
  require 'thinking_sphinx/active_record/callbacks/delta_callbacks'
34
35
  require 'thinking_sphinx/active_record/callbacks/update_callbacks'
@@ -1,8 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ThinkingSphinx::ActiveRecord::AssociationProxy
4
- extend ActiveSupport::Concern
5
-
6
4
  def search(query = nil, options = {})
7
5
  perform_search super(*normalise_search_arguments(query, options))
8
6
  end
@@ -12,6 +10,7 @@ module ThinkingSphinx::ActiveRecord::AssociationProxy
12
10
  end
13
11
 
14
12
  private
13
+
15
14
  def normalise_search_arguments(query, options)
16
15
  query, options = nil, query if query.is_a?(Hash)
17
16
  options[:ignore_scopes] = true
@@ -31,7 +31,7 @@ class ThinkingSphinx::ActiveRecord::AssociationProxy::AttributeFinder
31
31
  @indices ||= begin
32
32
  configuration.preload_indices
33
33
  configuration.indices_for_references(
34
- *ThinkingSphinx::IndexSet.reference_name(@association.klass)
34
+ *configuration.index_set_class.reference_name(@association.klass)
35
35
  ).reject &:distributed?
36
36
  end
37
37
  end
@@ -4,7 +4,7 @@ class ThinkingSphinx::ActiveRecord::Attribute::SphinxPresenter
4
4
  SPHINX_TYPES = {
5
5
  :integer => :uint,
6
6
  :boolean => :bool,
7
- :timestamp => :timestamp,
7
+ :timestamp => :uint,
8
8
  :float => :float,
9
9
  :string => :string,
10
10
  :bigint => :bigint,
@@ -4,13 +4,24 @@ module ThinkingSphinx::ActiveRecord::Base
4
4
  extend ActiveSupport::Concern
5
5
 
6
6
  included do
7
- after_destroy ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks
8
- before_save ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
9
- after_update ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks
10
- after_commit ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
7
+ if ActiveRecord::VERSION::STRING.to_i >= 5
8
+ [
9
+ ::ActiveRecord::Reflection::HasManyReflection,
10
+ ::ActiveRecord::Reflection::HasAndBelongsToManyReflection
11
+ ].each do |reflection_class|
12
+ reflection_class.include DefaultReflectionAssociations
13
+ end
14
+ else
15
+ ::ActiveRecord::Associations::CollectionProxy.include(
16
+ ThinkingSphinx::ActiveRecord::AssociationProxy
17
+ )
18
+ end
19
+ end
11
20
 
12
- ::ActiveRecord::Associations::CollectionProxy.send :include,
13
- ThinkingSphinx::ActiveRecord::AssociationProxy
21
+ module DefaultReflectionAssociations
22
+ def extensions
23
+ super + [ThinkingSphinx::ActiveRecord::AssociationProxy]
24
+ end
14
25
  end
15
26
 
16
27
  module ClassMethods
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ class ThinkingSphinx::ActiveRecord::Callbacks::AssociationDeltaCallbacks
4
+ def initialize(path)
5
+ @path = path
6
+ end
7
+
8
+ def after_commit(instance)
9
+ Array(objects_for(instance)).each do |object|
10
+ object.update :delta => true unless object.frozen?
11
+ end
12
+ end
13
+
14
+ private
15
+
16
+ attr_reader :path
17
+
18
+ def objects_for(instance)
19
+ path.inject(instance) { |object, method| object.send method }
20
+ end
21
+ end
@@ -27,7 +27,7 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks <
27
27
 
28
28
  def indices
29
29
  ThinkingSphinx::Configuration.instance.index_set_class.new(
30
- :classes => [instance.class]
30
+ :instances => [instance], :classes => [instance.class]
31
31
  ).to_a
32
32
  end
33
33
  end
@@ -43,8 +43,9 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks <
43
43
  end
44
44
 
45
45
  def indices
46
- @indices ||= config.index_set_class.new(:classes => [instance.class]).
47
- select { |index| index.type == "plain" }
46
+ @indices ||= config.index_set_class.new(
47
+ :instances => [instance], :classes => [instance.class]
48
+ ).select { |index| index.type == "plain" }
48
49
  end
49
50
 
50
51
  def new_or_changed?
@@ -47,7 +47,7 @@ class ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks <
47
47
  end
48
48
 
49
49
  def reference
50
- ThinkingSphinx::IndexSet.reference_name(instance.class)
50
+ configuration.index_set_class.reference_name(instance.class)
51
51
  end
52
52
 
53
53
  def update(index)
@@ -13,15 +13,15 @@ class ThinkingSphinx::ActiveRecord::Interpreter <
13
13
  end
14
14
 
15
15
  def has(*columns)
16
- __source.attributes += build_properties(
16
+ build_properties(
17
17
  ::ThinkingSphinx::ActiveRecord::Attribute, columns
18
- )
18
+ ).each { |attribute| __source.add_attribute attribute }
19
19
  end
20
20
 
21
21
  def indexes(*columns)
22
- __source.fields += build_properties(
22
+ build_properties(
23
23
  ::ThinkingSphinx::ActiveRecord::Field, columns
24
- )
24
+ ).each { |field| __source.add_field field }
25
25
  end
26
26
 
27
27
  def join(*columns)
@@ -39,6 +39,18 @@ module ThinkingSphinx
39
39
  @adapter ||= DatabaseAdapters.adapter_for(@model)
40
40
  end
41
41
 
42
+ def add_attribute(attribute)
43
+ attributes.delete_if { |existing| existing.name == attribute.name }
44
+
45
+ attributes << attribute
46
+ end
47
+
48
+ def add_field(field)
49
+ fields.delete_if { |existing| existing.name == field.name }
50
+
51
+ fields << field
52
+ end
53
+
42
54
  def delta_processor
43
55
  options[:delta_processor].try(:new, adapter, @options[:delta_options] || {})
44
56
  end
@@ -18,14 +18,14 @@ class ThinkingSphinx::ActiveRecord::SQLSource::Template
18
18
  private
19
19
 
20
20
  def add_attribute(column, name, type, options = {})
21
- source.attributes << ThinkingSphinx::ActiveRecord::Attribute.new(
21
+ source.add_attribute ThinkingSphinx::ActiveRecord::Attribute.new(
22
22
  source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
23
23
  options.merge(:as => name, :type => type)
24
24
  )
25
25
  end
26
26
 
27
27
  def add_field(column, name, options = {})
28
- source.fields << ThinkingSphinx::ActiveRecord::Field.new(
28
+ source.add_field ThinkingSphinx::ActiveRecord::Field.new(
29
29
  source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
30
30
  options.merge(:as => name)
31
31
  )
@@ -3,6 +3,13 @@
3
3
  class ThinkingSphinx::Callbacks
4
4
  attr_reader :instance
5
5
 
6
+ def self.append(model, reference = nil, options, &block)
7
+ reference ||= ThinkingSphinx::Configuration.instance.index_set_class.
8
+ reference_name(model)
9
+
10
+ ThinkingSphinx::Callbacks::Appender.call(model, reference, options, &block)
11
+ end
12
+
6
13
  def self.callbacks(*methods)
7
14
  mod = Module.new
8
15
  methods.each do |method|
@@ -33,3 +40,5 @@ class ThinkingSphinx::Callbacks
33
40
  @instance = instance
34
41
  end
35
42
  end
43
+
44
+ require "thinking_sphinx/callbacks/appender"
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ class ThinkingSphinx::Callbacks::Appender
4
+ def self.call(model, reference, options, &block)
5
+ new(model, reference, options, &block).call
6
+ end
7
+
8
+ def initialize(model, reference, options, &block)
9
+ @model = model
10
+ @reference = reference
11
+ @options = options
12
+ @block = block
13
+ end
14
+
15
+ def call
16
+ add_core_callbacks
17
+ add_delta_callbacks if behaviours.include?(:deltas)
18
+ add_real_time_callbacks if behaviours.include?(:real_time)
19
+ add_update_callbacks if behaviours.include?(:updates)
20
+ end
21
+
22
+ private
23
+
24
+ attr_reader :model, :reference, :options, :block
25
+
26
+ def add_core_callbacks
27
+ model.after_destroy ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks
28
+ end
29
+
30
+ def add_delta_callbacks
31
+ if path.empty?
32
+ model.before_save ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
33
+ model.after_commit ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
34
+ else
35
+ model.after_commit(
36
+ ThinkingSphinx::ActiveRecord::Callbacks::AssociationDeltaCallbacks
37
+ .new(path)
38
+ )
39
+ end
40
+ end
41
+
42
+ def add_real_time_callbacks
43
+ model.after_save ThinkingSphinx::RealTime.callback_for(
44
+ reference, path, &block
45
+ )
46
+ end
47
+
48
+ def add_update_callbacks
49
+ model.after_update ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks
50
+ end
51
+
52
+ def behaviours
53
+ options[:behaviours] || []
54
+ end
55
+
56
+ def path
57
+ options[:path] || []
58
+ end
59
+ end
@@ -2,9 +2,7 @@
2
2
 
3
3
  class ThinkingSphinx::Commands::IndexRealTime < ThinkingSphinx::Commands::Base
4
4
  def call
5
- options[:indices].each do |index|
6
- ThinkingSphinx::RealTime::Populator.populate index
7
-
5
+ ThinkingSphinx::RealTime.processor.call options[:indices] do
8
6
  command :rotate
9
7
  end
10
8
  end
@@ -16,6 +16,10 @@ module ThinkingSphinx::Connection
16
16
  connection_class.new options
17
17
  end
18
18
 
19
+ def self.clear
20
+ @pool = nil
21
+ end
22
+
19
23
  def self.connection_class
20
24
  return ThinkingSphinx::Connection::JRuby if RUBY_PLATFORM == 'java'
21
25
 
@@ -39,7 +39,7 @@ class ThinkingSphinx::Connection::Client
39
39
  private
40
40
 
41
41
  def check(statements)
42
- if statements.length > ThinkingSphinx::MAXIMUM_STATEMENT_LENGTH
42
+ if statements.length > maximum_statement_length
43
43
  exception = ThinkingSphinx::QueryLengthError.new
44
44
  exception.statement = statements
45
45
  raise exception
@@ -56,6 +56,11 @@ class ThinkingSphinx::Connection::Client
56
56
  @client = nil
57
57
  end
58
58
 
59
+ def maximum_statement_length
60
+ @maximum_statement_length ||= ThinkingSphinx::Configuration.instance.
61
+ settings['maximum_statement_length']
62
+ end
63
+
59
64
  def perform(statements)
60
65
  results_for statements
61
66
  rescue => error
@@ -11,7 +11,6 @@ module ThinkingSphinx::Core::Index
11
11
 
12
12
  def initialize(reference, options = {})
13
13
  @reference = reference.to_sym
14
- @docinfo = :extern unless config.settings["skip_docinfo"]
15
14
  @options = options
16
15
  @offset = config.next_offset(options[:offset_as] || reference)
17
16
  @type = 'plain'
@@ -38,7 +37,11 @@ module ThinkingSphinx::Core::Index
38
37
  end
39
38
 
40
39
  def interpret_definition!
41
- return unless model.table_exists?
40
+ table_exists = model.table_exists?
41
+ unless table_exists
42
+ Rails.logger.info "No table exists for #{model}. Index can not be created"
43
+ return
44
+ end
42
45
  return if @interpreted_definition
43
46
 
44
47
  apply_defaults!
@@ -22,10 +22,6 @@ class ThinkingSphinx::Deletion
22
22
 
23
23
  attr_reader :index, :ids
24
24
 
25
- def document_ids_for_keys
26
- ids.collect { |id| index.document_id_for_key id }
27
- end
28
-
29
25
  def execute(statement)
30
26
  statement = statement.gsub(/\s*\n\s*/, ' ').strip
31
27
 
@@ -36,11 +32,28 @@ class ThinkingSphinx::Deletion
36
32
  end
37
33
  end
38
34
 
35
+ class PlainDeletion < ThinkingSphinx::Deletion
36
+ def perform
37
+ ids.each_slice(1000) do |some_ids|
38
+ execute <<-SQL
39
+ UPDATE #{name}
40
+ SET sphinx_deleted = 1
41
+ WHERE sphinx_internal_id IN (#{some_ids.join(', ')})
42
+ SQL
43
+ end
44
+ end
45
+ end
46
+
39
47
  class RealtimeDeletion < ThinkingSphinx::Deletion
40
48
  def perform
41
49
  return unless callbacks_enabled?
42
50
 
43
- execute Riddle::Query::Delete.new(name, document_ids_for_keys).to_sql
51
+ ids.each_slice(1000) do |some_ids|
52
+ execute <<-SQL
53
+ DELETE FROM #{name}
54
+ WHERE sphinx_internal_id IN (#{some_ids.join(', ')})
55
+ SQL
56
+ end
44
57
  end
45
58
 
46
59
  private
@@ -54,16 +67,4 @@ class ThinkingSphinx::Deletion
54
67
  ThinkingSphinx::Configuration.instance
55
68
  end
56
69
  end
57
-
58
- class PlainDeletion < ThinkingSphinx::Deletion
59
- def perform
60
- document_ids_for_keys.each_slice(1000) do |document_ids|
61
- execute <<-SQL
62
- UPDATE #{name}
63
- SET sphinx_deleted = 1
64
- WHERE id IN (#{document_ids.join(', ')})
65
- SQL
66
- end
67
- end
68
- end
69
70
  end
@@ -38,7 +38,7 @@ end
38
38
  class ThinkingSphinx::QueryLengthError < ThinkingSphinx::SphinxError
39
39
  def message
40
40
  <<-MESSAGE
41
- The supplied SphinxQL statement is #{statement.length} characters long. The maximum allowed length is #{ThinkingSphinx::MAXIMUM_STATEMENT_LENGTH}.
41
+ The supplied SphinxQL statement is #{statement.length} characters long. The maximum allowed length is #{ThinkingSphinx::Configuration.instance.settings['maximum_statement_length']}.
42
42
 
43
43
  If this error has been raised during real-time index population, it's probably due to overly large batches of records being processed at once. The default is 1000, but you can lower it on a per-environment basis in config/thinking_sphinx.yml:
44
44
 
@@ -34,11 +34,11 @@ class ThinkingSphinx::IndexSet
34
34
  end
35
35
 
36
36
  def classes
37
- options[:classes] || []
37
+ options[:classes] || instances.collect(&:class)
38
38
  end
39
39
 
40
40
  def classes_specified?
41
- classes.any? || references_specified?
41
+ instances.any? || classes.any? || references_specified?
42
42
  end
43
43
 
44
44
  def classes_and_ancestors
@@ -68,6 +68,10 @@ class ThinkingSphinx::IndexSet
68
68
  all_indices.select { |index| references.include? index.reference }
69
69
  end
70
70
 
71
+ def instances
72
+ options[:instances] || []
73
+ end
74
+
71
75
  def mti_classes
72
76
  classes.reject { |klass|
73
77
  klass.column_names.include?(klass.inheritance_column)
@@ -76,7 +80,7 @@ class ThinkingSphinx::IndexSet
76
80
 
77
81
  def references
78
82
  options[:references] || classes_and_ancestors.collect { |klass|
79
- ThinkingSphinx::IndexSet.reference_name(klass)
83
+ self.class.reference_name(klass)
80
84
  }
81
85
  end
82
86