thinking-sphinx 4.4.1 → 5.4.0

data/lib/thinking_sphinx/active_record/callbacks/delete_callbacks.rb

@@ -3,7 +3,11 @@
 class ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks <
   ThinkingSphinx::Callbacks
 
-  callbacks :after_destroy, :after_rollback
+  callbacks :after_commit, :after_destroy, :after_rollback
+
+  def after_commit
+    delete_from_sphinx
+  end
 
   def after_destroy
     delete_from_sphinx
@@ -27,7 +31,7 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks <
 
   def indices
     ThinkingSphinx::Configuration.instance.index_set_class.new(
-      :classes => [instance.class]
+      :instances => [instance], :classes => [instance.class]
     ).to_a
   end
 end

data/lib/thinking_sphinx/active_record/callbacks/delta_callbacks.rb

@@ -43,8 +43,9 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks <
   end
 
   def indices
-    @indices ||= config.index_set_class.new(:classes => [instance.class]).
-      select { |index| index.type == "plain" }
+    @indices ||= config.index_set_class.new(
+      :instances => [instance], :classes => [instance.class]
+    ).select { |index| index.type == "plain" }
   end
 
   def new_or_changed?

data/lib/thinking_sphinx/active_record/callbacks/update_callbacks.rb

@@ -47,7 +47,7 @@ class ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks <
   end
 
   def reference
-    ThinkingSphinx::IndexSet.reference_name(instance.class)
+    configuration.index_set_class.reference_name(instance.class)
   end
 
   def update(index)

data/lib/thinking_sphinx/active_record/interpreter.rb

@@ -13,15 +13,15 @@ class ThinkingSphinx::ActiveRecord::Interpreter <
   end
 
   def has(*columns)
-    __source.attributes += build_properties(
+    build_properties(
       ::ThinkingSphinx::ActiveRecord::Attribute, columns
-    )
+    ).each { |attribute| __source.add_attribute attribute }
   end
 
   def indexes(*columns)
-    __source.fields += build_properties(
+    build_properties(
       ::ThinkingSphinx::ActiveRecord::Field, columns
-    )
+    ).each { |field| __source.add_field field }
   end
 
   def join(*columns)

data/lib/thinking_sphinx/active_record/sql_source/template.rb

@@ -18,14 +18,14 @@ class ThinkingSphinx::ActiveRecord::SQLSource::Template
   private
 
   def add_attribute(column, name, type, options = {})
-    source.attributes << ThinkingSphinx::ActiveRecord::Attribute.new(
+    source.add_attribute ThinkingSphinx::ActiveRecord::Attribute.new(
       source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
       options.merge(:as => name, :type => type)
     )
   end
 
   def add_field(column, name, options = {})
-    source.fields << ThinkingSphinx::ActiveRecord::Field.new(
+    source.add_field ThinkingSphinx::ActiveRecord::Field.new(
       source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
       options.merge(:as => name)
     )

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -39,6 +39,18 @@ module ThinkingSphinx
         @adapter ||= DatabaseAdapters.adapter_for(@model)
       end
 
+      def add_attribute(attribute)
+        attributes.delete_if { |existing| existing.name == attribute.name }
+
+        attributes << attribute
+      end
+
+      def add_field(field)
+        fields.delete_if { |existing| existing.name == field.name }
+
+        fields << field
+      end
+
       def delta_processor
         options[:delta_processor].try(:new, adapter, @options[:delta_options] || {})
       end

data/lib/thinking_sphinx/active_record.rb

@@ -29,6 +29,7 @@ require 'thinking_sphinx/active_record/source_joins'
 require 'thinking_sphinx/active_record/sql_builder'
 require 'thinking_sphinx/active_record/sql_source'
 
+require 'thinking_sphinx/active_record/callbacks/association_delta_callbacks'
 require 'thinking_sphinx/active_record/callbacks/delete_callbacks'
 require 'thinking_sphinx/active_record/callbacks/delta_callbacks'
 require 'thinking_sphinx/active_record/callbacks/update_callbacks'
@@ -39,3 +40,5 @@ require 'thinking_sphinx/active_record/depolymorph/conditions_reflection'
 require 'thinking_sphinx/active_record/depolymorph/overridden_reflection'
 require 'thinking_sphinx/active_record/depolymorph/scoped_reflection'
 require 'thinking_sphinx/active_record/filter_reflection'
+
+ActiveRecord::Base.include ThinkingSphinx::ActiveRecord::Base

data/lib/thinking_sphinx/callbacks/appender.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Callbacks::Appender
+  def self.call(model, reference, options, &block)
+    new(model, reference, options, &block).call
+  end
+
+  def initialize(model, reference, options, &block)
+    @model = model
+    @reference = reference
+    @options = options
+    @block = block
+  end
+
+  def call
+    add_core_callbacks
+    add_delta_callbacks     if behaviours.include?(:deltas)
+    add_real_time_callbacks if behaviours.include?(:real_time)
+    add_update_callbacks    if behaviours.include?(:updates)
+  end
+
+  private
+
+  attr_reader :model, :reference, :options, :block
+
+  def add_core_callbacks
+    model.after_commit(
+      ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks,
+      on: :destroy
+    )
+  end
+
+  def add_delta_callbacks
+    if path.empty?
+      model.before_save  ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
+      model.after_commit ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
+    else
+      model.after_commit(
+        ThinkingSphinx::ActiveRecord::Callbacks::AssociationDeltaCallbacks
+          .new(path)
+      )
+    end
+  end
+
+  def add_real_time_callbacks
+    model.after_commit(
+      ThinkingSphinx::RealTime.callback_for(reference, path, &block),
+      on: [:create, :update]
+    )
+  end
+
+  def add_update_callbacks
+    model.after_update ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks
+  end
+
+  def behaviours
+    options[:behaviours] || []
+  end
+
+  def path
+    options[:path] || []
+  end
+end

data/lib/thinking_sphinx/callbacks.rb

@@ -3,6 +3,13 @@
 class ThinkingSphinx::Callbacks
   attr_reader :instance
 
+  def self.append(model, reference = nil, options, &block)
+    reference ||= ThinkingSphinx::Configuration.instance.index_set_class.
+      reference_name(model)
+
+    ThinkingSphinx::Callbacks::Appender.call(model, reference, options, &block)
+  end
+
   def self.callbacks(*methods)
     mod = Module.new
     methods.each do |method|
@@ -33,3 +40,5 @@ class ThinkingSphinx::Callbacks
     @instance = instance
   end
 end
+
+require "thinking_sphinx/callbacks/appender"

data/lib/thinking_sphinx/connection/client.rb

@@ -39,7 +39,7 @@ class ThinkingSphinx::Connection::Client
   private
 
   def check(statements)
-    if statements.length > ThinkingSphinx::MAXIMUM_STATEMENT_LENGTH
+    if statements.length > maximum_statement_length
       exception           = ThinkingSphinx::QueryLengthError.new
       exception.statement = statements
       raise exception
@@ -56,6 +56,11 @@ class ThinkingSphinx::Connection::Client
     @client = nil
   end
 
+  def maximum_statement_length
+    @maximum_statement_length ||= ThinkingSphinx::Configuration.instance.
+      settings['maximum_statement_length']
+  end
+
   def perform(statements)
     results_for statements
   rescue => error

data/lib/thinking_sphinx/connection.rb

@@ -16,6 +16,10 @@ module ThinkingSphinx::Connection
     connection_class.new options
   end
 
+  def self.clear
+    @pool = nil
+  end
+
   def self.connection_class
     return ThinkingSphinx::Connection::JRuby if RUBY_PLATFORM == 'java'
 

data/lib/thinking_sphinx/core/index.rb

@@ -11,7 +11,6 @@ module ThinkingSphinx::Core::Index
 
   def initialize(reference, options = {})
     @reference    = reference.to_sym
-    @docinfo      = :extern unless config.settings["skip_docinfo"]
     @options      = options
     @offset       = config.next_offset(options[:offset_as] || reference)
     @type         = 'plain'
@@ -40,7 +39,7 @@ module ThinkingSphinx::Core::Index
   def interpret_definition!
     table_exists = model.table_exists?
     unless table_exists
-      Rails.logger.info "No table exists for #{model}. Index can not be created" 
+      Rails.logger.info "No table exists for #{model}. Index can not be created"
       return
     end
     return if @interpreted_definition

data/lib/thinking_sphinx/deletion.rb

@@ -22,10 +22,6 @@ class ThinkingSphinx::Deletion
 
   attr_reader :index, :ids
 
-  def document_ids_for_keys
-    ids.collect { |id| index.document_id_for_key id }
-  end
-
   def execute(statement)
     statement = statement.gsub(/\s*\n\s*/, ' ').strip
 
@@ -36,11 +32,28 @@ class ThinkingSphinx::Deletion
     end
   end
 
+  class PlainDeletion < ThinkingSphinx::Deletion
+    def perform
+      ids.each_slice(1000) do |some_ids|
+        execute <<-SQL
+UPDATE #{name}
+SET sphinx_deleted = 1
+WHERE sphinx_internal_id IN (#{some_ids.join(', ')})
+        SQL
+      end
+    end
+  end
+
   class RealtimeDeletion < ThinkingSphinx::Deletion
     def perform
       return unless callbacks_enabled?
 
-      execute Riddle::Query::Delete.new(name, document_ids_for_keys).to_sql
+      ids.each_slice(1000) do |some_ids|
+        execute <<-SQL
+DELETE FROM #{name}
+WHERE sphinx_internal_id IN (#{some_ids.join(', ')})
+        SQL
+      end
     end
 
     private
@@ -54,16 +67,4 @@ class ThinkingSphinx::Deletion
       ThinkingSphinx::Configuration.instance
     end
   end
-
-  class PlainDeletion < ThinkingSphinx::Deletion
-    def perform
-      document_ids_for_keys.each_slice(1000) do |document_ids|
-        execute <<-SQL
-UPDATE #{name}
-SET sphinx_deleted = 1
-WHERE id IN (#{document_ids.join(', ')})
-        SQL
-      end
-    end
-  end
 end

data/lib/thinking_sphinx/errors.rb

@@ -38,7 +38,7 @@ end
 class ThinkingSphinx::QueryLengthError < ThinkingSphinx::SphinxError
   def message
     <<-MESSAGE
-The supplied SphinxQL statement is #{statement.length} characters long. The maximum allowed length is #{ThinkingSphinx::MAXIMUM_STATEMENT_LENGTH}.
+The supplied SphinxQL statement is #{statement.length} characters long. The maximum allowed length is #{ThinkingSphinx::Configuration.instance.settings['maximum_statement_length']}.
 
 If this error has been raised during real-time index population, it's probably due to overly large batches of records being processed at once. The default is 1000, but you can lower it on a per-environment basis in config/thinking_sphinx.yml:
 

data/lib/thinking_sphinx/index_set.rb

@@ -34,11 +34,11 @@ class ThinkingSphinx::IndexSet
   end
 
   def classes
-    options[:classes] || []
+    options[:classes] || instances.collect(&:class)
   end
 
   def classes_specified?
-    classes.any? || references_specified?
+    instances.any? || classes.any? || references_specified?
   end
 
   def classes_and_ancestors
@@ -68,6 +68,10 @@ class ThinkingSphinx::IndexSet
     all_indices.select { |index| references.include? index.reference }
   end
 
+  def instances
+    options[:instances] || []
+  end
+
   def mti_classes
     classes.reject { |klass|
       klass.column_names.include?(klass.inheritance_column)
@@ -76,7 +80,7 @@ class ThinkingSphinx::IndexSet
 
   def references
     options[:references] || classes_and_ancestors.collect { |klass|
-      ThinkingSphinx::IndexSet.reference_name(klass)
+      self.class.reference_name(klass)
     }
   end
 

data/lib/thinking_sphinx/middlewares/sphinxql.rb

@@ -133,7 +133,7 @@ class ThinkingSphinx::Middlewares::SphinxQL <
 
     def indices_match_classes?
       indices.collect(&:reference).uniq.sort == classes.collect { |klass|
-        ThinkingSphinx::IndexSet.reference_name(klass)
+        configuration.index_set_class.reference_name(klass)
       }.sort
     end
 

data/lib/thinking_sphinx/railtie.rb

@@ -5,21 +5,34 @@ class ThinkingSphinx::Railtie < Rails::Railtie
     ThinkingSphinx::Configuration.reset
   end
 
+  config.after_initialize do
+    require 'thinking_sphinx/active_record'
+  end
+
   initializer 'thinking_sphinx.initialisation' do
     ActiveSupport.on_load(:active_record) do
-      ActiveRecord::Base.send :include, ThinkingSphinx::ActiveRecord::Base
+      require 'thinking_sphinx/active_record'
     end
 
-    if ActiveSupport::VERSION::MAJOR > 5
-      if Rails.application.config.autoloader == :zeitwerk
-        ActiveSupport::Dependencies.autoload_paths.delete(
-          Rails.root.join("app", "indices").to_s
-        )
-      end
+    if zeitwerk?
+      ActiveSupport::Dependencies.autoload_paths.delete(
+        Rails.root.join("app", "indices").to_s
+      )
     end
+
+    Rails.application.config.eager_load_paths -=
+      ThinkingSphinx::Configuration.instance.index_paths
+    Rails.application.config.eager_load_paths.freeze
   end
 
   rake_tasks do
     load File.expand_path('../tasks.rb', __FILE__)
   end
+
+  def zeitwerk?
+    return true if ActiveSupport::VERSION::MAJOR >= 7
+    return false if ActiveSupport::VERSION::MAJOR <= 5
+
+    Rails.application.config.autoloader == :zeitwerk
+  end
 end

data/lib/thinking_sphinx/real_time/index/template.rb

@@ -13,6 +13,10 @@ class ThinkingSphinx::RealTime::Index::Template
     add_attribute primary_key,  :sphinx_internal_id,    :bigint
     add_attribute class_column, :sphinx_internal_class, :string, :facet => true
     add_attribute 0,            :sphinx_deleted,        :integer
+
+    if tidying?
+      add_attribute -> (_) { Time.current.to_i }, :sphinx_updated_at, :timestamp
+    end
   end
 
   private
@@ -34,7 +38,15 @@ class ThinkingSphinx::RealTime::Index::Template
     [:class, :name]
   end
 
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+
   def primary_key
     index.primary_key.to_sym
   end
+
+  def tidying?
+    config.settings["real_time_tidy"]
+  end
 end

data/lib/thinking_sphinx/real_time/index.rb

@@ -16,10 +16,14 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
   end
 
   def add_attribute(attribute)
+    @attributes.delete_if { |existing| existing.name == attribute.name }
+
     @attributes << attribute
   end
 
   def add_field(field)
+    @fields.delete_if { |existing| existing.name == field.name }
+
     @fields << field
   end
 
@@ -61,12 +65,10 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
 
   def collection_for(attribute)
     case attribute.type
-    when :integer, :boolean
+    when :integer, :boolean, :timestamp
       attribute.multi? ? @rt_attr_multi : @rt_attr_uint
     when :string
       @rt_attr_string
-    when :timestamp
-      @rt_attr_timestamp
     when :float
       @rt_attr_float
     when :bigint

data/lib/thinking_sphinx/real_time/interpreter.rb

@@ -5,16 +5,18 @@ class ThinkingSphinx::RealTime::Interpreter <
 
   def has(*columns)
     options = columns.extract_options!
-    @index.attributes += columns.collect { |column|
+
+    columns.collect { |column|
       ::ThinkingSphinx::RealTime::Attribute.new column, options
-    }
+    }.each { |attribute| @index.add_attribute attribute }
   end
 
   def indexes(*columns)
     options = columns.extract_options!
-    @index.fields += columns.collect { |column|
+
+    columns.collect { |column|
       ::ThinkingSphinx::RealTime::Field.new column, options
-    }
+    }.each { |field| @index.add_field field }
 
     append_sortable_attributes columns, options if options[:sortable]
   end
@@ -39,7 +41,7 @@ class ThinkingSphinx::RealTime::Interpreter <
   def append_sortable_attributes(columns, options)
     options = options.except(:sortable).merge(:type => :string)
 
-    @index.attributes += columns.collect { |column|
+    columns.collect { |column|
       aliased_name   = options[:as]
       aliased_name ||= column.__name.to_sym if column.respond_to?(:__name)
       aliased_name ||= column
@@ -47,6 +49,6 @@ class ThinkingSphinx::RealTime::Interpreter <
       options[:as] = "#{aliased_name}_sort".to_sym
 
       ::ThinkingSphinx::RealTime::Attribute.new column, options
-    }
+    }.each { |attribute| @index.add_attribute attribute }
   end
 end

data/lib/thinking_sphinx/real_time/populator.rb

@@ -7,6 +7,7 @@ class ThinkingSphinx::RealTime::Populator
 
   def initialize(index)
     @index = index
+    @started_at = Time.current
   end
 
   def populate
@@ -17,12 +18,14 @@ class ThinkingSphinx::RealTime::Populator
       instrument 'populated', :instances => instances
     end
 
+    transcriber.clear_before(started_at) if configuration.settings["real_time_tidy"]
+
     instrument 'finish_populating'
   end
 
   private
 
-  attr_reader :index
+  attr_reader :index, :started_at
 
   delegate :controller, :batch_size, :to => :configuration
   delegate :scope,                   :to => :index

data/lib/thinking_sphinx/real_time/transcriber.rb

@@ -5,31 +5,20 @@ class ThinkingSphinx::RealTime::Transcriber
     @index = index
   end
 
+  def clear_before(time)
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_updated_at < #{time.to_i}
+    SQL
+  end
+
   def copy(*instances)
     items = instances.select { |instance|
       instance.persisted? && copy?(instance)
     }
     return unless items.present?
 
-    values = []
-    items.each do |instance|
-      begin
-        values << ThinkingSphinx::RealTime::TranscribeInstance.call(
-          instance, index, properties
-        )
-      rescue ThinkingSphinx::TranscriptionError => error
-        instrument 'error', :error => error
-      end
-    end
-
-    insert = Riddle::Query::Insert.new index.name, columns, values
-    sphinxql = insert.replace!.to_sql
-
-    ThinkingSphinx::Logger.log :query, sphinxql do
-      ThinkingSphinx::Connection.take do |connection|
-        connection.execute sphinxql
-      end
-    end
+    delete_existing items
+    insert_replacements items
   end
 
   private
@@ -55,6 +44,27 @@ class ThinkingSphinx::RealTime::Transcriber
     }
   end
 
+  def delete_existing(instances)
+    ids = instances.collect(&index.primary_key.to_sym)
+
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_internal_id IN (#{ids.join(', ')})
+    SQL
+  end
+
+  def execute(sphinxql)
+    ThinkingSphinx::Logger.log :query, sphinxql do
+      ThinkingSphinx::Connection.take do |connection|
+        connection.execute sphinxql
+      end
+    end
+  end
+
+  def insert_replacements(instances)
+    insert = Riddle::Query::Insert.new index.name, columns, values(instances)
+    execute insert.replace!.to_sql
+  end
+
   def instrument(message, options = {})
     ActiveSupport::Notifications.instrument(
       "#{message}.thinking_sphinx.real_time", options.merge(:index => index)
@@ -64,4 +74,16 @@ class ThinkingSphinx::RealTime::Transcriber
   def properties
     @properties ||= index.fields + index.attributes
   end
+
+  def values(instances)
+    instances.each_with_object([]) do |instance, array|
+      begin
+        array << ThinkingSphinx::RealTime::TranscribeInstance.call(
+          instance, index, properties
+        )
+      rescue ThinkingSphinx::TranscriptionError => error
+        instrument 'error', :error => error
+      end
+    end
+  end
 end

data/lib/thinking_sphinx/real_time/translator.rb

@@ -10,6 +10,7 @@ class ThinkingSphinx::RealTime::Translator
   end
 
   def call
+    return name.call(object) if name.is_a?(Proc)
     return name   unless name.is_a?(Symbol)
     return result unless result.is_a?(String)
 

data/lib/thinking_sphinx/search/stale_ids_exception.rb

@@ -9,6 +9,7 @@ class ThinkingSphinx::Search::StaleIdsException < StandardError
   end
 
   def message
-    "Record IDs found by Sphinx but not by ActiveRecord : #{ids.join(', ')}"
+    "Record IDs found by Sphinx but not by ActiveRecord : #{ids.join(', ')}\n" \
+    "https://freelancing-gods.com/thinking-sphinx/v5/common_issues.html#record-ids"
   end
 end

data/lib/thinking_sphinx/search.rb

@@ -2,7 +2,7 @@
 
 class ThinkingSphinx::Search < Array
   CORE_METHODS = %w( == class class_eval extend frozen? id instance_eval
-    instance_of? instance_values instance_variable_defined?
+    instance_exec instance_of? instance_values instance_variable_defined?
     instance_variable_get instance_variable_set instance_variables is_a?
     kind_of? member? method methods nil? object_id respond_to?
     respond_to_missing? send should should_not type )