thinking-sphinx 4.3.2 → 5.2.0

data/lib/thinking_sphinx/middlewares/sphinxql.rb

@@ -133,7 +133,7 @@ class ThinkingSphinx::Middlewares::SphinxQL <
 
     def indices_match_classes?
       indices.collect(&:reference).uniq.sort == classes.collect { |klass|
-        ThinkingSphinx::IndexSet.reference_name(klass)
+        configuration.index_set_class.reference_name(klass)
       }.sort
     end
 

data/lib/thinking_sphinx/railtie.rb

@@ -7,7 +7,20 @@ class ThinkingSphinx::Railtie < Rails::Railtie
 
   initializer 'thinking_sphinx.initialisation' do
     ActiveSupport.on_load(:active_record) do
-      ActiveRecord::Base.send :include, ThinkingSphinx::ActiveRecord::Base
+      require 'thinking_sphinx/active_record'
+      ActiveRecord::Base.include ThinkingSphinx::ActiveRecord::Base
+    end
+
+    if ActiveSupport::VERSION::MAJOR > 5
+      if Rails.application.config.autoloader == :zeitwerk
+        ActiveSupport::Dependencies.autoload_paths.delete(
+          Rails.root.join("app", "indices").to_s
+        )
+      end
+
+      Rails.application.config.eager_load_paths -=
+        ThinkingSphinx::Configuration.instance.index_paths
+      Rails.application.config.eager_load_paths.freeze
     end
   end
 

data/lib/thinking_sphinx/real_time.rb

@@ -8,6 +8,22 @@ module ThinkingSphinx::RealTime
   def self.callback_for(reference, path = [], &block)
     Callbacks::RealTimeCallbacks.new reference.to_sym, path, &block
   end
+
+  def self.populator
+    @populator ||= ThinkingSphinx::RealTime::Populator
+  end
+
+  def self.populator=(value)
+    @populator = value
+  end
+
+  def self.processor
+    @processor ||= ThinkingSphinx::RealTime::Processor
+  end
+
+  def self.processor=(value)
+    @processor = value
+  end
 end
 
 require 'thinking_sphinx/real_time/property'
@@ -16,6 +32,7 @@ require 'thinking_sphinx/real_time/field'
 require 'thinking_sphinx/real_time/index'
 require 'thinking_sphinx/real_time/interpreter'
 require 'thinking_sphinx/real_time/populator'
+require 'thinking_sphinx/real_time/processor'
 require 'thinking_sphinx/real_time/transcribe_instance'
 require 'thinking_sphinx/real_time/transcriber'
 require 'thinking_sphinx/real_time/translator'

data/lib/thinking_sphinx/real_time/index.rb

@@ -16,10 +16,14 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
   end
 
   def add_attribute(attribute)
+    @attributes.delete_if { |existing| existing.name == attribute.name }
+
     @attributes << attribute
   end
 
   def add_field(field)
+    @fields.delete_if { |existing| existing.name == field.name }
+
     @fields << field
   end
 
@@ -61,12 +65,10 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
 
   def collection_for(attribute)
     case attribute.type
-    when :integer, :boolean
+    when :integer, :boolean, :timestamp
       attribute.multi? ? @rt_attr_multi : @rt_attr_uint
     when :string
       @rt_attr_string
-    when :timestamp
-      @rt_attr_timestamp
     when :float
       @rt_attr_float
     when :bigint

data/lib/thinking_sphinx/real_time/index/template.rb

@@ -13,6 +13,10 @@ class ThinkingSphinx::RealTime::Index::Template
     add_attribute primary_key,  :sphinx_internal_id,    :bigint
     add_attribute class_column, :sphinx_internal_class, :string, :facet => true
     add_attribute 0,            :sphinx_deleted,        :integer
+
+    if tidying?
+      add_attribute -> (_) { Time.current.to_i }, :sphinx_updated_at, :timestamp
+    end
   end
 
   private
@@ -34,7 +38,15 @@ class ThinkingSphinx::RealTime::Index::Template
     [:class, :name]
   end
 
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+
   def primary_key
     index.primary_key.to_sym
   end
+
+  def tidying?
+    config.settings["real_time_tidy"]
+  end
 end

data/lib/thinking_sphinx/real_time/interpreter.rb

@@ -5,16 +5,18 @@ class ThinkingSphinx::RealTime::Interpreter <
 
   def has(*columns)
     options = columns.extract_options!
-    @index.attributes += columns.collect { |column|
+
+    columns.collect { |column|
       ::ThinkingSphinx::RealTime::Attribute.new column, options
-    }
+    }.each { |attribute| @index.add_attribute attribute }
   end
 
   def indexes(*columns)
     options = columns.extract_options!
-    @index.fields += columns.collect { |column|
+
+    columns.collect { |column|
       ::ThinkingSphinx::RealTime::Field.new column, options
-    }
+    }.each { |field| @index.add_field field }
 
     append_sortable_attributes columns, options if options[:sortable]
   end
@@ -39,7 +41,7 @@ class ThinkingSphinx::RealTime::Interpreter <
   def append_sortable_attributes(columns, options)
     options = options.except(:sortable).merge(:type => :string)
 
-    @index.attributes += columns.collect { |column|
+    columns.collect { |column|
       aliased_name   = options[:as]
       aliased_name ||= column.__name.to_sym if column.respond_to?(:__name)
       aliased_name ||= column
@@ -47,6 +49,6 @@ class ThinkingSphinx::RealTime::Interpreter <
       options[:as] = "#{aliased_name}_sort".to_sym
 
       ::ThinkingSphinx::RealTime::Attribute.new column, options
-    }
+    }.each { |attribute| @index.add_attribute attribute }
   end
 end

data/lib/thinking_sphinx/real_time/populator.rb

@@ -7,9 +7,10 @@ class ThinkingSphinx::RealTime::Populator
 
   def initialize(index)
     @index = index
+    @started_at = Time.current
   end
 
-  def populate(&block)
+  def populate
     instrument 'start_populating'
 
     scope.find_in_batches(:batch_size => batch_size) do |instances|
@@ -17,12 +18,14 @@ class ThinkingSphinx::RealTime::Populator
       instrument 'populated', :instances => instances
     end
 
+    transcriber.clear_before(started_at) if configuration.settings["real_time_tidy"]
+
     instrument 'finish_populating'
   end
 
   private
 
-  attr_reader :index
+  attr_reader :index, :started_at
 
   delegate :controller, :batch_size, :to => :configuration
   delegate :scope,                   :to => :index

data/lib/thinking_sphinx/real_time/processor.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Processor
+  def self.call(indices, &block)
+    new(indices).call(&block)
+  end
+
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def call(&block)
+    subscribe_to_progress
+
+    indices.each do |index|
+      ThinkingSphinx::RealTime.populator.populate index
+
+      block.call
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def command
+    ThinkingSphinx::Commander.call(
+      command, configuration, options, stream
+    )
+  end
+
+  def subscribe_to_progress
+    ThinkingSphinx::Subscribers::PopulatorSubscriber.
+      attach_to 'thinking_sphinx.real_time'
+  end
+end

data/lib/thinking_sphinx/real_time/transcriber.rb

@@ -5,31 +5,20 @@ class ThinkingSphinx::RealTime::Transcriber
     @index = index
   end
 
+  def clear_before(time)
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_updated_at < #{time.to_i}
+    SQL
+  end
+
   def copy(*instances)
     items = instances.select { |instance|
       instance.persisted? && copy?(instance)
     }
     return unless items.present?
 
-    values = []
-    items.each do |instance|
-      begin
-        values << ThinkingSphinx::RealTime::TranscribeInstance.call(
-          instance, index, properties
-        )
-      rescue ThinkingSphinx::TranscriptionError => error
-        instrument 'error', :error => error
-      end
-    end
-
-    insert = Riddle::Query::Insert.new index.name, columns, values
-    sphinxql = insert.replace!.to_sql
-
-    ThinkingSphinx::Logger.log :query, sphinxql do
-      ThinkingSphinx::Connection.take do |connection|
-        connection.execute sphinxql
-      end
-    end
+    delete_existing items
+    insert_replacements items
   end
 
   private
@@ -55,6 +44,27 @@ class ThinkingSphinx::RealTime::Transcriber
     }
   end
 
+  def delete_existing(instances)
+    ids = instances.collect(&index.primary_key.to_sym)
+
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_internal_id IN (#{ids.join(', ')})
+    SQL
+  end
+
+  def execute(sphinxql)
+    ThinkingSphinx::Logger.log :query, sphinxql do
+      ThinkingSphinx::Connection.take do |connection|
+        connection.execute sphinxql
+      end
+    end
+  end
+
+  def insert_replacements(instances)
+    insert = Riddle::Query::Insert.new index.name, columns, values(instances)
+    execute insert.replace!.to_sql
+  end
+
   def instrument(message, options = {})
     ActiveSupport::Notifications.instrument(
       "#{message}.thinking_sphinx.real_time", options.merge(:index => index)
@@ -64,4 +74,16 @@ class ThinkingSphinx::RealTime::Transcriber
   def properties
     @properties ||= index.fields + index.attributes
   end
+
+  def values(instances)
+    instances.each_with_object([]) do |instance, array|
+      begin
+        array << ThinkingSphinx::RealTime::TranscribeInstance.call(
+          instance, index, properties
+        )
+      rescue ThinkingSphinx::TranscriptionError => error
+        instrument 'error', :error => error
+      end
+    end
+  end
 end

data/lib/thinking_sphinx/real_time/translator.rb

@@ -10,6 +10,7 @@ class ThinkingSphinx::RealTime::Translator
   end
 
   def call
+    return name.call(object) if name.is_a?(Proc)
     return name   unless name.is_a?(Symbol)
     return result unless result.is_a?(String)
 

data/lib/thinking_sphinx/settings.rb

@@ -11,14 +11,16 @@ class ThinkingSphinx::Settings
     lemmatizer_base mysql_ssl_cert mysql_ssl_key mysql_ssl_ca
   ].freeze
   DEFAULTS = {
-    "configuration_file" => "config/ENVIRONMENT.sphinx.conf",
-    "indices_location"   => "db/sphinx/ENVIRONMENT",
-    "pid_file"           => "log/ENVIRONMENT.sphinx.pid",
-    "log"                => "log/ENVIRONMENT.searchd.log",
-    "query_log"          => "log/ENVIRONMENT.searchd.query.log",
-    "binlog_path"        => "tmp/binlog/ENVIRONMENT",
-    "workers"            => "threads",
-    "mysql_encoding"     => "utf8"
+    "configuration_file"       => "config/ENVIRONMENT.sphinx.conf",
+    "indices_location"         => "db/sphinx/ENVIRONMENT",
+    "pid_file"                 => "log/ENVIRONMENT.sphinx.pid",
+    "log"                      => "log/ENVIRONMENT.searchd.log",
+    "query_log"                => "log/ENVIRONMENT.searchd.query.log",
+    "binlog_path"              => "tmp/binlog/ENVIRONMENT",
+    "workers"                  => "threads",
+    "mysql_encoding"           => "utf8",
+    "maximum_statement_length" => (2 ** 23) - 5,
+    "real_time_tidy"           => false
   }.freeze
 
   def self.call(configuration)
@@ -62,7 +64,9 @@ class ThinkingSphinx::Settings
 
   def defaults
     DEFAULTS.inject({}) do |hash, (key, value)|
-      value = value.gsub("ENVIRONMENT", framework.environment)
+      if value.is_a?(String)
+        value = value.gsub("ENVIRONMENT", framework.environment)
+      end
 
       if FILE_KEYS.include?(key)
         hash[key] = absolute value

data/lib/thinking_sphinx/subscribers/populator_subscriber.rb

@@ -46,7 +46,3 @@ Error transcribing #{instance.class} #{instance.id}:
   delegate :output, :to => ThinkingSphinx
   delegate :puts, :print, :to => :output
 end
-
-ThinkingSphinx::Subscribers::PopulatorSubscriber.attach_to(
-  'thinking_sphinx.real_time'
-)

data/spec/acceptance/big_integers_spec.rb

@@ -52,7 +52,7 @@ describe '64 bit document ids', :live => true do
   context 'with Real-Time' do
     it 'handles large 32 bit integers with an offset multiplier' do
       product = Product.create! :name => "Widget"
-      product.update_attributes :id => 980190962
+      product.update :id => 980190962
       expect(
         Product.search('widget', :indices => ['product_core']).to_a
       ).to eq([product])

data/spec/acceptance/geosearching_spec.rb

@@ -38,10 +38,20 @@ describe 'Searching by latitude and longitude', :live => true do
       expected = {:mysql => 250326.906250, :postgresql => 250331.234375}
     end
 
-    if ActiveRecord::Base.configurations['test']['adapter'][/postgres/]
-      expect(cities.first.geodist).to eq(expected[:postgresql])
+    adapter = nil
+
+    if ActiveRecord::VERSION::STRING.to_f > 6.0
+      adapter = ActiveRecord::Base.configurations.configs_for.first.adapter
+    elsif ActiveRecord::VERSION::STRING.to_f > 5.2
+      adapter = ActiveRecord::Base.configurations.configs_for.first.config["adapter"]
+    else
+      adapter = ActiveRecord::Base.configurations['test']['adapter']
+    end
+
+    if adapter[/postgres/]
+      expect(cities.first.geodist).to be_within(0.01).of(expected[:postgresql])
     else # mysql
-      expect(cities.first.geodist).to eq(expected[:mysql])
+      expect(cities.first.geodist).to be_within(0.01).of(expected[:mysql])
     end
   end
 

data/spec/acceptance/merging_spec.rb

@@ -34,7 +34,7 @@ describe "Merging deltas", :live => true do
       Book.search("Space", :indices => ["book_core"]).to_a
     ).to eq([race])
 
-    race.reload.update_attributes :title => "The Hate Race"
+    race.reload.update :title => "The Hate Race"
     sleep 0.25
     expect(
       Book.search("Race", :indices => ["book_delta"]).to_a

data/spec/acceptance/real_time_updates_spec.rb

@@ -11,7 +11,7 @@ describe 'Updates to records in real-time indices', :live => true do
 
   it "handles attributes for sortable fields accordingly" do
     product = Product.create! :name => 'Red Fish'
-    product.update_attributes :name => 'Blue Fish'
+    product.update :name => 'Blue Fish'
 
     expect(Product.search('blue fish', :indices => ['product_core']).to_a).
       to eq([product])
@@ -22,7 +22,7 @@ describe 'Updates to records in real-time indices', :live => true do
 
     expect(Admin::Person.search('Death').to_a).to eq([person])
 
-    person.update_attributes :name => 'Mort'
+    person.update :name => 'Mort'
 
     expect(Admin::Person.search('Death').to_a).to be_empty
     expect(Admin::Person.search('Mort').to_a).to eq([person])

data/spec/acceptance/sql_deltas_spec.rb

@@ -25,7 +25,7 @@ describe 'SQL delta indexing', :live => true do
 
     expect(Book.search('Harry').to_a).to eq([book])
 
-    book.reload.update_attributes(:author => 'Terry Pratchett')
+    book.reload.update(:author => 'Terry Pratchett')
     sleep 0.25
 
     expect(Book.search('Terry').to_a).to eq([book])
@@ -37,7 +37,7 @@ describe 'SQL delta indexing', :live => true do
 
     expect(Book.search('Harry').to_a).to eq([book])
 
-    book.reload.update_attributes(:author => 'Terry Pratchett')
+    book.reload.update(:author => 'Terry Pratchett')
     sleep 0.25
 
     expect(Book.search('Harry')).to be_empty
@@ -49,7 +49,7 @@ describe 'SQL delta indexing', :live => true do
 
     expect(Album.search('Whitloms').to_a).to eq([album])
 
-    album.reload.update_attributes(:artist => 'The Whitlams')
+    album.reload.update(:artist => 'The Whitlams')
     sleep 0.25
 
     expect(Book.search('Whitloms')).to be_empty
@@ -63,4 +63,16 @@ describe 'SQL delta indexing', :live => true do
 
     expect(Book.search('Gaiman').to_a).to eq([book])
   end
+
+  it "updates associated models" do
+    colour = Colour.create(:name => 'green')
+    sleep 0.25
+
+    expect(Colour.search('green').to_a).to eq([colour])
+
+    tee = colour.tees.create
+    sleep 0.25
+
+    expect(Colour.search(:with => {:tee_ids => tee.id}).to_a).to eq([colour])
+  end
 end