thinking-sphinx 4.4.1 → 5.2.1

data/lib/thinking_sphinx/real_time/index.rb

@@ -16,10 +16,14 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
   end
 
   def add_attribute(attribute)
+    @attributes.delete_if { |existing| existing.name == attribute.name }
+
     @attributes << attribute
   end
 
   def add_field(field)
+    @fields.delete_if { |existing| existing.name == field.name }
+
     @fields << field
   end
 
@@ -61,12 +65,10 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
 
   def collection_for(attribute)
     case attribute.type
-    when :integer, :boolean
+    when :integer, :boolean, :timestamp
       attribute.multi? ? @rt_attr_multi : @rt_attr_uint
     when :string
       @rt_attr_string
-    when :timestamp
-      @rt_attr_timestamp
     when :float
       @rt_attr_float
     when :bigint

data/lib/thinking_sphinx/real_time/index/template.rb

@@ -13,6 +13,10 @@ class ThinkingSphinx::RealTime::Index::Template
     add_attribute primary_key,  :sphinx_internal_id,    :bigint
     add_attribute class_column, :sphinx_internal_class, :string, :facet => true
     add_attribute 0,            :sphinx_deleted,        :integer
+
+    if tidying?
+      add_attribute -> (_) { Time.current.to_i }, :sphinx_updated_at, :timestamp
+    end
   end
 
   private
@@ -34,7 +38,15 @@ class ThinkingSphinx::RealTime::Index::Template
     [:class, :name]
   end
 
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+
   def primary_key
     index.primary_key.to_sym
   end
+
+  def tidying?
+    config.settings["real_time_tidy"]
+  end
 end

data/lib/thinking_sphinx/real_time/interpreter.rb

@@ -5,16 +5,18 @@ class ThinkingSphinx::RealTime::Interpreter <
 
   def has(*columns)
     options = columns.extract_options!
-    @index.attributes += columns.collect { |column|
+
+    columns.collect { |column|
       ::ThinkingSphinx::RealTime::Attribute.new column, options
-    }
+    }.each { |attribute| @index.add_attribute attribute }
   end
 
   def indexes(*columns)
     options = columns.extract_options!
-    @index.fields += columns.collect { |column|
+
+    columns.collect { |column|
       ::ThinkingSphinx::RealTime::Field.new column, options
-    }
+    }.each { |field| @index.add_field field }
 
     append_sortable_attributes columns, options if options[:sortable]
   end
@@ -39,7 +41,7 @@ class ThinkingSphinx::RealTime::Interpreter <
   def append_sortable_attributes(columns, options)
     options = options.except(:sortable).merge(:type => :string)
 
-    @index.attributes += columns.collect { |column|
+    columns.collect { |column|
       aliased_name   = options[:as]
       aliased_name ||= column.__name.to_sym if column.respond_to?(:__name)
       aliased_name ||= column
@@ -47,6 +49,6 @@ class ThinkingSphinx::RealTime::Interpreter <
       options[:as] = "#{aliased_name}_sort".to_sym
 
       ::ThinkingSphinx::RealTime::Attribute.new column, options
-    }
+    }.each { |attribute| @index.add_attribute attribute }
   end
 end

data/lib/thinking_sphinx/real_time/populator.rb

@@ -7,6 +7,7 @@ class ThinkingSphinx::RealTime::Populator
 
   def initialize(index)
     @index = index
+    @started_at = Time.current
   end
 
   def populate
@@ -17,12 +18,14 @@ class ThinkingSphinx::RealTime::Populator
       instrument 'populated', :instances => instances
     end
 
+    transcriber.clear_before(started_at) if configuration.settings["real_time_tidy"]
+
     instrument 'finish_populating'
   end
 
   private
 
-  attr_reader :index
+  attr_reader :index, :started_at
 
   delegate :controller, :batch_size, :to => :configuration
   delegate :scope,                   :to => :index

data/lib/thinking_sphinx/real_time/transcriber.rb

@@ -5,31 +5,20 @@ class ThinkingSphinx::RealTime::Transcriber
     @index = index
   end
 
+  def clear_before(time)
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_updated_at < #{time.to_i}
+    SQL
+  end
+
   def copy(*instances)
     items = instances.select { |instance|
       instance.persisted? && copy?(instance)
     }
     return unless items.present?
 
-    values = []
-    items.each do |instance|
-      begin
-        values << ThinkingSphinx::RealTime::TranscribeInstance.call(
-          instance, index, properties
-        )
-      rescue ThinkingSphinx::TranscriptionError => error
-        instrument 'error', :error => error
-      end
-    end
-
-    insert = Riddle::Query::Insert.new index.name, columns, values
-    sphinxql = insert.replace!.to_sql
-
-    ThinkingSphinx::Logger.log :query, sphinxql do
-      ThinkingSphinx::Connection.take do |connection|
-        connection.execute sphinxql
-      end
-    end
+    delete_existing items
+    insert_replacements items
   end
 
   private
@@ -55,6 +44,27 @@ class ThinkingSphinx::RealTime::Transcriber
     }
   end
 
+  def delete_existing(instances)
+    ids = instances.collect(&index.primary_key.to_sym)
+
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_internal_id IN (#{ids.join(', ')})
+    SQL
+  end
+
+  def execute(sphinxql)
+    ThinkingSphinx::Logger.log :query, sphinxql do
+      ThinkingSphinx::Connection.take do |connection|
+        connection.execute sphinxql
+      end
+    end
+  end
+
+  def insert_replacements(instances)
+    insert = Riddle::Query::Insert.new index.name, columns, values(instances)
+    execute insert.replace!.to_sql
+  end
+
   def instrument(message, options = {})
     ActiveSupport::Notifications.instrument(
       "#{message}.thinking_sphinx.real_time", options.merge(:index => index)
@@ -64,4 +74,16 @@ class ThinkingSphinx::RealTime::Transcriber
   def properties
     @properties ||= index.fields + index.attributes
   end
+
+  def values(instances)
+    instances.each_with_object([]) do |instance, array|
+      begin
+        array << ThinkingSphinx::RealTime::TranscribeInstance.call(
+          instance, index, properties
+        )
+      rescue ThinkingSphinx::TranscriptionError => error
+        instrument 'error', :error => error
+      end
+    end
+  end
 end

data/lib/thinking_sphinx/real_time/translator.rb

@@ -10,6 +10,7 @@ class ThinkingSphinx::RealTime::Translator
   end
 
   def call
+    return name.call(object) if name.is_a?(Proc)
     return name   unless name.is_a?(Symbol)
     return result unless result.is_a?(String)
 

data/lib/thinking_sphinx/settings.rb

@@ -11,14 +11,16 @@ class ThinkingSphinx::Settings
     lemmatizer_base mysql_ssl_cert mysql_ssl_key mysql_ssl_ca
   ].freeze
   DEFAULTS = {
-    "configuration_file" => "config/ENVIRONMENT.sphinx.conf",
-    "indices_location"   => "db/sphinx/ENVIRONMENT",
-    "pid_file"           => "log/ENVIRONMENT.sphinx.pid",
-    "log"                => "log/ENVIRONMENT.searchd.log",
-    "query_log"          => "log/ENVIRONMENT.searchd.query.log",
-    "binlog_path"        => "tmp/binlog/ENVIRONMENT",
-    "workers"            => "threads",
-    "mysql_encoding"     => "utf8"
+    "configuration_file"       => "config/ENVIRONMENT.sphinx.conf",
+    "indices_location"         => "db/sphinx/ENVIRONMENT",
+    "pid_file"                 => "log/ENVIRONMENT.sphinx.pid",
+    "log"                      => "log/ENVIRONMENT.searchd.log",
+    "query_log"                => "log/ENVIRONMENT.searchd.query.log",
+    "binlog_path"              => "tmp/binlog/ENVIRONMENT",
+    "workers"                  => "threads",
+    "mysql_encoding"           => "utf8",
+    "maximum_statement_length" => (2 ** 23) - 5,
+    "real_time_tidy"           => false
   }.freeze
 
   def self.call(configuration)
@@ -62,7 +64,9 @@ class ThinkingSphinx::Settings
 
   def defaults
     DEFAULTS.inject({}) do |hash, (key, value)|
-      value = value.gsub("ENVIRONMENT", framework.environment)
+      if value.is_a?(String)
+        value = value.gsub("ENVIRONMENT", framework.environment)
+      end
 
       if FILE_KEYS.include?(key)
         hash[key] = absolute value

data/spec/acceptance/big_integers_spec.rb

@@ -52,7 +52,7 @@ describe '64 bit document ids', :live => true do
   context 'with Real-Time' do
     it 'handles large 32 bit integers with an offset multiplier' do
       product = Product.create! :name => "Widget"
-      product.update_attributes :id => 980190962
+      product.update :id => 980190962
       expect(
         Product.search('widget', :indices => ['product_core']).to_a
       ).to eq([product])

data/spec/acceptance/geosearching_spec.rb

@@ -38,10 +38,20 @@ describe 'Searching by latitude and longitude', :live => true do
       expected = {:mysql => 250326.906250, :postgresql => 250331.234375}
     end
 
-    if ActiveRecord::Base.configurations['test']['adapter'][/postgres/]
-      expect(cities.first.geodist).to eq(expected[:postgresql])
+    adapter = nil
+
+    if ActiveRecord::VERSION::STRING.to_f > 6.0
+      adapter = ActiveRecord::Base.configurations.configs_for.first.adapter
+    elsif ActiveRecord::VERSION::STRING.to_f > 5.2
+      adapter = ActiveRecord::Base.configurations.configs_for.first.config["adapter"]
+    else
+      adapter = ActiveRecord::Base.configurations['test']['adapter']
+    end
+
+    if adapter[/postgres/]
+      expect(cities.first.geodist).to be_within(0.01).of(expected[:postgresql])
     else # mysql
-      expect(cities.first.geodist).to eq(expected[:mysql])
+      expect(cities.first.geodist).to be_within(0.01).of(expected[:mysql])
     end
   end
 

data/spec/acceptance/merging_spec.rb

@@ -34,7 +34,7 @@ describe "Merging deltas", :live => true do
       Book.search("Space", :indices => ["book_core"]).to_a
     ).to eq([race])
 
-    race.reload.update_attributes :title => "The Hate Race"
+    race.reload.update :title => "The Hate Race"
     sleep 0.25
     expect(
       Book.search("Race", :indices => ["book_delta"]).to_a

data/spec/acceptance/real_time_updates_spec.rb

@@ -11,7 +11,7 @@ describe 'Updates to records in real-time indices', :live => true do
 
   it "handles attributes for sortable fields accordingly" do
     product = Product.create! :name => 'Red Fish'
-    product.update_attributes :name => 'Blue Fish'
+    product.update :name => 'Blue Fish'
 
     expect(Product.search('blue fish', :indices => ['product_core']).to_a).
       to eq([product])
@@ -22,7 +22,7 @@ describe 'Updates to records in real-time indices', :live => true do
 
     expect(Admin::Person.search('Death').to_a).to eq([person])
 
-    person.update_attributes :name => 'Mort'
+    person.update :name => 'Mort'
 
     expect(Admin::Person.search('Death').to_a).to be_empty
     expect(Admin::Person.search('Mort').to_a).to eq([person])

data/spec/acceptance/sql_deltas_spec.rb

@@ -25,7 +25,7 @@ describe 'SQL delta indexing', :live => true do
 
     expect(Book.search('Harry').to_a).to eq([book])
 
-    book.reload.update_attributes(:author => 'Terry Pratchett')
+    book.reload.update(:author => 'Terry Pratchett')
     sleep 0.25
 
     expect(Book.search('Terry').to_a).to eq([book])
@@ -37,7 +37,7 @@ describe 'SQL delta indexing', :live => true do
 
     expect(Book.search('Harry').to_a).to eq([book])
 
-    book.reload.update_attributes(:author => 'Terry Pratchett')
+    book.reload.update(:author => 'Terry Pratchett')
     sleep 0.25
 
     expect(Book.search('Harry')).to be_empty
@@ -49,7 +49,7 @@ describe 'SQL delta indexing', :live => true do
 
     expect(Album.search('Whitloms').to_a).to eq([album])
 
-    album.reload.update_attributes(:artist => 'The Whitlams')
+    album.reload.update(:artist => 'The Whitlams')
     sleep 0.25
 
     expect(Book.search('Whitloms')).to be_empty
@@ -63,4 +63,16 @@ describe 'SQL delta indexing', :live => true do
 
     expect(Book.search('Gaiman').to_a).to eq([book])
   end
+
+  it "updates associated models" do
+    colour = Colour.create(:name => 'green')
+    sleep 0.25
+
+    expect(Colour.search('green').to_a).to eq([colour])
+
+    tee = colour.tees.create
+    sleep 0.25
+
+    expect(Colour.search(:with => {:tee_ids => tee.id}).to_a).to eq([colour])
+  end
 end

data/spec/acceptance/support/sphinx_helpers.rb

@@ -6,22 +6,22 @@ module SphinxHelpers
   end
 
   def index(*indices)
-    sleep 0.5 if ENV['TRAVIS']
+    sleep 0.5 if ENV['CI']
 
     yield if block_given?
 
     sphinx.index *indices
     sleep 0.25
-    sleep 0.5 if ENV['TRAVIS']
+    sleep 0.5 if ENV['CI']
   end
 
   def merge
-    sleep 0.5 if ENV['TRAVIS']
+    sleep 0.5 if ENV['CI']
     sleep 0.5
 
     sphinx.merge
     sleep 1.5
-    sleep 0.5 if ENV['TRAVIS']
+    sleep 0.5 if ENV['CI']
   end
 end
 

data/spec/acceptance/suspended_deltas_spec.rb

@@ -10,7 +10,7 @@ describe 'Suspend deltas for a given action', :live => true do
     expect(Book.search('Harry').to_a).to eq([book])
 
     ThinkingSphinx::Deltas.suspend :book do
-      book.reload.update_attributes(:author => 'Terry Pratchett')
+      book.reload.update(:author => 'Terry Pratchett')
       sleep 0.25
 
       expect(Book.search('Terry').to_a).to eq([])
@@ -27,7 +27,7 @@ describe 'Suspend deltas for a given action', :live => true do
     expect(Book.search('Harry').to_a).to eq([book])
 
     ThinkingSphinx::Deltas.suspend :book do
-      book.reload.update_attributes(:author => 'Terry Pratchett')
+      book.reload.update(:author => 'Terry Pratchett')
       sleep 0.25
 
       expect(Book.search('Terry').to_a).to eq([])
@@ -44,7 +44,7 @@ describe 'Suspend deltas for a given action', :live => true do
     expect(Book.search('Harry').to_a).to eq([book])
 
     ThinkingSphinx::Deltas.suspend_and_update :book do
-      book.reload.update_attributes(:author => 'Terry Pratchett')
+      book.reload.update(:author => 'Terry Pratchett')
       sleep 0.25
 
       expect(Book.search('Terry').to_a).to eq([])

data/spec/internal/app/indices/article_index.rb

@@ -10,7 +10,6 @@ ThinkingSphinx::Index.define :article, :with => :active_record do
   has taggings.created_at, :as => :taggings_at, :type => :timestamp
 
   set_property :min_infix_len => 4
-  set_property :enable_star   => true
 end
 
 ThinkingSphinx::Index.define :article, :with => :active_record,

data/spec/internal/app/indices/colour_index.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+ThinkingSphinx::Index.define :colour, :with => :active_record, :delta => true do
+  indexes name
+
+  has tees.id, :as => :tee_ids
+end

data/spec/internal/app/models/admin/person.rb

@@ -3,5 +3,7 @@
 class Admin::Person < ActiveRecord::Base
   self.table_name = 'admin_people'
 
-  after_save ThinkingSphinx::RealTime.callback_for('admin/person')
+  ThinkingSphinx::Callbacks.append(
+    self, 'admin/person', :behaviours => [:sql, :real_time]
+  )
 end

data/spec/internal/app/models/album.rb

@@ -6,7 +6,9 @@ class Album < ActiveRecord::Base
   before_validation :set_id, :on => :create
   before_validation :set_integer_id, :on => :create
 
-  after_save ThinkingSphinx::RealTime.callback_for(:album)
+  ThinkingSphinx::Callbacks.append(
+    self, :behaviours => [:sql, :real_time, :deltas]
+  )
 
   validates :id,         :presence => true, :uniqueness => true
   validates :integer_id, :presence => true, :uniqueness => true