thinking-sphinx 3.1.4 → 6.0.0

data/lib/thinking_sphinx/middlewares/stale_id_filter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Middlewares::StaleIdFilter <
   ThinkingSphinx::Middlewares::Middleware
 
@@ -11,7 +13,7 @@ class ThinkingSphinx::Middlewares::StaleIdFilter <
     rescue ThinkingSphinx::Search::StaleIdsException => error
       raise error if @retries <= 0
 
-      append_stale_ids error.ids
+      append_stale_ids error.ids, error.context
       ThinkingSphinx::Logger.log :message, log_message
 
       @retries -= 1 and retry
@@ -20,7 +22,7 @@ class ThinkingSphinx::Middlewares::StaleIdFilter <
 
   private
 
-  def append_stale_ids(ids)
+  def append_stale_ids(ids, context)
     @stale_ids |= ids
 
     context.search.options[:without_ids] ||= []

data/lib/thinking_sphinx/middlewares/valid_options.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Middlewares::ValidOptions <
+  ThinkingSphinx::Middlewares::Middleware
+
+  def call(contexts)
+    contexts.each { |context| check_options context.search.options }
+
+    app.call contexts
+  end
+
+  private
+
+  def check_options(options)
+    unknown = invalid_keys options.keys
+    return if unknown.empty?
+
+    ThinkingSphinx::Logger.log :caution,
+      "Unexpected search options: #{unknown.inspect}"
+  end
+
+  def invalid_keys(keys)
+    keys - ThinkingSphinx::Search.valid_options
+  end
+end

data/lib/thinking_sphinx/middlewares.rb

@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Middlewares; end
 
-%w[middleware active_record_translator geographer glazier ids_only inquirer
-   sphinxql stale_id_checker stale_id_filter utf8].each do |middleware|
+%w[
+  middleware active_record_translator geographer glazier ids_only inquirer
+  sphinxql stale_id_checker stale_id_filter valid_options
+].each do |middleware|
   require "thinking_sphinx/middlewares/#{middleware}"
 end
 
@@ -10,7 +14,7 @@ module ThinkingSphinx::Middlewares
     middlewares.each { |m| builder.use m }
   end
 
-  BASE_MIDDLEWARES = [SphinxQL, Geographer, Inquirer]
+  BASE_MIDDLEWARES = [ValidOptions, SphinxQL, Geographer, Inquirer]
 
   DEFAULT = ::Middleware::Builder.new do
     use StaleIdFilter

data/lib/thinking_sphinx/panes/attributes_pane.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Panes::AttributesPane
   def initialize(context, object, raw)
     @raw = raw

data/lib/thinking_sphinx/panes/distance_pane.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Panes::DistancePane
   def initialize(context, object, raw)
     @raw = raw

data/lib/thinking_sphinx/panes/excerpts_pane.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Panes::ExcerptsPane
   def initialize(context, object, raw)
     @context, @object = context, object

data/lib/thinking_sphinx/panes/weight_pane.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Panes::WeightPane
   def initialize(context, object, raw)
     @raw = raw
   end
 
   def weight
-    @raw[ThinkingSphinx::SphinxQL.weight[:column]]
+    @raw["weight()"]
   end
 end

data/lib/thinking_sphinx/panes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Panes
   #
 end

data/lib/thinking_sphinx/processor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Processor
+  # @param instance [ActiveRecord::Base] an ActiveRecord object
+  # @param model [Class] the ActiveRecord model of the instance
+  # @param id [Integer] the instance indices primary key (might be different from model primary key)
+  def initialize(instance: nil, model: nil, id: nil)
+    raise ArgumentError if instance.nil? && (model.nil? || id.nil?)
+
+    @instance = instance
+    @model = model || instance.class
+    @id = id
+  end
+
+  def delete
+    return if instance&.new_record?
+
+    indices.each { |index| perform_deletion(index) }
+  end
+
+  # Will insert instance into all matching indices
+  def upsert
+    real_time_indices.each do |index|
+      found = loaded_instance(index)
+      ThinkingSphinx::RealTime::Transcriber.new(index).copy found if found
+    end
+  end
+
+  # Will upsert or delete instance into all matching indices based on index scope
+  def sync
+    real_time_indices.each do |index|
+      found = find_in(index)
+
+      if found
+        ThinkingSphinx::RealTime::Transcriber.new(index).copy found
+      else
+        ThinkingSphinx::Deletion.perform(index, index_id(index))
+      end
+    end
+  end
+
+  private
+
+  attr_reader :instance, :model, :id
+
+  def indices
+    ThinkingSphinx::Configuration.instance.index_set_class.new(
+      :instances => [instance].compact, :classes => [model]
+    ).to_a
+  end
+
+  def find_in(index)
+    index.scope.find_by(index.primary_key => index_id(index))
+  end
+
+  def loaded_instance(index)
+    instance || find_in(index)
+  end
+
+  def real_time_indices
+    indices.select { |index| index.is_a? ThinkingSphinx::RealTime::Index }
+  end
+
+  def perform_deletion(index)
+    ThinkingSphinx::Deletion.perform(index, index_id(index))
+  end
+
+  def index_id(index)
+    id || instance.public_send(index.primary_key)
+  end
+end

data/lib/thinking_sphinx/query.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Query
   def self.escape(query)
     Riddle::Query.escape query

data/lib/thinking_sphinx/railtie.rb

@@ -1,11 +1,38 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Railtie < Rails::Railtie
+  config.to_prepare do
+    ThinkingSphinx::Configuration.reset
+  end
+
+  config.after_initialize do
+    require 'thinking_sphinx/active_record'
+  end
+
   initializer 'thinking_sphinx.initialisation' do
-    if defined?(ActiveRecord::Base)
-      ActiveRecord::Base.send :include, ThinkingSphinx::ActiveRecord::Base
+    ActiveSupport.on_load(:active_record) do
+      require 'thinking_sphinx/active_record'
     end
+
+    if zeitwerk?
+      ActiveSupport::Dependencies.autoload_paths.delete(
+        Rails.root.join("app", "indices").to_s
+      )
+    end
+
+    Rails.application.config.eager_load_paths -=
+      ThinkingSphinx::Configuration.instance.index_paths
+    Rails.application.config.eager_load_paths.freeze
   end
 
   rake_tasks do
     load File.expand_path('../tasks.rb', __FILE__)
   end
+
+  def zeitwerk?
+    return true if ActiveSupport::VERSION::MAJOR >= 7
+    return false if ActiveSupport::VERSION::MAJOR <= 5
+
+    Rails.application.config.autoloader == :zeitwerk
+  end
 end

data/lib/thinking_sphinx/rake_interface.rb

@@ -1,87 +1,32 @@
-class ThinkingSphinx::RakeInterface
-  def clear_all
-    [
-      configuration.indices_location,
-      configuration.searchd.binlog_path
-    ].each do |path|
-      FileUtils.rm_r(path) if File.exists?(path)
-    end
-  end
+# frozen_string_literal: true
 
-  def clear_real_time
-    indices = configuration.indices.select { |index| index.type == 'rt' }
-    indices.each do |index|
-      index.render
-      Dir["#{index.path}.*"].each { |path| FileUtils.rm path }
-    end
+class ThinkingSphinx::RakeInterface
+  DEFAULT_OPTIONS = {:verbose => true}
 
-    path = configuration.searchd.binlog_path
-    FileUtils.rm_r(path) if File.exists?(path)
+  def initialize(options = {})
+    @options           = DEFAULT_OPTIONS.merge options
+    @options[:verbose] = false if @options[:silent]
   end
 
   def configure
-    puts "Generating configuration to #{configuration.configuration_file}"
-    configuration.render_to_file
-  end
-
-  def generate
-    indices = configuration.indices.select { |index| index.type == 'rt' }
-    indices.each do |index|
-      ThinkingSphinx::RealTime::Populator.populate index
-    end
-  end
-
-  def index(reconfigure = true, verbose = true)
-    configure if reconfigure
-    FileUtils.mkdir_p configuration.indices_location
-    ThinkingSphinx.before_index_hooks.each { |hook| hook.call }
-    controller.index :verbose => verbose
+    ThinkingSphinx::Commander.call :configure, configuration, options
   end
 
-  def prepare
-    configuration.preload_indices
-    configuration.render
-
-    FileUtils.mkdir_p configuration.indices_location
+  def daemon
+    @daemon ||= ThinkingSphinx::Interfaces::Daemon.new configuration, options
   end
 
-  def start
-    raise RuntimeError, 'searchd is already running' if controller.running?
-
-    FileUtils.mkdir_p configuration.indices_location
-    controller.start
-
-    if controller.running?
-      puts "Started searchd successfully (pid: #{controller.pid})."
-    else
-      puts "Failed to start searchd. Check the log files for more information."
-    end
+  def rt
+    @rt ||= ThinkingSphinx::Interfaces::RealTime.new configuration, options
   end
 
-  def status
-    if controller.running?
-      puts "The Sphinx daemon searchd is currently running."
-    else
-      puts "The Sphinx daemon searchd is not currently running."
-    end
-  end
-
-  def stop
-    unless controller.running?
-      puts 'searchd is not currently running.' and return
-    end
-
-    pid = controller.pid
-    until controller.stop do
-      sleep(0.5)
-    end
-
-    puts "Stopped searchd daemon (pid: #{pid})."
+  def sql
+    @sql ||= ThinkingSphinx::Interfaces::SQL.new configuration, options
   end
 
   private
 
-  delegate :controller, :to => :configuration
+  attr_reader :options
 
   def configuration
     ThinkingSphinx::Configuration.instance

data/lib/thinking_sphinx/real_time/attribute.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Attribute < ThinkingSphinx::RealTime::Property
   def multi?
     @options[:multi]
@@ -8,7 +10,9 @@ class ThinkingSphinx::RealTime::Attribute < ThinkingSphinx::RealTime::Property
   end
 
   def translate(object)
-    super || default_value
+    output = super || default_value
+
+    json? ? output.to_json : output
   end
 
   private
@@ -16,4 +20,8 @@ class ThinkingSphinx::RealTime::Attribute < ThinkingSphinx::RealTime::Property
   def default_value
     type == :string ? '' : 0
   end
+
+  def json?
+    type == :json
+  end
 end

data/lib/thinking_sphinx/real_time/callbacks/real_time_callbacks.rb

@@ -1,16 +1,16 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Callbacks::RealTimeCallbacks
   def initialize(reference, path = [], &block)
     @reference, @path, @block = reference, path, block
   end
 
-  def after_save(instance)
-    return unless real_time_indices? && callbacks_enabled?
+  def after_commit(instance)
+    persist_changes instance
+  end
 
-    real_time_indices.each do |index|
-      objects_for(instance).each do |object|
-        ThinkingSphinx::RealTime::Transcriber.new(index).copy object
-      end
-    end
+  def after_save(instance)
+    persist_changes instance
   end
 
   private
@@ -40,6 +40,16 @@ class ThinkingSphinx::RealTime::Callbacks::RealTimeCallbacks
     Array results
   end
 
+  def persist_changes(instance)
+    return unless real_time_indices? && callbacks_enabled?
+
+    real_time_indices.each do |index|
+      objects_for(instance).each do |object|
+        ThinkingSphinx::RealTime::Transcriber.new(index).copy object
+      end
+    end
+  end
+
   def real_time_indices?
     real_time_indices.any?
   end

data/lib/thinking_sphinx/real_time/field.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Field < ThinkingSphinx::RealTime::Property
   include ThinkingSphinx::Core::Field
 

data/lib/thinking_sphinx/real_time/index/template.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Index::Template
   attr_reader :index
 
@@ -8,9 +10,13 @@ class ThinkingSphinx::RealTime::Index::Template
   def apply
     add_field class_column, :sphinx_internal_class_name
 
-    add_attribute :id,          :sphinx_internal_id,    :bigint
+    add_attribute primary_key,  :sphinx_internal_id,    :bigint
     add_attribute class_column, :sphinx_internal_class, :string, :facet => true
     add_attribute 0,            :sphinx_deleted,        :integer
+
+    if tidying?
+      add_attribute -> (_) { Time.current.to_i }, :sphinx_updated_at, :timestamp
+    end
   end
 
   private
@@ -31,4 +37,16 @@ class ThinkingSphinx::RealTime::Index::Template
   def class_column
     [:class, :name]
   end
+
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+
+  def primary_key
+    index.primary_key.to_sym
+  end
+
+  def tidying?
+    config.settings["real_time_tidy"]
+  end
 end

data/lib/thinking_sphinx/real_time/index.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
   include ThinkingSphinx::Core::Index
 
@@ -8,16 +10,20 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
     @attributes = []
     @conditions = []
 
-    Template.new(self).apply
-
     super reference, options
+
+    Template.new(self).apply
   end
 
   def add_attribute(attribute)
+    @attributes.delete_if { |existing| existing.name == attribute.name }
+
     @attributes << attribute
   end
 
   def add_field(field)
+    @fields.delete_if { |existing| existing.name == field.name }
+
     @fields << field
   end
 
@@ -59,16 +65,16 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
 
   def collection_for(attribute)
     case attribute.type
-    when :integer, :boolean
+    when :integer, :boolean, :timestamp
       attribute.multi? ? @rt_attr_multi : @rt_attr_uint
     when :string
       @rt_attr_string
-    when :timestamp
-      @rt_attr_timestamp
     when :float
       @rt_attr_float
     when :bigint
       attribute.multi? ? @rt_attr_multi_64 : @rt_attr_bigint
+    when :json
+      @rt_attr_json
     else
       raise "Unknown attribute type '#{attribute.type}'"
     end

data/lib/thinking_sphinx/real_time/interpreter.rb

@@ -1,18 +1,22 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Interpreter <
   ::ThinkingSphinx::Core::Interpreter
 
   def has(*columns)
     options = columns.extract_options!
-    @index.attributes += columns.collect { |column|
+
+    columns.collect { |column|
       ::ThinkingSphinx::RealTime::Attribute.new column, options
-    }
+    }.each { |attribute| @index.add_attribute attribute }
   end
 
   def indexes(*columns)
     options = columns.extract_options!
-    @index.fields += columns.collect { |column|
+
+    columns.collect { |column|
       ::ThinkingSphinx::RealTime::Field.new column, options
-    }
+    }.each { |field| @index.add_field field }
 
     append_sortable_attributes columns, options if options[:sortable]
   end
@@ -37,7 +41,7 @@ class ThinkingSphinx::RealTime::Interpreter <
   def append_sortable_attributes(columns, options)
     options = options.except(:sortable).merge(:type => :string)
 
-    @index.attributes += columns.collect { |column|
+    columns.collect { |column|
       aliased_name   = options[:as]
       aliased_name ||= column.__name.to_sym if column.respond_to?(:__name)
       aliased_name ||= column
@@ -45,6 +49,6 @@ class ThinkingSphinx::RealTime::Interpreter <
       options[:as] = "#{aliased_name}_sort".to_sym
 
       ::ThinkingSphinx::RealTime::Attribute.new column, options
-    }
+    }.each { |attribute| @index.add_attribute attribute }
   end
 end

data/lib/thinking_sphinx/real_time/populator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Populator
   def self.populate(index)
     new(index).populate
@@ -5,28 +7,28 @@ class ThinkingSphinx::RealTime::Populator
 
   def initialize(index)
     @index = index
+    @started_at = Time.current
   end
 
-  def populate(&block)
+  def populate
     instrument 'start_populating'
 
-    remove_files
-
-    scope.find_each do |instance|
-      transcriber.copy instance
-      instrument 'populated', :instance => instance
+    scope.find_in_batches(:batch_size => batch_size) do |instances|
+      transcriber.copy *instances
+      instrument 'populated', :instances => instances
     end
 
-    controller.rotate
+    transcriber.clear_before(started_at) if configuration.settings["real_time_tidy"]
+
     instrument 'finish_populating'
   end
 
   private
 
-  attr_reader :index
+  attr_reader :index, :started_at
 
-  delegate :controller, :to => :configuration
-  delegate :scope,      :to => :index
+  delegate :controller, :batch_size, :to => :configuration
+  delegate :scope,                   :to => :index
 
   def configuration
     ThinkingSphinx::Configuration.instance
@@ -38,10 +40,6 @@ class ThinkingSphinx::RealTime::Populator
     )
   end
 
-  def remove_files
-    Dir["#{index.path}*"].each { |file| FileUtils.rm file }
-  end
-
   def transcriber
     @transcriber ||= ThinkingSphinx::RealTime::Transcriber.new index
   end

data/lib/thinking_sphinx/real_time/processor.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Processor
+  def self.call(indices, &block)
+    new(indices).call(&block)
+  end
+
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def call(&block)
+    subscribe_to_progress
+
+    indices.each do |index|
+      ThinkingSphinx::RealTime.populator.populate index
+
+      block.call
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def command
+    ThinkingSphinx::Commander.call(
+      command, configuration, options, stream
+    )
+  end
+
+  def subscribe_to_progress
+    ThinkingSphinx::Subscribers::PopulatorSubscriber.
+      attach_to 'thinking_sphinx.real_time'
+  end
+end

data/lib/thinking_sphinx/real_time/property.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Property
   include ThinkingSphinx::Core::Property
 
@@ -14,10 +16,6 @@ class ThinkingSphinx::RealTime::Property
   end
 
   def translate(object)
-    return @column.__name unless @column.__name.is_a?(Symbol)
-
-    base = @column.__stack.inject(object) { |base, node| base.try(node) }
-    base = base.try(@column.__name)
-    base.is_a?(String) ? base.gsub("\u0000", '') : base
+    ThinkingSphinx::RealTime::Translator.call(object, @column)
   end
 end