thinking-sphinx 1.2.13 → 6.0.0

data/lib/thinking_sphinx/processor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Processor
+  # @param instance [ActiveRecord::Base] an ActiveRecord object
+  # @param model [Class] the ActiveRecord model of the instance
+  # @param id [Integer] the instance indices primary key (might be different from model primary key)
+  def initialize(instance: nil, model: nil, id: nil)
+    raise ArgumentError if instance.nil? && (model.nil? || id.nil?)
+
+    @instance = instance
+    @model = model || instance.class
+    @id = id
+  end
+
+  def delete
+    return if instance&.new_record?
+
+    indices.each { |index| perform_deletion(index) }
+  end
+
+  # Will insert instance into all matching indices
+  def upsert
+    real_time_indices.each do |index|
+      found = loaded_instance(index)
+      ThinkingSphinx::RealTime::Transcriber.new(index).copy found if found
+    end
+  end
+
+  # Will upsert or delete instance into all matching indices based on index scope
+  def sync
+    real_time_indices.each do |index|
+      found = find_in(index)
+
+      if found
+        ThinkingSphinx::RealTime::Transcriber.new(index).copy found
+      else
+        ThinkingSphinx::Deletion.perform(index, index_id(index))
+      end
+    end
+  end
+
+  private
+
+  attr_reader :instance, :model, :id
+
+  def indices
+    ThinkingSphinx::Configuration.instance.index_set_class.new(
+      :instances => [instance].compact, :classes => [model]
+    ).to_a
+  end
+
+  def find_in(index)
+    index.scope.find_by(index.primary_key => index_id(index))
+  end
+
+  def loaded_instance(index)
+    instance || find_in(index)
+  end
+
+  def real_time_indices
+    indices.select { |index| index.is_a? ThinkingSphinx::RealTime::Index }
+  end
+
+  def perform_deletion(index)
+    ThinkingSphinx::Deletion.perform(index, index_id(index))
+  end
+
+  def index_id(index)
+    id || instance.public_send(index.primary_key)
+  end
+end

data/lib/thinking_sphinx/query.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ThinkingSphinx::Query
+  def self.escape(query)
+    Riddle::Query.escape query
+  end
+
+  def self.wildcard(query, pattern = true)
+    ThinkingSphinx::Wildcard.call query, pattern
+  end
+end

data/lib/thinking_sphinx/railtie.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Railtie < Rails::Railtie
+  config.to_prepare do
+    ThinkingSphinx::Configuration.reset
+  end
+
+  config.after_initialize do
+    require 'thinking_sphinx/active_record'
+  end
+
+  initializer 'thinking_sphinx.initialisation' do
+    ActiveSupport.on_load(:active_record) do
+      require 'thinking_sphinx/active_record'
+    end
+
+    if zeitwerk?
+      ActiveSupport::Dependencies.autoload_paths.delete(
+        Rails.root.join("app", "indices").to_s
+      )
+    end
+
+    Rails.application.config.eager_load_paths -=
+      ThinkingSphinx::Configuration.instance.index_paths
+    Rails.application.config.eager_load_paths.freeze
+  end
+
+  rake_tasks do
+    load File.expand_path('../tasks.rb', __FILE__)
+  end
+
+  def zeitwerk?
+    return true if ActiveSupport::VERSION::MAJOR >= 7
+    return false if ActiveSupport::VERSION::MAJOR <= 5
+
+    Rails.application.config.autoloader == :zeitwerk
+  end
+end

data/lib/thinking_sphinx/rake_interface.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RakeInterface
+  DEFAULT_OPTIONS = {:verbose => true}
+
+  def initialize(options = {})
+    @options           = DEFAULT_OPTIONS.merge options
+    @options[:verbose] = false if @options[:silent]
+  end
+
+  def configure
+    ThinkingSphinx::Commander.call :configure, configuration, options
+  end
+
+  def daemon
+    @daemon ||= ThinkingSphinx::Interfaces::Daemon.new configuration, options
+  end
+
+  def rt
+    @rt ||= ThinkingSphinx::Interfaces::RealTime.new configuration, options
+  end
+
+  def sql
+    @sql ||= ThinkingSphinx::Interfaces::SQL.new configuration, options
+  end
+
+  private
+
+  attr_reader :options
+
+  def configuration
+    ThinkingSphinx::Configuration.instance
+  end
+end

data/lib/thinking_sphinx/real_time/attribute.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Attribute < ThinkingSphinx::RealTime::Property
+  def multi?
+    @options[:multi]
+  end
+
+  def type
+    @options[:type]
+  end
+
+  def translate(object)
+    output = super || default_value
+
+    json? ? output.to_json : output
+  end
+
+  private
+
+  def default_value
+    type == :string ? '' : 0
+  end
+
+  def json?
+    type == :json
+  end
+end

data/lib/thinking_sphinx/real_time/callbacks/real_time_callbacks.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Callbacks::RealTimeCallbacks
+  def initialize(reference, path = [], &block)
+    @reference, @path, @block = reference, path, block
+  end
+
+  def after_commit(instance)
+    persist_changes instance
+  end
+
+  def after_save(instance)
+    persist_changes instance
+  end
+
+  private
+
+  attr_reader :reference, :path, :block
+
+  def callbacks_enabled?
+    setting = configuration.settings['real_time_callbacks']
+    setting.nil? || setting
+  end
+
+  def configuration
+    ThinkingSphinx::Configuration.instance
+  end
+
+  def indices
+    configuration.indices_for_references reference
+  end
+
+  def objects_for(instance)
+    if block
+      results = block.call instance
+    else
+      results = path.inject(instance) { |object, method| object.send method }
+    end
+
+    Array results
+  end
+
+  def persist_changes(instance)
+    return unless real_time_indices? && callbacks_enabled?
+
+    real_time_indices.each do |index|
+      objects_for(instance).each do |object|
+        ThinkingSphinx::RealTime::Transcriber.new(index).copy object
+      end
+    end
+  end
+
+  def real_time_indices?
+    real_time_indices.any?
+  end
+
+  def real_time_indices
+    indices.select { |index| index.is_a? ThinkingSphinx::RealTime::Index }
+  end
+end

data/lib/thinking_sphinx/real_time/field.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Field < ThinkingSphinx::RealTime::Property
+  include ThinkingSphinx::Core::Field
+
+  def translate(object)
+    Array(super || '').join(' ')
+  end
+end

data/lib/thinking_sphinx/real_time/index/template.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Index::Template
+  attr_reader :index
+
+  def initialize(index)
+    @index = index
+  end
+
+  def apply
+    add_field class_column, :sphinx_internal_class_name
+
+    add_attribute primary_key,  :sphinx_internal_id,    :bigint
+    add_attribute class_column, :sphinx_internal_class, :string, :facet => true
+    add_attribute 0,            :sphinx_deleted,        :integer
+
+    if tidying?
+      add_attribute -> (_) { Time.current.to_i }, :sphinx_updated_at, :timestamp
+    end
+  end
+
+  private
+
+  def add_attribute(column, name, type, options = {})
+    index.add_attribute ThinkingSphinx::RealTime::Attribute.new(
+      ThinkingSphinx::ActiveRecord::Column.new(*column),
+      options.merge(:as => name, :type => type)
+    )
+  end
+
+  def add_field(column, name)
+    index.add_field ThinkingSphinx::RealTime::Field.new(
+      ThinkingSphinx::ActiveRecord::Column.new(*column), :as => name
+    )
+  end
+
+  def class_column
+    [:class, :name]
+  end
+
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+
+  def primary_key
+    index.primary_key.to_sym
+  end
+
+  def tidying?
+    config.settings["real_time_tidy"]
+  end
+end

data/lib/thinking_sphinx/real_time/index.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
+  include ThinkingSphinx::Core::Index
+
+  attr_writer :fields, :attributes, :conditions, :scope
+
+  def initialize(reference, options = {})
+    @fields     = []
+    @attributes = []
+    @conditions = []
+
+    super reference, options
+
+    Template.new(self).apply
+  end
+
+  def add_attribute(attribute)
+    @attributes.delete_if { |existing| existing.name == attribute.name }
+
+    @attributes << attribute
+  end
+
+  def add_field(field)
+    @fields.delete_if { |existing| existing.name == field.name }
+
+    @fields << field
+  end
+
+  def attributes
+    interpret_definition!
+
+    @attributes
+  end
+
+  def conditions
+    interpret_definition!
+
+    @conditions
+  end
+
+  def facets
+    properties.select(&:facet?)
+  end
+
+  def fields
+    interpret_definition!
+
+    @fields
+  end
+
+  def scope
+    @scope.nil? ? model : @scope.call
+  end
+
+  def unique_attribute_names
+    attributes.collect(&:name)
+  end
+
+  private
+
+  def append_unique_attribute(collection, attribute)
+    collection << attribute.name unless collection.include?(attribute.name)
+  end
+
+  def collection_for(attribute)
+    case attribute.type
+    when :integer, :boolean, :timestamp
+      attribute.multi? ? @rt_attr_multi : @rt_attr_uint
+    when :string
+      @rt_attr_string
+    when :float
+      @rt_attr_float
+    when :bigint
+      attribute.multi? ? @rt_attr_multi_64 : @rt_attr_bigint
+    when :json
+      @rt_attr_json
+    else
+      raise "Unknown attribute type '#{attribute.type}'"
+    end
+  end
+
+  def interpreter
+    ThinkingSphinx::RealTime::Interpreter
+  end
+
+  def pre_render
+    super
+
+    @rt_field = fields.collect &:name
+
+    attributes.each do |attribute|
+      append_unique_attribute collection_for(attribute), attribute
+    end
+  end
+
+  def properties
+    fields + attributes
+  end
+end
+
+require 'thinking_sphinx/real_time/index/template'

data/lib/thinking_sphinx/real_time/interpreter.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Interpreter <
+  ::ThinkingSphinx::Core::Interpreter
+
+  def has(*columns)
+    options = columns.extract_options!
+
+    columns.collect { |column|
+      ::ThinkingSphinx::RealTime::Attribute.new column, options
+    }.each { |attribute| @index.add_attribute attribute }
+  end
+
+  def indexes(*columns)
+    options = columns.extract_options!
+
+    columns.collect { |column|
+      ::ThinkingSphinx::RealTime::Field.new column, options
+    }.each { |field| @index.add_field field }
+
+    append_sortable_attributes columns, options if options[:sortable]
+  end
+
+  def scope(&block)
+    @index.scope = block
+  end
+
+  def set_property(properties)
+    properties.each do |key, value|
+      @index.send("#{key}=", value) if @index.class.settings.include?(key)
+      @index.options[key] = value   if search_option?(key)
+    end
+  end
+
+  def where(condition)
+    @index.conditions << condition
+  end
+
+  private
+
+  def append_sortable_attributes(columns, options)
+    options = options.except(:sortable).merge(:type => :string)
+
+    columns.collect { |column|
+      aliased_name   = options[:as]
+      aliased_name ||= column.__name.to_sym if column.respond_to?(:__name)
+      aliased_name ||= column
+
+      options[:as] = "#{aliased_name}_sort".to_sym
+
+      ::ThinkingSphinx::RealTime::Attribute.new column, options
+    }.each { |attribute| @index.add_attribute attribute }
+  end
+end

data/lib/thinking_sphinx/real_time/populator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Populator
+  def self.populate(index)
+    new(index).populate
+  end
+
+  def initialize(index)
+    @index = index
+    @started_at = Time.current
+  end
+
+  def populate
+    instrument 'start_populating'
+
+    scope.find_in_batches(:batch_size => batch_size) do |instances|
+      transcriber.copy *instances
+      instrument 'populated', :instances => instances
+    end
+
+    transcriber.clear_before(started_at) if configuration.settings["real_time_tidy"]
+
+    instrument 'finish_populating'
+  end
+
+  private
+
+  attr_reader :index, :started_at
+
+  delegate :controller, :batch_size, :to => :configuration
+  delegate :scope,                   :to => :index
+
+  def configuration
+    ThinkingSphinx::Configuration.instance
+  end
+
+  def instrument(message, options = {})
+    ActiveSupport::Notifications.instrument(
+      "#{message}.thinking_sphinx.real_time", options.merge(:index => index)
+    )
+  end
+
+  def transcriber
+    @transcriber ||= ThinkingSphinx::RealTime::Transcriber.new index
+  end
+end

data/lib/thinking_sphinx/real_time/processor.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Processor
+  def self.call(indices, &block)
+    new(indices).call(&block)
+  end
+
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def call(&block)
+    subscribe_to_progress
+
+    indices.each do |index|
+      ThinkingSphinx::RealTime.populator.populate index
+
+      block.call
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def command
+    ThinkingSphinx::Commander.call(
+      command, configuration, options, stream
+    )
+  end
+
+  def subscribe_to_progress
+    ThinkingSphinx::Subscribers::PopulatorSubscriber.
+      attach_to 'thinking_sphinx.real_time'
+  end
+end

data/lib/thinking_sphinx/real_time/property.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Property
+  include ThinkingSphinx::Core::Property
+
+  attr_reader :column, :options
+
+  def initialize(column, options = {})
+    @options = options
+    @column  = column.respond_to?(:__name) ? column :
+      ThinkingSphinx::ActiveRecord::Column.new(column)
+  end
+
+  def name
+    (@options[:as] || @column.__name).to_s
+  end
+
+  def translate(object)
+    ThinkingSphinx::RealTime::Translator.call(object, @column)
+  end
+end

data/lib/thinking_sphinx/real_time/transcribe_instance.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::TranscribeInstance
+  def self.call(instance, index, properties)
+    new(instance, index, properties).call
+  end
+
+  def initialize(instance, index, properties)
+    @instance, @index, @properties = instance, index, properties
+  end
+
+  def call
+    properties.each_with_object([document_id]) do |property, instance_values|
+      begin
+        instance_values << property.translate(instance)
+      rescue StandardError => error
+        raise_wrapper error, property
+      end
+    end
+  end
+
+  private
+
+  attr_reader :instance, :index, :properties
+
+  def document_id
+    index.document_id_for_key instance.public_send(index.primary_key)
+  end
+
+  def raise_wrapper(error, property)
+    wrapper = ThinkingSphinx::TranscriptionError.new
+    wrapper.inner_exception = error
+    wrapper.instance        = instance
+    wrapper.property        = property
+
+    raise wrapper
+  end
+end

data/lib/thinking_sphinx/real_time/transcriber.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Transcriber
+  def initialize(index)
+    @index = index
+  end
+
+  def clear_before(time)
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_updated_at < #{time.to_i}
+    SQL
+  end
+
+  def copy(*instances)
+    items = instances.select { |instance|
+      instance.persisted? && copy?(instance)
+    }
+    return unless items.present?
+
+    delete_existing items
+    insert_replacements items
+  end
+
+  private
+
+  attr_reader :index
+
+  def columns
+    @columns ||= properties.each_with_object(['id']) do |property, columns|
+      columns << property.name
+    end
+  end
+
+  def copy?(instance)
+    index.conditions.empty? || index.conditions.all? { |condition|
+      case condition
+      when Symbol
+        instance.send(condition)
+      when Proc
+        condition.call instance
+      else
+        "Unexpected condition: #{condition}. Expecting Symbol or Proc."
+      end
+    }
+  end
+
+  def delete_existing(instances)
+    ids = instances.collect(&index.primary_key.to_sym)
+
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_internal_id IN (#{ids.join(', ')})
+    SQL
+  end
+
+  def execute(sphinxql)
+    ThinkingSphinx::Logger.log :query, sphinxql do
+      ThinkingSphinx::Connection.take do |connection|
+        connection.execute sphinxql
+      end
+    end
+  end
+
+  def insert_replacements(instances)
+    insert = Riddle::Query::Insert.new index.name, columns, values(instances)
+    execute insert.replace!.to_sql
+  end
+
+  def instrument(message, options = {})
+    ActiveSupport::Notifications.instrument(
+      "#{message}.thinking_sphinx.real_time", options.merge(:index => index)
+    )
+  end
+
+  def properties
+    @properties ||= index.fields + index.attributes
+  end
+
+  def values(instances)
+    instances.each_with_object([]) do |instance, array|
+      begin
+        array << ThinkingSphinx::RealTime::TranscribeInstance.call(
+          instance, index, properties
+        )
+      rescue ThinkingSphinx::TranscriptionError => error
+        instrument 'error', :error => error
+      end
+    end
+  end
+end