thinking-sphinx 3.1.4 → 6.0.0

data/lib/thinking_sphinx/real_time/transcribe_instance.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::TranscribeInstance
+  def self.call(instance, index, properties)
+    new(instance, index, properties).call
+  end
+
+  def initialize(instance, index, properties)
+    @instance, @index, @properties = instance, index, properties
+  end
+
+  def call
+    properties.each_with_object([document_id]) do |property, instance_values|
+      begin
+        instance_values << property.translate(instance)
+      rescue StandardError => error
+        raise_wrapper error, property
+      end
+    end
+  end
+
+  private
+
+  attr_reader :instance, :index, :properties
+
+  def document_id
+    index.document_id_for_key instance.public_send(index.primary_key)
+  end
+
+  def raise_wrapper(error, property)
+    wrapper = ThinkingSphinx::TranscriptionError.new
+    wrapper.inner_exception = error
+    wrapper.instance        = instance
+    wrapper.property        = property
+
+    raise wrapper
+  end
+end

data/lib/thinking_sphinx/real_time/transcriber.rb

@@ -1,31 +1,36 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Transcriber
   def initialize(index)
     @index = index
   end
 
-  def copy(instance)
-    return unless instance.persisted? && copy?(instance)
+  def clear_before(time)
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_updated_at < #{time.to_i}
+    SQL
+  end
 
-    columns, values = ['id'], [index.document_id_for_key(instance.id)]
-    (index.fields + index.attributes).each do |property|
-      columns << property.name
-      values  << property.translate(instance)
-    end
+  def copy(*instances)
+    items = instances.select { |instance|
+      instance.persisted? && copy?(instance)
+    }
+    return unless items.present?
 
-    insert = Riddle::Query::Insert.new index.name, columns, values
-    sphinxql = insert.replace!.to_sql
-    
-    ThinkingSphinx::Logger.log :query, sphinxql do
-      ThinkingSphinx::Connection.take do |connection|
-        connection.execute sphinxql
-      end 
-    end
+    delete_existing items
+    insert_replacements items
   end
 
   private
 
   attr_reader :index
 
+  def columns
+    @columns ||= properties.each_with_object(['id']) do |property, columns|
+      columns << property.name
+    end
+  end
+
   def copy?(instance)
     index.conditions.empty? || index.conditions.all? { |condition|
       case condition
@@ -38,4 +43,47 @@ class ThinkingSphinx::RealTime::Transcriber
       end
     }
   end
+
+  def delete_existing(instances)
+    ids = instances.collect(&index.primary_key.to_sym)
+
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_internal_id IN (#{ids.join(', ')})
+    SQL
+  end
+
+  def execute(sphinxql)
+    ThinkingSphinx::Logger.log :query, sphinxql do
+      ThinkingSphinx::Connection.take do |connection|
+        connection.execute sphinxql
+      end
+    end
+  end
+
+  def insert_replacements(instances)
+    insert = Riddle::Query::Insert.new index.name, columns, values(instances)
+    execute insert.replace!.to_sql
+  end
+
+  def instrument(message, options = {})
+    ActiveSupport::Notifications.instrument(
+      "#{message}.thinking_sphinx.real_time", options.merge(:index => index)
+    )
+  end
+
+  def properties
+    @properties ||= index.fields + index.attributes
+  end
+
+  def values(instances)
+    instances.each_with_object([]) do |instance, array|
+      begin
+        array << ThinkingSphinx::RealTime::TranscribeInstance.call(
+          instance, index, properties
+        )
+      rescue ThinkingSphinx::TranscriptionError => error
+        instrument 'error', :error => error
+      end
+    end
+  end
 end

data/lib/thinking_sphinx/real_time/translator.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::Translator
+  def self.call(object, column)
+    new(object, column).call
+  end
+
+  def initialize(object, column)
+    @object, @column = object, column
+  end
+
+  def call
+    return name.call(object) if name.is_a?(Proc)
+    return name   unless name.is_a?(Symbol)
+    return result unless result.is_a?(String)
+
+    result.gsub("\u0000", '').force_encoding "UTF-8"
+  end
+
+  private
+
+  attr_reader :object, :column
+
+  def name
+    @column.__name
+  end
+
+  def owner
+    stack.inject(object) { |previous, node| previous.try node }
+  end
+
+  def result
+    @result ||= owner.try name
+  end
+
+  def stack
+    @column.__stack
+  end
+end

data/lib/thinking_sphinx/real_time.rb

@@ -1,10 +1,28 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::RealTime
   module Callbacks
     #
   end
 
   def self.callback_for(reference, path = [], &block)
-    Callbacks::RealTimeCallbacks.new reference, path, &block
+    Callbacks::RealTimeCallbacks.new reference.to_sym, path, &block
+  end
+
+  def self.populator
+    @populator ||= ThinkingSphinx::RealTime::Populator
+  end
+
+  def self.populator=(value)
+    @populator = value
+  end
+
+  def self.processor
+    @processor ||= ThinkingSphinx::RealTime::Processor
+  end
+
+  def self.processor=(value)
+    @processor = value
   end
 end
 
@@ -14,6 +32,9 @@ require 'thinking_sphinx/real_time/field'
 require 'thinking_sphinx/real_time/index'
 require 'thinking_sphinx/real_time/interpreter'
 require 'thinking_sphinx/real_time/populator'
+require 'thinking_sphinx/real_time/processor'
+require 'thinking_sphinx/real_time/transcribe_instance'
 require 'thinking_sphinx/real_time/transcriber'
+require 'thinking_sphinx/real_time/translator'
 
 require 'thinking_sphinx/real_time/callbacks/real_time_callbacks'

data/lib/thinking_sphinx/scopes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Scopes
   extend ActiveSupport::Concern
 
@@ -24,5 +26,9 @@ module ThinkingSphinx::Scopes
       query, options = sphinx_scopes[method].call(*args)
       search query, (options || {})
     end
+
+    def respond_to_missing?(method, include_private = false)
+      super || sphinx_scopes.keys.include?(method)
+    end
   end
 end

data/lib/thinking_sphinx/search/batch_inquirer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::BatchInquirer
   def initialize(&block)
     @queries = []

data/lib/thinking_sphinx/search/context.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::Context
   attr_reader :search, :configuration
 
@@ -5,6 +7,7 @@ class ThinkingSphinx::Search::Context
     @search        = search
     @configuration = configuration || ThinkingSphinx::Configuration.instance
     @memory        = {
+      :raw     => [],
       :results => [],
       :panes   => ThinkingSphinx::Configuration::Defaults::PANES.clone
     }
@@ -17,4 +20,12 @@ class ThinkingSphinx::Search::Context
   def []=(key, value)
     @memory[key] = value
   end
+
+  def marshal_dump
+    [@memory.except(:raw, :indices)]
+  end
+
+  def marshal_load(array)
+    @memory = array.first
+  end
 end

data/lib/thinking_sphinx/search/glaze.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::Glaze < BasicObject
   def initialize(context, object, raw = {}, pane_classes = [])
     @object, @raw = object, raw

data/lib/thinking_sphinx/search/merger.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::Merger
   attr_reader :search
 

data/lib/thinking_sphinx/search/query.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::Query
   attr_reader :keywords, :conditions, :star
 
@@ -10,12 +12,18 @@ class ThinkingSphinx::Search::Query
     (star_keyword(keywords || '') + ' ' + conditions.keys.collect { |key|
        next if conditions[key].blank?
 
-      "@#{key} #{star_keyword conditions[key], key}"
+      "#{expand_key key} #{star_keyword conditions[key], key}"
     }.join(' ')).strip
   end
 
   private
 
+  def expand_key(key)
+    return "@#{key}" unless key.is_a?(Array)
+
+    "@(#{key.join(',')})"
+  end
+
   def star_keyword(keyword, key = nil)
     return keyword.to_s unless star
     return keyword.to_s if key.to_s == 'sphinx_internal_class_name'

data/lib/thinking_sphinx/search/stale_ids_exception.rb

@@ -1,11 +1,15 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::StaleIdsException < StandardError
-  attr_reader :ids
+  attr_reader :ids, :context
 
-  def initialize(ids)
+  def initialize(ids, context)
     @ids = ids
+    @context = context
   end
 
   def message
-    "Record IDs found by Sphinx but not by ActiveRecord : #{ids.join(', ')}"
+    "Record IDs found by Sphinx but not by ActiveRecord : #{ids.join(', ')}\n" \
+    "https://freelancing-gods.com/thinking-sphinx/v5/common_issues.html#record-ids"
   end
 end

data/lib/thinking_sphinx/search.rb

@@ -1,11 +1,23 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search < Array
   CORE_METHODS = %w( == class class_eval extend frozen? id instance_eval
-    instance_of? instance_values instance_variable_defined?
+    instance_exec instance_of? instance_values instance_variable_defined?
     instance_variable_get instance_variable_set instance_variables is_a?
     kind_of? member? method methods nil? object_id respond_to?
     respond_to_missing? send should should_not type )
   SAFE_METHODS = %w( partition private_methods protected_methods public_methods
     send class )
+  KNOWN_OPTIONS = (
+    [
+      :classes, :conditions, :excerpts, :geo, :group_by, :ids_only,
+      :ignore_scopes, :indices, :limit, :masks, :max_matches, :middleware,
+      :none, :offset, :order, :order_group_by, :page, :per_page, :populate,
+      :retry_stale, :select, :skip_sti, :sql, :star, :with, :with_all, :without,
+      :without_ids
+    ] +
+    ThinkingSphinx::Middlewares::SphinxQL::SELECT_OPTIONS
+  ).uniq
   DEFAULT_MASKS = [
     ThinkingSphinx::Masks::PaginationMask,
     ThinkingSphinx::Masks::ScopesMask,
@@ -21,6 +33,12 @@ class ThinkingSphinx::Search < Array
   attr_reader   :options
   attr_accessor :query
 
+  def self.valid_options
+    @valid_options
+  end
+
+  @valid_options = KNOWN_OPTIONS.dup
+
   def initialize(query = nil, options = {})
     query, options   = nil, query if query.is_a?(Hash)
     @query, @options = query, options
@@ -38,6 +56,16 @@ class ThinkingSphinx::Search < Array
     options[:page].to_i
   end
 
+  def marshal_dump
+    populate
+
+    [@populated, @query, @options, @context]
+  end
+
+  def marshal_load(array)
+    @populated, @query, @options, @context = array
+  end
+
   def masks
     @masks ||= @options[:masks] || DEFAULT_MASKS.clone
   end
@@ -64,7 +92,7 @@ class ThinkingSphinx::Search < Array
   def populate
     return self if @populated
 
-    middleware.call [context]
+    middleware.call [context] unless options[:none]
     @populated = true
 
     self

data/lib/thinking_sphinx/settings.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+class ThinkingSphinx::Settings
+  ALWAYS_ABSOLUTE = %w[ socket ]
+  FILE_KEYS = %w[
+    indices_location configuration_file bin_path log query_log pid_file
+    binlog_path snippets_file_prefix sphinxql_state path stopwords wordforms
+    exceptions global_idf rlp_context rlp_root rlp_environment plugin_dir
+    lemmatizer_base mysql_ssl_cert mysql_ssl_key mysql_ssl_ca
+  ].freeze
+  DEFAULTS = {
+    "configuration_file"       => "config/ENVIRONMENT.sphinx.conf",
+    "indices_location"         => "db/sphinx/ENVIRONMENT",
+    "pid_file"                 => "log/ENVIRONMENT.sphinx.pid",
+    "log"                      => "log/ENVIRONMENT.searchd.log",
+    "query_log"                => "log/ENVIRONMENT.searchd.query.log",
+    "binlog_path"              => "tmp/binlog/ENVIRONMENT",
+    "workers"                  => "threads",
+    "mysql_encoding"           => "utf8",
+    "maximum_statement_length" => (2 ** 23) - 5,
+    "real_time_tidy"           => false,
+    "cutoff"                   => 0
+  }.freeze
+  YAML_SAFE_LOAD = YAML.method(:safe_load).parameters.any? do |parameter|
+    parameter == [:key, :aliases]
+  end
+
+  def self.call(configuration)
+    new(configuration).call
+  end
+
+  def initialize(configuration)
+    @configuration = configuration
+  end
+
+  def call
+    return defaults unless File.exist? file
+
+    merged.inject({}) do |hash, (key, value)|
+      if absolute_key?(key)
+        hash[key] = absolute value
+      else
+        hash[key] = value
+      end
+      hash
+    end
+  end
+
+  private
+
+  attr_reader :configuration
+
+  delegate :framework, :to => :configuration
+
+  def absolute(relative)
+    return relative if relative.nil?
+
+    real_path File.absolute_path(relative, framework.root)
+  end
+
+  def absolute_key?(key)
+    return true if ALWAYS_ABSOLUTE.include?(key)
+
+    merged["absolute_paths"] && file_keys.include?(key)
+  end
+
+  def defaults
+    DEFAULTS.inject({}) do |hash, (key, value)|
+      if value.is_a?(String)
+        value = value.gsub("ENVIRONMENT", framework.environment)
+      end
+
+      if FILE_KEYS.include?(key)
+        hash[key] = absolute value
+      else
+        hash[key] = value
+      end
+
+      hash
+    end
+  end
+
+  def file
+    @file ||= Pathname.new(framework.root).join "config", "thinking_sphinx.yml"
+  end
+
+  def file_keys
+    @file_keys ||= FILE_KEYS + (original["file_keys"] || [])
+  end
+
+  def join(first, last)
+    return first if last.nil?
+
+    File.join first, last
+  end
+
+  def merged
+    @merged ||= defaults.merge original
+  end
+
+  def original
+    yaml_contents && yaml_contents[framework.environment] || {}
+  end
+
+  def real_path(base, nonexistent = nil)
+    if File.exist?(base)
+      join File.realpath(base), nonexistent
+    else
+      components = File.split base
+      real_path components.first, join(components.last, nonexistent)
+    end
+  end
+
+  def yaml_contents
+    @yaml_contents ||= begin
+      input = File.read file
+      input = ERB.new(input).result if defined?(ERB)
+
+      if YAML_SAFE_LOAD
+        YAML.safe_load(input, aliases: true)
+      else
+        YAML.load(input)
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/sinatra.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 require 'thinking_sphinx'
 
 ActiveSupport.on_load :active_record do
-  include ThinkingSphinx::ActiveRecord::Base
+  require 'thinking_sphinx/active_record'
 end

data/lib/thinking_sphinx/subscribers/populator_subscriber.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Subscribers::PopulatorSubscriber
   def self.attach_to(namespace)
     subscriber = new
@@ -16,19 +18,31 @@ class ThinkingSphinx::Subscribers::PopulatorSubscriber
       ActiveSupport::Notifications::Event.new(message, *args)
   end
 
+  def error(event)
+    error    = event.payload[:error].inner_exception
+    instance = event.payload[:error].instance
+
+    puts <<-MESSAGE
+
+Error transcribing #{instance.class} #{instance.id}:
+#{error.message}
+    MESSAGE
+  end
+
   def start_populating(event)
     puts "Generating index files for #{event.payload[:index].name}"
   end
 
   def populated(event)
-    print '.'
+    print '.' * event.payload[:instances].length
   end
 
   def finish_populating(event)
     print "\n"
   end
-end
 
-ThinkingSphinx::Subscribers::PopulatorSubscriber.attach_to(
-  'thinking_sphinx.real_time'
-)
+  private
+
+  delegate :output, :to => ThinkingSphinx
+  delegate :puts, :print, :to => :output
+end

data/lib/thinking_sphinx/tasks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 namespace :ts do
   desc 'Generate the Sphinx configuration file'
   task :configure => :environment do
@@ -5,54 +7,79 @@ namespace :ts do
   end
 
   desc 'Generate the Sphinx configuration file and process all indices'
-  task :index => :environment do
-    interface.index(
-      ENV['INDEX_ONLY'] != 'true',
-      !Rake.application.options.silent
-    )
-  end
+  task :index => ['ts:sql:index', 'ts:rt:index']
 
   desc 'Clear out Sphinx files'
-  task :clear => :environment do
-    interface.clear_all
-  end
+  task :clear => ['ts:sql:clear', 'ts:rt:clear']
 
-  desc 'Clear out real-time index files'
-  task :clear_rt => :environment do
-    interface.clear_real_time
-  end
+  desc "Merge all delta indices into their respective core indices"
+  task :merge => ["ts:sql:merge"]
 
-  desc 'Generate fresh index files for real-time indices'
-  task :generate => :environment do
-    interface.prepare
-    interface.generate
-  end
-
-  desc 'Stop Sphinx, index and then restart Sphinx'
-  task :rebuild => [:stop, :clear, :index, :start]
-
-  desc 'Stop Sphinx, clear files, reconfigure, start Sphinx, generate files'
-  task :regenerate => [:stop, :clear_rt, :configure, :start, :generate]
+  desc 'Delete and regenerate Sphinx files, restart the daemon'
+  task :rebuild => [
+    :stop, :clear, :configure, 'ts:sql:index', :start, 'ts:rt:index'
+  ]
 
   desc 'Restart the Sphinx daemon'
   task :restart => [:stop, :start]
 
   desc 'Start the Sphinx daemon'
   task :start => :environment do
-    interface.start
+    interface.daemon.start
   end
 
   desc 'Stop the Sphinx daemon'
   task :stop => :environment do
-    interface.stop
+    interface.daemon.stop
   end
 
   desc 'Determine whether Sphinx is running'
   task :status => :environment do
-    interface.status
+    interface.daemon.status
+  end
+
+  namespace :sql do
+    desc 'Delete SQL-backed Sphinx files'
+    task :clear => :environment do
+      interface.sql.clear
+    end
+
+    desc 'Generate fresh index files for SQL-backed indices'
+    task :index => :environment do
+      interface.sql.index(ENV['INDEX_ONLY'] != 'true')
+    end
+
+    task :merge => :environment do
+      interface.sql.merge
+    end
+
+    desc 'Delete and regenerate SQL-backed Sphinx files, restart the daemon'
+    task :rebuild => ['ts:stop', 'ts:sql:clear', 'ts:sql:index', 'ts:start']
+  end
+
+  namespace :rt do
+    desc 'Delete real-time Sphinx files'
+    task :clear => :environment do
+      interface.rt.clear
+    end
+
+    desc 'Generate fresh index files for real-time indices'
+    task :index => :environment do
+      interface.rt.index
+    end
+
+    desc 'Delete and regenerate real-time Sphinx files, restart the daemon'
+    task :rebuild => [
+      'ts:stop', 'ts:rt:clear', 'ts:configure', 'ts:start', 'ts:rt:index'
+    ]
   end
 
   def interface
-    @interface ||= ThinkingSphinx::RakeInterface.new
+    @interface ||= ThinkingSphinx.rake_interface.new(
+      :verbose     => Rake::FileUtilsExt.verbose_flag,
+      :silent      => Rake.application.options.silent,
+      :nodetach    => (ENV['NODETACH'] == 'true'),
+      :index_names => ENV.fetch('INDEX_FILTER', '').split(',')
+    )
   end
 end