thinking-sphinx 3.1.4 → 6.0.0

data/lib/thinking_sphinx/connection/jruby.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Connection::JRuby < ThinkingSphinx::Connection::Client
+  attr_reader :address, :options
+
+  def initialize(options)
+    options.delete :socket
+
+    super
+
+    @address = "jdbc:mysql://#{@options[:host]}:#{@options[:port]}/?allowMultiQueries=true"
+  end
+
+  def base_error
+    Java::JavaSql::SQLException
+  end
+
+  private
+
+  def client
+    @client ||= Java::ComMysqlJdbc::Driver.new.connect address, properties
+  rescue base_error => error
+    raise ThinkingSphinx::SphinxError.new_from_mysql error
+  end
+
+  def properties
+    object = Java::JavaUtil::Properties.new
+    object.setProperty "user", options[:username]     if options[:username]
+    object.setProperty "password", options[:password] if options[:password]
+    object
+  end
+
+  def results_for(statements)
+    statement = client.createStatement
+    statement.execute statements
+
+    results   = [set_to_array(statement.getResultSet)]
+    results  << set_to_array(statement.getResultSet) while statement.getMoreResults
+    results.compact
+  end
+
+  def set_to_array(set)
+    return nil if set.nil?
+
+    meta = set.getMetaData
+    rows = []
+
+    while set.next
+      rows << (1..meta.getColumnCount).inject({}) do |row, index|
+        name      = meta.getColumnName index
+        row[name] = set.getObject(index)
+        row
+      end
+    end
+
+    rows
+  end
+end

data/lib/thinking_sphinx/connection/mri.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Connection::MRI < ThinkingSphinx::Connection::Client
+  def base_error
+    Mysql2::Error
+  end
+
+  private
+
+  attr_reader :options
+
+  def client
+    @client ||= Mysql2::Client.new({
+      :flags           => Mysql2::Client::MULTI_STATEMENTS,
+      :connect_timeout => 5
+    }.merge(options))
+  rescue base_error => error
+    raise ThinkingSphinx::SphinxError.new_from_mysql error
+  end
+
+  def results_for(statements)
+    results  = [client.query(statements)]
+    results << client.store_result while client.next_result
+    results
+  end
+end

data/lib/thinking_sphinx/connection.rb

@@ -1,22 +1,25 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Connection
   MAXIMUM_RETRIES = 3
 
   def self.new
     configuration = ThinkingSphinx::Configuration.instance
-    # If you use localhost, MySQL insists on a socket connection, but Sphinx
-    # requires a TCP connection. Using 127.0.0.1 fixes that.
-    address = configuration.searchd.address || '127.0.0.1'
-    address = '127.0.0.1' if address == 'localhost'
 
     options = {
-      :host      => address,
+      :host      => configuration.searchd.address,
       :port      => configuration.searchd.mysql41,
+      :socket    => configuration.searchd.socket,
       :reconnect => true
     }.merge(configuration.settings['connection_options'] || {})
 
     connection_class.new options
   end
 
+  def self.clear
+    @pool = nil
+  end
+
   def self.connection_class
     return ThinkingSphinx::Connection::JRuby if RUBY_PLATFORM == 'java'
 
@@ -26,7 +29,7 @@ module ThinkingSphinx::Connection
   def self.pool
     @pool ||= Innertube::Pool.new(
       Proc.new { ThinkingSphinx::Connection.new },
-      Proc.new { |connection| connection.close }
+      Proc.new { |connection| connection.close! }
     )
   end
 
@@ -63,112 +66,8 @@ module ThinkingSphinx::Connection
   end
 
   @persistent = true
-
-  class Client
-    def close
-      client.close unless ThinkingSphinx::Connection.persistent?
-    end
-
-    def execute(statement)
-      query(statement).first
-    end
-
-    def query_all(*statements)
-      query *statements
-    end
-
-    private
-
-    def close_and_clear
-      client.close
-      @client = nil
-    end
-
-    def query(*statements)
-      results_for *statements
-    rescue => error
-      message           = "#{error.message} - #{statements.join('; ')}"
-      wrapper           = ThinkingSphinx::QueryExecutionError.new message
-      wrapper.statement = statements.join('; ')
-      raise wrapper
-    ensure
-      close_and_clear unless ThinkingSphinx::Connection.persistent?
-    end
-  end
-
-  class MRI < Client
-    def initialize(options)
-      @options = options
-    end
-
-    def base_error
-      Mysql2::Error
-    end
-
-    private
-
-    attr_reader :options
-
-    def client
-      @client ||= Mysql2::Client.new({
-        :flags => Mysql2::Client::MULTI_STATEMENTS
-      }.merge(options))
-    rescue base_error => error
-      raise ThinkingSphinx::SphinxError.new_from_mysql error
-    end
-
-    def results_for(*statements)
-      results  = [client.query(statements.join('; '))]
-      results << client.store_result while client.next_result
-      results
-    end
-  end
-
-  class JRuby < Client
-    attr_reader :address, :options
-
-    def initialize(options)
-      @address = "jdbc:mysql://#{options[:host]}:#{options[:port]}/?allowMultiQueries=true"
-      @options = options
-    end
-
-    def base_error
-      Java::JavaSql::SQLException
-    end
-
-    private
-
-    def client
-      @client ||= java.sql.DriverManager.getConnection address,
-        options[:username], options[:password]
-    rescue base_error => error
-      raise ThinkingSphinx::SphinxError.new_from_mysql error
-    end
-
-    def results_for(*statements)
-      statement = client.createStatement
-      statement.execute statements.join('; ')
-
-      results   = [set_to_array(statement.getResultSet)]
-      results  << set_to_array(statement.getResultSet) while statement.getMoreResults
-      results.compact
-    end
-
-    def set_to_array(set)
-      return nil if set.nil?
-
-      meta = set.meta_data
-      rows = []
-
-      while set.next
-        rows << (1..meta.column_count).inject({}) do |row, index|
-          name      = meta.column_name index
-          row[name] = set.get_object(index)
-          row
-        end
-      end
-
-      rows
-    end
-  end
 end
+
+require 'thinking_sphinx/connection/client'
+require 'thinking_sphinx/connection/jruby'
+require 'thinking_sphinx/connection/mri'

data/lib/thinking_sphinx/core/field.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Core::Field
   def infixing?
     options[:infixes]

data/lib/thinking_sphinx/core/index.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Core::Index
   extend ActiveSupport::Concern
   include ThinkingSphinx::Core::Settings
@@ -9,7 +11,6 @@ module ThinkingSphinx::Core::Index
 
   def initialize(reference, options = {})
     @reference    = reference.to_sym
-    @docinfo      = :extern
     @options      = options
     @offset       = config.next_offset(options[:offset_as] || reference)
     @type         = 'plain'
@@ -25,11 +26,22 @@ module ThinkingSphinx::Core::Index
     false
   end
 
+  def document_id_for_instance(instance)
+    document_id_for_key instance.public_send(primary_key)
+  end
+
   def document_id_for_key(key)
+    return nil if key.nil?
+
     key * config.indices.count + offset
   end
 
   def interpret_definition!
+    table_exists = model.table_exists?
+    unless table_exists
+      Rails.logger.info "No table exists for #{model}. Index can not be created"
+      return
+    end
     return if @interpreted_definition
 
     apply_defaults!
@@ -47,6 +59,11 @@ module ThinkingSphinx::Core::Index
     @options
   end
 
+  def primary_key
+    @primary_key ||= @options[:primary_key] ||
+      config.settings['primary_key'] || model.primary_key || :id
+  end
+
   def render
     pre_render
     set_path
@@ -84,7 +101,10 @@ module ThinkingSphinx::Core::Index
   end
 
   def set_path
-    FileUtils.mkdir_p path_prefix
+    unless config.settings['skip_directory_creation']
+      FileUtils.mkdir_p path_prefix
+    end
+
     @path = File.join path_prefix, name
   end
 end

data/lib/thinking_sphinx/core/interpreter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Core::Interpreter < BasicObject
   def self.translate!(index, block)
     new(index, block).translate!

data/lib/thinking_sphinx/core/property.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Core::Property
   def facet?
     options[:facet]

data/lib/thinking_sphinx/core/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Core::Settings
   private
   def apply_defaults!(defaults = self.class.settings)

data/lib/thinking_sphinx/core.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Core
   #
 end

data/lib/thinking_sphinx/deletion.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Deletion
   delegate :name, :to => :index
 
@@ -20,31 +22,49 @@ class ThinkingSphinx::Deletion
 
   attr_reader :index, :ids
 
-  def document_ids_for_keys
-    ids.collect { |id| index.document_id_for_key id }
-  end
-
   def execute(statement)
-    ThinkingSphinx::Connection.take do |connection|
-      connection.execute statement
-    end
-  end
+    statement = statement.gsub(/\s*\n\s*/, ' ').strip
 
-  class RealtimeDeletion < ThinkingSphinx::Deletion
-    def perform
-      execute Riddle::Query::Delete.new(name, document_ids_for_keys).to_sql
+    ThinkingSphinx::Logger.log :query, statement do
+      ThinkingSphinx::Connection.take do |connection|
+        connection.execute statement
+      end
     end
   end
 
   class PlainDeletion < ThinkingSphinx::Deletion
     def perform
-      document_ids_for_keys.each_slice(1000) do |document_ids|
+      ids.each_slice(1000) do |some_ids|
         execute <<-SQL
 UPDATE #{name}
 SET sphinx_deleted = 1
-WHERE id IN (#{document_ids.join(', ')})
+WHERE sphinx_internal_id IN (#{some_ids.join(', ')})
+        SQL
+      end
+    end
+  end
+
+  class RealtimeDeletion < ThinkingSphinx::Deletion
+    def perform
+      return unless callbacks_enabled?
+
+      ids.each_slice(1000) do |some_ids|
+        execute <<-SQL
+DELETE FROM #{name}
+WHERE sphinx_internal_id IN (#{some_ids.join(', ')})
         SQL
       end
     end
+
+    private
+
+    def callbacks_enabled?
+      setting = configuration.settings['real_time_callbacks']
+      setting.nil? || setting
+    end
+
+    def configuration
+      ThinkingSphinx::Configuration.instance
+    end
   end
 end

data/lib/thinking_sphinx/deltas/default_delta.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Deltas::DefaultDelta
   attr_reader :adapter, :options
 
@@ -13,7 +15,7 @@ class ThinkingSphinx::Deltas::DefaultDelta
 
   def delete(index, instance)
     ThinkingSphinx::Deltas::DeleteJob.new(
-      index.name, index.document_id_for_key(instance.id)
+      index.name, index.document_id_for_instance(instance)
     ).perform
   end
 

data/lib/thinking_sphinx/deltas/delete_job.rb

@@ -1,15 +1,27 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Deltas::DeleteJob
   def initialize(index_name, document_id)
     @index_name, @document_id = index_name, document_id
   end
 
   def perform
-    ThinkingSphinx::Connection.take do |connection|
-      connection.execute Riddle::Query.update(
-        @index_name, @document_id, :sphinx_deleted => true
-      )
+    return if @document_id.nil?
+
+    ThinkingSphinx::Logger.log :query, statement do
+      ThinkingSphinx::Connection.take do |connection|
+        connection.execute statement
+      end
     end
   rescue ThinkingSphinx::ConnectionError => error
     # This isn't vital, so don't raise the error.
   end
+
+  private
+
+  def statement
+    @statement ||= Riddle::Query.update(
+      @index_name, @document_id, :sphinx_deleted => true
+    )
+  end
 end

data/lib/thinking_sphinx/deltas/index_job.rb

@@ -1,16 +1,28 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Deltas::IndexJob
   def initialize(index_name)
     @index_name = index_name
   end
 
   def perform
-    configuration.controller.index @index_name,
-      :verbose => !configuration.settings['quiet_deltas']
+    ThinkingSphinx::Commander.call(
+      :index_sql, configuration,
+      :indices => [index_name],
+      :verbose => !quiet_deltas?
+    )
   end
 
   private
 
+  attr_reader :index_name
+
   def configuration
     @configuration ||= ThinkingSphinx::Configuration.instance
   end
+
+  def quiet_deltas?
+    configuration.settings['quiet_deltas'].nil? ||
+    configuration.settings['quiet_deltas']
+  end
 end

data/lib/thinking_sphinx/deltas.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Deltas
   def self.config
     ThinkingSphinx::Configuration.instance

data/lib/thinking_sphinx/distributed/index.rb

@@ -1,11 +1,14 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Distributed::Index <
   Riddle::Configuration::DistributedIndex
 
-  attr_reader :reference, :options
+  attr_reader :reference, :options, :local_index_objects
 
   def initialize(reference)
-    @reference = reference
-    @options   = {}
+    @reference           = reference
+    @options             = {}
+    @local_index_objects = []
 
     super reference.to_s.gsub('/', '_')
   end
@@ -18,7 +21,26 @@ class ThinkingSphinx::Distributed::Index <
     true
   end
 
+  def facets
+    local_index_objects.collect(&:facets).flatten
+  end
+
+  def local_index_objects=(indices)
+    self.local_indices = indices.collect(&:name)
+    @local_index_objects = indices
+  end
+
   def model
     @model ||= reference.to_s.camelize.constantize
   end
+
+  def primary_key
+    @primary_key ||= configuration.settings['primary_key'] || :id
+  end
+
+  private
+
+  def configuration
+    ThinkingSphinx::Configuration.instance
+  end
 end

data/lib/thinking_sphinx/distributed.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Distributed
   #
 end

data/lib/thinking_sphinx/errors.rb

@@ -1,18 +1,24 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::SphinxError < StandardError
   attr_accessor :statement
 
   def self.new_from_mysql(error)
     case error.message
-    when /parse error/
+    when /parse error/, /query is non-computable/
       replacement = ThinkingSphinx::ParseError.new(error.message)
     when /syntax error/
       replacement = ThinkingSphinx::SyntaxError.new(error.message)
-    when /query error/
+    when /query error/, /unknown column/
       replacement = ThinkingSphinx::QueryError.new(error.message)
-    when /Can't connect to MySQL server/, /Communications link failure/
+    when /Can't connect to( MySQL)? server/,
+      /Communications link failure/,
+      /Lost connection to( MySQL)? server/
       replacement = ThinkingSphinx::ConnectionError.new(
         "Error connecting to Sphinx via the MySQL protocol. #{error.message}"
       )
+    when /offset out of bounds/
+      replacement = ThinkingSphinx::OutOfBoundsError.new(error.message)
     else
       replacement = new(error.message)
     end
@@ -29,12 +35,28 @@ end
 class ThinkingSphinx::QueryError < ThinkingSphinx::SphinxError
 end
 
+class ThinkingSphinx::QueryLengthError < ThinkingSphinx::SphinxError
+  def message
+    <<-MESSAGE
+The supplied SphinxQL statement is #{statement.length} characters long. The maximum allowed length is #{ThinkingSphinx::Configuration.instance.settings['maximum_statement_length']}.
+
+If this error has been raised during real-time index population, it's probably due to overly large batches of records being processed at once. The default is 1000, but you can lower it on a per-environment basis in config/thinking_sphinx.yml:
+
+  development:
+    batch_size: 500
+    MESSAGE
+  end
+end
+
 class ThinkingSphinx::SyntaxError < ThinkingSphinx::QueryError
 end
 
 class ThinkingSphinx::ParseError < ThinkingSphinx::QueryError
 end
 
+class ThinkingSphinx::OutOfBoundsError < ThinkingSphinx::QueryError
+end
+
 class ThinkingSphinx::QueryExecutionError < StandardError
   attr_accessor :statement
 end
@@ -50,3 +72,25 @@ end
 
 class ThinkingSphinx::PopulatedResultsError < StandardError
 end
+
+class ThinkingSphinx::DuplicateNameError < StandardError
+end
+
+class ThinkingSphinx::InvalidDatabaseAdapter < StandardError
+end
+
+class ThinkingSphinx::SphinxAlreadyRunning < StandardError
+end
+
+class ThinkingSphinx::UnknownDatabaseAdapter < StandardError
+end
+
+class ThinkingSphinx::UnknownAttributeType < StandardError
+end
+
+class ThinkingSphinx::TranscriptionError < StandardError
+  attr_accessor :inner_exception, :instance, :property
+end
+
+class ThinkingSphinx::UnknownCommand < StandardError
+end

data/lib/thinking_sphinx/excerpter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Excerpter
   DefaultOptions = {
     :before_match    => '<span class="match">',

data/lib/thinking_sphinx/facet.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Facet
   attr_reader :name
 
@@ -11,7 +13,7 @@ class ThinkingSphinx::Facet
 
   def results_from(raw)
     raw.inject({}) { |hash, row|
-      hash[row[group_column]] = row[ThinkingSphinx::SphinxQL.count[:column]]
+      hash[row[group_column]] = row["sphinx_internal_count"]
       hash
     }
   end
@@ -19,8 +21,7 @@ class ThinkingSphinx::Facet
   private
 
   def group_column
-    @properties.any?(&:multi?) ?
-      ThinkingSphinx::SphinxQL.group_by[:column] : name
+    @properties.any?(&:multi?) ? "sphinx_internal_group" : name
   end
 
   def use_field?

data/lib/thinking_sphinx/facet_search.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::FacetSearch
   include Enumerable
 
@@ -112,8 +114,8 @@ class ThinkingSphinx::FacetSearch
   def options_for(facet)
     options.merge(
       :select      => [(options[:select] || '*'),
-        "#{ThinkingSphinx::SphinxQL.group_by[:select]}",
-        "#{ThinkingSphinx::SphinxQL.count[:select]}"
+        "groupby() AS sphinx_internal_group",
+        "id AS sphinx_document_id, count(DISTINCT sphinx_document_id) AS sphinx_internal_count"
       ].join(', '),
       :group_by    => facet.name,
       :indices     => index_names_for(facet),

data/lib/thinking_sphinx/float_formatter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::FloatFormatter
   PATTERN = /(\d+)e\-(\d+)$/
 

data/lib/thinking_sphinx/frameworks/plain.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Frameworks::Plain
   attr_accessor :environment, :root