thinking-sphinx 3.1.4 → 6.0.0

data/lib/thinking_sphinx/commands/merge_and_update.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::MergeAndUpdate < ThinkingSphinx::Commands::Base
+  def call
+    configuration.preload_indices
+    configuration.render
+
+    index_pairs.each do |(core_index, delta_index)|
+      command :merge,
+        :core_index  => core_index,
+        :delta_index => delta_index,
+        :filters     => {:sphinx_deleted => 0}
+
+      core_index.model.where(:delta => true).update_all(:delta => false)
+    end
+  end
+
+  private
+
+  delegate :controller, :to => :configuration
+
+  def core_indices
+    indices.select { |index| !index.delta? }.select do |index|
+      name_filters.empty? ||
+      name_filters.include?(index.name.gsub(/_core$/, ''))
+    end
+  end
+
+  def delta_for(core_index)
+    name = core_index.name.gsub(/_core$/, "_delta")
+    indices.detect { |index| index.name == name }
+  end
+
+  def index_pairs
+    core_indices.collect { |core_index|
+      [core_index, delta_for(core_index)]
+    }
+  end
+
+  def indices
+    @indices ||= configuration.indices.select { |index|
+      index.type == "plain" && index.options[:delta_processor]
+    }
+  end
+
+  def indices_exist?(*indices)
+    indices.all? { |index| File.exist?("#{index.path}.spi") }
+  end
+
+  def name_filters
+    @name_filters ||= options[:index_names] || []
+  end
+
+  def type
+    'merging_and_updating'
+  end
+end

data/lib/thinking_sphinx/commands/prepare.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::Prepare < ThinkingSphinx::Commands::Base
+  def call
+    return if skip_directories?
+
+    FileUtils.mkdir_p configuration.indices_location
+  end
+
+  private
+
+  def type
+    'prepare'
+  end
+end

data/lib/thinking_sphinx/commands/rotate.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::Rotate < ThinkingSphinx::Commands::Base
+  def call
+    controller.rotate
+  end
+
+  private
+
+  def type
+    'rotate'
+  end
+end

data/lib/thinking_sphinx/commands/running.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::Running < ThinkingSphinx::Commands::Base
+  def call
+    return true if configuration.settings['skip_running_check']
+
+    controller.running?
+  end
+
+  private
+
+  def type
+    'running'
+  end
+end

data/lib/thinking_sphinx/commands/start_attached.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::StartAttached < ThinkingSphinx::Commands::Base
+  def call
+    FileUtils.mkdir_p configuration.indices_location unless skip_directories?
+
+    unless pid = fork
+      controller.start :verbose => options[:verbose], :nodetach => true
+    end
+
+    Signal.trap('TERM') { Process.kill(:TERM, pid) }
+    Signal.trap('INT')  { Process.kill(:TERM, pid) }
+
+    Process.wait(pid)
+  end
+
+  private
+
+  def type
+    'start'
+  end
+end

data/lib/thinking_sphinx/commands/start_detached.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::StartDetached < ThinkingSphinx::Commands::Base
+  def call
+    FileUtils.mkdir_p configuration.indices_location unless skip_directories?
+
+    result = controller.start :verbose => options[:verbose]
+
+    if command :running
+      log "Started searchd successfully (pid: #{controller.pid})."
+    else
+      handle_failure result
+    end
+  end
+
+  private
+
+  def type
+    'start'
+  end
+end

data/lib/thinking_sphinx/commands/stop.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::Stop < ThinkingSphinx::Commands::Base
+  def call
+    unless command :running
+      log 'searchd is not currently running.'
+      return
+    end
+
+    pid = controller.pid
+    until !command :running do
+      controller.stop options
+      sleep(0.5)
+    end
+
+    log "Stopped searchd daemon (pid: #{pid})."
+  end
+
+  private
+
+  def type
+    'stop'
+  end
+end

data/lib/thinking_sphinx/commands.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ThinkingSphinx::Commands
+  #
+end
+
+require 'thinking_sphinx/commands/base'
+require 'thinking_sphinx/commands/clear_real_time'
+require 'thinking_sphinx/commands/clear_sql'
+require 'thinking_sphinx/commands/configure'
+require 'thinking_sphinx/commands/index_sql'
+require 'thinking_sphinx/commands/index_real_time'
+require 'thinking_sphinx/commands/merge'
+require 'thinking_sphinx/commands/merge_and_update'
+require 'thinking_sphinx/commands/prepare'
+require 'thinking_sphinx/commands/rotate'
+require 'thinking_sphinx/commands/running'
+require 'thinking_sphinx/commands/start_attached'
+require 'thinking_sphinx/commands/start_detached'
+require 'thinking_sphinx/commands/stop'

data/lib/thinking_sphinx/configuration/consistent_ids.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Configuration::ConsistentIds
   def initialize(indices)
     @indices = indices

data/lib/thinking_sphinx/configuration/defaults.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Configuration::Defaults
   ADDRESS = '127.0.0.1'
   PORT    = 9306

data/lib/thinking_sphinx/configuration/distributed_indices.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Configuration::DistributedIndices
   def initialize(indices)
     @indices = indices
@@ -19,7 +21,7 @@ class ThinkingSphinx::Configuration::DistributedIndices
 
   def distributed_index(reference, indices)
     index = ThinkingSphinx::Distributed::Index.new reference
-    index.local_indices += indices.collect &:name
+    index.local_index_objects = indices
     index
   end
 

data/lib/thinking_sphinx/configuration/duplicate_names.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Configuration::DuplicateNames
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def reconcile
+    indices.each do |index|
+      return if index.distributed?
+
+      counts_for(index).each do |name, count|
+        next if count <= 1
+
+        raise ThinkingSphinx::DuplicateNameError,
+          "Duplicate field/attribute name '#{name}' in index '#{index.name}'"
+      end
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def counts_for(index)
+    names_for(index).inject({}) do |hash, name|
+      hash[name] ||= 0
+      hash[name] += 1
+      hash
+    end
+  end
+
+  def names_for(index)
+    index.fields.collect(&:name) + index.attributes.collect(&:name)
+  end
+end

data/lib/thinking_sphinx/configuration/minimum_fields.rb

@@ -1,13 +1,13 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Configuration::MinimumFields
   def initialize(indices)
     @indices = indices
   end
 
   def reconcile
-    return unless no_inheritance_columns?
-
-    sources.each do |source|
-      source.fields.delete_if do |field|
+    field_collections.each do |collection|
+      collection.fields.delete_if do |field|
         field.name == 'sphinx_internal_class_name'
       end
     end
@@ -17,15 +17,20 @@ class ThinkingSphinx::Configuration::MinimumFields
 
   attr_reader :indices
 
-  def no_inheritance_columns?
-    indices.select { |index|
-      index.model.column_names.include?(index.model.inheritance_column)
-    }.empty?
+  def field_collections
+    indices_without_inheritance_of_type('plain').collect(&:sources).flatten +
+      indices_without_inheritance_of_type('rt')
+  end
+
+  def inheritance_columns?(index)
+    index.model.table_exists? && index.model.column_names.include?(index.model.inheritance_column)
+  end
+
+  def indices_without_inheritance_of_type(type)
+    indices_without_inheritance.select { |index| index.type == type }
   end
 
-  def sources
-    @sources ||= @indices.select { |index|
-      index.respond_to?(:sources)
-    }.collect(&:sources).flatten
+  def indices_without_inheritance
+    indices.reject(&method(:inheritance_columns?))
   end
 end

data/lib/thinking_sphinx/configuration.rb

@@ -1,16 +1,22 @@
+# frozen_string_literal: true
+
 require 'pathname'
 
 class ThinkingSphinx::Configuration < Riddle::Configuration
-  attr_accessor :configuration_file, :indices_location, :version
+  attr_accessor :configuration_file, :indices_location, :version, :batch_size
   attr_reader :index_paths
-  attr_writer :controller, :index_set_class
+  attr_writer :controller, :index_set_class, :indexing_strategy,
+    :guarding_strategy
 
   delegate :environment, :to => :framework
 
+  @@mutex = defined?(ActiveSupport::Concurrency::LoadInterlockAwareMonitor) ?
+    ActiveSupport::Concurrency::LoadInterlockAwareMonitor.new : Mutex.new
+
   def initialize
     super
 
-    setup
+    reset
   end
 
   def self.instance
@@ -27,7 +33,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
 
   def controller
     @controller ||= begin
-      rc = ThinkingSphinx::Controller.new self, configuration_file
+      rc = Riddle::Controller.new self, configuration_file
       rc.bin_path = bin_path.gsub(/([^\/])$/, '\1/') if bin_path.present?
       rc
     end
@@ -39,14 +45,14 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
 
   def framework=(framework)
     @framework = framework
-    setup
+    reset
     framework
   end
 
   def engine_index_paths
     return [] unless defined?(Rails)
 
-    engine_indice_paths.flatten.compact
+    engine_indice_paths.flatten.compact.sort
   end
 
   def engine_indice_paths
@@ -56,10 +62,18 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
     end
   end
 
+  def guarding_strategy
+    @guarding_strategy ||= ThinkingSphinx::Guard::Files
+  end
+
   def index_set_class
     @index_set_class ||= ThinkingSphinx::IndexSet
   end
 
+  def indexing_strategy
+    @indexing_strategy ||= ThinkingSphinx::IndexingStrategies::AllAtOnce
+  end
+
   def indices_for_references(*references)
     index_set_class.new(:references => references).to_a
   end
@@ -69,105 +83,105 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
   end
 
   def preload_indices
-    return if @preloaded_indices
+    @@mutex.synchronize do
+      return if @preloaded_indices
 
-    index_paths.each do |path|
-      Dir["#{path}/**/*.rb"].sort.each do |file|
-        ActiveSupport::Dependencies.require_or_load file
+      index_paths.each do |path|
+        Dir["#{path}/**/*.rb"].sort.each { |file| preload_index file }
       end
-    end
 
-    if settings['distributed_indices'].nil? || settings['distributed_indices']
-      ThinkingSphinx::Configuration::DistributedIndices.new(indices).reconcile
+      normalise
+      verify
+
+      @preloaded_indices = true
     end
+  end
 
-    @preloaded_indices = true
+  def preload_index(file)
+    if ActiveRecord::VERSION::MAJOR <= 5
+      ActiveSupport::Dependencies.require_or_load file
+    else
+      load file
+    end
   end
 
   def render
     preload_indices
 
-    ThinkingSphinx::Configuration::ConsistentIds.new(indices).reconcile
-    ThinkingSphinx::Configuration::MinimumFields.new(indices).reconcile
-
     super
   end
 
   def render_to_file
-    FileUtils.mkdir_p searchd.binlog_path unless searchd.binlog_path.blank?
+    unless settings['skip_directory_creation'] || searchd.binlog_path.blank?
+      FileUtils.mkdir_p searchd.binlog_path
+    end
 
     open(configuration_file, 'w') { |file| file.write render }
   end
 
   def settings
-    @settings ||= File.exists?(settings_file) ? settings_to_hash : {}
+    @settings ||= ThinkingSphinx::Settings.call self
   end
 
-  private
-
-  def configure_searchd
-    configure_searchd_log_files
+  def setup
+    @configuration_file = settings['configuration_file']
+    @index_paths = engine_index_paths +
+      [Pathname.new(framework.root).join('app', 'indices').to_s]
+    @indices_location = settings['indices_location']
+    @version = settings['version'] || '2.2.11'
+    @batch_size = settings['batch_size'] || 1000
 
-    searchd.binlog_path = tmp_path.join('binlog', environment).to_s
-    searchd.address = settings['address'].presence || Defaults::ADDRESS
-    searchd.mysql41 = settings['mysql41'] || settings['port'] || Defaults::PORT
-    searchd.workers = 'threads'
-    searchd.mysql_version_string = '5.5.21' if RUBY_PLATFORM == 'java'
-  end
+    if settings['common_sphinx_configuration']
+      common.common_sphinx_configuration  = true
+      indexer.common_sphinx_configuration = true
+    end
 
-  def configure_searchd_log_files
-    searchd.pid_file = log_root.join("#{environment}.sphinx.pid").to_s
-    searchd.log = log_root.join("#{environment}.searchd.log").to_s
-    searchd.query_log = log_root.join("#{environment}.searchd.query.log").to_s
-  end
+    configure_searchd
 
-  def log_root
-    real_path 'log'
-  end
+    apply_sphinx_settings!
 
-  def framework_root
-    Pathname.new(framework.root)
+    @offsets = {}
   end
 
-  def real_path(*arguments)
-    path = framework_root.join(*arguments)
-    path.exist? ? path.realpath : path
-  end
+  private
 
-  def settings_to_hash
-    contents = YAML.load(ERB.new(File.read(settings_file)).result)
-    contents && contents[environment] || {}
-  end
+  def apply_sphinx_settings!
+    sphinx_sections.each do |object|
+      settings.each do |key, value|
+        next unless object.class.settings.include?(key.to_sym)
 
-  def settings_file
-    framework_root.join 'config', 'thinking_sphinx.yml'
+        object.send("#{key}=", value)
+      end
+    end
   end
 
-  def setup
-    @settings = nil
-    @configuration_file = settings['configuration_file'] || framework_root.join(
-      'config', "#{environment}.sphinx.conf"
-    ).to_s
-    @index_paths = engine_index_paths + [framework_root.join('app', 'indices').to_s]
-    @indices_location = settings['indices_location'] || framework_root.join(
-      'db', 'sphinx', environment
-    ).to_s
-    @version = settings['version'] || '2.1.4'
+  def configure_searchd
+    searchd.socket = "#{settings["socket"]}:mysql41" if socket?
 
-    if settings['common_sphinx_configuration']
-      common.common_sphinx_configuration  = true
-      indexer.common_sphinx_configuration = true
+    if tcp?
+      searchd.address = settings['address'].presence || Defaults::ADDRESS
+      searchd.mysql41 = settings['mysql41'] || settings['port'] || Defaults::PORT
     end
 
-    configure_searchd
+    searchd.mysql_version_string = '5.5.21' if RUBY_PLATFORM == 'java'
+  end
 
-    apply_sphinx_settings!
+  def normalise
+    if settings['distributed_indices'].nil? || settings['distributed_indices']
+      ThinkingSphinx::Configuration::DistributedIndices.new(indices).reconcile
+    end
 
-    @offsets = {}
+    ThinkingSphinx::Configuration::ConsistentIds.new(indices).reconcile
+    ThinkingSphinx::Configuration::MinimumFields.new(indices).reconcile
+  end
+
+  def reset
+    @settings = nil
+    setup
   end
 
-  def tmp_path
-    real_path 'tmp'
+  def socket?
+    settings["socket"].present?
   end
 
   def sphinx_sections
@@ -176,18 +190,20 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
     sections
   end
 
-  def apply_sphinx_settings!
-    sphinx_sections.each do |object|
-      settings.each do |key, value|
-        next unless object.class.settings.include?(key.to_sym)
+  def tcp?
+    settings["socket"].nil?      ||
+    settings["address"].present? ||
+    settings["mysql41"].present? ||
+    settings["port"].present?
+  end
 
-        object.send("#{key}=", value)
-      end
-    end
+  def verify
+    ThinkingSphinx::Configuration::DuplicateNames.new(indices).reconcile
   end
 end
 
 require 'thinking_sphinx/configuration/consistent_ids'
 require 'thinking_sphinx/configuration/defaults'
 require 'thinking_sphinx/configuration/distributed_indices'
+require 'thinking_sphinx/configuration/duplicate_names'
 require 'thinking_sphinx/configuration/minimum_fields'

data/lib/thinking_sphinx/connection/client.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Connection::Client
+  def initialize(options)
+    if options[:socket].present?
+      options[:socket] = options[:socket].remove /:mysql41$/
+
+      options.delete :host
+      options.delete :port
+    else
+      options.delete :socket
+
+      # If you use localhost, MySQL insists on a socket connection, but in this
+      # situation we want a TCP connection. Using 127.0.0.1 fixes that.
+      if options[:host].nil? || options[:host] == "localhost"
+        options[:host] = "127.0.0.1"
+      end
+    end
+
+    @options = options
+  end
+
+  def close
+    close! unless ThinkingSphinx::Connection.persistent?
+  end
+
+  def close!
+    client.close
+  end
+
+  def execute(statement)
+    check_and_perform(statement).first
+  end
+
+  def query_all(*statements)
+    check_and_perform statements.join('; ')
+  end
+
+  private
+
+  def check(statements)
+    if statements.length > maximum_statement_length
+      exception           = ThinkingSphinx::QueryLengthError.new
+      exception.statement = statements
+      raise exception
+    end
+  end
+
+  def check_and_perform(statements)
+    check statements
+    perform statements
+  end
+
+  def close_and_clear
+    client.close
+    @client = nil
+  end
+
+  def maximum_statement_length
+    @maximum_statement_length ||= ThinkingSphinx::Configuration.instance.
+      settings['maximum_statement_length']
+  end
+
+  def perform(statements)
+    results_for statements
+  rescue => error
+    message           = "#{error.message} - #{statements}"
+    wrapper           = ThinkingSphinx::QueryExecutionError.new message
+    wrapper.statement = statements
+    raise wrapper
+  ensure
+    close_and_clear unless ThinkingSphinx::Connection.persistent?
+  end
+end