thinking-sphinx 1.2.13 → 6.0.0

data/lib/thinking_sphinx/configuration/consistent_ids.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Configuration::ConsistentIds
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def reconcile
+    return unless sphinx_internal_ids.any? { |attribute|
+      attribute.type == :bigint
+    }
+
+    sphinx_internal_ids.each do |attribute|
+      attribute.type = :bigint
+    end
+  end
+
+  private
+
+  def attributes
+    @attributes = sources.collect(&:attributes).flatten
+  end
+
+  def sphinx_internal_ids
+    @sphinx_internal_ids ||= attributes.select { |attribute|
+      attribute.name == 'sphinx_internal_id'
+    }
+  end
+
+  def sources
+    @sources ||= @indices.select { |index|
+      index.respond_to?(:sources)
+    }.collect(&:sources).flatten
+  end
+end

data/lib/thinking_sphinx/configuration/defaults.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ThinkingSphinx::Configuration::Defaults
+  ADDRESS = '127.0.0.1'
+  PORT    = 9306
+  PANES   = []
+end

data/lib/thinking_sphinx/configuration/distributed_indices.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Configuration::DistributedIndices
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def reconcile
+    grouped_indices.each do |reference, indices|
+      append distributed_index(reference, indices)
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def append(index)
+    ThinkingSphinx::Configuration.instance.indices << index
+  end
+
+  def distributed_index(reference, indices)
+    index = ThinkingSphinx::Distributed::Index.new reference
+    index.local_index_objects = indices
+    index
+  end
+
+  def grouped_indices
+    indices.group_by &:reference
+  end
+end

data/lib/thinking_sphinx/configuration/duplicate_names.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Configuration::DuplicateNames
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def reconcile
+    indices.each do |index|
+      return if index.distributed?
+
+      counts_for(index).each do |name, count|
+        next if count <= 1
+
+        raise ThinkingSphinx::DuplicateNameError,
+          "Duplicate field/attribute name '#{name}' in index '#{index.name}'"
+      end
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def counts_for(index)
+    names_for(index).inject({}) do |hash, name|
+      hash[name] ||= 0
+      hash[name] += 1
+      hash
+    end
+  end
+
+  def names_for(index)
+    index.fields.collect(&:name) + index.attributes.collect(&:name)
+  end
+end

data/lib/thinking_sphinx/configuration/minimum_fields.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Configuration::MinimumFields
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def reconcile
+    field_collections.each do |collection|
+      collection.fields.delete_if do |field|
+        field.name == 'sphinx_internal_class_name'
+      end
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def field_collections
+    indices_without_inheritance_of_type('plain').collect(&:sources).flatten +
+      indices_without_inheritance_of_type('rt')
+  end
+
+  def inheritance_columns?(index)
+    index.model.table_exists? && index.model.column_names.include?(index.model.inheritance_column)
+  end
+
+  def indices_without_inheritance_of_type(type)
+    indices_without_inheritance.select { |index| index.type == type }
+  end
+
+  def indices_without_inheritance
+    indices.reject(&method(:inheritance_columns?))
+  end
+end

data/lib/thinking_sphinx/configuration.rb

@@ -1,282 +1,209 @@
-require 'erb'
-require 'singleton'
-
-module ThinkingSphinx
-  # This class both keeps track of the configuration settings for Sphinx and
-  # also generates the resulting file for Sphinx to use.
-  # 
-  # Here are the default settings, relative to RAILS_ROOT where relevant:
-  #
-  # config file::           config/#{environment}.sphinx.conf
-  # searchd log file::      log/searchd.log
-  # query log file::        log/searchd.query.log
-  # pid file::              log/searchd.#{environment}.pid
-  # searchd files::         db/sphinx/#{environment}/
-  # address::               127.0.0.1
-  # port::                  3312
-  # allow star::            false
-  # min prefix length::     1
-  # min infix length::      1
-  # mem limit::             64M
-  # max matches::           1000
-  # morphology::            nil
-  # charset type::          utf-8
-  # charset table::         nil
-  # ignore chars::          nil
-  # html strip::            false
-  # html remove elements::  ''
-  # searchd_binary_name::   searchd
-  # indexer_binary_name::   indexer
-  #
-  # If you want to change these settings, create a YAML file at
-  # config/sphinx.yml with settings for each environment, in a similar
-  # fashion to database.yml - using the following keys: config_file,
-  # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
-  # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
-  # max_matches, morphology, charset_type, charset_table, ignore_chars,
-  # html_strip, html_remove_elements, delayed_job_priority,
-  # searchd_binary_name, indexer_binary_name.
-  #
-  # I think you've got the idea.
-  # 
-  # Each setting in the YAML file is optional - so only put in the ones you
-  # want to change.
-  #
-  # Keep in mind, if for some particular reason you're using a version of
-  # Sphinx older than 0.9.8 r871 (that's prior to the proper 0.9.8 release),
-  # don't set allow_star to true.
-  # 
-  class Configuration
-    include Singleton
-    
-    SourceOptions = %w( mysql_connect_flags sql_range_step sql_query_pre
-      sql_query_post sql_ranged_throttle sql_query_post_index )
-    
-    IndexOptions  = %w( charset_table charset_type docinfo enable_star
-      exceptions html_index_attrs html_remove_elements html_strip ignore_chars
-      min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
-      ngram_len phrase_boundary phrase_boundary_step preopen stopwords
-      wordforms )
-    
-    CustomOptions = %w( disable_range )
-        
-    attr_accessor :config_file, :searchd_log_file, :query_log_file,
-      :pid_file, :searchd_file_path, :address, :port, :allow_star,
-      :database_yml_file, :app_root, :bin_path, :model_directories,
-      :delayed_job_priority, :searchd_binary_name, :indexer_binary_name
-    
-    attr_accessor :source_options, :index_options
-    
-    attr_reader :environment, :configuration
-    
-    # Load in the configuration settings - this will look for config/sphinx.yml
-    # and parse it according to the current environment.
-    # 
-    def initialize(app_root = Dir.pwd)
-      self.reset
-    end
-    
-    def self.configure(&block)
-      yield instance
-      instance.reset(instance.app_root)
-    end
-    
-    def reset(custom_app_root=nil)
-      if custom_app_root
-        self.app_root = custom_app_root
-      else
-        self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
-        self.app_root          = Merb.root  if defined?(Merb)
-        self.app_root        ||= app_root
-      end
-      
-      @configuration = Riddle::Configuration.new
-      @configuration.searchd.address    = "127.0.0.1"
-      @configuration.searchd.port       = 3312
-      @configuration.searchd.pid_file   = "#{self.app_root}/log/searchd.#{environment}.pid"
-      @configuration.searchd.log        = "#{self.app_root}/log/searchd.log"
-      @configuration.searchd.query_log  = "#{self.app_root}/log/searchd.query.log"
-      
-      self.database_yml_file    = "#{self.app_root}/config/database.yml"
-      self.config_file          = "#{self.app_root}/config/#{environment}.sphinx.conf"
-      self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
-      self.allow_star           = false
-      self.bin_path             = ""
-      self.model_directories    = ["#{app_root}/app/models/"] +
-        Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
-      self.delayed_job_priority = 0
-      
-      self.source_options  = {}
-      self.index_options   = {
-        :charset_type => "utf-8"
-      }
-      
-      self.searchd_binary_name = "searchd"
-      self.indexer_binary_name = "indexer"
-            
-      parse_config
-      
-      self
-    end
-    
-    def self.environment
-      @@environment ||= (
-        defined?(Merb) ? Merb.environment : ENV['RAILS_ENV']
-      ) || "development"
-    end
-    
-    def environment
-      self.class.environment
-    end
-    
-    def controller
-      @controller ||= Riddle::Controller.new(@configuration, self.config_file)
+# frozen_string_literal: true
+
+require 'pathname'
+
+class ThinkingSphinx::Configuration < Riddle::Configuration
+  attr_accessor :configuration_file, :indices_location, :version, :batch_size
+  attr_reader :index_paths
+  attr_writer :controller, :index_set_class, :indexing_strategy,
+    :guarding_strategy
+
+  delegate :environment, :to => :framework
+
+  @@mutex = defined?(ActiveSupport::Concurrency::LoadInterlockAwareMonitor) ?
+    ActiveSupport::Concurrency::LoadInterlockAwareMonitor.new : Mutex.new
+
+  def initialize
+    super
+
+    reset
+  end
+
+  def self.instance
+    @instance ||= new
+  end
+
+  def self.reset
+    @instance = nil
+  end
+
+  def bin_path
+    settings['bin_path']
+  end
+
+  def controller
+    @controller ||= begin
+      rc = Riddle::Controller.new self, configuration_file
+      rc.bin_path = bin_path.gsub(/([^\/])$/, '\1/') if bin_path.present?
+      rc
     end
-    
-    # Generate the config file for Sphinx by using all the settings defined and
-    # looping through all the models with indexes to build the relevant
-    # indexer and searchd configuration, and sources and indexes details.
-    #
-    def build(file_path=nil)
-      load_models
-      file_path ||= "#{self.config_file}"
-      
-      @configuration.indexes.clear
-      
-      ThinkingSphinx.indexed_models.each_with_index do |model, model_index|
-        @configuration.indexes.concat model.constantize.to_riddle(model_index)
-      end
-      
-      open(file_path, "w") do |file|
-        file.write @configuration.render
-      end
+  end
+
+  def framework
+    @framework ||= ThinkingSphinx::Frameworks.current
+  end
+
+  def framework=(framework)
+    @framework = framework
+    reset
+    framework
+  end
+
+  def engine_index_paths
+    return [] unless defined?(Rails)
+
+    engine_indice_paths.flatten.compact.sort
+  end
+
+  def engine_indice_paths
+    Rails::Engine.subclasses.collect(&:instance).collect do |engine|
+      engine.paths.add 'app/indices' unless engine.paths['app/indices']
+      engine.paths['app/indices'].existent
     end
-    
-    # Make sure all models are loaded - without reloading any that
-    # ActiveRecord::Base is already aware of (otherwise we start to hit some
-    # messy dependencies issues).
-    # 
-    def load_models
-      return if defined?(Rails) &&
-        Rails::VERSION::STRING.to_f > 2.1 &&
-        Rails.configuration.cache_classes
-      
-      self.model_directories.each do |base|
-        Dir["#{base}**/*.rb"].each do |file|
-          model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
-        
-          next if model_name.nil?
-          next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
-            model.name == model_name
-          }
-        
-          begin
-            model_name.camelize.constantize
-          rescue LoadError
-            model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
-          rescue NameError
-            next
-          rescue StandardError
-            puts "Warning: Error loading #{file}"
-          end
-        end
+  end
+
+  def guarding_strategy
+    @guarding_strategy ||= ThinkingSphinx::Guard::Files
+  end
+
+  def index_set_class
+    @index_set_class ||= ThinkingSphinx::IndexSet
+  end
+
+  def indexing_strategy
+    @indexing_strategy ||= ThinkingSphinx::IndexingStrategies::AllAtOnce
+  end
+
+  def indices_for_references(*references)
+    index_set_class.new(:references => references).to_a
+  end
+
+  def next_offset(reference)
+    @offsets[reference] ||= @offsets.keys.count
+  end
+
+  def preload_indices
+    @@mutex.synchronize do
+      return if @preloaded_indices
+
+      index_paths.each do |path|
+        Dir["#{path}/**/*.rb"].sort.each { |file| preload_index file }
       end
+
+      normalise
+      verify
+
+      @preloaded_indices = true
     end
-    
-    def address
-      @configuration.searchd.address
-    end
-    
-    def address=(address)
-      @configuration.searchd.address = address
-    end
-    
-    def port
-      @configuration.searchd.port
-    end
-    
-    def port=(port)
-      @configuration.searchd.port = port
-    end
-    
-    def pid_file
-      @configuration.searchd.pid_file
-    end
-    
-    def pid_file=(pid_file)
-      @configuration.searchd.pid_file = pid_file
-    end
-    
-    def searchd_log_file
-      @configuration.searchd.log
-    end
-    
-    def searchd_log_file=(file)
-      @configuration.searchd.log = file
-    end
-    
-    def query_log_file
-      @configuration.searchd.query_log
+  end
+
+  def preload_index(file)
+    if ActiveRecord::VERSION::MAJOR <= 5
+      ActiveSupport::Dependencies.require_or_load file
+    else
+      load file
     end
-    
-    def query_log_file=(file)
-      @configuration.searchd.query_log = file
+  end
+
+  def render
+    preload_indices
+
+    super
+  end
+
+  def render_to_file
+    unless settings['skip_directory_creation'] || searchd.binlog_path.blank?
+      FileUtils.mkdir_p searchd.binlog_path
     end
-    
-    def client
-      client = Riddle::Client.new address, port
-      client.max_matches = configuration.searchd.max_matches || 1000
-      client
+
+    open(configuration_file, 'w') { |file| file.write render }
+  end
+
+  def settings
+    @settings ||= ThinkingSphinx::Settings.call self
+  end
+
+  def setup
+    @configuration_file = settings['configuration_file']
+    @index_paths = engine_index_paths +
+      [Pathname.new(framework.root).join('app', 'indices').to_s]
+    @indices_location = settings['indices_location']
+    @version = settings['version'] || '2.2.11'
+    @batch_size = settings['batch_size'] || 1000
+
+    if settings['common_sphinx_configuration']
+      common.common_sphinx_configuration  = true
+      indexer.common_sphinx_configuration = true
     end
-    
-    def models_by_crc
-      @models_by_crc ||= begin
-        ThinkingSphinx.indexed_models.inject({}) do |hash, model|
-          hash[model.constantize.to_crc32] = model
-          Object.subclasses_of(model.constantize).each { |subclass|
-            hash[subclass.to_crc32] = subclass.name
-          }
-          hash
-        end
+
+    configure_searchd
+
+    apply_sphinx_settings!
+
+    @offsets = {}
+  end
+
+  private
+
+  def apply_sphinx_settings!
+    sphinx_sections.each do |object|
+      settings.each do |key, value|
+        next unless object.class.settings.include?(key.to_sym)
+
+        object.send("#{key}=", value)
       end
     end
-    
-    private
-    
-    # Parse the config/sphinx.yml file - if it exists - then use the attribute
-    # accessors to set the appropriate values. Nothing too clever.
-    # 
-    def parse_config
-      path = "#{app_root}/config/sphinx.yml"
-      return unless File.exists?(path)
-      
-      conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
-      
-      conf.each do |key,value|
-        self.send("#{key}=", value) if self.respond_to?("#{key}=")
-        
-        set_sphinx_setting self.source_options, key, value, SourceOptions
-        set_sphinx_setting self.index_options,  key, value, IndexOptions
-        set_sphinx_setting self.index_options,  key, value, CustomOptions
-        set_sphinx_setting @configuration.searchd, key, value
-        set_sphinx_setting @configuration.indexer, key, value
-      end unless conf.nil?
-      
-      self.bin_path += '/' unless self.bin_path.blank?
-      
-      if self.allow_star
-        self.index_options[:enable_star]    = true
-        self.index_options[:min_prefix_len] = 1
-      end
+  end
+
+  def configure_searchd
+    searchd.socket = "#{settings["socket"]}:mysql41" if socket?
+
+    if tcp?
+      searchd.address = settings['address'].presence || Defaults::ADDRESS
+      searchd.mysql41 = settings['mysql41'] || settings['port'] || Defaults::PORT
     end
-    
-    def set_sphinx_setting(object, key, value, allowed = {})
-      if object.is_a?(Hash)
-        object[key.to_sym] = value if allowed.include?(key.to_s)
-      else
-        object.send("#{key}=", value) if object.respond_to?("#{key}")
-        send("#{key}=", value) if self.respond_to?("#{key}")
-      end
+
+    searchd.mysql_version_string = '5.5.21' if RUBY_PLATFORM == 'java'
+  end
+
+  def normalise
+    if settings['distributed_indices'].nil? || settings['distributed_indices']
+      ThinkingSphinx::Configuration::DistributedIndices.new(indices).reconcile
     end
+
+    ThinkingSphinx::Configuration::ConsistentIds.new(indices).reconcile
+    ThinkingSphinx::Configuration::MinimumFields.new(indices).reconcile
+  end
+
+  def reset
+    @settings = nil
+    setup
+  end
+
+  def socket?
+    settings["socket"].present?
+  end
+
+  def sphinx_sections
+    sections = [indexer, searchd]
+    sections.unshift common if settings['common_sphinx_configuration']
+    sections
+  end
+
+  def tcp?
+    settings["socket"].nil?      ||
+    settings["address"].present? ||
+    settings["mysql41"].present? ||
+    settings["port"].present?
+  end
+
+  def verify
+    ThinkingSphinx::Configuration::DuplicateNames.new(indices).reconcile
   end
 end
+
+require 'thinking_sphinx/configuration/consistent_ids'
+require 'thinking_sphinx/configuration/defaults'
+require 'thinking_sphinx/configuration/distributed_indices'
+require 'thinking_sphinx/configuration/duplicate_names'
+require 'thinking_sphinx/configuration/minimum_fields'