thinking-sphinx 3.1.4 → 6.0.0

data/lib/thinking_sphinx/active_record/sql_builder/query.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx
   module ActiveRecord
     class SQLBuilder::Query
@@ -18,10 +20,17 @@ module ThinkingSphinx
 
       def filter_by_query_pre
         scope_by_time_zone
+        scope_by_delta_processor
         scope_by_session
         scope_by_utf8
       end
 
+      def scope_by_delta_processor
+        return unless delta_processor && !source.delta?
+
+        self.scope << delta_processor.reset_query
+      end
+
       def scope_by_session
         return unless max_len = source.options[:group_concat_max_len]
 
@@ -38,6 +47,10 @@ module ThinkingSphinx
         self.scope += utf8_query_pre if source.options[:utf8?]
       end
 
+      def source
+        report.source
+      end
+      
       def method_missing(*args, &block)
         report.send *args, &block
       end

data/lib/thinking_sphinx/active_record/sql_builder/statement.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thinking_sphinx/active_record/sql_builder/clause_builder'
 
 module ThinkingSphinx

data/lib/thinking_sphinx/active_record/sql_builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx
   module ActiveRecord
     class SQLBuilder
@@ -20,12 +22,6 @@ module ThinkingSphinx
         query.to_query
       end
 
-      def sql_query_post_index
-        return [] unless delta_processor && !source.delta?
-
-        [delta_processor.reset_query]
-      end
-
       private
 
       delegate :adapter, :model, :delta_processor, :to => :source
@@ -49,18 +45,9 @@ module ThinkingSphinx
       end
 
       def associations
-        @associations ||= begin
-          joins = Joiner::Joins.new model
-          if joins.respond_to?(:join_association_class)
-            joins.join_association_class = ThinkingSphinx::ActiveRecord::JoinAssociation
-          end
-
-          source.associations.reject(&:string?).each do |association|
-            joins.add_join_to association.stack
-          end
-
-          joins
-        end
+        @associations ||= ThinkingSphinx::ActiveRecord::SourceJoins.call(
+          model, source
+        )
       end
 
       def quote_column(column)

data/lib/thinking_sphinx/active_record/sql_source/template.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::ActiveRecord::SQLSource::Template
   attr_reader :source
 
@@ -16,14 +18,14 @@ class ThinkingSphinx::ActiveRecord::SQLSource::Template
   private
 
   def add_attribute(column, name, type, options = {})
-    source.attributes << ThinkingSphinx::ActiveRecord::Attribute.new(
+    source.add_attribute ThinkingSphinx::ActiveRecord::Attribute.new(
       source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
       options.merge(:as => name, :type => type)
     )
   end
 
   def add_field(column, name, options = {})
-    source.fields << ThinkingSphinx::ActiveRecord::Field.new(
+    source.add_field ThinkingSphinx::ActiveRecord::Field.new(
       source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
       options.merge(:as => name)
     )
@@ -48,6 +50,6 @@ class ThinkingSphinx::ActiveRecord::SQLSource::Template
   end
 
   def primary_key
-    source.model.primary_key.to_sym
+    source.options[:primary_key].to_sym
   end
 end

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx
   module ActiveRecord
     class SQLSource < Riddle::Configuration::SQLSource
       include ThinkingSphinx::Core::Settings
-      attr_reader :model, :database_settings, :options
+
+      attr_reader :model, :options
       attr_accessor :fields, :attributes, :associations, :conditions,
         :groupings, :polymorphs
 
@@ -12,9 +15,8 @@ module ThinkingSphinx
 
       def initialize(model, options = {})
         @model             = model
-        @database_settings = model.connection.instance_variable_get(:@config).clone
         @options           = {
-          :utf8? => (@database_settings[:encoding] == 'utf8')
+          :utf8? => (database_settings[:encoding].to_s[/^utf8/])
         }.merge options
 
         @fields            = []
@@ -37,6 +39,18 @@ module ThinkingSphinx
         @adapter ||= DatabaseAdapters.adapter_for(@model)
       end
 
+      def add_attribute(attribute)
+        attributes.delete_if { |existing| existing.name == attribute.name }
+
+        attributes << attribute
+      end
+
+      def add_field(field)
+        fields.delete_if { |existing| existing.name == field.name }
+
+        fields << field
+      end
+
       def delta_processor
         options[:delta_processor].try(:new, adapter, @options[:delta_options] || {})
       end
@@ -61,6 +75,10 @@ module ThinkingSphinx
         options[:primary_key]
       end
 
+      def properties
+        fields + attributes
+      end
+
       def render
         prepare_for_render unless @prepared
 
@@ -74,6 +92,9 @@ module ThinkingSphinx
         @sql_db   ||= settings[:database]
         @sql_port ||= settings[:port]
         @sql_sock ||= settings[:socket]
+        @mysql_ssl_cert ||= settings[:sslcert]
+        @mysql_ssl_key  ||= settings[:sslkey]
+        @mysql_ssl_ca   ||= settings[:sslca]
       end
 
       def type
@@ -83,7 +104,7 @@ module ThinkingSphinx
         when DatabaseAdapters::PostgreSQLAdapter
           'pgsql'
         else
-          raise "Unknown Adapter Type: #{adapter.class.name}"
+          raise UnknownDatabaseAdapter, "Provided type: #{adapter.class.name}"
         end
       end
 
@@ -119,13 +140,22 @@ module ThinkingSphinx
         @sql_query             = builder.sql_query
         @sql_query_range     ||= builder.sql_query_range
         @sql_query_pre        += builder.sql_query_pre
-        @sql_query_post_index += builder.sql_query_post_index
       end
 
       def config
         ThinkingSphinx::Configuration.instance
       end
 
+      def database_settings
+        @database_settings ||= begin
+          if model.connection.respond_to?(:config)
+            model.connection.config.clone
+          else
+            model.connection.instance_variable_get(:@config).clone
+          end
+        end
+      end
+
       def prepare_for_render
         polymorphs.each &:morph!
         append_presenter_to_attribute_array
@@ -136,10 +166,6 @@ module ThinkingSphinx
 
         @prepared = true
       end
-
-      def properties
-        fields + attributes
-      end
     end
   end
 end

data/lib/thinking_sphinx/active_record.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 require 'active_record'
 require 'joiner'
 
 module ThinkingSphinx::ActiveRecord
   module Callbacks; end
+  module Depolymorph; end
 end
 
 require 'thinking_sphinx/active_record/property'
@@ -14,7 +17,6 @@ require 'thinking_sphinx/active_record/column'
 require 'thinking_sphinx/active_record/column_sql_presenter'
 require 'thinking_sphinx/active_record/database_adapters'
 require 'thinking_sphinx/active_record/field'
-require 'thinking_sphinx/active_record/filter_reflection'
 require 'thinking_sphinx/active_record/index'
 require 'thinking_sphinx/active_record/interpreter'
 require 'thinking_sphinx/active_record/join_association'
@@ -23,9 +25,20 @@ require 'thinking_sphinx/active_record/polymorpher'
 require 'thinking_sphinx/active_record/property_query'
 require 'thinking_sphinx/active_record/property_sql_presenter'
 require 'thinking_sphinx/active_record/simple_many_query'
+require 'thinking_sphinx/active_record/source_joins'
 require 'thinking_sphinx/active_record/sql_builder'
 require 'thinking_sphinx/active_record/sql_source'
 
+require 'thinking_sphinx/active_record/callbacks/association_delta_callbacks'
 require 'thinking_sphinx/active_record/callbacks/delete_callbacks'
 require 'thinking_sphinx/active_record/callbacks/delta_callbacks'
 require 'thinking_sphinx/active_record/callbacks/update_callbacks'
+
+require 'thinking_sphinx/active_record/depolymorph/base_reflection'
+require 'thinking_sphinx/active_record/depolymorph/association_reflection'
+require 'thinking_sphinx/active_record/depolymorph/conditions_reflection'
+require 'thinking_sphinx/active_record/depolymorph/overridden_reflection'
+require 'thinking_sphinx/active_record/depolymorph/scoped_reflection'
+require 'thinking_sphinx/active_record/filter_reflection'
+
+ActiveRecord::Base.include ThinkingSphinx::ActiveRecord::Base

data/lib/thinking_sphinx/attribute_types.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::AttributeTypes
+  def self.call
+    @call ||= new.call
+  end
+
+  def self.reset
+    @call = nil
+  end
+
+  def call
+    return {} unless File.exist?(configuration_file)
+
+    realtime_indices.each { |index|
+      map_types_with_prefix index, :rt,
+        [:uint, :bigint, :float, :timestamp, :string, :bool, :json]
+
+      index.rt_attr_multi.each     { |name| attributes[name] << :uint }
+      index.rt_attr_multi_64.each  { |name| attributes[name] << :bigint }
+    }
+
+    plain_sources.each { |source|
+      map_types_with_prefix source, :sql,
+        [:uint, :bigint, :float, :timestamp, :string, :bool, :json]
+
+      source.sql_attr_str2ordinal    { |name| attributes[name] << :uint }
+      source.sql_attr_str2wordcount  { |name| attributes[name] << :uint }
+      source.sql_attr_multi.each { |setting|
+        type, name, *ignored = setting.split(/\s+/)
+        attributes[name] << type.to_sym
+      }
+    }
+
+    attributes.values.each &:uniq!
+    attributes
+  end
+
+  private
+
+  def attributes
+    @attributes ||= Hash.new { |hash, key| hash[key] = [] }
+  end
+
+  def configuration
+    @configuration ||= Riddle::Configuration.parse!(
+      File.read(configuration_file)
+    )
+  end
+
+  def configuration_file
+    ThinkingSphinx::Configuration.instance.configuration_file
+  end
+
+  def map_types_with_prefix(object, prefix, types)
+    types.each do |type|
+      object.public_send("#{prefix}_attr_#{type}").each do |name|
+        attributes[name] << type
+      end
+    end
+  end
+
+  def plain_sources
+    configuration.indices.select { |index|
+      index.type == 'plain' || index.type.nil?
+    }.collect(&:sources).flatten
+  end
+
+  def realtime_indices
+    configuration.indices.select { |index| index.type == 'rt' }
+  end
+end

data/lib/thinking_sphinx/batched_search.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::BatchedSearch
   attr_accessor :searches
 

data/lib/thinking_sphinx/callbacks/appender.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Callbacks::Appender
+  def self.call(model, reference, options, &block)
+    new(model, reference, options, &block).call
+  end
+
+  def initialize(model, reference, options, &block)
+    @model = model
+    @reference = reference
+    @options = options
+    @block = block
+  end
+
+  def call
+    add_core_callbacks
+    add_delta_callbacks     if behaviours.include?(:deltas)
+    add_real_time_callbacks if behaviours.include?(:real_time)
+    add_update_callbacks    if behaviours.include?(:updates)
+  end
+
+  private
+
+  attr_reader :model, :reference, :options, :block
+
+  def add_core_callbacks
+    model.after_commit(
+      ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks,
+      on: :destroy
+    )
+  end
+
+  def add_delta_callbacks
+    if path.empty?
+      model.before_save  ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
+      model.after_commit ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
+    else
+      model.after_commit(
+        ThinkingSphinx::ActiveRecord::Callbacks::AssociationDeltaCallbacks
+          .new(path)
+      )
+    end
+  end
+
+  def add_real_time_callbacks
+    model.after_commit(
+      ThinkingSphinx::RealTime.callback_for(reference, path, &block),
+      on: [:create, :update]
+    )
+  end
+
+  def add_update_callbacks
+    model.after_update ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks
+  end
+
+  def behaviours
+    options[:behaviours] || []
+  end
+
+  def path
+    options[:path] || []
+  end
+end

data/lib/thinking_sphinx/callbacks.rb

@@ -1,6 +1,15 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Callbacks
   attr_reader :instance
 
+  def self.append(model, reference = nil, options, &block)
+    reference ||= ThinkingSphinx::Configuration.instance.index_set_class.
+      reference_name(model)
+
+    ThinkingSphinx::Callbacks::Appender.call(model, reference, options, &block)
+  end
+
   def self.callbacks(*methods)
     mod = Module.new
     methods.each do |method|
@@ -9,7 +18,27 @@ class ThinkingSphinx::Callbacks
     extend mod
   end
 
+  def self.resume!
+    @suspended = false
+  end
+
+  def self.suspend(&block)
+    suspend!
+    yield
+    resume!
+  end
+
+  def self.suspend!
+    @suspended = true
+  end
+
+  def self.suspended?
+    @suspended
+  end
+
   def initialize(instance)
     @instance = instance
   end
 end
+
+require "thinking_sphinx/callbacks/appender"

data/lib/thinking_sphinx/capistrano/v2.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Capistrano::Configuration.instance(:must_exist).load do
   _cset(:thinking_sphinx_roles)   { :db }
   _cset(:thinking_sphinx_options) { {:roles => fetch(:thinking_sphinx_roles)} }

data/lib/thinking_sphinx/capistrano/v3.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 namespace :load do
   task :defaults do
     set :thinking_sphinx_roles, :db

data/lib/thinking_sphinx/capistrano.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 if defined?(Capistrano::VERSION)
   if Gem::Version.new(Capistrano::VERSION).release >= Gem::Version.new('3.0.0')
     recipe_version = 3

data/lib/thinking_sphinx/commander.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commander
+  def self.call(command, configuration, options, stream = STDOUT)
+    raise ThinkingSphinx::UnknownCommand unless registry.keys.include?(command)
+
+    registry[command].call configuration, options, stream
+  end
+
+  def self.registry
+    @registry ||= {
+      :clear_real_time  => ThinkingSphinx::Commands::ClearRealTime,
+      :clear_sql        => ThinkingSphinx::Commands::ClearSQL,
+      :configure        => ThinkingSphinx::Commands::Configure,
+      :index_sql        => ThinkingSphinx::Commands::IndexSQL,
+      :index_real_time  => ThinkingSphinx::Commands::IndexRealTime,
+      :merge            => ThinkingSphinx::Commands::Merge,
+      :merge_and_update => ThinkingSphinx::Commands::MergeAndUpdate,
+      :prepare          => ThinkingSphinx::Commands::Prepare,
+      :rotate           => ThinkingSphinx::Commands::Rotate,
+      :running          => ThinkingSphinx::Commands::Running,
+      :start_attached   => ThinkingSphinx::Commands::StartAttached,
+      :start_detached   => ThinkingSphinx::Commands::StartDetached,
+      :stop             => ThinkingSphinx::Commands::Stop
+    }
+  end
+end

data/lib/thinking_sphinx/commands/base.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::Base
+  include ThinkingSphinx::WithOutput
+
+  def self.call(configuration, options, stream = STDOUT)
+    new(configuration, options, stream).call_with_handling
+  end
+
+  def call_with_handling
+    call
+  rescue Riddle::CommandFailedError => error
+    handle_failure error.command_result
+  end
+
+  private
+
+  delegate :controller, :to => :configuration
+
+  def command(command, extra_options = {})
+    ThinkingSphinx::Commander.call(
+      command, configuration, options.merge(extra_options), stream
+    )
+  end
+
+  def command_output(output)
+    return "See above\n" if output.nil?
+
+    "\n\t" + output.gsub("\n", "\n\t")
+  end
+
+  def handle_failure(result)
+    stream.puts <<-TXT
+
+The Sphinx #{type} command failed:
+  Command: #{result.command}
+  Status:  #{result.status}
+  Output:  #{command_output result.output}
+There may be more information about the failure in #{configuration.searchd.log}.
+    TXT
+    exit(result.status || 1)
+  end
+
+  def log(message)
+    return if options[:silent]
+
+    stream.puts message
+  end
+
+  def skip_directories?
+    configuration.settings['skip_directory_creation']
+  end
+end

data/lib/thinking_sphinx/commands/clear_real_time.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::ClearRealTime < ThinkingSphinx::Commands::Base
+  def call
+    options[:indices].each do |index|
+      index.render
+      Dir["#{index.path}.*"].each { |path| FileUtils.rm path }
+    end
+
+    FileUtils.rm_rf(binlog_path) if File.exist?(binlog_path)
+  end
+
+  private
+
+  def binlog_path
+    configuration.searchd.binlog_path
+  end
+
+  def type
+    'clear_realtime'
+  end
+end

data/lib/thinking_sphinx/commands/clear_sql.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::ClearSQL < ThinkingSphinx::Commands::Base
+  def call
+    options[:indices].each do |index|
+      index.render
+      Dir["#{index.path}.*"].each { |path| FileUtils.rm path }
+    end
+
+    FileUtils.rm_rf Dir["#{configuration.indices_location}/ts-*.tmp"]
+  end
+
+  private
+
+  def type
+    'clear_sql'
+  end
+end

data/lib/thinking_sphinx/commands/configure.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::Configure < ThinkingSphinx::Commands::Base
+  def call
+    log "Generating configuration to #{configuration.configuration_file}"
+
+    configuration.render_to_file
+  end
+
+  private
+
+  def type
+    'configure'
+  end
+end

data/lib/thinking_sphinx/commands/index_real_time.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::IndexRealTime < ThinkingSphinx::Commands::Base
+  def call
+    ThinkingSphinx::RealTime.processor.call options[:indices] do
+      command :rotate
+    end
+  end
+
+  private
+
+  def type
+    'indexing'
+  end
+end

data/lib/thinking_sphinx/commands/index_sql.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::IndexSQL < ThinkingSphinx::Commands::Base
+  def call
+    if indices.empty?
+      ThinkingSphinx.before_index_hooks.each { |hook| hook.call }
+    end
+
+    configuration.indexing_strategy.call(indices) do |index_names|
+      configuration.guarding_strategy.call(index_names) do |names|
+        controller.index *names, :verbose => options[:verbose]
+      end
+    end
+  end
+
+  private
+
+  def indices
+    options[:indices] || []
+  end
+
+  def type
+    'indexing'
+  end
+end

data/lib/thinking_sphinx/commands/merge.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Commands::Merge < ThinkingSphinx::Commands::Base
+  def call
+    return unless indices_exist?
+
+    controller.merge(
+      options[:core_index].name,
+      options[:delta_index].name,
+      :filters => options[:filters],
+      :verbose => options[:verbose]
+    )
+  end
+
+  private
+
+  delegate :controller, :to => :configuration
+
+  def indices_exist?
+    File.exist?("#{options[:core_index].path}.spi") &&
+    File.exist?("#{options[:delta_index].path}.spi")
+  end
+
+  def type
+    'merging'
+  end
+end