thinking-sphinx 3.1.1 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff053fbf548b20a5ffe44d2bc171d68d6553dd08
-  data.tar.gz: 62849df76201a3ba5c65d7a2dfc6de0d641a7347
+  metadata.gz: 629217e38872edc12e92be134409a509b84f1304
+  data.tar.gz: 4b19eec90cfe3b560b1fcdd56a7018f07704ec94
 SHA512:
-  metadata.gz: 0c1b3ffb995ccf4588ffc9a3354adfcdc9a5b6eb3feec6ddff4ec791f26fc043557dcfb14e240c47cc80a86202ab6e6d0371b3adca937b6b9e2f9ee740f22370
-  data.tar.gz: 63e2df341ccdc388d6227890ddb00073dfdd32611f7427706e886df59dd6483b61530029d16a50c335ad12a7288b43e140701afe4cd69183bd0dc74e21b245c0
+  metadata.gz: 5116aad817ad119c97f05ec5bcdf330ca89ecae39ecac5695c4b9573ba062a88e25b6c8db22c596a32ec57ca8c044b2805360696b4b6342196d19b775886e340
+  data.tar.gz: 6923c30ad7163f0ec5c286b5cf98403c27618dc57191bff659808000f9f51bcf2e1ef0eec22fbf6ecc54752a9e1e6474d6e62a65b65db6cc462d8657beae183e

data/.travis.yml

@@ -10,12 +10,12 @@ before_script:
   - "mysql -e 'create database thinking_sphinx;' > /dev/null"
   - "psql -c 'create database thinking_sphinx;' -U postgres >/dev/null"
 env:
-  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.0.9/bin/ SPHINX_VERSION=2.0.9
-  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.0.9/bin/ SPHINX_VERSION=2.0.9
-  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.1.3/bin/ SPHINX_VERSION=2.1.3
-  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.1.3/bin/ SPHINX_VERSION=2.1.3
-  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.2.1/bin/ SPHINX_VERSION=2.2.1
-  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.2.1/bin/ SPHINX_VERSION=2.2.1
+  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.0.10/bin/ SPHINX_VERSION=2.0.10
+  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.0.10/bin/ SPHINX_VERSION=2.0.10
+  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.1.8/bin/ SPHINX_VERSION=2.1.8
+  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.1.8/bin/ SPHINX_VERSION=2.1.8
+  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.2.2-beta/bin/ SPHINX_VERSION=2.2.2
+  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.2.2-beta/bin/ SPHINX_VERSION=2.2.2
 gemfile:
   - gemfiles/rails_3_2.gemfile
   - gemfiles/rails_4_0.gemfile

data/HISTORY

@@ -1,3 +1,24 @@
+2014-11-04: 3.1.2
+* [CHANGE] regenerate task now only deletes index files for real-time indices.
+* [CHANGE] Raise an exception when a populated search query is modified (as it can't be requeried).
+* [FEATURE] Allow for custom paths for index files using :path option in the ThinkingSphinx::Index.define call.
+* [FIX] Ensure indexing guard files are removed when an exception is raised (Bobby Uhlenbrock).
+* [FIX] Don't update real-time indices for objects that are not persisted (Chance Downs).
+* [FEATURE] Allow the binlog path to be an empty string (Bobby Uhlenbrock).
+* [FIX] Use STI base class for polymorphic association replacements.
+* [FIX] Convert database setting keys to symbols for consistency with Rails (@dimko).
+* [FIX] Field weights and other search options are now respected from set_property.
+* [CHANGE] Log indices that aren't processed due to guard files existing.
+* [FEATURE] Add status task to report on whether Sphinx is running.
+* [FIX] Models with more than one index have correct facet counts (using Sphinx 2.1.x or newer).
+* [FEATURE] Real-time index callbacks can take a block for dynamic scoping.
+* [FIX] Some association fixes for Rails 4.1.
+* [CHANGE] Paginate records by 1000 results at a time when flagging as deleted.
+* [CHANGE] Default the Capistrano TS Rails environment to use rails_env, and then fall back to stage.
+* [CHANGE] rebuild task uses clear between stopping the daemon and indexing.
+* [FIX] Clear connections when raising connection errors.
+* [FEATURE] Allow casting of document ids pre-offset as bigints (via big_documents_id option).
+
 2014-04-22: 3.1.1
 * [CHANGE] Include full statements when query execution errors are raised (uglier, but more useful when debugging).
 * [FEATURE] Allow for common section in generated Sphinx configuration files for Sphinx 2.2.x (disabled by default, though) (Trevor Smith).

data/lib/thinking_sphinx.rb

@@ -55,6 +55,7 @@ require 'thinking_sphinx/facet'
 require 'thinking_sphinx/facet_search'
 require 'thinking_sphinx/float_formatter'
 require 'thinking_sphinx/frameworks'
+require 'thinking_sphinx/guard'
 require 'thinking_sphinx/index'
 require 'thinking_sphinx/index_set'
 require 'thinking_sphinx/masks'

data/lib/thinking_sphinx/active_record.rb

@@ -17,6 +17,7 @@ require 'thinking_sphinx/active_record/field'
 require 'thinking_sphinx/active_record/filtered_reflection'
 require 'thinking_sphinx/active_record/index'
 require 'thinking_sphinx/active_record/interpreter'
+require 'thinking_sphinx/active_record/join_association'
 require 'thinking_sphinx/active_record/log_subscriber'
 require 'thinking_sphinx/active_record/polymorpher'
 require 'thinking_sphinx/active_record/property_query'

data/lib/thinking_sphinx/active_record/base.rb

@@ -25,8 +25,9 @@ module ThinkingSphinx::ActiveRecord::Base
     end
 
     def search_for_ids(query = nil, options = {})
-      search = search query, options
-      ThinkingSphinx::Search::Merger.new(search).merge! nil, :ids_only => true
+      ThinkingSphinx::Search::Merger.new(
+        search(query, options)
+      ).merge! nil, :ids_only => true
     end
 
     private

data/lib/thinking_sphinx/active_record/database_adapters/mysql_adapter.rb

@@ -5,6 +5,10 @@ class ThinkingSphinx::ActiveRecord::DatabaseAdapters::MySQLAdapter <
     value ? 1 : 0
   end
 
+  def cast_to_bigint(clause)
+    "CAST(#{clause} AS UNSIGNED INTEGER)"
+  end
+
   def cast_to_string(clause)
     "CAST(#{clause} AS char)"
   end

data/lib/thinking_sphinx/active_record/database_adapters/postgresql_adapter.rb

@@ -5,6 +5,10 @@ class ThinkingSphinx::ActiveRecord::DatabaseAdapters::PostgreSQLAdapter <
     value ? 'TRUE' : 'FALSE'
   end
 
+  def cast_to_bigint(clause)
+    "#{clause}::bigint"
+  end
+
   def cast_to_string(clause)
     "#{clause}::varchar"
   end

data/lib/thinking_sphinx/active_record/filtered_reflection.rb

@@ -17,7 +17,10 @@ class ThinkingSphinx::ActiveRecord::FilteredReflection <
       @options[:foreign_key] ||= "#{reflection.name}_id"
       @options[:foreign_type]  = reflection.foreign_type
 
-      return @options if reflection.respond_to?(:scope)
+      if reflection.respond_to?(:scope)
+        @options[:sphinx_internal_filtered] = true
+        return @options
+      end
 
       case @options[:conditions]
       when nil
@@ -34,11 +37,16 @@ class ThinkingSphinx::ActiveRecord::FilteredReflection <
     end
 
     def scope
+      if ::Joiner::Joins.instance_methods.include?(:join_association_class)
+        return nil
+      end
+
       lambda { |association|
         reflection = association.reflection
+        klass      = reflection.class_name.constantize
         where(
           association.parent.aliased_table_name.to_sym =>
-          {reflection.foreign_type => reflection.class_name}
+          {reflection.foreign_type => klass.base_class.name}
         )
       }
     end

data/lib/thinking_sphinx/active_record/interpreter.rb

@@ -39,7 +39,7 @@ class ThinkingSphinx::ActiveRecord::Interpreter <
   end
 
   def set_database(hash_or_key)
-    configuration = hash_or_key.is_a?(::Hash) ? hash_or_key :
+    configuration = hash_or_key.is_a?(::Hash) ? hash_or_key.symbolize_keys :
       ::ActiveRecord::Base.configurations[hash_or_key.to_s]
 
     __source.set_database_settings configuration
@@ -50,6 +50,7 @@ class ThinkingSphinx::ActiveRecord::Interpreter <
       @index.send("#{key}=", value)   if @index.class.settings.include?(key)
       __source.send("#{key}=", value) if __source.class.settings.include?(key)
       __source.options[key] = value   if source_option?(key)
+      @index.options[key] = value     if search_option?(key)
     end
   end
 

data/lib/thinking_sphinx/active_record/join_association.rb

@@ -0,0 +1,13 @@
+class ThinkingSphinx::ActiveRecord::JoinAssociation <
+  ::ActiveRecord::Associations::JoinDependency::JoinAssociation
+
+  def build_constraint(klass, table, key, foreign_table, foreign_key)
+    constraint = super
+
+    constraint = constraint.and(
+      foreign_table[reflection.options[:foreign_type]].eq(base_klass.name)
+    ) if reflection.options[:sphinx_internal_filtered]
+
+    constraint
+  end
+end

data/lib/thinking_sphinx/active_record/log_subscriber.rb

@@ -1,13 +1,18 @@
 class ThinkingSphinx::ActiveRecord::LogSubscriber < ActiveSupport::LogSubscriber
-  def query(event)
-    identifier = color('Sphinx Query (%.1fms)' % event.duration, GREEN, true)
-    debug "  #{identifier}  #{event.payload[:query]}"
+  def guard(event)
+    identifier = color 'Sphinx', GREEN, true
+    warn "  #{identifier}  #{event.payload[:guard]}"
   end
 
   def message(event)
     identifier = color 'Sphinx', GREEN, true
     debug "  #{identifier}  #{event.payload[:message]}"
   end
+
+  def query(event)
+    identifier = color('Sphinx Query (%.1fms)' % event.duration, GREEN, true)
+    debug "  #{identifier}  #{event.payload[:query]}"
+  end
 end
 
 ThinkingSphinx::ActiveRecord::LogSubscriber.attach_to :thinking_sphinx

data/lib/thinking_sphinx/active_record/polymorpher.rb

@@ -14,7 +14,14 @@ class ThinkingSphinx::ActiveRecord::Polymorpher
 
   def append_reflections
     mappings.each do |class_name, name|
-      klass.reflections[name] ||= clone_with name, class_name
+      next if klass.reflections[name]
+
+      reflection = clone_with name, class_name
+      if ActiveRecord::Reflection.respond_to?(:add_reflection)
+        ActiveRecord::Reflection.add_reflection klass, name, reflection
+      else
+        klass.reflections[name] = reflection
+      end
     end
   end
 

data/lib/thinking_sphinx/active_record/sql_builder.rb

@@ -34,7 +34,7 @@ module ThinkingSphinx
 
       delegate :adapter, :model, :delta_processor, :to => :source
       delegate :convert_nulls, :time_zone_query_pre, :utf8_query_pre,
-        :to => :adapter
+        :cast_to_bigint, :to => :adapter
 
       def query
         Query.new(self)
@@ -53,10 +53,17 @@ module ThinkingSphinx
       end
 
       def associations
-        @associations ||= Joiner::Joins.new(model).tap do |assocs|
+        @associations ||= begin
+          joins = Joiner::Joins.new model
+          if joins.respond_to?(:join_association_class)
+            joins.join_association_class = ThinkingSphinx::ActiveRecord::JoinAssociation
+          end
+
           source.associations.reject(&:string?).each do |association|
-            assocs.add_join_to association.stack
+            joins.add_join_to association.stack
           end
+
+          joins
         end
       end
 
@@ -76,9 +83,17 @@ module ThinkingSphinx
         ('SQL_NO_CACHE ' if source.type == 'mysql').to_s
       end
 
+      def big_document_ids?
+        source.options[:big_document_ids] || config.settings['big_document_ids']
+      end
+
       def document_id
         quoted_alias = quote_column source.primary_key
-        "#{quoted_primary_key} * #{config.indices.count} + #{source.offset} AS #{quoted_alias}"
+        column = quoted_primary_key
+        column = cast_to_bigint column if big_document_ids?
+        column = "#{column} * #{config.indices.count} + #{source.offset}"
+
+        "#{column} AS #{quoted_alias}"
       end
 
       def reversed_document_id

data/lib/thinking_sphinx/active_record/sql_builder/statement.rb

@@ -4,8 +4,8 @@ module ThinkingSphinx
   module ActiveRecord
     class SQLBuilder::Statement
       def initialize(report)
-        self.report = report
-        self.scope = relation
+        @report = report
+        @scope  = relation
       end
 
       def to_relation
@@ -32,8 +32,9 @@ module ThinkingSphinx
         scope
       end
 
-      protected
-      attr_accessor :report, :scope
+      private
+
+      attr_reader :report, :scope
 
       def custom_joins
         @custom_joins ||= source.associations.select(&:string?).collect(&:to_s)
@@ -43,11 +44,13 @@ module ThinkingSphinx
         minimum = convert_nulls "MIN(#{quoted_primary_key})", 1
         maximum = convert_nulls "MAX(#{quoted_primary_key})", 1
 
-        self.scope = scope.select("#{minimum}, #{maximum}").where(where_clause(true))
+        @scope = scope.select("#{minimum}, #{maximum}").where(
+          where_clause(true)
+        )
       end
 
       def filter_by_query_info
-        self.scope = scope.where("#{quoted_primary_key} = #{reversed_document_id}")
+        @scope = scope.where("#{quoted_primary_key} = #{reversed_document_id}")
       end
 
       def filter_by_scopes
@@ -86,27 +89,31 @@ module ThinkingSphinx
       end
 
       def scope_by_select
-        self.scope = scope.select(pre_select + select_clause)
+        @scope = scope.select(pre_select + select_clause)
       end
 
       def scope_by_where_clause
-        self.scope = scope.where where_clause
+        @scope = scope.where where_clause
       end
 
       def scope_by_group_clause
-        self.scope = scope.group(group_clause)
+        @scope = scope.group(group_clause)
       end
 
       def scope_by_joins
-        self.scope = scope.joins(associations.join_values)
+        @scope = scope.joins(associations.join_values)
       end
 
       def scope_by_custom_joins
-        self.scope = scope.joins(custom_joins) if custom_joins.any?
+        @scope = scope.joins(custom_joins) if custom_joins.any?
       end
 
       def scope_by_order
-        self.scope = scope.order('NULL') if source.type == 'mysql'
+        @scope = scope.order('NULL') if source.type == 'mysql'
+      end
+
+      def source
+        report.source
       end
 
       def method_missing(*args, &block)

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -7,8 +7,8 @@ module ThinkingSphinx
         :groupings, :polymorphs
 
       OPTIONS = [:name, :offset, :delta_processor, :delta?, :delta_options,
-        :disable_range?, :group_concat_max_len, :utf8?, :position, 
-        :minimal_group_by?]
+        :disable_range?, :group_concat_max_len, :utf8?, :position,
+        :minimal_group_by?, :big_document_ids]
 
       def initialize(model, options = {})
         @model             = model

data/lib/thinking_sphinx/capistrano/v3.rb

@@ -1,7 +1,7 @@
 namespace :load do
   task :defaults do
     set :thinking_sphinx_roles, :db
-    set :thinking_sphinx_rails_env, -> { fetch(:stage) }
+    set :thinking_sphinx_rails_env, -> { fetch(:rails_env) || fetch(:stage) }
   end
 end
 

data/lib/thinking_sphinx/configuration.rb

@@ -91,7 +91,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
   end
 
   def render_to_file
-    FileUtils.mkdir_p searchd.binlog_path
+    FileUtils.mkdir_p searchd.binlog_path unless searchd.binlog_path.blank?
 
     open(configuration_file, 'w') { |file| file.write render }
   end

data/lib/thinking_sphinx/connection.rb

@@ -1,4 +1,6 @@
 module ThinkingSphinx::Connection
+  MAXIMUM_RETRIES = 3
+
   def self.new
     configuration = ThinkingSphinx::Configuration.instance
     # If you use localhost, MySQL insists on a socket connection, but Sphinx
@@ -37,13 +39,13 @@ module ThinkingSphinx::Connection
           yield connection
         rescue ThinkingSphinx::QueryExecutionError, connection.base_error => error
           original = ThinkingSphinx::SphinxError.new_from_mysql error
-          raise original if original.is_a?(ThinkingSphinx::QueryError)
+          retries += MAXIMUM_RETRIES if original.is_a?(ThinkingSphinx::QueryError)
           raise Innertube::Pool::BadResource
         end
       end
     rescue Innertube::Pool::BadResource
       retries += 1
-      raise original unless retries < 3
+      raise original unless retries < MAXIMUM_RETRIES
 
       ActiveSupport::Notifications.instrument(
         "message.thinking_sphinx", :message => "Retrying query \"#{original.statement}\" after error: #{original.message}"

data/lib/thinking_sphinx/controller.rb

@@ -3,18 +3,8 @@ class ThinkingSphinx::Controller < Riddle::Controller
     options = indices.extract_options!
     indices << '--all' if indices.empty?
 
-    indices = indices.reject { |index| File.exists? guard_file(index) }
-    return if indices.empty?
-
-    indices.each { |index| FileUtils.touch guard_file(index) }
-    super(*(indices + [options]))
-    indices.each { |index| FileUtils.rm guard_file(index) }
-  end
-
-  def guard_file(index)
-    File.join(
-      ThinkingSphinx::Configuration.instance.indices_location,
-      "ts-#{index}.tmp"
-    )
+    ThinkingSphinx::Guard::Files.call(indices) do |names|
+      super(*(names + [options]))
+    end
   end
 end

data/lib/thinking_sphinx/core/index.rb

@@ -3,7 +3,7 @@ module ThinkingSphinx::Core::Index
   include ThinkingSphinx::Core::Settings
 
   included do
-    attr_reader :reference, :offset, :options
+    attr_reader :reference, :offset
     attr_writer :definition_block
   end
 
@@ -27,7 +27,7 @@ module ThinkingSphinx::Core::Index
   end
 
   def document_id_for_key(key)
-     key * config.indices.count + offset
+    key * config.indices.count + offset
   end
 
   def interpret_definition!
@@ -43,6 +43,11 @@ module ThinkingSphinx::Core::Index
     @model ||= reference.to_s.camelize.constantize
   end
 
+  def options
+    interpret_definition!
+    @options
+  end
+
   def render
     pre_render
     set_path
@@ -71,11 +76,16 @@ module ThinkingSphinx::Core::Index
     'core'
   end
 
+  def path_prefix
+    options[:path] || config.indices_location
+  end
+
   def pre_render
     interpret_definition!
   end
 
   def set_path
-    @path ||= File.join config.indices_location, name
+    FileUtils.mkdir_p path_prefix
+    @path = File.join path_prefix, name
   end
 end