thinking-sphinx 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f8b8aa71063530ab0e184d27cc0fd8afe17da77
-  data.tar.gz: c54952a5a6aa5a4b84e75ceb57071ddfc2244851
+  metadata.gz: d8c3f957070c43b828b3c6756b1d891e3abff4d9
+  data.tar.gz: af7a4930372048d0ace812880e262a609cd8ff3f
 SHA512:
-  metadata.gz: 5684d082b99b5e6131d3244711316d099df261a9b07a1c1452f47150344e2e5ed32f55bdc0fe69948d62f291ef8fd99aeb71b936aebd845d96a2cee78453bc08
-  data.tar.gz: 8d6de05c533ff85787d7d248a548e53fdc1c370c7a625a5a6dd717b2c1bb449c16a1a54b643e87f8ea16ad8b0b844c0ac7c31c453da4a23ec6c763b03bf74cbf
+  metadata.gz: 825e6672b6a264387ec4da67df3a8df35c429f2b26c6c66fbd17968c0837d5bd8b5a257761d0f69d38ca1284e9a26695a6590842e1fdb967341b4b8abddc68ab
+  data.tar.gz: a0dde9c41198468f7ed223f5113341af6ca63c475b655e22b4b9225746f423faa73e5ff303589a5860d40e720ce7efcacabc7a53769ac8fdc0d9068c5e046207

data/.travis.yml

@@ -2,8 +2,9 @@ language: ruby
 rvm:
   - 2.0.0
   - 2.1
-  - 2.2
-  - jruby-19mode
+  - 2.2.2
+  - 2.3.1
+  - jruby-9.0.5.0
 before_install:
   - gem update --system
   - gem install bundler
@@ -24,3 +25,6 @@ addons:
   postgresql: "9.4"
 services:
   - postgresql
+matrix:
+  allow_failures:
+  - rvm: jruby-9.0.5.0

data/Appraisals

@@ -1,15 +1,28 @@
 appraise 'rails_3_2' do
   gem 'rails',  '~> 3.2.22.2'
+  gem 'rack',   '~> 1.0', :platforms => [:ruby_20, :ruby_21]
 end
 
 appraise 'rails_4_0' do
   gem 'rails', '~> 4.0.13'
+  gem 'rack',   '~> 1.0', :platforms => [:ruby_20, :ruby_21]
 end
 
 appraise 'rails_4_1' do
   gem 'rails', '~> 4.1.15'
+  gem 'rack',  '~> 1.0', :platforms => [:ruby_20, :ruby_21]
 end
 
 appraise 'rails_4_2' do
   gem 'rails', '~> 4.2.6'
+  gem 'rack',  '~> 1.0', :platforms => [:ruby_20, :ruby_21]
 end
+
+appraise 'rails_5_0' do
+  gem 'rails', '~> 5.0.0'
+  # gem 'activerecord-jdbc-adapter',
+  #   :git      => 'git://github.com/jruby/activerecord-jdbc-adapter.git',
+  #   :branch   => 'rails-5',
+  #   :platform => :jruby,
+  #   :ref      => 'c3570ce730'
+end if RUBY_VERSION.to_f >= 2.2 && RUBY_PLATFORM != 'java'

data/Gemfile

@@ -3,9 +3,10 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'mysql2', '~> 0.3.12b4', :platform => :ruby
-gem 'pg',     '~> 0.16.0',   :platform => :ruby
+gem 'pg',     '~> 0.18.4',   :platform => :ruby
 
 gem 'jdbc-mysql',                          '5.1.35',   :platform => :jruby
 gem 'activerecord-jdbcmysql-adapter',      '~> 1.3.4', :platform => :jruby
 gem 'activerecord-jdbcpostgresql-adapter', '~> 1.3.4', :platform => :jruby
 
+gem 'rack', '~> 1.0' if RUBY_VERSION.to_f <= 2.1

data/HISTORY

@@ -1,3 +1,22 @@
+2016-12-13: 3.3.0
+* [FEATURE] Real-time callbacks can now be used with after_commit hooks if that's preferred over after_save.
+* [CHANGE] Only toggle the delta value if the record has changed or is new (rather than on every single save call).
+* [CHANGE] Delta indexing is now quiet by default (rather than verbose).
+* [FIX] Explicit source method in the SQLQuery Builder instead of relying on method missing, thus avoiding any global methods named 'source' (Asaf Bartov).
+* [CHANGE] Use Riddle's reworked command interface for interacting with Sphinx's command-line tools.
+* [CHANGE] Respect Rake's quiet and silent flags for the Thinking Sphinx rake tasks.
+* [CHANGE] ts:start and ts:stop tasks default to verbose.
+* [FIX] Load indices before deleting index files, to ensure the files are actually found and deleted.
+* [FIX] Avoid loading ActiveRecord earlier than necessary. This avoids loading Rails out of order, which caused problems with Rails 5.
+* [FEATURE] Allow for custom batch sizes when populating real-time indices.
+* [FIX] Handle queries that are too long for Sphinx.
+* [FIX] Improve Rails 5 / JRuby support.
+* [CHANGE] Sort engine paths for loading indices to ensure they're consistent.
+* [CHANGE] Custom exception class for invalid database adapters.
+* [FIX] Fixed handling of multiple field tokens in wildcarding logic.
+* [CHANGE] Memoize the default primary keys per context.
+* [FIX] Ensure custom primary key columns are handled consistently (Julio Monteiro).
+
 2016-05-13: 3.2.0
 * [FIX] Ensure SQL table aliases are reliable for SQL-backed index queries.
 * [FEATURE] Add JSON attribute support for real-time indices.

data/README.textile

@@ -1,11 +1,12 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v3.2.0.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v3.3.0.
 
 h2. Upgrading
 
 Please refer to the release notes for any changes you need to make when upgrading:
 
+* "v3.3.0":https://github.com/pat/thinking-sphinx/releases/tag/v3.3.0
 * "v3.2.0":https://github.com/pat/thinking-sphinx/releases/tag/v3.2.0
 * "v3.1.4":https://github.com/pat/thinking-sphinx/releases/tag/v3.1.4
 * "v3.1.3":https://github.com/pat/thinking-sphinx/releases/tag/v3.1.3
@@ -22,7 +23,7 @@ It's a gem, so install it like you would any other gem. You will also need to sp
 
 <pre><code>gem 'mysql2',          '~> 0.3.18', :platform => :ruby
 gem 'jdbc-mysql',      '= 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 3.2.0'</code></pre>
+gem 'thinking-sphinx', '~> 3.3.0'</code></pre>
 
 The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database. If you're using JRuby, there is "currently an issue with Sphinx and jdbc-mysql 5.1.36 or newer":http://sphinxsearch.com/forum/view.html?id=13939, so you'll need to stick to nothing more recent than 5.1.35.
 

data/gemfiles/rails_3_2.gemfile

@@ -3,10 +3,11 @@
 source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.12b4", :platform => :ruby
-gem "pg", "~> 0.16.0", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
 gem "jdbc-mysql", "5.1.35", :platform => :jruby
 gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
 gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
 gem "rails", "~> 3.2.22.2"
+gem "rack", "~> 1.0", :platforms => [:ruby_20, :ruby_21]
 
 gemspec :path => "../"

data/gemfiles/rails_4_0.gemfile

@@ -3,10 +3,11 @@
 source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.12b4", :platform => :ruby
-gem "pg", "~> 0.16.0", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
 gem "jdbc-mysql", "5.1.35", :platform => :jruby
 gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
 gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
 gem "rails", "~> 4.0.13"
+gem "rack", "~> 1.0", :platforms => [:ruby_20, :ruby_21]
 
 gemspec :path => "../"

data/gemfiles/rails_4_1.gemfile

@@ -3,10 +3,11 @@
 source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.12b4", :platform => :ruby
-gem "pg", "~> 0.16.0", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
 gem "jdbc-mysql", "5.1.35", :platform => :jruby
 gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
 gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
 gem "rails", "~> 4.1.15"
+gem "rack", "~> 1.0", :platforms => [:ruby_20, :ruby_21]
 
 gemspec :path => "../"

data/gemfiles/rails_4_2.gemfile

@@ -3,10 +3,11 @@
 source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.12b4", :platform => :ruby
-gem "pg", "~> 0.16.0", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
 gem "jdbc-mysql", "5.1.35", :platform => :jruby
 gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
 gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
 gem "rails", "~> 4.2.6"
+gem "rack", "~> 1.0", :platforms => [:ruby_20, :ruby_21]
 
 gemspec :path => "../"

data/gemfiles/rails_5_0.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "mysql2", "~> 0.3.12b4", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
+gem "jdbc-mysql", "5.1.35", :platform => :jruby
+gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
+gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
+gem "rails", "~> 5.0.0"
+
+gemspec :path => "../"

data/lib/thinking_sphinx.rb

@@ -15,6 +15,8 @@ require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/module/attribute_accessors'
 
 module ThinkingSphinx
+  MAXIMUM_STATEMENT_LENGTH = (2 ** 23) - 1
+
   def self.count(query = '', options = {})
     search_for_ids(query, options).total_entries
   end

data/lib/thinking_sphinx/active_record/attribute/type.rb

@@ -41,7 +41,7 @@ class ThinkingSphinx::ActiveRecord::Attribute::Type
   end
 
   def big_integer?
-    database_column.type == :integer && database_column.sql_type[/bigint/i]
+    type_symbol == :integer && database_column.sql_type[/bigint/i]
   end
 
   def column_name
@@ -76,15 +76,29 @@ class ThinkingSphinx::ActiveRecord::Attribute::Type
 
     return :bigint if big_integer?
 
-    case database_column.type
+    case type_symbol
     when :datetime, :date
       :timestamp
     when :text
       :string
     when :decimal
       :float
+    when :integer, :boolean, :timestamp, :float, :string, :bigint, :json
+      type_symbol
     else
-      database_column.type
+      raise ThinkingSphinx::UnknownAttributeType,
+        <<-ERROR
+Unable to determine an equivalent Sphinx attribute type from #{database_column.type.class.name} for attribute #{attribute.name}. You may want to manually set the type.
+
+e.g.
+  has my_column, :type => :integer
+        ERROR
     end
   end
+
+  def type_symbol
+    return database_column.type if database_column.type.is_a?(Symbol)
+
+    database_column.type.class.name.demodulize.downcase.to_sym
+  end
 end

data/lib/thinking_sphinx/active_record/callbacks/delta_callbacks.rb

@@ -4,9 +4,7 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks <
   callbacks :after_commit, :before_save
 
   def after_commit
-    return unless !suspended? && delta_indices? && processors.any? { |processor|
-      processor.toggled?(instance)
-    }
+    return unless !suspended? && delta_indices? && toggled?
 
     delta_indices.each do |index|
       index.delta_processor.index index
@@ -18,7 +16,9 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks <
   end
 
   def before_save
-    return unless !ThinkingSphinx::Callbacks.suspended? && delta_indices?
+    return unless new_or_changed?           &&
+      !ThinkingSphinx::Callbacks.suspended? &&
+      delta_indices?
 
     processors.each { |processor| processor.toggle instance }
   end
@@ -45,6 +45,10 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks <
     @indices ||= config.index_set_class.new :classes => [instance.class]
   end
 
+  def new_or_changed?
+    instance.new_record? || instance.changed?
+  end
+
   def processors
     delta_indices.collect &:delta_processor
   end
@@ -52,4 +56,8 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks <
   def suspended?
     ThinkingSphinx::Callbacks.suspended? || ThinkingSphinx::Deltas.suspended?
   end
+
+  def toggled?
+    processors.any? { |processor| processor.toggled?(instance) }
+  end
 end

data/lib/thinking_sphinx/active_record/database_adapters.rb

@@ -12,7 +12,7 @@ module ThinkingSphinx::ActiveRecord::DatabaseAdapters
       when :postgresql
         PostgreSQLAdapter
       else
-        raise "Invalid Database Adapter '#{adapter}': Thinking Sphinx only supports MySQL and PostgreSQL."
+        raise ThinkingSphinx::InvalidDatabaseAdapter, "Invalid adapter '#{adapter}': Thinking Sphinx only supports MySQL and PostgreSQL."
       end
 
       klass.new model

data/lib/thinking_sphinx/active_record/sql_builder/query.rb

@@ -45,6 +45,10 @@ module ThinkingSphinx
         self.scope += utf8_query_pre if source.options[:utf8?]
       end
 
+      def source
+        report.source
+      end
+      
       def method_missing(*args, &block)
         report.send *args, &block
       end

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -90,7 +90,7 @@ module ThinkingSphinx
         when DatabaseAdapters::PostgreSQLAdapter
           'pgsql'
         else
-          raise "Unknown Adapter Type: #{adapter.class.name}"
+          raise UnknownDatabaseAdapter, "Provided type: #{adapter.class.name}"
         end
       end
 

data/lib/thinking_sphinx/configuration.rb

@@ -1,7 +1,7 @@
 require 'pathname'
 
 class ThinkingSphinx::Configuration < Riddle::Configuration
-  attr_accessor :configuration_file, :indices_location, :version
+  attr_accessor :configuration_file, :indices_location, :version, :batch_size
   attr_reader :index_paths
   attr_writer :controller, :index_set_class, :indexing_strategy
 
@@ -48,7 +48,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
   def engine_index_paths
     return [] unless defined?(Rails)
 
-    engine_indice_paths.flatten.compact
+    engine_indice_paths.flatten.compact.sort
   end
 
   def engine_indice_paths
@@ -121,6 +121,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
       'db', 'sphinx', environment
     ).to_s
     @version = settings['version'] || '2.1.4'
+    @batch_size = settings['batch_size'] || 1000
 
     if settings['common_sphinx_configuration']
       common.common_sphinx_configuration  = true

data/lib/thinking_sphinx/connection.rb

@@ -70,26 +70,39 @@ module ThinkingSphinx::Connection
     end
 
     def execute(statement)
-      query(statement).first
+      check_and_perform(statement).first
     end
 
     def query_all(*statements)
-      query *statements
+      check_and_perform statements.join('; ')
     end
 
     private
 
+    def check(statements)
+      if statements.length > ThinkingSphinx::MAXIMUM_STATEMENT_LENGTH
+        exception           = ThinkingSphinx::QueryLengthError.new
+        exception.statement = statements
+        raise exception
+      end
+    end
+
+    def check_and_perform(statements)
+      check statements
+      perform statements
+    end
+
     def close_and_clear
       client.close
       @client = nil
     end
 
-    def query(*statements)
-      results_for *statements
+    def perform(statements)
+      results_for statements
     rescue => error
-      message           = "#{error.message} - #{statements.join('; ')}"
+      message           = "#{error.message} - #{statements}"
       wrapper           = ThinkingSphinx::QueryExecutionError.new message
-      wrapper.statement = statements.join('; ')
+      wrapper.statement = statements
       raise wrapper
     ensure
       close_and_clear unless ThinkingSphinx::Connection.persistent?
@@ -117,8 +130,8 @@ module ThinkingSphinx::Connection
       raise ThinkingSphinx::SphinxError.new_from_mysql error
     end
 
-    def results_for(*statements)
-      results  = [client.query(statements.join('; '))]
+    def results_for(statements)
+      results  = [client.query(statements)]
       results << client.store_result while client.next_result
       results
     end
@@ -145,9 +158,9 @@ module ThinkingSphinx::Connection
       raise ThinkingSphinx::SphinxError.new_from_mysql error
     end
 
-    def results_for(*statements)
+    def results_for(statements)
       statement = client.createStatement
-      statement.execute statements.join('; ')
+      statement.execute statements
 
       results   = [set_to_array(statement.getResultSet)]
       results  << set_to_array(statement.getResultSet) while statement.getMoreResults

data/lib/thinking_sphinx/deltas/index_job.rb

@@ -4,8 +4,7 @@ class ThinkingSphinx::Deltas::IndexJob
   end
 
   def perform
-    configuration.controller.index @index_name,
-      :verbose => !configuration.settings['quiet_deltas']
+    configuration.controller.index @index_name, :verbose => !quiet_deltas?
   end
 
   private
@@ -13,4 +12,9 @@ class ThinkingSphinx::Deltas::IndexJob
   def configuration
     @configuration ||= ThinkingSphinx::Configuration.instance
   end
+
+  def quiet_deltas?
+    configuration.settings['quiet_deltas'].nil? ||
+    configuration.settings['quiet_deltas']
+  end
 end

data/lib/thinking_sphinx/errors.rb

@@ -31,6 +31,19 @@ end
 class ThinkingSphinx::QueryError < ThinkingSphinx::SphinxError
 end
 
+class ThinkingSphinx::QueryLengthError < ThinkingSphinx::SphinxError
+  def message
+    <<-MESSAGE
+The supplied SphinxQL statement is #{statement.length} characters long. The maximum allowed length is #{ThinkingSphinx::MAXIMUM_STATEMENT_LENGTH}.
+
+If this error has been raised during real-time index population, it's probably due to overly large batches of records being processed at once. The default is 1000, but you can lower it on a per-environment basis in config/thinking_sphinx.yml:
+
+  development:
+    batch_size: 500
+    MESSAGE
+  end
+end
+
 class ThinkingSphinx::SyntaxError < ThinkingSphinx::QueryError
 end
 
@@ -58,3 +71,15 @@ end
 
 class ThinkingSphinx::DuplicateNameError < StandardError
 end
+
+class ThinkingSphinx::InvalidDatabaseAdapter < StandardError
+end
+
+class ThinkingSphinx::SphinxAlreadyRunning < StandardError
+end
+
+class ThinkingSphinx::UnknownDatabaseAdapter < StandardError
+end
+
+class ThinkingSphinx::UnknownAttributeType < StandardError
+end