thinking-sphinx 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8c3f957070c43b828b3c6756b1d891e3abff4d9
-  data.tar.gz: af7a4930372048d0ace812880e262a609cd8ff3f
+  metadata.gz: b763d739981ac8436e92fa07b67bae1e01912014
+  data.tar.gz: e11258a8602e0005be61a97da389365524bac728
 SHA512:
-  metadata.gz: 825e6672b6a264387ec4da67df3a8df35c429f2b26c6c66fbd17968c0837d5bd8b5a257761d0f69d38ca1284e9a26695a6590842e1fdb967341b4b8abddc68ab
-  data.tar.gz: a0dde9c41198468f7ed223f5113341af6ca63c475b655e22b4b9225746f423faa73e5ff303589a5860d40e720ce7efcacabc7a53769ac8fdc0d9068c5e046207
+  metadata.gz: 7c66d2e95dc7af7b45f4291e5c28e88b400442087add27c2fe43a55e5f4384781adf0c34326ae5e80d22882821f136b8f100b85e5337c335f6612a0fa12b9f40
+  data.tar.gz: 28909c68df0089f551a0fc2d5026b5259d7dfe4b1f8132ef0aea7fa191d9848a46f2d8e3ca08b8282eb1529bc670304aca2c850fde4a4fa561234369c6beb36d

data/.gitignore

@@ -1,6 +1,7 @@
 *.gem
 .bundle
 .rbx
+gemfiles
 Gemfile.lock
 *.sublime-*
 pkg/*

data/.travis.yml

@@ -1,30 +1,39 @@
 language: ruby
 rvm:
-  - 2.0.0
-  - 2.1
-  - 2.2.2
-  - 2.3.1
-  - jruby-9.0.5.0
+- 2.2.2
+- 2.3.4
+- 2.4.1
+- jruby-9.1.10.0
 before_install:
-  - gem update --system
-  - gem install bundler
+- pip install --upgrade --user awscli
+- gem update --system
+- gem install bundler
 before_script:
-  - "mysql -e 'create database thinking_sphinx;' > /dev/null"
-  - "psql -c 'create database thinking_sphinx;' -U postgres >/dev/null"
-  - bundle exec appraisal install
+- mysql -e 'create database thinking_sphinx;' > /dev/null
+- psql -c 'create database thinking_sphinx;' -U postgres >/dev/null
+- "./bin/loadsphinx $SPHINX_VERSION"
+- "./bin/literals"
+- bundle exec appraisal install
 script: bundle exec appraisal rspec
 env:
-  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.0.10/bin/ SPHINX_VERSION=2.0.10
-  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.0.10/bin/ SPHINX_VERSION=2.0.10
-  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.1.9/bin/ SPHINX_VERSION=2.1.9
-  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.1.9/bin/ SPHINX_VERSION=2.1.9
-  - DATABASE=mysql2 SPHINX_BIN=/usr/local/sphinx-2.2.6/bin/ SPHINX_VERSION=2.2.6
-  - DATABASE=postgresql SPHINX_BIN=/usr/local/sphinx-2.2.6/bin/ SPHINX_VERSION=2.2.6
+  global:
+  - SPHINX_BIN=ext/sphinx/bin/
+  - secure: cUPinkilBafqDSPsTkl/PXYc2aXNKUQKXGK8poBBMqKN9/wjfJx1DWgtowDKalekdZELxDhc85Ye3bL1xlW4nLjOu+U6Tkt8eNw2Nhs1flodHzA/RyENdBLr/tBHt43EjkrDehZx5sBHmWQY4miHs8AJz0oKO9Ae2inTOHx9Iuc=
+  matrix:
+  - DATABASE=mysql2 SPHINX_VERSION=2.0.10
+  - DATABASE=postgresql SPHINX_VERSION=2.0.10
+  - DATABASE=mysql2 SPHINX_VERSION=2.1.9
+  - DATABASE=postgresql SPHINX_VERSION=2.1.9
+  - DATABASE=mysql2 SPHINX_VERSION=2.2.6
+  - DATABASE=postgresql SPHINX_VERSION=2.2.6
 sudo: false
 addons:
-  postgresql: "9.4"
+  postgresql: '9.4'
 services:
-  - postgresql
+- postgresql
 matrix:
-  allow_failures:
-  - rvm: jruby-9.0.5.0
+  exclude:
+  - rvm: jruby-9.1.10.0
+    env: DATABASE=mysql2 SPHINX_VERSION=2.0.10
+  - rvm: jruby-9.1.10.0
+    env: DATABASE=postgresql SPHINX_VERSION=2.0.10

data/Appraisals

@@ -1,28 +1,32 @@
 appraise 'rails_3_2' do
   gem 'rails',  '~> 3.2.22.2'
   gem 'rack',   '~> 1.0', :platforms => [:ruby_20, :ruby_21]
-end
+end if RUBY_VERSION.to_f <= 2.3
 
 appraise 'rails_4_0' do
   gem 'rails', '~> 4.0.13'
   gem 'rack',   '~> 1.0', :platforms => [:ruby_20, :ruby_21]
-end
+end if RUBY_VERSION.to_f <= 2.3
 
 appraise 'rails_4_1' do
   gem 'rails', '~> 4.1.15'
   gem 'rack',  '~> 1.0', :platforms => [:ruby_20, :ruby_21]
-end
+end if RUBY_VERSION.to_f <= 2.3
 
 appraise 'rails_4_2' do
   gem 'rails', '~> 4.2.6'
   gem 'rack',  '~> 1.0', :platforms => [:ruby_20, :ruby_21]
-end
+end if RUBY_VERSION.to_f <= 2.3
 
 appraise 'rails_5_0' do
-  gem 'rails', '~> 5.0.0'
+  gem 'rails', '~> 5.0.2'
   # gem 'activerecord-jdbc-adapter',
   #   :git      => 'git://github.com/jruby/activerecord-jdbc-adapter.git',
   #   :branch   => 'rails-5',
   #   :platform => :jruby,
   #   :ref      => 'c3570ce730'
 end if RUBY_VERSION.to_f >= 2.2 && RUBY_PLATFORM != 'java'
+
+appraise 'rails_5_1' do
+  gem 'rails', '~> 5.1.0'
+end if RUBY_VERSION.to_f >= 2.2 && RUBY_PLATFORM != 'java'

data/Gemfile

@@ -6,7 +6,12 @@ gem 'mysql2', '~> 0.3.12b4', :platform => :ruby
 gem 'pg',     '~> 0.18.4',   :platform => :ruby
 
 gem 'jdbc-mysql',                          '5.1.35',   :platform => :jruby
-gem 'activerecord-jdbcmysql-adapter',      '~> 1.3.4', :platform => :jruby
-gem 'activerecord-jdbcpostgresql-adapter', '~> 1.3.4', :platform => :jruby
+gem 'activerecord-jdbcmysql-adapter',      '~> 1.3.23', :platform => :jruby
+gem 'activerecord-jdbcpostgresql-adapter', '~> 1.3.23', :platform => :jruby
 
-gem 'rack', '~> 1.0' if RUBY_VERSION.to_f <= 2.1
+if RUBY_VERSION.to_f <= 2.1
+  gem 'rack', '~> 1.0'
+  gem 'nokogiri', '1.6.8'
+end
+
+gem 'activerecord', '>= 3.2.22' if RUBY_PLATFORM == 'java'

data/HISTORY

@@ -1,3 +1,27 @@
+2017-08-28: 3.4.0
+* [CHANGE] Delta callback logic now prioritises checking for high level settings rather than model changes.
+* [FIX] Index normalisation now occurs consistently, and removes unneccesary sphinx_internal_class_name fields from real-time indices.
+* [FEATURE] Rake tasks are now unified, so the original tasks will operate on real-time indices as well.
+* [FEATURE] Output warnings when unknown options are used in search calls.
+* [CHANGE] Allow for unsaved records when calculating document ids (and return nil).
+* [CHANGE] Display SphinxQL deletion statements in the log.
+* [CHANGE] Add support for Ruby's frozen string literals feature.
+* [FIX] Fix Sphinx connections in JRuby.
+* [FIX] Fix long SphinxQL query handling in JRuby.
+* [FEATURE] Allow generation of a single real-time index (Tim Brown).
+* [FIX] Always close the SphinxQL connection if Innertube's asking (@cmaion).
+* [CHANGE] Use saved_changes if it's available (in Rails 5.1+).
+* [FEATURE] Automatically use UTF8 in Sphinx for encodings that are extensions of UTF8.
+* [FIX] Get bigint primary keys working in Rails 5.1.
+* [CHANGE] Set a default connection timeout of 5 seconds.
+* [FIX] Fix handling of attached starts of Sphinx (via Henne Vogelsang).
+* [FIX] Fix multi-field conditions.
+* [FEATURE] Basic type checking for attribute filters.
+* [CHANGE] Don't search multi-table inheritance ancestors.
+* [FIX] Use the base class of STI models for polymorphic join generation (via Andrés Cirugeda).
+* [CHANGE] Handle non-computable queries as parse errors.
+* [FIX] Ensure ts:index now respects rake silent/quiet flags.
+
 2016-12-13: 3.3.0
 * [FEATURE] Real-time callbacks can now be used with after_commit hooks if that's preferred over after_save.
 * [CHANGE] Only toggle the delta value if the record has changed or is new (rather than on every single save call).

data/README.textile

@@ -1,11 +1,12 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v3.3.0.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v3.4.0.
 
 h2. Upgrading
 
 Please refer to the release notes for any changes you need to make when upgrading:
 
+* "v3.4.0":https://github.com/pat/thinking-sphinx/releases/tag/v3.4.0
 * "v3.3.0":https://github.com/pat/thinking-sphinx/releases/tag/v3.3.0
 * "v3.2.0":https://github.com/pat/thinking-sphinx/releases/tag/v3.2.0
 * "v3.1.4":https://github.com/pat/thinking-sphinx/releases/tag/v3.1.4
@@ -21,9 +22,9 @@ h2. Installation
 
 It's a gem, so install it like you would any other gem. You will also need to specify the mysql2 gem if you're using MRI, or jdbc-mysql if you're using JRuby:
 
-<pre><code>gem 'mysql2',          '~> 0.3.18', :platform => :ruby
+<pre><code>gem 'mysql2',          '~> 0.3', :platform => :ruby
 gem 'jdbc-mysql',      '= 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 3.3.0'</code></pre>
+gem 'thinking-sphinx', '~> 3.4.0'</code></pre>
 
 The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database. If you're using JRuby, there is "currently an issue with Sphinx and jdbc-mysql 5.1.36 or newer":http://sphinxsearch.com/forum/view.html?id=13939, so you'll need to stick to nothing more recent than 5.1.35.
 
@@ -84,4 +85,4 @@ You can then run the unit tests with @rake spec:unit@, the acceptance tests with
 
 h2. Licence
 
-Copyright (c) 2007-2015, Thinking Sphinx is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/thinking-sphinx/contributors.
+Copyright (c) 2007-2017, Thinking Sphinx is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/thinking-sphinx/contributors.

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "thinking_sphinx"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/literals

@@ -0,0 +1,9 @@
+if (ruby -e "exit RUBY_VERSION.to_f >= 2.4")
+then
+  echo "Automatic frozen string literals are supported"
+  gem install pragmater -v 4.0.0
+  pragmater --add lib --comments "# frozen_string_literal: true" --whitelist "**/*.rb"
+  pragmater --add spec --comments "# frozen_string_literal: true" --whitelist "**/*.rb"
+else
+  echo "Automatic frozen string literals are not supported."
+fi

data/bin/loadsphinx

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+version=$1
+name="sphinx-$version"
+url="http://sphinxsearch.com/files/$name-release.tar.gz"
+bucket="thinking-sphinx"
+directory="ext/sphinx"
+prefix="`pwd`/$directory"
+file="ext/$name.tar.gz"
+
+download_and_compile_source () {
+  curl -O $url
+  tar -zxf $name-release.tar.gz
+  cd $name-release
+  ./configure --with-mysql --with-pgsql --enable-id64 --prefix=$prefix
+  make
+  make install
+  cd ..
+  rm -rf $name-release.tar.gz $name-release
+}
+
+load_cache () {
+  aws s3 cp s3://$bucket/bincaches/$name.tar.gz $file
+  tar -zxf $file
+}
+
+push_cache () {
+  tar -czf $file $directory
+  aws s3 cp $file s3://$bucket/bincaches/$name.tar.gz --acl public-read
+}
+
+if aws s3api head-object --bucket $bucket --key bincaches/$name.tar.gz
+then
+  load_cache
+else
+  download_and_compile_source
+  push_cache
+fi

data/lib/thinking_sphinx.rb

@@ -15,7 +15,7 @@ require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/module/attribute_accessors'
 
 module ThinkingSphinx
-  MAXIMUM_STATEMENT_LENGTH = (2 ** 23) - 1
+  MAXIMUM_STATEMENT_LENGTH = (2 ** 23) - 5
 
   def self.count(query = '', options = {})
     search_for_ids(query, options).total_entries
@@ -40,14 +40,24 @@ module ThinkingSphinx
 
   @before_index_hooks = []
 
-  module Subscribers; end
+  module Commands; end
   module IndexingStrategies; end
+  module Interfaces; end
+  module Subscribers; end
 end
 
 # Core
+require 'thinking_sphinx/attribute_types'
 require 'thinking_sphinx/batched_search'
 require 'thinking_sphinx/callbacks'
 require 'thinking_sphinx/core'
+require 'thinking_sphinx/with_output'
+require 'thinking_sphinx/commands/base'
+require 'thinking_sphinx/commands/configure'
+require 'thinking_sphinx/commands/index'
+require 'thinking_sphinx/commands/start_attached'
+require 'thinking_sphinx/commands/start_detached'
+require 'thinking_sphinx/commands/stop'
 require 'thinking_sphinx/configuration'
 require 'thinking_sphinx/connection'
 require 'thinking_sphinx/controller'
@@ -63,6 +73,9 @@ require 'thinking_sphinx/index'
 require 'thinking_sphinx/indexing_strategies/all_at_once'
 require 'thinking_sphinx/indexing_strategies/one_at_a_time'
 require 'thinking_sphinx/index_set'
+require 'thinking_sphinx/interfaces/daemon'
+require 'thinking_sphinx/interfaces/real_time'
+require 'thinking_sphinx/interfaces/sql'
 require 'thinking_sphinx/masks'
 require 'thinking_sphinx/middlewares'
 require 'thinking_sphinx/panes'

data/lib/thinking_sphinx/active_record/callbacks/delta_callbacks.rb

@@ -16,9 +16,8 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks <
   end
 
   def before_save
-    return unless new_or_changed?           &&
-      !ThinkingSphinx::Callbacks.suspended? &&
-      delta_indices?
+    return unless !ThinkingSphinx::Callbacks.suspended? && delta_indices? &&
+      new_or_changed?
 
     processors.each { |processor| processor.toggle instance }
   end

data/lib/thinking_sphinx/active_record/callbacks/update_callbacks.rb

@@ -1,6 +1,12 @@
 class ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks <
   ThinkingSphinx::Callbacks
 
+  if ActiveRecord::Base.instance_methods.grep(/saved_changes/).any?
+    CHANGED_ATTRIBUTES = lambda { |instance| instance.saved_changes.keys }
+  else
+    CHANGED_ATTRIBUTES = lambda { |instance| instance.changed }
+  end
+
   callbacks :after_update
 
   def after_update
@@ -15,7 +21,7 @@ class ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks <
 
   def attributes_hash_for(index)
     updateable_attributes_for(index).inject({}) do |hash, attribute|
-      if instance.changed.include?(attribute.columns.first.__name.to_s)
+      if changed_attributes.include?(attribute.columns.first.__name.to_s)
         hash[attribute.name] = attribute.value_for(instance)
       end
 
@@ -23,6 +29,10 @@ class ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks <
     end
   end
 
+  def changed_attributes
+    @changed_attributes ||= CHANGED_ATTRIBUTES.call instance
+  end
+
   def configuration
     ThinkingSphinx::Configuration.instance
   end

data/lib/thinking_sphinx/active_record/index.rb

@@ -63,7 +63,7 @@ class ThinkingSphinx::ActiveRecord::Index < Riddle::Configuration::Index
       :delta?          => @options[:delta?],
       :delta_processor => @options[:delta_processor],
       :delta_options   => @options[:delta_options],
-      :primary_key     => @options[:primary_key] || model.primary_key || :id
+      :primary_key     => primary_key
     }
   end
 end

data/lib/thinking_sphinx/active_record/join_association.rb

@@ -5,7 +5,9 @@ class ThinkingSphinx::ActiveRecord::JoinAssociation <
     constraint = super
 
     constraint = constraint.and(
-      foreign_table[reflection.options[:foreign_type]].eq(base_klass.name)
+      foreign_table[reflection.options[:foreign_type]].eq(
+        base_klass.base_class.name
+      )
     ) if reflection.options[:sphinx_internal_filtered]
 
     constraint

data/lib/thinking_sphinx/active_record/log_subscriber.rb

@@ -13,6 +13,11 @@ class ThinkingSphinx::ActiveRecord::LogSubscriber < ActiveSupport::LogSubscriber
     identifier = color('Sphinx Query (%.1fms)' % event.duration, GREEN, true)
     debug "  #{identifier}  #{event.payload[:query]}"
   end
+
+  def warn(event)
+    identifier = color 'Sphinx', GREEN, true
+    warn "  #{identifier}  #{event.payload[:guard]}"
+  end
 end
 
 ThinkingSphinx::ActiveRecord::LogSubscriber.attach_to :thinking_sphinx

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -14,7 +14,7 @@ module ThinkingSphinx
         @model             = model
         @database_settings = model.connection.instance_variable_get(:@config).clone
         @options           = {
-          :utf8? => (@database_settings[:encoding] == 'utf8')
+          :utf8? => (@database_settings[:encoding].to_s[/^utf8/])
         }.merge options
 
         @fields            = []

data/lib/thinking_sphinx/attribute_types.rb

@@ -0,0 +1,70 @@
+class ThinkingSphinx::AttributeTypes
+  def self.call
+    @call ||= new.call
+  end
+
+  def self.reset
+    @call = nil
+  end
+
+  def call
+    return {} unless File.exist?(configuration_file)
+
+    realtime_indices.each { |index|
+      map_types_with_prefix index, :rt,
+        [:uint, :bigint, :float, :timestamp, :string, :bool, :json]
+
+      index.rt_attr_multi.each     { |name| attributes[name] << :uint }
+      index.rt_attr_multi_64.each  { |name| attributes[name] << :bigint }
+    }
+
+    plain_sources.each { |source|
+      map_types_with_prefix source, :sql,
+        [:uint, :bigint, :float, :timestamp, :string, :bool, :json]
+
+      source.sql_attr_str2ordinal    { |name| attributes[name] << :uint }
+      source.sql_attr_str2wordcount  { |name| attributes[name] << :uint }
+      source.sql_attr_multi.each { |setting|
+        type, name, *ignored = setting.split(/\s+/)
+        attributes[name] << type.to_sym
+      }
+    }
+
+    attributes.values.each &:uniq!
+    attributes
+  end
+
+  private
+
+  def attributes
+    @attributes ||= Hash.new { |hash, key| hash[key] = [] }
+  end
+
+  def configuration
+    @configuration ||= Riddle::Configuration.parse!(
+      File.read(configuration_file)
+    )
+  end
+
+  def configuration_file
+    ThinkingSphinx::Configuration.instance.configuration_file
+  end
+
+  def map_types_with_prefix(object, prefix, types)
+    types.each do |type|
+      object.public_send("#{prefix}_attr_#{type}").each do |name|
+        attributes[name] << type
+      end
+    end
+  end
+
+  def plain_sources
+    configuration.indices.select { |index|
+      index.type == 'plain' || index.type.nil?
+    }.collect(&:sources).flatten
+  end
+
+  def realtime_indices
+    configuration.indices.select { |index| index.type == 'rt' }
+  end
+end