thinking-sphinx 3.0.5 → 3.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66940af3da082d3b4d69667b40b1b6495f226dd1
-  data.tar.gz: 52af0d5ae485c7caec6fab3206b28c34764ab1a5
+  metadata.gz: 7e8abde6121afcf9bad7eb33647ed1083d0f8bea
+  data.tar.gz: 2a372d2529cb95c232a5fc2d500aa9a5798eebd8
 SHA512:
-  metadata.gz: eb3099e4ef3987b78b20003ba8c7ebaa13846fae26cdb6c384f47962d584dd144e511d8dcaa81f9900a9d18d3020edc489a1fde377a1816ec0337ba6b3da0ce8
-  data.tar.gz: b43d0cb3be98bd7d738f4b8bf18492832e496a7c441368fb250ef8ecbfb4d13a5ebb66c6a14ce57b74d44cbcc6131b96f2a1581342852b7d576ccb6ffd75743c
+  metadata.gz: 2c22cc575eb577957271f00a0b04034871051634ddf7324132587be852ea35b995ad3e4f220dca5765d93766f69864c6d1df1a8379520fd9bc3637c239f04e28
+  data.tar.gz: c253ba339cf5dfc7903d6b98ebf63e77726194ab481d79ec009d3830f072598d6a65fee1efbc07dfa4ed6f6e561cef424883cbf834dbb76d0bf12dd5322f6214

data/.gitignore

@@ -6,4 +6,7 @@ Gemfile.lock
 pkg/*
 spec/internal/config/test.sphinx.conf
 spec/internal/db/sphinx
+!spec/internal/tmp/.gitkeep
+spec/internal/tmp/*
+!spec/internal/tmp/.gitkeep
 _site

data/HISTORY

@@ -1,3 +1,28 @@
+2013-10-20: 3.0.6
+* [FEATURE] Raise an error if no indices match the search criteria (Bryan Ricker).
+* [FEATURE] skip_time_zone setting is now available per environment via config/thinking_sphinx.yml to avoid the sql_query_pre time zone command.
+* [CHANGE] Updating Riddle dependency to be >= 1.5.9.
+* [FEATURE] Added new search options in Sphinx 2.1.x.
+* [FEATURE] Added ability to disable UTF-8 forced encoding, now that Sphinx 2.1.2 returns UTF-8 strings by default. This will be disabled by default in Thinking Sphinx 3.1.0.
+* [FEATURE] Added ability to switch between Sphinx special variables and the equivalent functions. Sphinx 2.1.x requires the latter, and that behaviour will become the default in Sphinx 3.1.0.
+* [FIX] Cast every column to a timestamp for timestamp attributes with multiple columns.
+* [CHANGE] Separated directory preparation from data generation for real-time index (re)generation tasks.
+* [CHANGE] Have tests index UTF-8 characters where appropriate (Pedro Cunha).
+* [FIX] Don't use Sphinx ordering if SQL order option is supplied to a search.
+* [CHANGE] Always use DISTINCT in group concatenation.
+* [CHANGE] Sphinx connection failures now have their own class, ThinkingSphinx::ConnectionError, instead of the standard Mysql2::Error.
+* [FIX] Custom middleware and mask options now function correctly with model-scoped searches.
+* [FEATURE] Adding search_for_ids on scoped search calls.
+* [CHANGE] Don't clobber custom :select options for facet searches (Timo Virkkala).
+* [CHANGE] Automatically load Riddle's Sphinx 2.0.5 compatability changes.
+* [FIX] Suspended deltas now no longer update core indices as well.
+* [CHANGE] Realtime fields and attributes now accept symbols as well as column objects, and fields can be sortable (with a _sort prefix for the matching attribute).
+* [FEATURE] MySQL users can enable a minimal GROUP BY statement, to speed up queries: set_property :minimal_group_by? => true.
+* [CHANGE] Insist on the log directory existing, to ensure correct behaviour for symlinked paths. (Michael Pearson).
+* [FIX] Use alphabetical ordering for index paths consistently (@grin).
+* [FIX] Convert very small floats to fixed format for geo-searches.
+* [CHANGE] Rake's silent mode is respected for indexing (@endoscient).
+
 2013-08-26: 3.0.5
 * [CHANGE] Updating Riddle dependency to be >= 1.5.8.
 * [FEATURE] Allow scoping of real-time index models.

data/README.textile

@@ -7,7 +7,7 @@ h2. Installation
 It's a gem, so install it like you would any other gem. You will also need to specify the Mysql2 gem as well (this is not an inbuilt dependency because JRuby, when supported, will need something different):
 
 <pre><code>gem 'mysql2',          '0.3.13'
-gem 'thinking-sphinx', '3.0.5'</code></pre>
+gem 'thinking-sphinx', '3.0.6'</code></pre>
 
 The mysql2 gem is required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database.
 
@@ -216,7 +216,7 @@ May or may not be added:
 
 h3. Sphinx Versions
 
-TS 3 is built for Sphinx 2.x only. You cannot use 1.10-beta, 0.9.9 or anything earlier than that. 2.0.5 or newer is recommended.
+TS 3 is built for Sphinx 2.0.5 or newer. You cannot use 1.10-beta, 0.9.9 or anything earlier than that.
 
 h3. Rails Versions
 

data/lib/thinking_sphinx.rb

@@ -6,10 +6,12 @@ else
 end
 
 require 'riddle'
+require 'riddle/2.1.0'
 require 'middleware'
 require 'active_record'
 require 'innertube'
 require 'active_support/core_ext/module/delegation'
+require 'active_support/core_ext/module/attribute_accessors'
 
 module ThinkingSphinx
   def self.count(query = '', options = {})
@@ -49,6 +51,7 @@ require 'thinking_sphinx/errors'
 require 'thinking_sphinx/excerpter'
 require 'thinking_sphinx/facet'
 require 'thinking_sphinx/facet_search'
+require 'thinking_sphinx/float_formatter'
 require 'thinking_sphinx/frameworks'
 require 'thinking_sphinx/index'
 require 'thinking_sphinx/index_set'
@@ -58,8 +61,10 @@ require 'thinking_sphinx/panes'
 require 'thinking_sphinx/rake_interface'
 require 'thinking_sphinx/scopes'
 require 'thinking_sphinx/search'
+require 'thinking_sphinx/sphinxql'
 require 'thinking_sphinx/subscribers/populator_subscriber'
 require 'thinking_sphinx/test'
+require 'thinking_sphinx/utf8'
 # Extended
 require 'thinking_sphinx/active_record'
 require 'thinking_sphinx/deltas'

data/lib/thinking_sphinx/active_record/callbacks/update_callbacks.rb

@@ -45,7 +45,7 @@ class ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks <
     ThinkingSphinx::Connection.take do |connection|
       connection.execute(sphinxql)
     end
-  rescue Mysql2::Error => error
+  rescue ThinkingSphinx::ConnectionError => error
     # This isn't vital, so don't raise the error.
   end
 

data/lib/thinking_sphinx/active_record/database_adapters/mysql_adapter.rb

@@ -22,7 +22,7 @@ class ThinkingSphinx::ActiveRecord::DatabaseAdapters::MySQLAdapter <
   end
 
   def group_concatenate(clause, separator = ' ')
-    "GROUP_CONCAT(#{clause} SEPARATOR '#{separator}')"
+    "GROUP_CONCAT(DISTINCT #{clause} SEPARATOR '#{separator}')"
   end
 
   def time_zone_query_pre

data/lib/thinking_sphinx/active_record/database_adapters/postgresql_adapter.rb

@@ -28,7 +28,7 @@ class ThinkingSphinx::ActiveRecord::DatabaseAdapters::PostgreSQLAdapter <
   end
 
   def group_concatenate(clause, separator = ' ')
-    "array_to_string(array_agg(#{clause}), '#{separator}')"
+    "array_to_string(array_agg(DISTINCT #{clause}), '#{separator}')"
   end
 
   def time_zone_query_pre

data/lib/thinking_sphinx/active_record/property_sql_presenter.rb

@@ -19,6 +19,8 @@ class ThinkingSphinx::ActiveRecord::PropertySQLPresenter
 
   private
 
+  delegate :multi?, :to => :property
+
   def aggregate?
     property.columns.any? { |column|
       associations.aggregate_for?(column.__stack)
@@ -26,12 +28,18 @@ class ThinkingSphinx::ActiveRecord::PropertySQLPresenter
   end
 
   def aggregate_separator
-    (property.multi?) ? ',' : ' '
+    multi? ? ',' : ' '
+  end
+
+  def cast_to_timestamp(clause)
+    clause.split(', ').collect { |part|
+      adapter.cast_to_timestamp part
+    }.join(', ')
   end
 
   def casted_column_with_table
     clause = columns_with_table
-    clause = adapter.cast_to_timestamp(clause) if property.type == :timestamp
+    clause = cast_to_timestamp clause if property.type == :timestamp
     clause = concatenate clause
     if aggregate?
       clause = adapter.group_concatenate(clause, aggregate_separator)

data/lib/thinking_sphinx/active_record/sql_builder/clause_builder.rb

@@ -2,27 +2,26 @@ module ThinkingSphinx
   module ActiveRecord
     class SQLBuilder::ClauseBuilder
       def initialize(first_element)
-        @first_element = first_element
+        @clauses = [first_element]
       end
 
       def compose(*additions)
         additions.each &method(:add_clause)
+
         self
       end
 
       def add_clause(clause)
-        self.clauses += Array(clause)
+        @clauses += Array(clause)
       end
 
       def separated(by = ', ')
         clauses.flatten.compact.join(by)
       end
 
-      protected
-      attr_accessor :clauses
-      def clauses
-        @clauses ||= [@first_element]
-      end
+      private
+
+      attr_reader :clauses
     end
   end
 end

data/lib/thinking_sphinx/active_record/sql_builder/query.rb

@@ -24,16 +24,20 @@ module ThinkingSphinx
       end
 
       def scope_by_delta_processor
-        self.scope << delta_processor.reset_query if delta_processor && !source.delta?
+        return unless delta_processor && !source.delta?
+
+        self.scope << delta_processor.reset_query
       end
 
       def scope_by_session
-        if max_len = source.options[:group_concat_max_len]
-          self.scope << "SET SESSION group_concat_max_len = #{max_len}"
-        end
+        return unless max_len = source.options[:group_concat_max_len]
+
+        self.scope << "SET SESSION group_concat_max_len = #{max_len}"
       end
 
       def scope_by_time_zone
+        return if config.settings['skip_time_zone']
+
         self.scope += time_zone_query_pre
       end
 

data/lib/thinking_sphinx/active_record/sql_builder/statement.rb

@@ -129,11 +129,14 @@ module ThinkingSphinx
       end
 
       def group_clause
-        SQLBuilder::ClauseBuilder.new(quoted_primary_key).compose(
+        builder = SQLBuilder::ClauseBuilder.new(quoted_primary_key)
+
+        builder.compose(
           presenters_to_group(field_presenters),
-          presenters_to_group(attribute_presenters),
-          groupings
-        ).separated
+          presenters_to_group(attribute_presenters)
+        ) unless source.options[:minimal_group_by?]
+
+        builder.compose(groupings).separated
       end
     end
   end

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -7,7 +7,7 @@ module ThinkingSphinx
         :groupings, :polymorphs
 
       OPTIONS = [:name, :offset, :delta_processor, :delta?, :disable_range?,
-        :group_concat_max_len, :utf8?, :position]
+        :group_concat_max_len, :utf8?, :position, :minimal_group_by?]
 
       def initialize(model, options = {})
         @model             = model

data/lib/thinking_sphinx/configuration.rb

@@ -68,7 +68,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
     return if @preloaded_indices
 
     index_paths.each do |path|
-      Dir["#{path}/**/*.rb"].each do |file|
+      Dir["#{path}/**/*.rb"].sort.each do |file|
         ActiveSupport::Dependencies.require_or_load file
       end
     end
@@ -100,7 +100,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
   def configure_searchd
     configure_searchd_log_files
 
-    searchd.binlog_path = framework_root.join('tmp', 'binlog',environment).to_s
+    searchd.binlog_path = tmp_path.join('binlog', environment).to_s
     searchd.address = settings['address'].presence || Defaults::ADDRESS
     searchd.mysql41 = settings['mysql41'] || settings['port'] || Defaults::PORT
     searchd.workers = 'threads'
@@ -113,7 +113,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
   end
 
   def log_root
-    framework_root.join('log')
+    framework_root.join('log').realpath
   end
 
   def framework_root
@@ -147,6 +147,11 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
     @offsets = {}
   end
 
+  def tmp_path
+    path = framework_root.join('tmp')
+    File.exists?(path) ? path.realpath : path
+  end
+
   def apply_sphinx_settings!
     [indexer, searchd].each do |object|
       settings.each do |key, value|

data/lib/thinking_sphinx/connection.rb

@@ -62,6 +62,8 @@ module ThinkingSphinx::Connection
         :port  => port,
         :flags => Mysql2::Client::MULTI_STATEMENTS
       }.merge(options))
+    rescue Mysql2::Error => error
+      raise ThinkingSphinx::SphinxError.new_from_mysql error
     end
 
     def close

data/lib/thinking_sphinx/deletion.rb

@@ -6,7 +6,7 @@ class ThinkingSphinx::Deletion
       'plain' => PlainDeletion,
       'rt'    => RealtimeDeletion
     }[index.type].new(index, instance).perform
-  rescue Mysql2::Error => error
+  rescue ThinkingSphinx::ConnectionError => error
     # This isn't vital, so don't raise the error.
   end
 

data/lib/thinking_sphinx/deltas.rb

@@ -26,7 +26,7 @@ module ThinkingSphinx::Deltas
     resume!
 
     config.indices_for_references(reference).each do |index|
-      index.delta_processor.index index
+      index.delta_processor.index index if index.delta?
     end
   end
 

data/lib/thinking_sphinx/deltas/delete_job.rb

@@ -9,7 +9,7 @@ class ThinkingSphinx::Deltas::DeleteJob
         @index_name, @document_id, :sphinx_deleted => true
       )
     end
-  rescue Mysql2::Error => error
+  rescue ThinkingSphinx::ConnectionError => error
     # This isn't vital, so don't raise the error.
   end
 end

data/lib/thinking_sphinx/errors.rb

@@ -9,6 +9,8 @@ class ThinkingSphinx::SphinxError < StandardError
       replacement = ThinkingSphinx::SyntaxError.new(error.message)
     when /query error/
       replacement = ThinkingSphinx::QueryError.new(error.message)
+    when /Can't connect to MySQL server/
+      replacement = ThinkingSphinx::ConnectionError.new(error.message)
     else
       replacement = new(error.message)
     end
@@ -19,6 +21,9 @@ class ThinkingSphinx::SphinxError < StandardError
   end
 end
 
+class ThinkingSphinx::ConnectionError < ThinkingSphinx::SphinxError
+end
+
 class ThinkingSphinx::QueryError < ThinkingSphinx::SphinxError
 end
 
@@ -34,3 +39,6 @@ end
 
 class ThinkingSphinx::MixedScopesError < StandardError
 end
+
+class ThinkingSphinx::NoIndicesError < StandardError
+end

data/lib/thinking_sphinx/excerpter.rb

@@ -18,8 +18,8 @@ class ThinkingSphinx::Excerpter
       connection.query(statement_for(text)).first['snippet']
     end
 
-    result.encode!("ISO-8859-1")
-    result.force_encoding("UTF-8")
+    ThinkingSphinx::Configuration.instance.settings['utf8'] ? result :
+      ThinkingSphinx::UTF8.encode(result)
   end
 
   private

data/lib/thinking_sphinx/facet.rb

@@ -11,7 +11,7 @@ class ThinkingSphinx::Facet
 
   def results_from(raw)
     raw.inject({}) { |hash, row|
-      hash[row[group_column]] = row['@count']
+      hash[row[group_column]] = row[ThinkingSphinx::SphinxQL.count]
       hash
     }
   end
@@ -19,7 +19,7 @@ class ThinkingSphinx::Facet
   private
 
   def group_column
-    @properties.any?(&:multi?) ? '@groupby' : name
+    @properties.any?(&:multi?) ? ThinkingSphinx::SphinxQL.group_by : name
   end
 
   def use_field?

data/lib/thinking_sphinx/facet_search.rb

@@ -101,7 +101,8 @@ class ThinkingSphinx::FacetSearch
 
   def options_for(facet)
     options.merge(
-      :select      => '*, @groupby, @count',
+      :select      => (options[:select] || '*') +
+        ", #{ThinkingSphinx::SphinxQL.group_by}, #{ThinkingSphinx::SphinxQL.count}",
       :group_by    => facet.name,
       :indices     => index_names_for(facet),
       :max_matches => limit,

data/lib/thinking_sphinx/float_formatter.rb

@@ -0,0 +1,33 @@
+class ThinkingSphinx::FloatFormatter
+  PATTERN = /(\d+)e\-(\d+)$/
+
+  def initialize(float)
+    @float = float
+  end
+
+  def fixed
+    return float.to_s unless exponent_present?
+
+    ("%0.#{decimal_places}f" % float).gsub(/0+$/, '')
+  end
+
+  private
+
+  attr_reader :float
+
+  def exponent_decimal_places
+    float.to_s[PATTERN, 1].length
+  end
+
+  def exponent_factor
+    float.to_s[PATTERN, 2].to_i
+  end
+
+  def exponent_present?
+    float.to_s['e']
+  end
+
+  def decimal_places
+    exponent_factor + exponent_decimal_places
+  end
+end

data/lib/thinking_sphinx/index_set.rb

@@ -1,6 +1,8 @@
 class ThinkingSphinx::IndexSet
   include Enumerable
 
+  delegate :each, :empty?, :to => :indices
+
   def initialize(classes, index_names, configuration = nil)
     @classes       = classes || []
     @index_names   = index_names
@@ -11,10 +13,6 @@ class ThinkingSphinx::IndexSet
     classes_and_ancestors - classes
   end
 
-  def each(&block)
-    indices.each { |index| yield index }
-  end
-
   def to_a
     indices
   end