thinking-sphinx 3.0.4 → 3.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e4abfc7a0ce650e92e51a842336ec1028f12e9ea
-  data.tar.gz: 2c51520fcd002486f51e127f981f5e7c60000706
+  metadata.gz: 66940af3da082d3b4d69667b40b1b6495f226dd1
+  data.tar.gz: 52af0d5ae485c7caec6fab3206b28c34764ab1a5
 SHA512:
-  metadata.gz: 384ba739298c2dc12431cf9c74e79a3f15191098d84b41afab0e94383ac9403384e9290e38c834ef17a1675eedf5835477f63c12aeeb440144421dab97e82959
-  data.tar.gz: 787f2a640087f91d7ffd7d39d1b092fb027551e6df58d2ab19062a78bfebe3b7bbbfe8ad4c65858d84fc590403eb0515296c6a5192fc3aaf6937e2f595b1ec3a
+  metadata.gz: eb3099e4ef3987b78b20003ba8c7ebaa13846fae26cdb6c384f47962d584dd144e511d8dcaa81f9900a9d18d3020edc489a1fde377a1816ec0337ba6b3da0ce8
+  data.tar.gz: b43d0cb3be98bd7d738f4b8bf18492832e496a7c441368fb250ef8ecbfb4d13a5ebb66c6a14ce57b74d44cbcc6131b96f2a1581342852b7d576ccb6ffd75743c

data/HISTORY

@@ -1,3 +1,16 @@
+2013-08-26: 3.0.5
+* [CHANGE] Updating Riddle dependency to be >= 1.5.8.
+* [FEATURE] Allow scoping of real-time index models.
+* [CHANGE] Real-time index population presentation and logic are now separated.
+* [CHANGE] Using the connection pool for update callbacks, excerpts, deletions.
+* [FIX] Respect existing sql_query_range/sql_query_info settings.
+* [CHANGE] Don't add the sphinx_internal_class_name unless STI models are indexed.
+* [FIX] Don't add select clauses or joins to sql_query if they're for query/ranged-query properties.
+* [CHANGE] Use Mysql2's reconnect option and have it turned on by default.
+* [FIX] Set database timezones as part of the indexing process.
+* [CHANGE] Improved auto-starring with escaped characters.
+* [FIX] Chaining scopes with just options works again.
+
 2013-07-09: 3.0.4
 * [CHANGE] Updating Riddle dependency to be >= 1.5.7.
 * [FEATURE] ts:regenerate rake task for rebuilding Sphinx when realtime indices are involved.

data/README.textile

@@ -6,8 +6,8 @@ h2. Installation
 
 It's a gem, so install it like you would any other gem. You will also need to specify the Mysql2 gem as well (this is not an inbuilt dependency because JRuby, when supported, will need something different):
 
-<pre><code>gem 'mysql2',          '0.3.12b4'
-gem 'thinking-sphinx', '3.0.0'</code></pre>
+<pre><code>gem 'mysql2',          '0.3.13'
+gem 'thinking-sphinx', '3.0.5'</code></pre>
 
 The mysql2 gem is required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database.
 
@@ -39,6 +39,8 @@ Other changes:
 * SphinxQL is now used instead of the old socket connections (hence the dependency on the @mysql2@ gem).
 * Specifying a different port for Sphinx to use (in @config/thinking_sphinx.yml@) should be done with the mysql41 setting, not the port setting.
 * The searchd_log and searchd_query_log settings are now log and query_log (matching their Sphinx names).
+* searchd_file_path is now indices_location.
+* config_file is now configuration_file.
 * You'll need to include @ThinkingSphinx::Scopes@ into your models if you want to use Sphinx scopes. Default scopes can be set as follows:
 
 <pre><code>class Person < ActiveRecord::Base
@@ -130,7 +132,7 @@ batch.searches #=> [[foo results], [bar results], [baz results]]</code></pre>
 * To search on specific indices, use the @:indices@ option, which expects an array of index names (including the @_core@ or @_delta@ suffixes).
 * @:without_any@ has become @:without_all@ - and is implemented, but Sphinx doesn't yet support the required logic.
 * If you're creating a multi-value attribute manually (using a SQL snippet), then in the definition pass in @:multi => true@, but @:type@ should be set as well, to one of the MVA types that Sphinx supports (@:integer@, @:timestamp@, or @:boolean@).
-* Automatic updates of non-string attributes are still limited to those from columns on the model in question, and is disabled by default. To enable it, just set attribute_updates to true in your @config/sphinx.yml@.
+* Automatic updates of non-string attributes are still limited to those from columns on the model in question, and is disabled by default. To enable it, just set attribute_updates to true in your @config/thinking_sphinx.yml@.
 * Search result helper methods are no longer injected into the actual result objects. Read the section below on search results, glazes and panes.
 * If you're using string facets, make sure they're defined as fields, not strings. There is currently no support for multi-value string facets.
 * To have fine-grained control over when deltas are invoked, create a sub-class of your chosen delta class (the standard is @ThinkingSphinx::Deltas::DefaultDelta@) and customise the @toggle@ and @toggled?@ methods, both of which accept a single parameter being the ActiveRecord instance.

data/lib/thinking_sphinx.rb

@@ -34,6 +34,8 @@ module ThinkingSphinx
   end
 
   @before_index_hooks = []
+
+  module Subscribers; end
 end
 
 # Core
@@ -56,6 +58,7 @@ require 'thinking_sphinx/panes'
 require 'thinking_sphinx/rake_interface'
 require 'thinking_sphinx/scopes'
 require 'thinking_sphinx/search'
+require 'thinking_sphinx/subscribers/populator_subscriber'
 require 'thinking_sphinx/test'
 # Extended
 require 'thinking_sphinx/active_record'

data/lib/thinking_sphinx/active_record/callbacks/update_callbacks.rb

@@ -42,7 +42,9 @@ class ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks <
     sphinxql = Riddle::Query.update(
       index.name, index.document_id_for_key(instance.id), attributes
     )
-    ThinkingSphinx::Connection.new.execute(sphinxql)
+    ThinkingSphinx::Connection.take do |connection|
+      connection.execute(sphinxql)
+    end
   rescue Mysql2::Error => error
     # This isn't vital, so don't raise the error.
   end

data/lib/thinking_sphinx/active_record/database_adapters/mysql_adapter.rb

@@ -25,6 +25,10 @@ class ThinkingSphinx::ActiveRecord::DatabaseAdapters::MySQLAdapter <
     "GROUP_CONCAT(#{clause} SEPARATOR '#{separator}')"
   end
 
+  def time_zone_query_pre
+    ["SET TIME_ZONE = '+0:00'"]
+  end
+
   def utf8_query_pre
     ['SET NAMES utf8']
   end

data/lib/thinking_sphinx/active_record/database_adapters/postgresql_adapter.rb

@@ -30,4 +30,8 @@ class ThinkingSphinx::ActiveRecord::DatabaseAdapters::PostgreSQLAdapter <
   def group_concatenate(clause, separator = ' ')
     "array_to_string(array_agg(#{clause}), '#{separator}')"
   end
+
+  def time_zone_query_pre
+    ['SET TIME ZONE UTC']
+  end
 end

data/lib/thinking_sphinx/active_record/property_query.rb

@@ -31,7 +31,7 @@ class ThinkingSphinx::ActiveRecord::PropertyQuery
   def base_association_class
     base_association.klass
   end
-  delegate :relation, :to => :base_association_class, :prefix => true
+  delegate :unscoped, :to => :base_association_class, :prefix => true
 
   def column
     @column ||= property.columns.first
@@ -93,7 +93,7 @@ class ThinkingSphinx::ActiveRecord::PropertyQuery
   end
 
   def range_sql
-    base_association_class_relation.select(
+    base_association_class_unscoped.select(
       "MIN(#{quoted_foreign_key}), MAX(#{quoted_foreign_key})"
     ).to_sql
   end
@@ -105,7 +105,7 @@ class ThinkingSphinx::ActiveRecord::PropertyQuery
   def to_sql
     raise "Could not determine SQL for MVA" if reflections.empty?
 
-    relation = base_association_class_relation.select("#{quoted_foreign_key} #{offset} AS #{quote_column('id')}, #{quoted_primary_key} AS #{quote_column(property.name)}"
+    relation = base_association_class_unscoped.select("#{quoted_foreign_key} #{offset} AS #{quote_column('id')}, #{quoted_primary_key} AS #{quote_column(property.name)}"
     )
     relation = relation.joins(joins) if joins.present?
     relation = relation.where("#{quoted_foreign_key} BETWEEN $start AND $end") if ranged?

data/lib/thinking_sphinx/active_record/property_sql_presenter.rb

@@ -6,12 +6,14 @@ class ThinkingSphinx::ActiveRecord::PropertySQLPresenter
   end
 
   def to_group
-    return nil unless group?
+    return nil if sourced_by_query? || !group?
 
     columns_with_table
   end
 
   def to_select
+    return nil if sourced_by_query?
+
     "#{casted_column_with_table} AS #{adapter.quote property.name}"
   end
 
@@ -76,4 +78,8 @@ class ThinkingSphinx::ActiveRecord::PropertySQLPresenter
   def group?
     !(aggregate? || property.columns.any?(&:string?))
   end
+
+  def sourced_by_query?
+    property.source_type.to_s[/query/]
+  end
 end

data/lib/thinking_sphinx/active_record/sql_builder.rb

@@ -26,6 +26,10 @@ module ThinkingSphinx
 
       private
 
+      delegate :adapter, :model, :delta_processor, :to => :source
+      delegate :convert_nulls, :time_zone_query_pre, :utf8_query_pre,
+        :to => :adapter
+
       def query
         Query.new(self)
       end
@@ -38,16 +42,10 @@ module ThinkingSphinx
         ThinkingSphinx::Configuration.instance
       end
 
-      delegate :adapter, :model, :delta_processor, :to => :source
-      delegate :convert_nulls, :utf8_query_pre, :to => :adapter
       def relation
         model.unscoped
       end
 
-      def base_join
-        @base_join ||= join_dependency_class.new model, [], initial_joins
-      end
-
       def associations
         @associations ||= ThinkingSphinx::ActiveRecord::Associations.new(model).tap do |assocs|
           source.associations.reject(&:string?).each do |association|
@@ -56,10 +54,6 @@ module ThinkingSphinx
         end
       end
 
-      def custom_joins
-        @custom_joins ||= source.associations.select(&:string?).collect(&:to_s)
-      end
-
       def quote_column(column)
         model.connection.quote_column_name(column)
       end
@@ -85,24 +79,6 @@ module ThinkingSphinx
         "($id - #{source.offset}) / #{config.indices.count}"
       end
 
-      def attribute_presenters
-        @attribute_presenters ||= property_sql_presenters_for(source.attributes)
-      end
-
-      def field_presenters
-        @field_presenters ||= property_sql_presenters_for(source.fields)
-      end
-
-      def property_sql_presenters_for(fields)
-        fields.collect { |field| property_sql_presenter_for(field) }
-      end
-
-      def property_sql_presenter_for(field)
-        ThinkingSphinx::ActiveRecord::PropertySQLPresenter.new(
-          field, source.adapter, associations
-        )
-      end
-
       def inheritance_column_condition
         "#{quoted_inheritance_column} = '#{model_name}'"
       end
@@ -114,14 +90,6 @@ module ThinkingSphinx
         condition
       end
 
-      def presenters_to_group(presenters)
-        presenters.collect(&:to_group)
-      end
-
-      def presenters_to_select(presenters)
-        presenters.collect(&:to_select)
-      end
-
       def groupings
         groupings = source.groupings
         if model.column_names.include?(model.inheritance_column)

data/lib/thinking_sphinx/active_record/sql_builder/query.rb

@@ -13,9 +13,11 @@ module ThinkingSphinx
       end
 
       protected
+
       attr_accessor :report, :scope
 
       def filter_by_query_pre
+        scope_by_time_zone
         scope_by_delta_processor
         scope_by_session
         scope_by_utf8
@@ -31,6 +33,10 @@ module ThinkingSphinx
         end
       end
 
+      def scope_by_time_zone
+        self.scope += time_zone_query_pre
+      end
+
       def scope_by_utf8
         self.scope += utf8_query_pre if source.options[:utf8?]
       end

data/lib/thinking_sphinx/active_record/sql_builder/statement.rb

@@ -35,6 +35,10 @@ module ThinkingSphinx
       protected
       attr_accessor :report, :scope
 
+      def custom_joins
+        @custom_joins ||= source.associations.select(&:string?).collect(&:to_s)
+      end
+
       def filter_by_query_range
         minimum = convert_nulls "MIN(#{quoted_primary_key})", 1
         maximum = convert_nulls "MAX(#{quoted_primary_key})", 1
@@ -55,6 +59,32 @@ module ThinkingSphinx
         scope_by_order
       end
 
+      def attribute_presenters
+        @attribute_presenters ||= property_sql_presenters_for source.attributes
+      end
+
+      def field_presenters
+        @field_presenters ||= property_sql_presenters_for source.fields
+      end
+
+      def presenters_to_group(presenters)
+        presenters.collect(&:to_group)
+      end
+
+      def presenters_to_select(presenters)
+        presenters.collect(&:to_select)
+      end
+
+      def property_sql_presenters_for(properties)
+        properties.collect { |property| property_sql_presenter_for(property) }
+      end
+
+      def property_sql_presenter_for(property)
+        ThinkingSphinx::ActiveRecord::PropertySQLPresenter.new(
+          property, source.adapter, associations
+        )
+      end
+
       def scope_by_select
         self.scope = scope.select(pre_select + select_clause)
       end

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -3,8 +3,8 @@ module ThinkingSphinx
     class SQLSource < Riddle::Configuration::SQLSource
       include ThinkingSphinx::Core::Settings
       attr_reader :model, :database_settings, :options
-      attr_accessor :fields, :attributes, :associations, :conditions, :groupings,
-        :polymorphs
+      attr_accessor :fields, :attributes, :associations, :conditions,
+        :groupings, :polymorphs
 
       OPTIONS = [:name, :offset, :delta_processor, :delta?, :disable_range?,
         :group_concat_max_len, :utf8?, :position]
@@ -107,10 +107,10 @@ module ThinkingSphinx
       end
 
       def build_sql_query
-        @sql_query       = builder.sql_query
-        @sql_query_range = builder.sql_query_range
-        @sql_query_info  = builder.sql_query_info
-        @sql_query_pre  += builder.sql_query_pre
+        @sql_query         = builder.sql_query
+        @sql_query_range ||= builder.sql_query_range
+        @sql_query_info  ||= builder.sql_query_info
+        @sql_query_pre    += builder.sql_query_pre
       end
 
       def config

data/lib/thinking_sphinx/configuration.rb

@@ -80,6 +80,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
     preload_indices
 
     ThinkingSphinx::Configuration::ConsistentIds.new(indices).reconcile
+    ThinkingSphinx::Configuration::MinimumFields.new(indices).reconcile
 
     super
   end
@@ -159,3 +160,4 @@ end
 
 require 'thinking_sphinx/configuration/consistent_ids'
 require 'thinking_sphinx/configuration/defaults'
+require 'thinking_sphinx/configuration/minimum_fields'

data/lib/thinking_sphinx/configuration/minimum_fields.rb

@@ -0,0 +1,31 @@
+class ThinkingSphinx::Configuration::MinimumFields
+  def initialize(indices)
+    @indices = indices
+  end
+
+  def reconcile
+    return unless no_inheritance_columns?
+
+    sources.each do |source|
+      source.fields.delete_if do |field|
+        field.name == 'sphinx_internal_class_name'
+      end
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def no_inheritance_columns?
+    indices.select { |index|
+      index.model.column_names.include?(index.model.inheritance_column)
+    }.empty?
+  end
+
+  def sources
+    @sources ||= @indices.select { |index|
+      index.respond_to?(:sources)
+    }.collect(&:sources).flatten
+  end
+end

data/lib/thinking_sphinx/connection.rb

@@ -7,8 +7,9 @@ module ThinkingSphinx::Connection
     address = '127.0.0.1' if address == 'localhost'
 
     options = {
-      :host  => address,
-      :port  => configuration.searchd.mysql41
+      :host      => address,
+      :port      => configuration.searchd.mysql41,
+      :reconnect => true
     }.merge(configuration.settings['connection_options'] || {})
 
     connection_class.new address, options[:port], options

data/lib/thinking_sphinx/deletion.rb

@@ -18,16 +18,12 @@ class ThinkingSphinx::Deletion
 
   attr_reader :index, :instance
 
-  def connection
-    @connection ||= ThinkingSphinx::Connection.new
-  end
-
   def document_id_for_key
     index.document_id_for_key instance.id
   end
 
   def execute(statement)
-    ThinkingSphinx::Connection.pool.take do |connection|
+    ThinkingSphinx::Connection.take do |connection|
       connection.execute statement
     end
   end

data/lib/thinking_sphinx/deltas/delete_job.rb

@@ -4,7 +4,7 @@ class ThinkingSphinx::Deltas::DeleteJob
   end
 
   def perform
-    ThinkingSphinx::Connection.pool.take do |connection|
+    ThinkingSphinx::Connection.take do |connection|
       connection.execute Riddle::Query.update(
         @index_name, @document_id, :sphinx_deleted => true
       )

data/lib/thinking_sphinx/excerpter.rb

@@ -14,7 +14,9 @@ class ThinkingSphinx::Excerpter
   end
 
   def excerpt!(text)
-    result = connection.query(statement_for(text)).first['snippet']
+    result = ThinkingSphinx::Connection.take do |connection|
+      connection.query(statement_for(text)).first['snippet']
+    end
 
     result.encode!("ISO-8859-1")
     result.force_encoding("UTF-8")
@@ -22,10 +24,6 @@ class ThinkingSphinx::Excerpter
 
   private
 
-  def connection
-    @connection ||= ThinkingSphinx::Connection.new
-  end
-
   def statement_for(text)
     Riddle::Query.snippets(text, index, words, options)
   end

data/lib/thinking_sphinx/masks/scopes_mask.rb

@@ -22,7 +22,8 @@ class ThinkingSphinx::Masks::ScopesMask
   private
 
   def apply_scope(scope, *args)
-    search *sphinx_scopes[scope].call(*args)
+    query, options = sphinx_scopes[scope].call(*args)
+    search query, options
   end
 
   def can_apply_scope?(scope)

data/lib/thinking_sphinx/middlewares/sphinxql.rb

@@ -55,6 +55,10 @@ class ThinkingSphinx::Middlewares::SphinxQL <
       "(#{classes_and_descendants_names.join('|')})"
     end
 
+    def class_condition_required?
+      classes.any? && !indices_match_classes?
+    end
+
     def constantize_inheritance_column(klass)
       klass.connection.select_values(inheritance_column_select(klass)).compact.each(&:constantize)
     end
@@ -70,6 +74,12 @@ class ThinkingSphinx::Middlewares::SphinxQL <
       end.flatten
     end
 
+    def indices_match_classes?
+      indices.collect(&:reference).uniq == classes.collect { |klass|
+        klass.name.underscore.to_sym
+      }
+    end
+
     def inheritance_column_select(klass)
       <<-SQL
 SELECT DISTINCT #{klass.inheritance_column}
@@ -85,7 +95,10 @@ SQL
 
     def extended_query
       conditions = options[:conditions] || {}
-      conditions[:sphinx_internal_class_name] = class_condition if classes.any?
+      if class_condition_required?
+        conditions[:sphinx_internal_class_name] = class_condition
+      end
+
       @extended_query ||= ThinkingSphinx::Search::Query.new(
         context.search.query, conditions, options[:star]
       ).to_s

data/lib/thinking_sphinx/rake_interface.rb

@@ -19,20 +19,9 @@ class ThinkingSphinx::RakeInterface
 
     FileUtils.mkdir_p configuration.indices_location
 
-    configuration.indices.each do |index|
-      next unless index.is_a?(ThinkingSphinx::RealTime::Index)
-
-      puts "Generating index files for #{index.name}"
-      transcriber = ThinkingSphinx::RealTime::Transcriber.new index
-      Dir["#{index.path}*"].each { |file| FileUtils.rm file }
-
-      index.model.find_each do |instance|
-        transcriber.copy instance
-        print "."
-      end
-      print "\n"
-
-      controller.rotate
+    indices = configuration.indices.select { |index| index.type == 'rt' }
+    indices.each do |index|
+      ThinkingSphinx::RealTime::Populator.populate index
     end
   end
 
@@ -71,11 +60,9 @@ class ThinkingSphinx::RakeInterface
 
   private
 
+  delegate :controller, :to => :configuration
+
   def configuration
     ThinkingSphinx::Configuration.instance
   end
-
-  def controller
-    configuration.controller
-  end
 end

data/lib/thinking_sphinx/real_time.rb

@@ -13,6 +13,7 @@ require 'thinking_sphinx/real_time/attribute'
 require 'thinking_sphinx/real_time/field'
 require 'thinking_sphinx/real_time/index'
 require 'thinking_sphinx/real_time/interpreter'
+require 'thinking_sphinx/real_time/populator'
 require 'thinking_sphinx/real_time/transcriber'
 
 require 'thinking_sphinx/real_time/callbacks/real_time_callbacks'

data/lib/thinking_sphinx/real_time/index.rb

@@ -1,7 +1,7 @@
 class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
   include ThinkingSphinx::Core::Index
 
-  attr_writer :fields, :attributes, :conditions
+  attr_writer :fields, :attributes, :conditions, :scope
 
   def initialize(reference, options = {})
     @fields     = []
@@ -43,6 +43,10 @@ class ThinkingSphinx::RealTime::Index < Riddle::Configuration::RealtimeIndex
     @fields
   end
 
+  def scope
+    @scope.nil? ? model : @scope.call
+  end
+
   def unique_attribute_names
     attributes.collect(&:name)
   end

data/lib/thinking_sphinx/real_time/interpreter.rb

@@ -15,6 +15,10 @@ class ThinkingSphinx::RealTime::Interpreter <
     }
   end
 
+  def scope(&block)
+    @index.scope = block
+  end
+
   def set_property(properties)
     properties.each do |key, value|
       @index.send("#{key}=", value)   if @index.class.settings.include?(key)

data/lib/thinking_sphinx/real_time/populator.rb

@@ -0,0 +1,48 @@
+class ThinkingSphinx::RealTime::Populator
+  def self.populate(index)
+    new(index).populate
+  end
+
+  def initialize(index)
+    @index = index
+  end
+
+  def populate(&block)
+    instrument 'start_populating'
+
+    remove_files
+
+    scope.find_each do |instance|
+      transcriber.copy instance
+      instrument 'populated', :instance => instance
+    end
+
+    controller.rotate
+    instrument 'finish_populating'
+  end
+
+  private
+
+  attr_reader :index
+
+  delegate :controller, :to => :configuration
+  delegate :scope,      :to => :index
+
+  def configuration
+    ThinkingSphinx::Configuration.instance
+  end
+
+  def instrument(message, options = {})
+    ActiveSupport::Notifications.instrument(
+      "#{message}.thinking_sphinx.real_time", options.merge(:index => index)
+    )
+  end
+
+  def remove_files
+    Dir["#{index.path}*"].each { |file| FileUtils.rm file }
+  end
+
+  def transcriber
+    @transcriber ||= ThinkingSphinx::RealTime::Transcriber.new index
+  end
+end

data/lib/thinking_sphinx/search/query.rb

@@ -28,7 +28,7 @@ class ThinkingSphinx::Search::Query
     keyword.gsub(/("#{token}(.*?#{token})?"|(?![!-])#{token})/u) do
       pre, proper, post = $`, $&, $'
       # E.g. "@foo", "/2", "~3", but not as part of a token
-      is_operator = pre.match(%r{(\W|^)[@~/]\Z}) ||
+      is_operator = pre.match(%r{\A(\W|^)[@~/]\Z}) ||
                     pre.match(%r{(\W|^)@\([^\)]*$})
       # E.g. "foo bar", with quotes
       is_quote    = proper[/^".*"$/]

data/lib/thinking_sphinx/subscribers/populator_subscriber.rb

@@ -0,0 +1,34 @@
+class ThinkingSphinx::Subscribers::PopulatorSubscriber
+  def self.attach_to(namespace)
+    subscriber = new
+
+    subscriber.public_methods(false).each do |event|
+      next if event == :call
+
+      ActiveSupport::Notifications.subscribe(
+        "#{event}.#{namespace}", subscriber
+      )
+    end
+  end
+
+  def call(message, *args)
+    send message.split('.').first,
+      ActiveSupport::Notifications::Event.new(message, *args)
+  end
+
+  def start_populating(event)
+    puts "Generating index files for #{event.payload[:index].name}"
+  end
+
+  def populated(event)
+    print '.'
+  end
+
+  def finish_populating(event)
+    print "\n"
+  end
+end
+
+ThinkingSphinx::Subscribers::PopulatorSubscriber.attach_to(
+  'thinking_sphinx.real_time'
+)

data/spec/acceptance/attribute_access_spec.rb

@@ -18,7 +18,7 @@ describe 'Accessing attributes directly via search results', :live => true do
     search = Book.search('gods', :select => '*, @weight')
     search.context[:panes] << ThinkingSphinx::Panes::WeightPane
 
-    search.first.weight.should == 3500
+    search.first.weight.should == 2500
   end
 
   it "can enumerate with the weight" do
@@ -28,7 +28,7 @@ describe 'Accessing attributes directly via search results', :live => true do
     search = Book.search('gods', :select => '*, @weight')
     search.masks << ThinkingSphinx::Masks::WeightEnumeratorMask
 
-    expectations = [[gods, 3500]]
+    expectations = [[gods, 2500]]
     search.each_with_weight do |result, weight|
       expectation = expectations.shift
 

data/spec/acceptance/index_options_spec.rb

@@ -106,7 +106,7 @@ describe 'Index options' do
     end
 
     it "respects sql_query_pre values" do
-      index.sources.first.sql_query_pre.should == ["DO STUFF"]
+      index.sources.first.sql_query_pre.should include("DO STUFF")
     end
   end
 

data/spec/acceptance/searching_with_filters_spec.rb

@@ -69,6 +69,26 @@ describe 'Searching with filters', :live => true do
     ).to_a.should == [pancakes]
   end
 
+  it "takes into account local timezones for timestamps" do
+    pancakes = Article.create :title => 'Pancakes'
+    waffles  = Article.create :title => 'Waffles'
+
+    food = Tag.create :name => 'food'
+    flat = Tag.create :name => 'flat'
+
+    Tagging.create(:tag => food, :article => pancakes).
+      update_column :created_at, 5.minutes.ago
+    Tagging.create :tag => flat, :article => pancakes
+    Tagging.create(:tag => food, :article => waffles).
+      update_column :created_at, 3.minute.ago
+
+    index
+
+    Article.search(
+      :with => {:taggings_at => 2.minutes.ago..Time.zone.now}
+    ).to_a.should == [pancakes]
+  end
+
   it "limits results with MVAs having all of the given values" do
     pancakes = Article.create :title => 'Pancakes'
     waffles  = Article.create :title => 'Waffles'

data/spec/acceptance/sphinx_scopes_spec.rb

@@ -25,6 +25,15 @@ describe 'Sphinx scopes', :live => true do
     grave = Book.create! :title => 'The Graveyard Book', :year => 2009
     index
 
+    Book.by_year(2001..2005).ordered.to_a.should == [boys, gods]
+  end
+
+  it "allows chaining of scopes that include queries" do
+    gods  = Book.create! :title => 'American Gods',      :year => 2001
+    boys  = Book.create! :title => 'Anansi Boys',        :year => 2005
+    grave = Book.create! :title => 'The Graveyard Book', :year => 2009
+    index
+
     Book.by_year(2001).by_query_and_year('Graveyard', 2009).to_a.
       should == [grave]
   end

data/spec/internal/app/indices/article_index.rb

@@ -4,7 +4,7 @@ ThinkingSphinx::Index.define :article, :with => :active_record do
   indexes user.articles.title, :as => :related_titles
 
   has published, user_id
-  has taggings.tag_id, :as => :tag_ids
+  has taggings.tag_id, :as => :tag_ids, :source => :query
   has taggings.created_at, :as => :taggings_at
 
   set_property :min_infix_len => 4

data/spec/internal/app/models/book.rb

@@ -10,4 +10,5 @@ class Book < ActiveRecord::Base
   sphinx_scope(:by_query_and_year) do |query, year|
     [query, {:with => {:year =>year}}]
   end
+  sphinx_scope(:ordered) { {:order => 'year DESC'} }
 end

data/spec/internal/app/models/city.rb

@@ -1,3 +1,3 @@
 class City < ActiveRecord::Base
-  scope :ordered, order(:name)
+  scope :ordered, lambda { order(:name) }
 end

data/spec/internal/db/schema.rb

@@ -33,8 +33,7 @@ ActiveRecord::Schema.define do
   end
 
   create_table(:categories, :force => true) do |t|
-    t.integer :id
-    t.string  :name
+    t.string :name
   end
 
   create_table(:categorisations, :force => true) do |t|

data/spec/thinking_sphinx/active_record/callbacks/update_callbacks_spec.rb

@@ -26,9 +26,11 @@ describe ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks do
     before :each do
       stub_const 'ThinkingSphinx::Configuration',
         double(:instance => configuration)
-      stub_const 'ThinkingSphinx::Connection', double(:new => connection)
+      stub_const 'ThinkingSphinx::Connection', double
       stub_const 'Riddle::Query', double(:update => 'SphinxQL')
 
+      ThinkingSphinx::Connection.stub(:take).and_yield(connection)
+
       source.attributes.replace([
         double(:name => 'foo', :updateable? => true,
           :columns => [double(:__name => 'foo_column')]),

data/spec/thinking_sphinx/active_record/property_sql_presenter_spec.rb

@@ -17,7 +17,7 @@ describe ThinkingSphinx::ActiveRecord::PropertySQLPresenter do
       )
     }
     let(:field)     { double('field', :name => 'title', :columns => [column],
-      :type => nil, :multi? => false) }
+      :type => nil, :multi? => false, :source_type => nil) }
     let(:column)    { double('column', :string? => false, :__stack => [],
       :__name => 'title') }
 
@@ -40,6 +40,12 @@ describe ThinkingSphinx::ActiveRecord::PropertySQLPresenter do
 
         presenter.to_group.should be_nil
       end
+
+      it "returns nil if the field is sourced via a separate query" do
+        field.stub :source_type => 'query'
+
+        presenter.to_group.should be_nil
+      end
     end
 
     describe '#to_select' do
@@ -95,6 +101,18 @@ describe ThinkingSphinx::ActiveRecord::PropertySQLPresenter do
         presenter.to_select.
           should == "CONCAT_WS(' ', articles.title) AS title"
       end
+
+      it "returns nil for query sourced fields" do
+        field.stub :source_type => :query
+
+        presenter.to_select.should be_nil
+      end
+
+      it "returns nil for ranged query sourced fields" do
+        field.stub :source_type => :ranged_query
+
+        presenter.to_select.should be_nil
+      end
     end
   end
 
@@ -105,7 +123,8 @@ describe ThinkingSphinx::ActiveRecord::PropertySQLPresenter do
       )
     }
     let(:attribute) { double('attribute', :name => 'created_at',
-      :columns => [column], :type => :integer, :multi? => false) }
+      :columns => [column], :type => :integer, :multi? => false,
+      :source_type => nil) }
     let(:column)    { double('column', :string? => false, :__stack => [],
       :__name => 'created_at') }
 
@@ -140,6 +159,12 @@ describe ThinkingSphinx::ActiveRecord::PropertySQLPresenter do
 
         presenter.to_group.should be_nil
       end
+
+      it "returns nil if the attribute is sourced via a separate query" do
+        attribute.stub :source_type => 'query'
+
+        presenter.to_group.should be_nil
+      end
     end
 
     describe '#to_select' do
@@ -196,6 +221,18 @@ describe ThinkingSphinx::ActiveRecord::PropertySQLPresenter do
 
         presenter.to_select.should == "CONCAT_WS(',', CAST(articles.created_at AS varchar), CAST(articles.created_at AS varchar)) AS created_at"
       end
+
+      it "returns nil for query sourced attributes" do
+        attribute.stub :source_type => :query
+
+        presenter.to_select.should be_nil
+      end
+
+      it "returns nil for ranged query sourced attributes" do
+        attribute.stub :source_type => :ranged_query
+
+        presenter.to_select.should be_nil
+      end
     end
   end
 end

data/spec/thinking_sphinx/active_record/sql_builder_spec.rb

@@ -15,7 +15,8 @@ describe ThinkingSphinx::ActiveRecord::SQLBuilder do
   let(:indices)      { double('indices', :count => 5) }
   let(:presenter)    { double('presenter', :to_select => '`name` AS `name`',
     :to_group => '`name`') }
-  let(:adapter)      { double('adapter') }
+  let(:adapter)      { double('adapter',
+    :time_zone_query_pre => ['SET TIME ZONE']) }
   let(:associations) { double('associations', :join_values => []) }
   let(:builder)      { ThinkingSphinx::ActiveRecord::SQLBuilder.new source }
 

data/spec/thinking_sphinx/deletion_spec.rb

@@ -6,13 +6,10 @@ describe ThinkingSphinx::Deletion do
     let(:index)      { double('index', :name => 'foo_core',
       :document_id_for_key => 14, :type => 'plain') }
     let(:instance)   { double('instance', :id => 7) }
-    let(:pool)       { double 'pool' }
 
     before :each do
-      ThinkingSphinx::Connection.stub :pool => pool
+      ThinkingSphinx::Connection.stub(:take).and_yield(connection)
       Riddle::Query.stub :update => 'UPDATE STATEMENT'
-
-      pool.stub(:take).and_yield(connection)
     end
 
     context 'index is SQL-backed' do

data/spec/thinking_sphinx/deltas/default_delta_spec.rb

@@ -23,12 +23,10 @@ describe ThinkingSphinx::Deltas::DefaultDelta do
     let(:index)      { double('index', :name => 'foo_core',
       :document_id_for_key => 14) }
     let(:instance)   { double('instance', :id => 7) }
-    let(:pool)       { double }
 
     before :each do
-      ThinkingSphinx::Connection.stub :pool => pool
+      ThinkingSphinx::Connection.stub(:take).and_yield(connection)
       Riddle::Query.stub :update => 'UPDATE STATEMENT'
-      pool.stub(:take).and_yield(connection)
     end
 
     it "updates the deleted flag to false" do

data/spec/thinking_sphinx/excerpter_spec.rb

@@ -7,7 +7,7 @@ describe ThinkingSphinx::Excerpter do
   }
 
   before :each do
-    ThinkingSphinx::Connection.stub :new => connection
+    ThinkingSphinx::Connection.stub(:take).and_yield(connection)
     Riddle::Query.stub :snippets => 'CALL SNIPPETS'
   end
 

data/spec/thinking_sphinx/middlewares/sphinxql_spec.rb

@@ -6,6 +6,7 @@ module ActiveRecord
   class Base; end
 end
 
+require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/string/inflections'
 require 'thinking_sphinx/middlewares/middleware'
@@ -49,6 +50,7 @@ describe ThinkingSphinx::Middlewares::SphinxQL do
         :name => 'User')
       search.options[:classes] = [klass]
       search.options[:indices] = ['user_core']
+      index_set.first.stub :reference => :user
 
       ThinkingSphinx::IndexSet.should_receive(:new).
         with([klass], ['user_core']).and_return(index_set)
@@ -83,6 +85,30 @@ describe ThinkingSphinx::Middlewares::SphinxQL do
       middleware.call [context]
     end
 
+    it "doesn't append a field condition by default" do
+      ThinkingSphinx::Search::Query.should_receive(:new) do |query, conditions, star|
+        conditions[:sphinx_internal_class_name].should be_nil
+        query
+      end
+
+      middleware.call [context]
+    end
+
+    it "doesn't append a field condition if all classes match index references" do
+      model = double('model', :connection => double,
+        :ancestors => [ActiveRecord::Base], :name => 'Animal')
+      index_set.first.stub :reference => :animal
+
+      search.options[:classes] = [model]
+
+      ThinkingSphinx::Search::Query.should_receive(:new) do |query, conditions, star|
+        conditions[:sphinx_internal_class_name].should be_nil
+        query
+      end
+
+      middleware.call [context]
+    end
+
     it "appends field conditions for the class when searching on subclasses" do
       db_connection = double('db connection', :select_values => [],
         :schema_cache => double('cache', :table_exists? => false))
@@ -98,6 +124,7 @@ describe ThinkingSphinx::Middlewares::SphinxQL do
       end
       submodel.stub :connection => db_connection, :column_names => ['type'],
         :descendants => []
+      index_set.first.stub :reference => :cat
 
       search.options[:classes] = [submodel]
 
@@ -123,6 +150,7 @@ describe ThinkingSphinx::Middlewares::SphinxQL do
       end
       submodel.stub :connection => db_connection, :column_names => ['type'],
         :descendants => []
+      index_set.first.stub :reference => :"animals/cat"
 
       search.options[:classes] = [submodel]
 
@@ -136,6 +164,7 @@ describe ThinkingSphinx::Middlewares::SphinxQL do
     it "does not query the database for subclasses if :skip_sti is set to true" do
       model = double('model', :connection => double,
         :ancestors => [ActiveRecord::Base], :name => 'Animal')
+      index_set.first.stub :reference => :animal
 
       search.options[:classes]  = [model]
       search.options[:skip_sti] = true

data/spec/thinking_sphinx/real_time/index_spec.rb

@@ -149,6 +149,22 @@ describe ThinkingSphinx::RealTime::Index do
     end
   end
 
+  describe '#scope' do
+    let(:model)  { double('model') }
+
+    it "returns the model by default" do
+      ActiveSupport::Inflector.stub(:constantize => model)
+
+      index.scope.should == model
+    end
+
+    it "returns the evaluated scope if provided" do
+      index.scope = lambda { :foo }
+
+      index.scope.should == :foo
+    end
+  end
+
   describe '#unique_attribute_names' do
     it "returns all attribute names" do
       index.unique_attribute_names.should == [

data/spec/thinking_sphinx/real_time/interpreter_spec.rb

@@ -119,6 +119,14 @@ describe ThinkingSphinx::RealTime::Interpreter do
     end
   end
 
+  describe '#scope' do
+    it "passes the scope block through to the index" do
+      index.should_receive(:scope=).with(instance_of(Proc))
+
+      instance.scope { :foo }
+    end
+  end
+
   describe '#set_property' do
     before :each do
       index.class.stub :settings => [:morphology]

data/spec/thinking_sphinx/search/query_spec.rb

@@ -69,6 +69,24 @@ describe ThinkingSphinx::Search::Query do
       query.to_s.should == "*\\@pan*"
     end
 
+    it "ignores escaped slashes" do
+      query = ThinkingSphinx::Search::Query.new "\\/\\/pan", {}, true
+
+      query.to_s.should == "\\/\\/*pan*"
+    end
+
+    it "does not star quorum operators" do
+      query = ThinkingSphinx::Search::Query.new "foo/3", {}, true
+
+      query.to_s.should == "*foo*/3"
+    end
+
+    it "does not star proximity operators or quoted strings" do
+      query = ThinkingSphinx::Search::Query.new %q{"hello world"~3}, {}, true
+
+      query.to_s.should == %q{"hello world"~3}
+    end
+
     it "handles null values by removing them from the conditions hash" do
       query = ThinkingSphinx::Search::Query.new '', :title => nil
 

data/thinking-sphinx.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = 'thinking-sphinx'
-  s.version     = '3.0.4'
+  s.version     = '3.0.5'
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Pat Allan"]
   s.email       = ["pat@freelancing-gods.com"]
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'builder',      '>= 2.1.2'
   s.add_runtime_dependency 'middleware',   '>= 0.1.0'
   s.add_runtime_dependency 'innertube',    '>= 1.0.2'
-  s.add_runtime_dependency 'riddle',       '>= 1.5.7'
+  s.add_runtime_dependency 'riddle',       '>= 1.5.8'
 
   s.add_development_dependency 'appraisal',        '~> 0.4.0'
   s.add_development_dependency 'combustion',       '~> 0.4.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: thinking-sphinx
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: 3.0.5
 platform: ruby
 authors:
 - Pat Allan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-09 00:00:00.000000000 Z
+date: 2013-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.5.7
+        version: 1.5.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.5.7
+        version: 1.5.8
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -198,6 +198,7 @@ files:
 - lib/thinking_sphinx/configuration.rb
 - lib/thinking_sphinx/configuration/consistent_ids.rb
 - lib/thinking_sphinx/configuration/defaults.rb
+- lib/thinking_sphinx/configuration/minimum_fields.rb
 - lib/thinking_sphinx/connection.rb
 - lib/thinking_sphinx/core.rb
 - lib/thinking_sphinx/core/field.rb
@@ -249,6 +250,7 @@ files:
 - lib/thinking_sphinx/real_time/index.rb
 - lib/thinking_sphinx/real_time/index/template.rb
 - lib/thinking_sphinx/real_time/interpreter.rb
+- lib/thinking_sphinx/real_time/populator.rb
 - lib/thinking_sphinx/real_time/property.rb
 - lib/thinking_sphinx/real_time/transcriber.rb
 - lib/thinking_sphinx/scopes.rb
@@ -260,6 +262,7 @@ files:
 - lib/thinking_sphinx/search/query.rb
 - lib/thinking_sphinx/search/stale_ids_exception.rb
 - lib/thinking_sphinx/sinatra.rb
+- lib/thinking_sphinx/subscribers/populator_subscriber.rb
 - lib/thinking_sphinx/tasks.rb
 - lib/thinking_sphinx/test.rb
 - sketchpad.rb