thinking-sphinx 1.2.13 → 1.4.0

data/lib/thinking_sphinx/search.rb

@@ -14,7 +14,7 @@ module ThinkingSphinx
       kind_of? member? method methods nil? object_id respond_to? send should
       type )
     SafeMethods = %w( partition private_methods protected_methods
-      public_methods send )
+      public_methods send class )
     
     instance_methods.select { |method|
       method.to_s[/^__/].nil? && !CoreMethods.include?(method.to_s)
@@ -57,10 +57,38 @@ module ThinkingSphinx
       ThinkingSphinx.facets *args
     end
     
+    def self.bundle_searches(enum = nil)
+      bundle = ThinkingSphinx::BundledSearch.new
+      
+      if enum.nil?
+        yield bundle
+      else
+        enum.each { |item| yield bundle, item }
+      end
+      
+      bundle.searches
+    end
+    
+    def self.matching_fields(fields, bitmask)
+      matches   = []
+      bitstring = bitmask.to_s(2).rjust(32, '0').reverse
+      
+      fields.each_with_index do |field, index|
+        matches << field if bitstring[index, 1] == '1'
+      end
+      matches
+    end
+    
     def initialize(*args)
+      ThinkingSphinx.context.define_indexes
+      
       @array    = []
       @options  = args.extract_options!
       @args     = args
+      
+      add_default_scope unless options[:ignore_default]
+      
+      populate if @options[:populate]
     end
     
     def to_a
@@ -68,6 +96,12 @@ module ThinkingSphinx
       @array
     end
     
+    def freeze
+      populate
+      @array.freeze
+      self
+    end
+    
     # Indication of whether the request has been made to Sphinx for the search
     # query.
     # 
@@ -90,6 +124,9 @@ module ThinkingSphinx
       if is_scope?(method)
         add_scope(method, *args, &block)
         return self
+      elsif method == :search_count
+        merge_search one_class.search(*args), self.args, options
+        return scoped_count
       elsif method.to_s[/^each_with_.*/].nil? && !@array.respond_to?(method)
         super
       elsif !SafeMethods.include?(method.to_s)
@@ -109,8 +146,8 @@ module ThinkingSphinx
     # @param [Symbol] method The method name
     # @return [Boolean] true if either Search or Array responds to the method.
     # 
-    def respond_to?(method)
-      super || @array.respond_to?(method)
+    def respond_to?(method, include_private = false)
+      super || @array.respond_to?(method, include_private)
     end
     
     # The current page number of the result set. Defaults to 1 if no page was
@@ -146,7 +183,9 @@ module ThinkingSphinx
     # @return [Integer]
     # 
     def per_page
-      @options[:limit] || @options[:per_page] || 20
+      @options[:limit] ||= @options[:per_page]
+      @options[:limit] ||= 20
+      @options[:limit].to_i
     end
     
     # The total number of pages available if the results are paginated.
@@ -155,33 +194,51 @@ module ThinkingSphinx
     # 
     def total_pages
       populate
+      return 0 if @results[:total].nil?
+      
       @total_pages ||= (@results[:total] / per_page.to_f).ceil
     end
     # Compatibility with older versions of will_paginate
     alias_method :page_count, :total_pages
     
+    # Query time taken
+    # 
+    # @return [Integer]
+    #
+    def query_time
+      populate
+      return 0 if @results[:time].nil?
+
+      @query_time ||= @results[:time]
+    end
+
     # The total number of search results available.
     # 
     # @return [Integer]
     # 
     def total_entries
       populate
+      return 0 if @results[:total_found].nil?
+      
       @total_entries ||= @results[:total_found]
     end
     
     # The current page's offset, based on the number of records per page.
+    # Or explicit :offset if given. 
     # 
     # @return [Integer]
     # 
     def offset
-      (current_page - 1) * per_page
+      @options[:offset] || ((current_page - 1) * per_page)
     end
     
     def indexes
       return options[:index] if options[:index]
       return '*' if classes.empty?
       
-      classes.collect { |klass| klass.sphinx_index_names }.flatten.join(',')
+      classes.collect { |klass|
+        klass.sphinx_index_names
+      }.flatten.uniq.join(',')
     end
     
     def each_with_groupby_and_count(&block)
@@ -201,6 +258,13 @@ module ThinkingSphinx
       end
     end
     
+    def each_with_match(&block)
+      populate
+      results[:matches].each_with_index do |match, index|
+        yield self[index], match
+      end
+    end
+    
     def excerpt_for(string, model = nil)
       if model.nil? && one_class
         model ||= one_class
@@ -208,17 +272,66 @@ module ThinkingSphinx
       
       populate
       client.excerpts(
-        :docs   => [string],
-        :words  => results[:words].keys.join(' '),
-        :index  => "#{model.source_of_sphinx_index.sphinx_name}_core"
+        {
+          :docs   => [string.to_s],
+          :words  => results[:words].keys.join(' '),
+          :index  => options[:index] || "#{model.source_of_sphinx_index.sphinx_name}_core"
+        }.merge(options[:excerpt_options] || {})
       ).first
     end
     
     def search(*args)
-      merge_search ThinkingSphinx::Search.new(*args)
+      args << args.extract_options!.merge(:ignore_default => true)
+      merge_search ThinkingSphinx::Search.new(*args), self.args, options
       self
     end
     
+    def search_for_ids(*args)
+      args << args.extract_options!.merge(
+        :ignore_default => true,
+        :ids_only       => true
+      )
+      merge_search ThinkingSphinx::Search.new(*args), self.args, options
+      self
+    end
+    
+    def facets(*args)
+      options = args.extract_options!
+      merge_search self, args, options
+      args << options
+      
+      ThinkingSphinx::FacetSearch.new *args
+    end
+    
+    def client
+      client = options[:client] || config.client
+      
+      prepare client
+    end
+    
+    def append_to(client)
+      prepare client
+      client.append_query query, indexes, comment
+      client.reset
+    end
+    
+    def populate_from_queue(results)
+      return if @populated
+      @populated = true
+      @results   = results
+      
+      if options[:ids_only]
+        replace @results[:matches].collect { |match|
+          match[:attributes]["sphinx_internal_id"]
+        }
+      else
+        replace instances_from_matches
+        add_excerpter
+        add_sphinx_attributes
+        add_matching_fields if client.rank_mode == :fieldmask
+      end
+    end
+    
     private
     
     def config
@@ -231,8 +344,12 @@ module ThinkingSphinx
       
       retry_on_stale_index do
         begin
-          log "Querying Sphinx: #{query}"
-          @results = client.query query, indexes, comment
+          log "Querying: '#{query}'"
+          runtime = Benchmark.realtime {
+            @results = client.query query, indexes, comment
+          }
+          log "Found #{@results[:total_found]} results", :debug,
+            "Sphinx (#{sprintf("%f", runtime)}s)"
         rescue Errno::ECONNREFUSED => err
           raise ThinkingSphinx::ConnectionError,
             'Connection to Sphinx Daemon (searchd) failed.'
@@ -246,52 +363,70 @@ module ThinkingSphinx
           replace instances_from_matches
           add_excerpter
           add_sphinx_attributes
+          add_matching_fields if client.rank_mode == :fieldmask
         end
       end
     end
     
     def add_excerpter
       each do |object|
-        next if object.respond_to?(:excerpts)
-        
-        excerpter = ThinkingSphinx::Excerpter.new self, object
-        block = lambda { excerpter }
+        next if object.nil?
         
-        object.metaclass.instance_eval do
-          define_method(:excerpts, &block)
-        end
+        object.excerpts = ThinkingSphinx::Excerpter.new self, object
       end
     end
     
     def add_sphinx_attributes
       each do |object|
-        next if object.nil? || object.respond_to?(:sphinx_attributes)
+        next if object.nil?
         
-        match = @results[:matches].detect { |match|
-          match[:attributes]['sphinx_internal_id'] == object.
-            primary_key_for_sphinx &&
-          match[:attributes]['class_crc'] == object.class.to_crc32
-        }
+        match = match_hash object
         next if match.nil?
         
-        object.metaclass.instance_eval do
-          define_method(:sphinx_attributes) { match[:attributes] }
-        end
+        object.sphinx_attributes = match[:attributes]
       end
     end
     
-    def self.log(message, method = :debug)
-      return if ::ActiveRecord::Base.logger.nil?
-      ::ActiveRecord::Base.logger.send method, message
+    def add_matching_fields
+      each do |object|
+        next if object.nil?
+        
+        match = match_hash object
+        next if match.nil?
+        object.matching_fields = ThinkingSphinx::Search.matching_fields(
+          @results[:fields], match[:weight]
+        )
+      end
     end
     
-    def log(message, method = :debug)
-      self.class.log(message, method)
+    def match_hash(object)
+      @results[:matches].detect { |match|
+        match[:attributes]['sphinx_internal_id'] == object.
+          primary_key_for_sphinx &&
+        match[:attributes]['class_crc'] == object.class.to_crc32
+      }
     end
     
-    def client
-      client = config.client
+    def self.log(message, method = :debug, identifier = 'Sphinx')
+      return if ::ActiveRecord::Base.logger.nil?
+      
+      info = ''
+      if ::ActiveRecord::Base.colorize_logging
+        identifier_color, message_color = "4;32;1", "0" # 0;1 = Bold
+        info << "  \e[#{identifier_color}m#{identifier}\e[0m   "
+        info << "\e[#{message_color}m#{message}\e[0m"
+      else
+        info = "#{identifier}   #{message}"
+      end
       
+      ::ActiveRecord::Base.logger.send method, info
+    end
+    
+    def log(*args)
+      self.class.log(*args)
+    end
+    
+    def prepare(client)
       index_options = one_class ?
         one_class.sphinx_indexes.first.local_options : {}
       
@@ -300,11 +435,12 @@ module ThinkingSphinx
         :group_distinct, :id_range, :cut_off, :retry_count, :retry_delay,
         :rank_mode, :max_query_time, :field_weights
       ].each do |key|
-        # puts "key: #{key}"
         value = options[key] || index_options[key]
-        # puts "value: #{value.inspect}"
         client.send("#{key}=", value) if value
       end
+
+      # treated non-standard as :select is already used for AR queries
+      client.select = options[:sphinx_select] || '*'
       
       client.limit      = per_page
       client.offset     = offset
@@ -381,7 +517,8 @@ module ThinkingSphinx
       query.gsub(/("#{token}(.*?#{token})?"|(?![!-])#{token})/u) do
         pre, proper, post = $`, $&, $'
         # E.g. "@foo", "/2", "~3", but not as part of a token
-        is_operator = pre.match(%r{(\W|^)[@~/]\Z})
+        is_operator = pre.match(%r{(\W|^)[@~/]\Z}) ||
+                      pre.match(%r{(\W|^)@\([^\)]*$})
         # E.g. "foo bar", with quotes
         is_quote    = proper.starts_with?('"') && proper.ends_with?('"')
         has_star    = pre.ends_with?("*") || post.starts_with?("*")
@@ -495,24 +632,8 @@ module ThinkingSphinx
       filters
     end
     
-    def condition_filters
-      (options[:conditions] || {}).collect { |attrib, value|
-        if attributes.include?(attrib.to_sym)
-          puts <<-MSG
-Deprecation Warning: filters on attributes should be done using the :with
-option, not :conditions. For example:
-  :with => {:#{attrib} => #{value.inspect}}
-MSG
-          Riddle::Client::Filter.new attrib.to_s, filter_value(value)
-        else
-          nil
-        end
-      }.compact
-    end
-    
     def filters
       internal_filters +
-      condition_filters +
       (options[:with] || {}).collect { |attrib, value|
         Riddle::Client::Filter.new attrib.to_s, filter_value(value)
       } +
@@ -527,14 +648,6 @@ MSG
     end
     
     # When passed a Time instance, returns the integer timestamp.
-    # 
-    # If using Rails 2.1+, need to handle timezones to translate them back to
-    # UTC, as that's what datetimes will be stored as by MySQL.
-    # 
-    # in_time_zone is a method that was added for the timezone support in
-    # Rails 2.1, which is why it's used for testing. I'm sure there's better
-    # ways, but this does the job.
-    # 
     def filter_value(value)
       case value
       when Range
@@ -542,7 +655,7 @@ MSG
       when Array
         value.collect { |v| filter_value(v) }.flatten
       when Time
-        value.respond_to?(:in_time_zone) ? [value.utc.to_i] : [value.to_i]
+        [value.to_i]
       when NilClass
         0
       else
@@ -608,6 +721,21 @@ MSG
       end
     end
     
+    def include_for_class(klass)
+      includes = options[:include] || klass.sphinx_index_options[:include]
+      
+      case includes
+      when NilClass
+        nil
+      when Array
+        includes.select { |inc| klass.reflections[inc] }
+      when Symbol
+        klass.reflections[includes].nil? ? nil : includes
+      else
+        includes
+      end
+    end
+    
     def instances_from_class(klass, matches)
       index_options = klass.sphinx_index_options
 
@@ -616,7 +744,7 @@ MSG
         :all,
         :joins      => options[:joins],
         :conditions => {klass.primary_key_for_sphinx.to_sym => ids},
-        :include    => (options[:include] || index_options[:include]),
+        :include    => include_for_class(klass),
         :select     => (options[:select]  || index_options[:select]),
         :order      => (options[:sql_order] || index_options[:sql_order])
       ) : []
@@ -684,11 +812,18 @@ MSG
       one_class && one_class.sphinx_scopes.include?(method)
     end
     
+    # Adds the default_sphinx_scope if set.
+    def add_default_scope
+      return unless one_class && one_class.has_default_sphinx_scope?
+      add_scope(one_class.get_default_sphinx_scope.to_sym)
+    end
+    
     def add_scope(method, *args, &block)
-      merge_search one_class.send(method, *args, &block)
+      method = "#{method}_without_default".to_sym
+      merge_search one_class.send(method, *args, &block), self.args, options
     end
     
-    def merge_search(search)
+    def merge_search(search, args, options)
       search.args.each { |arg| args << arg }
       
       search.options.keys.each do |key|
@@ -704,5 +839,16 @@ MSG
         end
       end
     end
+    
+    def scoped_count
+      return self.total_entries if @options[:ids_only]
+      
+      @options[:ids_only] = true
+      results_count = self.total_entries
+      @options[:ids_only] = false
+      @populated = false
+      
+      results_count
+    end
   end
 end

data/lib/thinking_sphinx/search_methods.rb

@@ -155,7 +155,7 @@ module ThinkingSphinx
       # asterisks. You need to make the config/sphinx.yml changes yourself.
       #
       # By default, the tokens are assumed to match the regular expression
-      # /\w+/u. If you've modified the charset_table, pass another regular
+      # /\w\+/u\+. If you've modified the charset_table, pass another regular
       # expression, e.g.
       #
       #   User.search("oo@bar.c", :star => /[\w@.]+/u)
@@ -313,13 +313,31 @@ module ThinkingSphinx
       # Once you've got your results set, you can access the distances as
       # follows:
       # 
-      # @results.each_with_geodist do |result, distance|
-      #   # ...
-      # end
+      #   @results.each_with_geodist do |result, distance|
+      #     # ...
+      #   end
       # 
       # The distance value is returned as a float, representing the distance in
       # metres.
       # 
+      # == Filtering by custom attributes
+      #
+      # Do note that this applies only to sphinx 0.9.9
+      # 
+      # Should you find yourself in desperate need of a filter that involves
+      # selecting either one of multiple conditions, one solution could be
+      # provided by the :sphinx_select option within the search. 
+      # This handles which fields are selected by sphinx from its store.
+      #
+      # The default value is "*", and you can add custom fields using syntax
+      # similar to sql:
+      #
+      #   Flower.search "foo",
+      #     :sphinx_select => "*, petals < 1 or color = 2 as grass"
+      #
+      # This will add the 'grass' attribute, which will now be usable in your
+      # filters.
+      #   
       # == Handling a Stale Index
       #
       # Especially if you don't use delta indexing, you risk having records in

data/lib/thinking_sphinx/source/internal_properties.rb

@@ -2,9 +2,9 @@ module ThinkingSphinx
   class Source
     module InternalProperties
       def add_internal_attributes_and_facets
-        add_internal_attribute :sphinx_internal_id, :integer, @model.primary_key_for_sphinx.to_sym
+        add_internal_attribute :sphinx_internal_id, nil,
+          @model.primary_key_for_sphinx.to_sym
         add_internal_attribute :class_crc,          :integer, crc_column, true
-        add_internal_attribute :subclass_crcs,      :multi,   subclasses_to_s
         add_internal_attribute :sphinx_deleted,     :integer, "0"
 
         add_internal_facet :class_crc

data/lib/thinking_sphinx/source/sql.rb

@@ -13,8 +13,10 @@ module ThinkingSphinx
       #   source.to_sql(:delta => true)
       #
       def to_sql(options={})
-        sql = <<-SQL
-SELECT #{ sql_select_clause options[:offset] }
+        sql = "SELECT "
+        sql += "SQL_NO_CACHE " if adapter.sphinx_identifier == "mysql"
+        sql += <<-SQL
+#{ sql_select_clause options[:offset] }
 FROM #{ @model.quoted_table_name }
   #{ all_associations.collect { |assoc| assoc.to_sql }.join(' ') }
 #{ sql_where_clause(options) }
@@ -53,7 +55,7 @@ GROUP BY #{ sql_group_clause }
       # 
       def to_sql_query_info(offset)
         "SELECT * FROM #{@model.quoted_table_name} WHERE " +
-        "#{quote_column(@model.primary_key_for_sphinx)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})"
+        "#{quote_column(@model.primary_key_for_sphinx)} = (($id - #{offset}) / #{ThinkingSphinx.context.indexed_models.size})"
       end
 
       def sql_select_clause(offset)

data/lib/thinking_sphinx/source.rb

@@ -6,19 +6,22 @@ module ThinkingSphinx
     include ThinkingSphinx::Source::InternalProperties
     include ThinkingSphinx::Source::SQL
     
-    attr_accessor :model, :fields, :attributes, :conditions, :groupings,
+    attr_accessor :model, :fields, :attributes, :joins, :conditions, :groupings,
       :options
-    attr_reader :base, :index
+    attr_reader :base, :index, :database_configuration
     
     def initialize(index, options = {})
       @index        = index
       @model        = index.model
       @fields       = []
       @attributes   = []
+      @joins        = []
       @conditions   = []
       @groupings    = []
       @options      = options
       @associations = {}
+      @database_configuration = @model.connection.
+        instance_variable_get(:@config).clone
       
       @base = ::ActiveRecord::Associations::ClassMethods::JoinDependency.new(
         @model, [], nil
@@ -33,30 +36,31 @@ module ThinkingSphinx
     end
     
     def name
-      @model.sphinx_name
+      index.name
     end
     
-    def to_riddle_for_core(offset, index)
+    def to_riddle_for_core(offset, position)
       source = Riddle::Configuration::SQLSource.new(
-        "#{name}_core_#{index}", adapter.sphinx_identifier
+        "#{index.core_name}_#{position}", adapter.sphinx_identifier
       )
       
       set_source_database_settings  source
       set_source_attributes         source, offset
-      set_source_sql                source, offset
       set_source_settings           source
+      set_source_sql                source, offset
       
       source
     end
     
-    def to_riddle_for_delta(offset, index)
+    def to_riddle_for_delta(offset, position)
       source = Riddle::Configuration::SQLSource.new(
-        "#{name}_delta_#{index}", adapter.sphinx_identifier
+        "#{index.delta_name}_#{position}", adapter.sphinx_identifier
       )
-      source.parent = "#{name}_core_#{index}"
+      source.parent = "#{index.core_name}_#{position}"
       
       set_source_database_settings  source
       set_source_attributes         source, offset, true
+      set_source_settings           source
       set_source_sql                source, offset, true
       
       source
@@ -79,7 +83,7 @@ module ThinkingSphinx
     end
     
     def set_source_database_settings(source)
-      config = @model.connection.instance_variable_get(:@config)
+      config = @database_configuration
       
       source.sql_host = config[:host]           || "localhost"
       source.sql_user = config[:username]       || config[:user] || 'root'
@@ -107,6 +111,7 @@ module ThinkingSphinx
       end
       
       source.sql_query_pre += [adapter.utf8_query_pre].compact if utf8?
+      source.sql_query_pre << adapter.utc_query_pre
     end
     
     def set_source_settings(source)
@@ -136,7 +141,11 @@ module ThinkingSphinx
         # attribute associations
         @attributes.collect { |attrib|
           attrib.associations.values if attrib.include_as_association?
-        }.compact.flatten
+        }.compact.flatten +
+        # explicit joins
+        @joins.collect { |join|
+          join.associations
+        }.flatten
       ).uniq.collect { |assoc|
         # get ancestors as well as column-level associations
         assoc.ancestors
@@ -144,7 +153,7 @@ module ThinkingSphinx
     end
     
     def utf8?
-      @index.options[:charset_type] == "utf-8"
+      @index.options[:charset_type] =~ /utf-8|zh_cn.utf-8/
     end
   end
 end