freelancing-god-thinking-sphinx 0.9.8 → 0.9.10

data/lib/thinking_sphinx/index.rb

@@ -9,7 +9,8 @@ module ThinkingSphinx
   # Enjoy.
   # 
   class Index
-    attr_accessor :model, :fields, :attributes, :conditions, :delta, :options
+    attr_accessor :model, :fields, :attributes, :conditions, :groupings,
+      :delta, :options
     
     # Create a new index instance by passing in the model it is tied to, and
     # a block to build it with (optional but recommended). For documentation
@@ -31,6 +32,7 @@ module ThinkingSphinx
       @fields       = []
       @attributes   = []
       @conditions   = []
+      @groupings    = []
       @options      = {}
       @delta        = false
       
@@ -38,16 +40,21 @@ module ThinkingSphinx
     end
     
     def name
+      self.class.name(@model)
+    end
+    
+    def self.name(model)
       model.name.underscore.tr(':/\\', '_')
     end
     
     def empty?(part = :core)
-      config = ThinkingSphinx::Configuration.new
+      config = ThinkingSphinx::Configuration.instance
       File.size?("#{config.searchd_file_path}/#{self.name}_#{part}.spa").nil?
     end
     
-    def to_config(index, database_conf, charset_type)
+    def to_config(model, index, database_conf, offset)
       # Set up associations and joins
+      add_internal_attributes
       link!
       
       attr_sources = attributes.collect { |attrib|
@@ -65,33 +72,36 @@ module ThinkingSphinx
       
       config = <<-SOURCE
 
-source #{model.indexes.first.name}_#{index}_core
+source #{self.class.name(model)}_#{index}_core
 {
 type     = #{db_adapter}
 sql_host = #{database_conf[:host] || "localhost"}
-sql_user = #{database_conf[:username]}
-sql_pass = #{database_conf[:password]}
+sql_user = #{database_conf[:username] || database_conf[:user]}
+sql_pass = #{(database_conf[:password] || "").gsub('#', '\#')}
 sql_db   = #{database_conf[:database]}
+#{"sql_port = #{database_conf[:port]}" unless database_conf[:port].blank? }
+#{"sql_sock = #{database_conf[:socket]}" unless database_conf[:socket].blank? }
 
-sql_query_pre    = #{charset_type == "utf-8" && adapter == :mysql ? "SET NAMES utf8" : ""}
+sql_query_pre    = #{utf8? && adapter == :mysql ? "SET NAMES utf8" : ""}
 #{"sql_query_pre    = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]}
 sql_query_pre    = #{to_sql_query_pre}
-sql_query        = #{to_sql.gsub(/\n/, ' ')}
+sql_query        = #{to_sql(:offset => offset).gsub(/\n/, ' ')}
 sql_query_range  = #{to_sql_query_range}
-sql_query_info   = #{to_sql_query_info}
+sql_query_info   = #{to_sql_query_info(offset)}
 #{attr_sources}
+#{ThinkingSphinx::Configuration.instance.hash_to_config(self.source_options)}
 }
       SOURCE
       
       if delta?
         config += <<-SOURCE
 
-source #{model.indexes.first.name}_#{index}_delta : #{model.indexes.first.name}_#{index}_core
+source #{self.class.name(model)}_#{index}_delta : #{self.class.name(model)}_#{index}_core
 {
 sql_query_pre    = 
-sql_query_pre    = #{charset_type == "utf-8" && adapter == :mysql ? "SET NAMES utf8" : ""}
+sql_query_pre    = #{utf8? && adapter == :mysql ? "SET NAMES utf8" : ""}
 #{"sql_query_pre    = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]}
-sql_query        = #{to_sql(:delta => true).gsub(/\n/, ' ')}
+sql_query        = #{to_sql(:delta => true, :offset => offset).gsub(/\n/, ' ')}
 sql_query_range  = #{to_sql_query_range :delta => true}
 }
         SOURCE
@@ -149,9 +159,16 @@ sql_query_range  = #{to_sql_query_range :delta => true}
         where_clause << " AND " << @conditions.join(" AND ")
       end
       
+      internal_groupings = []
+      if @model.column_names.include?(@model.inheritance_column)
+         internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}"
+      end
+      
+      unique_id_expr = "* #{ThinkingSphinx.indexed_models.size} + #{options[:offset] || 0}"
+      
       sql = <<-SQL
 SELECT #{ (
-  ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + 
+  ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] + 
   @fields.collect { |field| field.to_select_sql } +
   @attributes.collect { |attribute| attribute.to_select_sql }
 ).join(", ") }
@@ -163,7 +180,8 @@ WHERE #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start
 GROUP BY #{ (
   ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + 
   @fields.collect { |field| field.to_group_sql }.compact +
-  @attributes.collect { |attribute| attribute.to_group_sql }.compact
+  @attributes.collect { |attribute| attribute.to_group_sql }.compact +
+  @groupings + internal_groupings
 ).join(", ") }
       SQL
       
@@ -177,9 +195,9 @@ GROUP BY #{ (
     # Simple helper method for the query info SQL - which is a statement that
     # returns the single row for a corresponding id.
     # 
-    def to_sql_query_info
+    def to_sql_query_info(offset)
       "SELECT * FROM #{@model.quoted_table_name} WHERE " +
-      " #{quote_column(@model.primary_key)} = $id"
+      " #{quote_column(@model.primary_key)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})"
     end
     
     # Simple helper method for the query range SQL - which is a statement that
@@ -228,6 +246,10 @@ GROUP BY #{ (
       end
     end
     
+    def adapter_object
+      @adapter_object ||= ThinkingSphinx::AbstractAdapter.detect(@model)
+    end
+    
     def prefix_fields
       @fields.select { |field| field.prefixes }
     end
@@ -236,8 +258,37 @@ GROUP BY #{ (
       @fields.select { |field| field.infixes }
     end
     
+    def local_index_options
+      @options.keys.inject({}) do |local_options, key|
+        if ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s)
+          local_options[key.to_sym] = @options[key]
+        end
+        local_options
+      end
+    end
+    
+    def index_options
+      all_index_options = ThinkingSphinx::Configuration.instance.index_options.clone
+      @options.keys.select { |key|
+        ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s)
+      }.each { |key| all_index_options[key.to_sym] = @options[key] }
+      all_index_options
+    end
+    
+    def source_options
+      all_source_options = ThinkingSphinx::Configuration.instance.source_options.clone
+      @options.keys.select { |key|
+        ThinkingSphinx::Configuration::SourceOptions.include?(key.to_s)
+      }.each { |key| all_source_options[key.to_sym] = @options[key] }
+      all_source_options
+    end
+    
     private
     
+    def utf8?
+      self.index_options[:charset_type] == "utf-8"
+    end
+    
     def quote_column(column)
       @model.connection.quote_column_name(column)
     end
@@ -263,19 +314,20 @@ GROUP BY #{ (
       @fields     = builder.fields
       @attributes = builder.attributes
       @conditions = builder.conditions
+      @groupings  = builder.groupings
       @delta      = builder.properties[:delta]
       @options    = builder.properties.except(:delta)
       
-      @attributes << Attribute.new(
-        FauxColumn.new(@model.to_crc32.to_s),
-        :type => :integer,
-        :as   => :class_crc
-      )
-      @attributes << Attribute.new(
-        FauxColumn.new("0"),
-        :type => :integer,
-        :as   => :sphinx_deleted
-      )
+      # We want to make sure that if the database doesn't exist, then Thinking
+      # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop
+      # and db:migrate). It's a bit hacky, but I can't think of a better way.
+    rescue StandardError => err
+      case err.class.name
+      when "Mysql::Error", "ActiveRecord::StatementInvalid"
+        return
+      else
+        raise err
+      end
     end
     
     # Returns all associations used amongst all the fields and attributes.
@@ -335,5 +387,46 @@ GROUP BY #{ (
         val ? '1' : '0'
       end
     end
+    
+    def crc_column
+      if @model.column_names.include?(@model.inheritance_column)
+        case adapter
+        when :postgres
+          "COALESCE(crc32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})"
+        when :mysql
+          "IFNULL(CRC32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})"
+        end
+      else
+        @model.to_crc32.to_s
+      end
+    end
+    
+    def add_internal_attributes
+      @attributes << Attribute.new(
+        FauxColumn.new(@model.primary_key.to_sym),
+        :type => :integer,
+        :as   => :sphinx_internal_id
+      ) unless @attributes.detect { |attr| attr.alias == :sphinx_internal_id }
+      
+      @attributes << Attribute.new(
+        FauxColumn.new(crc_column),
+        :type => :integer,
+        :as   => :class_crc
+      ) unless @attributes.detect { |attr| attr.alias == :class_crc }
+      
+      @attributes << Attribute.new(
+        FauxColumn.new("'" + (@model.send(:subclasses).collect { |klass|
+          klass.to_crc32.to_s
+        } << @model.to_crc32.to_s).join(",") + "'"),
+        :type => :multi,
+        :as   => :subclass_crcs
+      ) unless @attributes.detect { |attr| attr.alias == :subclass_crcs }
+      
+      @attributes << Attribute.new(
+        FauxColumn.new("0"),
+        :type => :integer,
+        :as   => :sphinx_deleted
+      ) unless @attributes.detect { |attr| attr.alias == :sphinx_deleted }
+    end
   end
 end

data/lib/thinking_sphinx/index/builder.rb

@@ -18,9 +18,13 @@ module ThinkingSphinx
         # rails documentation. It's not needed though, so it gets undef'd.
         # Hopefully the list of methods that get in the way doesn't get too
         # long.
-        undef_method :parent
+        undef_method :parent  if respond_to?(:parent)
+        undef_method :name    if respond_to?(:name)
+        undef_method :id      if respond_to?(:id)
+        undef_method :type    if respond_to?(:type)
         
-        attr_accessor :fields, :attributes, :properties, :conditions
+        attr_accessor :fields, :attributes, :properties, :conditions,
+          :groupings
         
         # Set up all the collections. Consider this the equivalent of an
         # instance's initialize method.
@@ -30,6 +34,7 @@ module ThinkingSphinx
           @attributes = []
           @properties = {}
           @conditions = []
+          @groupings  = []
         end
         
         # This is how you add fields - the strings Sphinx looks at - to your
@@ -81,8 +86,7 @@ module ThinkingSphinx
         def indexes(*args)
           options = args.extract_options!
           args.each do |columns|
-            columns = FauxColumn.new(columns) if columns.is_a?(Symbol)
-            fields << Field.new(columns, options)
+            fields << Field.new(FauxColumn.coerce(columns), options)
             
             if fields.last.sortable
               attributes << Attribute.new(
@@ -137,23 +141,7 @@ module ThinkingSphinx
         def has(*args)
           options = args.extract_options!
           args.each do |columns|
-            columns = case columns
-            when Symbol, String
-              FauxColumn.new(columns)
-            when Array
-              columns.collect { |col|
-                case col
-                when Symbol, String
-                  FauxColumn.new(col)
-                else
-                  col
-                end
-              }
-            else
-              columns
-            end
-            
-            attributes << Attribute.new(columns, options)
+            attributes << Attribute.new(FauxColumn.coerce(columns), options)
           end
         end
         alias_method :attribute, :has
@@ -169,6 +157,16 @@ module ThinkingSphinx
           @conditions += args
         end
         
+        # Use this method to add some manual SQL strings to the GROUP BY
+        # clause. You can pass in as many strings as you'd like, they'll get
+        # joined together with commas later on.
+        # 
+        #   group_by "lat", "lng"
+        # 
+        def group_by(*args)
+          @groupings += args
+        end
+        
         # This is what to use to set properties on the index. Chief amongst
         # those is the delta property - to allow automatic updates to your
         # indexes as new models are added and edited - but also you can
@@ -185,7 +183,7 @@ module ThinkingSphinx
         # when defining the index, so you don't need to specify them for every
         # geo-related search.
         #
-        #   set_property :latitude_attr => "lt", :longitude => "lg"
+        #   set_property :latitude_attr => "lt", :longitude_attr => "lg"
         # 
         # Please don't forget to add a boolean field named 'delta' to your
         # model's database table if enabling the delta index for it.
@@ -206,6 +204,16 @@ module ThinkingSphinx
         def method_missing(method, *args)
           FauxColumn.new(method, *args)
         end
+        
+        # A method to allow adding fields from associations which have names
+        # that clash with method names in the Builder class (ie: properties,
+        # fields, attributes).
+        # 
+        # Example: indexes assoc(:properties).column
+        # 
+        def assoc(assoc)
+          FauxColumn.new(method)
+        end
       end
     end
   end

data/lib/thinking_sphinx/index/faux_column.rb

@@ -15,6 +15,19 @@ module ThinkingSphinx
         @stack = stack
       end
       
+      def self.coerce(columns)
+        case columns
+        when Symbol, String
+          FauxColumn.new(columns)
+        when Array
+          columns.collect { |col| FauxColumn.coerce(col) }
+        when FauxColumn
+          columns
+        else
+          nil
+        end
+      end
+      
       # Can't use normal method name, as that could be an association or
       # column name.
       # 

data/lib/thinking_sphinx/rails_additions.rb

@@ -53,4 +53,16 @@ end
 
 ActiveRecord::Base.extend(
   ThinkingSphinx::ActiveRecordQuotedName
-) unless ActiveRecord::Base.respond_to?("quoted_table_name")
+) unless ActiveRecord::Base.respond_to?("quoted_table_name")
+
+module ThinkingSphinx
+  module ActiveRecordStoreFullSTIClass
+    def store_full_sti_class
+      false
+    end
+  end
+end
+
+ActiveRecord::Base.extend(
+  ThinkingSphinx::ActiveRecordStoreFullSTIClass
+) unless ActiveRecord::Base.respond_to?(:store_full_sti_class)

data/lib/thinking_sphinx/search.rb

@@ -17,16 +17,8 @@ module ThinkingSphinx
         
         options = args.extract_options!
         page    = options[:page] ? options[:page].to_i : 1
-        
-        begin
-          pager = WillPaginate::Collection.create(page,
-            client.limit, results[:total_found] || 0) do |collection|
-            collection.replace results[:matches].collect { |match| match[:doc] }
-            collection.instance_variable_set :@total_entries, results[:total_found]
-          end
-        rescue
-          results[:matches].collect { |match| match[:doc] }
-        end
+
+        ThinkingSphinx::Collection.ids_from_results(results, page, client.limit, options)
       end
 
       # Searches through the Sphinx indexes for relevant matches. There's
@@ -149,8 +141,8 @@ module ThinkingSphinx
       # attributes. To search with that point, you can then use one of the
       # following syntax examples:
       # 
-      #   Address.search "Melbourne", :geo => [1.4, -2.217]
-      #   Address.search "Australia", :geo => [-0.55, 3.108],
+      #   Address.search "Melbourne", :geo => [1.4, -2.217], :order => "@geodist asc"
+      #   Address.search "Australia", :geo => [-0.55, 3.108], :order => "@geodist asc"
       #     :latitude_attr => "latit", :longitude_attr => "longit"
       # 
       # The first example applies when your latitude and longitude attributes
@@ -168,9 +160,9 @@ module ThinkingSphinx
       # 
       # Now, geo-location searching really only has an affect if you have a
       # filter, sort or grouping clause related to it - otherwise it's just a
-      # normal search. To make use of the positioning difference, use the
-      # special attribute "@geodist" in any of your filters or sorting or grouping
-      # clauses.
+      # normal search, and _will not_ return a distance value otherwise. To
+      # make use of the positioning difference, use the special attribute
+      # "@geodist" in any of your filters or sorting or grouping clauses.
       # 
       # And don't forget - both the latitude and longitude you use in your
       # search, and the values in your indexes, need to be stored as a float in radians,
@@ -182,6 +174,16 @@ module ThinkingSphinx
       #     # ...
       #   end
       # 
+      # Once you've got your results set, you can access the distances as
+      # follows:
+      # 
+      # @results.each_with_geodist do |result, distance|
+      #   # ...
+      # end
+      # 
+      # The distance value is returned as a float, representing the distance in
+      # metres.
+      # 
       def search(*args)
         results, client = search_results(*args.clone)
         
@@ -193,17 +195,14 @@ module ThinkingSphinx
         klass   = options[:class]
         page    = options[:page] ? options[:page].to_i : 1
         
-        begin
-          pager = WillPaginate::Collection.create(page,
-            client.limit, results[:total] || 0) do |collection|
-            collection.replace instances_from_results(results[:matches], options, klass)
-            collection.instance_variable_set :@total_entries, results[:total_found]
-          end
-        rescue StandardError => err
-          instances_from_results(results[:matches], options, klass)
-        end
+        ThinkingSphinx::Collection.create_from_results(results, page, client.limit, options)
       end
-      
+
+      def count(*args)
+        results, client = search_results(*args.clone)
+        results[:total] || 0
+      end
+
       # Checks if a document with the given id exists within a specific index.
       # Expected parameters:
       #
@@ -259,7 +258,7 @@ module ThinkingSphinx
         begin
           ::ActiveRecord::Base.logger.debug "Sphinx: #{query}"
           results = client.query query
-          ::ActiveRecord::Base.logger.debug "Sphinx Result: #{results[:matches].collect{|m| m[:doc]}.inspect}"
+          ::ActiveRecord::Base.logger.debug "Sphinx Result: #{results[:matches].collect{|m| m[:attributes]["sphinx_internal_id"]}.inspect}"
         rescue Errno::ECONNREFUSED => err
           raise ThinkingSphinx::ConnectionError, "Connection to Sphinx Daemon (searchd) failed."
         end
@@ -267,53 +266,14 @@ module ThinkingSphinx
         return results, client
       end
       
-      def instances_from_results(results, options = {}, klass = nil)
-        if klass.nil?
-          results.collect { |result| instance_from_result result, options }
-        else
-          ids = results.collect { |result| result[:doc] }
-          instances = klass.find(
-            :all,
-            :conditions => {klass.primary_key.to_sym => ids},
-            :include    => options[:include],
-            :select     => options[:select]
-          )
-          ids.collect { |obj_id| instances.detect { |obj| obj.id == obj_id } }
-        end
-      end
-      
-      # Either use the provided class to instantiate a result from a model, or
-      # get the result's CRC value and determine the class from that.
-      # 
-      def instance_from_result(result, options)
-        class_from_crc(result[:attributes]["class_crc"]).find(
-          result[:doc], :include => options[:include], :select => options[:select]
-        )
-      end
-      
-      # Convert a CRC value to the corresponding class.
-      # 
-      def class_from_crc(crc)
-        unless @models_by_crc
-          Configuration.new.load_models
-          
-          @models_by_crc = ThinkingSphinx.indexed_models.inject({}) do |hash, model|
-            hash[model.constantize.to_crc32] = model
-            hash
-          end
-        end
-        
-        @models_by_crc[crc].constantize
-      end
-      
       # Set all the appropriate settings for the client, using the provided
       # options hash.
       # 
       def client_from_options(options = {})
-        config = ThinkingSphinx::Configuration.new
+        config = ThinkingSphinx::Configuration.instance
         client = Riddle::Client.new config.address, config.port
         klass  = options[:class]
-        index_options = klass ? klass.indexes.last.options : {}
+        index_options = klass ? klass.sphinx_indexes.last.options : {}
         
         [
           :max_matches, :match_mode, :sort_mode, :sort_by, :id_range,
@@ -327,14 +287,17 @@ module ThinkingSphinx
           )
         end
         
+        options[:classes] = [klass] if klass
+        
         client.anchor = anchor_conditions(klass, options) || {} if client.anchor.empty?
         
         client.filters << Riddle::Client::Filter.new(
           "sphinx_deleted", [0]
         )
+        
         # class filters
         client.filters << Riddle::Client::Filter.new(
-          "class_crc", options[:classes].collect { |klass| klass.to_crc32 }
+          "class_crc", options[:classes].collect { |k| k.to_crc32s }.flatten
         ) if options[:classes]
         
         # normal attribute filters
@@ -365,7 +328,7 @@ module ThinkingSphinx
       # and filters.
       # 
       def search_conditions(klass, conditions={})
-        attributes = klass ? klass.indexes.collect { |index|
+        attributes = klass ? klass.sphinx_indexes.collect { |index|
           index.attributes.collect { |attrib| attrib.unique_name }
         }.flatten : []
         
@@ -375,18 +338,13 @@ module ThinkingSphinx
         conditions.each do |key,val|
           if attributes.include?(key.to_sym)
             filters << Riddle::Client::Filter.new(
-              key.to_s,
-              val.is_a?(Range) ? val : Array(val)
+              key.to_s, filter_value(val)
             )
           else
             search_string << "@#{key} #{val} "
           end
         end
         
-        filters << Riddle::Client::Filter.new(
-          "class_crc", [klass.to_crc32]
-        ) if klass
-        
         return search_string, filters
       end
       
@@ -395,15 +353,15 @@ module ThinkingSphinx
       # there's actually any values.
       # 
       def anchor_conditions(klass, options)
-        attributes = klass ? klass.indexes.collect { |index|
+        attributes = klass ? klass.sphinx_indexes.collect { |index|
           index.attributes.collect { |attrib| attrib.unique_name }
         }.flatten : []
         
-        lat_attr = klass ? klass.indexes.collect { |index|
+        lat_attr = klass ? klass.sphinx_indexes.collect { |index|
           index.options[:latitude_attr]
         }.compact.first : nil
         
-        lon_attr = klass ? klass.indexes.collect { |index|
+        lon_attr = klass ? klass.sphinx_indexes.collect { |index|
           index.options[:longitude_attr]
         }.compact.first : nil
         
@@ -412,6 +370,7 @@ module ThinkingSphinx
         lat_attr ||= :latitude  if attributes.include?(:latitude)
         
         lon_attr = options[:longitude_attr] if options[:longitude_attr]
+        lon_attr ||= :lng       if attributes.include?(:lng)
         lon_attr ||= :lon       if attributes.include?(:lon)
         lon_attr ||= :long      if attributes.include?(:long)
         lon_attr ||= :longitude if attributes.include?(:longitude)
@@ -425,9 +384,9 @@ module ThinkingSphinx
         end
         
         lat && lon ? {
-          :latitude_attribute   => lat_attr,
+          :latitude_attribute   => lat_attr.to_s,
           :latitude             => lat,
-          :longitude_attribute  => lon_attr,
+          :longitude_attribute  => lon_attr.to_s,
           :longitude            => lon
         } : nil
       end
@@ -437,7 +396,7 @@ module ThinkingSphinx
       # 
       def set_sort_options!(client, options)
         klass = options[:class]
-        fields = klass ? klass.indexes.collect { |index|
+        fields = klass ? klass.sphinx_indexes.collect { |index|
           index.fields.collect { |field| field.unique_name }
         }.flatten : []