dpickett-thinking-sphinx 1.1.4 → 1.1.12

data/lib/thinking_sphinx/index.rb

@@ -37,6 +37,19 @@ module ThinkingSphinx
       @delta_object = nil
       
       initialize_from_builder(&block) if block_given?
+      
+      add_internal_attributes_and_facets
+      
+      # We want to make sure that if the database doesn't exist, then Thinking
+      # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop
+      # and db:migrate). It's a bit hacky, but I can't think of a better way.
+    rescue StandardError => err
+      case err.class.name
+      when "Mysql::Error", "Java::JavaSql::SQLException", "ActiveRecord::StatementInvalid"
+        return
+      else
+        raise err
+      end
     end
     
     def name
@@ -48,7 +61,6 @@ module ThinkingSphinx
     end
     
     def to_riddle_for_core(offset, index)
-      add_internal_attributes
       link!
       
       source = Riddle::Configuration::SQLSource.new(
@@ -56,7 +68,7 @@ module ThinkingSphinx
       )
       
       set_source_database_settings  source
-      set_source_attributes         source
+      set_source_attributes         source, offset
       set_source_sql                source, offset
       set_source_settings           source
       
@@ -64,7 +76,6 @@ module ThinkingSphinx
     end
     
     def to_riddle_for_delta(offset, index)
-      add_internal_attributes
       link!
       
       source = Riddle::Configuration::SQLSource.new(
@@ -73,7 +84,7 @@ module ThinkingSphinx
       source.parent = "#{name}_core_#{index}"
       
       set_source_database_settings  source
-      set_source_attributes         source
+      set_source_attributes         source, offset
       set_source_sql                source, offset, true
       
       source
@@ -175,17 +186,6 @@ module ThinkingSphinx
       @model.sphinx_facets ||= []
       @fields.select(    &is_faceted).each &add_facet
       @attributes.select(&is_faceted).each &add_facet
-      
-      # We want to make sure that if the database doesn't exist, then Thinking
-      # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop
-      # and db:migrate). It's a bit hacky, but I can't think of a better way.
-    rescue StandardError => err
-      case err.class.name
-      when "Mysql::Error", "ActiveRecord::StatementInvalid"
-        return
-      else
-        raise err
-      end
     end
     
     # Returns all associations used amongst all the fields and attributes.
@@ -200,8 +200,8 @@ module ThinkingSphinx
         }.flatten +
         # attribute associations
         @attributes.collect { |attrib|
-          attrib.associations.values
-        }.flatten
+          attrib.associations.values if attrib.include_as_association?
+        }.compact.flatten
       ).uniq.collect { |assoc|
         # get ancestors as well as column-level associations
         assoc.ancestors
@@ -238,7 +238,7 @@ module ThinkingSphinx
     def crc_column
       if @model.column_names.include?(@model.inheritance_column)
         adapter.cast_to_unsigned(adapter.convert_nulls(
-          adapter.crc(adapter.quote_with_table(@model.inheritance_column)),
+          adapter.crc(adapter.quote_with_table(@model.inheritance_column), true),
           @model.to_crc32
         ))
       else
@@ -246,50 +246,45 @@ module ThinkingSphinx
       end
     end
     
-    def add_internal_attributes
-      @attributes << Attribute.new(
-        FauxColumn.new(@model.primary_key.to_sym),
-        :type => :integer,
-        :as   => :sphinx_internal_id
-      ) unless @attributes.detect { |attr| attr.alias == :sphinx_internal_id }
-      
-      unless @attributes.detect { |attr| attr.alias == :class_crc }
-        @attributes << Attribute.new(
-          FauxColumn.new(crc_column),
-          :type => :integer,
-          :as   => :class_crc,
-          :facet => true
-        ) 
-      
-        @model.sphinx_facets << ThinkingSphinx::ClassFacet.new(@attributes.last)
-      end
-      
-      if @model.column_names.include?(@model.inheritance_column)
-        class_col = FauxColumn.new(
-          adapter.convert_nulls(adapter.quote_with_table(@model.inheritance_column), @model.to_s)
-        )
-      else
-        class_col = FauxColumn.new("'#{@model.to_s}'")
-      end
+    def add_internal_attributes_and_facets
+      add_internal_attribute :sphinx_internal_id, :integer, @model.primary_key.to_sym
+      add_internal_attribute :class_crc,          :integer, crc_column, true
+      add_internal_attribute :subclass_crcs,      :multi,   subclasses_to_s
+      add_internal_attribute :sphinx_deleted,     :integer, "0"
       
-      @attributes << Attribute.new(class_col,
-        :type => :string,
-        :as   => :class
-      )
+      add_internal_facet :class_crc
+    end
+    
+    def add_internal_attribute(name, type, contents, facet = false)
+      return unless attribute_by_alias(name).nil?
       
       @attributes << Attribute.new(
-        FauxColumn.new("'" + (@model.send(:subclasses).collect { |klass|
-          klass.to_crc32.to_s
-        } << @model.to_crc32.to_s).join(",") + "'"),
-        :type => :multi,
-        :as   => :subclass_crcs
-      ) unless @attributes.detect { |attr| attr.alias == :subclass_crcs }
+        FauxColumn.new(contents),
+        :type   => type,
+        :as     => name,
+        :facet  => facet,
+        :admin  => true
+      )
+    end
+    
+    def add_internal_facet(name)
+      return unless facet_by_alias(name).nil?
       
-      @attributes << Attribute.new(
-        FauxColumn.new("0"),
-        :type => :integer,
-        :as   => :sphinx_deleted
-      ) unless @attributes.detect { |attr| attr.alias == :sphinx_deleted }
+      @model.sphinx_facets << ClassFacet.new(attribute_by_alias(name))
+    end
+    
+    def attribute_by_alias(attr_alias)
+      @attributes.detect { |attrib| attrib.alias == attr_alias }
+    end
+    
+    def facet_by_alias(name)
+      @model.sphinx_facets.detect { |facet| facet.name == name }
+    end
+    
+    def subclasses_to_s
+      "'" + (@model.send(:subclasses).collect { |klass|
+        klass.to_crc32.to_s
+      } << @model.to_crc32.to_s).join(",") + "'"
     end
         
     def set_source_database_settings(source)
@@ -303,9 +298,9 @@ module ThinkingSphinx
       source.sql_sock = config[:socket]
     end
     
-    def set_source_attributes(source)
+    def set_source_attributes(source, offset = nil)
       attributes.each do |attrib|
-        source.send(attrib.type_to_config) << attrib.config_value
+        source.send(attrib.type_to_config) << attrib.config_value(offset)
       end
     end
     
@@ -372,14 +367,14 @@ module ThinkingSphinx
          internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}"
       end
       
-      unique_id_expr = "* #{ThinkingSphinx.indexed_models.size} + #{options[:offset] || 0}"
+      unique_id_expr = ThinkingSphinx.unique_id_expression(options[:offset])
       
       sql = <<-SQL
 SELECT #{ (
   ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] + 
   @fields.collect { |field| field.to_select_sql } +
   @attributes.collect { |attribute| attribute.to_select_sql }
-).join(", ") }
+).compact.join(", ") }
 FROM #{ @model.table_name }
   #{ assocs.collect { |assoc| assoc.to_sql }.join(' ') }
 WHERE #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start
@@ -393,10 +388,7 @@ GROUP BY #{ (
 ).join(", ") }
       SQL
       
-      if @model.connection.class.name == "ActiveRecord::ConnectionAdapters::MysqlAdapter"
-        sql += " ORDER BY NULL"
-      end
-      
+      sql += " ORDER BY NULL" if adapter.sphinx_identifier == "mysql"
       sql
     end
     

data/lib/thinking_sphinx/property.rb

@@ -0,0 +1,133 @@
+module ThinkingSphinx
+  class Property
+    attr_accessor :alias, :columns, :associations, :model, :faceted, :admin
+    
+    def initialize(columns, options = {})
+      @columns      = Array(columns)
+      @associations = {}
+
+      raise "Cannot define a field with no columns. Maybe you are trying to index a field with a reserved name (id, name). You can fix this error by using a symbol rather than a bare name (:id instead of id)." if @columns.empty? || @columns.any? { |column| !column.respond_to?(:__stack) }
+      
+      @alias    = options[:as]
+      @faceted  = options[:facet]
+      @admin    = options[:admin]
+    end
+    
+    # Returns the unique name of the attribute - which is either the alias of
+    # the attribute, or the name of the only column - if there is only one. If
+    # there isn't, there should be an alias. Else things probably won't work.
+    # Consider yourself warned.
+    # 
+    def unique_name
+      if @columns.length == 1
+        @alias || @columns.first.__name
+      else
+        @alias
+      end
+    end
+    
+    def to_facet
+      return nil unless @faceted
+      
+      ThinkingSphinx::Facet.new(self)
+    end
+    
+    # Get the part of the GROUP BY clause related to this attribute - if one is
+    # needed. If not, all you'll get back is nil. The latter will happen if
+    # there isn't actually a real column to get data from, or if there's
+    # multiple data values (read: a has_many or has_and_belongs_to_many
+    # association).
+    # 
+    def to_group_sql
+      case
+      when is_many?, is_string?, ThinkingSphinx.use_group_by_shortcut?
+        nil
+      else
+        @columns.collect { |column|
+          column_with_prefix(column)
+        }
+      end
+    end
+    
+    def changed?(instance)
+      return true if is_string? || @columns.any? { |col| !col.__stack.empty? }
+      
+      !@columns.all? { |col|
+        instance.respond_to?("#{col.__name.to_s}_changed?") &&
+        !instance.send("#{col.__name.to_s}_changed?")
+      }
+    end
+    
+    def admin?
+      admin
+    end
+    
+    def public?
+      !admin
+    end
+    
+    private
+    
+    # Could there be more than one value related to the parent record? If so,
+    # then this will return true. If not, false. It's that simple.
+    # 
+    def is_many?
+      associations.values.flatten.any? { |assoc| assoc.is_many? }
+    end
+    
+    # Returns true if any of the columns are string values, instead of database
+    # column references.
+    def is_string?
+      columns.all? { |col| col.is_string? }
+    end
+    
+    def adapter
+      @adapter ||= @model.sphinx_database_adapter
+    end
+    
+    def quote_with_table(table, column)
+      "#{quote_table_name(table)}.#{quote_column(column)}"
+    end
+    
+    def quote_column(column)
+      @model.connection.quote_column_name(column)
+    end
+    
+    def quote_table_name(table_name)
+      @model.connection.quote_table_name(table_name)
+    end
+    
+    # Indication of whether the columns should be concatenated with a space
+    # between each value. True if there's either multiple sources or multiple
+    # associations.
+    # 
+    def concat_ws?
+      multiple_associations? || @columns.length > 1
+    end
+        
+    # Checks whether any column requires multiple associations (which only
+    # happens for polymorphic situations).
+    # 
+    def multiple_associations?
+      associations.any? { |col,assocs| assocs.length > 1 }
+    end
+    
+    # Builds a column reference tied to the appropriate associations. This
+    # dives into the associations hash and their corresponding joins to
+    # figure out how to correctly reference a column in SQL.
+    # 
+    def column_with_prefix(column)
+      if column.is_string?
+        column.__name
+      elsif associations[column].empty?
+        "#{@model.quoted_table_name}.#{quote_column(column.__name)}"
+      else
+        associations[column].collect { |assoc|
+          assoc.has_column?(column.__name) ?
+          "#{quote_with_table(assoc.join.aliased_table_name, column.__name)}" :
+          nil
+        }.compact.join(', ')
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/rails_additions.rb

@@ -49,10 +49,13 @@ module ThinkingSphinx
   end
 end
 
-if ActiveRecord::ConnectionAdapters.constants.include?("MysqlAdapter")
-  ActiveRecord::ConnectionAdapters::MysqlAdapter.send(
-    :include, ThinkingSphinx::MysqlQuotedTableName
-  ) unless ActiveRecord::ConnectionAdapters::MysqlAdapter.instance_methods.include?("quote_table_name")
+if ActiveRecord::ConnectionAdapters.constants.include?("MysqlAdapter") or ActiveRecord::Base.respond_to?(:jdbcmysql_connection)
+  adapter = ActiveRecord::ConnectionAdapters.const_get(
+    defined?(JRUBY_VERSION) ? :JdbcAdapter : :MysqlAdapter
+  )
+  unless adapter.instance_methods.include?("quote_table_name")
+    adapter.send(:include, ThinkingSphinx::MysqlQuotedTableName)
+  end
 end
 
 module ThinkingSphinx

data/lib/thinking_sphinx/search.rb

@@ -1,4 +1,4 @@
-module ThinkingSphinx  
+module ThinkingSphinx
   # Once you've got those indexes in and built, this is the stuff that
   # matters - how to search! This class provides a generic search
   # interface - which you can use to search all your indexed models at once.
@@ -7,6 +7,11 @@ module ThinkingSphinx
   # called from a model.
   # 
   class Search
+    GlobalFacetOptions = {
+      :all_attributes => false,
+      :class_facet    => true
+    }
+    
     class << self
       # Searches for results that match the parameters provided. Will only
       # return the ids for the matching objects. See #search for syntax
@@ -94,16 +99,24 @@ module ThinkingSphinx
       # == Searching by Attributes
       #
       # Also known as filters, you can limit your searches to documents that
-      # have specific values for their attributes. There are two ways to do
-      # this. The first is one that works in all scenarios - using the :with
-      # option.
-      #
-      #   ThinkingSphinx::Search.search :with => {:parent_id => 10}
-      #
-      # The second is only viable if you're searching with a specific model
-      # (not multi-model searching). With a single model, Thinking Sphinx
-      # can figure out what attributes and fields are available, so you can
-      # put it all in the :conditions hash, and it will sort it out.
+      # have specific values for their attributes. There are three ways to do
+      # this. The first two techniques work in all scenarios - using the :with
+      # or :with_all options.
+      #
+      #   ThinkingSphinx::Search.search :with => {:tag_ids => 10}
+      #   ThinkingSphinx::Search.search :with => {:tag_ids => [10,12]}
+      #   ThinkingSphinx::Search.search :with_all => {:tag_ids => [10,12]}
+      #
+      # The first :with search will match records with a tag_id attribute of 10.
+      # The second :with will match records with a tag_id attribute of 10 OR 12.
+      # If you need to find records that are tagged with ids 10 AND 12, you
+      # will need to use the :with_all search parameter. This is particuarly
+      # useful in conjunction with Multi Value Attributes (MVAs).
+      #
+      # The third filtering technique is only viable if you're searching with a
+      # specific model (not multi-model searching). With a single model,
+      # Thinking Sphinx can figure out what attributes and fields are available,
+      # so you can put it all in the :conditions hash, and it will sort it out.
       # 
       #   Node.search :conditions => {:parent_id => 10}
       # 
@@ -186,6 +199,12 @@ module ThinkingSphinx
       # documentation[http://sphinxsearch.com/doc.html] for that level of
       # detail though.
       #
+      # If desired, you can sort by a column in your model instead of a sphinx
+      # field or attribute. This sort only applies to the current page, so is
+      # most useful when performing a search with a single page of results.
+      #
+      #   User.search("pat", :sql_order => "name")
+      #
       # == Grouping
       # 
       # For this you can use the group_by, group_clause and group_function
@@ -194,7 +213,66 @@ module ThinkingSphinx
       # you read all the relevant
       # documentation[http://sphinxsearch.com/doc.html#clustering] first.
       # 
-      # Yes this section will be expanded, but this is a start.
+      # Grouping is done via three parameters within the options hash
+      # * <tt>:group_function</tt> determines the way grouping is done
+      # * <tt>:group_by</tt> determines the field which is used for grouping
+      # * <tt>:group_clause</tt> determines the sorting order 
+      #
+      # === group_function
+      #  
+      # Valid values for :group_function are
+      # * <tt>:day</tt>, <tt>:week</tt>, <tt>:month</tt>, <tt>:year</tt> - Grouping is done by the respective timeframes. 
+      # * <tt>:attr</tt>, <tt>:attrpair</tt> - Grouping is done by the specified attributes(s)
+      # 
+      # === group_by
+      #
+      # This parameter denotes the field by which grouping is done. Note that the
+      # specified field must be a sphinx attribute or index.
+      #
+      # === group_clause
+      #
+      # This determines the sorting order of the groups. In a grouping search,
+      # the matches within a group will sorted by the <tt>:sort_mode</tt> and <tt>:order</tt> parameters.
+      # The group matches themselves however, will be sorted by <tt>:group_clause</tt>. 
+      # 
+      # The syntax for this is the same as an order parameter in extended sort mode.
+      # Namely, you can specify an SQL-like sort expression with up to 5 attributes 
+      # (including internal attributes), eg: "@relevance DESC, price ASC, @id DESC"
+      #
+      # === Grouping by timestamp
+      # 
+      # Timestamp grouping groups off items by the day, week, month or year of the
+      # attribute given. In order to do this you need to define a timestamp attribute,
+      # which pretty much looks like the standard defintion for any attribute.
+      #
+      #   define_index do
+      #     #
+      #     # All your other stuff
+      #     #
+      #     has :created_at
+      #   end
+      #
+      # When you need to fire off your search, it'll go something to the tune of
+      #   
+      #   Fruit.search "apricot", :group_function => :day, :group_by => 'created_at'
+      #
+      # The <tt>@groupby</tt> special attribute will contain the date for that group.
+      # Depending on the <tt>:group_function</tt> parameter, the date format will be
+      #
+      # * <tt>:day</tt> - YYYYMMDD
+      # * <tt>:week</tt> - YYYYNNN (NNN is the first day of the week in question, 
+      #   counting from the start of the year )
+      # * <tt>:month</tt> - YYYYMM
+      # * <tt>:year</tt> - YYYY
+      #
+      #
+      # === Grouping by attribute
+      #
+      # The syntax is the same as grouping by timestamp, except for the fact that the 
+      # <tt>:group_function</tt> parameter is changed
+      #
+      #   Fruit.search "apricot", :group_function => :attr, :group_by => 'size'
+      # 
       #
       # == Geo/Location Searching
       #
@@ -291,8 +369,10 @@ module ThinkingSphinx
       def retry_search_on_stale_index(query, options, &block)
         stale_ids = []
         stale_retries_left = case options[:retry_stale]
-                              when true:       3  # default to three retries
-                              when nil, false: 0  # no retries
+                              when true
+                                3  # default to three retries
+                              when nil, false
+                                0  # no retries
                               else             options[:retry_stale].to_i
                               end
         begin
@@ -352,43 +432,23 @@ module ThinkingSphinx
         end
       end
       
+      # Model.facets *args
+      # ThinkingSphinx::Search.facets *args
+      # ThinkingSphinx::Search.facets *args, :all_attributes  => true
+      # ThinkingSphinx::Search.facets *args, :class_facet     => false
+      # 
       def facets(*args)
-        hash    = ThinkingSphinx::FacetCollection.new args
-        options = args.extract_options!.clone.merge! :group_function => :attr
-        
-        klasses = options[:classes] || [options[:class]]
-        klasses = [] if options[:class].nil?
-        
-        #no classes specified so get classes from resultset
-        if klasses.empty?
-          options[:group_by] = "class_crc"
-          results = search(*(args + [options]))
-
-          hash[:class] = {}
-          results.each_with_groupby_and_count do |result, group, count|
-            hash[:class][result.class.name] = count
-            klasses << result.class
-          end
-        end
+        options = args.extract_options!
         
-        klasses.each do |klass|
-          klass.sphinx_facets.inject(hash) do |hash, facet|
-            if facet.name != :class || options[:include_class_facets]
-              hash.add_from_results facet, 
-                search(*(args + 
-                  [options.merge(:group_by => facet.attribute_name)]))
-            end
-
-            hash
-          end
+        if options[:class]
+          facets_for_model options[:class], args, options
+        else
+          facets_for_all_models args, options
         end
-        
-        hash
       end
       
       private
       
-      
       # This method handles the common search functionality, and returns both
       # the result hash and the client. Not super elegant, but it'll do for
       # the moment.
@@ -412,6 +472,7 @@ module ThinkingSphinx
         
         client.limit  = options[:per_page].to_i if options[:per_page]
         page          = options[:page] ? options[:page].to_i : 1
+        page          = 1 if page <= 0
         client.offset = (page - 1) * client.limit
 
         begin
@@ -492,6 +553,13 @@ module ThinkingSphinx
           Riddle::Client::Filter.new attr.to_s, filter_value(val), true
         } if options[:without]
         
+        # every-match attribute filters
+        client.filters += options[:with_all].collect { |attr,vals|
+          Array(vals).collect { |val|
+            Riddle::Client::Filter.new attr.to_s, filter_value(val)
+          }
+        }.flatten if options[:with_all]
+        
         # exclusive attribute filter on primary key
         client.filters += Array(options[:without_ids]).collect { |id|
           Riddle::Client::Filter.new 'sphinx_internal_id', filter_value(id), true
@@ -649,6 +717,75 @@ module ThinkingSphinx
         
         string
       end
+      
+      def facets_for_model(klass, args, options)
+        hash    = ThinkingSphinx::FacetCollection.new args + [options]
+        options = options.clone.merge! facet_query_options
+        
+        klass.sphinx_facets.inject(hash) do |hash, facet|
+          unless facet.name == :class && !options[:class_facet]
+            options[:group_by]    = facet.attribute_name
+            hash.add_from_results facet, search(*(args + [options]))
+          end
+          
+          hash
+        end
+      end
+      
+      def facets_for_all_models(args, options)
+        options = GlobalFacetOptions.merge(options)
+        hash    = ThinkingSphinx::FacetCollection.new args + [options]
+        options = options.merge! facet_query_options
+        
+        facet_names(options).inject(hash) do |hash, name|
+          options[:group_by] = name
+          hash.add_from_results name, search(*(args + [options]))
+          hash
+        end
+      end
+      
+      def facet_query_options
+        config = ThinkingSphinx::Configuration.instance
+        max    = config.configuration.searchd.max_matches || 1000
+        
+        {
+          :group_function => :attr,
+          :limit          => max,
+          :max_matches    => max
+        }
+      end
+      
+      def facet_classes(options)
+        options[:classes] || ThinkingSphinx.indexed_models.collect { |model|
+          model.constantize
+        }
+      end
+      
+      def facet_names(options)
+        classes = facet_classes(options)
+        names   = options[:all_attributes] ?
+          facet_names_for_all_classes(classes) :
+          facet_names_common_to_all_classes(classes)
+        
+        names.delete "class_crc" unless options[:class_facet]
+        names
+      end
+      
+      def facet_names_for_all_classes(classes)
+        classes.collect { |klass|
+          klass.sphinx_facets.collect { |facet| facet.attribute_name }
+        }.flatten.uniq
+      end
+      
+      def facet_names_common_to_all_classes(classes)
+        facet_names_for_all_classes(classes).select { |name|
+          classes.all? { |klass|
+            klass.sphinx_facets.detect { |facet|
+              facet.attribute_name == name
+            }
+          }
+        }
+      end
     end
   end
 end

data/lib/thinking_sphinx/tasks.rb

@@ -8,8 +8,8 @@ namespace :thinking_sphinx do
   
   desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
   task :running_start => :app_env do
-   Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
-   Rake::Task["thinking_sphinx:start"].invoke
+    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
+    Rake::Task["thinking_sphinx:start"].invoke
   end
   
   desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
@@ -30,7 +30,7 @@ namespace :thinking_sphinx do
     if sphinx_running?
       puts "Started successfully (pid #{sphinx_pid})."
     else
-      puts "Failed to start searchd daemon. Check #{config.searchd_log_file}."
+      puts "Failed to start searchd daemon. Check #{config.searchd_log_file}"
     end
   end
   
@@ -39,7 +39,7 @@ namespace :thinking_sphinx do
     raise RuntimeError, "searchd is not running." unless sphinx_running?
     config = ThinkingSphinx::Configuration.instance
     pid    = sphinx_pid
-    system "searchd --stop --config #{config.config_file}"
+    system "#{config.bin_path}searchd --stop --config #{config.config_file}"
     puts "Stopped search daemon (pid #{pid})."
   end