dpickett-thinking-sphinx 1.1.4 → 1.1.12

data/README.textile

@@ -0,0 +1,126 @@
+h1. Thinking Sphinx
+
+h2. Usage
+
+First, if you haven't done so already, check out the main "usage":http://ts.freelancing-gods.com/usage.html page. Once you've done that, the next place to look for information is the specific method docs - ThinkingSphinx::Search and ThinkingSphinx::Index::Builder in particular.
+
+Keep in mind that while Thinking Sphinx works for ActiveRecord with Merb, it doesn't yet support DataMapper (although that is planned).
+
+h2. Contributing
+
+Fork on GitHub and after you've committed tested patches, send a pull request.
+
+To quickly see if your system is ready to run the thinking sphinx specs, run the contribute.rb script found in the project root directory. Use the following instructions to install any missing requirements.
+
+To get the spec suite running, you will need to install the not-a-mock gem if you don't already have it:
+
+  git clone git://github.com/freelancing-god/not-a-mock.git
+  cd not-a-mock
+  rake gem
+  gem install pkg/not_a_mock-1.1.0.gem
+  
+Then install the ginger gem. The steps are the same, except that you might need to sudo the gem install:
+
+  git clone git://github.com/freelancing-god/ginger.git
+  cd ginger
+  rake gem
+  sudo gem install pkg/ginger-1.1.0.gem
+
+Then set up your database:
+
+  cp spec/fixtures/database.yml.default spec/fixtures/database.yml
+  mysqladmin -u root create thinking_sphinx
+
+Make sure you don't have another Sphinx daemon (searchd) running. If you do, quit it with "rake ts:stop"
+in the app root.
+
+You should now have a passing test suite from which to build your patch on.
+
+  rake spec
+
+If you get the message "Failed to start searchd daemon", run the spec with sudo:
+
+  sudo rake spec
+
+If you quit the spec suite before it's completed, you may be left with data in the test
+database, causing the next run to have failures. Let that run complete and then try again.
+
+h2. Contributors
+
+Since I first released this library, there's been quite a few people who have submitted patches, to my immense gratitude. Others have suggested syntax changes and general improvements. So my thanks to the following people:
+
+* Joost Hietbrink
+* Jonathan Conway
+* Gregory Mirzayantz
+* Tung Nguyen
+* Sean Cribbs
+* Benoit Caccinolo
+* John Barton
+* Oliver Beddows
+* Arthur Zapparoli
+* Dusty Doris
+* Marcus Crafter
+* Patrick Lenz
+* Björn Andreasson
+* James Healy
+* Jae-Jun Hwang
+* Xavier Shay
+* Jason Rust
+* Gopal Patel
+* Chris Heald
+* Peter Vandenberk
+* Josh French
+* Andrew Bennett
+* Jordan Fowler
+* Seth Walker
+* Joe Noon
+* Wolfgang Postler
+* Rick Olson
+* Killian Murphy
+* Morten Primdahl
+* Ryan Bates
+* David Eisinger
+* Shay Arnett
+* Minh Tran
+* Jeremy Durham
+* Piotr Sarnacki
+* Matt Johnson
+* Nicolas Blanco
+* Max Lapshin
+* Josh Natanson
+* Philip Hallstrom
+* Christian Rishøj
+* Mike Flester
+* Jim Remsik
+* Kennon Ballou
+* Henrik Nyh
+* Emil Tin
+* Doug Cole
+* Ed Hickey
+* Evan Weaver
+* Thibaut Barrere
+* Kristopher Chambers
+* Dmitrij Smalko
+* Aleksey Yeschenko
+* Lachie Cox
+* Lourens Naude
+* Tom Davies
+* Dan Pickett
+* Alex Caudill
+* Jim Benton
+* John Aughey
+* Keith Pitty
+* Jeff Talbot
+* Dana Contreras
+* Menno van der Sman
+* Bill Harding
+* Isaac Feliu
+* Andrei Bocan
+* László Bácsi
+* Peter Wagenet
+* Max Lapshin
+* Martin Emde
+* David Wennergren
+* Mark Lane
+* Eric Lindvall
+* Lawrence Pit

data/lib/thinking_sphinx/active_record/attribute_updates.rb

@@ -0,0 +1,48 @@
+module ThinkingSphinx
+  module ActiveRecord
+    module AttributeUpdates
+      def self.included(base)
+        base.class_eval do
+          after_commit :update_attribute_values
+        end
+      end
+      
+      private
+            
+      def update_attribute_values
+        return unless ThinkingSphinx.updates_enabled? && ThinkingSphinx.sphinx_running?
+        
+        config = ThinkingSphinx::Configuration.instance
+        client = Riddle::Client.new config.address, config.port
+        
+        self.sphinx_indexes.each do |index|
+          attribute_pairs  = attribute_values_for_index(index)
+          attribute_names  = attribute_pairs.keys
+          attribute_values = attribute_names.collect { |key|
+            attribute_pairs[key]
+          }
+          
+          client.update "#{index.name}_core", attribute_names, {
+            sphinx_document_id => attribute_values
+          } if in_core_index?
+        end
+      end
+      
+      def updatable_attributes(index)
+        index.attributes.select { |attrib| attrib.updatable? }
+      end
+      
+      def attribute_values_for_index(index)
+        updatable_attributes(index).inject({}) { |hash, attrib|
+          if attrib.type == :datetime
+            hash[attrib.unique_name.to_s] = attrib.live_value(self).to_time.to_i
+          else
+            hash[attrib.unique_name.to_s] = attrib.live_value self
+          end
+          
+          hash
+        }
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/active_record/delta.rb

@@ -50,6 +50,10 @@ module ThinkingSphinx
             end
           end
           
+          def toggled_delta?
+            self.class.delta_object.toggled(self)
+          end
+          
           private
           
           # Set the delta value for the model to be true.
@@ -65,7 +69,16 @@ module ThinkingSphinx
           end
           
           def should_toggle_delta?
-            !self.respond_to?(:changed?) || self.changed? || self.new_record?
+            self.new_record? || indexed_data_changed?
+          end
+          
+          def indexed_data_changed?
+            sphinx_indexes.any? { |index|
+              index.fields.any? { |field| field.changed?(self) } ||
+              index.attributes.any? { |attrib|
+                attrib.public? && attrib.changed?(self) && !attrib.updatable?
+              }
+            }
           end
         end
       end

data/lib/thinking_sphinx/active_record.rb

@@ -1,3 +1,4 @@
+require 'thinking_sphinx/active_record/attribute_updates'
 require 'thinking_sphinx/active_record/delta'
 require 'thinking_sphinx/active_record/search'
 require 'thinking_sphinx/active_record/has_many_association'
@@ -79,6 +80,8 @@ module ThinkingSphinx
             
             after_destroy :toggle_deleted
             
+            include ThinkingSphinx::ActiveRecord::AttributeUpdates
+            
             index
           end
           alias_method :sphinx_index, :define_index
@@ -207,11 +210,20 @@ module ThinkingSphinx
       )
     end
     
+    def in_index?(suffix)
+      self.class.search_for_id self.sphinx_document_id, sphinx_index_name(suffix)
+    end
+    
     def in_core_index?
-      self.class.search_for_id(
-        self.sphinx_document_id,
-        "#{self.class.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_core"
-      )
+      in_index? "core"
+    end
+    
+    def in_delta_index?
+      in_index? "delta"
+    end
+    
+    def in_both_indexes?
+      in_core_index? && in_delta_index?
     end
     
     def toggle_deleted
@@ -232,7 +244,7 @@ module ThinkingSphinx
         {self.sphinx_document_id => 1}
       ) if ThinkingSphinx.deltas_enabled? &&
         self.class.sphinx_indexes.any? { |index| index.delta? } &&
-        self.delta
+        self.toggled_delta?
     rescue ::ThinkingSphinx::ConnectionError
       # nothing
     end
@@ -241,5 +253,11 @@ module ThinkingSphinx
       (self.id * ThinkingSphinx.indexed_models.size) +
         ThinkingSphinx.indexed_models.index(self.class.source_of_sphinx_index.name)
     end
+    
+    private
+    
+    def sphinx_index_name(suffix)
+      "#{self.class.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_#{suffix}"
+    end
   end
 end

data/lib/thinking_sphinx/adapters/abstract_adapter.rb

@@ -16,8 +16,16 @@ module ThinkingSphinx
         ThinkingSphinx::MysqlAdapter.new model
       when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
         ThinkingSphinx::PostgreSQLAdapter.new model
+      when "ActiveRecord::ConnectionAdapters::JdbcAdapter"
+        if model.connection.config[:adapter] == "jdbcmysql"
+          ThinkingSphinx::MysqlAdapter.new model
+        elsif model.connection.config[:adapter] == "jdbcpostgresql"
+          ThinkingSphinx::PostgreSQLAdapter.new model
+        else
+          raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL"
+        end
       else
-        raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL"
+        raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL, not #{model.connection.class.name}"
       end
     end
     

data/lib/thinking_sphinx/adapters/mysql_adapter.rb

@@ -13,7 +13,7 @@ module ThinkingSphinx
     end
     
     def group_concatenate(clause, separator = ' ')
-      "GROUP_CONCAT(#{clause} SEPARATOR '#{separator}')"
+      "GROUP_CONCAT(DISTINCT #{clause} SEPARATOR '#{separator}')"
     end
     
     def cast_to_string(clause)
@@ -38,7 +38,8 @@ module ThinkingSphinx
       value ? 1 : 0
     end
     
-    def crc(clause)
+    def crc(clause, blank_to_null = false)
+      clause = "NULLIF(#{clause},'')" if blank_to_null
       "CRC32(#{clause})"
     end
     

data/lib/thinking_sphinx/adapters/postgresql_adapter.rb

@@ -11,7 +11,7 @@ module ThinkingSphinx
     
     def concatenate(clause, separator = ' ')
       clause.split(', ').collect { |field|
-        "COALESCE(#{field}, '')"
+        "COALESCE(CAST(#{field} as varchar), '')"
       }.join(" || '#{separator}' || ")
     end
     
@@ -41,7 +41,8 @@ module ThinkingSphinx
       value ? 'TRUE' : 'FALSE'
     end
     
-    def crc(clause)
+    def crc(clause, blank_to_null = false)
+      clause = "NULLIF(#{clause},'')" if blank_to_null
       "crc32(#{clause})"
     end
     
@@ -69,7 +70,7 @@ module ThinkingSphinx
     end
     
     def create_array_accum_function
-      if connection.raw_connection.server_version > 80200
+      if connection.raw_connection.respond_to?(:server_version) && connection.raw_connection.server_version > 80200
         execute <<-SQL
           CREATE AGGREGATE array_accum (anyelement)
           (

data/lib/thinking_sphinx/association.rb

@@ -99,6 +99,23 @@ module ThinkingSphinx
       @reflection.klass.column_names.include?(column.to_s)
     end
     
+    def primary_key_from_reflection
+      if @reflection.options[:through]
+        @reflection.source_reflection.options[:foreign_key] ||
+        @reflection.source_reflection.primary_key_name
+      else
+        nil
+      end
+    end
+    
+    def table
+      if @reflection.options[:through]
+        @join.aliased_join_table_name
+      else
+        @join.aliased_table_name
+      end
+    end
+    
     private
     
     # Returns all the objects that could be currently instantiated from a

data/lib/thinking_sphinx/attribute.rb

@@ -8,15 +8,16 @@ module ThinkingSphinx
   # generate SQL statements, you'll need to set the base model, and all the
   # associations. Which can get messy. Use Index.link!, it really helps.
   # 
-  class Attribute
-    attr_accessor :alias, :columns, :associations, :model, :faceted
+  class Attribute < ThinkingSphinx::Property
+    attr_accessor :source
     
     # To create a new attribute, you'll need to pass in either a single Column
     # or an array of them, and some (optional) options.
     #
     # Valid options are:
-    # - :as   => :alias_name 
-    # - :type => :attribute_type
+    # - :as     => :alias_name
+    # - :type   => :attribute_type
+    # - :source => :field, :query, :ranged_query
     #
     # Alias is only required in three circumstances: when there's
     # another attribute or field with the same name, when the column name is
@@ -28,6 +29,13 @@ module ThinkingSphinx
     # can't be figured out by the column - ie: when not actually using a
     # database column as your source.
     # 
+    # Source is only used for multi-value attributes (MVA). By default this will
+    # use a left-join and a group_concat to obtain the values. For better performance
+    # during indexing it can be beneficial to let Sphinx use a separate query to retrieve
+    # all document,value-pairs.
+    # Either :query or :ranged_query will enable this feature, where :ranged_query will cause
+    # the query to be executed incremental.
+    #
     # Example usage:
     #
     #   Attribute.new(
@@ -40,6 +48,12 @@ module ThinkingSphinx
     #   )
     #
     #   Attribute.new(
+    #     Column.new(:posts, :id),
+    #     :as => :post_ids,
+    #     :source => :ranged_query
+    #   )
+    #
+    #   Attribute.new(
     #     [Column.new(:pages, :id), Column.new(:articles, :id)],
     #     :as => :content_ids
     #   )
@@ -54,14 +68,14 @@ module ThinkingSphinx
     # that Sphinx expects these values to be in radians.
     #  
     def initialize(columns, options = {})
-      @columns      = Array(columns)
-      @associations = {}
-      
-      raise "Cannot define a field with no columns. Maybe you are trying to index a field with a reserved name (id, name). You can fix this error by using a symbol rather than a bare name (:id instead of id)." if @columns.empty? || @columns.any? { |column| !column.respond_to?(:__stack) }
+      super
       
-      @alias    = options[:as]
       @type     = options[:type]
-      @faceted  = options[:facet]
+      @source   = options[:source]
+      @crc      = options[:crc]
+      
+      @type   ||= :multi    unless @source.nil?
+      @type     = :integer  if @type == :string && @crc
     end
     
     # Get the part of the SELECT clause related to this attribute. Don't forget
@@ -71,6 +85,8 @@ module ThinkingSphinx
     # datetimes to timestamps, as needed.
     # 
     def to_select_sql
+      return nil unless include_as_association?
+      
       clause = @columns.collect { |column|
         column_with_prefix(column)
       }.join(', ')
@@ -81,27 +97,11 @@ module ThinkingSphinx
       clause = adapter.group_concatenate(clause, separator) if is_many?
       clause = adapter.cast_to_datetime(clause)             if type == :datetime
       clause = adapter.convert_nulls(clause)                if type == :string
+      clause = adapter.crc(clause)                          if @crc
       
       "#{clause} AS #{quote_column(unique_name)}"
     end
     
-    # Get the part of the GROUP BY clause related to this attribute - if one is
-    # needed. If not, all you'll get back is nil. The latter will happen if
-    # there isn't actually a real column to get data from, or if there's
-    # multiple data values (read: a has_many or has_and_belongs_to_many
-    # association).
-    # 
-    def to_group_sql
-      case
-      when is_many?, is_string?, ThinkingSphinx.use_group_by_shortcut?
-        nil
-      else
-        @columns.collect { |column|
-          column_with_prefix(column)
-        }
-      end
-    end
-    
     def type_to_config
       {
         :multi    => :sql_attr_multi,
@@ -113,105 +113,116 @@ module ThinkingSphinx
       }[type]
     end
     
-    def config_value
-      if type == :multi
-        "uint #{unique_name} from field"
-      else
-        unique_name
-      end
+    def include_as_association?
+      ! (type == :multi && (source == :query || source == :ranged_query))
     end
     
-    # Returns the unique name of the attribute - which is either the alias of
-    # the attribute, or the name of the only column - if there is only one. If
-    # there isn't, there should be an alias. Else things probably won't work.
-    # Consider yourself warned.
+    # Returns the configuration value that should be used for
+    # the attribute.
+    # Special case is the multi-valued attribute that needs some
+    # extra configuration. 
     # 
-    def unique_name
-      if @columns.length == 1
-        @alias || @columns.first.__name
+    def config_value(offset = nil)
+      if type == :multi
+        multi_config = include_as_association? ? "field" :
+          source_value(offset).gsub(/\n\s*/, " ")
+        "uint #{unique_name} from #{multi_config}"
       else
-        @alias
+        unique_name
       end
     end
-    
+        
     # Returns the type of the column. If that's not already set, it returns
     # :multi if there's the possibility of more than one value, :string if
     # there's more than one association, otherwise it figures out what the
     # actual column's datatype is and returns that.
+    # 
     def type
-      @type ||= case
-      when is_many?
-        :multi
-      when @associations.values.flatten.length > 1
-        :string
-      else
-        translated_type_from_database
+      @type ||= begin
+        base_type = case
+        when is_many?, is_many_ints?
+          :multi
+        when @associations.values.flatten.length > 1
+          :string
+        else
+          translated_type_from_database
+        end
+        
+        if base_type == :string && @crc
+          :integer
+        else
+          @crc = false
+          base_type
+        end
       end
     end
     
-    def to_facet
-      return nil unless @faceted
-      
-      ThinkingSphinx::Facet.new(self)
+    def updatable?
+      [:integer, :datetime, :boolean].include?(type) && !is_string?
+    end
+    
+    def live_value(instance)
+      object = instance
+      column = @columns.first
+      column.__stack.each { |method| object = object.send(method) }
+      object.send(column.__name)
     end
     
     private
     
-    def adapter
-      @adapter ||= @model.sphinx_database_adapter
+    def source_value(offset)
+      if is_string?
+        "#{source.to_s.dasherize}; #{columns.first.__name}"
+      elsif source == :ranged_query
+        "ranged-query; #{query offset} #{query_clause}; #{range_query}"
+      else
+        "query; #{query offset}"
+      end
     end
     
-    def quote_column(column)
-      @model.connection.quote_column_name(column)
+    def query(offset)
+      assoc = association_for_mva
+      raise "Could not determine SQL for MVA" if assoc.nil?
+      
+      <<-SQL
+SELECT #{foreign_key_for_mva assoc}
+  #{ThinkingSphinx.unique_id_expression(offset)} AS #{quote_column('id')},
+  #{primary_key_for_mva(assoc)} AS #{quote_column(unique_name)}
+FROM #{quote_table_name assoc.table}
+      SQL
     end
     
-    # Indication of whether the columns should be concatenated with a space
-    # between each value. True if there's either multiple sources or multiple
-    # associations.
-    # 
-    def concat_ws?
-      multiple_associations? || @columns.length > 1
+    def query_clause
+      foreign_key = foreign_key_for_mva association_for_mva
+      "WHERE #{foreign_key} >= $start AND #{foreign_key} <= $end"
     end
-        
-    # Checks whether any column requires multiple associations (which only
-    # happens for polymorphic situations).
-    # 
-    def multiple_associations?
-      associations.any? { |col,assocs| assocs.length > 1 }
+    
+    def range_query
+      assoc       = association_for_mva
+      foreign_key = foreign_key_for_mva assoc
+      "SELECT MIN(#{foreign_key}), MAX(#{foreign_key}) FROM #{quote_table_name assoc.table}"
     end
     
-    # Builds a column reference tied to the appropriate associations. This
-    # dives into the associations hash and their corresponding joins to
-    # figure out how to correctly reference a column in SQL.
-    # 
-    def column_with_prefix(column)
-      if column.is_string?
-        column.__name
-      elsif associations[column].empty?
-        "#{@model.quoted_table_name}.#{quote_column(column.__name)}"
-      else
-        associations[column].collect { |assoc|
-          assoc.has_column?(column.__name) ?
-          "#{@model.connection.quote_table_name(assoc.join.aliased_table_name)}" + 
-          ".#{quote_column(column.__name)}" :
-          nil
-        }.compact.join(', ')
-      end
+    def primary_key_for_mva(assoc)
+      quote_with_table(
+        assoc.table, assoc.primary_key_from_reflection || columns.first.__name
+      )
     end
     
-    # Could there be more than one value related to the parent record? If so,
-    # then this will return true. If not, false. It's that simple.
-    # 
-    def is_many?
-      associations.values.flatten.any? { |assoc| assoc.is_many? }
+    def foreign_key_for_mva(assoc)
+      quote_with_table assoc.table, assoc.reflection.primary_key_name
     end
     
-    # Returns true if any of the columns are string values, instead of database
-    # column references.
-    def is_string?
-      columns.all? { |col| col.is_string? }
+    def association_for_mva
+      @association_for_mva ||= associations[columns.first].detect { |assoc|
+        assoc.has_column?(columns.first.__name)
+      }
     end
     
+    def is_many_ints?
+      concat_ws? && all_ints?
+    end
+        
     def all_ints?
       @columns.all? { |col|
         klasses = @associations[col].empty? ? [@model] :

data/lib/thinking_sphinx/class_facet.rb

@@ -10,11 +10,6 @@ module ThinkingSphinx
     
     def value(object, attribute_value)
       object.class.name
-      # ThinkingSphinx.indexed_models.each do |i|
-      #   return i if i.to_crc32 == attribute_value
-      # end
-      # 
-      # raise "Unknown class crc"
     end
   end
 end

data/lib/thinking_sphinx/collection.rb

@@ -46,9 +46,11 @@ module ThinkingSphinx
       ids = matches.collect { |match| match[:attributes]["sphinx_internal_id"] }
       instances = ids.length > 0 ? klass.find(
         :all,
+        :joins      => options[:joins],
         :conditions => {klass.primary_key.to_sym => ids},
         :include    => (options[:include] || index_options[:include]),
-        :select     => (options[:select]  || index_options[:select])
+        :select     => (options[:select]  || index_options[:select]),
+        :order      => (options[:sql_order] || index_options[:sql_order])
       ) : []
 
       # Raise an exception if we find records in Sphinx but not in the DB, so
@@ -59,6 +61,10 @@ module ThinkingSphinx
         raise StaleIdsException, stale_ids
       end
 
+      # if the user has specified an SQL order, return the collection
+      # without rearranging it into the Sphinx order
+      return instances if options[:sql_order]
+
       ids.collect { |obj_id|
         instances.detect { |obj| obj.id == obj_id }
       }