DrMark-thinking-sphinx 0.9.9 → 1.1.6

data/lib/thinking_sphinx/active_record/delta.rb

@@ -11,84 +11,65 @@ module ThinkingSphinx
       #
       def self.included(base)
         base.class_eval do
-          # The define_callbacks method was added post Rails 2.0.2 - if it
-          # doesn't exist, we define the callback manually
-          #
-          if respond_to?(:define_callbacks)
-            define_callbacks :after_commit
-          else
-            class << self
-              # Handle after_commit callbacks - call all the registered callbacks.
-              #
-              def after_commit(*callbacks, &block)
-                callbacks << block if block_given?
-                write_inheritable_array(:after_commit, callbacks)
+          class << self
+            # Temporarily disable delta indexing inside a block, then perform a single
+            # rebuild of index at the end.
+            #
+            # Useful when performing updates to batches of models to prevent
+            # the delta index being rebuilt after each individual update.
+            #
+            # In the following example, the delta index will only be rebuilt once,
+            # not 10 times.
+            #
+            #   SomeModel.suspended_delta do
+            #     10.times do
+            #       SomeModel.create( ... )
+            #     end
+            #   end
+            #
+            def suspended_delta(reindex_after = true, &block)
+              original_setting = ThinkingSphinx.deltas_enabled?
+              ThinkingSphinx.deltas_enabled = false
+              begin
+                yield
+              ensure
+                ThinkingSphinx.deltas_enabled = original_setting
+                self.index_delta if reindex_after
               end
             end
+
+            # Build the delta index for the related model. This won't be called
+            # if running in the test environment.
+            #
+            def index_delta(instance = nil)
+              delta_object.index(self, instance)
+            end
+            
+            def delta_object
+              self.sphinx_indexes.first.delta_object
+            end
           end
           
-          def after_commit
-            # Deliberately blank.
-          end
-          
-          # Normal boolean save wrapped in a handler for the after_commit
-          # callback.
-          # 
-          def save_with_after_commit_callback(*args)
-            value = save_without_after_commit_callback(*args)
-            callback(:after_commit) if value
-            return value
-          end
-          
-          alias_method_chain :save, :after_commit_callback
-          
-          # Forceful save wrapped in a handler for the after_commit callback.
-          #
-          def save_with_after_commit_callback!(*args)
-            value = save_without_after_commit_callback!(*args)
-            callback(:after_commit) if value
-            return value
-          end
-          
-          alias_method_chain :save!, :after_commit_callback
-          
-          # Normal destroy wrapped in a handler for the after_commit callback.
-          #
-          def destroy_with_after_commit_callback
-            value = destroy_without_after_commit_callback
-            callback(:after_commit) if value
-            return value
+          def toggled_delta?
+            self.class.delta_object.toggled(self)
           end
           
-          alias_method_chain :destroy, :after_commit_callback
-          
           private
           
           # Set the delta value for the model to be true.
           def toggle_delta
-            self.delta = true
+            self.class.delta_object.toggle(self) if should_toggle_delta?
           end
           
           # Build the delta index for the related model. This won't be called
           # if running in the test environment.
           # 
           def index_delta
-            return true unless ThinkingSphinx.updates_enabled? &&
-              ThinkingSphinx.deltas_enabled?
-            
-            config = ThinkingSphinx::Configuration.new
-            client = Riddle::Client.new config.address, config.port
-            
-            client.update(
-              "#{self.class.indexes.first.name}_core",
-              ['sphinx_deleted'],
-              {self.id => 1}
-            ) if self.in_core_index?
-            
-            configuration = ThinkingSphinx::Configuration.new
-            system "indexer --config #{configuration.config_file} --rotate #{self.class.indexes.first.name}_delta"
-            
-            true
+            self.class.index_delta(self) if self.class.delta_object.toggled(self)
+          end
+          
+          def should_toggle_delta?
+            !self.respond_to?(:changed?) || self.changed? || self.new_record?
           end
         end
       end

data/lib/thinking_sphinx/active_record/has_many_association.rb

@@ -6,7 +6,7 @@ module ThinkingSphinx
         stack = [@reflection.options[:through]].compact
         
         attribute   = nil
-        (@reflection.klass.indexes || []).each do |index|
+        (@reflection.klass.sphinx_indexes || []).each do |index|
           attribute = index.attributes.detect { |attrib|
             attrib.columns.length == 1 &&
             attrib.columns.first.__name  == foreign_key.to_sym &&

data/lib/thinking_sphinx/active_record/search.rb

@@ -42,6 +42,13 @@ module ThinkingSphinx
               args << options
               ThinkingSphinx::Search.search_for_id(*args)
             end
+            
+            def facets(*args)
+              options = args.extract_options!
+              options[:class] = self
+              args << options
+              ThinkingSphinx::Search.facets(*args)
+            end
           end
         end
       end

data/lib/thinking_sphinx/adapters/abstract_adapter.rb

@@ -0,0 +1,42 @@
+module ThinkingSphinx
+  class AbstractAdapter
+    def initialize(model)
+      @model = model
+    end
+    
+    def setup
+      # Deliberately blank - subclasses should do something though. Well, if
+      # they need to.
+    end
+      
+    def self.detect(model)
+      case model.connection.class.name
+      when "ActiveRecord::ConnectionAdapters::MysqlAdapter",
+           "ActiveRecord::ConnectionAdapters::MysqlplusAdapter"
+        ThinkingSphinx::MysqlAdapter.new model
+      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
+        ThinkingSphinx::PostgreSQLAdapter.new model
+      when "ActiveRecord::ConnectionAdapters::JdbcAdapter"
+        if model.connection.config[:adapter] == "jdbcmysql"
+          ThinkingSphinx::MysqlAdapter.new model
+        elsif model.connection.config[:adapter] == "jdbcpostgresql"
+          ThinkingSphinx::PostgreSQLAdapter.new model
+        else
+          raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL"
+        end
+      else
+        raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL, not #{model.connection.class.name}"
+      end
+    end
+    
+    def quote_with_table(column)
+      "#{@model.quoted_table_name}.#{@model.connection.quote_column_name(column)}"
+    end
+    
+    protected
+    
+    def connection
+      @connection ||= @model.connection
+    end
+  end
+end

data/lib/thinking_sphinx/adapters/mysql_adapter.rb

@@ -0,0 +1,54 @@
+module ThinkingSphinx
+  class MysqlAdapter < AbstractAdapter
+    def setup
+      # Does MySQL actually need to do anything?
+    end
+    
+    def sphinx_identifier
+      "mysql"
+    end
+    
+    def concatenate(clause, separator = ' ')
+      "CONCAT_WS('#{separator}', #{clause})"
+    end
+    
+    def group_concatenate(clause, separator = ' ')
+      "GROUP_CONCAT(DISTINCT #{clause} SEPARATOR '#{separator}')"
+    end
+    
+    def cast_to_string(clause)
+      "CAST(#{clause} AS CHAR)"
+    end
+    
+    def cast_to_datetime(clause)
+      "UNIX_TIMESTAMP(#{clause})"
+    end
+    
+    def cast_to_unsigned(clause)
+      "CAST(#{clause} AS UNSIGNED)"
+    end
+    
+    def convert_nulls(clause, default = '')
+      default = "'#{default}'" if default.is_a?(String)
+      
+      "IFNULL(#{clause}, #{default})"
+    end
+    
+    def boolean(value)
+      value ? 1 : 0
+    end
+    
+    def crc(clause, blank_to_null = false)
+      clause = "NULLIF(#{clause},'')" if blank_to_null
+      "CRC32(#{clause})"
+    end
+    
+    def utf8_query_pre
+      "SET NAMES utf8"
+    end
+    
+    def time_difference(diff)
+      "DATE_SUB(NOW(), INTERVAL #{diff} SECOND)"
+    end
+  end
+end

data/lib/thinking_sphinx/adapters/postgresql_adapter.rb

@@ -0,0 +1,130 @@
+module ThinkingSphinx
+  class PostgreSQLAdapter < AbstractAdapter
+    def setup
+      create_array_accum_function
+      create_crc32_function
+    end
+    
+    def sphinx_identifier
+      "pgsql"
+    end
+    
+    def concatenate(clause, separator = ' ')
+      clause.split(', ').collect { |field|
+        "COALESCE(CAST(#{field} as varchar), '')"
+      }.join(" || '#{separator}' || ")
+    end
+    
+    def group_concatenate(clause, separator = ' ')
+      "array_to_string(array_accum(#{clause}), '#{separator}')"
+    end
+    
+    def cast_to_string(clause)
+      clause
+    end
+    
+    def cast_to_datetime(clause)
+      "cast(extract(epoch from #{clause}) as int)"
+    end
+    
+    def cast_to_unsigned(clause)
+      clause
+    end
+    
+    def convert_nulls(clause, default = '')
+      default = "'#{default}'" if default.is_a?(String)
+      
+      "COALESCE(#{clause}, #{default})"
+    end
+    
+    def boolean(value)
+      value ? 'TRUE' : 'FALSE'
+    end
+    
+    def crc(clause, blank_to_null = false)
+      clause = "NULLIF(#{clause},'')" if blank_to_null
+      "crc32(#{clause})"
+    end
+    
+    def utf8_query_pre
+      nil
+    end
+    
+    def time_difference(diff)
+      "current_timestamp - interval '#{diff} seconds'"
+    end
+    
+    private
+    
+    def execute(command, output_error = false)
+      connection.execute "begin"
+      connection.execute "savepoint ts"
+      begin
+        connection.execute command
+      rescue StandardError => err
+        puts err if output_error
+        connection.execute "rollback to savepoint ts"
+      end
+      connection.execute "release savepoint ts"
+      connection.execute "commit"
+    end
+    
+    def create_array_accum_function
+      if connection.raw_connection.respond_to?(:server_version) && connection.raw_connection.server_version > 80200
+        execute <<-SQL
+          CREATE AGGREGATE array_accum (anyelement)
+          (
+              sfunc = array_append,
+              stype = anyarray,
+              initcond = '{}'
+          );
+        SQL
+      else
+        execute <<-SQL
+          CREATE AGGREGATE array_accum
+          (
+              basetype = anyelement,
+              sfunc = array_append,
+              stype = anyarray,
+              initcond = '{}'
+          );
+        SQL
+      end
+    end
+    
+    def create_crc32_function
+      execute "CREATE LANGUAGE 'plpgsql';"
+      function = <<-SQL
+        CREATE OR REPLACE FUNCTION crc32(word text)
+        RETURNS bigint AS $$
+          DECLARE tmp bigint;
+          DECLARE i int;
+          DECLARE j int;
+          DECLARE word_array bytea;
+          BEGIN
+            i = 0;
+            tmp = 4294967295;
+            word_array = decode(replace(word, E'\\\\', E'\\\\\\\\'), 'escape');
+            LOOP
+              tmp = (tmp # get_byte(word_array, i))::bigint;
+              i = i + 1;
+              j = 0;
+              LOOP
+                tmp = ((tmp >> 1) # (3988292384 * (tmp & 1)))::bigint;
+                j = j + 1;
+                IF j >= 8 THEN
+                  EXIT;
+                END IF;
+              END LOOP;
+              IF i >= char_length(word) THEN
+                EXIT;
+              END IF;
+            END LOOP;
+            return (tmp # 4294967295);
+          END
+        $$ IMMUTABLE STRICT LANGUAGE plpgsql;
+      SQL
+      execute function, true
+    end
+  end
+end

data/lib/thinking_sphinx/association.rb

@@ -99,6 +99,23 @@ module ThinkingSphinx
       @reflection.klass.column_names.include?(column.to_s)
     end
     
+    def primary_key_from_reflection
+      if @reflection.options[:through]
+        @reflection.source_reflection.options[:foreign_key] ||
+        @reflection.source_reflection.primary_key_name
+      else
+        nil
+      end
+    end
+    
+    def table
+      if @reflection.options[:through]
+        @join.aliased_join_table_name
+      else
+        @join.aliased_table_name
+      end
+    end
+    
     private
     
     # Returns all the objects that could be currently instantiated from a

data/lib/thinking_sphinx/attribute.rb

@@ -9,14 +9,15 @@ module ThinkingSphinx
   # associations. Which can get messy. Use Index.link!, it really helps.
   # 
   class Attribute
-    attr_accessor :alias, :columns, :associations, :model
+    attr_accessor :alias, :columns, :associations, :model, :faceted, :source
     
     # To create a new attribute, you'll need to pass in either a single Column
     # or an array of them, and some (optional) options.
     #
     # Valid options are:
-    # - :as   => :alias_name 
-    # - :type => :attribute_type
+    # - :as     => :alias_name
+    # - :type   => :attribute_type
+    # - :source => :field, :query, :ranged_query
     #
     # Alias is only required in three circumstances: when there's
     # another attribute or field with the same name, when the column name is
@@ -28,6 +29,13 @@ module ThinkingSphinx
     # can't be figured out by the column - ie: when not actually using a
     # database column as your source.
     # 
+    # Source is only used for multi-value attributes (MVA). By default this will
+    # use a left-join and a group_concat to obtain the values. For better performance
+    # during indexing it can be beneficial to let Sphinx use a separate query to retrieve
+    # all document,value-pairs.
+    # Either :query or :ranged_query will enable this feature, where :ranged_query will cause
+    # the query to be executed incremental.
+    #
     # Example usage:
     #
     #   Attribute.new(
@@ -40,6 +48,12 @@ module ThinkingSphinx
     #   )
     #
     #   Attribute.new(
+    #     Column.new(:posts, :id),
+    #     :as => :post_ids,
+    #     :source => :ranged_query
+    #   )
+    #
+    #   Attribute.new(
     #     [Column.new(:pages, :id), Column.new(:articles, :id)],
     #     :as => :content_ids
     #   )
@@ -59,8 +73,14 @@ module ThinkingSphinx
       
       raise "Cannot define a field with no columns. Maybe you are trying to index a field with a reserved name (id, name). You can fix this error by using a symbol rather than a bare name (:id instead of id)." if @columns.empty? || @columns.any? { |column| !column.respond_to?(:__stack) }
       
-      @alias        = options[:as]
-      @type         = options[:type]
+      @alias    = options[:as]
+      @type     = options[:type]
+      @faceted  = options[:facet]
+      @source   = options[:source]
+      @crc      = options[:crc]
+      
+      @type   ||= :multi    unless @source.nil?
+      @type     = :integer  if @type == :string && @crc
     end
     
     # Get the part of the SELECT clause related to this attribute. Don't forget
@@ -70,16 +90,19 @@ module ThinkingSphinx
     # datetimes to timestamps, as needed.
     # 
     def to_select_sql
+      return nil unless include_as_association?
+      
       clause = @columns.collect { |column|
         column_with_prefix(column)
       }.join(', ')
       
       separator = all_ints? ? ',' : ' '
       
-      clause = concatenate(clause, separator)       if concat_ws?
-      clause = group_concatenate(clause, separator) if is_many?
-      clause = cast_to_datetime(clause)             if type == :datetime
-      clause = convert_nulls(clause)                if type == :string
+      clause = adapter.concatenate(clause, separator)       if concat_ws?
+      clause = adapter.group_concatenate(clause, separator) if is_many?
+      clause = adapter.cast_to_datetime(clause)             if type == :datetime
+      clause = adapter.convert_nulls(clause)                if type == :string
+      clause = adapter.crc(clause)                          if @crc
       
       "#{clause} AS #{quote_column(unique_name)}"
     end
@@ -101,23 +124,33 @@ module ThinkingSphinx
       end
     end
     
-    # Generates the appropriate attribute statement for a Sphinx configuration
-    # file, depending on the attribute's type.
+    def type_to_config
+      {
+        :multi    => :sql_attr_multi,
+        :datetime => :sql_attr_timestamp,
+        :string   => :sql_attr_str2ordinal,
+        :float    => :sql_attr_float,
+        :boolean  => :sql_attr_bool,
+        :integer  => :sql_attr_uint
+      }[type]
+    end
+    
+    def include_as_association?
+      ! (type == :multi && (source == :query || source == :ranged_query))
+    end
+    
+    # Returns the configuration value that should be used for
+    # the attribute.
+    # Special case is the multi-valued attribute that needs some
+    # extra configuration. 
     # 
-    def to_sphinx_clause
-      case type
-      when :multi
-        "sql_attr_multi       = uint #{unique_name} from field"
-      when :datetime
-        "sql_attr_timestamp   = #{unique_name}"
-      when :string
-        "sql_attr_str2ordinal = #{unique_name}"
-      when :float
-        "sql_attr_float       = #{unique_name}"
-      when :boolean
-        "sql_attr_bool        = #{unique_name}"
+    def config_value(offset = nil)
+      if type == :multi
+        multi_config = include_as_association? ? "field" :
+          source_value(offset).gsub(/\n\s*/, " ")
+        "uint #{unique_name} from #{multi_config}"
       else
-        "sql_attr_uint        = #{unique_name}"
+        unique_name
       end
     end
     
@@ -134,67 +167,104 @@ module ThinkingSphinx
       end
     end
     
+    # Returns the type of the column. If that's not already set, it returns
+    # :multi if there's the possibility of more than one value, :string if
+    # there's more than one association, otherwise it figures out what the
+    # actual column's datatype is and returns that.
+    # 
+    def type
+      @type ||= begin
+        base_type = case
+        when is_many?, is_many_ints?
+          :multi
+        when @associations.values.flatten.length > 1
+          :string
+        else
+          translated_type_from_database
+        end
+        
+        if base_type == :string && @crc
+          :integer
+        else
+          @crc = false
+          base_type
+        end
+      end
+    end
+    
+    def to_facet
+      return nil unless @faceted
+      
+      ThinkingSphinx::Facet.new(self)
+    end
+    
     private
     
-    def concatenate(clause, separator = ' ')
-      case @model.connection.class.name
-      when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
-        "CONCAT_WS('#{separator}', #{clause})"
-      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-        clause.split(', ').join(" || '#{separator}' || ")
+    def source_value(offset)
+      if is_string?
+        "#{source.to_s.dasherize}; #{columns.first.__name}"
+      elsif source == :ranged_query
+        "ranged-query; #{query offset} #{query_clause}; #{range_query}"
       else
-        clause
+        "query; #{query offset}"
       end
     end
     
-    def group_concatenate(clause, separator = ' ')
-      case @model.connection.class.name
-      when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
-        "GROUP_CONCAT(#{clause} SEPARATOR '#{separator}')"
-      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-        "array_to_string(array_accum(#{clause}), '#{separator}')"
-      else
-        clause
-      end
+    def query(offset)
+      assoc = association_for_mva
+      raise "Could not determine SQL for MVA" if assoc.nil?
+      
+      <<-SQL
+SELECT #{foreign_key_for_mva assoc}
+  #{ThinkingSphinx.unique_id_expression(offset)} AS #{quote_column('id')},
+  #{primary_key_for_mva(assoc)} AS #{quote_column(unique_name)}
+FROM #{quote_table_name assoc.table}
+      SQL
     end
     
-    def cast_to_string(clause)
-      case @model.connection.class.name
-      when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
-        "CAST(#{clause} AS CHAR)"
-      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-        clause
-      else
-        clause
-      end
+    def query_clause
+      foreign_key = foreign_key_for_mva association_for_mva
+      "WHERE #{foreign_key} >= $start AND #{foreign_key} <= $end"
     end
     
-    def cast_to_datetime(clause)
-      case @model.connection.class.name
-      when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
-        "UNIX_TIMESTAMP(#{clause})"
-      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-        clause # Rails' datetimes are timestamps in PostgreSQL
-      else
-        clause
-      end
+    def range_query
+      assoc       = association_for_mva
+      foreign_key = foreign_key_for_mva assoc
+      "SELECT MIN(#{foreign_key}), MAX(#{foreign_key}) FROM #{quote_table_name assoc.table}"
     end
     
-    def convert_nulls(clause)
-      case @model.connection.class.name
-      when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
-        "IFNULL(#{clause}, '')"
-      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-        "COALESCE(#{clause}, '')"
-      else
-        clause
-      end
+    def primary_key_for_mva(assoc)
+      quote_with_table(
+        assoc.table, assoc.primary_key_from_reflection || columns.first.__name
+      )
+    end
+    
+    def foreign_key_for_mva(assoc)
+      quote_with_table assoc.table, assoc.reflection.primary_key_name
+    end
+    
+    def association_for_mva
+      @association_for_mva ||= associations[columns.first].detect { |assoc|
+        assoc.has_column?(columns.first.__name)
+      }
+    end
+    
+    def adapter
+      @adapter ||= @model.sphinx_database_adapter
+    end
+    
+    def quote_with_table(table, column)
+      "#{quote_table_name(table)}.#{quote_column(column)}"
     end
     
     def quote_column(column)
       @model.connection.quote_column_name(column)
     end
     
+    def quote_table_name(table_name)
+      @model.connection.quote_table_name(table_name)
+    end
+    
     # Indication of whether the columns should be concatenated with a space
     # between each value. True if there's either multiple sources or multiple
     # associations.
@@ -202,16 +272,7 @@ module ThinkingSphinx
     def concat_ws?
       multiple_associations? || @columns.length > 1
     end
-    
-    # Checks the association tree for each column - if they're all the same,
-    # returns false.
-    # 
-    def multiple_sources?
-      first = associations[@columns.first]
-      
-      !@columns.all? { |col| associations[col] == first }
-    end
-    
+        
     # Checks whether any column requires multiple associations (which only
     # happens for polymorphic situations).
     # 
@@ -231,7 +292,7 @@ module ThinkingSphinx
       else
         associations[column].collect { |assoc|
           assoc.has_column?(column.__name) ?
-          "#{@model.connection.quote_table_name(assoc.join.aliased_table_name)}" + 
+          "#{quote_table_name(assoc.join.aliased_table_name)}" + 
           ".#{quote_column(column.__name)}" :
           nil
         }.compact.join(', ')
@@ -245,31 +306,16 @@ module ThinkingSphinx
       associations.values.flatten.any? { |assoc| assoc.is_many? }
     end
     
+    def is_many_ints?
+      concat_ws? && all_ints?
+    end
+    
     # Returns true if any of the columns are string values, instead of database
     # column references.
     def is_string?
       columns.all? { |col| col.is_string? }
     end
     
-    # Returns the type of the column. If that's not already set, it returns
-    # :multi if there's the possibility of more than one value, :string if
-    # there's more than one association, otherwise it figures out what the
-    # actual column's datatype is and returns that.
-    def type
-      @type ||= case
-      when is_many?
-        :multi
-      when @associations.values.flatten.length > 1
-        :string
-      else
-        klass = @associations.values.flatten.first ? 
-          @associations.values.flatten.first.reflection.klass : @model
-        klass.columns.detect { |col|
-          @columns.collect { |c| c.__name.to_s }.include? col.name
-        }.type
-      end
-    end
-    
     def all_ints?
       @columns.all? { |col|
         klasses = @associations[col].empty? ? [@model] :
@@ -280,5 +326,33 @@ module ThinkingSphinx
         }
       }
     end
+    
+    def type_from_database
+      klass = @associations.values.flatten.first ? 
+        @associations.values.flatten.first.reflection.klass : @model
+      
+      klass.columns.detect { |col|
+        @columns.collect { |c| c.__name.to_s }.include? col.name
+      }.type
+    end
+    
+    def translated_type_from_database
+      case type_from_db = type_from_database
+      when :datetime, :string, :float, :boolean, :integer
+        type_from_db
+      when :decimal
+        :float
+      when :timestamp, :date
+        :datetime
+      else
+        raise <<-MESSAGE
+
+Cannot automatically map column type #{type_from_db} to an equivalent Sphinx
+type (integer, float, boolean, datetime, string as ordinal). You could try to
+explicitly convert the column's value in your define_index block:
+  has "CAST(column AS INT)", :type => :integer, :as => :column
+        MESSAGE
+      end
+    end
   end
 end