warp-thinking-sphinx 1.2.12

data/lib/thinking_sphinx/active_record/attribute_updates.rb

@@ -0,0 +1,48 @@
+module ThinkingSphinx
+  module ActiveRecord
+    module AttributeUpdates
+      def self.included(base)
+        base.class_eval do
+          after_commit :update_attribute_values
+        end
+      end
+      
+      private
+            
+      def update_attribute_values
+        return unless ThinkingSphinx.updates_enabled? && ThinkingSphinx.sphinx_running?
+        
+        config = ThinkingSphinx::Configuration.instance
+        client = Riddle::Client.new config.address, config.port
+        
+        self.sphinx_indexes.each do |index|
+          attribute_pairs  = attribute_values_for_index(index)
+          attribute_names  = attribute_pairs.keys
+          attribute_values = attribute_names.collect { |key|
+            attribute_pairs[key]
+          }
+          
+          client.update "#{index.name}_core", attribute_names, {
+            sphinx_document_id => attribute_values
+          } if in_core_index?
+        end
+      end
+      
+      def updatable_attributes(index)
+        index.attributes.select { |attrib| attrib.updatable? }
+      end
+      
+      def attribute_values_for_index(index)
+        updatable_attributes(index).inject({}) { |hash, attrib|
+          if attrib.type == :datetime
+            hash[attrib.unique_name.to_s] = attrib.live_value(self).to_time.to_i
+          else
+            hash[attrib.unique_name.to_s] = attrib.live_value self
+          end
+          
+          hash
+        }
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/active_record/delta.rb

@@ -0,0 +1,87 @@
+module ThinkingSphinx
+  module ActiveRecord
+    # This module contains all the delta-related code for models. There isn't
+    # really anything you need to call manually in here - except perhaps
+    # index_delta, but not sure what reason why.
+    # 
+    module Delta
+      # Code for after_commit callback is written by Eli Miller:
+      # http://elimiller.blogspot.com/2007/06/proper-cache-expiry-with-aftercommit.html
+      # with slight modification from Joost Hietbrink.
+      #
+      def self.included(base)
+        base.class_eval do
+          class << self
+            # Temporarily disable delta indexing inside a block, then perform a single
+            # rebuild of index at the end.
+            #
+            # Useful when performing updates to batches of models to prevent
+            # the delta index being rebuilt after each individual update.
+            #
+            # In the following example, the delta index will only be rebuilt once,
+            # not 10 times.
+            #
+            #   SomeModel.suspended_delta do
+            #     10.times do
+            #       SomeModel.create( ... )
+            #     end
+            #   end
+            #
+            def suspended_delta(reindex_after = true, &block)
+              original_setting = ThinkingSphinx.deltas_enabled?
+              ThinkingSphinx.deltas_enabled = false
+              begin
+                yield
+              ensure
+                ThinkingSphinx.deltas_enabled = original_setting
+                self.index_delta if reindex_after
+              end
+            end
+
+            # Build the delta index for the related model. This won't be called
+            # if running in the test environment.
+            #
+            def index_delta(instance = nil)
+              delta_object.index(self, instance)
+            end
+            
+            def delta_object
+              self.sphinx_indexes.first.delta_object
+            end
+          end
+          
+          def toggled_delta?
+            self.class.delta_object.toggled(self)
+          end
+          
+          private
+          
+          # Set the delta value for the model to be true.
+          def toggle_delta
+            self.class.delta_object.toggle(self) if should_toggle_delta?
+          end
+          
+          # Build the delta index for the related model. This won't be called
+          # if running in the test environment.
+          # 
+          def index_delta
+            self.class.index_delta(self) if self.class.delta_object.toggled(self)
+          end
+          
+          def should_toggle_delta?
+            self.new_record? || indexed_data_changed?
+          end
+          
+          def indexed_data_changed?
+            sphinx_indexes.any? { |index|
+              index.fields.any? { |field| field.changed?(self) } ||
+              index.attributes.any? { |attrib|
+                attrib.public? && attrib.changed?(self) && !attrib.updatable?
+              }
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/active_record/has_many_association.rb

@@ -0,0 +1,28 @@
+module ThinkingSphinx
+  module ActiveRecord
+    module HasManyAssociation
+      def search(*args)
+        foreign_key = @reflection.primary_key_name
+        stack = [@reflection.options[:through]].compact
+        
+        attribute   = nil
+        (@reflection.klass.sphinx_indexes || []).each do |index|
+          attribute = index.attributes.detect { |attrib|
+            attrib.columns.length == 1 &&
+            attrib.columns.first.__name  == foreign_key.to_sym
+          }
+          break if attribute
+        end
+        
+        raise "Missing Attribute for Foreign Key #{foreign_key}" unless attribute
+        
+        options = args.extract_options!
+        options[:with] ||= {}
+        options[:with][attribute.unique_name] = @owner.id
+        
+        args << options
+        @reflection.klass.search(*args)
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/active_record/scopes.rb

@@ -0,0 +1,39 @@
+module ThinkingSphinx
+  module ActiveRecord
+    module Scopes
+      def self.included(base)
+        base.class_eval do
+          extend ThinkingSphinx::ActiveRecord::Scopes::ClassMethods
+        end
+      end
+    
+      module ClassMethods
+        def sphinx_scope(method, &block)
+          @sphinx_scopes ||= []
+          @sphinx_scopes << method
+          
+          metaclass.instance_eval do
+            define_method(method) do |*args|
+              options = {:classes => classes_option}
+              options.merge! block.call(*args)
+              
+              ThinkingSphinx::Search.new(options)
+            end
+          end
+        end
+        
+        def sphinx_scopes
+          @sphinx_scopes || []
+        end
+        
+        def remove_sphinx_scopes
+          sphinx_scopes.each do |scope|
+            metaclass.send(:undef_method, scope)
+          end
+          
+          sphinx_scopes.clear
+        end
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/adapters/abstract_adapter.rb

@@ -0,0 +1,42 @@
+module ThinkingSphinx
+  class AbstractAdapter
+    def initialize(model)
+      @model = model
+    end
+    
+    def setup
+      # Deliberately blank - subclasses should do something though. Well, if
+      # they need to.
+    end
+      
+    def self.detect(model)
+      case model.connection.class.name
+      when "ActiveRecord::ConnectionAdapters::MysqlAdapter",
+           "ActiveRecord::ConnectionAdapters::MysqlplusAdapter"
+        ThinkingSphinx::MysqlAdapter.new model
+      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
+        ThinkingSphinx::PostgreSQLAdapter.new model
+      when "ActiveRecord::ConnectionAdapters::JdbcAdapter"
+        if model.connection.config[:adapter] == "jdbcmysql"
+          ThinkingSphinx::MysqlAdapter.new model
+        elsif model.connection.config[:adapter] == "jdbcpostgresql"
+          ThinkingSphinx::PostgreSQLAdapter.new model
+        else
+          raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL"
+        end
+      else
+        raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL, not #{model.connection.class.name}"
+      end
+    end
+    
+    def quote_with_table(column)
+      "#{@model.quoted_table_name}.#{@model.connection.quote_column_name(column)}"
+    end
+    
+    protected
+    
+    def connection
+      @connection ||= @model.connection
+    end
+  end
+end

data/lib/thinking_sphinx/adapters/mysql_adapter.rb

@@ -0,0 +1,54 @@
+module ThinkingSphinx
+  class MysqlAdapter < AbstractAdapter
+    def setup
+      # Does MySQL actually need to do anything?
+    end
+    
+    def sphinx_identifier
+      "mysql"
+    end
+    
+    def concatenate(clause, separator = ' ')
+      "CONCAT_WS('#{separator}', #{clause})"
+    end
+    
+    def group_concatenate(clause, separator = ' ')
+      "GROUP_CONCAT(DISTINCT #{clause} SEPARATOR '#{separator}')"
+    end
+    
+    def cast_to_string(clause)
+      "CAST(#{clause} AS CHAR)"
+    end
+    
+    def cast_to_datetime(clause)
+      "UNIX_TIMESTAMP(#{clause})"
+    end
+    
+    def cast_to_unsigned(clause)
+      "CAST(#{clause} AS UNSIGNED)"
+    end
+    
+    def convert_nulls(clause, default = '')
+      default = "'#{default}'" if default.is_a?(String)
+      
+      "IFNULL(#{clause}, #{default})"
+    end
+    
+    def boolean(value)
+      value ? 1 : 0
+    end
+    
+    def crc(clause, blank_to_null = false)
+      clause = "NULLIF(#{clause},'')" if blank_to_null
+      "CRC32(#{clause})"
+    end
+    
+    def utf8_query_pre
+      "SET NAMES utf8"
+    end
+    
+    def time_difference(diff)
+      "DATE_SUB(NOW(), INTERVAL #{diff} SECOND)"
+    end
+  end
+end

data/lib/thinking_sphinx/adapters/postgresql_adapter.rb

@@ -0,0 +1,136 @@
+module ThinkingSphinx
+  class PostgreSQLAdapter < AbstractAdapter
+    def setup
+      create_array_accum_function
+      create_crc32_function
+    end
+    
+    def sphinx_identifier
+      "pgsql"
+    end
+    
+    def concatenate(clause, separator = ' ')
+      clause.split(', ').collect { |field|
+        case field
+        when /COALESCE/, "'')"
+          field
+        else
+          "COALESCE(CAST(#{field} as varchar), '')"
+        end
+      }.join(" || '#{separator}' || ")
+    end
+    
+    def group_concatenate(clause, separator = ' ')
+      "array_to_string(array_accum(#{clause}), '#{separator}')"
+    end
+    
+    def cast_to_string(clause)
+      clause
+    end
+    
+    def cast_to_datetime(clause)
+      "cast(extract(epoch from #{clause}) as int)"
+    end
+    
+    def cast_to_unsigned(clause)
+      clause
+    end
+    
+    def convert_nulls(clause, default = '')
+      default = "'#{default}'"  if default.is_a?(String)
+      default = 'NULL'          if default.nil?
+      
+      "COALESCE(#{clause}, #{default})"
+    end
+    
+    def boolean(value)
+      value ? 'TRUE' : 'FALSE'
+    end
+    
+    def crc(clause, blank_to_null = false)
+      clause = "NULLIF(#{clause},'')" if blank_to_null
+      "crc32(#{clause})"
+    end
+    
+    def utf8_query_pre
+      nil
+    end
+    
+    def time_difference(diff)
+      "current_timestamp - interval '#{diff} seconds'"
+    end
+    
+    private
+    
+    def execute(command, output_error = false)
+      connection.execute "begin"
+      connection.execute "savepoint ts"
+      begin
+        connection.execute command
+      rescue StandardError => err
+        puts err if output_error
+        connection.execute "rollback to savepoint ts"
+      end
+      connection.execute "release savepoint ts"
+      connection.execute "commit"
+    end
+    
+    def create_array_accum_function
+      if connection.raw_connection.respond_to?(:server_version) && connection.raw_connection.server_version > 80200
+        execute <<-SQL
+          CREATE AGGREGATE array_accum (anyelement)
+          (
+              sfunc = array_append,
+              stype = anyarray,
+              initcond = '{}'
+          );
+        SQL
+      else
+        execute <<-SQL
+          CREATE AGGREGATE array_accum
+          (
+              basetype = anyelement,
+              sfunc = array_append,
+              stype = anyarray,
+              initcond = '{}'
+          );
+        SQL
+      end
+    end
+    
+    def create_crc32_function
+      execute "CREATE LANGUAGE 'plpgsql';"
+      function = <<-SQL
+        CREATE OR REPLACE FUNCTION crc32(word text)
+        RETURNS bigint AS $$
+          DECLARE tmp bigint;
+          DECLARE i int;
+          DECLARE j int;
+          DECLARE word_array bytea;
+          BEGIN
+            i = 0;
+            tmp = 4294967295;
+            word_array = decode(replace(word, E'\\\\', E'\\\\\\\\'), 'escape');
+            LOOP
+              tmp = (tmp # get_byte(word_array, i))::bigint;
+              i = i + 1;
+              j = 0;
+              LOOP
+                tmp = ((tmp >> 1) # (3988292384 * (tmp & 1)))::bigint;
+                j = j + 1;
+                IF j >= 8 THEN
+                  EXIT;
+                END IF;
+              END LOOP;
+              IF i >= char_length(word) THEN
+                EXIT;
+              END IF;
+            END LOOP;
+            return (tmp # 4294967295);
+          END
+        $$ IMMUTABLE STRICT LANGUAGE plpgsql;
+      SQL
+      execute function, true
+    end
+  end
+end

data/lib/thinking_sphinx/association.rb

@@ -0,0 +1,243 @@
+module ThinkingSphinx
+  # Association tracks a specific reflection and join to reference data that
+  # isn't in the base model. Very much an internal class for Thinking Sphinx -
+  # perhaps because I feel it's not as strong (or simple) as most of the rest.
+  #
+  class Association
+    attr_accessor :parent, :reflection, :join, :columns
+
+    # Create a new association by passing in the parent association, and the
+    # corresponding reflection instance. If there is no parent, pass in nil.
+    #
+    #   top   = Association.new nil, top_reflection
+    #   child = Association.new top, child_reflection
+    #
+    def initialize(parent, reflection)
+      @parent, @reflection = parent, reflection
+      @columns = []
+      @children = {}
+    end
+
+    # Get the children associations for a given association name. The only time
+    # that there'll actually be more than one association is when the
+    # relationship is polymorphic. To keep things simple though, it will always
+    # be an Array that gets returned (an empty one if no matches).
+    #
+    #   # where pages is an association on the class tied to the reflection.
+    #   association.children(:pages)
+    #
+    def children(assoc)
+      @children[assoc] ||= Association.children(@reflection.klass, assoc, self)
+    end
+
+    # Get the children associations for a given class, association name and
+    # parent association. Much like the instance method of the same name, it
+    # will return an empty array if no associations have the name, and only
+    # have multiple association instances if the underlying relationship is
+    # polymorphic.
+    #
+    #   Association.children(User, :pages, user_association)
+    #
+    def self.children(klass, assoc, parent=nil)
+      ref = klass.reflect_on_association(assoc)
+
+      return [] if ref.nil?
+      return [Association.new(parent, ref)] unless ref.options[:polymorphic]
+
+      # association is polymorphic - create associations for each
+      # non-polymorphic reflection.
+      polymorphic_classes(ref).collect { |klass|
+        Association.new parent, ::ActiveRecord::Reflection::AssociationReflection.new(
+          ref.macro,
+          "#{ref.name}_#{klass.name}".to_sym,
+          casted_options(klass, ref),
+          ref.active_record
+        )
+      }
+    end
+
+    # Link up the join for this model from a base join - and set parent
+    # associations' joins recursively.
+    #
+    def join_to(base_join)
+      parent.join_to(base_join) if parent && parent.join.nil?
+
+      @join ||= ::ActiveRecord::Associations::ClassMethods::JoinDependency::JoinAssociation.new(
+        @reflection, base_join, parent ? parent.join : base_join.joins.first
+      )
+
+      @join.aliased_table_name << '_join' if is_many? and !(@join.aliased_table_name =~ /_join$/)
+    end
+
+    def column_def_sql(column_name)
+      if column_name.to_s =~ /_id$/
+        "#{quote_column(column_name)} int(11) NOT NULL"
+      elsif column_name.to_s =~ /_ids$/
+        "#{quote_column(column_name)} BLOB default NULL"
+      else
+        "#{quote_column(column_name)} TEXT default NULL"
+      end
+    end
+
+    # Returns the association's flattened columns as an indexed tempory table
+    # to speed up the joins and avoid a huge group by statement
+    #
+    def to_flat_sql
+      primary_key = quote_column(@reflection.primary_key_name)
+      temp_table_name = quote_table_name(@join.aliased_table_name)
+
+      create = "CREATE TEMPORARY TABLE %s (%s, PRIMARY KEY (%s))" % [
+        temp_table_name,
+        (columns.uniq.map(&:__name) << @reflection.primary_key_name).map { |column| column_def_sql(column) }.join(', '),
+        primary_key
+      ]
+
+      insert = "INSERT INTO %s (%s)" % [
+        temp_table_name,
+        (columns.uniq.map { |column| quote_column(column.__name) } << primary_key).join(', ')
+      ]
+
+      insert << " SELECT %s FROM %s" % [
+        (columns.uniq.map { |column| column_sql(column.__name) } << primary_key).join(', '),
+        @reflection.quoted_table_name
+      ]
+
+      insert << " INNER JOIN %s ON %s.%s = %s.%s" % [
+        quote_table_name(@reflection.options[:join_table]),
+        quote_table_name(@reflection.options[:join_table]),
+        quote_column(@reflection.association_foreign_key),
+        @reflection.quoted_table_name,
+        quote_column(@join.primary_key)
+      ] if @reflection.macro == :has_and_belongs_to_many
+
+      insert << " GROUP BY %s" % [
+        primary_key
+      ]
+
+      [create, insert]
+    end
+
+    # Returns the association's join SQL statements - and it replaces
+    # ::ts_join_alias:: with the aliased table name so the generated reflection
+    # join conditions avoid column name collisions.
+    #
+    def to_sql
+      if is_many?
+        temp_table_name = quote_table_name(@join.aliased_table_name)
+        " LEFT OUTER JOIN %s ON %s.%s = %s.%s" % [
+          temp_table_name,
+          temp_table_name,
+          quote_column(@reflection.primary_key_name),
+          quote_table_name(@join.parent.aliased_table_name),
+          quote_column(@join.parent.primary_key)
+        ]
+      else
+        @join.association_join.gsub(/::ts_join_alias::/,
+          "#{@reflection.klass.connection.quote_table_name(@join.parent.aliased_table_name)}"
+        )
+      end
+    end
+
+    # Returns true if the association - or a parent - is a has_many or
+    # has_and_belongs_to_many.
+    #
+    def is_many?
+      case @reflection.macro
+      when :has_many, :has_and_belongs_to_many
+        true
+      else
+        @parent ? @parent.is_many? : false
+      end
+    end
+
+    def adapter
+      @adapter ||= @reflection.klass.sphinx_database_adapter
+    end
+
+    def column_sql(column)
+       separator = column == :id ? ',' : ' '
+      "#{adapter.group_concatenate(column, separator)} AS #{quote_column(column)}"
+    end
+
+    def quote_column(column)
+      @reflection.klass.connection.quote_column_name(column)
+    end
+
+    def quote_table_name(table_name)
+      @reflection.klass.connection.quote_table_name(table_name)
+    end
+
+    # Returns an array of all the associations that lead to this one - starting
+    # with the top level all the way to the current association object.
+    #
+    def ancestors
+      (parent ? parent.ancestors : []) << self
+    end
+
+    def has_column?(column)
+      @reflection.klass.column_names.include?(column.to_s)
+    end
+
+    def primary_key_from_reflection
+      if @reflection.options[:through]
+        @reflection.source_reflection.options[:foreign_key] ||
+        @reflection.source_reflection.primary_key_name
+      elsif @reflection.macro == :has_and_belongs_to_many
+        @reflection.association_foreign_key
+      else
+        nil
+      end
+    end
+
+    def table
+      if @reflection.options[:through] ||
+        @reflection.macro == :has_and_belongs_to_many
+        @join.aliased_join_table_name
+      else
+        @join.aliased_table_name
+      end
+    end
+
+    private
+
+    # Returns all the objects that could be currently instantiated from a
+    # polymorphic association. This is pretty damn fast if there's an index on
+    # the foreign type column - but if there isn't, it can take a while if you
+    # have a lot of data.
+    #
+    def self.polymorphic_classes(ref)
+      ref.active_record.connection.select_all(
+        "SELECT DISTINCT #{ref.options[:foreign_type]} " +
+        "FROM #{ref.active_record.table_name} " +
+        "WHERE #{ref.options[:foreign_type]} IS NOT NULL"
+      ).collect { |row|
+        row[ref.options[:foreign_type]].constantize
+      }
+    end
+
+    # Returns a new set of options for an association that mimics an existing
+    # polymorphic relationship for a specific class. It adds a condition to
+    # filter by the appropriate object.
+    #
+    def self.casted_options(klass, ref)
+      options = ref.options.clone
+      options[:polymorphic]   = nil
+      options[:class_name]    = klass.name
+      options[:foreign_key] ||= "#{ref.name}_id"
+
+      quoted_foreign_type = klass.connection.quote_column_name ref.options[:foreign_type]
+      case options[:conditions]
+      when nil
+        options[:conditions] = "::ts_join_alias::.#{quoted_foreign_type} = '#{klass.name}'"
+      when Array
+        options[:conditions] << "::ts_join_alias::.#{quoted_foreign_type} = '#{klass.name}'"
+      when Hash
+        options[:conditions].merge!(ref.options[:foreign_type] => klass.name)
+      else
+        options[:conditions] << " AND ::ts_join_alias::.#{quoted_foreign_type} = '#{klass.name}'"
+      end
+
+      options
+    end
+  end
+end