DrMark-thinking-sphinx 0.9.9 → 1.1.6

data/lib/thinking_sphinx/index.rb

@@ -9,7 +9,8 @@ module ThinkingSphinx
   # Enjoy.
   # 
   class Index
-    attr_accessor :model, :fields, :attributes, :conditions, :delta, :options
+    attr_accessor :model, :fields, :attributes, :conditions, :groupings,
+      :delta_object, :options
     
     # Create a new index instance by passing in the model it is tied to, and
     # a block to build it with (optional but recommended). For documentation
@@ -31,8 +32,9 @@ module ThinkingSphinx
       @fields       = []
       @attributes   = []
       @conditions   = []
+      @groupings    = []
       @options      = {}
-      @delta        = false
+      @delta_object = nil
       
       initialize_from_builder(&block) if block_given?
     end
@@ -45,65 +47,36 @@ module ThinkingSphinx
       model.name.underscore.tr(':/\\', '_')
     end
     
-    def empty?(part = :core)
-      config = ThinkingSphinx::Configuration.new
-      File.size?("#{config.searchd_file_path}/#{self.name}_#{part}.spa").nil?
+    def to_riddle_for_core(offset, index)
+      add_internal_attributes_and_facets
+      link!
+      
+      source = Riddle::Configuration::SQLSource.new(
+        "#{name}_core_#{index}", adapter.sphinx_identifier
+      )
+      
+      set_source_database_settings  source
+      set_source_attributes         source, offset
+      set_source_sql                source, offset
+      set_source_settings           source
+      
+      source
     end
     
-    def to_config(model, index, database_conf, charset_type, offset)
-      # Set up associations and joins
-      add_internal_attributes
+    def to_riddle_for_delta(offset, index)
+      add_internal_attributes_and_facets
       link!
       
-      attr_sources = attributes.collect { |attrib|
-        attrib.to_sphinx_clause
-      }.join("\n  ")
-      
-      db_adapter = case adapter
-      when :postgres
-        "pgsql"
-      when :mysql
-        "mysql"
-      else
-        raise "Unsupported Database Adapter: Sphinx only supports MySQL and PosgreSQL"
-      end
+      source = Riddle::Configuration::SQLSource.new(
+        "#{name}_delta_#{index}", adapter.sphinx_identifier
+      )
+      source.parent = "#{name}_core_#{index}"
       
-      config = <<-SOURCE
-
-source #{self.class.name(model)}_#{index}_core
-{
-type     = #{db_adapter}
-sql_host = #{database_conf[:host] || "localhost"}
-sql_user = #{database_conf[:username] || database_conf[:user]}
-sql_pass = #{(database_conf[:password] || "").gsub('#', '\#')}
-sql_db   = #{database_conf[:database]}
-#{"sql_sock = #{database_conf[:socket]}" unless database_conf[:socket].blank? }
-
-sql_query_pre    = #{charset_type == "utf-8" && adapter == :mysql ? "SET NAMES utf8" : ""}
-#{"sql_query_pre    = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]}
-sql_query_pre    = #{to_sql_query_pre}
-sql_query        = #{to_sql(:offset => offset).gsub(/\n/, ' ')}
-sql_query_range  = #{to_sql_query_range :offset => offset}
-sql_query_info   = #{to_sql_query_info(offset)}
-#{attr_sources}
-}
-      SOURCE
-      
-      if delta?
-        config += <<-SOURCE
-
-source #{self.class.name(model)}_#{index}_delta : #{self.class.name(model)}_#{index}_core
-{
-sql_query_pre    = 
-sql_query_pre    = #{charset_type == "utf-8" && adapter == :mysql ? "SET NAMES utf8" : ""}
-#{"sql_query_pre    = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]}
-sql_query        = #{to_sql(:delta => true, :offset => offset).gsub(/\n/, ' ')}
-sql_query_range  = #{to_sql_query_range :offset => offset, :delta => true}
-}
-        SOURCE
-      end
+      set_source_database_settings  source
+      set_source_attributes         source, offset
+      set_source_sql                source, offset, true
       
-      config
+      source
     end
     
     # Link all the fields and associations to their corresponding
@@ -133,111 +106,16 @@ sql_query_range  = #{to_sql_query_range :offset => offset, :delta => true}
       }
     end
     
-    # Generates the big SQL statement to get the data back for all the fields
-    # and attributes, using all the relevant association joins. If you want
-    # the version filtered for delta values, send through :delta => true in the
-    # options. Won't do much though if the index isn't set up to support a
-    # delta sibling.
-    # 
-    # Examples:
-    # 
-    #   index.to_sql
-    #   index.to_sql(:delta => true)
-    #
-    def to_sql(options={})
-      assocs = all_associations
-      
-      where_clause = ""
-      if self.delta?
-        where_clause << " AND #{@model.quoted_table_name}.#{quote_column('delta')}" +" = #{options[:delta] ? db_boolean(true) : db_boolean(false)}"
-      end
-      unless @conditions.empty?
-        where_clause << " AND " << @conditions.join(" AND ")
-      end
-      
-      unique_id_expr = "* #{ThinkingSphinx.indexed_models.size} + #{options[:offset] || 0}"
-      
-      sql = <<-SQL
-SELECT #{ (
-  ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] + 
-  @fields.collect { |field| field.to_select_sql } +
-  @attributes.collect { |attribute| attribute.to_select_sql }
-).join(", ") }
-FROM #{ @model.table_name }
-  #{ assocs.collect { |assoc| assoc.to_sql }.join(' ') }
-WHERE #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} >= $start
-  AND #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} <= $end
-  #{ where_clause }
-GROUP BY #{ (
-  ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + 
-  @fields.collect { |field| field.to_group_sql }.compact +
-  @attributes.collect { |attribute| attribute.to_group_sql }.compact
-).join(", ") }
-      SQL
-      
-      if @model.connection.class.name == "ActiveRecord::ConnectionAdapters::MysqlAdapter"
-        sql += " ORDER BY NULL"
-      end
-      
-      sql
-    end
-    
-    # Simple helper method for the query info SQL - which is a statement that
-    # returns the single row for a corresponding id.
-    # 
-    def to_sql_query_info(offset)
-      "SELECT * FROM #{@model.quoted_table_name} WHERE " +
-      " #{quote_column(@model.primary_key)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})"
-    end
-    
-    # Simple helper method for the query range SQL - which is a statement that
-    # returns minimum and maximum id values. These can be filtered by delta -
-    # so pass in :delta => true to get the delta version of the SQL.
-    # 
-    def to_sql_query_range(options={})
-      unique_id_expr = "* #{ThinkingSphinx.indexed_models.size} + #{options[:offset] || 0}"
-      
-      min_statement = "MIN(#{quote_column(@model.primary_key)} #{unique_id_expr})"
-      max_statement = "MAX(#{quote_column(@model.primary_key)} #{unique_id_expr})"
-      
-      # Fix to handle Sphinx PostgreSQL bug (it doesn't like NULLs or 0's)
-      if adapter == :postgres
-        min_statement = "COALESCE(#{min_statement}, 1)"
-        max_statement = "COALESCE(#{max_statement}, 1)"
-      end
-      
-      sql = "SELECT #{min_statement}, #{max_statement} " +
-            "FROM #{@model.quoted_table_name} "
-      sql << "WHERE #{@model.quoted_table_name}.#{quote_column('delta')} " + 
-            "= #{options[:delta] ? db_boolean(true) : db_boolean(false)}" if self.delta?
-      sql
-    end
-    
-    # Returns the SQL query to run before a full index - ie: nothing unless the
-    # index has a delta, and then it's an update statement to set delta values
-    # back to 0.
-    #
-    def to_sql_query_pre
-      self.delta? ? "UPDATE #{@model.quoted_table_name} SET #{quote_column('delta')} = #{db_boolean(false)}" : ""
-    end
-    
     # Flag to indicate whether this index has a corresponding delta index.
     #
     def delta?
-      @delta
+      !@delta_object.nil?
     end
     
     def adapter
-      @adapter ||= case @model.connection.class.name
-      when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
-        :mysql
-      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-        :postgres
-      else
-        raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL"
-      end
+      @adapter ||= @model.sphinx_database_adapter
     end
-    
+        
     def prefix_fields
       @fields.select { |field| field.prefixes }
     end
@@ -245,11 +123,23 @@ GROUP BY #{ (
     def infix_fields
       @fields.select { |field| field.infixes }
     end
+        
+    def index_options
+      all_index_options = ThinkingSphinx::Configuration.instance.index_options.clone
+      @options.keys.select { |key|
+        ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s)
+      }.each { |key| all_index_options[key.to_sym] = @options[key] }
+      all_index_options
+    end
+        
+    def quote_column(column)
+      @model.connection.quote_column_name(column)
+    end
     
     private
     
-    def quote_column(column)
-      @model.connection.quote_column_name(column)
+    def utf8?
+      self.index_options[:charset_type] == "utf-8"
     end
     
     # Does all the magic with the block provided to the base #initialize.
@@ -269,12 +159,33 @@ GROUP BY #{ (
         stored_class = @model.store_full_sti_class ? @model.name : @model.name.demodulize
         builder.where("#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)} = '#{stored_class}'")
       end
-
-      @fields     = builder.fields
-      @attributes = builder.attributes
-      @conditions = builder.conditions
-      @delta      = builder.properties[:delta]
-      @options    = builder.properties.except(:delta)
+      
+      set_model = Proc.new { |item| item.model = @model }
+      
+      @fields       = builder.fields &set_model
+      @attributes   = builder.attributes.each &set_model
+      @conditions   = builder.conditions
+      @groupings    = builder.groupings
+      @delta_object = ThinkingSphinx::Deltas.parse self, builder.properties
+      @options      = builder.properties
+      
+      is_faceted = Proc.new { |item| item.faceted }
+      add_facet  = Proc.new { |item| @model.sphinx_facets << item.to_facet }
+      
+      @model.sphinx_facets ||= []
+      @fields.select(    &is_faceted).each &add_facet
+      @attributes.select(&is_faceted).each &add_facet
+      
+      # We want to make sure that if the database doesn't exist, then Thinking
+      # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop
+      # and db:migrate). It's a bit hacky, but I can't think of a better way.
+    rescue StandardError => err
+      case err.class.name
+      when "Mysql::Error", "Java::JavaSql::SQLException", "ActiveRecord::StatementInvalid"
+        return
+      else
+        raise err
+      end
     end
     
     # Returns all associations used amongst all the fields and attributes.
@@ -289,8 +200,8 @@ GROUP BY #{ (
         }.flatten +
         # attribute associations
         @attributes.collect { |attrib|
-          attrib.associations.values
-        }.flatten
+          attrib.associations.values if attrib.include_as_association?
+        }.compact.flatten
       ).uniq.collect { |assoc|
         # get ancestors as well as column-level associations
         assoc.ancestors
@@ -324,53 +235,189 @@ GROUP BY #{ (
       @associations[key] ||= Association.children(@model, key)
     end
 
-    # Returns the proper boolean value string literal for the
-    # current database adapter.
-    #
-    def db_boolean(val)
-      if adapter == :postgres
-        val ? 'TRUE' : 'FALSE'
+    def crc_column
+      if @model.column_names.include?(@model.inheritance_column)
+        adapter.cast_to_unsigned(adapter.convert_nulls(
+          adapter.crc(adapter.quote_with_table(@model.inheritance_column), true),
+          @model.to_crc32
+        ))
       else
-        val ? '1' : '0'
+        @model.to_crc32.to_s
       end
     end
     
-    def crc_column
-      if adapter == :postgres
-        @model.to_crc32.to_s
-      elsif @model.column_names.include?(@model.inheritance_column)
-        "IFNULL(CRC32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})"
+    def add_internal_attributes_and_facets
+      add_internal_attribute :sphinx_internal_id, :integer, @model.primary_key.to_sym
+      add_internal_attribute :class_crc,          :integer, crc_column, true
+      add_internal_attribute :subclass_crcs,      :multi,   subclasses_to_s
+      add_internal_attribute :sphinx_deleted,     :integer, "0"
+      
+      add_internal_facet :class_crc
+    end
+    
+    def add_internal_attribute(name, type, contents, facet = false)
+      return unless attribute_by_alias(name).nil?
+      
+      @attributes << Attribute.new(
+        FauxColumn.new(contents),
+        :type   => type,
+        :as     => name,
+        :facet  => facet
+      )
+    end
+    
+    def add_internal_facet(name)
+      return unless facet_by_alias(name).nil?
+      
+      @model.sphinx_facets << ClassFacet.new(attribute_by_alias(name))
+    end
+    
+    def attribute_by_alias(attr_alias)
+      @attributes.detect { |attrib| attrib.alias == attr_alias }
+    end
+    
+    def facet_by_alias(name)
+      @model.sphinx_facets.detect { |facet| facet.name == name }
+    end
+    
+    def subclasses_to_s
+      "'" + (@model.send(:subclasses).collect { |klass|
+        klass.to_crc32.to_s
+      } << @model.to_crc32.to_s).join(",") + "'"
+    end
+        
+    def set_source_database_settings(source)
+      config = @model.connection.instance_variable_get(:@config)
+      
+      source.sql_host = config[:host]           || "localhost"
+      source.sql_user = config[:username]       || config[:user] || ""
+      source.sql_pass = (config[:password].to_s || "").gsub('#', '\#')
+      source.sql_db   = config[:database]
+      source.sql_port = config[:port]
+      source.sql_sock = config[:socket]
+    end
+    
+    def set_source_attributes(source, offset = nil)
+      attributes.each do |attrib|
+        source.send(attrib.type_to_config) << attrib.config_value(offset)
+      end
+    end
+    
+    def set_source_sql(source, offset, delta = false)
+      source.sql_query        = to_sql(:offset => offset, :delta => delta).gsub(/\n/, ' ')
+      source.sql_query_range  = to_sql_query_range(:delta => delta)
+      source.sql_query_info   = to_sql_query_info(offset)
+      
+      source.sql_query_pre += send(!delta ? :sql_query_pre_for_core : :sql_query_pre_for_delta)
+      
+      if @options[:group_concat_max_len]
+        source.sql_query_pre << "SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}"
+      end
+      
+      source.sql_query_pre += [adapter.utf8_query_pre].compact if utf8?
+    end
+    
+    def set_source_settings(source)
+      ThinkingSphinx::Configuration.instance.source_options.each do |key, value|
+        source.send("#{key}=".to_sym, value)
+      end
+      
+      @options.each do |key, value|
+        source.send("#{key}=".to_sym, value) if ThinkingSphinx::Configuration::SourceOptions.include?(key.to_s) && !value.nil?
+      end
+    end
+    
+    def sql_query_pre_for_core
+      if self.delta? && !@delta_object.reset_query(@model).blank?
+        [@delta_object.reset_query(@model)]
       else
-        @model.to_crc32.to_s
+        []
       end
     end
     
-    def add_internal_attributes
-      @attributes << Attribute.new(
-        FauxColumn.new(:id),
-        :type => :integer,
-        :as   => :sphinx_internal_id
-      ) unless @attributes.detect { |attr| attr.alias == :sphinx_internal_id }
+    def sql_query_pre_for_delta
+      [""]
+    end
+    
+    # Generates the big SQL statement to get the data back for all the fields
+    # and attributes, using all the relevant association joins. If you want
+    # the version filtered for delta values, send through :delta => true in the
+    # options. Won't do much though if the index isn't set up to support a
+    # delta sibling.
+    # 
+    # Examples:
+    # 
+    #   index.to_sql
+    #   index.to_sql(:delta => true)
+    #
+    def to_sql(options={})
+      assocs = all_associations
       
-      @attributes << Attribute.new(
-        FauxColumn.new(crc_column),
-        :type => :integer,
-        :as   => :class_crc
-      ) unless @attributes.detect { |attr| attr.alias == :class_crc }
+      where_clause = ""
+      if self.delta? && !@delta_object.clause(@model, options[:delta]).blank?
+        where_clause << " AND #{@delta_object.clause(@model, options[:delta])}"
+      end
+      unless @conditions.empty?
+        where_clause << " AND " << @conditions.join(" AND ")
+      end
       
-      @attributes << Attribute.new(
-        FauxColumn.new("'" + (@model.send(:subclasses).collect { |klass|
-          klass.to_crc32.to_s
-        } << @model.to_crc32.to_s).join(",") + "'"),
-        :type => :multi,
-        :as   => :subclass_crcs
-      ) unless @attributes.detect { |attr| attr.alias == :subclass_crcs }
+      internal_groupings = []
+      if @model.column_names.include?(@model.inheritance_column)
+         internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}"
+      end
       
-      @attributes << Attribute.new(
-        FauxColumn.new("0"),
-        :type => :integer,
-        :as   => :sphinx_deleted
-      ) unless @attributes.detect { |attr| attr.alias == :sphinx_deleted }
+      unique_id_expr = ThinkingSphinx.unique_id_expression(options[:offset])
+      
+      sql = <<-SQL
+SELECT #{ (
+  ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] + 
+  @fields.collect { |field| field.to_select_sql } +
+  @attributes.collect { |attribute| attribute.to_select_sql }
+).compact.join(", ") }
+FROM #{ @model.table_name }
+  #{ assocs.collect { |assoc| assoc.to_sql }.join(' ') }
+WHERE #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start
+  AND #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} <= $end
+  #{ where_clause }
+GROUP BY #{ (
+  ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + 
+  @fields.collect { |field| field.to_group_sql }.compact +
+  @attributes.collect { |attribute| attribute.to_group_sql }.compact +
+  @groupings + internal_groupings
+).join(", ") }
+      SQL
+      
+      sql += " ORDER BY NULL" if adapter.sphinx_identifier == "mysql"
+      sql
+    end
+    
+    # Simple helper method for the query info SQL - which is a statement that
+    # returns the single row for a corresponding id.
+    # 
+    def to_sql_query_info(offset)
+      "SELECT * FROM #{@model.quoted_table_name} WHERE " +
+      " #{quote_column(@model.primary_key)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})"
+    end
+    
+    # Simple helper method for the query range SQL - which is a statement that
+    # returns minimum and maximum id values. These can be filtered by delta -
+    # so pass in :delta => true to get the delta version of the SQL.
+    # 
+    def to_sql_query_range(options={})
+      min_statement = adapter.convert_nulls(
+        "MIN(#{quote_column(@model.primary_key)})", 1
+      )
+      max_statement = adapter.convert_nulls(
+        "MAX(#{quote_column(@model.primary_key)})", 1
+      )
+      
+      sql = "SELECT #{min_statement}, #{max_statement} " +
+            "FROM #{@model.quoted_table_name} "
+      if self.delta? && !@delta_object.clause(@model, options[:delta]).blank?
+        sql << "WHERE #{@delta_object.clause(@model, options[:delta])}"
+      end
+      
+      sql
     end
   end
 end