jaikoo-thinking-sphinx 0.9.10

data/lib/thinking_sphinx/index.rb

@@ -0,0 +1,432 @@
+require 'thinking_sphinx/index/builder'
+require 'thinking_sphinx/index/faux_column'
+
+module ThinkingSphinx
+  # The Index class is a ruby representation of a Sphinx source (not a Sphinx
+  # index - yes, I know it's a little confusing. You'll manage). This is
+  # another 'internal' Thinking Sphinx class - if you're using it directly,
+  # you either know what you're doing, or messing with things beyond your ken.
+  # Enjoy.
+  # 
+  class Index
+    attr_accessor :model, :fields, :attributes, :conditions, :groupings,
+      :delta, :options
+    
+    # Create a new index instance by passing in the model it is tied to, and
+    # a block to build it with (optional but recommended). For documentation
+    # on the syntax for inside the block, the Builder class is what you want.
+    #
+    # Quick Example:
+    #
+    #   Index.new(User) do
+    #     indexes login, email
+    #     
+    #     has created_at
+    #     
+    #     set_property :delta => true
+    #   end
+    #
+    def initialize(model, &block)
+      @model        = model
+      @associations = {}
+      @fields       = []
+      @attributes   = []
+      @conditions   = []
+      @groupings    = []
+      @options      = {}
+      @delta        = false
+      
+      initialize_from_builder(&block) if block_given?
+    end
+    
+    def name
+      self.class.name(@model)
+    end
+    
+    def self.name(model)
+      model.name.underscore.tr(':/\\', '_')
+    end
+    
+    def empty?(part = :core)
+      config = ThinkingSphinx::Configuration.instance
+      File.size?("#{config.searchd_file_path}/#{self.name}_#{part}.spa").nil?
+    end
+    
+    def to_config(model, index, database_conf, offset)
+      # Set up associations and joins
+      add_internal_attributes
+      link!
+      
+      attr_sources = attributes.collect { |attrib|
+        attrib.to_sphinx_clause
+      }.join("\n  ")
+      
+      db_adapter = case adapter
+      when :postgres
+        "pgsql"
+      when :mysql
+        "mysql"
+      else
+        raise "Unsupported Database Adapter: Sphinx only supports MySQL and PosgreSQL"
+      end
+      
+      config = <<-SOURCE
+
+source #{self.class.name(model)}_#{index}_core
+{
+type     = #{db_adapter}
+sql_host = #{database_conf[:host] || "localhost"}
+sql_user = #{database_conf[:username] || database_conf[:user]}
+sql_pass = #{(database_conf[:password] || "").gsub('#', '\#')}
+sql_db   = #{database_conf[:database]}
+#{"sql_port = #{database_conf[:port]}" unless database_conf[:port].blank? }
+#{"sql_sock = #{database_conf[:socket]}" unless database_conf[:socket].blank? }
+
+sql_query_pre    = #{utf8? && adapter == :mysql ? "SET NAMES utf8" : ""}
+#{"sql_query_pre    = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]}
+sql_query_pre    = #{to_sql_query_pre}
+sql_query        = #{to_sql(:offset => offset).gsub(/\n/, ' ')}
+sql_query_range  = #{to_sql_query_range}
+sql_query_info   = #{to_sql_query_info(offset)}
+#{attr_sources}
+#{ThinkingSphinx::Configuration.instance.hash_to_config(self.source_options)}
+}
+      SOURCE
+      
+      if delta?
+        config += <<-SOURCE
+
+source #{self.class.name(model)}_#{index}_delta : #{self.class.name(model)}_#{index}_core
+{
+sql_query_pre    = 
+sql_query_pre    = #{utf8? && adapter == :mysql ? "SET NAMES utf8" : ""}
+#{"sql_query_pre    = SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}" if @options[:group_concat_max_len]}
+sql_query        = #{to_sql(:delta => true, :offset => offset).gsub(/\n/, ' ')}
+sql_query_range  = #{to_sql_query_range :delta => true}
+}
+        SOURCE
+      end
+      
+      config
+    end
+    
+    # Link all the fields and associations to their corresponding
+    # associations and joins. This _must_ be called before interrogating
+    # the index's fields and associations for anything that may reference
+    # their SQL structure.
+    # 
+    def link!
+      base = ::ActiveRecord::Associations::ClassMethods::JoinDependency.new(
+        @model, [], nil
+      )
+      
+      @fields.each { |field|
+        field.model ||= @model
+        field.columns.each { |col|
+          field.associations[col] = associations(col.__stack.clone)
+          field.associations[col].each { |assoc| assoc.join_to(base) }
+        }
+      }
+      
+      @attributes.each { |attribute|
+        attribute.model ||= @model
+        attribute.columns.each { |col|
+          attribute.associations[col] = associations(col.__stack.clone)
+          attribute.associations[col].each { |assoc| assoc.join_to(base) }
+        }
+      }
+    end
+    
+    # Generates the big SQL statement to get the data back for all the fields
+    # and attributes, using all the relevant association joins. If you want
+    # the version filtered for delta values, send through :delta => true in the
+    # options. Won't do much though if the index isn't set up to support a
+    # delta sibling.
+    # 
+    # Examples:
+    # 
+    #   index.to_sql
+    #   index.to_sql(:delta => true)
+    #
+    def to_sql(options={})
+      assocs = all_associations
+      
+      where_clause = ""
+      if self.delta?
+        where_clause << " AND #{@model.quoted_table_name}.#{quote_column('delta')}" +" = #{options[:delta] ? db_boolean(true) : db_boolean(false)}"
+      end
+      unless @conditions.empty?
+        where_clause << " AND " << @conditions.join(" AND ")
+      end
+      
+      internal_groupings = []
+      if @model.column_names.include?(@model.inheritance_column)
+         internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}"
+      end
+      
+      unique_id_expr = "* #{ThinkingSphinx.indexed_models.size} + #{options[:offset] || 0}"
+      
+      sql = <<-SQL
+SELECT #{ (
+  ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] + 
+  @fields.collect { |field| field.to_select_sql } +
+  @attributes.collect { |attribute| attribute.to_select_sql }
+).join(", ") }
+FROM #{ @model.table_name }
+  #{ assocs.collect { |assoc| assoc.to_sql }.join(' ') }
+WHERE #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start
+  AND #{@model.quoted_table_name}.#{quote_column(@model.primary_key)} <= $end
+  #{ where_clause }
+GROUP BY #{ (
+  ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + 
+  @fields.collect { |field| field.to_group_sql }.compact +
+  @attributes.collect { |attribute| attribute.to_group_sql }.compact +
+  @groupings + internal_groupings
+).join(", ") }
+      SQL
+      
+      if @model.connection.class.name == "ActiveRecord::ConnectionAdapters::MysqlAdapter"
+        sql += " ORDER BY NULL"
+      end
+      
+      sql
+    end
+    
+    # Simple helper method for the query info SQL - which is a statement that
+    # returns the single row for a corresponding id.
+    # 
+    def to_sql_query_info(offset)
+      "SELECT * FROM #{@model.quoted_table_name} WHERE " +
+      " #{quote_column(@model.primary_key)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})"
+    end
+    
+    # Simple helper method for the query range SQL - which is a statement that
+    # returns minimum and maximum id values. These can be filtered by delta -
+    # so pass in :delta => true to get the delta version of the SQL.
+    # 
+    def to_sql_query_range(options={})
+      min_statement = "MIN(#{quote_column(@model.primary_key)})"
+      max_statement = "MAX(#{quote_column(@model.primary_key)})"
+      
+      # Fix to handle Sphinx PostgreSQL bug (it doesn't like NULLs or 0's)
+      if adapter == :postgres
+        min_statement = "COALESCE(#{min_statement}, 1)"
+        max_statement = "COALESCE(#{max_statement}, 1)"
+      end
+      
+      sql = "SELECT #{min_statement}, #{max_statement} " +
+            "FROM #{@model.quoted_table_name} "
+      sql << "WHERE #{@model.quoted_table_name}.#{quote_column('delta')} " + 
+            "= #{options[:delta] ? db_boolean(true) : db_boolean(false)}" if self.delta?
+      sql
+    end
+    
+    # Returns the SQL query to run before a full index - ie: nothing unless the
+    # index has a delta, and then it's an update statement to set delta values
+    # back to 0.
+    #
+    def to_sql_query_pre
+      self.delta? ? "UPDATE #{@model.quoted_table_name} SET #{quote_column('delta')} = #{db_boolean(false)}" : ""
+    end
+    
+    # Flag to indicate whether this index has a corresponding delta index.
+    #
+    def delta?
+      @delta
+    end
+    
+    def adapter
+      @adapter ||= case @model.connection.class.name
+      when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
+        :mysql
+      when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
+        :postgres
+      else
+        raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL"
+      end
+    end
+    
+    def adapter_object
+      @adapter_object ||= ThinkingSphinx::AbstractAdapter.detect(@model)
+    end
+    
+    def prefix_fields
+      @fields.select { |field| field.prefixes }
+    end
+    
+    def infix_fields
+      @fields.select { |field| field.infixes }
+    end
+    
+    def local_index_options
+      @options.keys.inject({}) do |local_options, key|
+        if ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s)
+          local_options[key.to_sym] = @options[key]
+        end
+        local_options
+      end
+    end
+    
+    def index_options
+      all_index_options = ThinkingSphinx::Configuration.instance.index_options.clone
+      @options.keys.select { |key|
+        ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s)
+      }.each { |key| all_index_options[key.to_sym] = @options[key] }
+      all_index_options
+    end
+    
+    def source_options
+      all_source_options = ThinkingSphinx::Configuration.instance.source_options.clone
+      @options.keys.select { |key|
+        ThinkingSphinx::Configuration::SourceOptions.include?(key.to_s)
+      }.each { |key| all_source_options[key.to_sym] = @options[key] }
+      all_source_options
+    end
+    
+    private
+    
+    def utf8?
+      self.index_options[:charset_type] == "utf-8"
+    end
+    
+    def quote_column(column)
+      @model.connection.quote_column_name(column)
+    end
+    
+    # Does all the magic with the block provided to the base #initialize.
+    # Creates a new class subclassed from Builder, and evaluates the block
+    # on it, then pulls all relevant settings - fields, attributes, conditions,
+    # properties - into the new index.
+    # 
+    # Also creates a CRC attribute for the model.
+    # 
+    def initialize_from_builder(&block)
+      builder = Class.new(Builder)
+      builder.setup
+      
+      builder.instance_eval &block
+      
+      unless @model.descends_from_active_record?
+        stored_class = @model.store_full_sti_class ? @model.name : @model.name.demodulize
+        builder.where("#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)} = '#{stored_class}'")
+      end
+
+      @fields     = builder.fields
+      @attributes = builder.attributes
+      @conditions = builder.conditions
+      @groupings  = builder.groupings
+      @delta      = builder.properties[:delta]
+      @options    = builder.properties.except(:delta)
+      
+      # We want to make sure that if the database doesn't exist, then Thinking
+      # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop
+      # and db:migrate). It's a bit hacky, but I can't think of a better way.
+    rescue StandardError => err
+      case err.class.name
+      when "Mysql::Error", "ActiveRecord::StatementInvalid"
+        return
+      else
+        raise err
+      end
+    end
+    
+    # Returns all associations used amongst all the fields and attributes.
+    # This includes all associations between the model and what the actual
+    # columns are from.
+    # 
+    def all_associations
+      @all_associations ||= (
+        # field associations
+        @fields.collect { |field|
+          field.associations.values
+        }.flatten +
+        # attribute associations
+        @attributes.collect { |attrib|
+          attrib.associations.values
+        }.flatten
+      ).uniq.collect { |assoc|
+        # get ancestors as well as column-level associations
+        assoc.ancestors
+      }.flatten.uniq
+    end
+    
+    # Gets a stack of associations for a specific path.
+    # 
+    def associations(path, parent = nil)
+      assocs = []
+      
+      if parent.nil?
+        assocs = association(path.shift)
+      else
+        assocs = parent.children(path.shift)
+      end
+      
+      until path.empty?
+        point = path.shift
+        assocs = assocs.collect { |assoc|
+          assoc.children(point)
+        }.flatten
+      end
+      
+      assocs
+    end
+    
+    # Gets the association stack for a specific key.
+    # 
+    def association(key)
+      @associations[key] ||= Association.children(@model, key)
+    end
+
+    # Returns the proper boolean value string literal for the
+    # current database adapter.
+    #
+    def db_boolean(val)
+      if adapter == :postgres
+        val ? 'TRUE' : 'FALSE'
+      else
+        val ? '1' : '0'
+      end
+    end
+    
+    def crc_column
+      if @model.column_names.include?(@model.inheritance_column)
+        case adapter
+        when :postgres
+          "COALESCE(crc32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})"
+        when :mysql
+          "IFNULL(CRC32(#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}), #{@model.to_crc32.to_s})"
+        end
+      else
+        @model.to_crc32.to_s
+      end
+    end
+    
+    def add_internal_attributes
+      @attributes << Attribute.new(
+        FauxColumn.new(@model.primary_key.to_sym),
+        :type => :integer,
+        :as   => :sphinx_internal_id
+      ) unless @attributes.detect { |attr| attr.alias == :sphinx_internal_id }
+      
+      @attributes << Attribute.new(
+        FauxColumn.new(crc_column),
+        :type => :integer,
+        :as   => :class_crc
+      ) unless @attributes.detect { |attr| attr.alias == :class_crc }
+      
+      @attributes << Attribute.new(
+        FauxColumn.new("'" + (@model.send(:subclasses).collect { |klass|
+          klass.to_crc32.to_s
+        } << @model.to_crc32.to_s).join(",") + "'"),
+        :type => :multi,
+        :as   => :subclass_crcs
+      ) unless @attributes.detect { |attr| attr.alias == :subclass_crcs }
+      
+      @attributes << Attribute.new(
+        FauxColumn.new("0"),
+        :type => :integer,
+        :as   => :sphinx_deleted
+      ) unless @attributes.detect { |attr| attr.alias == :sphinx_deleted }
+    end
+  end
+end

data/lib/thinking_sphinx/index/builder.rb

@@ -0,0 +1,220 @@
+module ThinkingSphinx
+  class Index
+    # The Builder class is the core for the index definition block processing.
+    # There are four methods you really need to pay attention to:
+    # - indexes (aliased to includes and attribute)
+    # - has (aliased to attribute)
+    # - where
+    # - set_property (aliased to set_properties)
+    #
+    # The first two of these methods allow you to define what data makes up
+    # your indexes. #where provides a method to add manual SQL conditions, and
+    # set_property allows you to set some settings on a per-index basis. Check
+    # out each method's documentation for better ideas of usage.
+    # 
+    class Builder
+      class << self
+        # No idea where this is coming from - haven't found it in any ruby or
+        # rails documentation. It's not needed though, so it gets undef'd.
+        # Hopefully the list of methods that get in the way doesn't get too
+        # long.
+        undef_method :parent  if respond_to?(:parent)
+        undef_method :name    if respond_to?(:name)
+        undef_method :id      if respond_to?(:id)
+        undef_method :type    if respond_to?(:type)
+        
+        attr_accessor :fields, :attributes, :properties, :conditions,
+          :groupings
+        
+        # Set up all the collections. Consider this the equivalent of an
+        # instance's initialize method.
+        # 
+        def setup
+          @fields     = []
+          @attributes = []
+          @properties = {}
+          @conditions = []
+          @groupings  = []
+        end
+        
+        # This is how you add fields - the strings Sphinx looks at - to your
+        # index. Technically, to use this method, you need to pass in some
+        # columns and options - but there's some neat method_missing stuff
+        # happening, so lets stick to the expected syntax within a define_index
+        # block.
+        #
+        # Expected options are :as, which points to a column alias in symbol
+        # form, and :sortable, which indicates whether you want to sort by this
+        # field.
+        #
+        # Adding Single-Column Fields:
+        # 
+        # You can use symbols or methods - and can chain methods together to
+        # get access down the associations tree.
+        # 
+        #   indexes :id, :as => :my_id
+        #   indexes :name, :sortable => true
+        #   indexes first_name, last_name, :sortable => true
+        #   indexes users.posts.content, :as => :post_content
+        #   indexes users(:id), :as => :user_ids
+        #
+        # Keep in mind that if any keywords for Ruby methods - such as id or 
+        # name - clash with your column names, you need to use the symbol
+        # version (see the first, second and last examples above).
+        #
+        # If you specify multiple columns (example #2), a field will be created
+        # for each. Don't use the :as option in this case. If you want to merge
+        # those columns together, continue reading.
+        # 
+        # Adding Multi-Column Fields:
+        # 
+        #   indexes [first_name, last_name], :as => :name
+        #   indexes [location, parent.location], :as => :location
+        #
+        # To combine multiple columns into a single field, you need to wrap
+        # them in an Array, as shown by the above examples. There's no
+        # limitations on whether they're symbols or methods or what level of
+        # associations they come from.
+        # 
+        # Adding SQL Fragment Fields
+        #
+        # You can also define a field using an SQL fragment, useful for when
+        # you would like to index a calculated value.
+        #
+        #   indexes "age < 18", :as => :minor
+        #
+        def indexes(*args)
+          options = args.extract_options!
+          args.each do |columns|
+            fields << Field.new(FauxColumn.coerce(columns), options)
+            
+            if fields.last.sortable
+              attributes << Attribute.new(
+                fields.last.columns.collect { |col| col.clone },
+                options.merge(
+                  :type => :string,
+                  :as => fields.last.unique_name.to_s.concat("_sort").to_sym
+                )
+              )
+            end
+          end
+        end
+        alias_method :field,    :indexes
+        alias_method :includes, :indexes
+        
+        # This is the method to add attributes to your index (hence why it is
+        # aliased as 'attribute'). The syntax is the same as #indexes, so use
+        # that as starting point, but keep in mind the following points.
+        # 
+        # An attribute can have an alias (the :as option), but it is always
+        # sortable - so you don't need to explicitly request that. You _can_
+        # specify the data type of the attribute (the :type option), but the
+        # code's pretty good at figuring that out itself from peering into the
+        # database.
+        # 
+        # Attributes are limited to the following types: integers, floats,
+        # datetimes (converted to timestamps), booleans and strings. Don't
+        # forget that Sphinx converts string attributes to integers, which are
+        # useful for sorting, but that's about it.
+        # 
+        # You can also have a collection of integers for multi-value attributes
+        # (MVAs). Generally these would be through a has_many relationship,
+        # like in this example:
+        # 
+        #   has posts(:id), :as => :post_ids
+        # 
+        # This allows you to filter on any of the values tied to a specific
+        # record. Might be best to read through the Sphinx documentation to get
+        # a better idea of that though.
+        # 
+        # Adding SQL Fragment Attributes
+        #
+        # You can also define an attribute using an SQL fragment, useful for
+        # when you would like to index a calculated value. Don't forget to set
+        # the type of the attribute though:
+        #
+        #   indexes "age < 18", :as => :minor, :type => :boolean
+        # 
+        # If you're creating attributes for latitude and longitude, don't
+        # forget that Sphinx expects these values to be in radians.
+        # 
+        def has(*args)
+          options = args.extract_options!
+          args.each do |columns|
+            attributes << Attribute.new(FauxColumn.coerce(columns), options)
+          end
+        end
+        alias_method :attribute, :has
+        
+        # Use this method to add some manual SQL conditions for your index
+        # request. You can pass in as many strings as you like, they'll get
+        # joined together with ANDs later on.
+        # 
+        #   where "user_id = 10"
+        #   where "parent_type = 'Article'", "created_at < NOW()"
+        # 
+        def where(*args)
+          @conditions += args
+        end
+        
+        # Use this method to add some manual SQL strings to the GROUP BY
+        # clause. You can pass in as many strings as you'd like, they'll get
+        # joined together with commas later on.
+        # 
+        #   group_by "lat", "lng"
+        # 
+        def group_by(*args)
+          @groupings += args
+        end
+        
+        # This is what to use to set properties on the index. Chief amongst
+        # those is the delta property - to allow automatic updates to your
+        # indexes as new models are added and edited - but also you can
+        # define search-related properties which will be the defaults for all
+        # searches on the model.
+        # 
+        #   set_property :delta => true
+        #   set_property :field_weights => {"name" => 100}
+        # 
+        # Also, the following two properties are particularly relevant for
+        # geo-location searching - latitude_attr and longitude_attr. If your
+        # attributes for these two values are named something other than
+        # lat/latitude or lon/long/longitude, you can dictate what they are
+        # when defining the index, so you don't need to specify them for every
+        # geo-related search.
+        #
+        #   set_property :latitude_attr => "lt", :longitude_attr => "lg"
+        # 
+        # Please don't forget to add a boolean field named 'delta' to your
+        # model's database table if enabling the delta index for it.
+        # 
+        def set_property(*args)
+          options = args.extract_options!
+          if options.empty?
+            @properties[args[0]] = args[1]
+          else
+            @properties.merge!(options)
+          end
+        end
+        alias_method :set_properties, :set_property
+        
+        # Handles the generation of new columns for the field and attribute
+        # definitions.
+        # 
+        def method_missing(method, *args)
+          FauxColumn.new(method, *args)
+        end
+        
+        # A method to allow adding fields from associations which have names
+        # that clash with method names in the Builder class (ie: properties,
+        # fields, attributes).
+        # 
+        # Example: indexes assoc(:properties).column
+        # 
+        def assoc(assoc)
+          FauxColumn.new(method)
+        end
+      end
+    end
+  end
+end