skalee-thinking-sphinx 1.3.14.1

data/lib/thinking_sphinx/adapters/postgresql_adapter.rb

@@ -0,0 +1,143 @@
+module ThinkingSphinx
+  class PostgreSQLAdapter < AbstractAdapter
+    def setup
+      create_array_accum_function
+      create_crc32_function
+    end
+    
+    def sphinx_identifier
+      "pgsql"
+    end
+    
+    def concatenate(clause, separator = ' ')
+      if clause[/^COALESCE/]
+        clause.split('), ').join(") || '#{separator}' || ")
+      else
+        clause.split(', ').collect { |field|
+          "CAST(COALESCE(#{field}, '') as varchar)"
+        }.join(" || '#{separator}' || ")
+      end
+    end
+    
+    def group_concatenate(clause, separator = ' ')
+      "array_to_string(array_accum(COALESCE(#{clause}, '0')), '#{separator}')"
+    end
+    
+    def cast_to_string(clause)
+      clause
+    end
+    
+    def cast_to_datetime(clause)
+      "cast(extract(epoch from #{clause}) as int)"
+    end
+    
+    def cast_to_unsigned(clause)
+      clause
+    end
+    
+    def convert_nulls(clause, default = '')
+      default = case default
+      when String
+        "'#{default}'"
+      when NilClass
+        'NULL'
+      when Fixnum
+        "#{default}::bigint"
+      else
+        default
+      end
+      
+      "COALESCE(#{clause}, #{default})"
+    end
+    
+    def boolean(value)
+      value ? 'TRUE' : 'FALSE'
+    end
+    
+    def crc(clause, blank_to_null = false)
+      clause = "NULLIF(#{clause},'')" if blank_to_null
+      "crc32(#{clause})"
+    end
+    
+    def utf8_query_pre
+      nil
+    end
+    
+    def time_difference(diff)
+      "current_timestamp - interval '#{diff} seconds'"
+    end
+    
+    private
+    
+    def execute(command, output_error = false)
+      connection.execute "begin"
+      connection.execute "savepoint ts"
+      begin
+        connection.execute command
+      rescue StandardError => err
+        puts err if output_error
+        connection.execute "rollback to savepoint ts"
+      end
+      connection.execute "release savepoint ts"
+      connection.execute "commit"
+    end
+    
+    def create_array_accum_function
+      if connection.raw_connection.respond_to?(:server_version) && connection.raw_connection.server_version > 80200
+        execute <<-SQL
+          CREATE AGGREGATE array_accum (anyelement)
+          (
+              sfunc = array_append,
+              stype = anyarray,
+              initcond = '{}'
+          );
+        SQL
+      else
+        execute <<-SQL
+          CREATE AGGREGATE array_accum
+          (
+              basetype = anyelement,
+              sfunc = array_append,
+              stype = anyarray,
+              initcond = '{}'
+          );
+        SQL
+      end
+    end
+    
+    def create_crc32_function
+      execute "CREATE LANGUAGE 'plpgsql';"
+      function = <<-SQL
+        CREATE OR REPLACE FUNCTION crc32(word text)
+        RETURNS bigint AS $$
+          DECLARE tmp bigint;
+          DECLARE i int;
+          DECLARE j int;
+          DECLARE word_array bytea;
+          BEGIN
+            i = 0;
+            tmp = 4294967295;
+            word_array = decode(replace(word, E'\\\\', E'\\\\\\\\'), 'escape');
+            LOOP
+              tmp = (tmp # get_byte(word_array, i))::bigint;
+              i = i + 1;
+              j = 0;
+              LOOP
+                tmp = ((tmp >> 1) # (3988292384 * (tmp & 1)))::bigint;
+                j = j + 1;
+                IF j >= 8 THEN
+                  EXIT;
+                END IF;
+              END LOOP;
+              IF i >= char_length(word) THEN
+                EXIT;
+              END IF;
+            END LOOP;
+            return (tmp # 4294967295);
+          END
+        $$ IMMUTABLE STRICT LANGUAGE plpgsql;
+      SQL
+      execute function, true
+    end
+  end
+end

data/lib/thinking_sphinx/association.rb

@@ -0,0 +1,164 @@
+module ThinkingSphinx
+  # Association tracks a specific reflection and join to reference data that
+  # isn't in the base model. Very much an internal class for Thinking Sphinx -
+  # perhaps because I feel it's not as strong (or simple) as most of the rest.
+  # 
+  class Association
+    attr_accessor :parent, :reflection, :join
+    
+    # Create a new association by passing in the parent association, and the
+    # corresponding reflection instance. If there is no parent, pass in nil.
+    # 
+    #   top   = Association.new nil, top_reflection
+    #   child = Association.new top, child_reflection
+    # 
+    def initialize(parent, reflection)
+      @parent, @reflection = parent, reflection
+      @children = {}
+    end
+    
+    # Get the children associations for a given association name. The only time
+    # that there'll actually be more than one association is when the
+    # relationship is polymorphic. To keep things simple though, it will always
+    # be an Array that gets returned (an empty one if no matches).
+    #
+    #   # where pages is an association on the class tied to the reflection.
+    #   association.children(:pages)
+    # 
+    def children(assoc)
+      @children[assoc] ||= Association.children(@reflection.klass, assoc, self)
+    end
+    
+    # Get the children associations for a given class, association name and
+    # parent association. Much like the instance method of the same name, it
+    # will return an empty array if no associations have the name, and only
+    # have multiple association instances if the underlying relationship is
+    # polymorphic.
+    # 
+    #   Association.children(User, :pages, user_association)
+    # 
+    def self.children(klass, assoc, parent=nil)
+      ref = klass.reflect_on_association(assoc)
+      
+      return [] if ref.nil?
+      return [Association.new(parent, ref)] unless ref.options[:polymorphic]
+      
+      # association is polymorphic - create associations for each
+      # non-polymorphic reflection.
+      polymorphic_classes(ref).collect { |klass|
+        Association.new parent, ::ActiveRecord::Reflection::AssociationReflection.new(
+          ref.macro,
+          "#{ref.name}_#{klass.name}".to_sym,
+          casted_options(klass, ref),
+          ref.active_record
+        )
+      }
+    end
+    
+    # Link up the join for this model from a base join - and set parent
+    # associations' joins recursively.
+    #
+    def join_to(base_join)
+      parent.join_to(base_join) if parent && parent.join.nil?
+      
+      @join ||= ::ActiveRecord::Associations::ClassMethods::JoinDependency::JoinAssociation.new(
+        @reflection, base_join, parent ? parent.join : base_join.joins.first
+      )
+    end
+    
+    # Returns the association's join SQL statements - and it replaces
+    # ::ts_join_alias:: with the aliased table name so the generated reflection
+    # join conditions avoid column name collisions.
+    # 
+    def to_sql
+      @join.association_join.gsub(/::ts_join_alias::/,
+        "#{@reflection.klass.connection.quote_table_name(@join.parent.aliased_table_name)}"
+      )
+    end
+    
+    # Returns true if the association - or a parent - is a has_many or
+    # has_and_belongs_to_many.
+    # 
+    def is_many?
+      case @reflection.macro
+      when :has_many, :has_and_belongs_to_many
+        true
+      else
+        @parent ? @parent.is_many? : false
+      end
+    end
+    
+    # Returns an array of all the associations that lead to this one - starting
+    # with the top level all the way to the current association object.
+    # 
+    def ancestors
+      (parent ? parent.ancestors : []) << self
+    end
+    
+    def has_column?(column)
+      @reflection.klass.column_names.include?(column.to_s)
+    end
+    
+    def primary_key_from_reflection
+      if @reflection.options[:through]
+        @reflection.source_reflection.options[:foreign_key] ||
+        @reflection.source_reflection.primary_key_name
+      elsif @reflection.macro == :has_and_belongs_to_many
+        @reflection.association_foreign_key
+      else
+        nil
+      end
+    end
+    
+    def table
+      if @reflection.options[:through] ||
+        @reflection.macro == :has_and_belongs_to_many
+        @join.aliased_join_table_name
+      else
+        @join.aliased_table_name
+      end
+    end
+    
+    private
+    
+    # Returns all the objects that could be currently instantiated from a
+    # polymorphic association. This is pretty damn fast if there's an index on
+    # the foreign type column - but if there isn't, it can take a while if you
+    # have a lot of data.
+    # 
+    def self.polymorphic_classes(ref)
+      ref.active_record.connection.select_all(
+        "SELECT DISTINCT #{ref.options[:foreign_type]} " +
+        "FROM #{ref.active_record.table_name} " +
+        "WHERE #{ref.options[:foreign_type]} IS NOT NULL"
+      ).collect { |row|
+        row[ref.options[:foreign_type]].constantize
+      }
+    end
+    
+    # Returns a new set of options for an association that mimics an existing
+    # polymorphic relationship for a specific class. It adds a condition to
+    # filter by the appropriate object.
+    # 
+    def self.casted_options(klass, ref)
+      options = ref.options.clone
+      options[:polymorphic]   = nil
+      options[:class_name]    = klass.name
+      options[:foreign_key] ||= "#{ref.name}_id"
+      
+      quoted_foreign_type = klass.connection.quote_column_name ref.options[:foreign_type]
+      case options[:conditions]
+      when nil
+        options[:conditions] = "::ts_join_alias::.#{quoted_foreign_type} = '#{klass.name}'"
+      when Array
+        options[:conditions] << "::ts_join_alias::.#{quoted_foreign_type} = '#{klass.name}'"
+      when Hash
+        options[:conditions].merge!(ref.options[:foreign_type] => klass.name)
+      else
+        options[:conditions] << " AND ::ts_join_alias::.#{quoted_foreign_type} = '#{klass.name}'"
+      end
+      
+      options
+    end
+  end
+end

data/lib/thinking_sphinx/attribute.rb

@@ -0,0 +1,362 @@
+module ThinkingSphinx
+  # Attributes - eternally useful when it comes to filtering, sorting or
+  # grouping. This class isn't really useful to you unless you're hacking
+  # around with the internals of Thinking Sphinx - but hey, don't let that
+  # stop you.
+  #
+  # One key thing to remember - if you're using the attribute manually to
+  # generate SQL statements, you'll need to set the base model, and all the
+  # associations. Which can get messy. Use Index.link!, it really helps.
+  # 
+  class Attribute < ThinkingSphinx::Property
+    attr_accessor :query_source
+    
+    # To create a new attribute, you'll need to pass in either a single Column
+    # or an array of them, and some (optional) options.
+    #
+    # Valid options are:
+    # - :as     => :alias_name
+    # - :type   => :attribute_type
+    # - :source => :field, :query, :ranged_query
+    #
+    # Alias is only required in three circumstances: when there's
+    # another attribute or field with the same name, when the column name is
+    # 'id', or when there's more than one column.
+    # 
+    # Type is not required, unless you want to force a column to be a certain
+    # type (but keep in mind the value will not be CASTed in the SQL
+    # statements). The only time you really need to use this is when the type
+    # can't be figured out by the column - ie: when not actually using a
+    # database column as your source.
+    # 
+    # Source is only used for multi-value attributes (MVA). By default this will
+    # use a left-join and a group_concat to obtain the values. For better performance
+    # during indexing it can be beneficial to let Sphinx use a separate query to retrieve
+    # all document,value-pairs.
+    # Either :query or :ranged_query will enable this feature, where :ranged_query will cause
+    # the query to be executed incremental.
+    #
+    # Example usage:
+    #
+    #   Attribute.new(
+    #     Column.new(:created_at)
+    #   )
+    #
+    #   Attribute.new(
+    #     Column.new(:posts, :id),
+    #     :as => :post_ids
+    #   )
+    #
+    #   Attribute.new(
+    #     Column.new(:posts, :id),
+    #     :as => :post_ids,
+    #     :source => :ranged_query
+    #   )
+    #
+    #   Attribute.new(
+    #     [Column.new(:pages, :id), Column.new(:articles, :id)],
+    #     :as => :content_ids
+    #   )
+    #
+    #   Attribute.new(
+    #     Column.new("NOW()"),
+    #     :as   => :indexed_at,
+    #     :type => :datetime
+    #   )
+    #
+    # If you're creating attributes for latitude and longitude, don't forget
+    # that Sphinx expects these values to be in radians.
+    #  
+    def initialize(source, columns, options = {})
+      super
+      
+      @type           = options[:type]
+      @query_source   = options[:source]
+      @crc            = options[:crc]
+      
+      @type         ||= :multi    unless @query_source.nil?
+      if @type == :string && @crc
+        @type = is_many? ? :multi : :integer
+      end
+      
+      source.attributes << self
+    end
+    
+    # Get the part of the SELECT clause related to this attribute. Don't forget
+    # to set your model and associations first though.
+    #
+    # This will concatenate strings and arrays of integers, and convert
+    # datetimes to timestamps, as needed.
+    # 
+    def to_select_sql
+      return nil unless include_as_association?
+      
+      separator = all_ints? || all_datetimes? || @crc ? ',' : ' '
+      
+      clause = @columns.collect { |column|
+        part = column_with_prefix(column)
+        case type
+        when :string
+          adapter.convert_nulls(part)
+        when :datetime
+          adapter.cast_to_datetime(part)
+        when :multi
+          part = adapter.cast_to_datetime(part)   if is_many_datetimes?
+          part = adapter.convert_nulls(part, '0') if is_many_ints?
+          part
+        else
+          part
+        end
+      }.join(', ')
+      
+      clause = adapter.crc(clause)                          if @crc
+      clause = adapter.concatenate(clause, separator)       if concat_ws?
+      clause = adapter.group_concatenate(clause, separator) if is_many?
+      
+      "#{clause} AS #{quote_column(unique_name)}"
+    end
+    
+    def type_to_config
+      {
+        :multi    => :sql_attr_multi,
+        :datetime => :sql_attr_timestamp,
+        :string   => :sql_attr_str2ordinal,
+        :float    => :sql_attr_float,
+        :boolean  => :sql_attr_bool,
+        :integer  => :sql_attr_uint
+      }[type]
+    end
+    
+    def include_as_association?
+      ! (type == :multi && (query_source == :query || query_source == :ranged_query))
+    end
+    
+    # Returns the configuration value that should be used for
+    # the attribute.
+    # Special case is the multi-valued attribute that needs some
+    # extra configuration. 
+    # 
+    def config_value(offset = nil, delta = false)
+      if type == :multi
+        multi_config = include_as_association? ? "field" :
+          source_value(offset, delta).gsub(/\s+/m, " ").strip
+        "uint #{unique_name} from #{multi_config}"
+      else
+        unique_name
+      end
+    end
+        
+    # Returns the type of the column. If that's not already set, it returns
+    # :multi if there's the possibility of more than one value, :string if
+    # there's more than one association, otherwise it figures out what the
+    # actual column's datatype is and returns that.
+    # 
+    def type
+      @type ||= begin
+        base_type = case
+        when is_many?, is_many_ints?
+          :multi
+        when @associations.values.flatten.length > 1
+          :string
+        else
+          translated_type_from_database
+        end
+        
+        if base_type == :string && @crc
+          base_type = :integer
+        else
+          @crc = false unless base_type == :multi && is_many_strings? && @crc
+        end
+        
+        base_type
+      end
+    end
+    
+    def updatable?
+      [:integer, :datetime, :boolean].include?(type) && !is_string?
+    end
+    
+    def live_value(instance)
+      object = instance
+      column = @columns.first
+      column.__stack.each { |method|
+        object = object.send(method)
+        return sphinx_value(nil) if object.nil?
+      }
+      
+      sphinx_value object.send(column.__name)
+    end
+    
+    def all_ints?
+      all_of_type?(:integer)
+    end
+    
+    def all_datetimes?
+      all_of_type?(:datetime, :date, :timestamp)
+    end
+    
+    def all_strings?
+      all_of_type?(:string, :text)
+    end
+    
+    private
+    
+    def source_value(offset, delta)
+      if is_string?
+        return "#{query_source.to_s.dasherize}; #{columns.first.__name}"
+      end
+      
+      query = query(offset)
+
+      if query_source == :ranged_query
+        query += query_clause
+        query += " AND #{query_delta.strip}" if delta
+        "ranged-query; #{query}; #{range_query}"
+      else
+        query += "WHERE #{query_delta.strip}" if delta
+        "query; #{query}"
+      end
+    end
+    
+    def query(offset)
+      base_assoc = base_association_for_mva
+      end_assoc  = end_association_for_mva
+      raise "Could not determine SQL for MVA" if base_assoc.nil?
+      
+      <<-SQL
+SELECT #{foreign_key_for_mva base_assoc}
+  #{ThinkingSphinx.unique_id_expression(offset)} AS #{quote_column('id')},
+  #{primary_key_for_mva(end_assoc)} AS #{quote_column(unique_name)}
+FROM #{quote_table_name base_assoc.table} #{association_joins}
+      SQL
+    end
+    
+    def query_clause
+      foreign_key = foreign_key_for_mva base_association_for_mva
+      "WHERE #{foreign_key} >= $start AND #{foreign_key} <= $end"
+    end
+    
+    def query_delta
+      foreign_key = foreign_key_for_mva base_association_for_mva
+      <<-SQL
+#{foreign_key} IN (SELECT #{quote_column model.primary_key}
+FROM #{model.quoted_table_name}
+WHERE #{@source.index.delta_object.clause(model, true)})
+      SQL
+    end
+    
+    def range_query
+      assoc       = base_association_for_mva
+      foreign_key = foreign_key_for_mva assoc
+      "SELECT MIN(#{foreign_key}), MAX(#{foreign_key}) FROM #{quote_table_name assoc.table}"
+    end
+    
+    def primary_key_for_mva(assoc)
+      quote_with_table(
+        assoc.table, assoc.primary_key_from_reflection || columns.first.__name
+      )
+    end
+    
+    def foreign_key_for_mva(assoc)
+      quote_with_table assoc.table, assoc.reflection.primary_key_name
+    end
+    
+    def end_association_for_mva
+      @association_for_mva ||= associations[columns.first].detect { |assoc|
+        assoc.has_column?(columns.first.__name)
+      }
+    end
+    
+    def base_association_for_mva
+      @first_association_for_mva ||= begin
+        assoc = end_association_for_mva
+        while !assoc.parent.nil?
+          assoc = assoc.parent
+        end
+        
+        assoc
+      end
+    end
+    
+    def association_joins
+      joins = []
+      assoc = end_association_for_mva
+      while assoc != base_association_for_mva
+        joins << assoc.to_sql
+        assoc = assoc.parent
+      end
+      
+      joins.join(' ')
+    end
+    
+    def is_many_ints?
+      concat_ws? && all_ints?
+    end
+    
+    def is_many_datetimes?
+      is_many? && all_datetimes?
+    end
+    
+    def is_many_strings?
+      is_many? && all_strings?
+    end
+       
+    def type_from_database
+      klass = @associations.values.flatten.first ? 
+        @associations.values.flatten.first.reflection.klass : @model
+      
+      column = klass.columns.detect { |col|
+        @columns.collect { |c| c.__name.to_s }.include? col.name
+      }
+      column.nil? ? nil : column.type
+    end
+    
+    def translated_type_from_database
+      case type_from_db = type_from_database
+      when :datetime, :string, :float, :boolean, :integer
+        type_from_db
+      when :decimal
+        :float
+      when :timestamp, :date
+        :datetime
+      else
+        raise <<-MESSAGE
+
+Cannot automatically map attribute #{unique_name} in #{@model.name} to an
+equivalent Sphinx type (integer, float, boolean, datetime, string as ordinal).
+You could try to explicitly convert the column's value in your define_index
+block:
+  has "CAST(column AS INT)", :type => :integer, :as => :column
+        MESSAGE
+      end
+    end
+    
+    def all_of_type?(*column_types)
+      @columns.all? { |col|
+        klasses = @associations[col].empty? ? [@model] :
+          @associations[col].collect { |assoc| assoc.reflection.klass }
+        klasses.all? { |klass|
+          column = klass.columns.detect { |column| column.name == col.__name.to_s }
+          !column.nil? && column_types.include?(column.type)
+        }
+      }
+    end
+    
+    def sphinx_value(value)
+      case value
+      when TrueClass
+        1
+      when FalseClass, NilClass
+        0
+      when Time
+        value.to_i
+      when Date
+        value.to_time.to_i
+      when String
+        value.to_crc32
+      else
+        value
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/auto_version.rb

@@ -0,0 +1,22 @@
+module ThinkingSphinx
+  class AutoVersion
+    def self.detect
+      version = ThinkingSphinx::Configuration.instance.controller.sphinx_version
+      case version
+      when '0.9.8', '0.9.9'
+        require "riddle/#{version}"
+      else
+        STDERR.puts %Q{
+Sphinx cannot be found on your system. You may need to configure the following
+settings in your config/sphinx.yml file:
+  * bin_path
+  * searchd_binary_name
+  * indexer_binary_name
+
+For more information, read the documentation:
+http://freelancing-god.github.com/ts/en/advanced_config.html
+}
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/class_facet.rb

@@ -0,0 +1,15 @@
+module ThinkingSphinx
+  class ClassFacet < ThinkingSphinx::Facet
+    def name
+      :class
+    end
+    
+    def attribute_name
+      "class_crc"
+    end
+    
+    def value(object, attribute_value)
+      object.class.name
+    end
+  end
+end