pager-ultrasphinx 1.0.20080510

data/lib/ultrasphinx/configure.rb

@@ -0,0 +1,367 @@
+
+module Ultrasphinx
+  class Configure  
+    class << self
+
+      include Associations
+  
+      # Force all the indexed models to load and register in the MODEL_CONFIGURATION hash.
+      def load_constants
+  
+        Dependencies.load_paths.grep(/\/models$/).each do |models_dir|
+          Dir.chdir(models_dir) do
+            Dir["**/*.rb"].each do |filename|
+              open(filename) do |file| 
+                begin
+                  if file.grep(/^\s+is_indexed/).any?
+                    filename = filename[0..-4]
+                    begin                
+                      File.basename(filename).camelize.constantize
+                    rescue NameError => e
+                      filename.camelize.constantize
+                    end
+                  end
+                rescue Object => e
+                  say "warning: critical autoload error on #{filename}; try referencing \"#{filename.camelize}\" directly in the console"
+                  #say e.backtrace.join("\n") if RAILS_ENV == "development"
+                end
+              end 
+            end
+          end
+        end
+  
+        # Build the field-to-type mappings.
+        Fields.instance.configure(MODEL_CONFIGURATION)
+      end
+      
+                    
+      # Main SQL builder.
+      def run       
+
+        load_constants
+              
+        say "rebuilding configurations for #{RAILS_ENV} environment" 
+        say "available models are #{MODEL_CONFIGURATION.keys.to_sentence}"
+        File.open(CONF_PATH, "w") do |conf|              
+          conf.puts global_header            
+          say "generating SQL"    
+
+          INDEXES.each do |index|
+            sources = []
+            cached_groups = Fields.instance.groups.join("\n")
+
+            MODEL_CONFIGURATION.each_with_index do |model_and_options, class_id|              
+              # This relies on hash sort order being deterministic per-machine
+              model, options = model_and_options
+              klass = model.constantize
+              source = "#{model.tableize.gsub('/', '__')}_#{index}"
+ 
+              if index != DELTA_INDEX or options['delta']
+                # If we are building the delta, we only want to include the models that requested it
+                conf.puts build_source(index, Fields.instance, model, options, class_id, klass, source, cached_groups)
+                sources << source                
+              end
+            end
+            
+            if sources.any?
+              # Don't generate a delta index if there are no delta tables
+              conf.puts build_index(index, sources)
+            end
+            
+          end
+        end              
+      end
+      
+      
+      ######
+      
+      private
+      
+      def global_header
+        ["\n# Auto-generated at #{Time.now}.",
+         "# Hand modifications will be overwritten.",
+         "# #{BASE_PATH}\n",
+         INDEXER_SETTINGS.except('delta')._to_conf_string('indexer'),
+         "",
+         DAEMON_SETTINGS._to_conf_string("searchd")]
+      end      
+      
+      
+      def setup_source_database(klass)
+        # Supporting Postgres now
+        connection_settings = klass.connection.instance_variable_get("@config")
+
+        adapter_defaults = DEFAULTS[ADAPTER]
+        raise ConfigurationError, "Unsupported database adapter" unless adapter_defaults
+
+        conf = [adapter_defaults]                  
+        connection_settings.reverse_merge(CONNECTION_DEFAULTS).each do |key, value|
+          conf << "#{CONFIG_MAP[key]} = #{value}" if CONFIG_MAP[key]          
+        end                 
+        conf.sort.join("\n")
+      end
+
+      
+      def build_delta_condition(index, klass, options)
+        if index == DELTA_INDEX and options['delta']
+          # Add delta condition if necessary
+          table, field = klass.table_name, options['delta']['field']
+          source_string = "#{table}.#{field}"
+          delta_column = klass.columns_hash[field] 
+
+          if delta_column 
+            raise ConfigurationError, "#{source_string} is not a :datetime" unless delta_column.type == :datetime
+            if (options['fields'] + options['concatenate'] + options['include']).detect { |entry| entry['sortable'] }
+              # Warning about the sortable problem
+              # XXX Kind of in an odd place, but I want to happen at index time
+              Ultrasphinx.say "warning; text sortable columns on #{klass.name} will return wrong results with partial delta indexing"
+            end
+
+            delta = INDEXER_SETTINGS['delta']
+            if delta 
+              string = "#{source_string} > #{SQL_FUNCTIONS[ADAPTER]['delta']._interpolate(delta)}";
+            else
+              raise ConfigurationError, "No 'indexer { delta }' setting specified in '#{BASE_PATH}'"
+            end
+          else
+            Ultrasphinx.say "warning; #{klass.name} will reindex the entire table during delta indexing"
+          end
+        end
+      end
+      
+      
+      def setup_source_arrays(index, klass, fields, class_id, conditions, order)
+        condition_strings = Array(conditions).map do |condition| 
+          "(#{condition})"
+        end
+        
+        column_strings = [
+          "(#{klass.table_name}.#{klass.primary_key} * #{MODEL_CONFIGURATION.size} + #{class_id}) AS id", 
+          "#{class_id} AS class_id", "'#{klass.name}' AS class"]
+        remaining_columns = fields.types.keys - ["class", "class_id"]        
+        [column_strings, [], condition_strings, [], false, remaining_columns, order]
+      end
+      
+      
+      def range_select_string(klass, delta_condition)
+        ["sql_query_range = SELECT",
+          SQL_FUNCTIONS[ADAPTER]['range_cast']._interpolate("MIN(#{klass.primary_key})"),
+          ",",
+          SQL_FUNCTIONS[ADAPTER]['range_cast']._interpolate("MAX(#{klass.primary_key})"),
+          "FROM #{klass.table_name}",
+          ("WHERE #{delta_condition}" if delta_condition),
+        ].join(" ")
+      end
+      
+      
+      def query_info_string(klass, class_id)
+        "sql_query_info = SELECT * FROM #{klass.table_name} WHERE #{klass.table_name}.#{klass.primary_key} = (($id - #{class_id}) / #{MODEL_CONFIGURATION.size})"
+      end      
+      
+            
+      def build_source(index, fields, model, options, class_id, klass, source, groups)
+                
+        column_strings, join_strings, condition_strings, group_bys, use_distinct, remaining_columns, order = 
+          setup_source_arrays(
+            index, klass, fields, class_id, options['conditions'], options['order'])
+            
+        delta_condition = 
+          build_delta_condition(
+            index, klass, options)             
+        condition_strings << delta_condition if delta_condition
+
+        column_strings, join_strings, group_bys, remaining_columns = 
+          build_regular_fields(
+            klass, fields, options['fields'], column_strings, join_strings, group_bys, remaining_columns)
+            
+        column_strings, join_strings, group_bys, remaining_columns = 
+          build_includes(
+            klass, fields, options['include'], column_strings, join_strings, group_bys, remaining_columns)
+            
+        column_strings, join_strings, group_bys, use_distinct, remaining_columns = 
+          build_concatenations(
+            klass, fields, options['concatenate'], column_strings, join_strings, group_bys, use_distinct, remaining_columns)
+        
+        column_strings = add_missing_columns(fields, remaining_columns, column_strings)
+       
+        ["\n# Source configuration\n\n",
+         "source #{source}\n{",
+          SOURCE_SETTINGS._to_conf_string,
+          setup_source_database(klass),
+          range_select_string(klass, delta_condition),
+          build_query(klass, column_strings, join_strings, condition_strings, use_distinct, group_bys, order),
+          "\n" + groups,
+          query_info_string(klass, class_id),
+          "}\n\n"]
+      end
+      
+      
+      def build_query(klass, column_strings, join_strings, condition_strings, use_distinct, group_bys, order)
+        
+        primary_key = "#{klass.table_name}.#{klass.primary_key}"
+        group_bys = case ADAPTER
+          when 'mysql'
+            primary_key
+          when 'postgresql'
+            # Postgres is very fussy about GROUP_BY 
+            ([primary_key] + group_bys.reject {|s| s == primary_key}.uniq.sort).join(', ')
+          end
+        
+        ["sql_query =", 
+          "SELECT",
+          # Avoid DISTINCT; it destroys performance
+          column_strings.sort_by do |string| 
+            # Sphinx wants them always in the same order, but "id" must be first
+            (field = string[/.*AS (.*)/, 1]) == "id" ? "*" : field
+          end.join(", "),
+          "FROM #{klass.table_name}",
+          join_strings.uniq,
+          "WHERE #{primary_key} >= $start AND #{primary_key} <= $end",
+          condition_strings.uniq.map {|condition| "AND #{condition}" },
+          "GROUP BY #{group_bys}",
+          ("ORDER BY #{order}" if order)
+        ].flatten.compact.join(" ")
+      end
+      
+      
+      def add_missing_columns(fields, remaining_columns, column_strings)
+        remaining_columns.each do |field|
+          column_strings << fields.null(field)
+        end
+        column_strings
+      end
+      
+
+      def build_regular_fields(klass, fields, entries, column_strings, join_strings, group_bys, remaining_columns)          
+        entries.to_a.each do |entry|          
+          source_string = "#{entry['table_alias']}.#{entry['field']}"          
+          group_bys << source_string
+          column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], entry['sortable'], column_strings, remaining_columns)
+        end
+        
+        [column_strings, join_strings, group_bys, remaining_columns]
+      end
+      
+
+      def build_includes(klass, fields, entries, column_strings, join_strings, group_bys, remaining_columns)                  
+        entries.to_a.each do |entry|
+          raise ConfigurationError, "You must identify your association with either class_name or association_name, but not both" if entry['class_name'] && entry ['association_name']          
+        
+          association = get_association(klass, entry)
+
+          # You can use 'class_name' and 'association_sql' to associate to a model that doesn't actually 
+          # have an association.
+          join_klass = association ? association.class_name.constantize : entry['class_name'].constantize
+                        
+          raise ConfigurationError, "Unknown association from #{klass} to #{entry['class_name'] || entry['association_name']}" if not association and not entry['association_sql']
+          
+          join_strings = install_join_unless_association_sql(entry['association_sql'], nil, join_strings) do 
+            "LEFT OUTER JOIN #{join_klass.table_name} AS #{entry['table_alias']} ON " + 
+            if (macro = association.macro) == :belongs_to 
+              "#{entry['table_alias']}.#{join_klass.primary_key} = #{klass.table_name}.#{association.primary_key_name}" 
+            elsif macro == :has_one
+              "#{klass.table_name}.#{klass.primary_key} = #{entry['table_alias']}.#{association.primary_key_name}" 
+            else
+              raise ConfigurationError, "Unidentified association macro #{macro.inspect}. Please use the :association_sql key to manually specify the JOIN syntax."
+            end
+          end
+          
+          source_string = "#{entry['table_alias']}.#{entry['field']}"
+          group_bys << source_string
+          column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], entry['sortable'], column_strings, remaining_columns)                         
+        end
+        
+        [column_strings, join_strings, group_bys, remaining_columns]
+      end
+      
+        
+      def build_concatenations(klass, fields, entries, column_strings, join_strings, group_bys, use_distinct, remaining_columns)
+        entries.to_a.each do |entry|
+          if entry['field']
+            # Group concats
+  
+            # Only has_many's or explicit sql right now.
+            association = get_association(klass, entry)
+            
+            # You can use 'class_name' and 'association_sql' to associate to a model that doesn't actually 
+            # have an association. The automatic choice of a table alias chosen might be kind of strange.
+            join_klass = association ? association.class_name.constantize : entry['class_name'].constantize
+        
+            join_strings = install_join_unless_association_sql(entry['association_sql'], nil, join_strings) do 
+              # XXX The foreign key is not verified for polymorphic relationships.
+              association = get_association(klass, entry)
+              "LEFT OUTER JOIN #{join_klass.table_name} AS #{entry['table_alias']} ON #{klass.table_name}.#{klass.primary_key} = #{entry['table_alias']}.#{association.primary_key_name}" + 
+                # XXX Is this valid?
+                (entry['conditions'] ? " AND (#{entry['conditions']})" : "") 
+            end
+            
+            source_string = "#{entry['table_alias']}.#{entry['field']}"
+            order_string = ("ORDER BY #{entry['order']}" if entry['order'])
+            # We are using the field in an aggregate, so we don't want to add it to group_bys
+            source_string = SQL_FUNCTIONS[ADAPTER]['group_concat']._interpolate(source_string, order_string)
+            use_distinct = true
+            
+            column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], entry['sortable'], column_strings, remaining_columns)
+            
+          elsif entry['fields']
+            # Regular concats
+            source_string = "CONCAT_WS(' ', " + entry['fields'].map do |subfield| 
+              "#{entry['table_alias']}.#{subfield}"
+            end.each do |subsource_string|
+              group_bys << subsource_string
+            end.join(', ') + ")"
+            
+            column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], entry['sortable'], column_strings, remaining_columns)              
+
+          else
+            raise ConfigurationError, "Invalid concatenate parameters for #{model}: #{entry.inspect}."
+          end
+        end
+        
+        [column_strings, join_strings, group_bys, use_distinct, remaining_columns]
+      end
+      
+          
+      def build_index(index, sources)
+        ["\n# Index configuration\n\n",
+          "index #{index}\n{",
+          sources.sort.map do |source| 
+            "  source = #{source}"
+          end.join("\n"),          
+          INDEX_SETTINGS.merge('path' => INDEX_SETTINGS['path'] + "/sphinx_index_#{index}")._to_conf_string,
+         "}\n\n"]
+      end
+      
+    
+      def install_field(fields, source_string, as, function_sql, with_facet, with_sortable, column_strings, remaining_columns)
+        source_string = function_sql._interpolate(source_string) if function_sql
+
+        column_strings << fields.cast(source_string, as)
+        remaining_columns.delete(as)
+        
+        # Generate duplicate text fields for sorting
+        if with_sortable
+          column_strings << fields.cast(source_string, "#{as}_sortable")
+          remaining_columns.delete("#{as}_sortable")
+        end
+        
+        # Generate hashed integer fields for text grouping
+        if with_facet
+          column_strings << "#{SQL_FUNCTIONS[ADAPTER]['hash']._interpolate(source_string)} AS #{as}_facet"
+          remaining_columns.delete("#{as}_facet")
+        end
+        [column_strings, remaining_columns]
+      end
+      
+      
+      def install_join_unless_association_sql(association_sql, join_string, join_strings)
+        join_strings << (association_sql or join_string or yield)
+      end
+      
+      def say(s)
+        Ultrasphinx.say s
+      end
+      
+    end 
+  end
+end

data/lib/ultrasphinx/core_extensions.rb

@@ -0,0 +1,132 @@
+
+class Array
+  # Only flatten the first level of an array
+  def _flatten_once
+    self.inject([]) do |set, element| 
+      set + Array(element)
+    end
+  end
+  
+  def _sum
+    self.inject(0) do |acc, element|
+      acc + element
+    end
+  end
+end
+
+class Object
+  def _metaclass 
+    class << self
+      self
+    end
+  end
+  
+  def _deep_dup
+    # Cause Ruby's clone/dup sucks.
+    Marshal.load(Marshal.dump(self))
+  end
+end
+
+#class HashWithIndifferentAccess
+#  # Returns a regular Hash with all string keys. Much faster
+#  # than HWIA#merge.
+#  def _fast_merge(right)
+#    left = Hash[self]
+#    left.merge!(self.class.new(right))
+#  end
+#end
+
+class Hash
+  def _coerce_basic_types
+    # XXX To remove
+    Hash[*self.map do |key, value|
+      [key.to_s,
+        if value.respond_to?(:to_i) && value.to_i.to_s == value
+          value.to_i
+        elsif value == ""
+          nil
+        else
+          value
+        end]
+      end._flatten_once]
+  end
+  
+  def _stringify_all!(*skip)
+    # Stringifies all keys, and stringifies all values except those slotted for keys in 'skip'
+    stringify_keys!
+    self.except(*skip).each do |key, value|
+      self[key] = value.to_s
+    end
+    self
+  end
+  
+  # Delete by multiple keys
+  def _delete(*args)
+    args.map do |key|
+      self.delete key
+    end    
+  end
+  
+  # Convert a hash to a Sphinx-style conf string
+  def _to_conf_string(section = nil)
+    inner = self.map do |key, value|
+      "  #{key} = #{value}"
+    end.join("\n")
+    section ? "#{section} {\n#{inner}\n}\n" : inner
+  end
+  
+  unless Hash.new.respond_to? :except
+    # Rails 1.2.6 compatibility
+    def except(*keys)
+      rejected = Set.new(respond_to?(:convert_key) ? keys.map { |key| convert_key(key) } : keys)
+      reject { |key,| rejected.include?(key) }
+    end
+    def except!(*keys)
+      replace(except(*keys))
+    end
+  end 
+  
+end
+
+### Filter type coercion methods
+
+class String
+  # XXX Not used enough to justify such a strange abstraction
+  def _to_numeric
+    zeroless = self.squeeze(" ").strip.sub(/^0+(\d)/, '\1')
+    zeroless.sub!(/(\...*?)0+$/, '\1')
+    if zeroless.to_i.to_s == zeroless
+      zeroless.to_i
+    elsif zeroless.to_f.to_s == zeroless
+      zeroless.to_f
+    elsif date = Chronic.parse(self.gsub(/(\d)([^\d\:\s])/, '\1 \2')) # Improve Chronic's flexibility a little
+      date.to_i
+    else
+      raise Ultrasphinx::UsageError, "#{self.inspect} could not be coerced into a numeric value"
+    end
+  end
+  
+  # Interpolate SQL field names into functions
+  def _interpolate(*values)
+    if values.size == 1
+      self.gsub('?', values.first)
+    else
+      values.inject(self) { |result, value| result.sub('?', value.to_s) }
+    end
+  end
+  
+end
+
+module Ultrasphinx::NumericSelf
+  def _to_numeric; self; end
+end
+
+module Ultrasphinx::DateSelf
+  def _to_numeric; self.to_i; end
+end
+
+class Fixnum; include Ultrasphinx::NumericSelf; end
+class Bignum; include Ultrasphinx::NumericSelf; end
+class Float; include Ultrasphinx::NumericSelf; end
+class Date; include Ultrasphinx::DateSelf; end
+class Time; include Ultrasphinx::DateSelf; end

data/lib/ultrasphinx/fields.rb

@@ -0,0 +1,198 @@
+
+module Ultrasphinx
+
+=begin rdoc
+This is a special singleton configuration class that stores the index field configurations. Rather than using a magic hash and including relevant behavior in Ultrasphinx::Configure and Ultrasphinx::Search, we unify it here.
+=end
+
+  class Fields
+    # XXX Class needs a big refactoring; one of the worst parts of Ultrasphinx
+    
+    include Singleton
+    include Associations
+    
+    TYPE_MAP = {
+      'string' => 'text', 
+      'text' => 'text', 
+      'integer' => 'integer', 
+      'date' => 'date', 
+      'datetime' => 'date',
+      'timestamp' => 'date',
+      'float' => 'float',
+      'boolean' => 'bool'
+    }
+        
+    attr_accessor :classes, :types
+    
+    def initialize
+      @types = {}
+      @classes = Hash.new([])
+      @groups = []
+    end
+    
+    
+    def groups
+      @groups.compact.sort_by do |string| 
+        string[/= (.*)/, 1]
+      end
+    end
+  
+  
+    def save_and_verify_type(field, new_type, string_sortable, klass, msg = nil)
+      # Smoosh fields together based on their name in the Sphinx query schema
+      field, new_type = field.to_s, TYPE_MAP[new_type.to_s]
+
+      if types[field]
+        # Existing field name; verify its type
+        msg ||= "Column type mismatch for #{field.inspect}; was already #{types[field].inspect}, but is now #{new_type.inspect}."
+        raise ConfigurationError, msg unless types[field] == new_type
+        classes[field] = (classes[field] + [klass]).uniq
+
+      else
+        # New field      
+        types[field] = new_type
+        classes[field] = [klass]
+
+        @groups << case new_type
+          when 'integer'
+            "sql_attr_uint = #{field}"
+          when 'float'
+            "sql_attr_float = #{field}"
+          when 'bool'
+            "sql_attr_bool = #{field}"
+          when 'date'
+            "sql_attr_timestamp = #{field}"
+          when 'text' 
+            "sql_attr_str2ordinal = #{field}" if string_sortable
+        end
+      end
+    end
+    
+    
+    def cast(source_string, field)
+      if types[field] == "date"
+        "UNIX_TIMESTAMP(#{source_string})"
+      elsif types[field] == "integer"
+        source_string # "CAST(#{source_string} AS UNSIGNED)"
+      else
+        source_string              
+      end + " AS #{field}"
+    end    
+      
+      
+    def null(field)      
+      case types[field]
+        when 'text'
+          "''"
+        when 'integer', 'float', 'bool'
+          "0"
+        when 'date'
+          "18000" # Midnight on 1/1/1970
+        when nil
+          raise "Field #{field} is missing"
+        else
+          raise "Field #{field} does not have a valid type #{types[field]}."
+      end + " AS #{field}"
+    end
+    
+    
+    def configure(configuration)
+
+      configuration.each do |model, options|        
+
+        klass = model.constantize        
+        save_and_verify_type('class_id', 'integer', nil, klass)
+        save_and_verify_type('class', 'string', nil, klass)
+                
+        begin
+        
+          # Fields are from the model
+          options['fields'] = options['fields'].to_a.map do |entry|
+            extract_table_alias!(entry, klass)
+            extract_field_alias!(entry, klass)
+            
+            unless klass.columns_hash[entry['field']]
+              # XXX I think this is here for migrations
+              Ultrasphinx.say "warning: field #{entry['field']} is not present in #{model}"
+            else
+              save_and_verify_type(entry['as'], klass.columns_hash[entry['field']].type, nil, klass)
+              install_duplicate_fields!(entry, klass)
+            end            
+          end  
+          
+          # Joins are whatever they are in the target       
+          options['include'].to_a.each do |entry|
+            extract_table_alias!(entry, klass)
+            extract_field_alias!(entry, klass)
+            
+            association_model = get_association_model(klass, entry)
+            
+            save_and_verify_type(entry['as'] || entry['field'], association_model.columns_hash[entry['field']].type, nil, klass)
+            install_duplicate_fields!(entry, klass)
+          end  
+          
+          # Regular concats are CHAR, group_concats are BLOB and need to be cast to CHAR
+          options['concatenate'].to_a.each do |entry|
+            extract_table_alias!(entry, klass)
+            save_and_verify_type(entry['as'], 'text', nil, klass) 
+            install_duplicate_fields!(entry, klass)
+          end          
+          
+        rescue ActiveRecord::StatementInvalid
+          Ultrasphinx.say "warning: model #{model} does not exist in the database yet"
+        end  
+      end
+      
+      self
+    end
+    
+    
+    def install_duplicate_fields!(entry, klass)
+      if entry['facet']
+        # Source must be a string
+        save_and_verify_type(entry['as'], 'text', nil, klass, 
+          "#{klass}##{entry['as']}: 'facet' option is only valid for text fields; numeric fields are enabled by default")
+        # Install facet column                
+        save_and_verify_type("#{entry['as']}_facet", 'integer', nil, klass)
+      end
+
+      if entry['sortable']
+        # Source must be a string
+        save_and_verify_type(entry['as'], 'text', nil, klass, 
+          "#{klass}##{entry['as']}: 'sortable' option is only valid for text columns; numeric fields are enabled by default")
+        # Install sortable column        
+        save_and_verify_type("#{entry['as']}_sortable", 'text', true, klass)      
+      end
+      entry
+    end
+    
+    
+    def extract_field_alias!(entry, klass)
+      unless entry['as']    
+        entry['as'] = entry['field'] 
+      end
+    end
+    
+    
+    def extract_table_alias!(entry, klass)
+      unless entry['table_alias']
+        entry['table_alias'] = if entry['field'] and entry['field'].include? "." and entry['association_sql']
+          # This field is referenced by a table alias in association_sql
+          table_alias, entry['field'] = entry['field'].split(".")
+          table_alias
+        elsif get_association(klass, entry)
+          # Refers to the association
+          get_association(klass, entry).name
+        elsif entry['association_sql']
+          # Refers to the association_sql class's table
+          entry['class_name'].constantize.table_name
+        else
+          # Refers to this class
+          klass.table_name
+        end
+      end
+    end
+    
+  end
+end
+