freelancing-god-thinking-sphinx 0.9.8 → 0.9.10

data/lib/thinking_sphinx/attribute.rb

@@ -230,9 +230,11 @@ module ThinkingSphinx
         "#{@model.quoted_table_name}.#{quote_column(column.__name)}"
       else
         associations[column].collect { |assoc|
+          assoc.has_column?(column.__name) ?
           "#{@model.connection.quote_table_name(assoc.join.aliased_table_name)}" + 
-          ".#{quote_column(column.__name)}"
-        }.join(', ')
+          ".#{quote_column(column.__name)}" :
+          nil
+        }.compact.join(', ')
       end
     end
     

data/lib/thinking_sphinx/collection.rb

@@ -0,0 +1,105 @@
+module ThinkingSphinx
+  class Collection < ::Array
+    attr_reader :total_entries, :total_pages, :current_page
+    attr_accessor :results
+    
+    def initialize(page, per_page, entries, total_entries)
+      @current_page, @per_page, @total_entries = page, per_page, total_entries
+      
+      @total_pages = (entries / @per_page.to_f).ceil
+    end
+    
+    def self.ids_from_results(results, page, limit, options)
+      collection = self.new(page, limit,
+        results[:total] || 0, results[:total_found] || 0
+      )
+      collection.results = results
+      collection.replace results[:matches].collect { |match|
+        match[:attributes]["sphinx_internal_id"]
+      }
+      return collection
+    end
+    
+    def self.create_from_results(results, page, limit, options)
+      collection = self.new(page, limit,
+        results[:total] || 0, results[:total_found] || 0
+      )
+      collection.results = results
+      collection.replace instances_from_matches(results[:matches], options)
+      return collection
+    end
+    
+    def self.instances_from_matches(matches, options = {})
+      return matches.collect { |match|
+        instance_from_match match, options
+      } unless klass = options[:class]
+      
+      ids = matches.collect { |match| match[:attributes]["sphinx_internal_id"] }
+      instances = ids.length > 0 ? klass.find(
+        :all,
+        :conditions => {klass.primary_key.to_sym => ids},
+        :include    => options[:include],
+        :select     => options[:select]
+      ) : []
+      ids.collect { |obj_id|
+        instances.detect { |obj| obj.id == obj_id }
+      }
+    end
+    
+    def self.instance_from_match(match, options)
+      # puts "ARGS: #{match[:attributes]["sphinx_internal_id"].inspect}, {:include => #{options[:include].inspect}, :select => #{options[:select].inspect}}"
+      class_from_crc(match[:attributes]["class_crc"]).find(
+        match[:attributes]["sphinx_internal_id"],
+        :include => options[:include],
+        :select  => options[:select]
+      )
+    end
+    
+    def self.class_from_crc(crc)
+      @@models_by_crc ||= ThinkingSphinx.indexed_models.inject({}) do |hash, model|
+        hash[model.constantize.to_crc32] = model
+        model.constantize.subclasses.each { |subclass|
+          hash[subclass.to_crc32] = subclass.name
+        }
+        hash
+      end
+      @@models_by_crc[crc].constantize
+    end
+    
+    def previous_page
+      current_page > 1 ? (current_page - 1) : nil
+    end
+    
+    def next_page
+      current_page < total_pages ? (current_page + 1): nil
+    end
+    
+    def offset
+      (current_page - 1) * @per_page
+    end
+    
+    def method_missing(method, *args, &block)
+      super unless method.to_s[/^each_with_.*/]
+      
+      each_with_attribute method.to_s.gsub(/^each_with_/, ''), &block
+    end
+    
+    def each_with_group_and_count(&block)
+      results[:matches].each_with_index do |match, index|
+        yield self[index], match[:attributes]["@group"], match[:attributes]["@count"]
+      end
+    end
+    
+    def each_with_attribute(attribute, &block)
+      results[:matches].each_with_index do |match, index|
+        yield self[index], (match[:attributes][attribute] || match[:attributes]["@#{attribute}"])
+      end
+    end
+    
+    def each_with_weighting(&block)
+      results[:matches].each_with_index do |match, index|
+        yield self[index], match[:weight]
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/configuration.rb

@@ -1,4 +1,5 @@
 require 'erb'
+require 'singleton'
 
 module ThinkingSphinx
   # This class both keeps track of the configuration settings for Sphinx and
@@ -29,10 +30,9 @@ module ThinkingSphinx
   # config/sphinx.yml with settings for each environment, in a similar
   # fashion to database.yml - using the following keys: config_file,
   # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
-  # allow_star, min_prefix_len, min_infix_len, mem_limit, max_matches,
-  # morphology, charset_type, charset_table, ignore_chars, html_strip,
-  # html_remove_elements. I think you've got
-  # the idea.
+  # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
+  # max_matches, # morphology, charset_type, charset_table, ignore_chars,
+  # html_strip, # html_remove_elements. I think you've got the idea.
   # 
   # Each setting in the YAML file is optional - so only put in the ones you
   # want to change.
@@ -42,11 +42,28 @@ module ThinkingSphinx
   # don't set allow_star to true.
   # 
   class Configuration
+    include Singleton
+    
+    SourceOptions = %w( mysql_connect_flags sql_range_step sql_query_pre
+      sql_query_post sql_ranged_throttle sql_query_post_index )
+    
+    IndexOptions  = %w( charset_table charset_type docinfo enable_star
+      exceptions html_index_attrs html_remove_elements html_strip ignore_chars
+      min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
+      ngram_len phrase_boundary phrase_boundary_step preopen stopwords
+      wordforms )
+    
+    IndexerOptions = %w( max_iops max_iosize mem_limit )
+    
+    SearchdOptions = %w( read_timeout max_children max_matches seamless_rotate
+      preopen_indexes unlink_old )
+    
     attr_accessor :config_file, :searchd_log_file, :query_log_file,
       :pid_file, :searchd_file_path, :address, :port, :allow_star,
-      :min_prefix_len, :min_infix_len, :mem_limit, :max_matches, :morphology,
-      :charset_type, :charset_table, :ignore_chars, :html_strip,
-      :html_remove_elements, :app_root
+      :database_yml_file, :app_root, :bin_path
+    
+    attr_accessor :source_options, :index_options, :indexer_options,
+      :searchd_options
     
     attr_reader :environment
     
@@ -54,10 +71,15 @@ module ThinkingSphinx
     # and parse it according to the current environment.
     # 
     def initialize(app_root = Dir.pwd)
+      self.reset
+    end
+    
+    def reset
       self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
       self.app_root          = Merb.root  if defined?(Merb)
       self.app_root        ||= app_root
       
+      self.database_yml_file    = "#{self.app_root}/config/database.yml"
       self.config_file          = "#{self.app_root}/config/#{environment}.sphinx.conf"
       self.searchd_log_file     = "#{self.app_root}/log/searchd.log"
       self.query_log_file       = "#{self.app_root}/log/searchd.query.log"
@@ -66,23 +88,24 @@ module ThinkingSphinx
       self.address              = "127.0.0.1"
       self.port                 = 3312
       self.allow_star           = false
-      self.min_prefix_len       = 1
-      self.min_infix_len        = 1
-      self.mem_limit            = "64M"
-      self.max_matches          = 1000
-      self.morphology           = "stem_en"
-      self.charset_type         = "utf-8"
-      self.charset_table        = nil
-      self.ignore_chars         = nil
-      self.html_strip           = false
-      self.html_remove_elements = ""
+      self.bin_path             = ""
+      
+      self.source_options  = {}
+      self.index_options   = {
+        :charset_type => "utf-8",
+        :morphology   => "stem_en"
+      }
+      self.indexer_options = {}
+      self.searchd_options = {}
       
       parse_config
+      
+      self
     end
     
     def self.environment
       @@environment ||= (
-        defined?(Merb) ? ENV['MERB_ENV'] : ENV['RAILS_ENV']
+        defined?(Merb) ? Merb.environment : ENV['RAILS_ENV']
       ) || "development"
     end
     
@@ -97,7 +120,7 @@ module ThinkingSphinx
     def build(file_path=nil)
       load_models
       file_path ||= "#{self.config_file}"
-      database_confs = YAML::load(ERB.new(IO.read("#{app_root}/config/database.yml")).result)
+      database_confs = YAML::load(ERB.new(IO.read("#{self.database_yml_file}")).result)
       database_confs.symbolize_keys!
       database_conf  = database_confs[environment.to_sym]
       database_conf.symbolize_keys!
@@ -106,7 +129,7 @@ module ThinkingSphinx
         file.write <<-CONFIG
 indexer
 {
-  mem_limit = #{self.mem_limit}
+#{hash_to_config(self.indexer_options)}
 }
 
 searchd
@@ -115,29 +138,30 @@ searchd
   port = #{self.port}
   log = #{self.searchd_log_file}
   query_log = #{self.query_log_file}
-  read_timeout = 5
-  max_children = 30
   pid_file = #{self.pid_file}
-  max_matches = #{self.max_matches}
+#{hash_to_config(self.searchd_options)}
 }
         CONFIG
         
-        ThinkingSphinx.indexed_models.each do |model|
+        ThinkingSphinx.indexed_models.each_with_index do |model, model_index|
           model           = model.constantize
           sources         = []
           delta_sources   = []
           prefixed_fields = []
           infixed_fields  = []
           
-          model.indexes.each_with_index do |index, i|
-            file.write index.to_config(i, database_conf, charset_type)
+          model.sphinx_indexes.select { |index| index.model == model }.each_with_index do |index, i|
+            file.write index.to_config(model, i, database_conf, model_index)
+            
+            index.adapter_object.setup
             
-            create_array_accum if index.adapter == :postgres
-            sources << "#{model.indexes.first.name}_#{i}_core"
-            delta_sources << "#{model.indexes.first.name}_#{i}_delta" if index.delta?
+            sources << "#{ThinkingSphinx::Index.name(model)}_#{i}_core"
+            delta_sources << "#{ThinkingSphinx::Index.name(model)}_#{i}_delta" if index.delta?
           end
           
-          source_list = sources.collect { |s| "source = #{s}" }.join("\n")
+          next if sources.empty?
+          
+          source_list = sources.collect       { |s| "source = #{s}" }.join("\n")
           delta_list  = delta_sources.collect { |s| "source = #{s}" }.join("\n")
           
           file.write core_index_for_model(model, source_list)
@@ -167,14 +191,38 @@ searchd
         begin
           model_name.camelize.constantize
         rescue LoadError
-          model_name.gsub!(/.*[\/\\]/, '')
-          retry
+          model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
         rescue NameError
           next
         end
       end
     end
     
+    def hash_to_config(hash)
+      hash.collect { |key, value|
+        translated_value = case value
+        when TrueClass
+          "1"
+        when FalseClass
+          "0"
+        when NilClass, ""
+          next
+        else
+          value
+        end
+        "  #{key} = #{translated_value}"
+      }.join("\n")
+    end
+    
+    def self.options_merge(base, extra)
+      base = base.clone
+      extra.each do |key, value|
+        next if value.nil? || value == ""
+        base[key] = value
+      end
+      base
+    end
+    
     private
     
     # Parse the config/sphinx.yml file - if it exists - then use the attribute
@@ -188,90 +236,79 @@ searchd
       
       conf.each do |key,value|
         self.send("#{key}=", value) if self.methods.include?("#{key}=")
+        
+        self.source_options[key.to_sym] = value  if SourceOptions.include?(key.to_s)
+        self.index_options[key.to_sym] = value   if IndexOptions.include?(key.to_s)
+        self.indexer_options[key.to_sym] = value if IndexerOptions.include?(key.to_s)
+        self.searchd_options[key.to_sym] = value if SearchdOptions.include?(key.to_s)
       end unless conf.nil?
+      
+      self.bin_path += '/' unless self.bin_path.blank?
     end
     
     def core_index_for_model(model, sources)
       output = <<-INDEX
 
-index #{model.indexes.first.name}_core
+index #{ThinkingSphinx::Index.name(model)}_core
 {
 #{sources}
-path = #{self.searchd_file_path}/#{model.indexes.first.name}_core
-charset_type = #{self.charset_type}
+path = #{self.searchd_file_path}/#{ThinkingSphinx::Index.name(model)}_core
 INDEX
       
-      morphology  = model.indexes.inject(self.morphology) { |morph, index|
-        index.options[:morphology] || morph
-      }
-      output += "  morphology     = #{morphology}\n"         unless morphology.blank?
-      output += "  charset_table  = #{self.charset_table}\n" unless self.charset_table.nil?
-      output += "  ignore_chars   = #{self.ignore_chars}\n"  unless self.ignore_chars.nil?
+      unless combined_index_options(model).empty?
+        output += hash_to_config(combined_index_options(model))
+      end
       
       if self.allow_star
+        # Ye Olde way of turning on enable_star
         output += "  enable_star    = 1\n"
-        output += "  min_prefix_len = #{self.min_prefix_len}\n"
-        output += "  min_infix_len  = #{self.min_infix_len}\n"
+        output += "  min_prefix_len = #{self.combined_index_options[:min_prefix_len]}\n"
       end
       
-      output += "  html_strip     = 1\n" if self.html_strip
-      output += "  html_remove_elements = #{self.html_remove_elements}\n" unless self.html_remove_elements.blank?
-
-      unless model.indexes.collect(&:prefix_fields).flatten.empty?
-        output += "  prefix_fields = #{model.indexes.collect(&:prefix_fields).flatten.join(', ')}\n"
+      unless model.sphinx_indexes.collect(&:prefix_fields).flatten.empty?
+        output += "  prefix_fields = #{model.sphinx_indexes.collect(&:prefix_fields).flatten.map(&:unique_name).join(', ')}\n"
+      else
+        output += " prefix_fields = _\n" unless model.sphinx_indexes.collect(&:infix_fields).flatten.empty?
       end
       
-      unless model.indexes.collect(&:infix_fields).flatten.empty?
-        output += "  infix_fields  = #{model.indexes.collect(&:infix_fields).flatten.join(', ')}\n"
+      unless model.sphinx_indexes.collect(&:infix_fields).flatten.empty?
+        output += "  infix_fields  = #{model.sphinx_indexes.collect(&:infix_fields).flatten.map(&:unique_name).join(', ')}\n"
+      else
+        output += " infix_fields = -\n" unless model.sphinx_indexes.collect(&:prefix_fields).flatten.empty?
       end
       
-      output + "}\n"
+      output + "\n}\n"
     end
     
     def delta_index_for_model(model, sources)
       <<-INDEX
-index #{model.indexes.first.name}_delta : #{model.indexes.first.name}_core
+index #{ThinkingSphinx::Index.name(model)}_delta : #{ThinkingSphinx::Index.name(model)}_core
 {
   #{sources}
-  path = #{self.searchd_file_path}/#{model.indexes.first.name}_delta
+  path = #{self.searchd_file_path}/#{ThinkingSphinx::Index.name(model)}_delta
 }
       INDEX
     end
     
     def distributed_index_for_model(model)
-      sources = ["local = #{model.indexes.first.name}_core"]
-      if model.indexes.any? { |index| index.delta? }
-        sources << "local = #{model.indexes.first.name}_delta"
+      sources = ["local = #{ThinkingSphinx::Index.name(model)}_core"]
+      if model.sphinx_indexes.any? { |index| index.delta? }
+        sources << "local = #{ThinkingSphinx::Index.name(model)}_delta"
       end
       
       <<-INDEX
-index #{model.indexes.first.name}
+index #{ThinkingSphinx::Index.name(model)}
 {
   type = distributed
   #{ sources.join("\n  ") }
-  charset_type = #{self.charset_type}
 }
       INDEX
     end
     
-    def create_array_accum
-      ::ActiveRecord::Base.connection.execute "begin"
-      ::ActiveRecord::Base.connection.execute "savepoint ts"
-      begin
-        ::ActiveRecord::Base.connection.execute <<-SQL
-          CREATE AGGREGATE array_accum (anyelement)
-          (
-              sfunc = array_append,
-              stype = anyarray,
-              initcond = '{}'
-          );
-        SQL
-      rescue
-        raise unless $!.to_s =~ /already exists with same argument types/
-        ::ActiveRecord::Base.connection.execute "rollback to savepoint ts"
+    def combined_index_options(model)
+      model.sphinx_indexes.inject(self.index_options) do |options, index|
+        self.class.options_merge(options, index.local_index_options)
       end
-      ::ActiveRecord::Base.connection.execute "release savepoint ts"
-      ::ActiveRecord::Base.connection.execute "commit"
     end
   end
 end

data/lib/thinking_sphinx/field.rb

@@ -178,13 +178,17 @@ module ThinkingSphinx
     # figure out how to correctly reference a column in SQL.
     #
     def column_with_prefix(column)
-      if associations[column].empty?
+      if column.is_string?
+        column.__name
+      elsif associations[column].empty?
         "#{@model.quoted_table_name}.#{quote_column(column.__name)}"
       else
         associations[column].collect { |assoc|
+          assoc.has_column?(column.__name) ?
           "#{@model.connection.quote_table_name(assoc.join.aliased_table_name)}" + 
-          ".#{quote_column(column.__name)}"
-        }.join(', ')
+          ".#{quote_column(column.__name)}" :
+          nil
+        }.compact.join(', ')
       end
     end
     
@@ -194,5 +198,9 @@ module ThinkingSphinx
     def is_many?
       associations.values.flatten.any? { |assoc| assoc.is_many? }
     end
+    
+    def is_string?
+      columns.all? { |col| col.is_string? }
+    end
   end
 end