freelancing-god-thinking-sphinx 0.9.12 → 0.9.13

data/README

@@ -92,3 +92,5 @@ Since I first released this library, there's been quite a few people who have su
 - Jim Remsik
 - Kennon Ballou
 - Henrik Nyh
+- Emil Tin
+- Doug Cole

data/lib/thinking_sphinx.rb

@@ -30,7 +30,7 @@ module ThinkingSphinx
   module Version #:nodoc:
     Major = 0
     Minor = 9
-    Tiny  = 12
+    Tiny  = 13
     
     String = [Major, Minor, Tiny].join('.')
   end
@@ -40,6 +40,15 @@ module ThinkingSphinx
   class ConnectionError < StandardError
   end
   
+  # A StaleIdsException is thrown by Collection.instances_from_matches if there
+  # are records in Sphinx but not in the database, so the search can be retried.
+  class StaleIdsException < StandardError
+    attr_accessor :ids
+    def initialize(ids)
+      self.ids = ids
+    end
+  end
+  
   # The collection of indexed models. Keep in mind that Rails lazily loads
   # its classes, so this may not actually be populated with _all_ the models
   # that have Sphinx indexes.
@@ -97,6 +106,16 @@ module ThinkingSphinx
     @@updates_enabled = value
   end
   
+  @@suppress_delta_output = false
+  
+  def self.suppress_delta_output?
+    @@suppress_delta_output
+  end
+  
+  def self.suppress_delta_output=(value)
+    @@suppress_delta_output = value
+  end
+  
   # Checks to see if MySQL will allow simplistic GROUP BY statements. If not,
   # or if not using MySQL, this will return false.
   # 
@@ -113,7 +132,7 @@ module ThinkingSphinx
   def self.sphinx_running?
     pid_file = ThinkingSphinx::Configuration.instance.pid_file
     
-    if File.exists?(pid_file)
+    if pid_file && File.exists?(pid_file)
       pid = `cat #{pid_file}`[/\d+/]
       `ps -p #{pid} | wc -l`.to_i > 1
     else

data/lib/thinking_sphinx/active_record.rb

@@ -83,6 +83,10 @@ module ThinkingSphinx
           end
           alias_method :sphinx_index, :define_index
           
+          def sphinx_index_options
+            sphinx_indexes.last.options
+          end
+          
           # Generate a unique CRC value for the model's name, to use to
           # determine which Sphinx documents belong to which AR records.
           # 
@@ -115,6 +119,82 @@ module ThinkingSphinx
 
             return parent
           end
+          
+          def to_riddle(offset)
+            ThinkingSphinx::AbstractAdapter.detect(self).setup
+            
+            indexes = [to_riddle_for_core(offset)]
+            indexes << to_riddle_for_delta(offset) if sphinx_delta?
+            indexes << to_riddle_for_distributed
+          end
+          
+          private
+          
+          def sphinx_name
+            self.name.underscore.tr(':/\\', '_')
+          end
+          
+          def sphinx_delta?
+            self.sphinx_indexes.any? { |index| index.delta? }
+          end
+          
+          def to_riddle_for_core(offset)
+            index = Riddle::Configuration::Index.new("#{sphinx_name}_core")
+            index.path = File.join(
+              ThinkingSphinx::Configuration.instance.searchd_file_path, index.name
+            )
+            
+            set_configuration_options_for_indexes index
+            set_field_settings_for_indexes        index
+            
+            self.sphinx_indexes.select { |ts_index|
+              ts_index.model == self
+            }.each_with_index do |ts_index, i|
+              index.sources << ts_index.to_riddle_for_core(offset, i)
+            end
+            
+            index
+          end
+          
+          def to_riddle_for_delta(offset)
+            index = Riddle::Configuration::Index.new("#{sphinx_name}_delta")
+            index.parent = "#{sphinx_name}_core"
+            index.path = File.join(ThinkingSphinx::Configuration.instance.searchd_file_path, index.name)
+            
+            self.sphinx_indexes.each_with_index do |ts_index, i|
+              index.sources << ts_index.to_riddle_for_delta(offset, i) if ts_index.delta?
+            end
+            
+            index
+          end
+          
+          def to_riddle_for_distributed
+            index = Riddle::Configuration::DistributedIndex.new(sphinx_name)
+            index.local_indexes << "#{sphinx_name}_core"
+            index.local_indexes.unshift "#{sphinx_name}_delta" if sphinx_delta?
+            index
+          end
+          
+          def set_configuration_options_for_indexes(index)
+            ThinkingSphinx::Configuration.instance.index_options.each do |key, value|
+              index.send("#{key}=".to_sym, value)
+            end
+            
+            self.sphinx_indexes.each do |ts_index|
+              ts_index.options.each do |key, value|
+                index.send("#{key}=".to_sym, value) if ThinkingSphinx::Configuration::IndexOptions.include?(key.to_s) && !value.nil?
+              end
+            end
+          end
+          
+          def set_field_settings_for_indexes(index)
+            field_names = lambda { |field| field.unique_name.to_s }
+            
+            self.sphinx_indexes.each do |ts_index|
+              index.prefix_field_names += ts_index.prefix_fields.collect(&field_names)
+              index.infix_field_names  += ts_index.infix_fields.collect(&field_names)
+            end
+          end
         end
       end
       
@@ -155,6 +235,8 @@ module ThinkingSphinx
       ) if ThinkingSphinx.deltas_enabled? &&
         self.class.sphinx_indexes.any? { |index| index.delta? } &&
         self.delta?
+    rescue ::ThinkingSphinx::ConnectionError
+      # nothing
     end
     
     def sphinx_document_id

data/lib/thinking_sphinx/active_record/delta.rb

@@ -54,7 +54,8 @@ module ThinkingSphinx
                 {instance.sphinx_document_id => 1}
               ) if instance && instance.in_core_index?
               
-              system "#{config.bin_path}indexer --config #{config.config_file} --rotate #{self.sphinx_indexes.first.name}_delta"
+              output = `#{config.bin_path}indexer --config #{config.config_file} --rotate #{self.sphinx_indexes.first.name}_delta`
+              puts output unless ThinkingSphinx.suppress_delta_output?
 
               true
             end

data/lib/thinking_sphinx/adapters/postgresql_adapter.rb

@@ -8,76 +8,75 @@ module ThinkingSphinx
       
       private
       
-      def create_array_accum_function
+      def execute(command, output_error = false)
         connection.execute "begin"
         connection.execute "savepoint ts"
         begin
-          # See http://www.postgresql.org/docs/8.2/interactive/sql-createaggregate.html
-          if connection.raw_connection.server_version > 80200
-            connection.execute <<-SQL
-              CREATE AGGREGATE array_accum (anyelement)
-              (
-                  sfunc = array_append,
-                  stype = anyarray,
-                  initcond = '{}'
-              );
-            SQL
-          else
-            connection.execute <<-SQL
-              CREATE AGGREGATE array_accum
-              (
-                  basetype = anyelement,
-                  sfunc = array_append,
-                  stype = anyarray,
-                  initcond = '{}'
-              );
-            SQL
-          end
-        rescue
+          connection.execute command
+        rescue StandardError => err
+          puts err if output_error
           connection.execute "rollback to savepoint ts"
         end
         connection.execute "release savepoint ts"
         connection.execute "commit"
       end
       
+      def create_array_accum_function
+        if connection.raw_connection.server_version > 80200
+          execute <<-SQL
+            CREATE AGGREGATE array_accum (anyelement)
+            (
+                sfunc = array_append,
+                stype = anyarray,
+                initcond = '{}'
+            );
+          SQL
+        else
+          execute <<-SQL
+            CREATE AGGREGATE array_accum
+            (
+                basetype = anyelement,
+                sfunc = array_append,
+                stype = anyarray,
+                initcond = '{}'
+            );
+          SQL
+        end
+      end
+      
       def create_crc32_function
-        connection.execute "begin"
-        connection.execute "savepoint ts"
-        begin
-          connection.execute "CREATE LANGUAGE 'plpgsql';"
-          connection.execute <<-SQL
-            CREATE OR REPLACE FUNCTION crc32(word text)
-            RETURNS bigint AS $$
-              DECLARE tmp bigint;
-              DECLARE i int;
-              DECLARE j int;
-              BEGIN
-                i = 0;
-                tmp = 4294967295;
+        execute "CREATE LANGUAGE 'plpgsql';"
+        function = <<-SQL
+          CREATE OR REPLACE FUNCTION crc32(word text)
+          RETURNS bigint AS $$
+            DECLARE tmp bigint;
+            DECLARE i int;
+            DECLARE j int;
+            DECLARE word_array bytea;
+            BEGIN
+              i = 0;
+              tmp = 4294967295;
+              word_array = decode(replace(word, E'\\\\', E'\\\\\\\\'), 'escape');
+              LOOP
+                tmp = (tmp # get_byte(word_array, i))::bigint;
+                i = i + 1;
+                j = 0;
                 LOOP
-                  tmp = (tmp # get_byte(word::bytea, i))::bigint;
-                  i = i + 1;
-                  j = 0;
-                  LOOP
-                    tmp = ((tmp >> 1) # (3988292384 * (tmp & 1)))::bigint;
-                    j = j + 1;
-                    IF j >= 8 THEN
-                      EXIT;
-                    END IF;
-                  END LOOP;
-                  IF i >= char_length(word) THEN
+                  tmp = ((tmp >> 1) # (3988292384 * (tmp & 1)))::bigint;
+                  j = j + 1;
+                  IF j >= 8 THEN
                     EXIT;
                   END IF;
                 END LOOP;
-                return (tmp # 4294967295);
-              END
-            $$ IMMUTABLE STRICT LANGUAGE plpgsql;
-          SQL
-        rescue
-          connection.execute "rollback to savepoint ts"
-        end
-        connection.execute "release savepoint ts"
-        connection.execute "commit"
+                IF i >= char_length(word) THEN
+                  EXIT;
+                END IF;
+              END LOOP;
+              return (tmp # 4294967295);
+            END
+          $$ IMMUTABLE STRICT LANGUAGE plpgsql;
+        SQL
+        execute function, true
       end
     end
   end

data/lib/thinking_sphinx/attribute.rb

@@ -101,23 +101,22 @@ module ThinkingSphinx
       end
     end
     
-    # Generates the appropriate attribute statement for a Sphinx configuration
-    # file, depending on the attribute's type.
-    # 
-    def to_sphinx_clause
-      case type
-      when :multi
-        "sql_attr_multi       = uint #{unique_name} from field"
-      when :datetime
-        "sql_attr_timestamp   = #{unique_name}"
-      when :string
-        "sql_attr_str2ordinal = #{unique_name}"
-      when :float
-        "sql_attr_float       = #{unique_name}"
-      when :boolean
-        "sql_attr_bool        = #{unique_name}"
+    def type_to_config
+      {
+        :multi    => :sql_attr_multi,
+        :datetime => :sql_attr_timestamp,
+        :string   => :sql_attr_str2ordinal,
+        :float    => :sql_attr_float,
+        :boolean  => :sql_attr_bool,
+        :integer  => :sql_attr_uint
+      }[type]
+    end
+    
+    def config_value
+      if type == :multi
+        "uint #{unique_name} from field"
       else
-        "sql_attr_uint        = #{unique_name}"
+        unique_name
       end
     end
     
@@ -141,7 +140,9 @@ module ThinkingSphinx
       when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
         "CONCAT_WS('#{separator}', #{clause})"
       when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-        clause.split(', ').join(" || '#{separator}' || ")
+        clause.split(', ').collect { |attribute|
+          "COALESCE(#{attribute}, '')"
+        }.join(" || ' ' || ")
       else
         clause
       end

data/lib/thinking_sphinx/collection.rb

@@ -1,7 +1,10 @@
 module ThinkingSphinx
   class Collection < ::Array
-    attr_reader :total_entries, :total_pages, :current_page
+    attr_reader :total_entries, :total_pages, :current_page, :per_page
     attr_accessor :results
+
+    # Compatibility with older versions of will_paginate
+    alias_method :page_count, :total_pages
     
     def initialize(page, per_page, entries, total_entries)
       @current_page, @per_page, @total_entries = page, per_page, total_entries
@@ -34,25 +37,36 @@ module ThinkingSphinx
         instance_from_match match, options
       } unless klass = options[:class]
       
+      index_options = klass.sphinx_index_options
+      
       ids = matches.collect { |match| match[:attributes]["sphinx_internal_id"] }
       instances = ids.length > 0 ? klass.find(
         :all,
         :conditions => {klass.primary_key.to_sym => ids},
-        :include    => options[:include],
-        :select     => options[:select]
+        :include    => (options[:include] || index_options[:include]),
+        :select     => (options[:select] || index_options[:select])
       ) : []
+      
+      # Raise an exception if we find records in Sphinx but not in the DB, so the search method
+      # can retry without them. See ThinkingSphinx::Search.retry_search_on_stale_index.
+      if options[:raise_on_stale] && instances.length < ids.length
+        stale_ids = ids - instances.map {|i| i.id }
+        raise StaleIdsException, stale_ids
+      end
+
       ids.collect { |obj_id|
         instances.detect { |obj| obj.id == obj_id }
       }
     end
     
     def self.instance_from_match(match, options)
-      # puts "ARGS: #{match[:attributes]["sphinx_internal_id"].inspect}, {:include => #{options[:include].inspect}, :select => #{options[:select].inspect}}"
       class_from_crc(match[:attributes]["class_crc"]).find(
         match[:attributes]["sphinx_internal_id"],
         :include => options[:include],
         :select  => options[:select]
       )
+    rescue ::ActiveRecord::RecordNotFound
+      nil
     end
     
     def self.class_from_crc(crc)

data/lib/thinking_sphinx/configuration.rb

@@ -52,20 +52,14 @@ module ThinkingSphinx
       min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
       ngram_len phrase_boundary phrase_boundary_step preopen stopwords
       wordforms )
-    
-    IndexerOptions = %w( max_iops max_iosize mem_limit )
-    
-    SearchdOptions = %w( read_timeout max_children max_matches seamless_rotate
-      preopen_indexes unlink_old )
-    
+        
     attr_accessor :config_file, :searchd_log_file, :query_log_file,
       :pid_file, :searchd_file_path, :address, :port, :allow_star,
       :database_yml_file, :app_root, :bin_path
     
-    attr_accessor :source_options, :index_options, :indexer_options,
-      :searchd_options
+    attr_accessor :source_options, :index_options
     
-    attr_reader :environment
+    attr_reader :environment, :configuration
     
     # Load in the configuration settings - this will look for config/sphinx.yml
     # and parse it according to the current environment.
@@ -79,14 +73,16 @@ module ThinkingSphinx
       self.app_root          = Merb.root  if defined?(Merb)
       self.app_root        ||= app_root
       
+      @configuration = Riddle::Configuration.new
+      @configuration.searchd.address    = "127.0.0.1"
+      @configuration.searchd.port       = 3312
+      @configuration.searchd.pid_file   = "#{self.app_root}/log/searchd.#{environment}.pid"
+      @configuration.searchd.log        = "#{self.app_root}/log/searchd.log"
+      @configuration.searchd.query_log  = "#{self.app_root}/log/searchd.query.log"
+      
       self.database_yml_file    = "#{self.app_root}/config/database.yml"
       self.config_file          = "#{self.app_root}/config/#{environment}.sphinx.conf"
-      self.searchd_log_file     = "#{self.app_root}/log/searchd.log"
-      self.query_log_file       = "#{self.app_root}/log/searchd.query.log"
-      self.pid_file             = "#{self.app_root}/log/searchd.#{environment}.pid"
       self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
-      self.address              = "127.0.0.1"
-      self.port                 = 3312
       self.allow_star           = false
       self.bin_path             = ""
       
@@ -95,9 +91,7 @@ module ThinkingSphinx
         :charset_type => "utf-8",
         :morphology   => "stem_en"
       }
-      self.indexer_options = {}
-      self.searchd_options = {}
-      
+            
       parse_config
       
       self
@@ -113,6 +107,10 @@ module ThinkingSphinx
       self.class.environment
     end
     
+    def controller
+      @controller ||= Riddle::Controller.new(@configuration, self.config_file)
+    end
+    
     # Generate the config file for Sphinx by using all the settings defined and
     # looping through all the models with indexes to build the relevant
     # indexer and searchd configuration, and sources and indexes details.
@@ -120,57 +118,15 @@ module ThinkingSphinx
     def build(file_path=nil)
       load_models
       file_path ||= "#{self.config_file}"
-      database_confs = YAML::load(ERB.new(IO.read("#{self.database_yml_file}")).result)
-      database_confs.symbolize_keys!
-      database_conf  = database_confs[environment.to_sym]
-      database_conf.symbolize_keys!
+      
+      @configuration.indexes.clear
+      
+      ThinkingSphinx.indexed_models.each_with_index do |model, model_index|
+        @configuration.indexes.concat model.constantize.to_riddle(model_index)
+      end
       
       open(file_path, "w") do |file|
-        file.write <<-CONFIG
-indexer
-{
-#{hash_to_config(self.indexer_options)}
-}
-
-searchd
-{
-  address = #{self.address}
-  port = #{self.port}
-  log = #{self.searchd_log_file}
-  query_log = #{self.query_log_file}
-  pid_file = #{self.pid_file}
-#{hash_to_config(self.searchd_options)}
-}
-        CONFIG
-        
-        ThinkingSphinx.indexed_models.each_with_index do |model, model_index|
-          model           = model.constantize
-          sources         = []
-          delta_sources   = []
-          prefixed_fields = []
-          infixed_fields  = []
-          
-          model.sphinx_indexes.select { |index| index.model == model }.each_with_index do |index, i|
-            file.write index.to_config(model, i, database_conf, model_index)
-            
-            index.adapter_object.setup
-            
-            sources << "#{ThinkingSphinx::Index.name(model)}_#{i}_core"
-            delta_sources << "#{ThinkingSphinx::Index.name(model)}_#{i}_delta" if index.delta?
-          end
-          
-          next if sources.empty?
-          
-          source_list = sources.collect       { |s| "source = #{s}" }.join("\n")
-          delta_list  = delta_sources.collect { |s| "source = #{s}" }.join("\n")
-          
-          file.write core_index_for_model(model, source_list)
-          unless delta_list.blank?
-            file.write delta_index_for_model(model, delta_list)
-          end
-          
-          file.write distributed_index_for_model(model)
-        end
+        file.write @configuration.render
       end
     end
     
@@ -223,6 +179,46 @@ searchd
       base
     end
     
+    def address
+      @configuration.searchd.address
+    end
+    
+    def address=(address)
+      @configuration.searchd.address = address
+    end
+    
+    def port
+      @configuration.searchd.port
+    end
+    
+    def port=(port)
+      @configuration.searchd.port = port
+    end
+    
+    def pid_file
+      @configuration.searchd.pid_file
+    end
+    
+    def pid_file=(pid_file)
+      @configuration.searchd.pid_file = pid_file
+    end
+    
+    def searchd_log_file
+      @configuration.searchd.log
+    end
+    
+    def searchd_log_file=(file)
+      @configuration.searchd.log = file
+    end
+    
+    def query_log_file
+      @configuration.searchd.query_log
+    end
+    
+    def query_log_file=(file)
+      @configuration.searchd.query_log = file
+    end
+    
     private
     
     # Parse the config/sphinx.yml file - if it exists - then use the attribute
@@ -237,77 +233,21 @@ searchd
       conf.each do |key,value|
         self.send("#{key}=", value) if self.methods.include?("#{key}=")
         
-        self.source_options[key.to_sym] = value  if SourceOptions.include?(key.to_s)
-        self.index_options[key.to_sym] = value   if IndexOptions.include?(key.to_s)
-        self.indexer_options[key.to_sym] = value if IndexerOptions.include?(key.to_s)
-        self.searchd_options[key.to_sym] = value if SearchdOptions.include?(key.to_s)
+        set_sphinx_setting self.source_options, key, value, SourceOptions
+        set_sphinx_setting self.index_options,  key, value, IndexOptions
+        set_sphinx_setting @configuration.searchd, key, value
+        set_sphinx_setting @configuration.indexer, key, value
       end unless conf.nil?
       
       self.bin_path += '/' unless self.bin_path.blank?
     end
     
-    def core_index_for_model(model, sources)
-      output = <<-INDEX
-
-index #{ThinkingSphinx::Index.name(model)}_core
-{
-#{sources}
-path = #{self.searchd_file_path}/#{ThinkingSphinx::Index.name(model)}_core
-INDEX
-      
-      unless combined_index_options(model).empty?
-        output += hash_to_config(combined_index_options(model))
-      end
-      
-      if self.allow_star
-        # Ye Olde way of turning on enable_star
-        output += "  enable_star    = 1\n"
-        output += "  min_prefix_len = #{combined_index_options[:min_prefix_len]}\n"
-      end
-      
-      unless model.sphinx_indexes.collect(&:prefix_fields).flatten.empty?
-        output += "  prefix_fields = #{model.sphinx_indexes.collect(&:prefix_fields).flatten.map(&:unique_name).join(', ')}\n"
-      else
-        output += " prefix_fields = _\n" unless model.sphinx_indexes.collect(&:infix_fields).flatten.empty?
-      end
-      
-      unless model.sphinx_indexes.collect(&:infix_fields).flatten.empty?
-        output += "  infix_fields  = #{model.sphinx_indexes.collect(&:infix_fields).flatten.map(&:unique_name).join(', ')}\n"
+    def set_sphinx_setting(object, key, value, allowed = {})
+      if object.is_a?(Hash)
+        object[key.to_sym] = value if allowed.include?(key.to_s)
       else
-        output += " infix_fields = -\n" unless model.sphinx_indexes.collect(&:prefix_fields).flatten.empty?
-      end
-      
-      output + "\n}\n"
-    end
-    
-    def delta_index_for_model(model, sources)
-      <<-INDEX
-index #{ThinkingSphinx::Index.name(model)}_delta : #{ThinkingSphinx::Index.name(model)}_core
-{
-  #{sources}
-  path = #{self.searchd_file_path}/#{ThinkingSphinx::Index.name(model)}_delta
-}
-      INDEX
-    end
-    
-    def distributed_index_for_model(model)
-      sources = ["local = #{ThinkingSphinx::Index.name(model)}_core"]
-      if model.sphinx_indexes.any? { |index| index.delta? }
-        sources << "local = #{ThinkingSphinx::Index.name(model)}_delta"
-      end
-      
-      <<-INDEX
-index #{ThinkingSphinx::Index.name(model)}
-{
-  type = distributed
-  #{ sources.join("\n  ") }
-}
-      INDEX
-    end
-    
-    def combined_index_options(model)
-      model.sphinx_indexes.inject(self.index_options) do |options, index|
-        self.class.options_merge(options, index.local_index_options)
+        object.send("#{key}=", value) if object.methods.include?("#{key}")
+        send("#{key}=", value) if self.methods.include?("#{key}")
       end
     end
   end