freelancing-god-thinking-sphinx 0.9.7 → 0.9.8

data/README

@@ -4,6 +4,8 @@
 
 First, if you haven't done so already, check out the main usage[http://ts.freelancing-gods.com/usage.html] page. Once you've done that, the next place to look for information is the specific method docs - ThinkingSphinx::Search and ThinkingSphinx::Index::Builder in particular.
 
+Keep in mind that while Thinking Sphinx works for ActiveRecord with Merb, it doesn't yet support DataMapper (although that is planned).
+
 == Contributing
 
 Fork on GitHub and after you've committed tested patches, send a pull request.
@@ -44,4 +46,12 @@ Since I first released this library, there's been quite a few people who have su
 - James Healy
 - Jae-Jun Hwang
 - Xavier Shay
-- Jason Rust
+- Jason Rust
+- Gopal Patel
+- Chris Heald
+- Peter Vandenberk
+- Josh French
+- Andrew Bennett
+- Jordan Fowler
+- Seth Walker
+- Joe Noon

data/lib/riddle.rb

@@ -1,4 +1,5 @@
 require 'socket'
+require 'timeout'
 require 'riddle/client'
 require 'riddle/client/filter'
 require 'riddle/client/message'
@@ -9,14 +10,17 @@ module Riddle #:nodoc:
   end
   
   module Version #:nodoc:
-    Major = 0
-    Minor = 9
-    Tiny  = 8
+    Major   = 0
+    Minor   = 9
+    Tiny    = 8
     # Revision number for RubyForge's sake, taken from what Sphinx
     # outputs to the command line.
-    Rev   = 1198
+    Rev     = 1371
+    # Release number to mark my own fixes, beyond feature parity with
+    # Sphinx itself.
+    Release = 0
     
-    String      = [Major, Minor, Tiny].join('.') + "rc1"
-    GemVersion  = [Major, Minor, Tiny, Rev].join('.')
+    String      = [Major, Minor, Tiny].join('.')
+    GemVersion  = [Major, Minor, Tiny, Rev, Release].join('.')
   end
 end

data/lib/riddle/client.rb

@@ -100,7 +100,7 @@ module Riddle
       :match_mode, :sort_mode, :sort_by, :weights, :id_range, :filters,
       :group_by, :group_function, :group_clause, :group_distinct, :cut_off,
       :retry_count, :retry_delay, :anchor, :index_weights, :rank_mode,
-      :max_query_time, :field_weights
+      :max_query_time, :field_weights, :timeout
     attr_reader :queue
     
     # Can instantiate with a specific server and port - otherwise it assumes
@@ -134,10 +134,40 @@ module Riddle
       @max_query_time = 0
       # string keys are field names, integer values are weightings
       @field_weights  = {}
+      @timeout        = 0
       
       @queue = []
     end
     
+    # Reset attributes and settings to defaults.
+    def reset
+      # defaults
+      @offset         = 0
+      @limit          = 20
+      @max_matches    = 1000
+      @match_mode     = :all
+      @sort_mode      = :relevance
+      @sort_by        = ''
+      @weights        = []
+      @id_range       = 0..0
+      @filters        = []
+      @group_by       = ''
+      @group_function = :day
+      @group_clause   = '@group desc'
+      @group_distinct = ''
+      @cut_off        = 0
+      @retry_count    = 0
+      @retry_delay    = 0
+      @anchor         = {}
+      # string keys are index names, integer values are weightings
+      @index_weights  = {}
+      @rank_mode      = :proximity_bm25
+      @max_query_time = 0
+      # string keys are field names, integer values are weightings
+      @field_weights  = {}
+      @timeout        = 0
+    end
+    
     # Set the geo-anchor point - with the names of the attributes that contain
     # the latitude and longitude (in radians), and the reference position.
     # Note that for geocoding to work properly, you must also set
@@ -384,6 +414,26 @@ module Riddle
     # Connects to the Sphinx daemon, and yields a socket to use. The socket is
     # closed at the end of the block.
     def connect(&block)
+      socket = nil
+      if @timeout == 0
+        socket = initialise_connection
+      else
+        begin
+          Timeout.timeout(@timeout) { socket = initialise_connection }
+        rescue Timeout::Error
+          raise Riddle::ConnectionError,
+            "Connection to #{@server} on #{@port} timed out after #{@timeout} seconds"
+        end
+      end
+      
+      begin
+        yield socket
+      ensure
+        socket.close
+      end
+    end
+    
+    def initialise_connection
       socket = TCPSocket.new @server, @port
       
       # Checking version
@@ -396,11 +446,7 @@ module Riddle
       # Send version
       socket.send [1].pack('N'), 0
       
-      begin
-        yield socket
-      ensure
-        socket.close
-      end
+      socket
     end
     
     # Send a collection of messages, for a command type (eg, search, excerpts,
@@ -507,7 +553,7 @@ module Riddle
       # Per Index Weights
       message.append_int @index_weights.length
       @index_weights.each do |key,val|
-        message.append_string key
+        message.append_string key.to_s
         message.append_int val
       end
       
@@ -517,7 +563,7 @@ module Riddle
       # Per Field Weights
       message.append_int @field_weights.length
       @field_weights.each do |key,val|
-        message.append_string key
+        message.append_string key.to_s
         message.append_int val
       end
       

data/lib/riddle/client/filter.rb

@@ -18,7 +18,7 @@ module Riddle
       def query_message
         message = Message.new
         
-        message.append_string self.attribute
+        message.append_string self.attribute.to_s
         case self.values
         when Range
           if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)

data/lib/thinking_sphinx.rb

@@ -20,7 +20,7 @@ module ThinkingSphinx
   module Version #:nodoc:
     Major = 0
     Minor = 9
-    Tiny  = 7
+    Tiny  = 8
     
     String = [Major, Minor, Tiny].join('.')
   end
@@ -52,10 +52,12 @@ module ThinkingSphinx
     @@define_indexes = value
   end
   
+  @@deltas_enabled = nil
+
   # Check if delta indexing is enabled.
   # 
   def self.deltas_enabled?
-    @@deltas_enabled =  true unless defined?(@@deltas_enabled)
+    @@deltas_enabled  = (ThinkingSphinx::Configuration.environment != 'test') if @@deltas_enabled.nil?
     @@deltas_enabled == true
   end
   
@@ -79,4 +81,4 @@ module ThinkingSphinx
       "SELECT @@global.sql_mode, @@session.sql_mode;"
     ).all? { |key,value| value.nil? || value[/ONLY_FULL_GROUP_BY/].nil? }
   end
-end
+end

data/lib/thinking_sphinx/active_record.rb

@@ -10,9 +10,8 @@ module ThinkingSphinx
   module ActiveRecord
     def self.included(base)
       base.class_eval do
+        class_inheritable_array :indexes
         class << self
-          attr_accessor :indexes
-          
           # Allows creation of indexes for Sphinx. If you don't do this, there
           # isn't much point trying to search (or using this plugin at all,
           # really).
@@ -65,10 +64,10 @@ module ThinkingSphinx
           def define_index(&block)
             return unless ThinkingSphinx.define_indexes?
             
-            @indexes ||= []
+            self.indexes ||= []
             index = Index.new(self, &block)
             
-            @indexes << index
+            self.indexes << index
             unless ThinkingSphinx.indexed_models.include?(self.name)
               ThinkingSphinx.indexed_models << self.name
             end
@@ -114,21 +113,27 @@ module ThinkingSphinx
       )
     end
     
+    def in_core_index?
+      @in_core_index ||= self.class.search_for_id(self.id, "#{self.class.name.downcase}_core")
+    end
+    
     def toggle_deleted
       config = ThinkingSphinx::Configuration.new
       client = Riddle::Client.new config.address, config.port
       
       client.update(
-        "#{self.class.name.downcase}_core",
+        "#{self.class.indexes.first.name}_core",
         ['sphinx_deleted'],
         {self.id => 1}
-      )
+      ) if self.in_core_index?
       
       client.update(
-        "#{self.class.name.downcase}_delta",
+        "#{self.class.indexes.first.name}_delta",
         ['sphinx_deleted'],
         {self.id => 1}
-      ) if self.class.indexes.any? { |index| index.delta? }
+      ) if ThinkingSphinx.deltas_enabled? &&
+        self.class.indexes.any? { |index| index.delta? } &&
+        self.delta?
     end
   end
-end
+end

data/lib/thinking_sphinx/active_record/delta.rb

@@ -73,13 +73,10 @@ module ThinkingSphinx
           # if running in the test environment.
           # 
           def index_delta
-            if ThinkingSphinx::Configuration.environment == "test" ||
-              !ThinkingSphinx.deltas_enabled?
-              return true
-            end
+            return true unless ThinkingSphinx.deltas_enabled?
             
             configuration = ThinkingSphinx::Configuration.new
-            system "indexer --config #{configuration.config_file} --rotate #{self.class.name.downcase}_delta"
+            system "indexer --config #{configuration.config_file} --rotate #{self.class.indexes.first.name}_delta"
             
             true
           end

data/lib/thinking_sphinx/active_record/search.rb

@@ -28,6 +28,13 @@ module ThinkingSphinx
               args << options
               ThinkingSphinx::Search.search(*args)
             end
+            
+            def search_for_id(*args)
+              options = args.extract_options!
+              options[:class] = self
+              args << options
+              ThinkingSphinx::Search.search_for_id(*args)
+            end
           end
         end
       end

data/lib/thinking_sphinx/configuration.rb

@@ -1,30 +1,38 @@
+require 'erb'
+
 module ThinkingSphinx
   # This class both keeps track of the configuration settings for Sphinx and
   # also generates the resulting file for Sphinx to use.
   # 
   # Here are the default settings, relative to RAILS_ROOT where relevant:
   #
-  # config file::      config/#{environment}.sphinx.conf
-  # searchd log file:: log/searchd.log
-  # query log file::   log/searchd.query.log
-  # pid file::         log/searchd.#{environment}.pid
-  # searchd files::    db/sphinx/#{environment}/
-  # address::          0.0.0.0 (all)
-  # port::             3312
-  # allow star::       false
-  # mem limit::        64M
-  # max matches::      1000
-  # morphology::       stem_en
-  # charset type::     utf-8
-  # charset table::    nil
-  # ignore chars::     nil
+  # config file::           config/#{environment}.sphinx.conf
+  # searchd log file::      log/searchd.log
+  # query log file::        log/searchd.query.log
+  # pid file::              log/searchd.#{environment}.pid
+  # searchd files::         db/sphinx/#{environment}/
+  # address::               127.0.0.1
+  # port::                  3312
+  # allow star::            false
+  # min prefix length::     1
+  # min infix length::      1
+  # mem limit::             64M
+  # max matches::           1000
+  # morphology::            stem_en
+  # charset type::          utf-8
+  # charset table::         nil
+  # ignore chars::          nil
+  # html strip::            false
+  # html remove elements::  ''
   #
   # If you want to change these settings, create a YAML file at
   # config/sphinx.yml with settings for each environment, in a similar
   # fashion to database.yml - using the following keys: config_file,
   # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
-  # allow_star, mem_limit, max_matches, morphology, charset_type,
-  # charset_table, ignore_chars. I think you've got the idea.
+  # allow_star, min_prefix_len, min_infix_len, mem_limit, max_matches,
+  # morphology, charset_type, charset_table, ignore_chars, html_strip,
+  # html_remove_elements. I think you've got
+  # the idea.
   # 
   # Each setting in the YAML file is optional - so only put in the ones you
   # want to change.
@@ -35,9 +43,10 @@ module ThinkingSphinx
   # 
   class Configuration
     attr_accessor :config_file, :searchd_log_file, :query_log_file,
-      :pid_file, :searchd_file_path, :address, :port, :allow_star, :mem_limit,
-      :max_matches, :morphology, :charset_type, :charset_table, :ignore_chars,
-      :app_root
+      :pid_file, :searchd_file_path, :address, :port, :allow_star,
+      :min_prefix_len, :min_infix_len, :mem_limit, :max_matches, :morphology,
+      :charset_type, :charset_table, :ignore_chars, :html_strip,
+      :html_remove_elements, :app_root
     
     attr_reader :environment
     
@@ -49,20 +58,24 @@ module ThinkingSphinx
       self.app_root          = Merb.root  if defined?(Merb)
       self.app_root        ||= app_root
       
-      self.config_file       = "#{self.app_root}/config/#{environment}.sphinx.conf"
-      self.searchd_log_file  = "#{self.app_root}/log/searchd.log"
-      self.query_log_file    = "#{self.app_root}/log/searchd.query.log"
-      self.pid_file          = "#{self.app_root}/log/searchd.#{environment}.pid"
-      self.searchd_file_path = "#{self.app_root}/db/sphinx/#{environment}"
-      self.address           = "0.0.0.0"
-      self.port              = 3312
-      self.allow_star        = false
-      self.mem_limit         = "64M"
-      self.max_matches       = 1000
-      self.morphology        = "stem_en"
-      self.charset_type      = "utf-8"
-      self.charset_table     = nil
-      self.ignore_chars      = nil
+      self.config_file          = "#{self.app_root}/config/#{environment}.sphinx.conf"
+      self.searchd_log_file     = "#{self.app_root}/log/searchd.log"
+      self.query_log_file       = "#{self.app_root}/log/searchd.query.log"
+      self.pid_file             = "#{self.app_root}/log/searchd.#{environment}.pid"
+      self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
+      self.address              = "127.0.0.1"
+      self.port                 = 3312
+      self.allow_star           = false
+      self.min_prefix_len       = 1
+      self.min_infix_len        = 1
+      self.mem_limit            = "64M"
+      self.max_matches          = 1000
+      self.morphology           = "stem_en"
+      self.charset_type         = "utf-8"
+      self.charset_table        = nil
+      self.ignore_chars         = nil
+      self.html_strip           = false
+      self.html_remove_elements = ""
       
       parse_config
     end
@@ -84,7 +97,7 @@ module ThinkingSphinx
     def build(file_path=nil)
       load_models
       file_path ||= "#{self.config_file}"
-      database_confs = YAML.load(File.open("#{app_root}/config/database.yml"))
+      database_confs = YAML::load(ERB.new(IO.read("#{app_root}/config/database.yml")).result)
       database_confs.symbolize_keys!
       database_conf  = database_confs[environment.to_sym]
       database_conf.symbolize_keys!
@@ -120,8 +133,8 @@ searchd
             file.write index.to_config(i, database_conf, charset_type)
             
             create_array_accum if index.adapter == :postgres
-            sources << "#{model.name.downcase}_#{i}_core"
-            delta_sources << "#{model.name.downcase}_#{i}_delta" if index.delta?
+            sources << "#{model.indexes.first.name}_#{i}_core"
+            delta_sources << "#{model.indexes.first.name}_#{i}_delta" if index.delta?
           end
           
           source_list = sources.collect { |s| "source = #{s}" }.join("\n")
@@ -152,9 +165,10 @@ searchd
         }
         
         begin
-          model_name.classify.constantize
+          model_name.camelize.constantize
         rescue LoadError
-          model_name.gsub(/.*[\/\\]/, '').classify.constantize
+          model_name.gsub!(/.*[\/\\]/, '')
+          retry
         rescue NameError
           next
         end
@@ -170,7 +184,7 @@ searchd
       path = "#{app_root}/config/sphinx.yml"
       return unless File.exists?(path)
       
-      conf = YAML.load(File.open(path))[environment]
+      conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
       
       conf.each do |key,value|
         self.send("#{key}=", value) if self.methods.include?("#{key}=")
@@ -180,23 +194,29 @@ searchd
     def core_index_for_model(model, sources)
       output = <<-INDEX
 
-index #{model.name.downcase}_core
+index #{model.indexes.first.name}_core
 {
 #{sources}
-path = #{self.searchd_file_path}/#{model.name.downcase}_core
+path = #{self.searchd_file_path}/#{model.indexes.first.name}_core
 charset_type = #{self.charset_type}
 INDEX
       
-      output += "  morphology     = #{self.morphology}\n"    unless self.morphology.blank?
+      morphology  = model.indexes.inject(self.morphology) { |morph, index|
+        index.options[:morphology] || morph
+      }
+      output += "  morphology     = #{morphology}\n"         unless morphology.blank?
       output += "  charset_table  = #{self.charset_table}\n" unless self.charset_table.nil?
       output += "  ignore_chars   = #{self.ignore_chars}\n"  unless self.ignore_chars.nil?
       
       if self.allow_star
         output += "  enable_star    = 1\n"
-        output += "  min_prefix_len = 1\n"
-        output += "  min_infix_len  = 1\n"
+        output += "  min_prefix_len = #{self.min_prefix_len}\n"
+        output += "  min_infix_len  = #{self.min_infix_len}\n"
       end
       
+      output += "  html_strip     = 1\n" if self.html_strip
+      output += "  html_remove_elements = #{self.html_remove_elements}\n" unless self.html_remove_elements.blank?
+
       unless model.indexes.collect(&:prefix_fields).flatten.empty?
         output += "  prefix_fields = #{model.indexes.collect(&:prefix_fields).flatten.join(', ')}\n"
       end
@@ -210,22 +230,22 @@ INDEX
     
     def delta_index_for_model(model, sources)
       <<-INDEX
-index #{model.name.downcase}_delta : #{model.name.downcase}_core
+index #{model.indexes.first.name}_delta : #{model.indexes.first.name}_core
 {
   #{sources}
-  path = #{self.searchd_file_path}/#{model.name.downcase}_delta
+  path = #{self.searchd_file_path}/#{model.indexes.first.name}_delta
 }
       INDEX
     end
     
     def distributed_index_for_model(model)
-      sources = ["local = #{model.name.downcase}_core"]
+      sources = ["local = #{model.indexes.first.name}_core"]
       if model.indexes.any? { |index| index.delta? }
-        sources << "local = #{model.name.downcase}_delta"
+        sources << "local = #{model.indexes.first.name}_delta"
       end
       
       <<-INDEX
-index #{model.name.downcase}
+index #{model.indexes.first.name}
 {
   type = distributed
   #{ sources.join("\n  ") }