thinking-sphinx 1.3.20 → 1.4.0

data/README.textile

@@ -172,3 +172,6 @@ Since I first released this library, there's been quite a few people who have su
 * Ben Hutton
 * Alfonso Jiménez
 * Szymon Nowak
+* Keith Pitt
+* Lee Capps
+* Sam Goldstein

data/VERSION

@@ -1 +1 @@
-1.3.20
+1.4.0

data/features/excerpts.feature

@@ -11,3 +11,11 @@ Feature: Generate excerpts for search results
     And I am searching on comments
     And I search for "lorem"
     Then calling content on the first result excerpts object should return "de un sitio mientras que mira su diseño. El punto de usar <span class="match">Lorem</span> Ipsum es que tiene una distribución"
+
+  Scenario: Excerpt Options
+    Given Sphinx is running
+    And I am searching on comments
+    And I search for "lorem"
+    And I provide excerpt option "before_match" with value "<em>"
+    And I provide excerpt option "after_match" with value "</em>"
+    Then calling content on the first result excerpts object should return "de un sitio mientras que mira su diseño. El punto de usar <em>Lorem</em> Ipsum es que tiene una distribución"

data/features/field_sorting.feature

@@ -0,0 +1,18 @@
+Feature: Field Sorting
+  In order to sort by strings
+  As a developer
+  I want to enable sorting by existing fields
+  
+  Background:
+    Given Sphinx is running
+    And I am searching on people
+    
+  Scenario: Searching with ordering on a sortable field
+    When I order by first_name
+    Then I should get 20 results
+    And the first_name of each result should indicate order
+  
+  Scenario: Sort on a case insensitive sortable field
+    When I order by last_name
+    Then the first result's "last_name" should be "abbott"
+  

data/features/searching_across_models.feature

@@ -7,7 +7,7 @@ Feature: Searching across multiple model
     Given Sphinx is running
     When I search for James
     And I am retrieving the result count
-    Then I should get a value of 3
+    Then I should get a value of 6
   
   Scenario: Confirming existance of a document id in a given index
     Given Sphinx is running

data/features/searching_by_model.feature

@@ -97,13 +97,6 @@ Feature: Searching on a single model
     Then I should get 10 results
     And the value of each result should indicate order
   
-  Scenario: Searching with ordering on a sortable field
-    Given Sphinx is running
-    And I am searching on people
-    And I order by first_name
-    Then I should get 20 results
-    And the first_name of each result should indicate order
-  
   Scenario: Intepreting Sphinx Internal Identifiers
     Given Sphinx is running
     And I am searching on people

data/features/sphinx_scopes.feature

@@ -48,3 +48,21 @@ Feature: Sphinx Scopes
     And I am retrieving the scoped result count for "Byrne"
     Then I should get a value of 1
   
+  Scenario: Default Scope
+    Given Sphinx is running
+    And I am searching on andrews
+    Then I should get 7 results
+  
+  Scenario: Default Scope and additional query terms
+    Given Sphinx is running
+    And I am searching on andrews
+    When I search for "Byrne"
+    Then I should get 1 result
+  
+  Scenario: Explicit scope plus search over a default scope
+    Given Sphinx is running
+    And I am searching on andrews
+    When I use the locked_last_name scope
+    And I search for "Cecil"
+    Then I should get 1 result
+  

data/features/step_definitions/common_steps.rb

@@ -155,6 +155,10 @@ Then /^the (\w+) of each result should indicate order$/ do |attribute|
   end
 end
 
+Then /^the first result's "([^"]*)" should be "([^"]*)"$/ do |attribute, value|
+  results.first.send(attribute.to_sym).should == value
+end
+
 Then /^I can iterate by result and (\w+)$/ do |attribute|
   iteration = lambda { |result, attr_value|
     result.should be_kind_of(@model)

data/features/step_definitions/search_steps.rb

@@ -87,3 +87,8 @@ end
 Then /^the first result should have a (\w+\s?\w*) of (\d+)$/ do |attribute, value|
   results.first.sphinx_attributes[attribute.gsub(/\s+/, '_')].should == value.to_i
 end
+
+Given /^I provide excerpt option "([a-z_]*)" with value "([^"]*)"$/ do |k, v|
+  @options[:excerpt_options] ||= {}
+  @options[:excerpt_options][k.to_sym] = v
+end

data/features/support/env.rb

@@ -1,10 +1,8 @@
 require 'rubygems'
-require 'cucumber'
-require 'spec'
 require 'fileutils'
-require 'ginger'
-require 'will_paginate'
-require 'active_record'
+require 'bundler'
+
+Bundler.require :default, :development
 
 $:.unshift File.dirname(__FILE__) + '/../../lib'
 Dir[File.join(File.dirname(__FILE__), '../../vendor/*/lib')].each do |path|

data/features/thinking_sphinx/db/fixtures/people.rb

@@ -18,7 +18,7 @@ Person.create :gender => "male", :first_name => "Peter", :middle_initial => "C",
 Person.create :gender => "female", :first_name => "Hollie", :middle_initial => "C", :last_name => "Hunter", :street_address => "34 Cornish Street", :city => "Kensington", :state => "VIC", :postcode => "3031", :email => "Hollie.C.Hunter@mailinator.com", :birthday => "1954/2/16 00:00:00"
 Person.create :gender => "male", :first_name => "Jonathan", :middle_initial => "C", :last_name => "Turner", :street_address => "2 Kopkes Road", :city => "Carngham", :state => "VIC", :postcode => "3351", :email => "Jonathan.C.Turner@trashymail.com", :birthday => "1963/8/26 00:00:00"
 Person.create :gender => "female", :first_name => "Kate", :middle_initial => "S", :last_name => "Doyle", :street_address => "42 Gregory Way", :city => "Mungalup", :state => "WA", :postcode => "6225", :email => "Kate.S.Doyle@mailinator.com", :birthday => "1974/1/5 00:00:00"
-Person.create :gender => "male", :first_name => "Harley", :middle_initial => "M", :last_name => "Abbott", :street_address => "39 Faulkner Street", :city => "Tilbuster", :state => "NSW", :postcode => "2350", :email => "Harley.M.Abbott@trashymail.com", :birthday => "1953/10/4 00:00:00"
+Person.create :gender => "male", :first_name => "Harley", :middle_initial => "M", :last_name => "abbott", :street_address => "39 Faulkner Street", :city => "Tilbuster", :state => "NSW", :postcode => "2350", :email => "Harley.M.Abbott@trashymail.com", :birthday => "1953/10/4 00:00:00"
 Person.create :gender => "male", :first_name => "Morgan", :middle_initial => "E", :last_name => "Iqbal", :street_address => "64 Carlisle Street", :city => "Dysart", :state => "VIC", :postcode => "3660", :email => "Morgan.E.Iqbal@spambob.com", :birthday => "1954/7/6 00:00:00"
 Person.create :gender => "female", :first_name => "Phoebe", :middle_initial => "T", :last_name => "Wells", :street_address => "10 Mnimbah Road", :city => "Eccleston", :state => "NSW", :postcode => "2311", :email => "Phoebe.T.Wells@trashymail.com", :birthday => "1949/5/27 00:00:00"
 Person.create :gender => "male", :first_name => "Finley", :middle_initial => "I", :last_name => "Martin", :street_address => "15 Thomas Lane", :city => "Epping", :state => "VIC", :postcode => "3076", :email => "Finley.I.Martin@dodgit.com", :birthday => "1983/3/12 00:00:00"

data/features/thinking_sphinx/models/andrew.rb

@@ -0,0 +1,17 @@
+require "#{File.dirname(__FILE__)}/person"
+
+class Andrew < ActiveRecord::Base
+  set_table_name 'people'
+  
+  define_index do
+    indexes first_name, last_name, street_address
+  end
+  
+  sphinx_scope(:locked_first_name) {
+    {:conditions => {:first_name => 'Andrew'}}
+  }
+  sphinx_scope(:locked_last_name) {
+    {:conditions => {:last_name => 'Byrne'}}
+  }
+  default_sphinx_scope :locked_first_name
+end

data/features/thinking_sphinx/models/person.rb

@@ -1,6 +1,7 @@
 class Person < ActiveRecord::Base
   define_index do
-    indexes first_name, last_name, :sortable => true
+    indexes first_name, :sortable => true
+    indexes last_name,  :sortable => :insensitive
     
     has [first_name, middle_initial, last_name], :as => :name_sort
     has birthday

data/lib/thinking_sphinx.rb

@@ -37,6 +37,8 @@ Merb::Plugins.add_rakefiles(
 ) if defined?(Merb)
 
 module ThinkingSphinx
+  mattr_accessor :database_adapter
+  
   # A ConnectionError will get thrown when a connection to Sphinx can't be
   # made.
   class ConnectionError < StandardError

data/lib/thinking_sphinx/active_record.rb

@@ -363,7 +363,7 @@ module ThinkingSphinx
     # @return [Integer] Unique record id for the purposes of Sphinx.
     # 
     def primary_key_for_sphinx
-      @primary_key_for_sphinx ||= read_attribute(self.class.primary_key_for_sphinx)
+      read_attribute(self.class.primary_key_for_sphinx)
     end
     
     def sphinx_document_id

data/lib/thinking_sphinx/active_record/scopes.rb

@@ -53,6 +53,13 @@ module ThinkingSphinx
               
               ThinkingSphinx::Search.new(options)
             end
+            
+            define_method("#{method}_without_default".to_sym) do |*args|
+              options = {:classes => classes_option, :ignore_default => true}
+              options.merge! block.call(*args)
+              
+              ThinkingSphinx::Search.new(options)
+            end
           end
         end
 

data/lib/thinking_sphinx/adapters/abstract_adapter.rb

@@ -10,24 +10,50 @@ module ThinkingSphinx
     end
       
     def self.detect(model)
+      adapter = adapter_for_model model
+      case adapter
+      when :mysql
+        ThinkingSphinx::MysqlAdapter.new model
+      when :postgresql
+        ThinkingSphinx::PostgreSQLAdapter.new model
+      else
+        raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL, not #{adapter}"
+      end
+    end
+    
+    def self.adapter_for_model(model)
+      case ThinkingSphinx.database_adapter
+      when String
+        ThinkingSphinx.database_adapter.to_sym
+      when NilClass
+        standard_adapter_for_model model
+      when Proc
+        ThinkingSphinx.database_adapter.call model
+      else
+        ThinkingSphinx.database_adapter
+      end
+    end
+    
+    def self.standard_adapter_for_model(model)
       case model.connection.class.name
       when "ActiveRecord::ConnectionAdapters::MysqlAdapter",
            "ActiveRecord::ConnectionAdapters::MysqlplusAdapter",
            "ActiveRecord::ConnectionAdapters::Mysql2Adapter",
            "ActiveRecord::ConnectionAdapters::NullDBAdapter"
-        ThinkingSphinx::MysqlAdapter.new model
+        :mysql
       when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-        ThinkingSphinx::PostgreSQLAdapter.new model
+        :postgresql
       when "ActiveRecord::ConnectionAdapters::JdbcAdapter"
-        if model.connection.config[:adapter] == "jdbcmysql"
-          ThinkingSphinx::MysqlAdapter.new model
-        elsif model.connection.config[:adapter] == "jdbcpostgresql"
-          ThinkingSphinx::PostgreSQLAdapter.new model
+        case model.connection.config[:adapter]
+        when "jdbcmysql"
+          :mysql
+        when "jdbcpostgresql"
+          :postgresql
         else
-          raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL"
+          model.connection.config[:adapter]
         end
       else
-        raise "Invalid Database Adapter: Sphinx only supports MySQL and PostgreSQL, not #{model.connection.class.name}"
+        model.connection.class.name
       end
     end
     
@@ -39,6 +65,10 @@ module ThinkingSphinx
       /bigint/i
     end
     
+    def downcase(clause)
+      "LOWER(#{clause})"
+    end
+    
     protected
     
     def connection

data/lib/thinking_sphinx/adapters/postgresql_adapter.rb

@@ -68,7 +68,7 @@ module ThinkingSphinx
     end
     
     def utc_query_pre
-      'SET TIME ZONE UTC'
+      "SET TIME ZONE 'UTC'"
     end
     
     private
@@ -119,6 +119,10 @@ module ThinkingSphinx
           DECLARE j int;
           DECLARE word_array bytea;
           BEGIN
+            IF COALESCE(word, '') = '' THEN
+              return 0;
+            END IF;
+          
             i = 0;
             tmp = 4294967295;
             word_array = decode(replace(word, E'\\\\', E'\\\\\\\\'), 'escape');
@@ -139,7 +143,7 @@ module ThinkingSphinx
             END LOOP;
             return (tmp # 4294967295);
           END
-        $$ IMMUTABLE STRICT LANGUAGE plpgsql;
+        $$ IMMUTABLE LANGUAGE plpgsql;
       SQL
       execute function, true
     end

data/lib/thinking_sphinx/attribute.rb

@@ -111,6 +111,7 @@ module ThinkingSphinx
       clause = adapter.crc(clause)                          if @crc
       clause = adapter.concatenate(clause, separator)       if concat_ws?
       clause = adapter.group_concatenate(clause, separator) if is_many?
+      clause = adapter.downcase(clause)                     if insensitive?
       
       "#{clause} AS #{quote_column(unique_name)}"
     end
@@ -376,5 +377,9 @@ block:
         value
       end
     end
+    
+    def insensitive?
+      @sortable == :insensitive
+    end
   end
 end

data/lib/thinking_sphinx/configuration.rb

@@ -54,13 +54,14 @@ module ThinkingSphinx
       sql_query_killlist sql_ranged_throttle sql_query_post_index unpack_zlib
       unpack_mysqlcompress unpack_mysqlcompress_maxsize )
     
-    IndexOptions  = %w( charset_table charset_type charset_dictpath docinfo
-      enable_star exceptions html_index_attrs html_remove_elements html_strip
-      index_exact_words ignore_chars inplace_docinfo_gap inplace_enable
-      inplace_hit_gap inplace_reloc_factor inplace_write_factor min_infix_len
-      min_prefix_len min_stemming_len min_word_len mlock morphology ngram_chars
-      ngram_len ondisk_dict overshort_step phrase_boundary phrase_boundary_step
-      preopen stopwords stopwords_step wordforms )
+    IndexOptions  = %w( blend_chars charset_table charset_type charset_dictpath
+      docinfo enable_star exceptions expand_keywords hitless_words
+      html_index_attrs html_remove_elements html_strip index_exact_words
+      ignore_chars inplace_docinfo_gap inplace_enable inplace_hit_gap
+      inplace_reloc_factor inplace_write_factor min_infix_len min_prefix_len
+      min_stemming_len min_word_len mlock morphology ngram_chars ngram_len
+      ondisk_dict overshort_step phrase_boundary phrase_boundary_step preopen
+      stopwords stopwords_step wordforms )
     
     CustomOptions = %w( disable_range )
         
@@ -233,9 +234,12 @@ module ThinkingSphinx
       @controller.indexer_binary_name = name
     end
     
+    attr_accessor :timeout
+
     def client
       client = Riddle::Client.new address, port
       client.max_matches = configuration.searchd.max_matches || 1000
+      client.timeout = timeout || 0
       client
     end
     

data/lib/thinking_sphinx/context.rb

@@ -65,8 +65,10 @@ class ThinkingSphinx::Context
           model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
         rescue NameError
           next
-        rescue StandardError
-          STDERR.puts "Warning: Error loading #{file}"
+        rescue StandardError => err
+          STDERR.puts "Warning: Error loading #{file}:"
+          STDERR.puts err.message
+          STDERR.puts err.backtrace.join("\n"), ''
         end
       end
     end

data/lib/thinking_sphinx/property.rb

@@ -13,6 +13,7 @@ module ThinkingSphinx
       @alias    = options[:as]
       @faceted  = options[:facet]
       @admin    = options[:admin]
+      @sortable = options[:sortable] || false
       
       @alias    = @alias.to_sym unless @alias.blank?
       

data/lib/thinking_sphinx/search.rb

@@ -86,6 +86,8 @@ module ThinkingSphinx
       @options  = args.extract_options!
       @args     = args
       
+      add_default_scope unless options[:ignore_default]
+      
       populate if @options[:populate]
     end
     
@@ -123,7 +125,7 @@ module ThinkingSphinx
         add_scope(method, *args, &block)
         return self
       elsif method == :search_count
-        merge_search one_class.search(*args)
+        merge_search one_class.search(*args), self.args, options
         return scoped_count
       elsif method.to_s[/^each_with_.*/].nil? && !@array.respond_to?(method)
         super
@@ -270,18 +272,43 @@ module ThinkingSphinx
       
       populate
       client.excerpts(
-        :docs   => [string],
-        :words  => results[:words].keys.join(' '),
-        :index  => options[:index] || "#{model.source_of_sphinx_index.sphinx_name}_core"
+        {
+          :docs   => [string.to_s],
+          :words  => results[:words].keys.join(' '),
+          :index  => options[:index] || "#{model.source_of_sphinx_index.sphinx_name}_core"
+        }.merge(options[:excerpt_options] || {})
       ).first
     end
     
     def search(*args)
-      add_default_scope
-      merge_search ThinkingSphinx::Search.new(*args)
+      args << args.extract_options!.merge(:ignore_default => true)
+      merge_search ThinkingSphinx::Search.new(*args), self.args, options
       self
     end
     
+    def search_for_ids(*args)
+      args << args.extract_options!.merge(
+        :ignore_default => true,
+        :ids_only       => true
+      )
+      merge_search ThinkingSphinx::Search.new(*args), self.args, options
+      self
+    end
+    
+    def facets(*args)
+      options = args.extract_options!
+      merge_search self, args, options
+      args << options
+      
+      ThinkingSphinx::FacetSearch.new *args
+    end
+    
+    def client
+      client = options[:client] || config.client
+      
+      prepare client
+    end
+    
     def append_to(client)
       prepare client
       client.append_query query, indexes, comment
@@ -382,9 +409,16 @@ module ThinkingSphinx
     
     def self.log(message, method = :debug, identifier = 'Sphinx')
       return if ::ActiveRecord::Base.logger.nil?
-      identifier_color, message_color = "4;32;1", "0" # 0;1 = Bold
-      info = "  \e[#{identifier_color}m#{identifier}\e[0m   "
-      info << "\e[#{message_color}m#{message}\e[0m"
+      
+      info = ''
+      if ::ActiveRecord::Base.colorize_logging
+        identifier_color, message_color = "4;32;1", "0" # 0;1 = Bold
+        info << "  \e[#{identifier_color}m#{identifier}\e[0m   "
+        info << "\e[#{message_color}m#{message}\e[0m"
+      else
+        info = "#{identifier}   #{message}"
+      end
+      
       ::ActiveRecord::Base.logger.send method, info
     end
     
@@ -392,12 +426,6 @@ module ThinkingSphinx
       self.class.log(*args)
     end
     
-    def client
-      client = config.client
-      
-      prepare client
-    end
-    
     def prepare(client)
       index_options = one_class ?
         one_class.sphinx_indexes.first.local_options : {}
@@ -489,7 +517,8 @@ module ThinkingSphinx
       query.gsub(/("#{token}(.*?#{token})?"|(?![!-])#{token})/u) do
         pre, proper, post = $`, $&, $'
         # E.g. "@foo", "/2", "~3", but not as part of a token
-        is_operator = pre.match(%r{(\W|^)[@~/]\Z})
+        is_operator = pre.match(%r{(\W|^)[@~/]\Z}) ||
+                      pre.match(%r{(\W|^)@\([^\)]*$})
         # E.g. "foo bar", with quotes
         is_quote    = proper.starts_with?('"') && proper.ends_with?('"')
         has_star    = pre.ends_with?("*") || post.starts_with?("*")
@@ -603,24 +632,8 @@ module ThinkingSphinx
       filters
     end
     
-    def condition_filters
-      (options[:conditions] || {}).collect { |attrib, value|
-        if attributes.include?(attrib.to_sym)
-          puts <<-MSG
-Deprecation Warning: filters on attributes should be done using the :with
-option, not :conditions. For example:
-  :with => {:#{attrib} => #{value.inspect}}
-MSG
-          Riddle::Client::Filter.new attrib.to_s, filter_value(value)
-        else
-          nil
-        end
-      }.compact
-    end
-    
     def filters
       internal_filters +
-      condition_filters +
       (options[:with] || {}).collect { |attrib, value|
         Riddle::Client::Filter.new attrib.to_s, filter_value(value)
       } +
@@ -708,6 +721,21 @@ MSG
       end
     end
     
+    def include_for_class(klass)
+      includes = options[:include] || klass.sphinx_index_options[:include]
+      
+      case includes
+      when NilClass
+        nil
+      when Array
+        includes.select { |inc| klass.reflections[inc] }
+      when Symbol
+        klass.reflections[includes].nil? ? nil : includes
+      else
+        includes
+      end
+    end
+    
     def instances_from_class(klass, matches)
       index_options = klass.sphinx_index_options
 
@@ -716,7 +744,7 @@ MSG
         :all,
         :joins      => options[:joins],
         :conditions => {klass.primary_key_for_sphinx.to_sym => ids},
-        :include    => (options[:include] || index_options[:include]),
+        :include    => include_for_class(klass),
         :select     => (options[:select]  || index_options[:select]),
         :order      => (options[:sql_order] || index_options[:sql_order])
       ) : []
@@ -786,14 +814,16 @@ MSG
     
     # Adds the default_sphinx_scope if set.
     def add_default_scope
-      add_scope(one_class.get_default_sphinx_scope) if one_class && one_class.has_default_sphinx_scope?
+      return unless one_class && one_class.has_default_sphinx_scope?
+      add_scope(one_class.get_default_sphinx_scope.to_sym)
     end
     
     def add_scope(method, *args, &block)
-      merge_search one_class.send(method, *args, &block)
+      method = "#{method}_without_default".to_sym
+      merge_search one_class.send(method, *args, &block), self.args, options
     end
     
-    def merge_search(search)
+    def merge_search(search, args, options)
       search.args.each { |arg| args << arg }
       
       search.options.keys.each do |key|