thinking-sphinx 3.0.0 → 3.0.1

data/lib/thinking_sphinx/active_record/sql_builder.rb

@@ -12,8 +12,9 @@ class ThinkingSphinx::ActiveRecord::SQLBuilder
     relation = relation.group group_clause
     relation = relation.order('NULL') if source.type == 'mysql'
     relation = relation.joins associations.join_values
+    relation = relation.joins custom_joins.collect(&:to_s) if custom_joins.any?
 
-    relation.to_sql
+    relation.to_sql.gsub(/\n/, "\\\n")
   end
 
   def sql_query_range
@@ -67,12 +68,16 @@ class ThinkingSphinx::ActiveRecord::SQLBuilder
 
   def associations
     @associations ||= ThinkingSphinx::ActiveRecord::Associations.new(model).tap do |assocs|
-      source.associations.each do |association|
+      source.associations.reject(&:string?).each do |association|
         assocs.add_join_to association.stack
       end
     end
   end
 
+  def custom_joins
+    @custom_joins ||= source.associations.select &:string?
+  end
+
   def quote_column(column)
     model.connection.quote_column_name(column)
   end

data/lib/thinking_sphinx/active_record/sql_source.rb

@@ -1,6 +1,7 @@
 class ThinkingSphinx::ActiveRecord::SQLSource < Riddle::Configuration::SQLSource
   attr_reader :model, :database_settings, :options
-  attr_accessor :fields, :attributes, :associations, :conditions, :groupings
+  attr_accessor :fields, :attributes, :associations, :conditions, :groupings,
+    :polymorphs
 
   OPTIONS = [:name, :offset, :delta_processor, :delta?, :disable_range?,
     :group_concat_max_len, :utf8?, :position]
@@ -15,6 +16,7 @@ class ThinkingSphinx::ActiveRecord::SQLSource < Riddle::Configuration::SQLSource
     @associations      = []
     @conditions        = []
     @groupings         = []
+    @polymorphs        = []
 
     Template.new(self).apply
 
@@ -89,6 +91,8 @@ class ThinkingSphinx::ActiveRecord::SQLSource < Riddle::Configuration::SQLSource
   end
 
   def prepare_for_render
+    polymorphs.each &:morph!
+
     set_database_settings
 
     fields.each do |field|

data/lib/thinking_sphinx/capistrano.rb

@@ -0,0 +1,64 @@
+Capistrano::Configuration.instance(:must_exist).load do
+  namespace :thinking_sphinx do
+    desc 'Generate the Sphinx configuration file.'
+    task :configure do
+      rake 'thinking_sphinx:configure'
+    end
+
+    desc 'Build Sphinx indexes into the shared path and symlink them into your release.'
+    task :index do
+      rake 'thinking_sphinx:index'
+    end
+    after 'thinking_sphinx:index', 'thinking_sphinx:symlink_indexes'
+
+    desc 'Start the Sphinx search daemon.'
+    task :start do
+      rake 'thinking_sphinx:start'
+    end
+    before 'thinking_sphinx:start', 'thinking_sphinx:configure'
+
+    desc 'Stop the Sphinx search daemon.'
+    task :stop do
+      rake 'thinking_sphinx:stop'
+    end
+
+    desc 'Restart the Sphinx search daemon.'
+    task :restart do
+      rake 'thinking_sphinx:stop thinking_sphinx:configure thinking_sphinx:start'
+    end
+
+    desc <<-DESC
+Stop, reindex, and then start the Sphinx search daemon. This task must be executed \
+if you alter the structure of your indexes.
+    DESC
+    task :rebuild do
+      rake 'thinking_sphinx:stop thinking_sphinx:reindex'
+    end
+    after 'thinking_sphinx:rebuild', 'thinking_sphinx:symlink_indexes'
+    after 'thinking_sphinx:rebuild', 'thinking_sphinx:start'
+
+    desc 'Create the shared folder for sphinx indexes.'
+    task :shared_sphinx_folder do
+      rails_env = fetch(:rails_env, 'production')
+      run "mkdir -p #{shared_path}/db/sphinx/#{rails_env}"
+    end
+
+    desc 'Symlink Sphinx indexes from the shared folder to the latest release.'
+    task :symlink_indexes do
+      run "if [ -d #{release_path} ]; then ln -nfs #{shared_path}/db/sphinx #{release_path}/db/sphinx; else ln -nfs #{shared_path}/db/sphinx #{current_path}/db/sphinx; fi;"
+    end
+
+    # Logical flow for deploying an app
+    after  'deploy:cold',            'thinking_sphinx:index'
+    after  'deploy:cold',            'thinking_sphinx:start'
+    after  'deploy:setup',           'thinking_sphinx:shared_sphinx_folder'
+    after  'deploy:finalize_update', 'thinking_sphinx:symlink_indexes'
+
+    def rake(tasks)
+      rails_env = fetch(:rails_env, 'production')
+      rake = fetch(:rake, 'rake')
+
+      run "if [ -d #{release_path} ]; then cd #{release_path}; else cd #{current_path}; fi; if [ -f Rakefile ]; then #{rake} RAILS_ENV=#{rails_env} #{tasks}; fi;"
+    end
+  end
+end

data/lib/thinking_sphinx/configuration.rb

@@ -1,5 +1,5 @@
 class ThinkingSphinx::Configuration < Riddle::Configuration
-  attr_accessor :configuration_file, :indices_location
+  attr_accessor :configuration_file, :indices_location, :version
   attr_reader :index_paths
   attr_writer :controller, :framework
 
@@ -11,6 +11,7 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
     @index_paths        = [File.join(framework.root, 'app', 'indices')]
     @indices_location   = File.join framework.root, 'db', 'sphinx',
       framework.environment
+    @version            = settings['version'] || '2.0.6'
 
     searchd.pid_file    = File.join framework.root, 'log',
       "#{framework.environment}.sphinx.pid"
@@ -27,6 +28,14 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
       Defaults::PORT
     searchd.workers   = 'threads'
 
+   [indexer, searchd].each do |object|
+      settings.each do |key, value|
+        next unless object.class.settings.include?(key.to_sym)
+
+        object.send("#{key}=", value)
+      end
+    end
+
     @offsets = {}
   end
 

data/lib/thinking_sphinx/connection.rb

@@ -28,6 +28,26 @@ module ThinkingSphinx::Connection
     )
   end
 
+  def self.take
+    retries  = 0
+    original = nil
+    begin
+      pool.take do |connection|
+        begin
+          yield connection
+        rescue Mysql2::Error => error
+          original = ThinkingSphinx::SphinxError.new_from_mysql error
+          raise original if original.is_a?(ThinkingSphinx::QueryError)
+          raise Innertube::Pool::BadResource
+        end
+      end
+    rescue Innertube::Pool::BadResource
+      retries += 1
+      retry if retries < 3
+      raise original
+    end
+  end
+
   class MRI
     attr_reader :client
 

data/lib/thinking_sphinx/errors.rb

@@ -0,0 +1,24 @@
+class ThinkingSphinx::SphinxError < StandardError
+  def self.new_from_mysql(error)
+    case error.message
+    when /parse error/
+      replacement = ThinkingSphinx::ParseError.new(error.message)
+    when /syntax error/
+      replacement = ThinkingSphinx::SyntaxError.new(error.message)
+    else
+      replacement = new(error.message)
+    end
+
+    replacement.set_backtrace error.backtrace
+    replacement
+  end
+end
+
+class ThinkingSphinx::QueryError < ThinkingSphinx::SphinxError
+end
+
+class ThinkingSphinx::SyntaxError < ThinkingSphinx::QueryError
+end
+
+class ThinkingSphinx::ParseError < ThinkingSphinx::QueryError
+end

data/lib/thinking_sphinx/middlewares/sphinxql.rb

@@ -40,7 +40,10 @@ class ThinkingSphinx::Middlewares::SphinxQL <
     end
 
     def class_condition
-      '(' + classes_and_descendants.collect(&:name).join('|') + ')'
+      class_names = classes_and_descendants.collect(&:name).collect { |name|
+        name[/:/] ? "\"#{name}\"" : name
+      }
+      '(' + class_names.join('|') + ')'
     end
 
     def descendants

data/lib/thinking_sphinx/real_time/transcriber.rb

@@ -13,7 +13,7 @@ class ThinkingSphinx::RealTime::Transcriber
     end
 
     sphinxql = Riddle::Query::Insert.new index.name, columns, values
-    ThinkingSphinx::Connection.pool.take do |connection|
+    ThinkingSphinx::Connection.take do |connection|
       connection.execute sphinxql.replace!.to_sql
     end
   end

data/lib/thinking_sphinx/search/batch_inquirer.rb

@@ -13,7 +13,9 @@ class ThinkingSphinx::Search::BatchInquirer
     @results ||= begin
       @queries.freeze
 
-      ThinkingSphinx::Connection.new.query_all *@queries
+      ThinkingSphinx::Connection.take do |connection|
+        connection.query_all *@queries
+      end
     end
   end
 end

data/lib/thinking_sphinx/search/glaze.rb

@@ -22,11 +22,11 @@ class ThinkingSphinx::Search::Glaze < BasicObject
   private
 
   def method_missing(method, *args, &block)
-    if @object.respond_to?(method)
+    pane = @panes.detect { |pane| pane.respond_to?(method) }
+    if @object.respond_to?(method) || pane.nil?
       @object.send(method, *args, &block)
     else
-      pane = @panes.detect { |pane| pane.respond_to?(method) }
-      pane.nil? ? super : pane.send(method, *args, &block)
+      pane.send(method, *args, &block)
     end
   end
 end

data/spec/acceptance/index_options_spec.rb

@@ -92,6 +92,7 @@ describe 'Index options' do
 
         set_property :sql_range_step => 5
         set_property :disable_range? => true
+        set_property :sql_query_pre => ["DO STUFF"]
       }
       index.render
     end
@@ -103,5 +104,9 @@ describe 'Index options' do
     it "allows for source options" do
       index.sources.first.disable_range?.should be_true
     end
+
+    it "respects sql_query_pre values" do
+      index.sources.first.sql_query_pre.should == ["DO STUFF"]
+    end
   end
 end

data/spec/acceptance/searching_on_fields_spec.rb

@@ -50,6 +50,7 @@ describe 'Searching on fields', :live => true do
     File.open(file_path, 'w') { |file| file.print 'Cyberpunk at its best' }
 
     book = Book.create! :title => 'Accelerando', :blurb_file => file_path.to_s
+    index
 
     Book.search('cyberpunk').to_a.should == [book]
   end

data/spec/acceptance/specifying_sql_spec.rb

@@ -36,6 +36,18 @@ describe 'specifying SQL for index definitions' do
     query.should match(/LEFT OUTER JOIN .tags./)
   end
 
+  it "handles custom join SQL statements" do
+    index = ThinkingSphinx::ActiveRecord::Index.new(:article)
+    index.definition_block = Proc.new {
+      indexes title
+      join "INNER JOIN foo ON foo.x = bar.y"
+    }
+    index.render
+
+    query = index.sources.first.sql_query
+    query.should match(/INNER JOIN foo ON foo.x = bar.y/)
+  end
+
   it "handles GROUP BY clauses" do
     index = ThinkingSphinx::ActiveRecord::Index.new(:article)
     index.definition_block = Proc.new {
@@ -83,6 +95,66 @@ describe 'specifying SQL for index definitions' do
     query = index.sources.first.sql_query
     query.should match(/WHERE .+title != 'secret'.+ GROUP BY/)
   end
+
+  it "escapes new lines in SQL snippets" do
+    index = ThinkingSphinx::ActiveRecord::Index.new(:article)
+    index.definition_block = Proc.new {
+      indexes title
+      has <<-SQL, as: :custom_attribute,  type: :integer
+      ARRAY_AGG(
+        CONCAT(
+          something
+        )
+      )
+      SQL
+    }
+    index.render
+
+    query = index.sources.first.sql_query
+    query.should match(/\\\n/)
+  end
+
+  it "joins each polymorphic relation" do
+    index = ThinkingSphinx::ActiveRecord::Index.new(:event)
+    index.definition_block = Proc.new {
+      indexes eventable.title, :as => :title
+      polymorphs eventable, :to => %w(Article Book)
+    }
+    index.render
+
+    query = index.sources.first.sql_query
+    query.should match(/LEFT OUTER JOIN .articles. ON .articles.\..id. = .events.\..eventable_id. AND .events.\..eventable_type. = 'Article'/)
+    query.should match(/LEFT OUTER JOIN .books. ON .books.\..id. = .events.\..eventable_id. AND .events.\..eventable_type. = 'Book'/)
+    query.should match(/articles\..title., books\..title./)
+  end
+
+  it "concatenates references where that have column" do
+    index = ThinkingSphinx::ActiveRecord::Index.new(:event)
+    index.definition_block = Proc.new {
+      indexes eventable.title, :as => :title
+      polymorphs eventable, :to => %w(Article User)
+    }
+    index.render
+
+    query = index.sources.first.sql_query
+    query.should match(/LEFT OUTER JOIN .articles. ON .articles.\..id. = .events.\..eventable_id. AND .events.\..eventable_type. = 'Article'/)
+    query.should match(/LEFT OUTER JOIN .users. ON .users.\..id. = .events.\..eventable_id. AND .events.\..eventable_type. = 'User'/)
+    query.should_not match(/articles\..title., users\..title./)
+  end
+
+  it "respects deeper associations through polymorphic joins" do
+    index = ThinkingSphinx::ActiveRecord::Index.new(:event)
+    index.definition_block = Proc.new {
+      indexes eventable.user.name, :as => :user_name
+      polymorphs eventable, :to => %w(Article Book)
+    }
+    index.render
+
+    query = index.sources.first.sql_query
+    query.should match(/LEFT OUTER JOIN .articles. ON .articles.\..id. = .events.\..eventable_id. AND .events.\..eventable_type. = 'Article'/)
+    query.should match(/LEFT OUTER JOIN .users. ON .users.\..id. = .articles.\..user_id./)
+    query.should match(/users\..name./)
+  end
 end
 
 describe 'separate queries for MVAs' do
@@ -247,6 +319,25 @@ describe 'separate queries for MVAs' do
     query.should == 'My Custom SQL Query'
     range.should == 'And a Range'
   end
+
+  it "escapes new lines in custom SQL snippets" do
+    index.definition_block = Proc.new {
+      indexes title
+      has <<-SQL, :as => :tag_ids, :source => :query, :type => :integer, :multi => true
+My Custom
+SQL Query
+      SQL
+    }
+    index.render
+
+    attribute = source.sql_attr_multi.detect { |attribute|
+      attribute[/tag_ids/]
+    }
+    declaration, query = attribute.split(/;\s+/)
+
+    declaration.should == 'uint tag_ids from query'
+    query.should == "My Custom\\\nSQL Query"
+  end
 end
 
 describe 'separate queries for field' do
@@ -336,4 +427,20 @@ describe 'separate queries for field' do
     query.should == 'My Custom SQL Query'
     range.should == 'And a Range'
   end
+
+  it "escapes new lines in custom SQL snippets" do
+    index.definition_block = Proc.new {
+      indexes <<-SQL, :as => :tags, :source => :query
+My Custom
+SQL Query
+      SQL
+    }
+    index.render
+
+    field = source.sql_joined_field.detect { |field| field[/tags/] }
+    declaration, query = field.split(/;\s+/)
+
+    declaration.should == 'tags from query'
+    query.should == "My Custom\\\nSQL Query"
+  end
 end

data/spec/acceptance/sql_deltas_spec.rb

@@ -40,4 +40,13 @@ describe 'SQL delta indexing', :live => true do
 
     Book.search('Harry').should be_empty
   end
+
+  it "automatically indexes new records of subclasses" do
+    book = Hardcover.create(
+      :title => 'American Gods', :author => 'Neil Gaiman'
+    )
+    sleep 0.25
+
+    Book.search('Gaiman').to_a.should == [book]
+  end
 end

data/spec/acceptance/support/sphinx_controller.rb

@@ -18,6 +18,7 @@ class SphinxController
     config.searchd.mysql41 = 9307
     config.settings['quiet_deltas']      = true
     config.settings['attribute_updates'] = true
+    config.controller.bin_path           = ENV['SPHINX_BIN'] || ''
   end
 
   def start

data/spec/internal/app/models/event.rb

@@ -0,0 +1,3 @@
+class Event < ActiveRecord::Base
+  belongs_to :eventable, :polymorphic => true
+end

data/spec/internal/app/models/hardcover.rb

@@ -0,0 +1,3 @@
+class Hardcover < Book
+  #
+end

data/spec/internal/db/schema.rb

@@ -22,7 +22,8 @@ ActiveRecord::Schema.define do
     t.string  :author
     t.integer :year
     t.string  :blurb_file
-    t.boolean :delta, :default => true, :null => false
+    t.boolean :delta, :default => true,   :null => false
+    t.string  :type,  :default => 'Book', :null => false
     t.timestamps
   end
 
@@ -42,6 +43,11 @@ ActiveRecord::Schema.define do
     t.timestamps
   end
 
+  create_table(:events, :force => true) do |t|
+    t.string  :eventable_type
+    t.integer :eventable_id
+  end
+
   create_table(:genres, :force => true) do |t|
     t.string :name
   end