thinking-sphinx 3.3.0 → 3.4.0

data/lib/thinking_sphinx/middlewares/valid_options.rb

@@ -0,0 +1,23 @@
+class ThinkingSphinx::Middlewares::ValidOptions <
+  ThinkingSphinx::Middlewares::Middleware
+
+  def call(contexts)
+    contexts.each { |context| check_options context.search.options }
+
+    app.call contexts
+  end
+
+  private
+
+  def check_options(options)
+    unknown = invalid_keys options.keys
+    return if unknown.empty?
+
+    ThinkingSphinx::Logger.log :warn,
+      "Unexpected search options: #{unknown.inspect}"
+  end
+
+  def invalid_keys(keys)
+    keys - ThinkingSphinx::Search.valid_options
+  end
+end

data/lib/thinking_sphinx/rake_interface.rb

@@ -1,146 +1,32 @@
 class ThinkingSphinx::RakeInterface
+  DEFAULT_OPTIONS = {:verbose => true}
+
   def initialize(options = {})
-    @options           = options
+    @options           = DEFAULT_OPTIONS.merge options
     @options[:verbose] = false if @options[:silent]
   end
 
-  def clear_all
-    [
-      configuration.indices_location,
-      configuration.searchd.binlog_path
-    ].each do |path|
-      FileUtils.rm_r(path) if File.exists?(path)
-    end
-  end
-
-  def clear_real_time
-    configuration.preload_indices
-    indices = configuration.indices.select { |index| index.type == 'rt' }
-    indices.each do |index|
-      index.render
-      Dir["#{index.path}.*"].each { |path| FileUtils.rm path }
-    end
-
-    path = configuration.searchd.binlog_path
-    FileUtils.rm_r(path) if File.exists?(path)
-  end
-
   def configure
-    log "Generating configuration to #{configuration.configuration_file}"
-    configuration.render_to_file
-  end
-
-  def generate
-    indices = configuration.indices.select { |index| index.type == 'rt' }
-    indices.each do |index|
-      ThinkingSphinx::RealTime::Populator.populate index
-    end
+    ThinkingSphinx::Commands::Configure.call configuration, options
   end
 
-  def index(reconfigure = true, verbose = true)
-    configure if reconfigure
-    FileUtils.mkdir_p configuration.indices_location
-    ThinkingSphinx.before_index_hooks.each { |hook| hook.call }
-    controller.index :verbose => verbose
-  rescue Riddle::CommandFailedError => error
-    handle_command_failure 'indexing', error.command_result
+  def daemon
+    @daemon ||= ThinkingSphinx::Interfaces::Daemon.new configuration, options
   end
 
-  def prepare
-    configuration.preload_indices
-    configuration.render
-
-    FileUtils.mkdir_p configuration.indices_location
+  def rt
+    @rt ||= ThinkingSphinx::Interfaces::RealTime.new configuration, options
   end
 
-  def start
-    if running?
-      raise ThinkingSphinx::SphinxAlreadyRunning, 'searchd is already running'
-    end
-
-    FileUtils.mkdir_p configuration.indices_location
-
-    options[:nodetach] ? start_attached : start_detached
-  end
-
-  def status
-    if running?
-      puts "The Sphinx daemon searchd is currently running."
-    else
-      puts "The Sphinx daemon searchd is not currently running."
-    end
-  end
-
-  def stop
-    unless running?
-      log 'searchd is not currently running.' and return
-    end
-
-    pid = controller.pid
-    until !running? do
-      controller.stop options
-      sleep(0.5)
-    end
-
-    log "Stopped searchd daemon (pid: #{pid})."
-  rescue Riddle::CommandFailedError => error
-    handle_command_failure 'stop', error.command_result
+  def sql
+    @sql ||= ThinkingSphinx::Interfaces::SQL.new configuration, options
   end
 
   private
 
   attr_reader :options
 
-  delegate :controller, :to => :configuration
-  delegate :running?,   :to => :controller
-
-  def command_output(output)
-    return "See above\n" if output.nil?
-
-    "\n\t" + output.gsub("\n", "\n\t")
-  end
-
   def configuration
     ThinkingSphinx::Configuration.instance
   end
-
-  def handle_command_failure(type, result)
-    puts <<-TXT
-
-The Sphinx #{type} command failed:
-  Command: #{result.command}
-  Status:  #{result.status}
-  Output:  #{command_output result.output}
-There may be more information about the failure in #{configuration.searchd.log}.
-    TXT
-    exit result.status
-  end
-
-  def log(message)
-    return if options[:silent]
-
-    puts message
-  end
-
-  def start_attached
-    unless pid = fork
-      controller.start :verbose => options[:verbose]
-    end
-
-    Signal.trap('TERM') { Process.kill(:TERM, pid); }
-    Signal.trap('INT')  { Process.kill(:TERM, pid); }
-    Process.wait(pid)
-  end
-
-  def start_detached
-    result = controller.start :verbose => options[:verbose]
-
-    if running?
-      log "Started searchd successfully (pid: #{controller.pid})."
-    else
-      handle_command_failure 'start', result
-    end
-  rescue Riddle::CommandFailedError => error
-    handle_command_failure 'start', error.command_result
-  end
 end

data/lib/thinking_sphinx/search.rb

@@ -21,6 +21,17 @@ class ThinkingSphinx::Search < Array
   attr_reader   :options
   attr_accessor :query
 
+  def self.valid_options
+    @valid_options
+  end
+
+  @valid_options = [
+    :classes, :conditions, :geo, :group_by, :ids_only, :ignore_scopes, :indices,
+    :limit, :masks, :max_matches, :middleware, :offset, :order, :order_group_by,
+    :page, :per_page, :populate, :retry_stale, :select, :skip_sti, :sql, :star,
+    :with, :with_all, :without, :without_ids
+  ]
+
   def initialize(query = nil, options = {})
     query, options   = nil, query if query.is_a?(Hash)
     @query, @options = query, options

data/lib/thinking_sphinx/search/query.rb

@@ -10,12 +10,18 @@ class ThinkingSphinx::Search::Query
     (star_keyword(keywords || '') + ' ' + conditions.keys.collect { |key|
        next if conditions[key].blank?
 
-      "@#{key} #{star_keyword conditions[key], key}"
+      "#{expand_key key} #{star_keyword conditions[key], key}"
     }.join(' ')).strip
   end
 
   private
 
+  def expand_key(key)
+    return "@#{key}" unless key.is_a?(Array)
+
+    "@(#{key.join(',')})"
+  end
+
   def star_keyword(keyword, key = nil)
     return keyword.to_s unless star
     return keyword.to_s if key.to_s == 'sphinx_internal_class_name'

data/lib/thinking_sphinx/tasks.rb

@@ -5,55 +5,114 @@ namespace :ts do
   end
 
   desc 'Generate the Sphinx configuration file and process all indices'
-  task :index => :environment do
-    interface.index(ENV['INDEX_ONLY'] != 'true')
-  end
+  task :index => ['ts:sql:index', 'ts:rt:index']
 
   desc 'Clear out Sphinx files'
-  task :clear => :environment do
-    interface.clear_all
-  end
+  task :clear => ['ts:sql:clear', 'ts:rt:clear']
 
-  desc 'Clear out real-time index files'
+  desc 'DEPRECATED: Clear out real-time index files'
   task :clear_rt => :environment do
-    interface.clear_real_time
+    puts <<-TXT
+The ts:clear_rt task is now deprecated due to the unified task approach, and
+invokes ts:rt:clear.
+* To delete all indices (both SQL-backed and real-time), use ts:clear.
+* To delete just real-time indices, use ts:rt:clear.
+* To delete just SQL-backed indices, use ts:sql:clear.
+
+    TXT
+
+    Rake::Task['ts:rt:clear'].invoke
   end
 
-  desc 'Generate fresh index files for real-time indices'
+  desc 'DEPRECATED: Generate fresh index files for all indices'
   task :generate => :environment do
-    interface.prepare
-    interface.generate
+    puts <<-TXT
+The ts:generate task is now deprecated due to the unified task approach, and
+invokes ts:index.
+* To process all indices (both SQL-backed and real-time), use ts:index.
+* To process just real-time indices, use ts:rt:index.
+* To process just SQL-backed indices, use ts:sql:index.
+
+    TXT
+
+    Rake::Task['ts:index'].invoke
   end
 
-  desc 'Stop Sphinx, index and then restart Sphinx'
-  task :rebuild => [:stop, :clear, :index, :start]
+  desc 'Delete and regenerate Sphinx files, restart the daemon'
+  task :rebuild => [
+    :stop, :clear, :configure, 'ts:sql:index', :start, 'ts:rt:index'
+  ]
+
+  desc 'DEPRECATED: Delete and regenerate Sphinx files, restart the daemon'
+  task :regenerate do
+        puts <<-TXT
+The ts:regenerate task is now deprecated due to the unified task approach, and
+invokes ts:rebuild.
+* To rebuild all indices (both SQL-backed and real-time), use ts:rebuild.
+* To rebuild just real-time indices, use ts:rt:rebuild.
+* To rebuild just SQL-backed indices, use ts:sql:rebuild.
 
-  desc 'Stop Sphinx, clear files, reconfigure, start Sphinx, generate files'
-  task :regenerate => [:stop, :clear_rt, :configure, :start, :generate]
+    TXT
+
+    Rake::Task['ts:rebuild'].invoke
+  end
 
   desc 'Restart the Sphinx daemon'
   task :restart => [:stop, :start]
 
   desc 'Start the Sphinx daemon'
   task :start => :environment do
-    interface.start
+    interface.daemon.start
   end
 
   desc 'Stop the Sphinx daemon'
   task :stop => :environment do
-    interface.stop
+    interface.daemon.stop
   end
 
   desc 'Determine whether Sphinx is running'
   task :status => :environment do
-    interface.status
+    interface.daemon.status
+  end
+
+  namespace :sql do
+    desc 'Delete SQL-backed Sphinx files'
+    task :clear => :environment do
+      interface.sql.clear
+    end
+
+    desc 'Generate fresh index files for SQL-backed indices'
+    task :index => :environment do
+      interface.sql.index(ENV['INDEX_ONLY'] != 'true')
+    end
+
+    desc 'Delete and regenerate SQL-backed Sphinx files, restart the daemon'
+    task :rebuild => ['ts:stop', 'ts:sql:clear', 'ts:sql:index', 'ts:start']
+  end
+
+  namespace :rt do
+    desc 'Delete real-time Sphinx files'
+    task :clear => :environment do
+      interface.rt.clear
+    end
+
+    desc 'Generate fresh index files for real-time indices'
+    task :index => :environment do
+      interface.rt.index
+    end
+
+    desc 'Delete and regenerate real-time Sphinx files, restart the daemon'
+    task :rebuild => [
+      'ts:stop', 'ts:rt:clear', 'ts:configure', 'ts:start', 'ts:rt:index'
+    ]
   end
 
   def interface
     @interface ||= ThinkingSphinx::RakeInterface.new(
-      :verbose  => Rake::FileUtilsExt.verbose_flag,
-      :silent   => Rake.application.options.silent,
-      :nodetach => (ENV['NODETACH'] == 'true')
+      :verbose      => Rake::FileUtilsExt.verbose_flag,
+      :silent       => Rake.application.options.silent,
+      :nodetach     => (ENV['NODETACH'] == 'true'),
+      :index_filter => ENV['INDEX_FILTER']
     )
   end
 end

data/lib/thinking_sphinx/with_output.rb

@@ -0,0 +1,11 @@
+module ThinkingSphinx::WithOutput
+  def initialize(configuration, options = {}, stream = STDOUT)
+    @configuration = configuration
+    @options       = options
+    @stream        = stream
+  end
+
+  private
+
+  attr_reader :configuration, :options, :stream
+end

data/spec/acceptance/connection_spec.rb

@@ -1,8 +1,8 @@
 require 'acceptance/spec_helper'
 
 RSpec.describe 'Connections', :live => true do
-  let(:maximum)    { (2 ** 23) - 1 }
-  let(:query)      { "SELECT * FROM book_core WHERE MATCH('')" }
+  let(:maximum)    { (2 ** 23) - 5 }
+  let(:query)      { String.new "SELECT * FROM book_core WHERE MATCH('')" }
   let(:difference) { maximum - query.length }
 
   it 'allows normal length queries through' do
@@ -10,13 +10,13 @@ RSpec.describe 'Connections', :live => true do
       ThinkingSphinx::Connection.take do |connection|
         connection.execute query.insert(-3, 'a' * difference)
       end
-    }.to_not raise_error#(ThinkingSphinx::QueryLengthError)
+    }.to_not raise_error
   end
 
   it 'does not allow overly long queries' do
     expect {
       ThinkingSphinx::Connection.take do |connection|
-        connection.execute query.insert(-3, 'a' * (difference + 1))
+        connection.execute query.insert(-3, 'a' * (difference + 5))
       end
     }.to raise_error(ThinkingSphinx::QueryLengthError)
   end

data/spec/acceptance/searching_within_a_model_spec.rb

@@ -79,6 +79,13 @@ describe 'Searching within a model', :live => true do
       Category.search.to_a
     }.to raise_error(ThinkingSphinx::NoIndicesError)
   end
+
+  it "handles models with alternative id columns" do
+    album = Album.create! :name => 'The Seldom Seen Kid', :artist => 'Elbow'
+    index
+
+    expect(Album.search.first).to eq(album)
+  end
 end
 
 describe 'Searching within a model with a realtime index', :live => true do

data/spec/acceptance/specifying_sql_spec.rb

@@ -155,9 +155,27 @@ describe 'specifying SQL for index definitions' do
     expect(query).to match(/LEFT OUTER JOIN .users. ON .users.\..id. = .articles.\..user_id./)
     expect(query).to match(/.users.\..name./)
   end
+
+  it "allows for STI mixed with polymorphic joins" do
+    index = ThinkingSphinx::ActiveRecord::Index.new(:event)
+    index.definition_block = Proc.new {
+      indexes eventable.name, :as => :name
+      polymorphs eventable, :to => %w(Bird Car)
+    }
+    index.render
+
+    query = index.sources.first.sql_query
+    expect(query).to match(/LEFT OUTER JOIN .animals. ON .animals.\..id. = .events.\..eventable_id. .* AND .events.\..eventable_type. = 'Animal'/)
+    expect(query).to match(/LEFT OUTER JOIN .cars. ON .cars.\..id. = .events.\..eventable_id. AND .events.\..eventable_type. = 'Car'/)
+    expect(query).to match(/.animals.\..name., .cars.\..name./)
+  end
 end if ActiveRecord::VERSION::MAJOR > 3
 
 describe 'separate queries for MVAs' do
+  def id_type
+    ActiveRecord::VERSION::STRING.to_f > 5.0 ? 'bigint' : 'uint'
+  end
+
   let(:index)  { ThinkingSphinx::ActiveRecord::Index.new(:article) }
   let(:count)  { ThinkingSphinx::Configuration.instance.indices.count }
   let(:source) { index.sources.first }
@@ -174,7 +192,7 @@ describe 'separate queries for MVAs' do
     }
     declaration, query = attribute.split(/;\s+/)
 
-    expect(declaration).to eq('uint tag_ids from query')
+    expect(declaration).to eq("uint tag_ids from query")
     expect(query).to match(/^SELECT .taggings.\..article_id. \* #{count} \+ #{source.offset} AS .id., .taggings.\..tag_id. AS .tag_ids. FROM .taggings.\s? WHERE \(.taggings.\..article_id. IS NOT NULL\)$/)
   end
 
@@ -190,7 +208,7 @@ describe 'separate queries for MVAs' do
     }
     declaration, query = attribute.split(/;\s+/)
 
-    expect(declaration).to eq('uint tag_ids from query')
+    expect(declaration).to eq("#{id_type} tag_ids from query")
     expect(query).to match(/^SELECT .taggings.\..article_id. \* #{count} \+ #{source.offset} AS .id., .tags.\..id. AS .tag_ids. FROM .taggings. INNER JOIN .tags. ON .tags.\..id. = .taggings.\..tag_id. WHERE \(.taggings.\..article_id. IS NOT NULL\)\s?$/)
   end
 
@@ -206,7 +224,7 @@ describe 'separate queries for MVAs' do
     }
     declaration, query = attribute.split(/;\s+/)
 
-    expect(declaration).to eq('uint tag_ids from query')
+    expect(declaration).to eq("#{id_type} tag_ids from query")
     expect(query).to match(/^SELECT .taggings.\..article_id. \* #{count} \+ #{source.offset} AS .id., .tags.\..id. AS .tag_ids. FROM .taggings. INNER JOIN .tags. ON .tags.\..id. = .taggings.\..tag_id. WHERE \(.taggings.\..article_id. IS NOT NULL\)\s?$/)
   end
 
@@ -224,7 +242,7 @@ describe 'separate queries for MVAs' do
     }
     declaration, query = attribute.split(/;\s+/)
 
-    expect(declaration).to eq('uint tag_ids from query')
+    expect(declaration).to eq("#{id_type} tag_ids from query")
     expect(query).to match(/^SELECT .articles.\..user_id. \* #{count} \+ #{source.offset} AS .id., .tags.\..id. AS .tag_ids. FROM .articles. INNER JOIN .taggings. ON .taggings.\..article_id. = .articles.\..id. INNER JOIN .tags. ON .tags.\..id. = .taggings.\..tag_id. WHERE \(.articles.\..user_id. IS NOT NULL\)\s?$/)
   end
 
@@ -242,7 +260,7 @@ describe 'separate queries for MVAs' do
     }
     declaration, query = attribute.split(/;\s+/)
 
-    expect(declaration).to eq('uint genre_ids from query')
+    expect(declaration).to eq("#{id_type} genre_ids from query")
     expect(query).to match(/^SELECT .books_genres.\..book_id. \* #{count} \+ #{source.offset} AS .id., .books_genres.\..genre_id. AS .genre_ids. FROM .books_genres.\s?$/)
   end if ActiveRecord::VERSION::MAJOR > 3
 
@@ -258,7 +276,7 @@ describe 'separate queries for MVAs' do
     }
     declaration, query, range = attribute.split(/;\s+/)
 
-    expect(declaration).to eq('uint tag_ids from ranged-query')
+    expect(declaration).to eq("uint tag_ids from ranged-query")
     expect(query).to match(/^SELECT .taggings.\..article_id. \* #{count} \+ #{source.offset} AS .id., .taggings.\..tag_id. AS .tag_ids. FROM .taggings. \s?WHERE \(.taggings.\..article_id. BETWEEN \$start AND \$end\) AND \(.taggings.\..article_id. IS NOT NULL\)$/)
     expect(range).to match(/^SELECT MIN\(.taggings.\..article_id.\), MAX\(.taggings.\..article_id.\) FROM .taggings.\s?$/)
   end
@@ -275,7 +293,7 @@ describe 'separate queries for MVAs' do
     }
     declaration, query, range = attribute.split(/;\s+/)
 
-    expect(declaration).to eq('uint tag_ids from ranged-query')
+    expect(declaration).to eq("#{id_type} tag_ids from ranged-query")
     expect(query).to match(/^SELECT .taggings.\..article_id. \* #{count} \+ #{source.offset} AS .id., .tags.\..id. AS .tag_ids. FROM .taggings. INNER JOIN .tags. ON .tags.\..id. = .taggings.\..tag_id. \s?WHERE \(.taggings.\..article_id. BETWEEN \$start AND \$end\) AND \(.taggings.\..article_id. IS NOT NULL\)$/)
     expect(range).to match(/^SELECT MIN\(.taggings.\..article_id.\), MAX\(.taggings.\..article_id.\) FROM .taggings.\s?$/)
   end
@@ -294,7 +312,7 @@ describe 'separate queries for MVAs' do
     }
     declaration, query, range = attribute.split(/;\s+/)
 
-    expect(declaration).to eq('uint genre_ids from ranged-query')
+    expect(declaration).to eq("#{id_type} genre_ids from ranged-query")
     expect(query).to match(/^SELECT .books_genres.\..book_id. \* #{count} \+ #{source.offset} AS .id., .books_genres.\..genre_id. AS .genre_ids. FROM .books_genres. WHERE \(.books_genres.\..book_id. BETWEEN \$start AND \$end\)$/)
     expect(range).to match(/^SELECT MIN\(.books_genres.\..book_id.\), MAX\(.books_genres.\..book_id.\) FROM .books_genres.$/)
   end if ActiveRecord::VERSION::MAJOR > 3