thinking-sphinx 2.1.0 → 3.0.0.pre

data/lib/thinking_sphinx/search/batch_inquirer.rb

@@ -0,0 +1,27 @@
+class ThinkingSphinx::Search::BatchInquirer
+  def initialize(&block)
+    @queries = []
+
+    yield self if block_given?
+  end
+
+  def append_query(query)
+    @queries << query
+  end
+
+  def results
+    @results ||= begin
+      @queries.freeze
+
+      results  = [connection.query(@queries.join('; '))]
+      results << connection.store_result while connection.next_result
+      results
+    end
+  end
+
+  private
+
+  def connection
+    @connection ||= ThinkingSphinx::Configuration.instance.connection
+  end
+end

data/lib/thinking_sphinx/search/context.rb

@@ -0,0 +1,26 @@
+class ThinkingSphinx::Search::Context
+  attr_reader :search, :configuration
+
+  def initialize(search, configuration = nil)
+    @search        = search
+    @configuration = configuration || ThinkingSphinx::Configuration.instance
+    @memory        = {
+      :results => [],
+      :panes   => ThinkingSphinx::Configuration::Defaults::PANES
+    }
+  end
+
+  def [](key)
+    @memory[key]
+  end
+
+  def []=(key, value)
+    @memory[key] = value
+  end
+
+  def log(notification, message, &block)
+    ActiveSupport::Notifications.instrument(
+      "#{notification}.thinking_sphinx", notification => message, &block
+    )
+  end
+end

data/lib/thinking_sphinx/search/glaze.rb

@@ -0,0 +1,32 @@
+class ThinkingSphinx::Search::Glaze < BasicObject
+  def initialize(context, object, raw = {}, pane_classes = [])
+    @object, @raw = object, raw
+
+    @panes = pane_classes.collect { |klass|
+      klass.new context, object, @raw
+    }
+  end
+
+  def ==(object)
+    (@object == object) || super
+  end
+
+  def equal?(object)
+    @object.equal? object
+  end
+
+  def unglazed
+    @object
+  end
+
+  private
+
+  def method_missing(method, *args, &block)
+    if @object.respond_to?(method)
+      @object.send(method, *args, &block)
+    else
+      pane = @panes.detect { |pane| pane.respond_to?(method) }
+      pane.nil? ? super : pane.send(method, *args, &block)
+    end
+  end
+end

data/lib/thinking_sphinx/search/merger.rb

@@ -0,0 +1,24 @@
+class ThinkingSphinx::Search::Merger
+  def initialize(search)
+    @search = search
+  end
+
+  def merge!(query = nil, options = {})
+    @search.query = query unless query.nil?
+    options.each do |key, value|
+      case key
+      when :conditions, :with, :without, :with_all, :without_all
+        @search.options[key] ||= {}
+        @search.options[key].merge! value
+      when :without_ids, :classes
+        @search.options[key] ||= []
+        @search.options[key] += value
+        @search.options[key].uniq!
+      else
+        @search.options[key] = value
+      end
+    end
+
+    @search
+  end
+end

data/lib/thinking_sphinx/search/query.rb

@@ -0,0 +1,43 @@
+class ThinkingSphinx::Search::Query
+  if Regexp.instance_methods.include?(:encoding)
+    DefaultToken = Regexp.new('\p{Word}+')
+  else
+    DefaultToken = Regexp.new('\w+', nil, 'u')
+  end
+
+  attr_reader :keywords, :conditions, :star
+
+  def initialize(keywords = '', conditions = {}, star = false)
+    @keywords, @conditions, @star = keywords, conditions, star
+  end
+
+  def to_s
+    (star_keyword(keywords) + ' ' + conditions.keys.collect { |key|
+      "@#{key} #{star_keyword conditions[key], key}"
+    }.join(' ')).strip
+  end
+
+  private
+
+  def star_keyword(keyword, key = nil)
+    unless star && (key.nil? || key.to_s != 'sphinx_internal_class')
+      return keyword.to_s
+    end
+
+    token = star.is_a?(Regexp) ? star : DefaultToken
+    keyword.gsub(/("#{token}(.*?#{token})?"|(?![!-])#{token})/u) do
+      pre, proper, post = $`, $&, $'
+      # E.g. "@foo", "/2", "~3", but not as part of a token
+      is_operator = pre.match(%r{(\W|^)[@~/]\Z}) ||
+                    pre.match(%r{(\W|^)@\([^\)]*$})
+      # E.g. "foo bar", with quotes
+      is_quote    = proper[/^".*"$/]
+      has_star    = post[/\*$/] || pre[/^\*/]
+      if is_operator || is_quote || has_star
+        proper
+      else
+        "*#{proper}*"
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/search/stale_ids_exception.rb

@@ -0,0 +1,11 @@
+class ThinkingSphinx::Search::StaleIdsException < StandardError
+  attr_reader :ids
+
+  def initialize(ids)
+    @ids = ids
+  end
+
+  def message
+    "Record IDs found by Sphinx but not by ActiveRecord : #{ids.join(', ')}"
+  end
+end

data/lib/thinking_sphinx/search/translator.rb

@@ -0,0 +1,50 @@
+class ThinkingSphinx::Search::Translator
+  attr_reader :raw, :excerpter
+
+  def initialize(raw, excerpter)
+    @raw, @excerpter = raw, excerpter
+  end
+
+  def to_active_record
+    results_for_models # load now to avoid segfaults
+    raw.collect { |row|
+      ThinkingSphinx::Search::Glaze.new result_for(row), excerpter, row
+    }
+  end
+
+  private
+
+  def ids_for_model(model_name)
+    raw.select { |row|
+      row['sphinx_internal_class'] == model_name
+    }.collect { |row|
+      row['sphinx_internal_id']
+    }
+  end
+
+  def model_names
+    @model_names ||= raw.collect { |row|
+      row['sphinx_internal_class']
+    }.uniq
+  end
+
+  def result_for(row)
+    results_for_models[row['sphinx_internal_class']].detect { |record|
+      record.id == row['sphinx_internal_id']
+    }
+  end
+
+  def results_for_models
+    @results_for_models ||= model_names.inject({}) { |hash, name|
+      ids = ids_for_model(name)
+      hash[name] = name.constantize.where(:id => ids)
+
+      stale_ids = ids - hash[name].collect(&:id)
+      if stale_ids.any?
+        raise ThinkingSphinx::Search::StaleIdsException, stale_ids
+      end
+
+      hash
+    }
+  end
+end

data/lib/thinking_sphinx/tasks.rb

@@ -1,138 +1,35 @@
-require 'fileutils'
-require 'timeout'
-
-namespace :thinking_sphinx do
-  task :app_env do
-    if defined?(Rails)
-      Rails.application.require_environment!
-      Rails.configuration.cache_classes = false
-    elsif defined?(Merb)
-      Rake::Task[:merb_env].invoke
-    elsif defined?(Sinatra)
-      Sinatra::Application.environment = ENV['RACK_ENV']
-    end
-  end
-
-  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
-  task :running_start => :app_env do
-    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
-    Rake::Task["thinking_sphinx:start"].invoke
-  end
-
-  desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
-  task :start => :app_env do
-    config = ThinkingSphinx::Configuration.instance
-
-    FileUtils.mkdir_p config.searchd_file_path
-    raise RuntimeError, "searchd is already running." if sphinx_running?
-
-    Dir["#{config.searchd_file_path}/*.spl"].each { |file| File.delete(file) }
-
-    if ENV["NODETACH"] == "true"
-      unless pid = fork
-        config.controller.start(:nodetach => true)
-      end
-      Signal.trap('TERM') { Process.kill(:QUIT, pid) }
-      Signal.trap('INT')  { Process.kill(:QUIT, pid) }
-      Process.wait(pid)
-    else
-      config.controller.start
-
-      if sphinx_running?
-        puts "Started successfully (pid #{sphinx_pid})."
-      else
-        puts "Failed to start searchd daemon. Check #{config.searchd_log_file}"
-        puts "Be sure to run thinking_sphinx:index before thinking_sphinx:start"
-      end
-    end
+namespace :ts do
+  desc 'The current version of Thinking Sphinx'
+  task :version do
+    puts "Thinking Sphinx v#{ThinkingSphinx::VERSION}"
   end
 
-  desc "Stop Sphinx using Thinking Sphinx's settings"
-  task :stop => :app_env do
-    unless sphinx_running?
-      puts "searchd is not running"
-    else
-      config = ThinkingSphinx::Configuration.instance
-      pid    = sphinx_pid
-      config.controller.stop
-
-      # Ensure searchd is stopped, but don't try too hard
-      Timeout.timeout(config.stop_timeout) do
-        sleep(1) until config.controller.stop
-      end
-
-      puts "Stopped search daemon (pid #{pid})."
-    end
+  desc 'Generate the Sphinx configuration file'
+  task :configure => :environment do
+    interface.configure
   end
 
-  desc "Restart Sphinx"
-  task :restart => [:app_env, :stop, :start]
-
-  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
-  task :configure => :app_env do
-    config = ThinkingSphinx::Configuration.instance
-    puts "Generating Configuration to #{config.config_file}"
-    config.build
+  desc 'Generate the Sphinx configuration file and process all indices'
+  task :index => :environment do
+    interface.index(ENV['INDEX_ONLY'] != 'true')
   end
 
-  desc "Index data for Sphinx using Thinking Sphinx's settings"
-  task :index => :app_env do
-    config = ThinkingSphinx::Configuration.instance
-    unless ENV["INDEX_ONLY"] == "true"
-      puts "Generating Configuration to #{config.config_file}"
-      config.build
-    end
+  desc 'Stop Sphinx, index and then restart Sphinx'
+  task :rebuild => [:stop, :index, :start]
+  desc 'Restart the Sphinx daemon'
+  task :restart => [:stop, :start]
 
-    FileUtils.mkdir_p config.searchd_file_path
-    ThinkingSphinx.before_index_hooks.each { |hook| hook.call }
-    config.controller.index :verbose => true
+  desc 'Start the Sphinx daemon'
+  task :start => :environment do
+    interface.start
   end
 
-  desc "Reindex Sphinx without regenerating the configuration file"
-  task :reindex => :app_env do
-    config = ThinkingSphinx::Configuration.instance
-    FileUtils.mkdir_p config.searchd_file_path
-    output = config.controller.index
-    puts output
-    config.touch_reindex_file(output)
+  desc 'Stop the Sphinx daemon'
+  task :stop => :environment do
+    interface.stop
   end
 
-  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
-  task :rebuild => :app_env do
-    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
-    Rake::Task["thinking_sphinx:index"].invoke
-    Rake::Task["thinking_sphinx:start"].invoke
+  def interface
+    @interface ||= ThinkingSphinx::RakeInterface.new
   end
 end
-
-namespace :ts do
-  desc "Output the current Thinking Sphinx version"
-  task :version => "thinking_sphinx:version"
-  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
-  task :run     => "thinking_sphinx:running_start"
-  desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
-  task :start   => "thinking_sphinx:start"
-  desc "Stop Sphinx using Thinking Sphinx's settings"
-  task :stop    => "thinking_sphinx:stop"
-  desc "Index data for Sphinx using Thinking Sphinx's settings"
-  task :in      => "thinking_sphinx:index"
-  task :index   => "thinking_sphinx:index"
-  desc "Reindex Sphinx without regenerating the configuration file"
-  task :reindex => "thinking_sphinx:reindex"
-  desc "Restart Sphinx"
-  task :restart => "thinking_sphinx:restart"
-  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
-  task :conf    => "thinking_sphinx:configure"
-  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
-  task :config  => "thinking_sphinx:configure"
-  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
-  task :rebuild => "thinking_sphinx:rebuild"
-end
-
-def sphinx_pid
-  ThinkingSphinx.sphinx_pid
-end
-
-def sphinx_running?
-  ThinkingSphinx.sphinx_running?
-end

data/lib/thinking_sphinx/test.rb

@@ -1,31 +1,31 @@
 class ThinkingSphinx::Test
   def self.init(suppress_delta_output = true)
-    set_flags suppress_delta_output
-    create_indexes_folder
+    FileUtils.mkdir_p config.indices_location
+    config.settings['quiet_deltas'] = true
   end
-  
+
   def self.start
     config.build
     config.controller.index
     config.controller.start
   end
-  
+
   def self.start_with_autostop
     autostop
     start
   end
-  
+
   def self.stop
     config.controller.stop
     sleep(0.5) # Ensure Sphinx has shut down completely
   end
-  
+
   def self.autostop
     Kernel.at_exit do
       ThinkingSphinx::Test.stop
     end
   end
-  
+
   def self.run(&block)
     begin
       start
@@ -34,22 +34,12 @@ class ThinkingSphinx::Test
       stop
     end
   end
-  
+
   def self.config
     @config ||= ::ThinkingSphinx::Configuration.instance
   end
-  
+
   def self.index(*indexes)
     config.controller.index *indexes
   end
-  
-  def self.set_flags(suppress_delta_output)
-    ::ThinkingSphinx.deltas_enabled        = true
-    ::ThinkingSphinx.updates_enabled       = true
-    ::ThinkingSphinx.suppress_delta_output = suppress_delta_output
-  end
-  
-  def self.create_indexes_folder
-    FileUtils.mkdir_p config.searchd_file_path
-  end
 end

data/sketchpad.rb

@@ -0,0 +1,58 @@
+# LOADING ALL CLASSES is invoked by:
+# - call define_index
+#   - loading any model with an index definition
+#   - because need to add to list of which models have indices.
+# - sphinx document id for an instance is requested
+#   - updating attributes
+#   - toggling as deleted
+# - when generating configuration
+#   - need to build full configuration file.
+# - when loading models_by_crc
+#   - searches and facet searches pre Sphinx 2
+# - facet search
+# - search
+#
+# WHY?
+# - need to know which classes are searchable
+# - need to track number of indices (multiplier) - STI matters here
+#   - are a subclass and superclass STI pair both indexed? One offset for both.
+# - need to track position of current model within all indexed models (offset)
+# - need to generate configuration
+# - need to know subclasses available to search on.
+#
+# SO:
+# - load all indices *and* related models when needing document id
+# - load all indices *and* related models when generating configuration
+# - maybe can avoid loading all indices and/or related models when searching?
+#   - would need to track ancestor models - harder when no multi-string
+#     attributes - for filtering on classes. Can we be smart about this and
+#     only force a full model load if absolutely necessary? And/or forced by
+#     a flag so developers need to request it?
+
+
+
+# Use symbols - avoid loading classes until we need to.
+ThinkingSphinx.define_index_for :article do
+  indexes subject, content
+  indexes user.name, :as => :user_name
+
+  has user_id
+end
+
+# Or better - let's not add everything to the root of TS module, and make it
+# clear of the index driver.
+ThinkingSphinx::Index.define :article, :with => :active_record do
+  indexes subject, content
+  indexes user.name, :as => :user_name
+
+  has user_id
+end
+
+# Built-in index driver support could be auto-mapped to methods:
+ThinkingSphinx::Index.define_with_active_record, :article
+
+ThinkingSphinx::Index.define_with_realtime :article do
+  indexes :subject, :content, :user_name
+
+  has user_id => :integer
+end