thinking-sphinx 1.2.13 → 6.0.0

data/lib/thinking_sphinx/settings.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+class ThinkingSphinx::Settings
+  ALWAYS_ABSOLUTE = %w[ socket ]
+  FILE_KEYS = %w[
+    indices_location configuration_file bin_path log query_log pid_file
+    binlog_path snippets_file_prefix sphinxql_state path stopwords wordforms
+    exceptions global_idf rlp_context rlp_root rlp_environment plugin_dir
+    lemmatizer_base mysql_ssl_cert mysql_ssl_key mysql_ssl_ca
+  ].freeze
+  DEFAULTS = {
+    "configuration_file"       => "config/ENVIRONMENT.sphinx.conf",
+    "indices_location"         => "db/sphinx/ENVIRONMENT",
+    "pid_file"                 => "log/ENVIRONMENT.sphinx.pid",
+    "log"                      => "log/ENVIRONMENT.searchd.log",
+    "query_log"                => "log/ENVIRONMENT.searchd.query.log",
+    "binlog_path"              => "tmp/binlog/ENVIRONMENT",
+    "workers"                  => "threads",
+    "mysql_encoding"           => "utf8",
+    "maximum_statement_length" => (2 ** 23) - 5,
+    "real_time_tidy"           => false,
+    "cutoff"                   => 0
+  }.freeze
+  YAML_SAFE_LOAD = YAML.method(:safe_load).parameters.any? do |parameter|
+    parameter == [:key, :aliases]
+  end
+
+  def self.call(configuration)
+    new(configuration).call
+  end
+
+  def initialize(configuration)
+    @configuration = configuration
+  end
+
+  def call
+    return defaults unless File.exist? file
+
+    merged.inject({}) do |hash, (key, value)|
+      if absolute_key?(key)
+        hash[key] = absolute value
+      else
+        hash[key] = value
+      end
+      hash
+    end
+  end
+
+  private
+
+  attr_reader :configuration
+
+  delegate :framework, :to => :configuration
+
+  def absolute(relative)
+    return relative if relative.nil?
+
+    real_path File.absolute_path(relative, framework.root)
+  end
+
+  def absolute_key?(key)
+    return true if ALWAYS_ABSOLUTE.include?(key)
+
+    merged["absolute_paths"] && file_keys.include?(key)
+  end
+
+  def defaults
+    DEFAULTS.inject({}) do |hash, (key, value)|
+      if value.is_a?(String)
+        value = value.gsub("ENVIRONMENT", framework.environment)
+      end
+
+      if FILE_KEYS.include?(key)
+        hash[key] = absolute value
+      else
+        hash[key] = value
+      end
+
+      hash
+    end
+  end
+
+  def file
+    @file ||= Pathname.new(framework.root).join "config", "thinking_sphinx.yml"
+  end
+
+  def file_keys
+    @file_keys ||= FILE_KEYS + (original["file_keys"] || [])
+  end
+
+  def join(first, last)
+    return first if last.nil?
+
+    File.join first, last
+  end
+
+  def merged
+    @merged ||= defaults.merge original
+  end
+
+  def original
+    yaml_contents && yaml_contents[framework.environment] || {}
+  end
+
+  def real_path(base, nonexistent = nil)
+    if File.exist?(base)
+      join File.realpath(base), nonexistent
+    else
+      components = File.split base
+      real_path components.first, join(components.last, nonexistent)
+    end
+  end
+
+  def yaml_contents
+    @yaml_contents ||= begin
+      input = File.read file
+      input = ERB.new(input).result if defined?(ERB)
+
+      if YAML_SAFE_LOAD
+        YAML.safe_load(input, aliases: true)
+      else
+        YAML.load(input)
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/sinatra.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'thinking_sphinx'
+
+ActiveSupport.on_load :active_record do
+  require 'thinking_sphinx/active_record'
+end

data/lib/thinking_sphinx/subscribers/populator_subscriber.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Subscribers::PopulatorSubscriber
+  def self.attach_to(namespace)
+    subscriber = new
+
+    subscriber.public_methods(false).each do |event|
+      next if event == :call
+
+      ActiveSupport::Notifications.subscribe(
+        "#{event}.#{namespace}", subscriber
+      )
+    end
+  end
+
+  def call(message, *args)
+    send message.split('.').first,
+      ActiveSupport::Notifications::Event.new(message, *args)
+  end
+
+  def error(event)
+    error    = event.payload[:error].inner_exception
+    instance = event.payload[:error].instance
+
+    puts <<-MESSAGE
+
+Error transcribing #{instance.class} #{instance.id}:
+#{error.message}
+    MESSAGE
+  end
+
+  def start_populating(event)
+    puts "Generating index files for #{event.payload[:index].name}"
+  end
+
+  def populated(event)
+    print '.' * event.payload[:instances].length
+  end
+
+  def finish_populating(event)
+    print "\n"
+  end
+
+  private
+
+  delegate :output, :to => ThinkingSphinx
+  delegate :puts, :print, :to => :output
+end

data/lib/thinking_sphinx/tasks.rb

@@ -1,165 +1,85 @@
-require 'fileutils'
+# frozen_string_literal: true
 
-namespace :thinking_sphinx do
-  task :app_env do
-    if defined?(RAILS_ROOT)
-      Rake::Task[:environment].invoke
-      Rails.configuration.cache_classes = false
-    end
-    
-    Rake::Task[:merb_env].invoke    if defined?(Merb)
+namespace :ts do
+  desc 'Generate the Sphinx configuration file'
+  task :configure => :environment do
+    interface.configure
   end
-  
-  desc "Output the current Thinking Sphinx version"
-  task :version => :app_env do
-    puts "Thinking Sphinx v" + ThinkingSphinx.version
+
+  desc 'Generate the Sphinx configuration file and process all indices'
+  task :index => ['ts:sql:index', 'ts:rt:index']
+
+  desc 'Clear out Sphinx files'
+  task :clear => ['ts:sql:clear', 'ts:rt:clear']
+
+  desc "Merge all delta indices into their respective core indices"
+  task :merge => ["ts:sql:merge"]
+
+  desc 'Delete and regenerate Sphinx files, restart the daemon'
+  task :rebuild => [
+    :stop, :clear, :configure, 'ts:sql:index', :start, 'ts:rt:index'
+  ]
+
+  desc 'Restart the Sphinx daemon'
+  task :restart => [:stop, :start]
+
+  desc 'Start the Sphinx daemon'
+  task :start => :environment do
+    interface.daemon.start
   end
-  
-  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
-  task :running_start => :app_env do
-    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
-    Rake::Task["thinking_sphinx:start"].invoke
+
+  desc 'Stop the Sphinx daemon'
+  task :stop => :environment do
+    interface.daemon.stop
   end
-  
-  desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
-  task :start => :app_env do
-    config = ThinkingSphinx::Configuration.instance
-    
-    FileUtils.mkdir_p config.searchd_file_path
-    raise RuntimeError, "searchd is already running." if sphinx_running?
-    
-    Dir["#{config.searchd_file_path}/*.spl"].each { |file| File.delete(file) }
-
-    system! "#{config.bin_path}#{config.searchd_binary_name} --pidfile --config \"#{config.config_file}\""
-    
-    sleep(2)
-    
-    if sphinx_running?
-      puts "Started successfully (pid #{sphinx_pid})."
-    else
-      puts "Failed to start searchd daemon. Check #{config.searchd_log_file}"
-    end
+
+  desc 'Determine whether Sphinx is running'
+  task :status => :environment do
+    interface.daemon.status
   end
-  
-  desc "Stop Sphinx using Thinking Sphinx's settings"
-  task :stop => :app_env do
-    unless sphinx_running?
-      puts "searchd is not running"
-    else
-      config = ThinkingSphinx::Configuration.instance
-      pid    = sphinx_pid
-      system! "#{config.bin_path}#{config.searchd_binary_name} --stop --config \"#{config.config_file}\""
-      puts "Stopped search daemon (pid #{pid})."
+
+  namespace :sql do
+    desc 'Delete SQL-backed Sphinx files'
+    task :clear => :environment do
+      interface.sql.clear
     end
-  end
-  
-  desc "Restart Sphinx"
-  task :restart => [:app_env, :stop, :start]
-  
-  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
-  task :configure => :app_env do
-    config = ThinkingSphinx::Configuration.instance
-    puts "Generating Configuration to #{config.config_file}"
-    config.build
-  end
-  
-  desc "Index data for Sphinx using Thinking Sphinx's settings"
-  task :index => :app_env do
-    ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
-    
-    config = ThinkingSphinx::Configuration.instance
-    unless ENV["INDEX_ONLY"] == "true"
-      puts "Generating Configuration to #{config.config_file}"
-      config.build
+
+    desc 'Generate fresh index files for SQL-backed indices'
+    task :index => :environment do
+      interface.sql.index(ENV['INDEX_ONLY'] != 'true')
     end
-        
-    FileUtils.mkdir_p config.searchd_file_path
-    cmd = "#{config.bin_path}#{config.indexer_binary_name} --config \"#{config.config_file}\" --all"
-    cmd << " --rotate" if sphinx_running?
-    
-    system! cmd
-  end
-  
-  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
-  task :rebuild => :app_env do
-    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
-    Rake::Task["thinking_sphinx:index"].invoke
-    Rake::Task["thinking_sphinx:start"].invoke
-  end
-  
-  namespace :index do
-    task :delta => :app_env do
-      ThinkingSphinx.indexed_models.select { |model|
-        model.constantize.sphinx_indexes.any? { |index| index.delta? }
-      }.each do |model|
-        model.constantize.sphinx_indexes.select { |index|
-          index.delta? && index.delta_object.respond_to?(:delayed_index)
-        }.each { |index|
-          index.delta_object.delayed_index(index.model)
-        }
-      end
+
+    task :merge => :environment do
+      interface.sql.merge
     end
-  end
-  
-  desc "Process stored delta index requests"
-  task :delayed_delta => :app_env do
-    require 'delayed/worker'
-    
-    Delayed::Worker.new(
-      :min_priority => ENV['MIN_PRIORITY'],
-      :max_priority => ENV['MAX_PRIORITY']
-    ).start
-  end
-end
 
-namespace :ts do
-  desc "Output the current Thinking Sphinx version"
-  task :version => "thinking_sphinx:version"
-  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
-  task :run     => "thinking_sphinx:running_start"
-  desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
-  task :start   => "thinking_sphinx:start"
-  desc "Stop Sphinx using Thinking Sphinx's settings"
-  task :stop    => "thinking_sphinx:stop"
-  desc "Index data for Sphinx using Thinking Sphinx's settings"
-  task :in      => "thinking_sphinx:index"
-  namespace :in do
-    desc "Index Thinking Sphinx datetime delta indexes"
-    task :delta => "thinking_sphinx:index:delta"
+    desc 'Delete and regenerate SQL-backed Sphinx files, restart the daemon'
+    task :rebuild => ['ts:stop', 'ts:sql:clear', 'ts:sql:index', 'ts:start']
   end
-  task :index   => "thinking_sphinx:index"
-  desc "Restart Sphinx"
-  task :restart => "thinking_sphinx:restart"
-  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
-  task :conf    => "thinking_sphinx:configure"
-  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
-  task :config  => "thinking_sphinx:configure"
-  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
-  task :rebuild => "thinking_sphinx:rebuild"
-  desc "Process stored delta index requests"
-  task :dd      => "thinking_sphinx:delayed_delta"
-end
 
-def sphinx_pid
-  ThinkingSphinx.sphinx_pid
-end
+  namespace :rt do
+    desc 'Delete real-time Sphinx files'
+    task :clear => :environment do
+      interface.rt.clear
+    end
 
-def sphinx_running?
-  ThinkingSphinx.sphinx_running?
-end
+    desc 'Generate fresh index files for real-time indices'
+    task :index => :environment do
+      interface.rt.index
+    end
+
+    desc 'Delete and regenerate real-time Sphinx files, restart the daemon'
+    task :rebuild => [
+      'ts:stop', 'ts:rt:clear', 'ts:configure', 'ts:start', 'ts:rt:index'
+    ]
+  end
 
-# a fail-fast, hopefully helpful version of system
-def system!(cmd)
-  unless system(cmd)
-    raise <<-SYSTEM_CALL_FAILED
-The following command failed:
-  #{cmd}
-
-This could be caused by a PATH issue in the environment of cron/passenger/etc. Your current PATH:
-  #{ENV['PATH']}
-You can set the path to your indexer and searchd binaries using the bin_path property in config/sphinx.yml:
-  production:
-    bin_path: '/usr/local/bin'
-SYSTEM_CALL_FAILED
+  def interface
+    @interface ||= ThinkingSphinx.rake_interface.new(
+      :verbose     => Rake::FileUtilsExt.verbose_flag,
+      :silent      => Rake.application.options.silent,
+      :nodetach    => (ENV['NODETACH'] == 'true'),
+      :index_names => ENV.fetch('INDEX_FILTER', '').split(',')
+    )
   end
 end

data/lib/thinking_sphinx/test.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Test
+  def self.init(suppress_delta_output = true)
+    FileUtils.mkdir_p config.indices_location
+    config.settings['quiet_deltas'] = suppress_delta_output
+  end
+
+  def self.start(options = {})
+    config.render_to_file
+    config.controller.index if options[:index].nil? || options[:index]
+    config.controller.start
+  end
+
+  def self.start_with_autostop
+    autostop
+    start
+  end
+
+  def self.stop
+    config.controller.stop
+    sleep(0.5) # Ensure Sphinx has shut down completely
+  end
+
+  def self.autostop
+    Kernel.at_exit do
+      ThinkingSphinx::Test.stop
+    end
+  end
+
+  def self.run(&block)
+    begin
+      start
+      yield
+    ensure
+      stop
+    end
+  end
+
+  def self.clear
+    [
+      config.indices_location,
+      config.searchd.binlog_path
+    ].each do |path|
+      FileUtils.rm_rf(path) if File.exist?(path)
+    end
+  end
+
+  def self.config
+    @config ||= ::ThinkingSphinx::Configuration.instance
+  end
+
+  def self.index(*indexes)
+    config.controller.index *indexes
+  end
+end

data/lib/thinking_sphinx/utf8.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::UTF8
+  attr_reader :string
+
+  def self.encode(string)
+    new(string).encode
+  end
+
+  def initialize(string)
+    @string = string
+  end
+
+  def encode
+    string.encode!('ISO-8859-1')
+    string.force_encoding('UTF-8')
+  end
+end

data/lib/thinking_sphinx/wildcard.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Wildcard
+  DEFAULT_TOKEN = /\p{Word}+/
+
+  def self.call(query, pattern = DEFAULT_TOKEN)
+    new(query, pattern).call
+  end
+
+  def initialize(query, pattern = DEFAULT_TOKEN)
+    @query   = query || ''
+    @pattern = pattern.is_a?(Regexp) ? pattern : DEFAULT_TOKEN
+  end
+
+  def call
+    query.gsub(extended_pattern) do
+      pre, proper, post = $`, $&, $'
+      # E.g. "@foo", "/2", "~3", but not as part of a token pattern
+      is_operator = pre.match(%r{@$}) ||
+                    pre.match(%r{([^\\]+|\A)[~/]\Z}) ||
+                    pre.match(%r{(\W|^)@\([^\)]*$})
+      # E.g. "foo bar", with quotes
+      is_quote    = proper[/^".*"$/]
+      has_star    = post[/\*$/] || pre[/^\*/]
+      if is_operator || is_quote || has_star
+        proper
+      else
+        "*#{proper}*"
+      end
+    end
+  end
+
+  private
+
+  attr_reader :query, :pattern
+
+  def extended_pattern
+    Regexp.new(
+      "(\"#{pattern}(.*?#{pattern})?\"|(?![!-])#{pattern})".encode('UTF-8')
+    )
+  end
+end

data/lib/thinking_sphinx/with_output.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module ThinkingSphinx::WithOutput
+  def initialize(configuration, options = {}, stream = STDOUT)
+    @configuration = configuration
+    @options       = options
+    @stream        = stream
+  end
+
+  private
+
+  attr_reader :configuration, :options, :stream
+end