thinking-sphinx 3.4.2 → 4.0.0

data/lib/thinking_sphinx/real_time/interpreter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Interpreter <
   ::ThinkingSphinx::Core::Interpreter
 

data/lib/thinking_sphinx/real_time/populator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Populator
   def self.populate(index)
     new(index).populate
@@ -15,7 +17,6 @@ class ThinkingSphinx::RealTime::Populator
       instrument 'populated', :instances => instances
     end
 
-    controller.rotate
     instrument 'finish_populating'
   end
 

data/lib/thinking_sphinx/real_time/property.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Property
   include ThinkingSphinx::Core::Property
 

data/lib/thinking_sphinx/real_time/transcribe_instance.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::RealTime::TranscribeInstance
+  def self.call(instance, index, properties)
+    new(instance, index, properties).call
+  end
+
+  def initialize(instance, index, properties)
+    @instance, @index, @properties = instance, index, properties
+  end
+
+  def call
+    properties.each_with_object([document_id]) do |property, instance_values|
+      begin
+        instance_values << property.translate(instance)
+      rescue StandardError => error
+        raise_wrapper error, property
+      end
+    end
+  end
+
+  private
+
+  attr_reader :instance, :index, :properties
+
+  def document_id
+    index.document_id_for_key instance.id
+  end
+
+  def raise_wrapper(error, property)
+    wrapper = ThinkingSphinx::TranscriptionError.new
+    wrapper.inner_exception = error
+    wrapper.instance        = instance
+    wrapper.property        = property
+
+    raise wrapper
+  end
+end

data/lib/thinking_sphinx/real_time/transcriber.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Transcriber
   def initialize(index)
     @index = index
@@ -9,9 +11,16 @@ class ThinkingSphinx::RealTime::Transcriber
     }
     return unless items.present?
 
-    values = items.collect { |instance|
-      TranscribeInstance.call(instance, index, properties)
-    }
+    values = []
+    items.each do |instance|
+      begin
+        values << ThinkingSphinx::RealTime::TranscribeInstance.call(
+          instance, index, properties
+        )
+      rescue ThinkingSphinx::TranscriptionError => error
+        instrument 'error', :error => error
+      end
+    end
 
     insert = Riddle::Query::Insert.new index.name, columns, values
     sphinxql = insert.replace!.to_sql
@@ -23,30 +32,6 @@ class ThinkingSphinx::RealTime::Transcriber
     end
   end
 
-  class TranscribeInstance
-    def self.call(instance, index, properties)
-      new(instance, index, properties).call
-    end
-
-    def initialize(instance, index, properties)
-      @instance, @index, @properties = instance, index, properties
-    end
-
-    def call
-      properties.each_with_object([document_id]) do |property, instance_values|
-        instance_values << property.translate(instance)
-      end
-    end
-
-    private
-
-    attr_reader :instance, :index, :properties
-
-    def document_id
-      index.document_id_for_key instance.id
-    end
-  end
-
   private
 
   attr_reader :index
@@ -70,6 +55,12 @@ class ThinkingSphinx::RealTime::Transcriber
     }
   end
 
+  def instrument(message, options = {})
+    ActiveSupport::Notifications.instrument(
+      "#{message}.thinking_sphinx.real_time", options.merge(:index => index)
+    )
+  end
+
   def properties
     @properties ||= index.fields + index.attributes
   end

data/lib/thinking_sphinx/real_time/translator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::RealTime::Translator
   def self.call(object, column)
     new(object, column).call
@@ -11,7 +13,7 @@ class ThinkingSphinx::RealTime::Translator
     return name   unless name.is_a?(Symbol)
     return result unless result.is_a?(String)
 
-    result.gsub "\u0000", ''
+    result.gsub("\u0000", '').force_encoding "UTF-8"
   end
 
   private

data/lib/thinking_sphinx/scopes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ThinkingSphinx::Scopes
   extend ActiveSupport::Concern
 

data/lib/thinking_sphinx/search.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search < Array
   CORE_METHODS = %w( == class class_eval extend frozen? id instance_eval
     instance_of? instance_values instance_variable_defined?

data/lib/thinking_sphinx/search/batch_inquirer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::BatchInquirer
   def initialize(&block)
     @queries = []

data/lib/thinking_sphinx/search/context.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::Context
   attr_reader :search, :configuration
 

data/lib/thinking_sphinx/search/glaze.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::Glaze < BasicObject
   def initialize(context, object, raw = {}, pane_classes = [])
     @object, @raw = object, raw

data/lib/thinking_sphinx/search/merger.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::Merger
   attr_reader :search
 

data/lib/thinking_sphinx/search/query.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::Query
   attr_reader :keywords, :conditions, :star
 

data/lib/thinking_sphinx/search/stale_ids_exception.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Search::StaleIdsException < StandardError
   attr_reader :ids, :context
 

data/lib/thinking_sphinx/settings.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+class ThinkingSphinx::Settings
+  ALWAYS_ABSOLUTE = %w[ socket ]
+  FILE_KEYS = %w[
+    indices_location configuration_file bin_path log query_log pid_file
+    binlog_path snippets_file_prefix sphinxql_state path stopwords wordforms
+    exceptions global_idf rlp_context rlp_root rlp_environment plugin_dir
+    lemmatizer_base mysql_ssl_cert mysql_ssl_key mysql_ssl_ca
+  ].freeze
+  DEFAULTS = {
+    "configuration_file" => "config/ENVIRONMENT.sphinx.conf",
+    "indices_location"   => "db/sphinx/ENVIRONMENT",
+    "pid_file"           => "log/ENVIRONMENT.sphinx.pid",
+    "log"                => "log/ENVIRONMENT.searchd.log",
+    "query_log"          => "log/ENVIRONMENT.searchd.query.log",
+    "binlog_path"        => "tmp/binlog/ENVIRONMENT",
+    "workers"            => "threads"
+  }.freeze
+
+  def self.call(configuration)
+    new(configuration).call
+  end
+
+  def initialize(configuration)
+    @configuration = configuration
+  end
+
+  def call
+    return defaults unless File.exists? file
+
+    merged.inject({}) do |hash, (key, value)|
+      if absolute_key?(key)
+        hash[key] = absolute value
+      else
+        hash[key] = value
+      end
+      hash
+    end
+  end
+
+  private
+
+  attr_reader :configuration
+
+  delegate :framework, :to => :configuration
+
+  def absolute(relative)
+    return relative if relative.nil?
+
+    real_path File.absolute_path(relative, framework.root)
+  end
+
+  def absolute_key?(key)
+    return true if ALWAYS_ABSOLUTE.include?(key)
+
+    merged["absolute_paths"] && file_keys.include?(key)
+  end
+
+  def defaults
+    DEFAULTS.inject({}) do |hash, (key, value)|
+      value = value.gsub("ENVIRONMENT", framework.environment)
+
+      if FILE_KEYS.include?(key)
+        hash[key] = absolute value
+      else
+        hash[key] = value
+      end
+
+      hash
+    end
+  end
+
+  def file
+    @file ||= Pathname.new(framework.root).join "config", "thinking_sphinx.yml"
+  end
+
+  def file_keys
+    @file_keys ||= FILE_KEYS + (original["file_keys"] || [])
+  end
+
+  def join(first, last)
+    return first if last.nil?
+
+    File.join first, last
+  end
+
+  def merged
+    @merged ||= defaults.merge original
+  end
+
+  def original
+    input = File.read file
+    input = ERB.new(input).result if defined?(ERB)
+
+    contents = YAML.load input
+    contents && contents[framework.environment] || {}
+  end
+
+  def real_path(base, nonexistent = nil)
+    if File.exist?(base)
+      join File.realpath(base), nonexistent
+    else
+      components = File.split base
+      real_path components.first, join(components.last, nonexistent)
+    end
+  end
+end

data/lib/thinking_sphinx/sinatra.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thinking_sphinx'
 
 ActiveSupport.on_load :active_record do

data/lib/thinking_sphinx/subscribers/populator_subscriber.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ThinkingSphinx::Subscribers::PopulatorSubscriber
   def self.attach_to(namespace)
     subscriber = new
@@ -16,6 +18,17 @@ class ThinkingSphinx::Subscribers::PopulatorSubscriber
       ActiveSupport::Notifications::Event.new(message, *args)
   end
 
+  def error(event)
+    error    = event.payload[:error].inner_exception
+    instance = event.payload[:error].instance
+
+    puts <<-MESSAGE
+
+Error transcribing #{instance.class} #{instance.id}:
+#{error.message}
+    MESSAGE
+  end
+
   def start_populating(event)
     puts "Generating index files for #{event.payload[:index].name}"
   end
@@ -27,6 +40,11 @@ class ThinkingSphinx::Subscribers::PopulatorSubscriber
   def finish_populating(event)
     print "\n"
   end
+
+  private
+
+  delegate :output, :to => ThinkingSphinx
+  delegate :puts, :print, :to => :output
 end
 
 ThinkingSphinx::Subscribers::PopulatorSubscriber.attach_to(

data/lib/thinking_sphinx/tasks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 namespace :ts do
   desc 'Generate the Sphinx configuration file'
   task :configure => :environment do
@@ -10,53 +12,14 @@ namespace :ts do
   desc 'Clear out Sphinx files'
   task :clear => ['ts:sql:clear', 'ts:rt:clear']
 
-  desc 'DEPRECATED: Clear out real-time index files'
-  task :clear_rt => :environment do
-    puts <<-TXT
-The ts:clear_rt task is now deprecated due to the unified task approach, and
-invokes ts:rt:clear.
-* To delete all indices (both SQL-backed and real-time), use ts:clear.
-* To delete just real-time indices, use ts:rt:clear.
-* To delete just SQL-backed indices, use ts:sql:clear.
-
-    TXT
-
-    Rake::Task['ts:rt:clear'].invoke
-  end
-
-  desc 'DEPRECATED: Generate fresh index files for all indices'
-  task :generate => :environment do
-    puts <<-TXT
-The ts:generate task is now deprecated due to the unified task approach, and
-invokes ts:index.
-* To process all indices (both SQL-backed and real-time), use ts:index.
-* To process just real-time indices, use ts:rt:index.
-* To process just SQL-backed indices, use ts:sql:index.
-
-    TXT
-
-    Rake::Task['ts:index'].invoke
-  end
+  desc "Merge all delta indices into their respective core indices"
+  task :merge => ["ts:sql:merge"]
 
   desc 'Delete and regenerate Sphinx files, restart the daemon'
   task :rebuild => [
     :stop, :clear, :configure, 'ts:sql:index', :start, 'ts:rt:index'
   ]
 
-  desc 'DEPRECATED: Delete and regenerate Sphinx files, restart the daemon'
-  task :regenerate do
-        puts <<-TXT
-The ts:regenerate task is now deprecated due to the unified task approach, and
-invokes ts:rebuild.
-* To rebuild all indices (both SQL-backed and real-time), use ts:rebuild.
-* To rebuild just real-time indices, use ts:rt:rebuild.
-* To rebuild just SQL-backed indices, use ts:sql:rebuild.
-
-    TXT
-
-    Rake::Task['ts:rebuild'].invoke
-  end
-
   desc 'Restart the Sphinx daemon'
   task :restart => [:stop, :start]
 
@@ -86,6 +49,10 @@ invokes ts:rebuild.
       interface.sql.index(ENV['INDEX_ONLY'] != 'true')
     end
 
+    task :merge => :environment do
+      interface.sql.merge
+    end
+
     desc 'Delete and regenerate SQL-backed Sphinx files, restart the daemon'
     task :rebuild => ['ts:stop', 'ts:sql:clear', 'ts:sql:index', 'ts:start']
   end
@@ -108,11 +75,11 @@ invokes ts:rebuild.
   end
 
   def interface
-    @interface ||= ThinkingSphinx::RakeInterface.new(
-      :verbose      => Rake::FileUtilsExt.verbose_flag,
-      :silent       => Rake.application.options.silent,
-      :nodetach     => (ENV['NODETACH'] == 'true'),
-      :index_filter => ENV['INDEX_FILTER']
+    @interface ||= ThinkingSphinx.rake_interface.new(
+      :verbose     => Rake::FileUtilsExt.verbose_flag,
+      :silent      => Rake.application.options.silent,
+      :nodetach    => (ENV['NODETACH'] == 'true'),
+      :index_names => ENV.fetch('INDEX_FILTER', '').split(',')
     )
   end
 end