flying-sphinx 0.6.6 → 0.7.0

data/HISTORY

@@ -1,3 +1,10 @@
+0.7.0 - 16th July 2012
+* Print the indexing log.
+* Distinguish between search server and SSH/indexing server, which allows for load balancers as the former.
+* Send the gem version through as a header on API calls.
+* Let flying-sphinx.com wrangle the Sphinx configuration.
+* Use v3 of the flying-sphinx.com API
+
 0.6.6 - 14th July 2012
 * Don't complain about 201s for starting/stopping Sphinx.
 * Relaxing the faraday_middleware dependency to allow 0.8 releases (Matthew Zikherman).

data/flying-sphinx.gemspec

@@ -25,10 +25,10 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'faraday_middleware', ['~> 0.7']
   s.add_runtime_dependency 'rash',               ['~> 0.3.0']
 
-  s.add_development_dependency 'rake',            ['0.8.7']
+  s.add_development_dependency 'rake',            ['~> 0.9.2']
+  s.add_development_dependency 'rspec',           ['~> 2.11']
+  s.add_development_dependency 'rspec-fire',      ['~> 1.1.0']
   s.add_development_dependency 'yajl-ruby',       ['~> 0.8.2']
-  s.add_development_dependency 'rspec',           ['~> 2.5.0']
-  s.add_development_dependency 'rcov',            ['~> 0.9.9']
   s.add_development_dependency 'fakeweb',         ['~> 1.3.0']
   s.add_development_dependency 'fakeweb-matcher', ['~> 1.2.2']
   s.add_development_dependency 'delayed_job',     ['~> 2.1.4']

data/lib/flying_sphinx.rb

@@ -2,6 +2,8 @@ module FlyingSphinx
   #
 end
 
+require 'faraday'
+require 'faraday_middleware'
 require 'net/ssh'
 require 'riddle/0.9.9'
 
@@ -11,6 +13,8 @@ require 'flying_sphinx/delayed_delta'
 require 'flying_sphinx/flag_as_deleted_job'
 require 'flying_sphinx/heroku_shared_adapter'
 require 'flying_sphinx/index_request'
+require 'flying_sphinx/setting_files'
+require 'flying_sphinx/sphinx_configuration'
 require 'flying_sphinx/tunnel'
 require 'flying_sphinx/version'
 

data/lib/flying_sphinx/api.rb

@@ -1,14 +1,8 @@
-require 'faraday'
-require 'faraday_middleware'
-
 class FlyingSphinx::API
-
-  APIServer        = 'https://flying-sphinx.com'
-  APIStagingServer = 'https://staging.flying-sphinx.com'
-  APIPath          = "/api/my/app"
-  APIVersion       = 2
-
-  attr_reader :api_key, :identifier, :adapter
+  SERVER         = 'https://flying-sphinx.com'
+  STAGING_SERVER = 'https://staging.flying-sphinx.com'
+  PATH           = "/api/my/app"
+  VERSION        = 3
 
   def initialize(identifier, api_key, adapter = Faraday.default_adapter)
     @api_key    = api_key
@@ -38,22 +32,25 @@ class FlyingSphinx::API
 
   private
 
+  attr_reader :api_key, :identifier, :adapter
+
   def normalize_path(path)
-    path = (path == '/' ? nil : "/#{path}")
-    "#{APIPath}#{path}"
+    path = (path == '/' ? '' : "/#{path.gsub(/^\//, '')}")
+    "#{PATH}#{path}"
   end
 
   def api_headers
     {
-      'Accept' => "application/vnd.flying-sphinx-v#{APIVersion}+json",
-      'X-Flying-Sphinx-Token' => "#{identifier}:#{api_key}"
+      'Accept' => "application/vnd.flying-sphinx-v#{VERSION}+json",
+      'X-Flying-Sphinx-Token'   => "#{identifier}:#{api_key}",
+      'X-Flying-Sphinx-Version' => FlyingSphinx::Version
     }
   end
 
   def connection(connection_options = {})
     options = {
       :ssl     => {:verify => false},
-      :url     => (ENV['STAGED_SPHINX_API_KEY'] ? APIStagingServer : APIServer),
+      :url     => (ENV['STAGED_SPHINX_API_KEY'] ? STAGING_SERVER : SERVER),
       :headers => api_headers
     }
 

data/lib/flying_sphinx/configuration.rb

@@ -1,41 +1,17 @@
 class FlyingSphinx::Configuration
-  attr_reader :identifier, :api_key, :host, :port, :database_port, :mem_limit
-
-  FileIndexSettings  = [:stopwords, :wordforms, :exceptions]
-  FileSourceSettings = [:mysql_ssl_cert, :mysql_ssl_key, :mysql_ssl_ca]
-  FileSettings       = FileIndexSettings + FileSourceSettings
+  attr_reader :host, :port, :ssh_server, :database_port
 
   def initialize(identifier = nil, api_key = nil)
     @identifier = identifier || identifier_from_env
     @api_key    = api_key    || api_key_from_env
 
     set_from_server
-    setup_environment_settings
   end
 
   def api
     @api ||= FlyingSphinx::API.new(identifier, api_key)
   end
 
-  def sphinx_configuration
-    thinking_sphinx.generate
-    set_database_settings
-    set_file_settings
-
-    riddle.render
-  end
-
-  def file_setting_pairs(setting)
-    @file_setting_pairs ||= {}
-    @file_setting_pairs[setting] ||= begin
-      pairs = {}
-      file_setting_sources(setting).each_with_index do |source, index|
-        pairs[source] = "#{base_path}/#{setting}/#{index}.txt"
-      end
-      pairs
-    end
-  end
-
   def start_sphinx
     api.post('start').success?
   end
@@ -56,14 +32,16 @@ class FlyingSphinx::Configuration
 
   private
 
+  attr_reader :identifier, :api_key
+
   def set_from_server
     response = api.get '/'
     raise 'Invalid Flying Sphinx credentials' if response.status == 403
 
     @host          = response.body.server
     @port          = response.body.port
+    @ssh_server    = response.body.ssh_server
     @database_port = response.body.database_port
-    @mem_limit     = response.body.mem_limit
   rescue
     # If the central Flying Sphinx server is down, let's use the environment
     # variables so searching is still going to work.
@@ -71,103 +49,6 @@ class FlyingSphinx::Configuration
     @port = port_from_env
   end
 
-  def base_path
-    "/mnt/sphinx/flying-sphinx/#{identifier}"
-  end
-
-  def log_path
-    "#{base_path}/log"
-  end
-
-  def thinking_sphinx
-    ThinkingSphinx::Configuration.instance
-  end
-
-  def riddle
-    thinking_sphinx.configuration
-  end
-
-  def setup_environment_settings
-    ThinkingSphinx.remote_sphinx = true
-
-    set_searchd_settings
-    set_indexer_settings
-    set_path_settings
-  end
-
-  def set_path_settings
-    thinking_sphinx.searchd_file_path = "#{base_path}/indexes"
-
-    riddle.searchd.pid_file  = "#{base_path}/searchd.pid"
-    riddle.searchd.log       = "#{log_path}/searchd.log"
-    riddle.searchd.query_log = "#{log_path}/searchd.query.log"
-  end
-
-  def set_searchd_settings
-    thinking_sphinx.port    = port
-    thinking_sphinx.address = host
-
-    if riddle.searchd.respond_to?(:client_key)
-      riddle.searchd.client_key = client_key
-    end
-  end
-
-  def set_indexer_settings
-    riddle.indexer.mem_limit = mem_limit.to_s + 'M'
-  end
-
-  def set_database_settings
-    return unless FlyingSphinx::Tunnel.required?
-
-    riddle.indices.each do |index|
-      next unless index.respond_to?(:sources)
-
-      index.sources.each do |source|
-        source.sql_host = '127.0.0.1'
-        source.sql_port = database_port
-      end
-    end
-  end
-
-  def set_file_settings
-    riddle.indices.each do |index|
-      set_file_settings_for index, FileIndexSettings
-
-      next unless index.respond_to?(:sources)
-
-      index.sources.each do |source|
-        set_file_settings_for source, FileSourceSettings
-      end
-    end
-  end
-
-  def set_file_settings_for(object, settings)
-    settings.each do |setting|
-      next unless object.respond_to?(setting)
-      object.send "#{setting}=",
-        file_setting_pairs(setting)[object.send(setting)]
-    end
-  end
-
-  def file_setting_sources(setting)
-    @file_setting_sources ||= {}
-    @file_setting_sources[setting] ||= riddle.indices.collect { |index|
-      file_settings_for_index(index, setting)
-    }.flatten.compact.uniq
-  end
-
-  def file_settings_for_index(index, setting)
-    settings = Array(file_setting_for(index, setting))
-    settings += index.sources.collect { |source|
-      file_setting_for(source, setting)
-    } if index.respond_to?(:sources)
-    settings
-  end
-
-  def file_setting_for(object, setting)
-    object.respond_to?(setting) ? object.send(setting) : nil
-  end
-
   def identifier_from_env
     ENV['STAGED_SPHINX_IDENTIFIER'] || ENV['FLYING_SPHINX_IDENTIFIER']
   end

data/lib/flying_sphinx/delayed_delta.rb

@@ -2,19 +2,19 @@ class FlyingSphinx::DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
   # Adds a job to the queue, if it doesn't already exist. This is to ensure
   # multiple indexing requests for the same delta index don't get added, as the
   # index only needs to be processed once.
-  # 
+  #
   # Because indexing jobs are all the same object, they all get serialised to
   # the same YAML value.
-  # 
+  #
   # @param [Object] object The job, which must respond to the #perform method.
   # @param [Integer] priority (0)
-  # 
+  #
   def self.enqueue(object, priority = 0)
     return if duplicates_exist? object
-    
+
     enqueue_without_duplicates_check object, priority
   end
-  
+
   def self.enqueue_without_duplicates_check(object, priority = 0)
     if defined?(Rails) && Rails.version.to_i <= 2
       ::Delayed::Job.enqueue(object, priority)
@@ -22,12 +22,12 @@ class FlyingSphinx::DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
       ::Delayed::Job.enqueue(object, :priority => priority)
     end
   end
-  
+
   # Checks whether a given job already exists in the queue.
-  # 
+  #
   # @param [Object] object The job
   # @return [Boolean] True if a duplicate of the job already exists in the queue
-  # 
+  #
   def self.duplicates_exist?(object)
     ::Delayed::Job.count(
       :conditions => {
@@ -36,16 +36,16 @@ class FlyingSphinx::DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
       }
     ) > 0
   end
-  
+
   # Adds a job to the queue for processing the given model's delta index. A job
   # for hiding the instance in the core index is also created, if an instance is
   # provided.
-  # 
-  # Neither job will be queued if updates or deltas are disabled, or if the 
+  #
+  # Neither job will be queued if updates or deltas are disabled, or if the
   # instance (when given) is not toggled to be in the delta index. The first two
   # options are controlled via ThinkingSphinx.updates_enabled? and
   # ThinkingSphinx.deltas_enabled?.
-  # 
+  #
   # @param [Class] model the ActiveRecord model to index.
   # @param [ActiveRecord::Base] instance the instance of the given model that
   #   has changed. Optional.
@@ -53,38 +53,34 @@ class FlyingSphinx::DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
   #
   def index(model, instance = nil)
     return true if skip? instance
-    
+
     self.class.enqueue(
       FlyingSphinx::IndexRequest.new(model.delta_index_names),
       delayed_job_priority
     )
-    
+
     self.class.enqueue_without_duplicates_check(
       FlyingSphinx::FlagAsDeletedJob.new(
         model.core_index_names, instance.sphinx_document_id
       ),
       delayed_job_priority
     ) if instance
-    
+
     true
   end
-  
+
   private
-  
-  def config
-    @config ||= ThinkingSphinx::Configuration.instance
-  end
-  
+
   def delayed_job_priority
-    config.delayed_job_priority
+    ThinkingSphinx::Configuration.instance.delayed_job_priority
   end
-  
+
   # Checks whether jobs should be enqueued. Only true if updates and deltas are
   # enabled, and the instance (if there is one) is toggled.
-  # 
+  #
   # @param [ActiveRecord::Base, NilClass] instance
   # @return [Boolean]
-  # 
+  #
   def skip?(instance)
     !ThinkingSphinx.updates_enabled? ||
     !ThinkingSphinx.deltas_enabled?  ||

data/lib/flying_sphinx/index_request.rb

@@ -29,8 +29,9 @@ class FlyingSphinx::IndexRequest
   end
 
   def update_and_index
-    update_sphinx_configuration
-    update_sphinx_reference_files
+    FlyingSphinx::SphinxConfiguration.new.upload_to api, tunnel?
+    FlyingSphinx::SettingFiles.new.upload_to api
+
     index
   end
 
@@ -40,7 +41,7 @@ class FlyingSphinx::IndexRequest
     status = request_status
     case status
     when 'FINISHED'
-      'Index Request has completed.'
+      "Index Request has completed:\n#{request_log}"
     when 'FAILED'
       'Index Request failed.'
     when 'PENDING'
@@ -66,25 +67,8 @@ class FlyingSphinx::IndexRequest
     @configuration ||= FlyingSphinx::Configuration.new
   end
 
-  def update_sphinx_configuration
-    api.put '/',
-      :configuration  => configuration.sphinx_configuration,
-      :sphinx_version => ThinkingSphinx::Configuration.instance.version
-  end
-
-  def update_sphinx_reference_files
-    FlyingSphinx::Configuration::FileSettings.each do |setting|
-      configuration.file_setting_pairs(setting).each do |local, remote|
-        api.post '/add_file',
-          :setting   => setting.to_s,
-          :file_name => remote.split('/').last,
-          :content   => open(local).read
-      end
-    end
-  end
-
   def index
-    if FlyingSphinx::Tunnel.required?
+    if tunnel?
       tunnelled_index
     else
       direct_index
@@ -136,8 +120,17 @@ class FlyingSphinx::IndexRequest
     end
   end
 
+  def request_log
+    @request.log
+  end
+
   def request_status
-    api.get("indices/#{index_id}").body.status
+    @request = api.get("indices/#{index_id}").body
+    @request.status
+  end
+
+  def tunnel?
+    FlyingSphinx::Tunnel.required?
   end
 
   def cancel_request

data/lib/flying_sphinx/rails.rb

@@ -4,6 +4,7 @@ if ENV['FLYING_SPHINX_IDENTIFIER'] || ENV['STAGED_SPHINX_IDENTIFIER']
   ActionController::Dispatcher.to_prepare :flying_sphinx do
     config = FlyingSphinx::Configuration.new
 
+    ThinkingSphinx.remote_sphinx = true
     ThinkingSphinx::Configuration.instance.address = config.host
     ThinkingSphinx::Configuration.instance.port    = config.port
     ThinkingSphinx::Configuration.instance.configuration.searchd.client_key =

data/lib/flying_sphinx/railtie.rb

@@ -6,6 +6,7 @@ class FlyingSphinx::Railtie < Rails::Railtie
   initializer "flying_sphinx.set_sphinx_host_and_port" do |app|
     config = FlyingSphinx::Configuration.new
 
+    ThinkingSphinx.remote_sphinx = true
     ThinkingSphinx::Configuration.instance.address = config.host
     ThinkingSphinx::Configuration.instance.port    = config.port
     ThinkingSphinx::Configuration.instance.configuration.searchd.client_key =