flying-sphinx 0.8.5 → 1.0.0

data/Gemfile

@@ -1,3 +1,3 @@
-source :rubygems
+source 'https://rubygems.org'
 
 gemspec

data/HISTORY

@@ -1,3 +1,10 @@
+1.0.0 - 7th May 2012
+* Updating Riddle dependency to >= 1.5.6.
+* Support for Thinking Sphinx v1/2/3.
+* Delayed Job support pushed back to ts-delayed-delta.
+* Updating MultiJson dependency to ensure MultiJson.load is always available.
+* All actions are now tracked through Pusher, instead of polling or using slow HTTP requests.
+
 0.8.5 - 10th December 2012
 * Daemon actions (start/stop) are now asynchronous.
 * More forgiving when environment variables aren't around. Particularly helpful for Padrino and Sinatra.

data/flying-sphinx.gemspec

@@ -17,11 +17,12 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
   s.executables   = ['flying-sphinx']
 
-  s.add_runtime_dependency 'thinking-sphinx',    ['>= 0']
-  s.add_runtime_dependency 'riddle',             ['>= 1.5.0']
-  s.add_runtime_dependency 'multi_json',         ['>= 1.0.1']
+  s.add_runtime_dependency 'thinking-sphinx'
+  s.add_runtime_dependency 'riddle',             ['>= 1.5.6']
+  s.add_runtime_dependency 'multi_json',         ['>= 1.3.0']
   s.add_runtime_dependency 'faraday_middleware', ['~> 0.7']
   s.add_runtime_dependency 'rash',               ['~> 0.3.0']
+  s.add_runtime_dependency 'pusher-client',      ['~> 0.3']
 
   s.add_development_dependency 'rake',            ['~> 0.9.2']
   s.add_development_dependency 'rspec',           ['~> 2.11']
@@ -29,7 +30,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'yajl-ruby',       ['~> 0.8.2']
   s.add_development_dependency 'fakeweb',         ['~> 1.3.0']
   s.add_development_dependency 'fakeweb-matcher', ['~> 1.2.2']
-  s.add_development_dependency 'delayed_job',     ['~> 2.1.4']
 
   s.post_install_message = <<-MESSAGE
 If you're upgrading, you should rebuild your Sphinx setup when deploying:

data/lib/flying_sphinx.rb

@@ -1,28 +1,54 @@
+require 'logger'
+
 module FlyingSphinx
-  #
+  module Translators
+    #
+  end
+
+  @logger       = Logger.new(STDOUT)
+  @logger.level = Logger::INFO
+  if ENV['VERBOSE_LOGGING'] && ENV['VERBOSE_LOGGING'] == 'true'
+    @logger.level = Logger::DEBUG
+  end
+
+  def self.logger
+    @logger
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  def self.translator
+    @translator
+  end
+
+  def self.translator=(translator)
+    @translator = translator
+  end
 end
 
+require 'multi_json'
 require 'faraday'
 require 'faraday_middleware'
 require 'riddle'
 require 'riddle/0.9.9'
+require 'pusher-client'
+
+PusherClient.logger = FlyingSphinx.logger
 
+require 'flying_sphinx/action'
 require 'flying_sphinx/api'
+require 'flying_sphinx/binary'
 require 'flying_sphinx/cli'
 require 'flying_sphinx/configuration'
-require 'flying_sphinx/flag_as_deleted_job'
-require 'flying_sphinx/index_request'
+require 'flying_sphinx/controller'
 require 'flying_sphinx/setting_files'
-require 'flying_sphinx/sphinx_configuration'
+require 'flying_sphinx/sphinxql'
 require 'flying_sphinx/version'
 
-if defined?(ThinkingSphinx)
-  require 'flying_sphinx/delayed_delta'
-  require 'flying_sphinx/heroku_shared_adapter'
-end
-
 if defined?(Rails) && defined?(Rails::Railtie)
   require 'flying_sphinx/railtie'
-elsif defined?(Rails)
+elsif defined?(Rails) && defined?(Rails::Plugin)
   require 'flying_sphinx/rails'
 end

data/lib/flying_sphinx/action.rb

@@ -0,0 +1,90 @@
+require 'timeout'
+
+class FlyingSphinx::Action
+  def self.perform(identifier, timeout = 60, &block)
+    new(identifier, timeout, &block).perform
+  end
+
+  def initialize(identifier, timeout, &block)
+    @identifier, @timeout, @block = identifier, timeout, block
+    @action_id = 0
+    @finished  = false
+  end
+
+  def perform
+    Timeout.timeout(timeout) do
+      socket.connect true
+
+      subscribe_to_events
+
+      sleep 0.5 until socket.connected
+      start
+      sleep 0.5 until finished
+    end
+
+    true
+  rescue Timeout::Error => error
+    FlyingSphinx.logger.warn "Action timed out. If this is happening regularly, please contact Flying Sphinx support: http://support.flying-sphinx.com"
+
+    return false
+  ensure
+    socket.disconnect
+  end
+
+  private
+
+  attr_reader :identifier, :block, :action_id, :finished, :timeout
+
+  def completion(data)
+    FlyingSphinx.logger.debug "Completion: #{data}"
+    data = MultiJson.load(data)
+
+    @finished = (data['id'] == action_id)
+  end
+
+  def debug(data)
+    FlyingSphinx.logger.debug "Progress: #{data}"
+    data = MultiJson.load(data)
+
+    puts data['data'] if data['data'] && data['data'].length > 0
+  end
+
+  def failure(data)
+    FlyingSphinx.logger.debug "Failure: #{data}"
+    data = MultiJson.load(data)
+
+    if data['id'] == action_id
+      FlyingSphinx.logger.warn 'Action failed.'
+      @finished = true
+    end
+  end
+
+  def response
+    attempts = 0
+    @response ||= begin
+      block.call
+    rescue
+      attempts += 1
+      retry if attempts <= 3
+      raise
+    end
+  end
+
+  def socket
+    @socket ||= PusherClient::Socket.new FlyingSphinx::API::PUSHER_KEY,
+      :encrypted => true
+  end
+
+  def start
+    raise "Action blocked" if response.body.status == 'BLOCKED'
+
+    @action_id = response.body.id
+  end
+
+  def subscribe_to_events
+    socket.subscribe identifier
+    socket[identifier].bind 'debug',      &method(:debug)
+    socket[identifier].bind 'completion', &method(:completion)
+    socket[identifier].bind 'failure',    &method(:failure)
+  end
+end

data/lib/flying_sphinx/api.rb

@@ -1,8 +1,16 @@
 class FlyingSphinx::API
-  SERVER         = 'https://flying-sphinx.com'
-  STAGING_SERVER = 'https://staging.flying-sphinx.com'
-  PATH           = "/api/my/app"
-  VERSION        = 3
+  unless ENV['STAGED_SPHINX_API_KEY']
+    SERVER     = 'https://flying-sphinx.com'
+    PUSHER_KEY = 'a8518107ea8a18fe5559'
+  else
+    SERVER     = 'https://staging.flying-sphinx.com'
+    PUSHER_KEY = 'c5602d4909b5144321ce'
+  end
+
+  PATH    = '/api/my/app'
+  VERSION = 4
+
+  attr_reader :api_key, :identifier
 
   def initialize(identifier, api_key, adapter = Faraday.default_adapter)
     @api_key    = api_key
@@ -32,7 +40,7 @@ class FlyingSphinx::API
 
   private
 
-  attr_reader :api_key, :identifier, :adapter
+  attr_reader :adapter
 
   def normalize_path(path)
     path = (path == '/' ? '' : "/#{path.gsub(/^\//, '')}")
@@ -50,7 +58,7 @@ class FlyingSphinx::API
   def connection(connection_options = {})
     options = {
       :ssl     => {:verify => false},
-      :url     => (ENV['STAGED_SPHINX_API_KEY'] ? STAGING_SERVER : SERVER),
+      :url     => SERVER,
       :headers => api_headers
     }
 
@@ -63,20 +71,9 @@ class FlyingSphinx::API
   end
 
   def log(method, path, data = {}, option = {}, &block)
-    return block.call unless log?
-
-    log_message "API Request: #{method} '#{path}'; params: #{data.inspect}"
+    FlyingSphinx.logger.debug "API Request: #{method} '#{path}'; params: #{data.inspect}"
     response = block.call
-    log_message "API Response: #{response.body.inspect}"
+    FlyingSphinx.logger.debug "API Response: #{response.body.inspect}"
     return response
   end
-
-  def log_message(message)
-    time = (Time.respond_to?(:zone) && Time.zone) ? Time.zone.now : Time.now.utc
-    puts "[#{time.to_s}] #{message}"
-  end
-
-  def log?
-    ENV['VERBOSE_LOGGING'] && ENV['VERBOSE_LOGGING'].length > 0
-  end
 end

data/lib/flying_sphinx/binary.rb

@@ -0,0 +1,7 @@
+module FlyingSphinx::Binary
+  def self.load
+    require 'flying_sphinx/binary/translator'
+
+    FlyingSphinx.translator = Translator.new FlyingSphinx::Configuration.new
+  end
+end

data/lib/flying_sphinx/binary/translator.rb

@@ -0,0 +1,48 @@
+class FlyingSphinx::Binary::Translator
+  def initialize(configuration)
+    ThinkingSphinx.remote_sphinx = true
+
+    thinking_sphinx.controller = FlyingSphinx::Controller.new configuration.api
+    thinking_sphinx.address    = configuration.host
+    thinking_sphinx.port       = configuration.port
+    thinking_sphinx.configuration.searchd.client_key =
+      configuration.client_key
+
+    if ENV['DATABASE_URL'] && ENV['DATABASE_URL'][/^mysql/].nil?
+      ThinkingSphinx.database_adapter = Class.new(ThinkingSphinx::PostgreSQLAdapter) do
+        def setup
+          create_array_accum_function
+        end
+      end
+    end
+  end
+
+  def sphinx_configuration
+    @sphinx_configuration ||= begin
+      generate_configuration
+      thinking_sphinx.configuration.searchd.client_key =
+        FlyingSphinx::Configuration.new.client_key
+      thinking_sphinx.configuration.render
+    end
+  end
+
+  def sphinx_indices
+    @sphinx_indices ||= begin
+      generate_configuration
+      thinking_sphinx.configuration.indices
+    end
+  end
+
+  private
+
+  def thinking_sphinx
+    ThinkingSphinx::Configuration.instance
+  end
+
+  def generate_configuration
+    return if @generated_configuration
+
+    thinking_sphinx.generate
+    @generated_configuration = true
+  end
+end

data/lib/flying_sphinx/cli.rb

@@ -1,14 +1,20 @@
+require 'forwardable'
+
 class FlyingSphinx::CLI
+  extend Forwardable
+
   COMMANDS = {
     'configure' => [:configure],
     'index'     => [:index],
     'setup'     => [:configure, :index],
     'start'     => [:start],
     'stop'      => [:stop],
-    'restart'   => [:stop, :start],
-    'rebuild'   => [:stop, :configure, :index, :start]
+    'restart'   => [:restart],
+    'rebuild'   => [:rebuild]
   }
 
+  def_delegators :controller, :start, :stop, :restart
+
   def initialize(command, arguments = [])
     @command, @arguments = command, arguments
   end
@@ -17,7 +23,13 @@ class FlyingSphinx::CLI
     methods = COMMANDS[@command]
     raise "Unknown command #{@command}" if methods.nil?
 
-    methods.all? { |method| send method }
+    methods.all? do |method|
+      FlyingSphinx.logger.info "Executing Action: #{method}"
+      result = send method
+      FlyingSphinx.logger.info "Action Finished: #{method}"
+
+      result
+    end
   end
 
   private
@@ -29,50 +41,45 @@ class FlyingSphinx::CLI
   def configure
     if @arguments.empty?
       load_rails
-      FlyingSphinx::SphinxConfiguration.new.upload_to configuration.api
-      FlyingSphinx::SettingFiles.new.upload_to configuration.api
+
+      controller.configure
     else
-      FlyingSphinx::SphinxConfiguration.new.upload_file_to configuration.api,
-        @arguments.first
+      controller.configure File.read(@arguments.first)
     end
 
-    puts "Sent configuration to Sphinx"
     true
   end
 
-  def index
-    FlyingSphinx::IndexRequest.cancel_jobs
-
-    request = FlyingSphinx::IndexRequest.new @arguments
-    request.index
-    puts request.status_message
+  def controller
+    @controller ||= FlyingSphinx::Controller.new configuration.api
+  end
 
-    true
+  def index
+    indices = @arguments + [{:verbose => true}]
+    controller.index *indices
   end
 
   def load_rails
     return unless ENV['RAILS_ENV']
 
     require File.expand_path('config/boot', Dir.pwd)
-    require File.expand_path('config/application', Dir.pwd)
-    Rails.application.require_environment!
 
-    require 'flying_sphinx/delayed_delta'
-  end
+    if defined?(Rails) && !defined?(Rails::Railtie)
+      require File.expand_path('config/environment', Dir.pwd)
+      require 'flying_sphinx/rails'
 
-  def start
-    if configuration.start_sphinx
-      puts "Started Sphinx"
-      true
+      FlyingSphinx::Binary.load
     else
-      puts "Sphinx failed to start... have you indexed first?"
-      false
+      require File.expand_path('config/application', Dir.pwd)
+      require 'flying_sphinx/railtie'
+
+      Rails.application.require_environment!
     end
   end
 
-  def stop
-    configuration.stop_sphinx
-    puts "Stopped Sphinx"
-    true
+  def rebuild
+    load_rails
+
+    controller.rebuild
   end
 end

data/lib/flying_sphinx/configuration.rb

@@ -1,39 +1,33 @@
 class FlyingSphinx::Configuration
-  attr_reader :host, :port, :ssh_server, :database_port
-
   def initialize(identifier = nil, api_key = nil)
     @identifier = identifier || identifier_from_env
     @api_key    = api_key    || api_key_from_env
-
-    set_from_server
   end
 
   def api
     @api ||= FlyingSphinx::API.new(identifier, api_key)
   end
 
-  def start_sphinx
-    change 'starting', 'started'
+  def client_key
+    "#{identifier}:#{api_key}"
   end
 
-  def stop_sphinx
-    change 'stopping', 'stopped'
+  def host
+    @host ||= response_body.server rescue host_from_env
   end
 
-  def client_key
-    "#{identifier}:#{api_key}"
+  def port
+    @port ||= response_body.port rescue port_from_env
   end
 
-  def output_recent_actions
-    api.get('actions').body.each do |action|
-      puts "#{action.created_at}  #{action.name}"
-    end
+  def username
+    "#{identifier}#{api_key}"
   end
 
   private
 
   attr_reader :identifier, :api_key
-  
+
   def change(initial, expected)
     api.post(initial)
 
@@ -46,19 +40,12 @@ class FlyingSphinx::Configuration
     response.body.status == expected
   end
 
-  def set_from_server
-    response = api.get '/'
-    raise 'Invalid Flying Sphinx credentials' if response.status == 403
-
-    @host          = response.body.server
-    @port          = response.body.port
-    @ssh_server    = response.body.ssh_server
-    @database_port = response.body.database_port
-  rescue
-    # If the central Flying Sphinx server is down, let's use the environment
-    # variables so searching is still going to work.
-    @host = host_from_env
-    @port = port_from_env
+  def response_body
+    @response_body ||= begin
+      response = api.get '/'
+      raise 'Invalid Flying Sphinx credentials' if response.status == 403
+      response.body
+    end
   end
 
   def identifier_from_env
@@ -74,6 +61,6 @@ class FlyingSphinx::Configuration
   end
 
   def port_from_env
-    (ENV['STAGED_SPHINX_PORT'] || ENV['FLYING_SPHINX_PORT'] || 9306).dup
+    (ENV['STAGED_SPHINX_PORT'] || ENV['FLYING_SPHINX_PORT'] || '9306').dup
   end
 end