flying-sphinx 0.4.4 → 0.5.0

data/.gitignore

@@ -0,0 +1,4 @@
+.bundle
+Gemfile.lock
+*.tmproj
+pkg

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/LICENCE

@@ -1,4 +1,4 @@
-Copyright (c) 2010 Pat Allan
+Copyright (c) 2010-2011 Pat Allan
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -0,0 +1,27 @@
+require 'rubygems'
+begin
+  require 'bundler'
+  Bundler::GemHelper.install_tasks
+rescue LoadError
+  puts "Although not required, it's recommended you use bundler during development"
+end
+
+require 'rake'
+
+task :default => :test
+task :test    => :spec
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+RSpec::Core::RakeTask.new(:rspec) do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+  t.rcov = true
+  t.rcov_opts = [
+    '--exclude', 'spec',   '--exclude', 'gems',
+    '--exclude', 'riddle', '--exclude', 'ruby'
+  ]
+end

data/VERSION

@@ -1 +1 @@
-0.4.4
+0.5.0

data/flying-sphinx.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |s|
+  s.name        = 'flying-sphinx'
+  s.version     = '0.5.0'
+  s.authors     = ['Pat Allan']
+  s.email       = 'pat@freelancing-gods.com'
+  s.date        = '2011-05-12'
+  s.summary     = 'Sphinx in the Cloud'
+  s.description = 'Hooks Thinking Sphinx into the Flying Sphinx service'
+  s.homepage    = 'https://flying-sphinx.com'
+
+  s.extra_rdoc_files = ['README.textile']
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ['lib']
+
+  s.rubygems_version = %q{1.3.7}
+
+  s.add_runtime_dependency 'thinking-sphinx',    ['>= 0']
+  s.add_runtime_dependency 'net-ssh',            ['~> 2.0.23']
+  s.add_runtime_dependency 'multi_json',         ['~> 1.0.1']
+  s.add_runtime_dependency 'faraday',            ['~> 0.6.1']
+  s.add_runtime_dependency 'faraday_middleware', ['~> 0.6.3']
+  s.add_runtime_dependency 'rash',               ['~> 0.3.0']
+  
+  s.add_development_dependency 'yajl-ruby',       ['~> 0.8.2']
+  s.add_development_dependency 'rspec',           ['~> 2.5.0']
+  s.add_development_dependency 'rcov',            ['~> 0.9.9']
+  s.add_development_dependency 'fakeweb',         ['~> 1.3.0']
+  s.add_development_dependency 'fakeweb-matcher', ['~> 1.2.2']
+  s.add_development_dependency 'delayed_job',     ['~> 2.1.4']
+  
+  s.post_install_message = <<-MESSAGE
+If you're upgrading, you should rebuild your Sphinx setup when deploying:
+
+  $ heroku rake fs:rebuild
+MESSAGE
+end

data/lib/flying_sphinx.rb

@@ -2,7 +2,6 @@ module FlyingSphinx
   #
 end
 
-require 'httparty'
 require 'net/ssh'
 require 'riddle/0.9.9'
 

data/lib/flying_sphinx/api.rb

@@ -1,33 +1,79 @@
+require 'faraday'
+require 'faraday_middleware'
+
 class FlyingSphinx::API
-  include HTTParty
-  
-  APIServer = 'https://flying-sphinx.com/heroku'
-  
-  attr_reader :api_key, :identifier
-  
-  def initialize(identifier, api_key)
-    @api_key   = api_key
+
+  APIServer  = 'https://flying-sphinx.com'
+  APIPath    = "/api/my/app"
+  APIVersion = 2
+
+  attr_reader :api_key, :identifier, :adapter
+
+  def initialize(identifier, api_key, adapter = Faraday.default_adapter)
+    @api_key    = api_key
     @identifier = identifier
+    @adapter    = adapter
   end
-  
+
   def get(path, data = {})
-    self.class.get "#{APIServer}#{path}", :query => data.merge(api_options)
+    log('GET', path, data) do
+      connection.get do |request|
+        request.url normalize_path(path), data
+      end
+    end
   end
-  
+
   def post(path, data = {})
-    self.class.post "#{APIServer}#{path}", :body => data.merge(api_options)
+    log('POST', path, data) do
+      connection.post normalize_path(path), data
+    end
   end
-  
+
   def put(path, data = {})
-    self.class.put "#{APIServer}#{path}", :body => data.merge(api_options)
+    log('PUT', path, data) do
+      connection.put normalize_path(path), data
+    end
   end
-  
+
   private
-  
-  def api_options
+
+  def normalize_path(path)
+    path = (path == '/' ? nil : "/#{path}")
+    "#{APIPath}#{path}"
+  end
+
+  def api_headers
     {
-      :api_key   => api_key,
-      :identifier => identifier
+      'Accept' => "application/vnd.flying-sphinx-v#{APIVersion}+json",
+      'X-Flying-Sphinx-Token' => "#{identifier}:#{api_key}"
+    }
+  end
+
+  def connection(connection_options = {})
+    options = {
+      :ssl     => {:verify => false},
+      :url     => APIServer,
+      :headers => api_headers
     }
+
+    Faraday.new(options) do |builder|
+      builder.use Faraday::Request::UrlEncoded
+      builder.use Faraday::Response::Rashify
+      builder.use Faraday::Response::ParseJson
+      builder.adapter(adapter)
+    end
+  end
+
+  def log(method, path, data = {}, option = {}, &block)
+    return block.call unless log?
+    
+    puts "API Request: #{method} '#{path}'; params: #{data.inspect}"
+    response = block.call
+    puts "API Response: #{response.body.to_hash.inspect}"
+    return response
+  end
+  
+  def log?
+    ENV['VERBOSE_LOGGING'].present?
   end
 end

data/lib/flying_sphinx/configuration.rb

@@ -1,104 +1,131 @@
 class FlyingSphinx::Configuration
   attr_reader :identifier, :api_key, :host, :port, :database_port, :mem_limit
-  
+
   def initialize(identifier = nil, api_key = nil)
     @identifier = identifier || identifier_from_env
     @api_key    = api_key    || api_key_from_env
-    
+
     set_from_server
     setup_environment_settings
   end
-  
+
   def api
     @api ||= FlyingSphinx::API.new(identifier, api_key)
   end
-  
+
   def sphinx_configuration
     thinking_sphinx.generate
     set_database_settings
-    
+
     riddle.render
   end
-  
+
   def start_sphinx
-    api.post('/app/start')
+    api.post('start')
   end
-  
+
   def stop_sphinx
-    api.post('/app/stop')
+    api.post('stop')
   end
   
-  private
-  
-  def set_from_server
-    response = api.get('/app')
-    raise 'Invalid Flying Sphinx credentials' if response.code == 403
-    
-    json = JSON.parse response.body
-    
-    @host          = json['server']
-    @port          = json['port']
-    @database_port = json['database_port']
-    @mem_limit     = json['mem_limit']
+  def client_key
+    "#{identifier}:#{api_key}"
+  end
+
+  def output_recent_actions
+    api.get('actions').body.each do |action|
+      puts "#{action.created_at}  #{action.name}"
+    end
   end
   
+  private
+
+  def set_from_server
+    response = api.get '/'
+    raise 'Invalid Flying Sphinx credentials' if response.status == 403
+
+    @host          = response.body.server
+    @port          = response.body.port
+    @database_port = response.body.database_port
+    @mem_limit     = response.body.mem_limit
+  rescue
+    # If the central Flying Sphinx server is down, let's use the environment
+    # variables so searching is still going to work.
+    @host = host_from_env
+    @port = port_from_env
+  end
+
   def base_path
     "/mnt/sphinx/flying-sphinx/#{identifier}"
   end
-  
+
   def log_path
     "#{base_path}/log"
   end
-  
+
   def thinking_sphinx
     ThinkingSphinx::Configuration.instance
   end
-  
+
   def riddle
     thinking_sphinx.configuration
   end
-  
+
   def setup_environment_settings
     ThinkingSphinx.remote_sphinx = true
-    
+
     set_searchd_settings
     set_indexer_settings
     set_path_settings
   end
-  
+
   def set_path_settings
     thinking_sphinx.searchd_file_path = "#{base_path}/indexes"
-    
+
     riddle.searchd.pid_file  = "#{base_path}/searchd.pid"
     riddle.searchd.log       = "#{log_path}/searchd.log"
     riddle.searchd.query_log = "#{log_path}/searchd.query.log"
   end
-  
+
   def set_searchd_settings
     thinking_sphinx.port    = port
     thinking_sphinx.address = host
+    
+    if riddle.searchd.respond_to?(:client_key)
+      riddle.searchd.client_key = client_key
+    end
   end
-  
+
   def set_indexer_settings
     riddle.indexer.mem_limit = mem_limit.to_s + 'M'
   end
-  
+
   def set_database_settings
+    return unless FlyingSphinx::Tunnel.required?
+    
     riddle.indexes.each do |index|
       next unless index.respond_to?(:sources)
-      
+
       index.sources.each do |source|
         source.sql_host = '127.0.0.1'
         source.sql_port = database_port
       end
     end
   end
-  
+
   def identifier_from_env
     ENV['FLYING_SPHINX_IDENTIFIER']
   end
-  
+
   def api_key_from_env
     ENV['FLYING_SPHINX_API_KEY']
   end
+  
+  def host_from_env
+    ENV['FLYING_SPHINX_HOST']
+  end
+  
+  def port_from_env
+    ENV['FLYING_SPHINX_PORT']
+  end
 end

data/lib/flying_sphinx/delayed_delta.rb

@@ -12,7 +12,11 @@ class FlyingSphinx::DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
   def self.enqueue(object, priority = 0)
     return if duplicates_exist? object
     
-    ::Delayed::Job.enqueue(object, :priority => priority)
+    if defined?(Rails) && Rails.version.to_i <= 2
+      ::Delayed::Job.enqueue(object, priority)
+    else
+      ::Delayed::Job.enqueue(object, :priority => priority)
+    end
   end
   
   # Checks whether a given job already exists in the queue.

data/lib/flying_sphinx/index_request.rb

@@ -1,67 +1,107 @@
 class FlyingSphinx::IndexRequest
   attr_reader :index_id, :indices
-  
+
+  INDEX_COMPLETE_CHECKING_INTERVAL = 3
+
   # Remove all Delta jobs from the queue. If the
   # delayed_jobs table does not exist, this method will do nothing.
-  # 
+  #
   def self.cancel_jobs
     return unless defined?(::Delayed) && ::Delayed::Job.table_exists?
-    
+
     ::Delayed::Job.delete_all "handler LIKE '--- !ruby/object:FlyingSphinx::%'"
   end
-  
+
+  def self.output_last_index
+    index = FlyingSphinx::Configuration.new.api.get('indices/last').body
+    puts "Index Job Status: #{index.status}"
+    puts "Index Log:\n#{index.log}"
+  end
+
   def initialize(indices = [])
     @indices = indices
   end
-  
+
   # Shows index name in Delayed::Job#name.
-  # 
+  #
   def display_name
     "#{self.class.name} for #{indices.join(', ')}"
   end
-  
+
   def update_and_index
     update_sphinx_configuration
     index
   end
   
+  def status_message
+    status = request_status
+    case status
+    when 'FINISHED'
+      'Index Request has completed.'
+    when 'FAILED'
+      'Index Request failed.'
+    when 'PENDING'
+      'Index Request is still pending - something has gone wrong.'
+    else
+      "Unknown index response: '#{status}'."
+    end
+  end
+
   # Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx is
   # running.
-  # 
+  #
   # @return [Boolean] true
-  # 
+  #
   def perform
     index
     true
   end
-  
+
   private
-  
+
   def configuration
     @configuration ||= FlyingSphinx::Configuration.new
   end
-  
+
   def update_sphinx_configuration
-    api.put '/app', :configuration => configuration.sphinx_configuration
+    api.put('/', :configuration => configuration.sphinx_configuration)
   end
-  
+
   def index
-    FlyingSphinx::Tunnel.connect(configuration) do
-      begin_request unless request_begun?
-      
-      !request_complete?
+    if FlyingSphinx::Tunnel.required?
+      tunnelled_index
+    else
+      direct_index
     end
   rescue Net::SSH::Exception
-    cancel_request
+    # Server closed the connection on us. That's (hopefully) expected, nothing
+    # to worry about.
   rescue RuntimeError => err
     puts err.message
   end
   
+  def tunnelled_index
+    FlyingSphinx::Tunnel.connect(configuration) do
+      begin_request unless request_begun?
+    
+      true
+    end
+  end
+  
+  def direct_index
+    begin_request
+    while !request_complete?
+      sleep 1
+    end
+  end
+  
   def begin_request
-    @index_id      = api.post('/app/indices', :indices => indices.join(','))
+    response = api.post 'indices', :indices => indices.join(',')
+
+    @index_id = response.body.id
     @request_begun = true
-    
-    raise RuntimeError, 'Your account does not support delta indexing. Upgrading plans is probably the best way around this.' if @index_id == 'BLOCKED'
+
+    raise RuntimeError, 'Your account does not support delta indexing. Upgrading plans is probably the best way around this.' if response.body.status == 'BLOCKED'
   end
   
   def request_begun?
@@ -69,10 +109,8 @@ class FlyingSphinx::IndexRequest
   end
   
   def request_complete?
-    response = api.get("/app/indices/#{index_id}")
-    case response.body
+    case request_status
     when 'FINISHED', 'FAILED'
-      puts "Indexing request failed." if response.body == 'FAILED'
       true
     when 'PENDING'
       false
@@ -81,14 +119,19 @@ class FlyingSphinx::IndexRequest
     end
   end
   
+  def request_status
+    api.get("indices/#{index_id}").body.status
+  end
+
   def cancel_request
     return if index_id.nil?
-    
+
     puts "Connecting Flying Sphinx to the Database failed"
     puts "Cancelling Index Request..."
-    api.put "/app/indices/#{index_id}", :status => 'CANCELLED'
+
+    api.put("indices/#{index_id}", :status => 'CANCELLED')
   end
-  
+
   def api
     configuration.api
   end