flying-sphinx 0.6.6 → 0.7.0

data/lib/flying_sphinx/setting_files.rb

@@ -0,0 +1,55 @@
+class FlyingSphinx::SettingFiles
+  INDEX_SETTINGS  = [:stopwords, :wordforms, :exceptions]
+  SOURCE_SETTINGS = [:mysql_ssl_cert, :mysql_ssl_key, :mysql_ssl_ca]
+
+  def initialize(indices = nil)
+    @indices = indices ||
+      ThinkingSphinx::Configuration.instance.configuration.indices
+  end
+
+  def upload_to(api)
+    each_file_for_setting do |setting, file|
+      api.post '/add_file',
+        :setting   => setting.to_s,
+        :file_name => File.basename(file),
+        :content   => File.read(file)
+    end
+  end
+
+  private
+
+  attr_reader :indices
+
+  def each_file_for_setting(&block)
+    index_settings  &block
+    source_settings &block
+  end
+
+  def index_settings(&block)
+    settings_in_list_from_collection INDEX_SETTINGS, indices, &block
+  end
+
+  def sources
+    @sources ||= indices.collect { |index|
+      index.respond_to?(:sources) ? index.sources : []
+    }.flatten
+  end
+
+  def source_settings(&block)
+    settings_in_list_from_collection SOURCE_SETTINGS, sources, &block
+  end
+
+  def setting_from(collection, setting)
+    collection.collect { |object|
+      object.respond_to?(setting) ? object.send(setting).to_s.split(' ') : []
+    }.flatten.uniq.compact
+  end
+
+  def settings_in_list_from_collection(settings, collection, &block)
+    settings.each do |setting|
+      setting_from(collection, setting).each { |file|
+        block.call setting, file
+      }
+    end
+  end
+end

data/lib/flying_sphinx/sphinx_configuration.rb

@@ -0,0 +1,23 @@
+class FlyingSphinx::SphinxConfiguration
+  def initialize(thinking_sphinx = ThinkingSphinx::Configuration.instance)
+    @thinking_sphinx = thinking_sphinx
+  end
+
+  def upload_to(api, tunnel = false)
+    api.put '/',
+      :configuration  => content,
+      :sphinx_version => thinking_sphinx.version,
+      :tunnel         => tunnel.to_s
+  end
+
+  private
+
+  attr_reader :thinking_sphinx
+
+  def content
+    @content ||= begin
+      thinking_sphinx.generate
+      thinking_sphinx.configuration.render
+    end
+  end
+end

data/lib/flying_sphinx/tasks.rb

@@ -5,7 +5,6 @@ namespace :fs do
     request = FlyingSphinx::IndexRequest.new
     request.update_and_index
     puts request.status_message
-
   end
 
   task :start => :environment do

data/lib/flying_sphinx/tunnel.rb

@@ -18,7 +18,7 @@ class FlyingSphinx::Tunnel
   end
 
   def open(&block)
-    session = Net::SSH.start(@configuration.host, 'sphinx', ssh_options)
+    session = Net::SSH.start(@configuration.ssh_server, 'sphinx', ssh_options)
     session.forward.remote(
       db_port, db_host, @configuration.database_port, '0.0.0.0'
     )

data/lib/flying_sphinx/version.rb

@@ -1,3 +1,3 @@
 module FlyingSphinx
-  Version = '0.6.6'
+  Version = '0.7.0'
 end

data/spec/light_spec_helper.rb

@@ -0,0 +1,7 @@
+require 'bundler/setup'
+require 'rspec/fire'
+
+RSpec.configure do |config|
+  config.mock_with :rspec
+  config.include(RSpec::Fire)
+end

data/spec/spec_helper.rb

@@ -1,12 +1,7 @@
 require 'rubygems'
-begin
-  require 'bundler'
-rescue LoadError
-  puts "although not required, it's recommended you use bundler during development"
-end
+require 'bundler'
 
 require 'timeout'
-
 require 'thinking-sphinx'
 require 'flying_sphinx'
 require 'delayed_job'
@@ -15,9 +10,4 @@ require 'fakeweb'
 require 'fakeweb_matcher'
 
 FakeWeb.allow_net_connect = false
-
-Delayed::Worker.backend = :active_record
-
-# we don't want a checking of interval in testing
-FlyingSphinx::IndexRequest.send(:remove_const, :INDEX_COMPLETE_CHECKING_INTERVAL)
-FlyingSphinx::IndexRequest::INDEX_COMPLETE_CHECKING_INTERVAL = 0
+Delayed::Worker.backend   = :active_record

data/spec/specs/api_spec.rb

@@ -0,0 +1,92 @@
+require 'light_spec_helper'
+require 'flying_sphinx/api'
+require 'flying_sphinx/version'
+
+describe FlyingSphinx::API do
+  let(:api)        { FlyingSphinx::API.new 'foo', 'bar', adapter }
+  let(:faraday)    { fire_class_double('Faraday', :new => connection) }
+  let(:adapter)    { double('adapter') }
+  let(:connection) { double('connection') }
+
+  before :each do
+    faraday.as_replaced_constant
+  end
+
+  shared_examples_for 'an API call' do
+    it "sets up a connection with the appropriate headers" do
+      faraday.should_receive(:new) do |options|
+        options[:headers].should == {
+          'Accept'                  => 'application/vnd.flying-sphinx-v3+json',
+          'X-Flying-Sphinx-Token'   => 'foo:bar',
+          'X-Flying-Sphinx-Version' => FlyingSphinx::Version
+        }
+
+        connection
+      end
+
+      send_request
+    end
+
+    it "connects to flying-sphinx.com with SSL" do
+      faraday.should_receive(:new) do |options|
+        options[:url].should == 'https://flying-sphinx.com'
+
+        connection
+      end
+
+      send_request
+    end
+  end
+
+  describe '#get' do
+    let(:request)      { double('request', :url => true) }
+    let(:send_request) { api.get '/resource', 'param' => 'value'}
+
+    before :each do
+      connection.stub(:get).and_yield(request)
+    end
+
+    it_should_behave_like 'an API call'
+
+    it "sends the GET request with the given path and data" do
+      request.should_receive(:url).
+        with('/api/my/app/resource', 'param' => 'value')
+
+      send_request
+    end
+  end
+
+  describe '#post' do
+    let(:send_request) { api.post '/resource', 'param' => 'value' }
+
+    before :each do
+      connection.stub(:post => true)
+    end
+
+    it_should_behave_like 'an API call'
+
+    it "sends the POST request with the given path and data" do
+      connection.should_receive(:post).
+        with('/api/my/app/resource', 'param' => 'value')
+
+      send_request
+    end
+  end
+
+  describe '#put' do
+    let(:send_request) { api.put '/resource', 'param' => 'value' }
+
+    before :each do
+      connection.stub(:put => true)
+    end
+
+    it_should_behave_like 'an API call'
+
+    it "sends the PUT request with the given path and data" do
+      connection.should_receive(:put).
+        with('/api/my/app/resource', 'param' => 'value')
+
+      send_request
+    end
+  end
+end

data/spec/specs/configuration_spec.rb

@@ -1,184 +1,35 @@
-require 'spec_helper'
-require 'multi_json'
+require 'light_spec_helper'
+require 'flying_sphinx/configuration'
 
 describe FlyingSphinx::Configuration do
-  let(:api_server) { 'https://flying-sphinx.com/api/my' }
+  let(:api)  { fire_double('FlyingSphinx::API',
+    :get => double(:body => body, :status => 200)) }
+  let(:body) { double(:server => 'foo.bar.com', :port => 9319,
+    :ssh_server => 'ssh.bar.com', :database_port => 10319) }
 
   before :each do
-    FakeWeb.register_uri(:get, "#{api_server}/app",
-      :body => MultiJson.encode(
-        :server        => 'foo.bar.com',
-        :port          => 9319,
-        :database_port => 10001
-      )
-    )
+    fire_class_double('FlyingSphinx::API', :new => api).as_replaced_constant
   end
 
   describe '#initialize' do
     let(:api_key)    { 'foo-bar-baz' }
-    let(:identifier) { 'app@heroku.com' }
+    let(:identifier) { 'my-identifier' }
     let(:config)     { FlyingSphinx::Configuration.new identifier, api_key }
 
-    it "requests details from the server with the given API key" do
-      config
-      FakeWeb.should have_requested :get, "#{api_server}/app"
-    end
-
-    it "sets the host from the server information" do
+    it "sets the host from the API information" do
       config.host.should == 'foo.bar.com'
     end
 
-    it "sets the port from the server information" do
+    it "sets the port from the API information" do
       config.port.should == 9319
     end
 
-    it "sets the port from the server information" do
-      config.database_port.should == 10001
+    it "sets the ssh server address" do
+      config.ssh_server.should == 'ssh.bar.com'
     end
-  end
-
-  describe '#file_setting_pairs' do
-    let(:config)    { FlyingSphinx::Configuration.new 'ident' }
-    let(:riddle)    { ThinkingSphinx::Configuration.instance.configuration }
-    let(:base_path) { '/mnt/sphinx/flying-sphinx/ident' }
-
-    context 'index setting' do
-      it "pairs each local file path to a server file path" do
-        riddle.stub! :indices => [
-          double('index', :wordforms => '/path/to/wordforms-foo.txt',
-            :sources => [double('source'), double('source')]),
-          double('index', :wordforms => '/path/to/wordforms-bar.txt',
-            :sources => [double('source'), double('source')])
-        ]
-
-        config.file_setting_pairs(:wordforms).should == {
-          '/path/to/wordforms-foo.txt' => "#{base_path}/wordforms/0.txt",
-          '/path/to/wordforms-bar.txt' => "#{base_path}/wordforms/1.txt"
-        }
-      end
-
-      it "doesn't duplicate multiple references to the same local file" do
-        riddle.stub! :indices => [
-          double('index', :wordforms => '/path/to/wordforms-foo.txt',
-            :sources => [double('source'), double('source')]),
-          double('index', :wordforms => '/path/to/wordforms-foo.txt',
-            :sources => [double('source'), double('source')])
-        ]
-
-        config.file_setting_pairs(:wordforms).should == {
-          '/path/to/wordforms-foo.txt' => "#{base_path}/wordforms/0.txt"
-        }
-      end
-    end
-
-    context 'source setting' do
-      it "pairs each local file path to a server file path" do
-        riddle.stub! :indices => [
-          double('index', :sources => [
-            double('source', :mysql_ssl_cert => '/path/to/cert-foo.txt'),
-            double('source', :mysql_ssl_cert => '/path/to/cert-bar.txt')
-          ]),
-          double('index', :sources => [
-            double('source', :mysql_ssl_cert => '/path/to/cert-baz.txt'),
-            double('source', :mysql_ssl_cert => nil)
-          ])
-        ]
-
-        config.file_setting_pairs(:mysql_ssl_cert).should == {
-          '/path/to/cert-foo.txt' => "#{base_path}/mysql_ssl_cert/0.txt",
-          '/path/to/cert-bar.txt' => "#{base_path}/mysql_ssl_cert/1.txt",
-          '/path/to/cert-baz.txt' => "#{base_path}/mysql_ssl_cert/2.txt"
-        }
-      end
-
-      it "doesn't duplicate multiple references to the same local file" do
-        riddle.stub! :indices => [
-          double('index', :sources => [
-            double('source', :mysql_ssl_cert => '/path/to/cert-foo.txt'),
-            double('source', :mysql_ssl_cert => '/path/to/cert-bar.txt')
-          ]),
-          double('index', :sources => [
-            double('source', :mysql_ssl_cert => '/path/to/cert-foo.txt'),
-            double('source', :mysql_ssl_cert => nil)
-          ])
-        ]
-
-        config.file_setting_pairs(:mysql_ssl_cert).should == {
-          '/path/to/cert-foo.txt' => "#{base_path}/mysql_ssl_cert/0.txt",
-          '/path/to/cert-bar.txt' => "#{base_path}/mysql_ssl_cert/1.txt"
-        }
-      end
-    end
-  end
-
-  describe '#sphinx_configuration' do
-    let(:config)    { FlyingSphinx::Configuration.new 'ident' }
-    let(:riddle)    { double('riddle configuration').as_null_object }
-    let(:base_path) { '/mnt/sphinx/flying-sphinx/ident' }
-    let(:source)    { double('source') }
-
-    before :each do
-      ThinkingSphinx::Configuration.instance.stub!(
-        :generate      => nil,
-        :configuration => riddle
-      )
-      FlyingSphinx::Tunnel.stub! :required? => false
-    end
-
-    it "sets database settings to match Flying Sphinx port forward" do
-      FlyingSphinx::Tunnel.stub! :required? => true
-
-      riddle.stub! :indices => [
-        double('distributed index'),
-        double('index', :sources => [source])
-      ]
-
-      source.should_receive(:sql_host=).with('127.0.0.1')
-      source.should_receive(:sql_port=).with(10001)
-
-      config.sphinx_configuration
-    end
-
-    it "sets file path to match server directories for index settings" do
-      riddle.stub! :indices => [
-        double('index', :wordforms => '/path/to/wordforms-foo.txt',
-          :sources => [double('source'), double('source')]),
-        double('index', :wordforms => '/path/to/wordforms-bar.txt',
-          :sources => [double('source'), double('source')]),
-        double('distributed index')
-      ]
-
-      riddle.indices[0].should_receive(:wordforms=).
-        with("#{base_path}/wordforms/0.txt")
-      riddle.indices[1].should_receive(:wordforms=).
-        with("#{base_path}/wordforms/1.txt")
-
-      config.sphinx_configuration
-    end
-
-    it "sets file path to match server directories for source settings" do
-      riddle.stub! :indices => [
-        double('index', :sources => [
-          double('source', :mysql_ssl_cert => '/path/to/cert-foo.txt'),
-          double('source', :mysql_ssl_cert => '/path/to/cert-bar.txt')
-        ]),
-        double('index', :sources => [
-          double('source', :mysql_ssl_cert => '/path/to/cert-baz.txt'),
-          double('source', :mysql_ssl_cert => nil)
-        ]),
-        double('distributed index')
-      ]
-
-      riddle.indices[0].sources[0].should_receive(:mysql_ssl_cert=).
-        with("#{base_path}/mysql_ssl_cert/0.txt")
-      riddle.indices[0].sources[1].should_receive(:mysql_ssl_cert=).
-        with("#{base_path}/mysql_ssl_cert/1.txt")
-      riddle.indices[1].sources[0].should_receive(:mysql_ssl_cert=).
-        with("#{base_path}/mysql_ssl_cert/2.txt")
-      riddle.indices[1].sources[1].should_receive(:mysql_ssl_cert=).
-        with(nil)
 
-      config.sphinx_configuration
+    it "sets the database port" do
+      config.database_port.should == 10319
     end
   end
 end

data/spec/specs/index_request_spec.rb

@@ -1,11 +1,13 @@
-require 'spec_helper'
+require 'light_spec_helper'
+require 'net/ssh/errors'
+require 'timeout'
+require 'flying_sphinx/index_request'
 
 describe FlyingSphinx::IndexRequest do
-  let(:api)           { FlyingSphinx::API.new 'foo', 'bar' }
-  let(:configuration) {
-    stub(:configuration, :api => api, :sphinx_configuration => 'foo {}',
-      :file_setting_pairs => {})
-  }
+  let(:api)           { fire_double('FlyingSphinx::API') }
+  let(:configuration) { stub(:configuration, :api => api) }
+  let(:tunnel_class)  { fire_class_double('FlyingSphinx::Tunnel').
+    as_replaced_constant }
 
   let(:index_response)    {
     stub(:response, :body => stub(:body, :id => 42, :status => 'OK'))
@@ -15,34 +17,39 @@ describe FlyingSphinx::IndexRequest do
   }
 
   before :each do
-    ThinkingSphinx.database_adapter = FlyingSphinx::HerokuSharedAdapter
+    stub_const 'FlyingSphinx::Configuration', double(:new => configuration)
 
-    FlyingSphinx::Configuration.stub!(:new => configuration)
-    FlyingSphinx::Tunnel.stub(:connect) { |config, block| block.call }
+    stub_const 'FlyingSphinx::IndexRequest::INDEX_COMPLETE_CHECKING_INTERVAL',
+      0
   end
 
   describe '.cancel_jobs' do
+    let(:job_class) { fire_class_double('Delayed::Job').as_replaced_constant }
+
     before :each do
-      Delayed::Job.stub!(:delete_all => true)
+      job_class.stub :delete_all => true
     end
 
     it "should not delete any rows if the delayed_jobs table does not exist" do
-      Delayed::Job.stub!(:table_exists? => false)
-      Delayed::Job.should_not_receive(:delete_all)
+      job_class.stub :table_exists? => false
+
+      job_class.should_not_receive(:delete_all)
 
       FlyingSphinx::IndexRequest.cancel_jobs
     end
 
     it "should delete rows if the delayed_jobs table does exist" do
-      Delayed::Job.stub!(:table_exists? => true)
-      Delayed::Job.should_receive(:delete_all)
+      job_class.stub :table_exists? => true
+
+      job_class.should_receive(:delete_all)
 
       FlyingSphinx::IndexRequest.cancel_jobs
     end
 
     it "should delete only Thinking Sphinx jobs" do
-      Delayed::Job.stub!(:table_exists? => true)
-      Delayed::Job.should_receive(:delete_all) do |sql|
+      job_class.stub :table_exists? => true
+
+      job_class.should_receive(:delete_all) do |sql|
         sql.should match(/handler LIKE '--- !ruby\/object:FlyingSphinx::\%'/)
       end
 
@@ -55,16 +62,23 @@ describe FlyingSphinx::IndexRequest do
     let(:conf_params)   { { :configuration => 'foo {}',
       :sphinx_version => '2.1.0-dev' } }
     let(:index_params)  { { :indices => '' } }
-    let(:ts_config)     { double('config', :version => '2.1.0-dev') }
+    let(:sphinx)        { fire_double('FlyingSphinx::SphinxConfiguration',
+      :upload_to => true)}
+    let(:setting_files) { fire_double('FlyingSphinx::SettingFiles',
+      :upload_to => true) }
 
     before :each do
-      ThinkingSphinx::Configuration.stub :instance => ts_config
+      stub_const 'FlyingSphinx::SettingFiles', double(:new => setting_files)
+      stub_const 'FlyingSphinx::SphinxConfiguration', double(:new => sphinx)
+
+      api.stub :post => index_response
+
+      tunnel_class.stub :required? => true
+      tunnel_class.stub(:connect).and_yield
     end
 
-    it "makes a new request" do
-      api.should_receive(:put).with('/', conf_params).and_return('ok')
-      api.should_receive(:post).
-        with('indices', index_params).and_return(index_response)
+    it "uploads the configuration file" do
+      sphinx.should_receive(:upload_to).with(api, true)
 
       begin
         Timeout::timeout(0.2) {
@@ -74,44 +88,31 @@ describe FlyingSphinx::IndexRequest do
       end
     end
 
-    [
-      :stopwords, :wordforms, :exceptions, :mysql_ssl_cert, :mysql_ssl_key,
-      :mysql_ssl_ca
-    ].each do |setting|
-      context "with #{setting}" do
-        let(:file_params) {
-          {:setting => setting.to_s, :file_name => 'bar.txt', :content => 'baz'}
+    it "uploads setting files" do
+      setting_files.should_receive(:upload_to).with(api)
+
+      begin
+        Timeout::timeout(0.2) {
+          index_request.update_and_index
         }
+      rescue Timeout::Error
+      end
+    end
+
+    it "makes a new request" do
+      api.should_receive(:post).
+        with('indices', index_params).and_return(index_response)
 
-        before :each do
-          configuration.stub!(:file_setting_pairs => {})
-          index_request.stub!(:open => double('file', :read => 'baz'))
-        end
-
-        it "sends the #{setting} file" do
-          api.should_receive(:put).with('/', conf_params).and_return('ok')
-          api.should_receive(:post).with('/add_file', file_params).
-            and_return('ok')
-          api.should_receive(:post).
-            with('indices', index_params).and_return(index_response)
-
-          configuration.should_receive(:file_setting_pairs).
-            with(setting).and_return({'foo.txt' => 'bar.txt'})
-
-          begin
-            Timeout::timeout(0.2) {
-              index_request.update_and_index
-            }
-          rescue Timeout::Error
-          end
-        end
+      begin
+        Timeout::timeout(0.2) {
+          index_request.update_and_index
+        }
+      rescue Timeout::Error
       end
     end
 
     context 'delta request without delta support' do
       it "should explain why the request failed" do
-        api.should_receive(:put).
-          with('/', conf_params).and_return('ok')
         api.should_receive(:post).
           with('indices', index_params).and_return(blocked_response)
         index_request.should_receive(:puts).
@@ -123,17 +124,12 @@ describe FlyingSphinx::IndexRequest do
 
     context 'request for a MySQL database' do
       before :each do
-        ThinkingSphinx.database_adapter = nil
-      end
-
-      after :each do
-        ThinkingSphinx.database_adapter = FlyingSphinx::HerokuSharedAdapter
+        tunnel_class.stub :required? => false
       end
 
       it "should not establish an SSH connection" do
         FlyingSphinx::Tunnel.should_not_receive(:connect)
 
-        api.should_receive(:put).with('/', conf_params).and_return('ok')
         api.should_receive(:post).
           with('indices', index_params).and_return(index_response)
         api.should_receive(:get).with('indices/42').
@@ -148,6 +144,11 @@ describe FlyingSphinx::IndexRequest do
     let(:index_request) { FlyingSphinx::IndexRequest.new ['foo_delta'] }
     let(:index_params)  { { :indices => 'foo_delta' } }
 
+    before :each do
+      tunnel_class.stub :required? => true
+      tunnel_class.stub(:connect).and_yield
+    end
+
     it "makes a new request" do
       api.should_receive(:post).
         with('indices', index_params).and_return(index_response)
@@ -164,7 +165,8 @@ describe FlyingSphinx::IndexRequest do
   describe '#status_message' do
     let(:index_request)     { FlyingSphinx::IndexRequest.new }
     let(:finished_response) {
-      stub(:response, :body => stub(:body, :status => 'FINISHED'))
+      stub(:response, :body => stub(:body, :status => 'FINISHED',
+        :log => 'Sphinx log'))
     }
     let(:failure_response)  {
       stub(:response, :body => stub(:body, :status => 'FAILED'))
@@ -185,7 +187,7 @@ describe FlyingSphinx::IndexRequest do
     it "returns with a positive message on success" do
       api.stub(:get => finished_response)
 
-      index_request.status_message.should == 'Index Request has completed.'
+      index_request.status_message.should == "Index Request has completed:\nSphinx log"
     end
 
     it "returns with a failure message on failure" do