flying-sphinx 0.5.2 → 0.6.0

data/HISTORY

@@ -1,3 +1,7 @@
+0.6.0 - 31st July 2011
+* Version in a separate file.
+* Support for all file-based Sphinx settings: stopwords, wordforms, exceptions, and mysql ssl settings for SQL sources.
+
 0.5.2 - 28th July 2011
 * Log SSH exceptions during indexing (when we're being verbose).
 * This history file now exists (pre-populated).

data/flying-sphinx.gemspec

@@ -1,7 +1,10 @@
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+require 'flying_sphinx/version'
+
 Gem::Specification.new do |s|
   s.name        = 'flying-sphinx'
-  s.version     = '0.5.2'
+  s.version     = FlyingSphinx::Version
   s.authors     = ['Pat Allan']
   s.email       = 'pat@freelancing-gods.com'
   s.summary     = 'Sphinx in the Cloud'

data/lib/flying_sphinx.rb

@@ -12,6 +12,7 @@ require 'flying_sphinx/flag_as_deleted_job'
 require 'flying_sphinx/heroku_shared_adapter'
 require 'flying_sphinx/index_request'
 require 'flying_sphinx/tunnel'
+require 'flying_sphinx/version'
 
 if defined?(Rails) && defined?(Rails::Railtie)
   require 'flying_sphinx/railtie'

data/lib/flying_sphinx/configuration.rb

@@ -1,5 +1,9 @@
 class FlyingSphinx::Configuration
   attr_reader :identifier, :api_key, :host, :port, :database_port, :mem_limit
+  
+  FileIndexSettings  = [:stopwords, :wordforms, :exceptions]
+  FileSourceSettings = [:mysql_ssl_cert, :mysql_ssl_key, :mysql_ssl_ca]
+  FileSettings       = FileIndexSettings + FileSourceSettings
 
   def initialize(identifier = nil, api_key = nil)
     @identifier = identifier || identifier_from_env
@@ -16,16 +20,17 @@ class FlyingSphinx::Configuration
   def sphinx_configuration
     thinking_sphinx.generate
     set_database_settings
-    set_wordforms
+    set_file_settings
 
     riddle.render
   end
   
-  def wordform_file_pairs
-    @wordform_file_pairs ||= begin
+  def file_setting_pairs(setting)
+    @file_setting_pairs ||= {}
+    @file_setting_pairs[setting] ||= begin
       pairs = {}
-      wordform_sources.each_with_index do |source, index|
-        pairs[source] = "#{base_path}/wordforms/#{index}.txt"
+      file_setting_sources(setting).each_with_index do |source, index|
+        pairs[source] = "#{base_path}/#{setting}/#{index}.txt"
       end
       pairs
     end
@@ -124,19 +129,45 @@ class FlyingSphinx::Configuration
     end
   end
   
-  def set_wordforms
+  def set_file_settings
     riddle.indexes.each do |index|
-      next unless index.respond_to?(:wordforms)
-      index.wordforms = wordform_file_pairs[index.wordforms]
+      set_file_settings_for index, FileIndexSettings
+      
+      next unless index.respond_to?(:sources)
+            
+      index.sources.each do |source|
+        set_file_settings_for source, FileSourceSettings
+      end
+    end
+  end
+  
+  def set_file_settings_for(object, settings)
+    settings.each do |setting|
+      next unless object.respond_to?(setting)
+      object.send "#{setting}=",
+        file_setting_pairs(setting)[object.send(setting)]
     end
   end
   
-  def wordform_sources
-    @wordform_sources ||= riddle.indexes.collect { |index|
-      index.respond_to?(:wordforms) ? index.wordforms : nil
+  def file_setting_sources(setting)
+    @file_setting_sources ||= {}
+    @file_setting_sources[setting] ||= riddle.indexes.collect { |index|
+      file_settings_for_index(index, setting)
     }.flatten.compact.uniq
   end
-
+  
+  def file_settings_for_index(index, setting)
+    settings = Array(file_setting_for(index, setting))
+    settings += index.sources.collect { |source|
+      file_setting_for(source, setting)
+    } if index.respond_to?(:sources)
+    settings
+  end
+  
+  def file_setting_for(object, setting)
+    object.respond_to?(setting) ? object.send(setting) : nil
+  end
+  
   def identifier_from_env
     ENV['FLYING_SPHINX_IDENTIFIER']
   end

data/lib/flying_sphinx/index_request.rb

@@ -71,11 +71,13 @@ class FlyingSphinx::IndexRequest
   end
   
   def update_sphinx_reference_files
-    configuration.wordform_file_pairs.each do |local, remote|
-      api.post '/add_file',
-        :setting   => 'wordforms',
-        :file_name => remote.split('/').last,
-        :content   => open(local).read
+    FlyingSphinx::Configuration::FileSettings.each do |setting|
+      configuration.file_setting_pairs(setting).each do |local, remote|
+        api.post '/add_file',
+          :setting   => setting.to_s,
+          :file_name => remote.split('/').last,
+          :content   => open(local).read
+      end
     end
   end
 

data/lib/flying_sphinx/version.rb

@@ -0,0 +1,3 @@
+module FlyingSphinx
+  Version = '0.6.0'
+end

data/spec/specs/configuration_spec.rb

@@ -2,22 +2,23 @@ require 'spec_helper'
 require 'multi_json'
 
 describe FlyingSphinx::Configuration do
+  let(:api_server) { 'https://flying-sphinx.com/api/my' }
+  
+  before :each do
+    FakeWeb.register_uri(:get, "#{api_server}/app",
+      :body => MultiJson.encode(
+        :server        => 'foo.bar.com',
+        :port          => 9319,
+        :database_port => 10001
+      )
+    )
+  end
+  
   describe '#initialize' do
-    let(:api_server) { 'https://flying-sphinx.com/api/my' }
     let(:api_key)    { 'foo-bar-baz' }
     let(:identifier) { 'app@heroku.com' }
     let(:config)     { FlyingSphinx::Configuration.new identifier, api_key }
     
-    before :each do
-      FakeWeb.register_uri(:get, "#{api_server}/app",
-        :body => MultiJson.encode(
-          :server        => 'foo.bar.com',
-          :port          => 9319,
-          :database_port => 10001
-        )
-      )
-    end
-    
     it "requests details from the server with the given API key" do
       config
       FakeWeb.should have_requested :get, "#{api_server}/app"
@@ -35,4 +36,149 @@ describe FlyingSphinx::Configuration do
       config.database_port.should == 10001
     end
   end
+  
+  describe '#file_setting_pairs' do
+    let(:config)    { FlyingSphinx::Configuration.new 'ident' }
+    let(:riddle)    { ThinkingSphinx::Configuration.instance.configuration }
+    let(:base_path) { '/mnt/sphinx/flying-sphinx/ident' }
+    
+    context 'index setting' do
+      it "pairs each local file path to a server file path" do
+        riddle.stub! :indexes => [
+          double('index', :wordforms => '/path/to/wordforms-foo.txt',
+            :sources => [double('source'), double('source')]),
+          double('index', :wordforms => '/path/to/wordforms-bar.txt',
+            :sources => [double('source'), double('source')])
+        ]
+        
+        config.file_setting_pairs(:wordforms).should == {
+          '/path/to/wordforms-foo.txt' => "#{base_path}/wordforms/0.txt",
+          '/path/to/wordforms-bar.txt' => "#{base_path}/wordforms/1.txt"
+        }
+      end
+      
+      it "doesn't duplicate multiple references to the same local file" do
+        riddle.stub! :indexes => [
+          double('index', :wordforms => '/path/to/wordforms-foo.txt',
+            :sources => [double('source'), double('source')]),
+          double('index', :wordforms => '/path/to/wordforms-foo.txt',
+            :sources => [double('source'), double('source')])
+        ]
+        
+        config.file_setting_pairs(:wordforms).should == {
+          '/path/to/wordforms-foo.txt' => "#{base_path}/wordforms/0.txt"
+        }
+      end
+    end
+    
+    context 'source setting' do
+      it "pairs each local file path to a server file path" do
+        riddle.stub! :indexes => [
+          double('index', :sources => [
+            double('source', :mysql_ssl_cert => '/path/to/cert-foo.txt'),
+            double('source', :mysql_ssl_cert => '/path/to/cert-bar.txt')
+          ]),
+          double('index', :sources => [
+            double('source', :mysql_ssl_cert => '/path/to/cert-baz.txt'),
+            double('source', :mysql_ssl_cert => nil)
+          ])
+        ]
+        
+        config.file_setting_pairs(:mysql_ssl_cert).should == {
+          '/path/to/cert-foo.txt' => "#{base_path}/mysql_ssl_cert/0.txt",
+          '/path/to/cert-bar.txt' => "#{base_path}/mysql_ssl_cert/1.txt",
+          '/path/to/cert-baz.txt' => "#{base_path}/mysql_ssl_cert/2.txt"
+        }
+      end
+      
+      it "doesn't duplicate multiple references to the same local file" do
+        riddle.stub! :indexes => [
+          double('index', :sources => [
+            double('source', :mysql_ssl_cert => '/path/to/cert-foo.txt'),
+            double('source', :mysql_ssl_cert => '/path/to/cert-bar.txt')
+          ]),
+          double('index', :sources => [
+            double('source', :mysql_ssl_cert => '/path/to/cert-foo.txt'),
+            double('source', :mysql_ssl_cert => nil)
+          ])
+        ]
+        
+        config.file_setting_pairs(:mysql_ssl_cert).should == {
+          '/path/to/cert-foo.txt' => "#{base_path}/mysql_ssl_cert/0.txt",
+          '/path/to/cert-bar.txt' => "#{base_path}/mysql_ssl_cert/1.txt"
+        }
+      end
+    end
+  end
+  
+  describe '#sphinx_configuration' do
+    let(:config)    { FlyingSphinx::Configuration.new 'ident' }
+    let(:riddle)    { double('riddle configuration').as_null_object }
+    let(:base_path) { '/mnt/sphinx/flying-sphinx/ident' }
+    let(:source)    { double('source') }
+    
+    before :each do
+      ThinkingSphinx::Configuration.instance.stub!(
+        :generate      => nil,
+        :configuration => riddle
+      )
+      FlyingSphinx::Tunnel.stub! :required? => false
+    end
+    
+    it "sets database settings to match Flying Sphinx port forward" do
+      FlyingSphinx::Tunnel.stub! :required? => true
+      
+      riddle.stub! :indexes => [
+        double('distributed index'),
+        double('index', :sources => [source])
+      ]
+      
+      source.should_receive(:sql_host=).with('127.0.0.1')
+      source.should_receive(:sql_port=).with(10001)
+      
+      config.sphinx_configuration
+    end
+    
+    it "sets file path to match server directories for index settings" do
+      riddle.stub! :indexes => [
+        double('index', :wordforms => '/path/to/wordforms-foo.txt',
+          :sources => [double('source'), double('source')]),
+        double('index', :wordforms => '/path/to/wordforms-bar.txt',
+          :sources => [double('source'), double('source')]),
+        double('distributed index')
+      ]
+      
+      riddle.indexes[0].should_receive(:wordforms=).
+        with("#{base_path}/wordforms/0.txt")
+      riddle.indexes[1].should_receive(:wordforms=).
+        with("#{base_path}/wordforms/1.txt")
+      
+      config.sphinx_configuration
+    end
+    
+    it "sets file path to match server directories for source settings" do
+      riddle.stub! :indexes => [
+        double('index', :sources => [
+          double('source', :mysql_ssl_cert => '/path/to/cert-foo.txt'),
+          double('source', :mysql_ssl_cert => '/path/to/cert-bar.txt')
+        ]),
+        double('index', :sources => [
+          double('source', :mysql_ssl_cert => '/path/to/cert-baz.txt'),
+          double('source', :mysql_ssl_cert => nil)
+        ]),
+        double('distributed index')
+      ]
+      
+      riddle.indexes[0].sources[0].should_receive(:mysql_ssl_cert=).
+        with("#{base_path}/mysql_ssl_cert/0.txt")
+      riddle.indexes[0].sources[1].should_receive(:mysql_ssl_cert=).
+        with("#{base_path}/mysql_ssl_cert/1.txt")
+      riddle.indexes[1].sources[0].should_receive(:mysql_ssl_cert=).
+        with("#{base_path}/mysql_ssl_cert/2.txt")
+      riddle.indexes[1].sources[1].should_receive(:mysql_ssl_cert=).
+        with(nil)
+      
+      config.sphinx_configuration
+    end
+  end
 end

data/spec/specs/index_request_spec.rb

@@ -4,7 +4,7 @@ describe FlyingSphinx::IndexRequest do
   let(:api)           { FlyingSphinx::API.new 'foo', 'bar' }
   let(:configuration) {
     stub(:configuration, :api => api, :sphinx_configuration => 'foo {}',
-      :wordform_file_pairs => {})
+      :file_setting_pairs => {})
   }
   
   let(:index_response)    {
@@ -68,28 +68,36 @@ describe FlyingSphinx::IndexRequest do
       end
     end
     
-    context 'with wordforms' do
-      let(:file_params) {
-        {:setting => 'wordforms', :file_name => 'bar.txt', :content => 'baz'}
-      }
-      
-      before :each do
-        configuration.stub!(:wordform_file_pairs => {'foo.txt' => 'bar.txt'})
-        index_request.stub!(:open => double('file', :read => 'baz'))
-      end
-      
-      it "sends the wordform file" do
-        api.should_receive(:put).with('/', conf_params).and_return('ok')
-        api.should_receive(:post).with('/add_file', file_params).
-          and_return('ok')
-        api.should_receive(:post).
-          with('indices', index_params).and_return(index_response)
+    [
+      :stopwords, :wordforms, :exceptions, :mysql_ssl_cert, :mysql_ssl_key,
+      :mysql_ssl_ca
+    ].each do |setting|
+      context "with #{setting}" do
+        let(:file_params) {
+          {:setting => setting.to_s, :file_name => 'bar.txt', :content => 'baz'}
+        }
+        
+        before :each do
+          configuration.stub!(:file_setting_pairs => {})
+          index_request.stub!(:open => double('file', :read => 'baz'))
+        end
         
-        begin
-          Timeout::timeout(0.2) {
-            index_request.update_and_index
-          }
-        rescue Timeout::Error
+        it "sends the #{setting} file" do
+          api.should_receive(:put).with('/', conf_params).and_return('ok')
+          api.should_receive(:post).with('/add_file', file_params).
+            and_return('ok')
+          api.should_receive(:post).
+            with('indices', index_params).and_return(index_response)
+          
+          configuration.should_receive(:file_setting_pairs).
+            with(setting).and_return({'foo.txt' => 'bar.txt'})
+          
+          begin
+            Timeout::timeout(0.2) {
+              index_request.update_and_index
+            }
+          rescue Timeout::Error
+          end
         end
       end
     end

metadata

@@ -2,7 +2,7 @@
 name: flying-sphinx
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors: 
 - Pat Allan
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-28 00:00:00 +10:00
+date: 2011-07-31 00:00:00 +10:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -187,6 +187,7 @@ files:
 - lib/flying_sphinx/sinatra.rb
 - lib/flying_sphinx/tasks.rb
 - lib/flying_sphinx/tunnel.rb
+- lib/flying_sphinx/version.rb
 - spec/spec_helper.rb
 - spec/specs/configuration_spec.rb
 - spec/specs/delayed_delta_spec.rb