pyramid_scheme 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.gitignore +21 -0
- data/LICENSE +20 -0
- data/README.rdoc +40 -0
- data/Rakefile +75 -0
- data/VERSION +1 -0
- data/lib/pyramid_scheme/configuration.rb +35 -0
- data/lib/pyramid_scheme/index_client.rb +28 -0
- data/lib/pyramid_scheme/index_lock_file.rb +25 -0
- data/lib/pyramid_scheme/index_provider/file_system.rb +83 -0
- data/lib/pyramid_scheme/index_server.rb +35 -0
- data/lib/pyramid_scheme/indexer.rb +14 -0
- data/lib/pyramid_scheme/required_configuration_not_found.rb +5 -0
- data/lib/pyramid_scheme/tasks/rake_tasks.rake +11 -0
- data/lib/pyramid_scheme/tasks.rb +1 -0
- data/lib/pyramid_scheme/thinking_sphinx_indexer.rb +7 -0
- data/lib/pyramid_scheme/ultrasphinx_indexer.rb +7 -0
- data/lib/pyramid_scheme.rb +30 -0
- data/pyramid_scheme.gemspec +94 -0
- data/spec/pyramid_scheme/index_client_spec.rb +19 -0
- data/spec/pyramid_scheme/index_provider/file_system_spec.rb +111 -0
- data/spec/pyramid_scheme/index_provider_configuration_spec.rb +29 -0
- data/spec/pyramid_scheme/index_server_spec.rb +45 -0
- data/spec/pyramid_scheme/thinking_sphinx_indexer_spec.rb +11 -0
- data/spec/pyramid_scheme/ultrasphinx_indexer_spec.rb +15 -0
- data/spec/pyramid_scheme.rb +17 -0
- data/spec/spec.opts +2 -0
- data/spec/spec_helper.rb +22 -0
- data/tasks/pyramid_scheme.rake +1 -0
- metadata +150 -0
data/.document
ADDED
data/.gitignore
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Dan Pickett
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
= pyramid_scheme
|
2
|
+
|
3
|
+
Index propagation for sphinx to eliminate redundant indexing. Run indexing once on an index server and have index clients retrieve the required files.
|
4
|
+
|
5
|
+
Use this if you have a distributed system for your application servers where each one requires a dedicated searchd process locally.
|
6
|
+
|
7
|
+
Supports ThinkingSphinx and Ultrasphinx
|
8
|
+
|
9
|
+
== Filesystem Configuration
|
10
|
+
|
11
|
+
Toss this in an initalizer like config/initializers/pyramid_scheme.rb
|
12
|
+
|
13
|
+
PyramidScheme.configure do |config|
|
14
|
+
config.client_source_path = '/some/client/source'
|
15
|
+
config.client_destination_path = '/some/client/destination'
|
16
|
+
config.server_source_path = '/some/server/source'
|
17
|
+
config.server_destination_path = '/some/server/destination'
|
18
|
+
end
|
19
|
+
|
20
|
+
In your Rakefile, require 'pyramid_scheme/tasks' after your thinking sphinx or ultrasphinx require statement
|
21
|
+
|
22
|
+
Your client source and server destination paths should point to the same storage path for the transfer to behave the way you want it to.
|
23
|
+
|
24
|
+
The server must run an index via rake pyramid_scheme:index for files to be placed properly for the client to pick up. A lock file will be created in the server destination directory
|
25
|
+
|
26
|
+
The client must run an index via rake pyramid_scheme:retrieve_index - upon retrieval, will send a sighup to your searchd process to rotate the indexes
|
27
|
+
|
28
|
+
== Note on Patches/Pull Requests
|
29
|
+
|
30
|
+
* Fork the project.
|
31
|
+
* Make your feature addition or bug fix.
|
32
|
+
* Add tests for it. This is important so I don't break it in a
|
33
|
+
future version unintentionally.
|
34
|
+
* Commit, do not mess with rakefile, version, or history.
|
35
|
+
(if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
36
|
+
* Send me a pull request. Bonus points for topic branches.
|
37
|
+
|
38
|
+
== Copyright
|
39
|
+
|
40
|
+
Copyright (c) 2010 Dan Pickett. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "pyramid_scheme"
|
8
|
+
gem.summary = %Q{Sphinx index propagation}
|
9
|
+
gem.description = %Q{Sphinx index propagtion (currently via the filesystem)}
|
10
|
+
gem.email = "dpickett@enlightsolutions.com"
|
11
|
+
gem.homepage = "http://github.com/dpickett/pyramid_scheme"
|
12
|
+
gem.files += FileList['lib/**/*.rake']
|
13
|
+
gem.authors = ["Dan Pickett"]
|
14
|
+
gem.add_dependency "rake", ">= 0.8.7"
|
15
|
+
gem.add_dependency "configatron"
|
16
|
+
gem.add_development_dependency "rspec", ">= 1.2.9"
|
17
|
+
gem.add_development_dependency "yard", ">= 0"
|
18
|
+
gem.add_development_dependency "mocha", "0.9.8"
|
19
|
+
gem.add_development_dependency "fakefs", "0.2.1"
|
20
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
21
|
+
end
|
22
|
+
Jeweler::GemcutterTasks.new
|
23
|
+
rescue LoadError
|
24
|
+
puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
|
25
|
+
end
|
26
|
+
|
27
|
+
require 'spec/rake/spectask'
|
28
|
+
Spec::Rake::SpecTask.new(:spec) do |spec|
|
29
|
+
spec.libs << 'lib' << 'spec'
|
30
|
+
spec.spec_files = FileList['spec/**/*_spec.rb']
|
31
|
+
end
|
32
|
+
|
33
|
+
Spec::Rake::SpecTask.new(:rcov) do |spec|
|
34
|
+
spec.libs << 'lib' << 'spec'
|
35
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
36
|
+
spec.rcov = true
|
37
|
+
end
|
38
|
+
|
39
|
+
task :spec => :check_dependencies
|
40
|
+
|
41
|
+
begin
|
42
|
+
require 'reek/adapters/rake_task'
|
43
|
+
Reek::RakeTask.new do |t|
|
44
|
+
t.fail_on_error = true
|
45
|
+
t.verbose = false
|
46
|
+
t.source_files = 'lib/**/*.rb'
|
47
|
+
end
|
48
|
+
rescue LoadError
|
49
|
+
task :reek do
|
50
|
+
abort "Reek is not available. In order to run reek, you must: sudo gem install reek"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
begin
|
55
|
+
require 'roodi'
|
56
|
+
require 'roodi_task'
|
57
|
+
RoodiTask.new do |t|
|
58
|
+
t.verbose = false
|
59
|
+
end
|
60
|
+
rescue LoadError
|
61
|
+
task :roodi do
|
62
|
+
abort "Roodi is not available. In order to run roodi, you must: sudo gem install roodi"
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
task :default => :spec
|
67
|
+
|
68
|
+
begin
|
69
|
+
require 'yard'
|
70
|
+
YARD::Rake::YardocTask.new
|
71
|
+
rescue LoadError
|
72
|
+
task :yardoc do
|
73
|
+
abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
|
74
|
+
end
|
75
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module PyramidScheme
|
2
|
+
class Configuration
|
3
|
+
INDEX_FILE_EXTENSIONS = [
|
4
|
+
'.spa',
|
5
|
+
'.spd',
|
6
|
+
'.sph',
|
7
|
+
'.spi',
|
8
|
+
'.spm',
|
9
|
+
'.spp'
|
10
|
+
]
|
11
|
+
|
12
|
+
def initialize(options = {})
|
13
|
+
self.class.set do |config|
|
14
|
+
options.each do |key, value|
|
15
|
+
config.send("#{key}=", value)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.set(&block)
|
21
|
+
yield(configatron.pyramid_scheme)
|
22
|
+
defaults.each do |key, value|
|
23
|
+
configatron.pyramid_scheme.send("#{key}=", value) unless configatron.pyramid_scheme.to_hash[key]
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.defaults
|
28
|
+
{ :lock_file_name => 'pyramid_scheme_index_in_progress.txt' }
|
29
|
+
end
|
30
|
+
|
31
|
+
def [](key)
|
32
|
+
configatron.pyramid_scheme.to_hash[key]
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module PyramidScheme
|
2
|
+
class IndexClient
|
3
|
+
attr_reader :index_provider
|
4
|
+
def initialize(options = {})
|
5
|
+
@index_provider = PyramidScheme::IndexProvider::FileSystem.new
|
6
|
+
end
|
7
|
+
|
8
|
+
def retrieve_index
|
9
|
+
@index_provider.retrieve_index
|
10
|
+
bounce_pids
|
11
|
+
end
|
12
|
+
|
13
|
+
def searchd_pids
|
14
|
+
ps_output = `ps ax | grep searchd`
|
15
|
+
ps_output.split("\n").collect{|p| /^(\d*)/.match(p)[0]}
|
16
|
+
end
|
17
|
+
|
18
|
+
def bounce_pids
|
19
|
+
searchd_pids.each do |pid|
|
20
|
+
begin
|
21
|
+
Process.kill("HUP", pid)
|
22
|
+
rescue
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module PyramidScheme
|
2
|
+
class IndexLockFile
|
3
|
+
def self.server_path
|
4
|
+
File.join(PyramidScheme.configuration[:server_destination_path],
|
5
|
+
PyramidScheme.configuration[:lock_file_name])
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.client_path
|
9
|
+
File.join(PyramidScheme.configuration[:client_source_path],
|
10
|
+
PyramidScheme.configuration[:lock_file_name])
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.exists?
|
14
|
+
File.exists?(client_path)
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.create
|
18
|
+
FileUtils.touch(server_path)
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.destroy
|
22
|
+
FileUtils.rm_f(server_path)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module PyramidScheme
|
2
|
+
module IndexProvider
|
3
|
+
class FileSystem
|
4
|
+
MAXIMUM_COPY_ATTEMPTS = 5
|
5
|
+
REQUIRED_OPTIONS = [
|
6
|
+
:server_source_path,
|
7
|
+
:server_destination_path,
|
8
|
+
:client_source_path,
|
9
|
+
:client_destination_path
|
10
|
+
]
|
11
|
+
|
12
|
+
attr_reader :configuration, :copy_attempts
|
13
|
+
|
14
|
+
def initialize(options = {})
|
15
|
+
@configuration = PyramidScheme::Configuration.new(options)
|
16
|
+
ensure_required_options_are_present
|
17
|
+
@copy_attempts = 0
|
18
|
+
end
|
19
|
+
|
20
|
+
def index_in_progress?
|
21
|
+
PyramidScheme::IndexLockFile.exists?
|
22
|
+
end
|
23
|
+
|
24
|
+
def process_index
|
25
|
+
server_copy
|
26
|
+
end
|
27
|
+
|
28
|
+
def retrieve_index
|
29
|
+
client_copy
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
def server_copy
|
34
|
+
Configuration::INDEX_FILE_EXTENSIONS.each do |ext|
|
35
|
+
Dir[File.join(self.configuration[:server_source_path], "*#{ext}")].each do |f|
|
36
|
+
FileUtils.cp_r(f, "#{self.configuration[:server_destination_path]}")
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def client_copy
|
42
|
+
if !exceeded_maximum_copy_attempts?
|
43
|
+
attempt_to_copy
|
44
|
+
else
|
45
|
+
raise "copying sphinx indexes failed after maximum number of attempts"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def ensure_required_options_are_present
|
50
|
+
REQUIRED_OPTIONS.each do |opt|
|
51
|
+
if configuration[opt].nil?
|
52
|
+
raise PyramidScheme::RequiredConfigurationNotFound, "the #{opt} setting was not found"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def attempt_to_copy
|
58
|
+
@copy_attempts += 1
|
59
|
+
if index_in_progress?
|
60
|
+
Kernel.sleep(5)
|
61
|
+
client_copy
|
62
|
+
else
|
63
|
+
copy_client_files
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def copy_client_files
|
68
|
+
Configuration::INDEX_FILE_EXTENSIONS.each do |ext|
|
69
|
+
Dir[File.join(self.configuration[:client_source_path], "*#{ext}")].each do |f|
|
70
|
+
new_filename = File.basename(f.gsub(/\./, ".new."))
|
71
|
+
FileUtils.cp_r(f,
|
72
|
+
"#{self.configuration[:client_destination_path]}/#{new_filename}")
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
def exceeded_maximum_copy_attempts?
|
79
|
+
copy_attempts > MAXIMUM_COPY_ATTEMPTS
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module PyramidScheme
|
2
|
+
class IndexServer
|
3
|
+
attr_reader :indexer_class, :configuration, :index_provider
|
4
|
+
|
5
|
+
# initializes a new index server
|
6
|
+
# @param options [Hash] takes an optional :indexer_class (defaults to PyramidScheme::ThinkingSphinxIndexer
|
7
|
+
def initialize(options = {})
|
8
|
+
@indexer_class = options[:indexer_class] || PyramidScheme::ThinkingSphinxIndexer
|
9
|
+
@index_provider = PyramidScheme::IndexProvider::FileSystem.new
|
10
|
+
@configuration = PyramidScheme::Configuration.new
|
11
|
+
end
|
12
|
+
|
13
|
+
# @returns [PyramidScheme::Indexer] an instance of the specified indexer_class from initialization
|
14
|
+
def indexer
|
15
|
+
@indexer ||= @indexer_class.new
|
16
|
+
end
|
17
|
+
|
18
|
+
# run the index
|
19
|
+
def index
|
20
|
+
create_lock_file
|
21
|
+
indexer.index
|
22
|
+
destroy_lock_file
|
23
|
+
index_provider.process_index
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
def create_lock_file
|
28
|
+
PyramidScheme::IndexLockFile.create
|
29
|
+
end
|
30
|
+
|
31
|
+
def destroy_lock_file
|
32
|
+
PyramidScheme::IndexLockFile.destroy
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module PyramidScheme
|
2
|
+
#defines an interface for indexers
|
3
|
+
class Indexer
|
4
|
+
attr_reader :rake_task_name
|
5
|
+
|
6
|
+
def initialize(rake_task_name = nil)
|
7
|
+
@rake_task_name = rake_task_name || self.class.default_task_name
|
8
|
+
end
|
9
|
+
|
10
|
+
def index
|
11
|
+
Rake::Task[@rake_task_name].invoke
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
namespace :pyramid_scheme do
|
2
|
+
desc "retrieve new sphinx indexes as the client"
|
3
|
+
task :retrieve => :environment do
|
4
|
+
PyramidScheme::IndexClient.new.retrieve_index
|
5
|
+
end
|
6
|
+
|
7
|
+
desc "create new sphinx indexes as the server"
|
8
|
+
task :index => :environment do
|
9
|
+
PyramidScheme::IndexServer.new.index
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
load(File.join(File.dirname(__FILE__), 'tasks/rake_tasks.rake'))
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'configatron'
|
4
|
+
|
5
|
+
require 'pyramid_scheme/required_configuration_not_found'
|
6
|
+
|
7
|
+
require 'pyramid_scheme/indexer'
|
8
|
+
require 'pyramid_scheme/thinking_sphinx_indexer'
|
9
|
+
require 'pyramid_scheme/ultrasphinx_indexer'
|
10
|
+
|
11
|
+
require 'pyramid_scheme/index_lock_file'
|
12
|
+
require 'pyramid_scheme/configuration'
|
13
|
+
require 'pyramid_scheme/index_provider/file_system'
|
14
|
+
|
15
|
+
require 'pyramid_scheme/index_server'
|
16
|
+
|
17
|
+
require 'pyramid_scheme/index_client'
|
18
|
+
|
19
|
+
require 'pyramid_scheme/tasks/rake_tasks.rake'
|
20
|
+
require 'pyramid_scheme/tasks'
|
21
|
+
|
22
|
+
module PyramidScheme
|
23
|
+
def self.configure(&block)
|
24
|
+
PyramidScheme::Configuration.set(&block)
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.configuration
|
28
|
+
PyramidScheme::Configuration.new
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{pyramid_scheme}
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Dan Pickett"]
|
12
|
+
s.date = %q{2010-02-08}
|
13
|
+
s.description = %q{Sphinx index propagtion (currently via the filesystem)}
|
14
|
+
s.email = %q{dpickett@enlightsolutions.com}
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE",
|
17
|
+
"README.rdoc"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".document",
|
21
|
+
".gitignore",
|
22
|
+
"LICENSE",
|
23
|
+
"README.rdoc",
|
24
|
+
"Rakefile",
|
25
|
+
"VERSION",
|
26
|
+
"lib/pyramid_scheme.rb",
|
27
|
+
"lib/pyramid_scheme/configuration.rb",
|
28
|
+
"lib/pyramid_scheme/index_client.rb",
|
29
|
+
"lib/pyramid_scheme/index_lock_file.rb",
|
30
|
+
"lib/pyramid_scheme/index_provider/file_system.rb",
|
31
|
+
"lib/pyramid_scheme/index_server.rb",
|
32
|
+
"lib/pyramid_scheme/indexer.rb",
|
33
|
+
"lib/pyramid_scheme/required_configuration_not_found.rb",
|
34
|
+
"lib/pyramid_scheme/tasks.rb",
|
35
|
+
"lib/pyramid_scheme/tasks/rake_tasks.rake",
|
36
|
+
"lib/pyramid_scheme/thinking_sphinx_indexer.rb",
|
37
|
+
"lib/pyramid_scheme/ultrasphinx_indexer.rb",
|
38
|
+
"pyramid_scheme.gemspec",
|
39
|
+
"spec/pyramid_scheme.rb",
|
40
|
+
"spec/pyramid_scheme/index_client_spec.rb",
|
41
|
+
"spec/pyramid_scheme/index_provider/file_system_spec.rb",
|
42
|
+
"spec/pyramid_scheme/index_provider_configuration_spec.rb",
|
43
|
+
"spec/pyramid_scheme/index_server_spec.rb",
|
44
|
+
"spec/pyramid_scheme/thinking_sphinx_indexer_spec.rb",
|
45
|
+
"spec/pyramid_scheme/ultrasphinx_indexer_spec.rb",
|
46
|
+
"spec/spec.opts",
|
47
|
+
"spec/spec_helper.rb",
|
48
|
+
"tasks/pyramid_scheme.rake"
|
49
|
+
]
|
50
|
+
s.homepage = %q{http://github.com/dpickett/pyramid_scheme}
|
51
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
52
|
+
s.require_paths = ["lib"]
|
53
|
+
s.rubygems_version = %q{1.3.5}
|
54
|
+
s.summary = %q{Sphinx index propagation}
|
55
|
+
s.test_files = [
|
56
|
+
"spec/pyramid_scheme/index_client_spec.rb",
|
57
|
+
"spec/pyramid_scheme/index_provider/file_system_spec.rb",
|
58
|
+
"spec/pyramid_scheme/index_provider_configuration_spec.rb",
|
59
|
+
"spec/pyramid_scheme/index_server_spec.rb",
|
60
|
+
"spec/pyramid_scheme/thinking_sphinx_indexer_spec.rb",
|
61
|
+
"spec/pyramid_scheme/ultrasphinx_indexer_spec.rb",
|
62
|
+
"spec/pyramid_scheme.rb",
|
63
|
+
"spec/spec_helper.rb"
|
64
|
+
]
|
65
|
+
|
66
|
+
if s.respond_to? :specification_version then
|
67
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
68
|
+
s.specification_version = 3
|
69
|
+
|
70
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
71
|
+
s.add_runtime_dependency(%q<rake>, [">= 0.8.7"])
|
72
|
+
s.add_runtime_dependency(%q<configatron>, [">= 0"])
|
73
|
+
s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
|
74
|
+
s.add_development_dependency(%q<yard>, [">= 0"])
|
75
|
+
s.add_development_dependency(%q<mocha>, ["= 0.9.8"])
|
76
|
+
s.add_development_dependency(%q<fakefs>, ["= 0.2.1"])
|
77
|
+
else
|
78
|
+
s.add_dependency(%q<rake>, [">= 0.8.7"])
|
79
|
+
s.add_dependency(%q<configatron>, [">= 0"])
|
80
|
+
s.add_dependency(%q<rspec>, [">= 1.2.9"])
|
81
|
+
s.add_dependency(%q<yard>, [">= 0"])
|
82
|
+
s.add_dependency(%q<mocha>, ["= 0.9.8"])
|
83
|
+
s.add_dependency(%q<fakefs>, ["= 0.2.1"])
|
84
|
+
end
|
85
|
+
else
|
86
|
+
s.add_dependency(%q<rake>, [">= 0.8.7"])
|
87
|
+
s.add_dependency(%q<configatron>, [">= 0"])
|
88
|
+
s.add_dependency(%q<rspec>, [">= 1.2.9"])
|
89
|
+
s.add_dependency(%q<yard>, [">= 0"])
|
90
|
+
s.add_dependency(%q<mocha>, ["= 0.9.8"])
|
91
|
+
s.add_dependency(%q<fakefs>, ["= 0.2.1"])
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe PyramidScheme::IndexClient do
|
4
|
+
before(:each) do
|
5
|
+
@client = PyramidScheme::IndexClient.new
|
6
|
+
end
|
7
|
+
|
8
|
+
it 'should have an index provider' do
|
9
|
+
@client.index_provider.should_not be_nil
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should send SIGHUP to active searchd processes' do
|
13
|
+
@client.expects(:searchd_pids).returns(["4345"])
|
14
|
+
Process.expects(:kill).with("HUP", "4345")
|
15
|
+
@client.bounce_pids
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe PyramidScheme::IndexProvider::FileSystem do
|
4
|
+
before(:each) do
|
5
|
+
@provider = PyramidScheme::IndexProvider::FileSystem.new
|
6
|
+
end
|
7
|
+
include FakeFS::SpecHelpers
|
8
|
+
|
9
|
+
it 'should have a configuration' do
|
10
|
+
@provider.configuration.should_not be_nil
|
11
|
+
end
|
12
|
+
|
13
|
+
[
|
14
|
+
:server_source_path,
|
15
|
+
:server_destination_path,
|
16
|
+
:client_source_path,
|
17
|
+
:client_destination_path
|
18
|
+
].each do |option|
|
19
|
+
it "should require the #{option} configuration option" do
|
20
|
+
PyramidScheme.configure do |config|
|
21
|
+
config.send("#{option}=", nil)
|
22
|
+
end
|
23
|
+
|
24
|
+
lambda { PyramidScheme::IndexProvider::FileSystem.new
|
25
|
+
}.should raise_error(PyramidScheme::RequiredConfigurationNotFound)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'should indicate that an index is in process if a lock file is present' do
|
30
|
+
@configuration = PyramidScheme::Configuration.new
|
31
|
+
FileUtils.mkdir_p(@configuration[:client_source_path])
|
32
|
+
FileUtils.touch(File.join(
|
33
|
+
@configuration[:client_source_path], @configuration[:lock_file_name]))
|
34
|
+
@provider.index_in_progress?.should be_true
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should sleep if the server is still indexing' do
|
38
|
+
@provider.stubs(:index_in_progress?).returns(true).then.returns(false)
|
39
|
+
Kernel.expects(:sleep).once
|
40
|
+
@provider.retrieve_index
|
41
|
+
@provider.copy_attempts.should eql(2)
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'should raise an error if the server is stuck indexing' do
|
45
|
+
Kernel.stub!(:sleep)
|
46
|
+
@provider.stubs(:index_in_progress?).returns(true)
|
47
|
+
lambda { @provider.retrieve_index }.should raise_error
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'should copy all sphinx files from server source to server destination' do
|
51
|
+
FileUtils.mkdir_p(PyramidScheme.configuration[:server_source_path])
|
52
|
+
FileUtils.mkdir_p(PyramidScheme.configuration[:server_destination_path])
|
53
|
+
@filenames = [
|
54
|
+
'.spi',
|
55
|
+
'.spd',
|
56
|
+
'.spa',
|
57
|
+
'.sph',
|
58
|
+
'.spm',
|
59
|
+
'.spp'
|
60
|
+
].collect{|s| "some_index#{s}" }
|
61
|
+
|
62
|
+
@filenames.each do |f|
|
63
|
+
FileUtils.touch(File.join(PyramidScheme.configuration[:server_source_path], f))
|
64
|
+
File.exists?(File.join(PyramidScheme.configuration[:server_source_path], f)).should be_true
|
65
|
+
end
|
66
|
+
|
67
|
+
@provider.process_index
|
68
|
+
|
69
|
+
@filenames.each do |f|
|
70
|
+
File.exists?(File.join(PyramidScheme.configuration[:server_destination_path], f)).should be_true
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe "copying from the filesytem" do
|
76
|
+
include FakeFS::SpecHelpers
|
77
|
+
|
78
|
+
before(:each) do
|
79
|
+
@configuration = PyramidScheme::Configuration.new
|
80
|
+
@provider = PyramidScheme::IndexProvider::FileSystem.new
|
81
|
+
FileUtils.mkdir_p(@configuration[:client_source_path])
|
82
|
+
FileUtils.mkdir_p(@configuration[:client_destination_path])
|
83
|
+
|
84
|
+
end
|
85
|
+
|
86
|
+
it 'should copy the files with .new extensions' do
|
87
|
+
@filenames = [
|
88
|
+
'.spi',
|
89
|
+
'.spd',
|
90
|
+
'.spa',
|
91
|
+
'.sph',
|
92
|
+
'.spm',
|
93
|
+
'.spp'
|
94
|
+
].collect{|s| "some_index#{s}" }
|
95
|
+
|
96
|
+
@filenames.each do |f|
|
97
|
+
FileUtils.touch(File.join(PyramidScheme.configuration[:client_source_path], f))
|
98
|
+
File.exists?(File.join(PyramidScheme.configuration[:client_source_path], f)).should be_true
|
99
|
+
end
|
100
|
+
|
101
|
+
@provider.retrieve_index
|
102
|
+
|
103
|
+
@filenames.each do |f|
|
104
|
+
new_filename = File.basename(f).gsub(/\./, ".new.")
|
105
|
+
File.exists?(File.join(PyramidScheme.configuration[:client_destination_path],
|
106
|
+
new_filename)).should be_true
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe PyramidScheme::Configuration do
|
4
|
+
before(:each) do
|
5
|
+
@source_path = '/some_mount_location'
|
6
|
+
@destination_path = '/data/app/current/db/sphinx/production'
|
7
|
+
PyramidScheme.configure do |config|
|
8
|
+
config.source_path = @source_path
|
9
|
+
config.destination_path = @destination_path
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'should use allow me to set global configuration options' do
|
14
|
+
@configuration = PyramidScheme::Configuration.new
|
15
|
+
@configuration[:source_path].should eql(@source_path)
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should allow me to override configuration options' do
|
19
|
+
@new_dest = '/some/other/path'
|
20
|
+
@configuration = PyramidScheme::Configuration.new(
|
21
|
+
:destination_path => @new_dest)
|
22
|
+
@configuration[:destination_path].should eql(@new_dest)
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should set a default index lock file' do
|
26
|
+
@configuration = PyramidScheme::Configuration.new
|
27
|
+
@configuration[:lock_file_name].should_not be_nil
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe PyramidScheme::IndexServer do
|
4
|
+
include FakeFS::SpecHelpers
|
5
|
+
|
6
|
+
before(:each) do
|
7
|
+
@server = PyramidScheme::IndexServer.new
|
8
|
+
@server.indexer.stub(:index)
|
9
|
+
FileUtils.mkdir_p(PyramidScheme.configuration[:server_destination_path])
|
10
|
+
FileUtils.mkdir_p(PyramidScheme.configuration[:client_source_path])
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'should have an indexer' do
|
14
|
+
@server.indexer.should_not be_nil
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should default to a thinking sphinx indexer' do
|
18
|
+
@server.indexer.should be_kind_of(PyramidScheme::ThinkingSphinxIndexer)
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'should allow me to specify an ultrasphinx indexer' do
|
22
|
+
@server = PyramidScheme::IndexServer.new(:indexer_class => PyramidScheme::UltrasphinxIndexer)
|
23
|
+
@server.indexer.should be_kind_of(PyramidScheme::UltrasphinxIndexer)
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'should index via the indexer' do
|
27
|
+
@server.indexer.expects(:index).once
|
28
|
+
@server.index
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'should touch a lock file before indexing' do
|
32
|
+
FileUtils.should_receive(:touch)
|
33
|
+
@server.index
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should remove the lock file after indexing' do
|
37
|
+
FileUtils.should_receive(:rm_f)
|
38
|
+
@server.index
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should have the index provider process the index' do
|
42
|
+
@server.index_provider.expects(:process_index)
|
43
|
+
@server.index
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe PyramidScheme::ThinkingSphinxIndexer do
|
4
|
+
before(:each) do
|
5
|
+
@indexer = PyramidScheme::ThinkingSphinxIndexer.new
|
6
|
+
end
|
7
|
+
|
8
|
+
it 'should have the default rake task of ts:in' do
|
9
|
+
@indexer.class.default_task_name.should eql('ts:in')
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe PyramidScheme::UltrasphinxIndexer do
|
4
|
+
before(:each) do
|
5
|
+
@indexer = PyramidScheme::UltrasphinxIndexer.new
|
6
|
+
end
|
7
|
+
|
8
|
+
it 'should have the default rake task of ultrasphinx:index' do
|
9
|
+
@indexer.class.default_task_name.should eql('ultrasphinx:index')
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should default to the default rake task' do
|
13
|
+
@indexer.rake_task_name.should eql(@indexer.class.default_task_name)
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe PyramidScheme do
|
4
|
+
it 'should have a shorthand delegate for setting the configuration' do
|
5
|
+
@path = '/a/path'
|
6
|
+
|
7
|
+
PyramidScheme.configure do |config|
|
8
|
+
config[:source_path] = @path
|
9
|
+
end
|
10
|
+
|
11
|
+
PyramidScheme::Configuration.new[:source_path].should eql(@path)
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'should have a shorthand delegate for getting the configuration' do
|
15
|
+
PyramidScheme.configuration[:source_path].should eql(PyramidScheme::Configuration.new[:source_path])
|
16
|
+
end
|
17
|
+
end
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
2
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
3
|
+
require 'pyramid_scheme'
|
4
|
+
require 'spec'
|
5
|
+
require 'spec/autorun'
|
6
|
+
|
7
|
+
require 'rubygems'
|
8
|
+
require 'mocha'
|
9
|
+
require 'fakefs'
|
10
|
+
require 'fakefs/spec_helpers'
|
11
|
+
|
12
|
+
|
13
|
+
Spec::Runner.configure do |config|
|
14
|
+
config.before(:each) do
|
15
|
+
PyramidScheme.configure do |config|
|
16
|
+
config.client_source_path = '/some/default/source'
|
17
|
+
config.client_destination_path = '/some/default/destination'
|
18
|
+
config.server_source_path = '/some/server/source'
|
19
|
+
config.server_destination_path = '/some/server/destination'
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require File.join(File.dirname(__FILE), '../lib/pyramid_scheme/tasks'
|
metadata
ADDED
@@ -0,0 +1,150 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pyramid_scheme
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Dan Pickett
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2010-02-08 00:00:00 -05:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: rake
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.8.7
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: configatron
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: "0"
|
34
|
+
version:
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: rspec
|
37
|
+
type: :development
|
38
|
+
version_requirement:
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 1.2.9
|
44
|
+
version:
|
45
|
+
- !ruby/object:Gem::Dependency
|
46
|
+
name: yard
|
47
|
+
type: :development
|
48
|
+
version_requirement:
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: "0"
|
54
|
+
version:
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mocha
|
57
|
+
type: :development
|
58
|
+
version_requirement:
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - "="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 0.9.8
|
64
|
+
version:
|
65
|
+
- !ruby/object:Gem::Dependency
|
66
|
+
name: fakefs
|
67
|
+
type: :development
|
68
|
+
version_requirement:
|
69
|
+
version_requirements: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - "="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: 0.2.1
|
74
|
+
version:
|
75
|
+
description: Sphinx index propagtion (currently via the filesystem)
|
76
|
+
email: dpickett@enlightsolutions.com
|
77
|
+
executables: []
|
78
|
+
|
79
|
+
extensions: []
|
80
|
+
|
81
|
+
extra_rdoc_files:
|
82
|
+
- LICENSE
|
83
|
+
- README.rdoc
|
84
|
+
files:
|
85
|
+
- .document
|
86
|
+
- .gitignore
|
87
|
+
- LICENSE
|
88
|
+
- README.rdoc
|
89
|
+
- Rakefile
|
90
|
+
- VERSION
|
91
|
+
- lib/pyramid_scheme.rb
|
92
|
+
- lib/pyramid_scheme/configuration.rb
|
93
|
+
- lib/pyramid_scheme/index_client.rb
|
94
|
+
- lib/pyramid_scheme/index_lock_file.rb
|
95
|
+
- lib/pyramid_scheme/index_provider/file_system.rb
|
96
|
+
- lib/pyramid_scheme/index_server.rb
|
97
|
+
- lib/pyramid_scheme/indexer.rb
|
98
|
+
- lib/pyramid_scheme/required_configuration_not_found.rb
|
99
|
+
- lib/pyramid_scheme/tasks.rb
|
100
|
+
- lib/pyramid_scheme/tasks/rake_tasks.rake
|
101
|
+
- lib/pyramid_scheme/thinking_sphinx_indexer.rb
|
102
|
+
- lib/pyramid_scheme/ultrasphinx_indexer.rb
|
103
|
+
- pyramid_scheme.gemspec
|
104
|
+
- spec/pyramid_scheme.rb
|
105
|
+
- spec/pyramid_scheme/index_client_spec.rb
|
106
|
+
- spec/pyramid_scheme/index_provider/file_system_spec.rb
|
107
|
+
- spec/pyramid_scheme/index_provider_configuration_spec.rb
|
108
|
+
- spec/pyramid_scheme/index_server_spec.rb
|
109
|
+
- spec/pyramid_scheme/thinking_sphinx_indexer_spec.rb
|
110
|
+
- spec/pyramid_scheme/ultrasphinx_indexer_spec.rb
|
111
|
+
- spec/spec.opts
|
112
|
+
- spec/spec_helper.rb
|
113
|
+
- tasks/pyramid_scheme.rake
|
114
|
+
has_rdoc: true
|
115
|
+
homepage: http://github.com/dpickett/pyramid_scheme
|
116
|
+
licenses: []
|
117
|
+
|
118
|
+
post_install_message:
|
119
|
+
rdoc_options:
|
120
|
+
- --charset=UTF-8
|
121
|
+
require_paths:
|
122
|
+
- lib
|
123
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - ">="
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: "0"
|
128
|
+
version:
|
129
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
130
|
+
requirements:
|
131
|
+
- - ">="
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: "0"
|
134
|
+
version:
|
135
|
+
requirements: []
|
136
|
+
|
137
|
+
rubyforge_project:
|
138
|
+
rubygems_version: 1.3.5
|
139
|
+
signing_key:
|
140
|
+
specification_version: 3
|
141
|
+
summary: Sphinx index propagation
|
142
|
+
test_files:
|
143
|
+
- spec/pyramid_scheme/index_client_spec.rb
|
144
|
+
- spec/pyramid_scheme/index_provider/file_system_spec.rb
|
145
|
+
- spec/pyramid_scheme/index_provider_configuration_spec.rb
|
146
|
+
- spec/pyramid_scheme/index_server_spec.rb
|
147
|
+
- spec/pyramid_scheme/thinking_sphinx_indexer_spec.rb
|
148
|
+
- spec/pyramid_scheme/ultrasphinx_indexer_spec.rb
|
149
|
+
- spec/pyramid_scheme.rb
|
150
|
+
- spec/spec_helper.rb
|