auser-poolparty 0.0.9 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +3 -2
- data/Rakefile +61 -4
- data/bin/instance +3 -1
- data/bin/pool +6 -2
- data/config/sample-config.yml +4 -4
- data/lib/core/object.rb +3 -0
- data/lib/helpers/plugin_spec_helper.rb +59 -0
- data/lib/modules/ec2_wrapper.rb +3 -1
- data/lib/modules/file_writer.rb +1 -1
- data/lib/modules/vlad_override.rb +83 -83
- data/lib/poolparty.rb +31 -13
- data/lib/poolparty/application.rb +22 -15
- data/lib/poolparty/init.rb +1 -1
- data/lib/poolparty/master.rb +41 -28
- data/lib/poolparty/monitors.rb +1 -3
- data/lib/poolparty/monitors/cpu.rb +7 -3
- data/lib/poolparty/monitors/memory.rb +14 -7
- data/lib/poolparty/monitors/web.rb +11 -5
- data/lib/poolparty/provider/packages/essential.rb +1 -1
- data/lib/poolparty/provider/packages/heartbeat.rb +1 -1
- data/lib/poolparty/provider/packages/ruby.rb +1 -10
- data/lib/poolparty/remote_instance.rb +5 -18
- data/lib/poolparty/remoter.rb +55 -4
- data/lib/poolparty/scheduler.rb +15 -25
- data/lib/poolparty/thread_pool.rb +94 -0
- data/poolparty.gemspec +9 -6
- data/spec/application_spec.rb +32 -13
- data/spec/callback_spec.rb +20 -1
- data/spec/file_writer_spec.rb +1 -0
- data/spec/kernel_spec.rb +13 -0
- data/spec/master_spec.rb +50 -20
- data/spec/monitors/cpu_monitor_spec.rb +1 -1
- data/spec/plugin_manager_spec.rb +9 -17
- data/spec/plugin_spec.rb +34 -34
- data/spec/poolparty_spec.rb +41 -1
- data/spec/remote_instance_spec.rb +5 -18
- data/spec/scheduler_spec.rb +7 -6
- data/spec/spec_helper.rb +8 -18
- metadata +19 -6
- data/lib/poolparty/tasks/package.rake +0 -53
data/CHANGELOG
CHANGED
@@ -1,11 +1,12 @@
|
|
1
|
+
v0.1.0 * Added poolnotify
|
2
|
+
* Updating gem on github
|
1
3
|
v0.0.9 * Changed configuration style to rsync all files across every instance
|
2
4
|
* Moved configuration back to a configure scrip
|
3
5
|
* Added in vlad configuration
|
4
6
|
* Added plugin ability to install custom software
|
5
7
|
* Updated configuration style
|
6
8
|
* Added cloud_list to pool
|
7
|
-
* Changed listing of the clouds through pool
|
8
|
-
* Added poolnotify
|
9
|
+
* Changed listing of the clouds through pool
|
9
10
|
v0.0.8 * Added plugin_manager
|
10
11
|
* Moved remoting to rake remote task (from vlad)
|
11
12
|
* Changed configuration from instance-based to cloud-based
|
data/Rakefile
CHANGED
@@ -7,9 +7,8 @@ begin
|
|
7
7
|
s.author = "Ari Lerner"
|
8
8
|
s.email = "ari.lerner@citrusbyte.com"
|
9
9
|
s.summary = "Run your entire application off EC2, managed and auto-scaling"
|
10
|
-
s.url = "http://
|
11
|
-
s.
|
12
|
-
s.development_dependencies = []
|
10
|
+
s.url = "http://poolpartyrb.com"
|
11
|
+
s.dependencies = ["aws-s3", "amazon-ec2", "auser-aska", "git", "crafterm-sprinkle", "SystemTimer"]
|
13
12
|
s.install_message = %q{
|
14
13
|
|
15
14
|
Get ready to jump in the pool, you just installed PoolParty!
|
@@ -33,4 +32,62 @@ end
|
|
33
32
|
|
34
33
|
task :default => :test
|
35
34
|
|
36
|
-
PoolParty.include_tasks
|
35
|
+
PoolParty.include_tasks
|
36
|
+
|
37
|
+
namespace(:pkg) do
|
38
|
+
## Rake task to create/update a .manifest file in your project, as well as update *.gemspec
|
39
|
+
desc %{Update ".manifest" with the latest list of project filenames. Respect\
|
40
|
+
.gitignore by excluding everything that git ignores. Update `files` and\
|
41
|
+
`test_files` arrays in "*.gemspec" file if it's present.}
|
42
|
+
task :manifest do
|
43
|
+
list = Dir['**/*'].sort
|
44
|
+
spec_file = Dir['*.gemspec'].first
|
45
|
+
list -= [spec_file] if spec_file
|
46
|
+
|
47
|
+
File.read('.gitignore').each_line do |glob|
|
48
|
+
glob = glob.chomp.sub(/^\//, '')
|
49
|
+
list -= Dir[glob]
|
50
|
+
list -= Dir["#{glob}/**/*"] if File.directory?(glob) and !File.symlink?(glob)
|
51
|
+
puts "excluding #{glob}"
|
52
|
+
end
|
53
|
+
|
54
|
+
if spec_file
|
55
|
+
spec = File.read spec_file
|
56
|
+
spec.gsub! /^(\s* s.(test_)?files \s* = \s* )( \[ [^\]]* \] | %w\( [^)]* \) )/mx do
|
57
|
+
assignment = $1
|
58
|
+
bunch = $2 ? list.grep(/^test\//) : list
|
59
|
+
'%s%%w(%s)' % [assignment, bunch.join(' ')]
|
60
|
+
end
|
61
|
+
|
62
|
+
File.open(spec_file, 'w') {|f| f << spec }
|
63
|
+
end
|
64
|
+
File.open('Manifest', 'w') {|f| f << list.join("\n") }
|
65
|
+
end
|
66
|
+
desc "Build gemspec for github"
|
67
|
+
task :gemspec => :manifest do
|
68
|
+
require "yaml"
|
69
|
+
`rake manifest gem`
|
70
|
+
data = YAML.load(open("poolparty.gemspec").read).to_ruby
|
71
|
+
File.open("poolparty.gemspec", "w+") {|f| f << data }
|
72
|
+
end
|
73
|
+
desc "Update gemspec with the time"
|
74
|
+
task :gemspec_update => :gemspec do
|
75
|
+
data = open("poolparty.gemspec").read
|
76
|
+
str = "Updated at #{Time.now.strftime("%I:%M%p, %D")}"
|
77
|
+
|
78
|
+
if data.scan(/Updated at/).empty?
|
79
|
+
data = data.gsub(/you just installed PoolParty\!/, '\0'+" (#{str})")
|
80
|
+
end
|
81
|
+
|
82
|
+
File.open("poolparty.gemspec", "w+") {|f| f << data }
|
83
|
+
end
|
84
|
+
desc "Get ready to release the gem"
|
85
|
+
task :prerelease => :gemspec_update do
|
86
|
+
`git add .`
|
87
|
+
`git ci -a -m "Updated gemspec for github"`
|
88
|
+
end
|
89
|
+
desc "Release them gem to the gem server"
|
90
|
+
task :release => :prerelease do
|
91
|
+
`git push origin master`
|
92
|
+
end
|
93
|
+
end
|
data/bin/instance
CHANGED
@@ -12,7 +12,9 @@ Usage: instance [OPTIONS] { #{commandables.join(" | ")} }
|
|
12
12
|
-----------------------------------------------------------------
|
13
13
|
EOU
|
14
14
|
})
|
15
|
-
|
15
|
+
|
16
|
+
PoolParty.load
|
17
|
+
|
16
18
|
master = PoolParty::Master.new
|
17
19
|
list = PoolParty::Optioner.parse(ARGV.dup, %w(-v --verbose))
|
18
20
|
num = list.reject {|a| commandables.include?(a) }.pop
|
data/bin/pool
CHANGED
@@ -13,6 +13,9 @@ Starting #{PoolParty::Application.app_name ? "#{PoolParty::Application.app_name}
|
|
13
13
|
Keypair: #{PoolParty::Application.keypair}
|
14
14
|
Access key: #{PoolParty::Application.access_key}
|
15
15
|
size: #{PoolParty::Application.size}
|
16
|
+
Monitors available:
|
17
|
+
--------------
|
18
|
+
#{PoolParty.registered_monitors.collect {|a| " #{a}"}}
|
16
19
|
Plugins:
|
17
20
|
--------------
|
18
21
|
#{Dir["#{PoolParty::Application.plugin_dir}/*"].collect {|a| " #{File.basename(a)}"}.join("\n")}
|
@@ -27,8 +30,8 @@ Usage: pool [OPTIONS] {start | stop | list | clouds_list | maintain | restart |
|
|
27
30
|
-----------------------------------------------------------------
|
28
31
|
EOU
|
29
32
|
})
|
30
|
-
|
31
|
-
PoolParty.
|
33
|
+
|
34
|
+
PoolParty.load
|
32
35
|
master = PoolParty::Master.new
|
33
36
|
list = PoolParty::Optioner.parse(ARGV.dup, %w(-v))
|
34
37
|
|
@@ -69,6 +72,7 @@ list.each do |cmd|
|
|
69
72
|
master.request_termination_of_all_instances
|
70
73
|
master.start_cloud!
|
71
74
|
when "switch"
|
75
|
+
list.shift
|
72
76
|
context = list.shift
|
73
77
|
context ? `source ~/.#{context}_pool_keys` : puts("You must supply a context to switch to")
|
74
78
|
else
|
data/config/sample-config.yml
CHANGED
data/lib/core/object.rb
CHANGED
@@ -3,6 +3,9 @@
|
|
3
3
|
Add returning to the object
|
4
4
|
=end
|
5
5
|
class Object
|
6
|
+
def my_methods
|
7
|
+
self.methods.sort - (self.class.methods + self.class.superclass.methods)
|
8
|
+
end
|
6
9
|
def alias_method(new_id, original_id)
|
7
10
|
original = self.method(original_id).to_proc
|
8
11
|
define_method(new_id){|*args| original.call(*args)}
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module PoolParty
|
2
|
+
class PluginSpecHelper
|
3
|
+
def self.define_stubs(klass, num=1)
|
4
|
+
require File.dirname(__FILE__) + "/../../spec/helpers/ec2_mock"
|
5
|
+
|
6
|
+
@klass = klass.send :new
|
7
|
+
klass.stub!(:new).and_return @klass
|
8
|
+
|
9
|
+
define_master
|
10
|
+
@instances = define_instances(num)
|
11
|
+
|
12
|
+
@master.stub!(:execute_tasks).and_return true
|
13
|
+
@master.stub!(:launch_minimum_instances).and_return true
|
14
|
+
@master.stub!(:number_of_pending_instances).and_return 0
|
15
|
+
@master.stub!(:get_node).with(0).and_return @instance0
|
16
|
+
|
17
|
+
@master.stub!(:nodes).and_return @instances
|
18
|
+
|
19
|
+
Kernel.stub!(:system).and_return "true"
|
20
|
+
|
21
|
+
Provider.stub!(:install_poolparty).and_return true
|
22
|
+
Provider.stub!(:install_userpackages).and_return true
|
23
|
+
|
24
|
+
[@klass, @master, @instances]
|
25
|
+
end
|
26
|
+
def self.define_master
|
27
|
+
@master ||= Master.new
|
28
|
+
end
|
29
|
+
def self.define_instances(num)
|
30
|
+
# Too many gross evals
|
31
|
+
returning [] do |arr|
|
32
|
+
num.times do |i|
|
33
|
+
eval <<-EOE
|
34
|
+
@instance#{i} = RemoteInstance.new
|
35
|
+
@instance#{i}.stub!(:ssh).and_return "true"
|
36
|
+
@instance#{i}.stub!(:scp).and_return "true"
|
37
|
+
@instance#{i}.stub!(:name).and_return "node#{i}"
|
38
|
+
@instance#{i}.stub!(:ip).and_return "127.0.0.#{i}"
|
39
|
+
EOE
|
40
|
+
arr << eval("@instance#{i}")
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
module Spec
|
48
|
+
module Mocks
|
49
|
+
module Methods
|
50
|
+
def should_receive_at_least_once(sym, opts={}, &block)
|
51
|
+
begin
|
52
|
+
e = __mock_proxy.add_message_expectation(opts[:expected_from] || caller(1)[0], sym.to_sym, opts, &block).at_least(1)
|
53
|
+
__mock_proxy.add_message_expectation(opts[:expected_from] || caller(1)[0], sym.to_sym, opts, &block).any_number_of_times
|
54
|
+
e
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/modules/ec2_wrapper.rb
CHANGED
@@ -43,7 +43,9 @@ module PoolParty
|
|
43
43
|
def get_instances_description
|
44
44
|
@cached_descriptions ||= EC2ResponseObject.get_descriptions(ec2.describe_instances)
|
45
45
|
end
|
46
|
-
|
46
|
+
def reset!
|
47
|
+
@cached_descriptions = nil
|
48
|
+
end
|
47
49
|
# EC2 connections
|
48
50
|
def ec2
|
49
51
|
@ec2 ||= EC2::Base.new(:access_key_id => Application.access_key, :secret_access_key => Application.secret_access_key)
|
data/lib/modules/file_writer.rb
CHANGED
@@ -1,83 +1,83 @@
|
|
1
|
-
require "vlad"
|
2
|
-
class Rake::RemoteTask < Rake::Task
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
end
|
1
|
+
# require "vlad"
|
2
|
+
# class Rake::RemoteTask < Rake::Task
|
3
|
+
# def run command
|
4
|
+
# cmd = [ssh_cmd, ssh_flags, target_host].compact
|
5
|
+
# result = []
|
6
|
+
#
|
7
|
+
# commander = cmd.join(" ") << " \"#{command}\""
|
8
|
+
# warn commander if $TRACE
|
9
|
+
#
|
10
|
+
# pid, inn, out, err = popen4(commander)
|
11
|
+
#
|
12
|
+
# inn.sync = true
|
13
|
+
# streams = [out, err]
|
14
|
+
# out_stream = {
|
15
|
+
# out => $stdout,
|
16
|
+
# err => $stderr,
|
17
|
+
# }
|
18
|
+
#
|
19
|
+
# # Handle process termination ourselves
|
20
|
+
# status = nil
|
21
|
+
# Thread.start do
|
22
|
+
# status = Process.waitpid2(pid).last
|
23
|
+
# end
|
24
|
+
#
|
25
|
+
# until streams.empty? do
|
26
|
+
# # don't busy loop
|
27
|
+
# selected, = select streams, nil, nil, 0.1
|
28
|
+
#
|
29
|
+
# next if selected.nil? or selected.empty?
|
30
|
+
#
|
31
|
+
# selected.each do |stream|
|
32
|
+
# if stream.eof? then
|
33
|
+
# streams.delete stream if status # we've quit, so no more writing
|
34
|
+
# next
|
35
|
+
# end
|
36
|
+
#
|
37
|
+
# data = stream.readpartial(1024)
|
38
|
+
# out_stream[stream].write data
|
39
|
+
#
|
40
|
+
# if stream == err and data =~ /^Password:/ then
|
41
|
+
# inn.puts sudo_password
|
42
|
+
# data << "\n"
|
43
|
+
# $stderr.write "\n"
|
44
|
+
# end
|
45
|
+
#
|
46
|
+
# result << data
|
47
|
+
# end
|
48
|
+
# end
|
49
|
+
#
|
50
|
+
# PoolParty.message "execution failed with status #{status.exitstatus}: #{cmd.join ' '}" unless status.success?
|
51
|
+
#
|
52
|
+
# result.join
|
53
|
+
# end
|
54
|
+
#
|
55
|
+
# def rsync local, remote
|
56
|
+
# cmd = [rsync_cmd, rsync_flags, local, "#{@target_host}:#{remote}"].flatten.compact
|
57
|
+
#
|
58
|
+
# success = system(*cmd.join(" "))
|
59
|
+
#
|
60
|
+
# unless success then
|
61
|
+
# raise Vlad::CommandFailedError, "execution failed: #{cmd.join ' '}"
|
62
|
+
# end
|
63
|
+
# end
|
64
|
+
# def set name, val = nil, &b
|
65
|
+
# rt.set name, val, &b
|
66
|
+
# end
|
67
|
+
# def rt
|
68
|
+
# @rt ||= Rake::RemoteTask
|
69
|
+
# end
|
70
|
+
#
|
71
|
+
# def target_hosts
|
72
|
+
# if hosts = ENV["HOSTS"] then
|
73
|
+
# hosts.strip.gsub(/\s+/, '').split(",")
|
74
|
+
# elsif options[:single]
|
75
|
+
# @roles = {}; @roles[:app] = {}
|
76
|
+
# @roles[:app][options[:single]] = options[:single]
|
77
|
+
# roles = Rake::RemoteTask.hosts_for(@roles)
|
78
|
+
# else
|
79
|
+
# roles = options[:roles]
|
80
|
+
# roles ? Rake::RemoteTask.hosts_for(roles) : Rake::RemoteTask.all_hosts
|
81
|
+
# end
|
82
|
+
# end
|
83
|
+
# end
|
data/lib/poolparty.rb
CHANGED
@@ -3,15 +3,14 @@
|
|
3
3
|
=end
|
4
4
|
$:.unshift File.dirname(__FILE__) # For use/testing when no gem is installed
|
5
5
|
|
6
|
+
$TRACE = true
|
7
|
+
|
6
8
|
# rubygems
|
7
9
|
require 'rubygems'
|
8
10
|
require "aws/s3"
|
9
|
-
require "sqs"
|
10
11
|
require "EC2"
|
11
12
|
require "aska"
|
12
13
|
require 'sprinkle'
|
13
|
-
|
14
|
-
require 'thread'
|
15
14
|
require "pp"
|
16
15
|
require "tempfile"
|
17
16
|
|
@@ -20,6 +19,7 @@ begin
|
|
20
19
|
require 'system_timer'
|
21
20
|
@@timer = SystemTimer
|
22
21
|
rescue LoadError
|
22
|
+
require 'thread'
|
23
23
|
require 'timeout'
|
24
24
|
@@timer = Timeout
|
25
25
|
end
|
@@ -30,7 +30,7 @@ pwd = File.dirname(__FILE__)
|
|
30
30
|
# Load the required files
|
31
31
|
# If there is an init file, load that, otherwise
|
32
32
|
# require all the files in each directory
|
33
|
-
%w(core modules s3 poolparty).each do |dir|
|
33
|
+
%w(core modules s3 helpers poolparty).each do |dir|
|
34
34
|
Dir["#{pwd}/#{dir}"].each do |dir|
|
35
35
|
begin
|
36
36
|
require File.join(dir, "init")
|
@@ -41,12 +41,14 @@ pwd = File.dirname(__FILE__)
|
|
41
41
|
end
|
42
42
|
|
43
43
|
module PoolParty
|
44
|
-
|
44
|
+
class Version #:nodoc:
|
45
45
|
@major = 0
|
46
|
-
@minor =
|
47
|
-
@tiny =
|
46
|
+
@minor = 1
|
47
|
+
@tiny = 0
|
48
48
|
|
49
|
-
|
49
|
+
def self.string
|
50
|
+
[@major, @minor, @tiny].join('.')
|
51
|
+
end
|
50
52
|
end
|
51
53
|
def timer
|
52
54
|
@@timer
|
@@ -69,7 +71,7 @@ module PoolParty
|
|
69
71
|
end
|
70
72
|
# User directory
|
71
73
|
def user_dir
|
72
|
-
|
74
|
+
Application.working_directory
|
73
75
|
end
|
74
76
|
# Write string to a tempfile
|
75
77
|
def write_to_temp_file(str="")
|
@@ -80,18 +82,34 @@ module PoolParty
|
|
80
82
|
end
|
81
83
|
def register_monitor(*names)
|
82
84
|
names.each do |name|
|
83
|
-
|
85
|
+
unless registered_monitor?(name)
|
86
|
+
PoolParty::Monitors.extend name
|
84
87
|
|
85
|
-
|
86
|
-
|
88
|
+
PoolParty::Master.send :include, name::Master
|
89
|
+
PoolParty::RemoteInstance.send :include, name::Remote
|
90
|
+
|
91
|
+
registered_monitors << name
|
92
|
+
end
|
87
93
|
end
|
88
94
|
end
|
95
|
+
def registered_monitor?(name); registered_monitors.include?(name); end
|
96
|
+
def registered_monitors; @@registered_monitors ||= [];end
|
97
|
+
|
98
|
+
def load
|
99
|
+
load_monitors
|
100
|
+
load_plugins
|
101
|
+
end
|
102
|
+
def load_monitors
|
103
|
+
loc = File.directory?("#{user_dir}/monitors") ? "#{user_dir}/monitors" : "#{root_dir}/lib/poolparty/monitors"
|
104
|
+
Dir["#{loc}/*"].each {|f| require f}
|
105
|
+
end
|
106
|
+
|
89
107
|
def load_plugins
|
90
108
|
Dir["#{plugin_dir}/**/init.rb"].each {|a| require a} if File.directory?(plugin_dir)
|
91
109
|
end
|
92
110
|
def reset!
|
111
|
+
@@registered_monitors = nil
|
93
112
|
@@installed_plugins = nil
|
94
|
-
Application.options = nil
|
95
113
|
end
|
96
114
|
def plugin_dir
|
97
115
|
"#{user_dir}/vendor"
|