auser-poolparty 0.0.9 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/CHANGELOG +3 -2
  2. data/Rakefile +61 -4
  3. data/bin/instance +3 -1
  4. data/bin/pool +6 -2
  5. data/config/sample-config.yml +4 -4
  6. data/lib/core/object.rb +3 -0
  7. data/lib/helpers/plugin_spec_helper.rb +59 -0
  8. data/lib/modules/ec2_wrapper.rb +3 -1
  9. data/lib/modules/file_writer.rb +1 -1
  10. data/lib/modules/vlad_override.rb +83 -83
  11. data/lib/poolparty.rb +31 -13
  12. data/lib/poolparty/application.rb +22 -15
  13. data/lib/poolparty/init.rb +1 -1
  14. data/lib/poolparty/master.rb +41 -28
  15. data/lib/poolparty/monitors.rb +1 -3
  16. data/lib/poolparty/monitors/cpu.rb +7 -3
  17. data/lib/poolparty/monitors/memory.rb +14 -7
  18. data/lib/poolparty/monitors/web.rb +11 -5
  19. data/lib/poolparty/provider/packages/essential.rb +1 -1
  20. data/lib/poolparty/provider/packages/heartbeat.rb +1 -1
  21. data/lib/poolparty/provider/packages/ruby.rb +1 -10
  22. data/lib/poolparty/remote_instance.rb +5 -18
  23. data/lib/poolparty/remoter.rb +55 -4
  24. data/lib/poolparty/scheduler.rb +15 -25
  25. data/lib/poolparty/thread_pool.rb +94 -0
  26. data/poolparty.gemspec +9 -6
  27. data/spec/application_spec.rb +32 -13
  28. data/spec/callback_spec.rb +20 -1
  29. data/spec/file_writer_spec.rb +1 -0
  30. data/spec/kernel_spec.rb +13 -0
  31. data/spec/master_spec.rb +50 -20
  32. data/spec/monitors/cpu_monitor_spec.rb +1 -1
  33. data/spec/plugin_manager_spec.rb +9 -17
  34. data/spec/plugin_spec.rb +34 -34
  35. data/spec/poolparty_spec.rb +41 -1
  36. data/spec/remote_instance_spec.rb +5 -18
  37. data/spec/scheduler_spec.rb +7 -6
  38. data/spec/spec_helper.rb +8 -18
  39. metadata +19 -6
  40. data/lib/poolparty/tasks/package.rake +0 -53
data/CHANGELOG CHANGED
@@ -1,11 +1,12 @@
1
+ v0.1.0 * Added poolnotify
2
+ * Updating gem on github
1
3
  v0.0.9 * Changed configuration style to rsync all files across every instance
2
4
  * Moved configuration back to a configure scrip
3
5
  * Added in vlad configuration
4
6
  * Added plugin ability to install custom software
5
7
  * Updated configuration style
6
8
  * Added cloud_list to pool
7
- * Changed listing of the clouds through pool
8
- * Added poolnotify
9
+ * Changed listing of the clouds through pool
9
10
  v0.0.8 * Added plugin_manager
10
11
  * Moved remoting to rake remote task (from vlad)
11
12
  * Changed configuration from instance-based to cloud-based
data/Rakefile CHANGED
@@ -7,9 +7,8 @@ begin
7
7
  s.author = "Ari Lerner"
8
8
  s.email = "ari.lerner@citrusbyte.com"
9
9
  s.summary = "Run your entire application off EC2, managed and auto-scaling"
10
- s.url = "http://blog.citrusbyte.com"
11
- s.runtime_dependencies = ["aws-s3", "amazon-ec2", "auser-aska", "git", "crafterm-sprinkle", "SystemTimer"]
12
- s.development_dependencies = []
10
+ s.url = "http://poolpartyrb.com"
11
+ s.dependencies = ["aws-s3", "amazon-ec2", "auser-aska", "git", "crafterm-sprinkle", "SystemTimer"]
13
12
  s.install_message = %q{
14
13
 
15
14
  Get ready to jump in the pool, you just installed PoolParty!
@@ -33,4 +32,62 @@ end
33
32
 
34
33
  task :default => :test
35
34
 
36
- PoolParty.include_tasks
35
+ PoolParty.include_tasks
36
+
37
+ namespace(:pkg) do
38
+ ## Rake task to create/update a .manifest file in your project, as well as update *.gemspec
39
+ desc %{Update ".manifest" with the latest list of project filenames. Respect\
40
+ .gitignore by excluding everything that git ignores. Update `files` and\
41
+ `test_files` arrays in "*.gemspec" file if it's present.}
42
+ task :manifest do
43
+ list = Dir['**/*'].sort
44
+ spec_file = Dir['*.gemspec'].first
45
+ list -= [spec_file] if spec_file
46
+
47
+ File.read('.gitignore').each_line do |glob|
48
+ glob = glob.chomp.sub(/^\//, '')
49
+ list -= Dir[glob]
50
+ list -= Dir["#{glob}/**/*"] if File.directory?(glob) and !File.symlink?(glob)
51
+ puts "excluding #{glob}"
52
+ end
53
+
54
+ if spec_file
55
+ spec = File.read spec_file
56
+ spec.gsub! /^(\s* s.(test_)?files \s* = \s* )( \[ [^\]]* \] | %w\( [^)]* \) )/mx do
57
+ assignment = $1
58
+ bunch = $2 ? list.grep(/^test\//) : list
59
+ '%s%%w(%s)' % [assignment, bunch.join(' ')]
60
+ end
61
+
62
+ File.open(spec_file, 'w') {|f| f << spec }
63
+ end
64
+ File.open('Manifest', 'w') {|f| f << list.join("\n") }
65
+ end
66
+ desc "Build gemspec for github"
67
+ task :gemspec => :manifest do
68
+ require "yaml"
69
+ `rake manifest gem`
70
+ data = YAML.load(open("poolparty.gemspec").read).to_ruby
71
+ File.open("poolparty.gemspec", "w+") {|f| f << data }
72
+ end
73
+ desc "Update gemspec with the time"
74
+ task :gemspec_update => :gemspec do
75
+ data = open("poolparty.gemspec").read
76
+ str = "Updated at #{Time.now.strftime("%I:%M%p, %D")}"
77
+
78
+ if data.scan(/Updated at/).empty?
79
+ data = data.gsub(/you just installed PoolParty\!/, '\0'+" (#{str})")
80
+ end
81
+
82
+ File.open("poolparty.gemspec", "w+") {|f| f << data }
83
+ end
84
+ desc "Get ready to release the gem"
85
+ task :prerelease => :gemspec_update do
86
+ `git add .`
87
+ `git ci -a -m "Updated gemspec for github"`
88
+ end
89
+ desc "Release them gem to the gem server"
90
+ task :release => :prerelease do
91
+ `git push origin master`
92
+ end
93
+ end
@@ -12,7 +12,9 @@ Usage: instance [OPTIONS] { #{commandables.join(" | ")} }
12
12
  -----------------------------------------------------------------
13
13
  EOU
14
14
  })
15
- PoolParty.load_plugins
15
+
16
+ PoolParty.load
17
+
16
18
  master = PoolParty::Master.new
17
19
  list = PoolParty::Optioner.parse(ARGV.dup, %w(-v --verbose))
18
20
  num = list.reject {|a| commandables.include?(a) }.pop
data/bin/pool CHANGED
@@ -13,6 +13,9 @@ Starting #{PoolParty::Application.app_name ? "#{PoolParty::Application.app_name}
13
13
  Keypair: #{PoolParty::Application.keypair}
14
14
  Access key: #{PoolParty::Application.access_key}
15
15
  size: #{PoolParty::Application.size}
16
+ Monitors available:
17
+ --------------
18
+ #{PoolParty.registered_monitors.collect {|a| " #{a}"}}
16
19
  Plugins:
17
20
  --------------
18
21
  #{Dir["#{PoolParty::Application.plugin_dir}/*"].collect {|a| " #{File.basename(a)}"}.join("\n")}
@@ -27,8 +30,8 @@ Usage: pool [OPTIONS] {start | stop | list | clouds_list | maintain | restart |
27
30
  -----------------------------------------------------------------
28
31
  EOU
29
32
  })
30
-
31
- PoolParty.load_plugins
33
+
34
+ PoolParty.load
32
35
  master = PoolParty::Master.new
33
36
  list = PoolParty::Optioner.parse(ARGV.dup, %w(-v))
34
37
 
@@ -69,6 +72,7 @@ list.each do |cmd|
69
72
  master.request_termination_of_all_instances
70
73
  master.start_cloud!
71
74
  when "switch"
75
+ list.shift
72
76
  context = list.shift
73
77
  context ? `source ~/.#{context}_pool_keys` : puts("You must supply a context to switch to")
74
78
  else
@@ -15,9 +15,9 @@
15
15
  :shared_bucket: "poolparty-app-data"
16
16
  :services: nginx
17
17
  :environment: production
18
- :contract_when:
18
+ :contract_when: |
19
19
  web > 10
20
20
  cpu < 0.2
21
- :expand_when:
22
- cpu > 0.45
23
- web < 10
21
+ :expand_when: |
22
+ cpu > 0.85
23
+ web < 5
@@ -3,6 +3,9 @@
3
3
  Add returning to the object
4
4
  =end
5
5
  class Object
6
+ def my_methods
7
+ self.methods.sort - (self.class.methods + self.class.superclass.methods)
8
+ end
6
9
  def alias_method(new_id, original_id)
7
10
  original = self.method(original_id).to_proc
8
11
  define_method(new_id){|*args| original.call(*args)}
@@ -0,0 +1,59 @@
1
+ module PoolParty
2
+ class PluginSpecHelper
3
+ def self.define_stubs(klass, num=1)
4
+ require File.dirname(__FILE__) + "/../../spec/helpers/ec2_mock"
5
+
6
+ @klass = klass.send :new
7
+ klass.stub!(:new).and_return @klass
8
+
9
+ define_master
10
+ @instances = define_instances(num)
11
+
12
+ @master.stub!(:execute_tasks).and_return true
13
+ @master.stub!(:launch_minimum_instances).and_return true
14
+ @master.stub!(:number_of_pending_instances).and_return 0
15
+ @master.stub!(:get_node).with(0).and_return @instance0
16
+
17
+ @master.stub!(:nodes).and_return @instances
18
+
19
+ Kernel.stub!(:system).and_return "true"
20
+
21
+ Provider.stub!(:install_poolparty).and_return true
22
+ Provider.stub!(:install_userpackages).and_return true
23
+
24
+ [@klass, @master, @instances]
25
+ end
26
+ def self.define_master
27
+ @master ||= Master.new
28
+ end
29
+ def self.define_instances(num)
30
+ # Too many gross evals
31
+ returning [] do |arr|
32
+ num.times do |i|
33
+ eval <<-EOE
34
+ @instance#{i} = RemoteInstance.new
35
+ @instance#{i}.stub!(:ssh).and_return "true"
36
+ @instance#{i}.stub!(:scp).and_return "true"
37
+ @instance#{i}.stub!(:name).and_return "node#{i}"
38
+ @instance#{i}.stub!(:ip).and_return "127.0.0.#{i}"
39
+ EOE
40
+ arr << eval("@instance#{i}")
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+
47
+ module Spec
48
+ module Mocks
49
+ module Methods
50
+ def should_receive_at_least_once(sym, opts={}, &block)
51
+ begin
52
+ e = __mock_proxy.add_message_expectation(opts[:expected_from] || caller(1)[0], sym.to_sym, opts, &block).at_least(1)
53
+ __mock_proxy.add_message_expectation(opts[:expected_from] || caller(1)[0], sym.to_sym, opts, &block).any_number_of_times
54
+ e
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -43,7 +43,9 @@ module PoolParty
43
43
  def get_instances_description
44
44
  @cached_descriptions ||= EC2ResponseObject.get_descriptions(ec2.describe_instances)
45
45
  end
46
-
46
+ def reset!
47
+ @cached_descriptions = nil
48
+ end
47
49
  # EC2 connections
48
50
  def ec2
49
51
  @ec2 ||= EC2::Base.new(:access_key_id => Application.access_key, :secret_access_key => Application.secret_access_key)
@@ -27,7 +27,7 @@ module PoolParty
27
27
  end
28
28
  def remote_base_tmp_dir
29
29
  "~/tmp"
30
- end
30
+ end
31
31
  def make_base_directory
32
32
  `mkdir -p #{base_tmp_dir}` unless File.directory?(base_tmp_dir)
33
33
  end
@@ -1,83 +1,83 @@
1
- require "vlad"
2
- class Rake::RemoteTask < Rake::Task
3
- def run command
4
- cmd = [ssh_cmd, ssh_flags, target_host].compact
5
- result = []
6
-
7
- commander = cmd.join(" ") << " \"#{command}\""
8
- warn commander if $TRACE
9
-
10
- pid, inn, out, err = popen4(commander)
11
-
12
- inn.sync = true
13
- streams = [out, err]
14
- out_stream = {
15
- out => $stdout,
16
- err => $stderr,
17
- }
18
-
19
- # Handle process termination ourselves
20
- status = nil
21
- Thread.start do
22
- status = Process.waitpid2(pid).last
23
- end
24
-
25
- until streams.empty? do
26
- # don't busy loop
27
- selected, = select streams, nil, nil, 0.1
28
-
29
- next if selected.nil? or selected.empty?
30
-
31
- selected.each do |stream|
32
- if stream.eof? then
33
- streams.delete stream if status # we've quit, so no more writing
34
- next
35
- end
36
-
37
- data = stream.readpartial(1024)
38
- out_stream[stream].write data
39
-
40
- if stream == err and data =~ /^Password:/ then
41
- inn.puts sudo_password
42
- data << "\n"
43
- $stderr.write "\n"
44
- end
45
-
46
- result << data
47
- end
48
- end
49
-
50
- PoolParty.message "execution failed with status #{status.exitstatus}: #{cmd.join ' '}" unless status.success?
51
-
52
- result.join
53
- end
54
-
55
- def rsync local, remote
56
- cmd = [rsync_cmd, rsync_flags, local, "#{@target_host}:#{remote}"].flatten.compact
57
-
58
- success = system(*cmd.join(" "))
59
-
60
- unless success then
61
- raise Vlad::CommandFailedError, "execution failed: #{cmd.join ' '}"
62
- end
63
- end
64
- def set name, val = nil, &b
65
- rt.set name, val, &b
66
- end
67
- def rt
68
- @rt ||= Rake::RemoteTask
69
- end
70
-
71
- def target_hosts
72
- if hosts = ENV["HOSTS"] then
73
- hosts.strip.gsub(/\s+/, '').split(",")
74
- elsif options[:single]
75
- @roles = {}; @roles[:app] = {}
76
- @roles[:app][options[:single]] = options[:single]
77
- roles = Rake::RemoteTask.hosts_for(@roles)
78
- else
79
- roles = options[:roles]
80
- roles ? Rake::RemoteTask.hosts_for(roles) : Rake::RemoteTask.all_hosts
81
- end
82
- end
83
- end
1
+ # require "vlad"
2
+ # class Rake::RemoteTask < Rake::Task
3
+ # def run command
4
+ # cmd = [ssh_cmd, ssh_flags, target_host].compact
5
+ # result = []
6
+ #
7
+ # commander = cmd.join(" ") << " \"#{command}\""
8
+ # warn commander if $TRACE
9
+ #
10
+ # pid, inn, out, err = popen4(commander)
11
+ #
12
+ # inn.sync = true
13
+ # streams = [out, err]
14
+ # out_stream = {
15
+ # out => $stdout,
16
+ # err => $stderr,
17
+ # }
18
+ #
19
+ # # Handle process termination ourselves
20
+ # status = nil
21
+ # Thread.start do
22
+ # status = Process.waitpid2(pid).last
23
+ # end
24
+ #
25
+ # until streams.empty? do
26
+ # # don't busy loop
27
+ # selected, = select streams, nil, nil, 0.1
28
+ #
29
+ # next if selected.nil? or selected.empty?
30
+ #
31
+ # selected.each do |stream|
32
+ # if stream.eof? then
33
+ # streams.delete stream if status # we've quit, so no more writing
34
+ # next
35
+ # end
36
+ #
37
+ # data = stream.readpartial(1024)
38
+ # out_stream[stream].write data
39
+ #
40
+ # if stream == err and data =~ /^Password:/ then
41
+ # inn.puts sudo_password
42
+ # data << "\n"
43
+ # $stderr.write "\n"
44
+ # end
45
+ #
46
+ # result << data
47
+ # end
48
+ # end
49
+ #
50
+ # PoolParty.message "execution failed with status #{status.exitstatus}: #{cmd.join ' '}" unless status.success?
51
+ #
52
+ # result.join
53
+ # end
54
+ #
55
+ # def rsync local, remote
56
+ # cmd = [rsync_cmd, rsync_flags, local, "#{@target_host}:#{remote}"].flatten.compact
57
+ #
58
+ # success = system(*cmd.join(" "))
59
+ #
60
+ # unless success then
61
+ # raise Vlad::CommandFailedError, "execution failed: #{cmd.join ' '}"
62
+ # end
63
+ # end
64
+ # def set name, val = nil, &b
65
+ # rt.set name, val, &b
66
+ # end
67
+ # def rt
68
+ # @rt ||= Rake::RemoteTask
69
+ # end
70
+ #
71
+ # def target_hosts
72
+ # if hosts = ENV["HOSTS"] then
73
+ # hosts.strip.gsub(/\s+/, '').split(",")
74
+ # elsif options[:single]
75
+ # @roles = {}; @roles[:app] = {}
76
+ # @roles[:app][options[:single]] = options[:single]
77
+ # roles = Rake::RemoteTask.hosts_for(@roles)
78
+ # else
79
+ # roles = options[:roles]
80
+ # roles ? Rake::RemoteTask.hosts_for(roles) : Rake::RemoteTask.all_hosts
81
+ # end
82
+ # end
83
+ # end
@@ -3,15 +3,14 @@
3
3
  =end
4
4
  $:.unshift File.dirname(__FILE__) # For use/testing when no gem is installed
5
5
 
6
+ $TRACE = true
7
+
6
8
  # rubygems
7
9
  require 'rubygems'
8
10
  require "aws/s3"
9
- require "sqs"
10
11
  require "EC2"
11
12
  require "aska"
12
13
  require 'sprinkle'
13
-
14
- require 'thread'
15
14
  require "pp"
16
15
  require "tempfile"
17
16
 
@@ -20,6 +19,7 @@ begin
20
19
  require 'system_timer'
21
20
  @@timer = SystemTimer
22
21
  rescue LoadError
22
+ require 'thread'
23
23
  require 'timeout'
24
24
  @@timer = Timeout
25
25
  end
@@ -30,7 +30,7 @@ pwd = File.dirname(__FILE__)
30
30
  # Load the required files
31
31
  # If there is an init file, load that, otherwise
32
32
  # require all the files in each directory
33
- %w(core modules s3 poolparty).each do |dir|
33
+ %w(core modules s3 helpers poolparty).each do |dir|
34
34
  Dir["#{pwd}/#{dir}"].each do |dir|
35
35
  begin
36
36
  require File.join(dir, "init")
@@ -41,12 +41,14 @@ pwd = File.dirname(__FILE__)
41
41
  end
42
42
 
43
43
  module PoolParty
44
- module Version #:nodoc:
44
+ class Version #:nodoc:
45
45
  @major = 0
46
- @minor = 0
47
- @tiny = 9
46
+ @minor = 1
47
+ @tiny = 0
48
48
 
49
- STRING = [@major, @minor, @tiny].join('.')
49
+ def self.string
50
+ [@major, @minor, @tiny].join('.')
51
+ end
50
52
  end
51
53
  def timer
52
54
  @@timer
@@ -69,7 +71,7 @@ module PoolParty
69
71
  end
70
72
  # User directory
71
73
  def user_dir
72
- Dir.pwd
74
+ Application.working_directory
73
75
  end
74
76
  # Write string to a tempfile
75
77
  def write_to_temp_file(str="")
@@ -80,18 +82,34 @@ module PoolParty
80
82
  end
81
83
  def register_monitor(*names)
82
84
  names.each do |name|
83
- PoolParty::Monitors.extend name
85
+ unless registered_monitor?(name)
86
+ PoolParty::Monitors.extend name
84
87
 
85
- PoolParty::Master.send :include, name::Master
86
- PoolParty::RemoteInstance.send :include, name::Remote
88
+ PoolParty::Master.send :include, name::Master
89
+ PoolParty::RemoteInstance.send :include, name::Remote
90
+
91
+ registered_monitors << name
92
+ end
87
93
  end
88
94
  end
95
+ def registered_monitor?(name); registered_monitors.include?(name); end
96
+ def registered_monitors; @@registered_monitors ||= [];end
97
+
98
+ def load
99
+ load_monitors
100
+ load_plugins
101
+ end
102
+ def load_monitors
103
+ loc = File.directory?("#{user_dir}/monitors") ? "#{user_dir}/monitors" : "#{root_dir}/lib/poolparty/monitors"
104
+ Dir["#{loc}/*"].each {|f| require f}
105
+ end
106
+
89
107
  def load_plugins
90
108
  Dir["#{plugin_dir}/**/init.rb"].each {|a| require a} if File.directory?(plugin_dir)
91
109
  end
92
110
  def reset!
111
+ @@registered_monitors = nil
93
112
  @@installed_plugins = nil
94
- Application.options = nil
95
113
  end
96
114
  def plugin_dir
97
115
  "#{user_dir}/vendor"