auser-poolparty 0.1.2 → 0.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (199) hide show
  1. data/History.txt +22 -0
  2. data/License.txt +20 -0
  3. data/README.txt +52 -0
  4. data/Rakefile +4 -109
  5. data/bin/cloud +31 -0
  6. data/bin/cloud-add-keypair +23 -0
  7. data/bin/cloud-configure +35 -0
  8. data/bin/cloud-contract +27 -0
  9. data/bin/cloud-expand +27 -0
  10. data/bin/cloud-list +32 -0
  11. data/bin/cloud-maintain +36 -0
  12. data/bin/cloud-provision +30 -0
  13. data/bin/cloud-reconfigure +24 -0
  14. data/bin/cloud-ssh +18 -0
  15. data/bin/cloud-start +29 -0
  16. data/bin/pool +23 -75
  17. data/bin/pool-console +12 -0
  18. data/bin/pool-describe +9 -0
  19. data/bin/pool-list +28 -0
  20. data/bin/pool-provision +34 -0
  21. data/bin/pool-spec +17 -0
  22. data/bin/pool-start +32 -0
  23. data/examples/basic.rb +20 -0
  24. data/examples/plugin_without_plugin_directory.rb +13 -0
  25. data/examples/poolparty.rb +12 -0
  26. data/examples/with_apache_plugin.rb +22 -0
  27. data/generators/poolspec/USAGE +5 -0
  28. data/generators/poolspec/poolspec_generator.rb +65 -0
  29. data/generators/poolspec/templates/pool_spec_template.erb +9 -0
  30. data/lib/erlang/eb_server.erl +27 -0
  31. data/lib/poolparty.rb +40 -116
  32. data/lib/poolparty/base_packages/haproxy.rb +41 -0
  33. data/lib/poolparty/base_packages/heartbeat.rb +43 -0
  34. data/lib/poolparty/base_packages/poolparty.rb +18 -0
  35. data/lib/poolparty/base_packages/ruby.rb +27 -0
  36. data/lib/poolparty/core/array.rb +24 -0
  37. data/lib/{core → poolparty/core}/exception.rb +0 -0
  38. data/lib/{core → poolparty/core}/float.rb +0 -0
  39. data/lib/poolparty/core/hash.rb +29 -0
  40. data/lib/poolparty/core/kernel.rb +34 -0
  41. data/lib/{core → poolparty/core}/module.rb +18 -0
  42. data/lib/poolparty/core/my_open_struct.rb +18 -0
  43. data/lib/poolparty/core/object.rb +54 -0
  44. data/lib/poolparty/core/proc.rb +2 -0
  45. data/lib/poolparty/core/string.rb +72 -0
  46. data/lib/poolparty/core/symbol.rb +8 -0
  47. data/lib/{core → poolparty/core}/time.rb +15 -0
  48. data/lib/poolparty/exceptions/RemoteException.rb +12 -0
  49. data/lib/poolparty/exceptions/ResourceException.rb +7 -0
  50. data/lib/poolparty/exceptions/RuntimeException.rb +7 -0
  51. data/lib/poolparty/exceptions/SpecException.rb +7 -0
  52. data/lib/poolparty/exceptions/TemplateNotFound.rb +7 -0
  53. data/lib/poolparty/helpers/binary.rb +30 -0
  54. data/lib/poolparty/helpers/console.rb +30 -0
  55. data/lib/poolparty/helpers/display.rb +25 -0
  56. data/lib/poolparty/helpers/optioner.rb +61 -0
  57. data/lib/poolparty/helpers/provisioner_base.rb +226 -0
  58. data/lib/poolparty/helpers/provisioners/master.rb +120 -0
  59. data/lib/poolparty/helpers/provisioners/slave.rb +52 -0
  60. data/lib/poolparty/modules/cloud_resourcer.rb +72 -0
  61. data/lib/poolparty/modules/configurable.rb +34 -0
  62. data/lib/poolparty/modules/definable_resource.rb +59 -0
  63. data/lib/poolparty/modules/file_writer.rb +55 -0
  64. data/lib/poolparty/modules/method_missing_sugar.rb +17 -0
  65. data/lib/poolparty/modules/output.rb +13 -0
  66. data/lib/poolparty/modules/pretty_printer.rb +38 -0
  67. data/lib/{core/string.rb → poolparty/modules/s3_string.rb} +5 -29
  68. data/lib/{modules → poolparty/modules}/safe_instance.rb +0 -0
  69. data/lib/poolparty/monitors/base_monitor.rb +16 -0
  70. data/lib/poolparty/net/remote.rb +35 -0
  71. data/lib/poolparty/net/remote_bases/ec2.rb +145 -0
  72. data/lib/poolparty/net/remote_instance.rb +68 -0
  73. data/lib/poolparty/net/remoter.rb +209 -0
  74. data/lib/poolparty/net/remoter_base.rb +117 -0
  75. data/lib/poolparty/plugins/gem_package.rb +39 -0
  76. data/lib/poolparty/plugins/line.rb +76 -0
  77. data/lib/poolparty/plugins/svn.rb +48 -0
  78. data/lib/poolparty/pool/base.rb +74 -0
  79. data/lib/poolparty/pool/cloud.rb +132 -0
  80. data/lib/poolparty/pool/custom_resource.rb +61 -0
  81. data/lib/poolparty/pool/loggable.rb +29 -0
  82. data/lib/poolparty/pool/plugin.rb +42 -0
  83. data/lib/poolparty/pool/plugin_model.rb +48 -0
  84. data/lib/poolparty/pool/pool.rb +55 -0
  85. data/lib/poolparty/pool/resource.rb +235 -0
  86. data/lib/poolparty/pool/resources/class_package.rb +60 -0
  87. data/lib/poolparty/pool/resources/cron.rb +14 -0
  88. data/lib/poolparty/pool/resources/directory.rb +23 -0
  89. data/lib/poolparty/pool/resources/exec.rb +26 -0
  90. data/lib/poolparty/pool/resources/file.rb +23 -0
  91. data/lib/poolparty/pool/resources/gem.rb +14 -0
  92. data/lib/poolparty/pool/resources/host.rb +14 -0
  93. data/lib/poolparty/pool/resources/package.rb +14 -0
  94. data/lib/poolparty/pool/resources/remote_file.rb +20 -0
  95. data/lib/poolparty/pool/resources/service.rb +21 -0
  96. data/lib/poolparty/pool/resources/sshkey.rb +19 -0
  97. data/lib/poolparty/pool/resources/variable.rb +27 -0
  98. data/lib/poolparty/pool/script.rb +21 -0
  99. data/{config/heartbeat_authkeys.conf → lib/poolparty/templates/authkeys} +0 -0
  100. data/lib/poolparty/templates/cib.xml +1 -0
  101. data/lib/poolparty/templates/fileserver.conf +4 -0
  102. data/{config/heartbeat.conf → lib/poolparty/templates/ha.cf} +3 -1
  103. data/{config → lib/poolparty/templates}/haproxy.conf +13 -6
  104. data/lib/poolparty/templates/namespaceauth.conf +19 -0
  105. data/lib/poolparty/templates/puppet.conf +13 -0
  106. data/lib/poolparty/version.rb +9 -0
  107. data/lib/poolpartycl.rb +3 -0
  108. data/script/destroy +14 -0
  109. data/script/generate +14 -0
  110. data/script/txt2html +82 -0
  111. data/{lib/poolparty/tasks → tasks}/cloud.rake +1 -1
  112. data/tasks/deployment.rake +34 -0
  113. data/tasks/development.rake +78 -0
  114. data/{lib/poolparty/tasks → tasks}/ec2.rake +1 -1
  115. data/tasks/environment.rake +7 -0
  116. data/{lib/poolparty/tasks → tasks}/instance.rake +0 -0
  117. data/{lib/poolparty/tasks → tasks}/server.rake +0 -0
  118. data/tasks/spec.rake +17 -0
  119. data/tasks/website.rake +17 -0
  120. metadata +154 -249
  121. data/CHANGELOG +0 -23
  122. data/LICENSE +0 -22
  123. data/README +0 -139
  124. data/assets/clouds.png +0 -0
  125. data/bin/instance +0 -68
  126. data/bin/poolnotify +0 -34
  127. data/config/cloud_master_takeover +0 -17
  128. data/config/create_proxy_ami.sh +0 -582
  129. data/config/installers/ubuntu_install.sh +0 -77
  130. data/config/monit.conf +0 -9
  131. data/config/monit/haproxy.monit.conf +0 -8
  132. data/config/monit/nginx.monit.conf +0 -0
  133. data/config/nginx.conf +0 -24
  134. data/config/reconfigure_instances_script.sh +0 -37
  135. data/config/sample-config.yml +0 -23
  136. data/config/scp_instances_script.sh +0 -12
  137. data/lib/core/array.rb +0 -16
  138. data/lib/core/hash.rb +0 -11
  139. data/lib/core/kernel.rb +0 -12
  140. data/lib/core/object.rb +0 -21
  141. data/lib/core/proc.rb +0 -15
  142. data/lib/helpers/plugin_spec_helper.rb +0 -58
  143. data/lib/modules/callback.rb +0 -133
  144. data/lib/modules/ec2_wrapper.rb +0 -108
  145. data/lib/modules/file_writer.rb +0 -38
  146. data/lib/modules/sprinkle_overrides.rb +0 -27
  147. data/lib/modules/vlad_override.rb +0 -83
  148. data/lib/poolparty/application.rb +0 -199
  149. data/lib/poolparty/init.rb +0 -6
  150. data/lib/poolparty/master.rb +0 -492
  151. data/lib/poolparty/monitors.rb +0 -11
  152. data/lib/poolparty/monitors/cpu.rb +0 -23
  153. data/lib/poolparty/monitors/memory.rb +0 -33
  154. data/lib/poolparty/monitors/web.rb +0 -29
  155. data/lib/poolparty/optioner.rb +0 -20
  156. data/lib/poolparty/plugin.rb +0 -78
  157. data/lib/poolparty/provider.rb +0 -104
  158. data/lib/poolparty/provider/essential.rb +0 -6
  159. data/lib/poolparty/provider/git.rb +0 -8
  160. data/lib/poolparty/provider/haproxy.rb +0 -9
  161. data/lib/poolparty/provider/heartbeat.rb +0 -6
  162. data/lib/poolparty/provider/rsync.rb +0 -8
  163. data/lib/poolparty/provider/ruby.rb +0 -65
  164. data/lib/poolparty/provider/s3fuse.rb +0 -22
  165. data/lib/poolparty/remote_instance.rb +0 -250
  166. data/lib/poolparty/remoter.rb +0 -171
  167. data/lib/poolparty/remoting.rb +0 -137
  168. data/lib/poolparty/scheduler.rb +0 -93
  169. data/lib/poolparty/tasks.rb +0 -47
  170. data/lib/poolparty/tasks/development.rake +0 -78
  171. data/lib/poolparty/tasks/plugins.rake +0 -30
  172. data/lib/poolparty/thread_pool.rb +0 -94
  173. data/lib/s3/s3_object_store_folders.rb +0 -44
  174. data/poolparty.gemspec +0 -71
  175. data/spec/files/describe_response +0 -37
  176. data/spec/files/multi_describe_response +0 -69
  177. data/spec/files/remote_desc_response +0 -37
  178. data/spec/helpers/ec2_mock.rb +0 -57
  179. data/spec/lib/core/core_spec.rb +0 -26
  180. data/spec/lib/core/kernel_spec.rb +0 -24
  181. data/spec/lib/core/string_spec.rb +0 -28
  182. data/spec/lib/modules/callback_spec.rb +0 -213
  183. data/spec/lib/modules/file_writer_spec.rb +0 -74
  184. data/spec/lib/poolparty/application_spec.rb +0 -135
  185. data/spec/lib/poolparty/ec2_wrapper_spec.rb +0 -110
  186. data/spec/lib/poolparty/master_spec.rb +0 -479
  187. data/spec/lib/poolparty/optioner_spec.rb +0 -34
  188. data/spec/lib/poolparty/plugin_spec.rb +0 -115
  189. data/spec/lib/poolparty/poolparty_spec.rb +0 -60
  190. data/spec/lib/poolparty/provider_spec.rb +0 -74
  191. data/spec/lib/poolparty/remote_instance_spec.rb +0 -178
  192. data/spec/lib/poolparty/remoter_spec.rb +0 -72
  193. data/spec/lib/poolparty/remoting_spec.rb +0 -148
  194. data/spec/lib/poolparty/scheduler_spec.rb +0 -70
  195. data/spec/monitors/cpu_monitor_spec.rb +0 -39
  196. data/spec/monitors/memory_spec.rb +0 -51
  197. data/spec/monitors/misc_monitor_spec.rb +0 -51
  198. data/spec/monitors/web_spec.rb +0 -40
  199. data/spec/spec_helper.rb +0 -53
@@ -1,11 +0,0 @@
1
- =begin rdoc
2
- Basic monitors for the master
3
- =end
4
- module PoolParty
5
- module Monitors
6
- module Master
7
- end
8
- module Remote
9
- end
10
- end
11
- end
@@ -1,23 +0,0 @@
1
- =begin rdoc
2
- Basic monitor on the cpu stats
3
- =end
4
- require "poolparty"
5
-
6
- module Cpu
7
- module Master
8
- def cpu
9
- nodes.size > 0 ? nodes.inject(0) {|i,a| i+=a.cpu } / nodes.size : 0.0
10
- end
11
- end
12
-
13
- module Remote
14
- def cpu
15
- str = run("uptime").split(/\s+/)[-3].to_f rescue 0.0
16
- PoolParty.message "Cpu usage: #{str}"
17
- str
18
- end
19
- end
20
-
21
- end
22
-
23
- PoolParty.register_monitor Cpu
@@ -1,33 +0,0 @@
1
- =begin rdoc
2
- Basic monitor on the cpu stats
3
- =end
4
- require "poolparty"
5
-
6
- module Memory
7
- module Master
8
- # Get the average memory usage over the cloud
9
- def memory
10
- nodes.size > 0 ? nodes.inject(0) {|i,a| i += a.memory } / nodes.size : 0.0
11
- end
12
- end
13
-
14
- module Remote
15
- def memory
16
- out = begin
17
- str = run("free -m | grep -i mem")
18
-
19
- total_memory = str.split[1].to_f
20
- used_memory = str.split[2].to_f
21
-
22
- used_memory / total_memory
23
- rescue Exception => e
24
- 0.0
25
- end
26
- PoolParty.message "Memory: #{out}"
27
- out
28
- end
29
- end
30
-
31
- end
32
-
33
- PoolParty.register_monitor Memory
@@ -1,29 +0,0 @@
1
- =begin rdoc
2
- Basic monitor on the cpu stats
3
- =end
4
- require "poolparty"
5
-
6
- module Web
7
- module Master
8
- # Get the average web request capabilities over the cloud
9
- def web
10
- nodes.size > 0 ? nodes.inject(0) {|i,a| i += a.web } / nodes.size : 0.0
11
- end
12
- end
13
-
14
- module Remote
15
- def web
16
- out = begin
17
- str = run("httperf --server localhost --port #{Application.client_port} --num-conn 3 --timeout 5 | grep 'Request rate'")
18
- str[/[.]* ([\d]*\.[\d]*) [.]*/, 0].chomp.to_f
19
- rescue Exception => e
20
- 0.0
21
- end
22
- PoolParty.message "Web requests: #{out}"
23
- out
24
- end
25
- end
26
-
27
- end
28
-
29
- PoolParty.register_monitor Web
@@ -1,20 +0,0 @@
1
- module PoolParty
2
- class Optioner
3
- # Parse the command line options for options without a switch
4
- def self.parse(argv, safe=[])
5
- args = []
6
- # Default options
7
- safe.push "-v"
8
- safe.push "-i"
9
-
10
- argv.each_with_index do |arg,i|
11
- unless arg.index("-") == 0# && !arg.match(/(?:[^"']+)/)
12
- args << arg
13
- else
14
- argv.shift unless safe.include?(arg)
15
- end
16
- end
17
- args
18
- end
19
- end
20
- end
@@ -1,78 +0,0 @@
1
- =begin rdoc
2
- Allow for plugins based in callbacks
3
-
4
- A plugin should be able to hook into any method and run their command
5
- either before or after the plugin.
6
- =end
7
- module PoolParty
8
- class Plugin
9
-
10
- # Create a class-level method for the name on the class
11
- # For instance:
12
- # create_methods :install, RemoteInstance
13
- # will give the following methods to the class
14
- # before_install and after_install on the RemoteInstance
15
- def self.create_methods(name, klass, opts={})
16
- str = ""
17
- %w(before after).each do |time|
18
- str << <<-EOE
19
- def self.#{time}_#{name}(*meth)
20
- callee = self
21
- #{klass}.class_eval do
22
- meth.each {|m| #{time} :#{name}, {m.to_sym => callee.to_s }}
23
- end
24
- end
25
- EOE
26
- end
27
- eval str
28
- end
29
-
30
- def self.user_tasks str
31
- PoolParty::RemoteInstance.user_tasks << str
32
- end
33
-
34
- def self.define_custom_package name=:userpackage, &block
35
- PoolParty::Provider.define_custom_package name, &block
36
- end
37
-
38
- def self.define_global_file(name, &block)
39
- PoolParty::Master.define_global_user_file(name, &block)
40
- end
41
-
42
- def self.define_node_file(name, &block)
43
- PoolParty::Master.define_node_user_file(name, &block)
44
- end
45
-
46
- def add_user_data(h={})
47
- case h.class.to_s
48
- when "String"
49
- PoolParty::Application.user_data << h
50
- when "Hash"
51
- PoolParty::Application.hash_to_launch_with.merge!(h)
52
- end
53
- end
54
-
55
- def read_config_file(filename)
56
- return {} unless filename
57
- YAML.load(open(filename).read)
58
- end
59
-
60
- %w(install configure associate_public_ip become_master).each do |method|
61
- create_methods method, RemoteInstance
62
- end
63
- %w(start start_monitor install_cloud configure_cloud scale_cloud reconfiguration add_instance terminate_instance check_stats).each do |method|
64
- create_methods method, Master
65
- end
66
- %w(define_tasks).each do |method|
67
- create_methods method, Tasks
68
- end
69
- %w(run_tasks).each do |method|
70
- create_methods method, Scheduler
71
- end
72
-
73
- %w(load_packages).each do |method|
74
- create_methods method, Provider
75
- end
76
-
77
- end
78
- end
@@ -1,104 +0,0 @@
1
- module PoolParty
2
- class Provider
3
- include Sprinkle
4
-
5
- def install_poolparty(testing=false)
6
- PoolParty.message "Installing required poolparty paraphernalia"
7
- load_packages
8
- user_packages.map {|blk| blk.call if blk }
9
-
10
- policy :poolparty, :roles => :app do
11
- requires :ruby
12
- requires :heartbeat
13
- requires :haproxy
14
- requires :s3fs
15
- requires :rsync
16
- requires :required_gems
17
-
18
- PoolParty::Provider.user_install_packages.each do |req|
19
- self.send :requires, req.to_sym
20
- end
21
- end
22
-
23
- testing ? show_process : process
24
- end
25
-
26
- def self.define_custom_package name=:userpackages, &block
27
- (user_install_packages << name).uniq!
28
- user_packages << yield if block_given?
29
- end
30
-
31
- def user_packages
32
- self.class.user_packages
33
- end
34
-
35
- def user_install_packages
36
- self.class.user_install_packages
37
- end
38
-
39
- def self.user_packages
40
- @user_packages ||= []
41
- end
42
-
43
- def self.user_install_packages
44
- @load_strings ||= []
45
- end
46
-
47
- def load_packages
48
- Dir["#{File.expand_path(File.dirname(__FILE__))}/provider/*"].each do |f|
49
- load f
50
- end
51
- end
52
-
53
- def set_start_with_sprinkle
54
- deployment do
55
- delivery :vlad do
56
- set :user, "#{Application.username}"
57
-
58
- Master.cloud_ips.each do |ip|
59
- role :app, "#{Application.username}@#{ip}"
60
- end
61
- end
62
-
63
- source do
64
- prefix '/usr/local'
65
- archives '/root/sources'
66
- builds '/root/builds'
67
- end
68
- end
69
-
70
- end
71
-
72
- def process
73
- set_start_with_sprinkle
74
- @deployment.process if @deployment
75
- end
76
-
77
- def show_process
78
- Sprinkle::OPTIONS[:testing] = true
79
- Sprinkle::OPTIONS[:verbose] = true
80
- Object.logger.level = ActiveSupport::BufferedLogger::Severity::DEBUG
81
- deployment do
82
- delivery :vlad do
83
- end
84
- source do
85
- prefix '/usr/local'
86
- archives '/root/sources'
87
- builds '/root/builds'
88
- end
89
- end.process
90
- end
91
-
92
- class << self
93
- require "sprinkle"
94
-
95
- def install_poolparty(testing=false)
96
- singleton.install_poolparty(testing)
97
- end
98
- def singleton
99
- @singleton ||= new
100
- end
101
- end
102
-
103
- end
104
- end
@@ -1,6 +0,0 @@
1
- ## Special package, anything that defines a 'source' package means build-essential should be installed for Ubuntu
2
-
3
- package :build_essential do
4
- description 'Build tools'
5
- apt %w( build-essential )
6
- end
@@ -1,8 +0,0 @@
1
- package :git do
2
- description "Git SCM"
3
- apt %w( zlib1g-dev git-core )
4
-
5
- verify do
6
- has_executable 'git'
7
- end
8
- end
@@ -1,9 +0,0 @@
1
- # Install haproxy
2
- package :haproxy, :provides => :proxy do
3
- description 'Haproxy proxy'
4
- apt %w( haproxy )
5
-
6
- verify do
7
- has_executable 'haproxy'
8
- end
9
- end
@@ -1,6 +0,0 @@
1
- package :heartbeat, :provides => :failover do
2
- description "Heartbeat Linux HA project"
3
- apt %w( heartbeat )
4
-
5
- has_executable 'heartbeat'
6
- end
@@ -1,8 +0,0 @@
1
- package :rsync do
2
- description "Rsync"
3
- apt %w( rsync )
4
-
5
- verify do
6
- has_executable 'rsync'
7
- end
8
- end
@@ -1,65 +0,0 @@
1
- package :ruby do
2
- description 'Ruby Virtual Machine'
3
- apt %w( ruby ruby1.8-dev )
4
- requires :ruby_dependencies
5
-
6
- verify do
7
- has_executable 'ruby'
8
- end
9
- end
10
-
11
- package :ruby_dependencies do
12
- description 'Ruby Virtual Machine Build Dependencies'
13
- apt %w( bison zlib1g-dev libssl-dev libreadline5-dev libncurses5-dev file )
14
- end
15
-
16
- package :rubygems do
17
- description 'Ruby Gems Package Management System'
18
- version '1.2.0'
19
- source "http://rubyforge.org/frs/download.php/38646/rubygems-#{version}.tgz" do
20
- custom_install 'ruby setup.rb'
21
- end
22
-
23
- post :install, "sed -i s/require\ 'rubygems'/require\ 'rubygems'\nrequire\ 'rubygems\/gem_runner'/g", "gem update --system", "gem sources -a http://gems.github.com"
24
-
25
- requires :ruby
26
-
27
- verify do
28
- has_executable 'gem'
29
- end
30
- end
31
-
32
- package :required_gems do
33
- description "Poolparty required gem"
34
- gem 'auser-poolparty'
35
- requires :s3
36
- requires :ec2
37
- requires :aska
38
-
39
- has_gem 'auser-poolparty'
40
- end
41
-
42
- package :s3 do
43
- description "Amazon s3"
44
- gem 'aws-s3'
45
-
46
- has_gem 'aws-s3'
47
- end
48
- package :ec2 do
49
- description "Amazon EC2"
50
- gem 'amazon-ec2'
51
-
52
- has_gem 'amazon-ec2'
53
- end
54
- package :aska do
55
- description "Aska - Expert System"
56
- gem 'auser-aska'
57
-
58
- has_gem 'auser-aska'
59
- end
60
- package :rake do
61
- description "Rake"
62
- gem 'rake'
63
-
64
- has_gem 'rake'
65
- end
@@ -1,22 +0,0 @@
1
- package :s3fs do
2
- description "S3 Fuse project"
3
- source "http://s3fs.googlecode.com/files/s3fs-r166-source.tar.gz" do
4
- custom_dir 's3fs'
5
- custom_install "make"
6
-
7
- post :install, "mv s3fs /usr/bin"
8
- end
9
-
10
- verify do
11
- has_executable 's3fs'
12
- end
13
-
14
- requires :s3fs_deps
15
- end
16
-
17
- package :s3fs_deps do
18
- description "S3 Fuse project dependencies"
19
- apt %w( libcurl4-openssl-dev libxml2-dev libfuse-dev )
20
-
21
- requires :build_essential
22
- end