dreadpiratepj-poolparty 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data/CHANGELOG +12 -0
  2. data/Manifest +115 -0
  3. data/README.txt +140 -0
  4. data/Rakefile +27 -0
  5. data/bin/instance +61 -0
  6. data/bin/pool +62 -0
  7. data/config/cloud_master_takeover +17 -0
  8. data/config/create_proxy_ami.sh +582 -0
  9. data/config/haproxy.conf +29 -0
  10. data/config/heartbeat.conf +8 -0
  11. data/config/heartbeat_authkeys.conf +2 -0
  12. data/config/installers/ubuntu_install.sh +77 -0
  13. data/config/monit.conf +9 -0
  14. data/config/monit/haproxy.monit.conf +7 -0
  15. data/config/monit/nginx.monit.conf +0 -0
  16. data/config/nginx.conf +24 -0
  17. data/config/reconfigure_instances_script.sh +18 -0
  18. data/config/sample-config.yml +23 -0
  19. data/config/scp_instances_script.sh +12 -0
  20. data/lib/core/array.rb +13 -0
  21. data/lib/core/exception.rb +9 -0
  22. data/lib/core/float.rb +13 -0
  23. data/lib/core/hash.rb +11 -0
  24. data/lib/core/kernel.rb +12 -0
  25. data/lib/core/module.rb +22 -0
  26. data/lib/core/object.rb +18 -0
  27. data/lib/core/proc.rb +15 -0
  28. data/lib/core/string.rb +49 -0
  29. data/lib/core/time.rb +41 -0
  30. data/lib/modules/callback.rb +133 -0
  31. data/lib/modules/ec2_wrapper.rb +82 -0
  32. data/lib/modules/safe_instance.rb +31 -0
  33. data/lib/modules/vlad_override.rb +82 -0
  34. data/lib/poolparty.rb +105 -0
  35. data/lib/poolparty/application.rb +170 -0
  36. data/lib/poolparty/init.rb +6 -0
  37. data/lib/poolparty/master.rb +329 -0
  38. data/lib/poolparty/monitors.rb +13 -0
  39. data/lib/poolparty/monitors/cpu.rb +19 -0
  40. data/lib/poolparty/monitors/memory.rb +26 -0
  41. data/lib/poolparty/monitors/web.rb +23 -0
  42. data/lib/poolparty/optioner.rb +16 -0
  43. data/lib/poolparty/plugin.rb +43 -0
  44. data/lib/poolparty/plugin_manager.rb +67 -0
  45. data/lib/poolparty/provider.rb +2 -0
  46. data/lib/poolparty/provider/packages/essential.rb +6 -0
  47. data/lib/poolparty/provider/packages/git.rb +4 -0
  48. data/lib/poolparty/provider/packages/haproxy.rb +20 -0
  49. data/lib/poolparty/provider/packages/heartbeat.rb +4 -0
  50. data/lib/poolparty/provider/packages/monit.rb +6 -0
  51. data/lib/poolparty/provider/packages/rsync.rb +4 -0
  52. data/lib/poolparty/provider/packages/ruby.rb +37 -0
  53. data/lib/poolparty/provider/packages/s3fuse.rb +11 -0
  54. data/lib/poolparty/provider/provider.rb +60 -0
  55. data/lib/poolparty/remote_instance.rb +216 -0
  56. data/lib/poolparty/remoter.rb +106 -0
  57. data/lib/poolparty/remoting.rb +112 -0
  58. data/lib/poolparty/scheduler.rb +103 -0
  59. data/lib/poolparty/tasks.rb +29 -0
  60. data/lib/poolparty/tasks/cloud.rake +57 -0
  61. data/lib/poolparty/tasks/development.rake +38 -0
  62. data/lib/poolparty/tasks/ec2.rake +20 -0
  63. data/lib/poolparty/tasks/instance.rake +63 -0
  64. data/lib/poolparty/tasks/plugins.rake +30 -0
  65. data/lib/poolparty/tasks/server.rake +42 -0
  66. data/lib/poolparty/tmp.rb +46 -0
  67. data/lib/s3/s3_object_store_folders.rb +44 -0
  68. data/misc/basics_tutorial.txt +142 -0
  69. data/poolparty.gemspec +72 -0
  70. data/spec/application_spec.rb +39 -0
  71. data/spec/callback_spec.rb +194 -0
  72. data/spec/core_spec.rb +15 -0
  73. data/spec/helpers/ec2_mock.rb +44 -0
  74. data/spec/kernel_spec.rb +11 -0
  75. data/spec/master_spec.rb +203 -0
  76. data/spec/monitors/cpu_monitor_spec.rb +38 -0
  77. data/spec/monitors/memory_spec.rb +50 -0
  78. data/spec/monitors/misc_monitor_spec.rb +50 -0
  79. data/spec/monitors/web_spec.rb +39 -0
  80. data/spec/optioner_spec.rb +22 -0
  81. data/spec/plugin_manager_spec.rb +31 -0
  82. data/spec/plugin_spec.rb +101 -0
  83. data/spec/pool_binary_spec.rb +10 -0
  84. data/spec/poolparty_spec.rb +15 -0
  85. data/spec/provider_spec.rb +17 -0
  86. data/spec/remote_instance_spec.rb +149 -0
  87. data/spec/remoter_spec.rb +65 -0
  88. data/spec/remoting_spec.rb +84 -0
  89. data/spec/scheduler_spec.rb +75 -0
  90. data/spec/spec_helper.rb +39 -0
  91. data/spec/string_spec.rb +28 -0
  92. data/web/static/conf/nginx.conf +22 -0
  93. data/web/static/site/images/balloon.png +0 -0
  94. data/web/static/site/images/cb.png +0 -0
  95. data/web/static/site/images/clouds.png +0 -0
  96. data/web/static/site/images/railsconf_preso_img.png +0 -0
  97. data/web/static/site/index.html +71 -0
  98. data/web/static/site/javascripts/application.js +3 -0
  99. data/web/static/site/javascripts/corner.js +178 -0
  100. data/web/static/site/javascripts/jquery-1.2.6.pack.js +11 -0
  101. data/web/static/site/misc.html +42 -0
  102. data/web/static/site/storage/pool_party_presentation.pdf +0 -0
  103. data/web/static/site/stylesheets/application.css +100 -0
  104. data/web/static/site/stylesheets/reset.css +17 -0
  105. data/web/static/src/layouts/application.haml +25 -0
  106. data/web/static/src/pages/index.haml +25 -0
  107. data/web/static/src/pages/misc.haml +5 -0
  108. data/web/static/src/stylesheets/application.sass +100 -0
  109. metadata +260 -0
@@ -0,0 +1,67 @@
1
+ =begin rdoc
2
+ A convenience method for working with plugins.
3
+
4
+ Sits on top of github.
5
+ =end
6
+ require "git"
7
+ module PoolParty
8
+ def installed_plugins
9
+ @@installed_plugins ||= PluginManager.extract_git_repos_from_plugin_dirs.uniq
10
+ end
11
+ class PluginManager
12
+ include Callbacks
13
+
14
+ def self.install_plugin(location)
15
+ unless File.directory?(plugin_directory(location))
16
+ begin
17
+ Git.clone(location, plugin_directory(location))
18
+ reset!
19
+ rescue Exception => e
20
+ puts "There was an error"
21
+ puts e
22
+ end
23
+ else
24
+ puts "Plugin already installed"
25
+ end
26
+ end
27
+
28
+ def self.remove_plugin(name)
29
+ Dir["#{PoolParty.root_dir}/#{PoolParty.plugin_dir}/*"].select {|a| a =~ /#{name}/}.each do |dir|
30
+ FileUtils.rm_rf dir
31
+ end
32
+ end
33
+
34
+ def self.scan
35
+ returning Array.new do |a|
36
+ plugin_dirs.each do |plugin|
37
+ a << File.basename(plugin)
38
+ end
39
+ end
40
+ end
41
+
42
+ def self.extract_git_repos_from_plugin_dirs
43
+ returning [] do |arr|
44
+ plugin_dirs.each do |dir|
45
+ begin
46
+ arr << open(File.join(dir, ".git", "config")).read[/url[\s]*=[\s](.*)/,1]
47
+ rescue Exception => e
48
+ end
49
+ end
50
+ end
51
+ end
52
+
53
+ def self.plugin_dirs
54
+ Dir["#{PoolParty.user_dir}/vendor/*"]
55
+ end
56
+
57
+ def self.plugin_directory(path)
58
+ File.join(base_plugin_dir, File.basename(path, File.extname(path)))
59
+ end
60
+ def self.create_plugin_directory
61
+ FileUtils.mkdir_p base_plugin_dir rescue ""
62
+ end
63
+ def self.base_plugin_dir
64
+ File.join(PoolParty.root_dir, "vendor")
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,2 @@
1
+ $:.unshift(File.dirname(__FILE__))
2
+ require "provider/provider"
@@ -0,0 +1,6 @@
1
+ ## Special package, anything that defines a 'source' package means build-essential should be installed for Ubuntu
2
+
3
+ package :build_essential do
4
+ description 'Build tools'
5
+ apt %w(build-essential)
6
+ end
@@ -0,0 +1,4 @@
1
+ package :git do
2
+ description "Git SCM"
3
+ apt %w( zlib1g-dev git-core )
4
+ end
@@ -0,0 +1,20 @@
1
+ # Install haproxy
2
+
3
+ def post_install_string
4
+ %w(
5
+ "echo 'Configuring haproxy logging'"
6
+ "sed -i 's/ENABLED=0/ENABLED=1/g' /etc/default/haproxy"
7
+ "sed -i 's/SYSLOGD=\"\"/SYSLOGD=\"-r\"/g' /etc/default/syslogd"
8
+ "echo 'local0.* /var/log/haproxy.log' >> /etc/syslog.conf && /etc/init.d/sysklogd restart"
9
+ "/etc/init.d/haproxy restart"
10
+ )
11
+ end
12
+
13
+ package :haproxy, :provides => :proxy do
14
+ description 'Haproxy proxy'
15
+ # version '1.2.18'
16
+ # source "http://haproxy.1wt.eu/download/1.2/src/haproxy-#{version}.tar.gz"
17
+ apt %w( haproxy )
18
+
19
+ post :install, post_install_string
20
+ end
@@ -0,0 +1,4 @@
1
+ package :heartbeat do
2
+ description "Heartbeat Linux HA project"
3
+ apt %w(heartbeat-2)
4
+ end
@@ -0,0 +1,6 @@
1
+ package :monit do
2
+ description "Monit monitoring service"
3
+ apt %w( monit )
4
+
5
+ post :apt, "sudo mkdir /etc/monit", "sed -i 's/startup=0/startup=1/g' /etc/default/monit"
6
+ end
@@ -0,0 +1,4 @@
1
+ package :rsync do
2
+ description "Rsync"
3
+ apt %w( rsync )
4
+ end
@@ -0,0 +1,37 @@
1
+ package :ruby do
2
+ description 'Ruby Virtual Machine'
3
+ # version '1.8.6'
4
+ # source "ftp://ftp.ruby-lang.org/pub/ruby/1.8/ruby-#{version}-p111.tar.gz" # implicit :style => :gnu
5
+ apt %w( ruby )
6
+ requires :ruby_dependencies
7
+ end
8
+
9
+ package :ruby_dependencies do
10
+ description 'Ruby Virtual Machine Build Dependencies'
11
+ apt %w( bison zlib1g-dev libssl-dev libreadline5-dev libncurses5-dev file )
12
+ end
13
+
14
+ package :rubygems do
15
+ description 'Ruby Gems Package Management System'
16
+ # version '1.0.1'
17
+ # source "http://rubyforge.org/frs/download.php/29548/rubygems-#{version}.tgz" do
18
+ # custom_install 'ruby setup.rb'
19
+ # end
20
+ apt %w( rubygems )
21
+ post :install, "gem update --system"
22
+ requires :ruby
23
+ end
24
+
25
+ package :poolparty_required_gems do
26
+ description "required gems"
27
+ gems %w(SQS aws-s3 amazon-ec2 aska rake)
28
+ end
29
+
30
+ package :poolparty do
31
+ description "Pool party gem"
32
+ gem "auser-pool-party" do
33
+ source 'http://gems.github.com'
34
+ end
35
+
36
+ required :poolparty_required_gems
37
+ end
@@ -0,0 +1,11 @@
1
+ package :s3fs do
2
+ description "S3 Fuse project"
3
+ source "http://s3fs.googlecode.com/files/s3fs-r166-source.tar.gz"
4
+
5
+ requires :s3fs_deps
6
+ end
7
+
8
+ package :s3fs_deps do
9
+ description "S3 Fuse project dependencies"
10
+ apt %w( libcurl4-openssl-dev libxml2-dev libfuse-dev )
11
+ end
@@ -0,0 +1,60 @@
1
+ require "sprinkle"
2
+
3
+ module PoolParty
4
+ class Provider
5
+
6
+ def self.install_poolparty(ips)
7
+
8
+ $:.unshift( File.dirname(__FILE__) )
9
+
10
+ load_str = []
11
+
12
+ Dir["#{File.expand_path(File.dirname(__FILE__))}/packages/*"].each {|f| load_str << open(f).read}
13
+
14
+ script=<<-EOS
15
+
16
+ #{load_str.join("\n")}
17
+
18
+ policy :poolparty, :roles => :app do
19
+ requires :git
20
+ requires :ruby
21
+ requires :monit
22
+ requires :s3fs
23
+ requires :rsync
24
+ requires :heartbeat
25
+ requires :poolparty
26
+ end
27
+
28
+ deployment do
29
+ delivery :vlad do
30
+
31
+ set :ssh_cmd, '#{RemoteInstance.ssh_string}'
32
+
33
+ #{string_roles_from_ips(ips)}
34
+ end
35
+
36
+ source do
37
+ prefix '/usr/local'
38
+ archives '/root/sources'
39
+ builds '/root/builds'
40
+ end
41
+
42
+ end
43
+ EOS
44
+
45
+ PoolParty.message "Installing required poolparty paraphernalia"
46
+ install_from_sprinkle_string script
47
+ end
48
+
49
+ def self.install_from_sprinkle_string(str)
50
+ Sprinkle::Script.sprinkle str
51
+ end
52
+
53
+ def self.string_roles_from_ips(ips)
54
+ ips.collect do |ip|
55
+ "role :app, '#{ip}'"
56
+ end.join("\n")
57
+ end
58
+
59
+ end
60
+ end
@@ -0,0 +1,216 @@
1
+ module PoolParty
2
+ class RemoteInstance
3
+ # ############################
4
+ include Remoter
5
+ # ############################
6
+ include PoolParty
7
+ include Callbacks
8
+
9
+ attr_reader :ip, :instance_id, :name, :status, :launching_time, :stack_installed
10
+ attr_accessor :name, :number, :scp_configure_file, :configure_file, :plugin_string
11
+
12
+ # CALLBACKS
13
+ after :install, :mark_installed
14
+
15
+ def initialize(obj={})
16
+ super
17
+
18
+ @ip = obj[:ip]
19
+ @instance_id = obj[:instance_id]
20
+ @name = obj[:name] || "node"
21
+ @number = obj[:number] || 0 # Defaults to the master
22
+ @status = obj[:status] || "running"
23
+ @launching_time = obj[:launching_time] || Time.now
24
+ end
25
+
26
+ # Host entry for this instance
27
+ def hosts_entry
28
+ "#{@ip} #{name}"
29
+ end
30
+ # Internal host entry for this instance
31
+ def local_hosts_entry
32
+ "127.0.0.1 #{name}\n127.0.0.1 localhost.localdomain localhost ubuntu"
33
+ end
34
+ # Node entry for heartbeat
35
+ def node_entry
36
+ "node #{name}"
37
+ end
38
+ # Internal naming scheme
39
+ def name
40
+ "#{@name}#{@number}"
41
+ end
42
+ # Entry for the heartbeat config file
43
+ def heartbeat_entry
44
+ "#{name} #{ip} #{Application.managed_services}"
45
+ end
46
+ # Entry for haproxy
47
+ def haproxy_entry
48
+ "\tserver #{name} #{@ip}:#{Application.client_port} weight 1 minconn 3 maxconn 6 check inter 20000 check"
49
+ end
50
+ def haproxy_resources_entry
51
+ "#{name} #{@ip}"
52
+ end
53
+ # Is this the master?
54
+ def master?
55
+ @number == 0
56
+ end
57
+ def secondary?
58
+ @number == 1
59
+ end
60
+ def set_hosts(c)
61
+ end
62
+ # Let's define some stuff for monit
63
+ %w(stop start restart).each do |cmd|
64
+ define_method "#{cmd}_with_monit" do
65
+ ssh("monit #{cmd} all")
66
+ end
67
+ end
68
+ # Configure the server with the new, sexy shell script
69
+ # This compiles all the scp commands into a shell script and then executes it
70
+ # then it will compile a list of the commands to operate on the instance
71
+ # and execute it
72
+ # This is how the cloud reconfigures itself
73
+ def configure(caller=nil)
74
+ associate_public_ip
75
+ scp_basic_config_files
76
+
77
+ Master.with_nodes do |node|
78
+ # These are node-specific
79
+ PoolParty.message "configuring #{node.name}"
80
+ node.scp_specific_config_files
81
+ end
82
+ configure_basics_through_ssh
83
+ end
84
+
85
+ def configure_basics_through_ssh
86
+ # -l #{Application.plugin_dir}
87
+ cmd=<<-EOC
88
+ #{update_plugin_string}
89
+ chmod 700 /etc/monit/monitrc
90
+ pool maintain -c ~/.config
91
+ hostname -v #{name}
92
+ /usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data
93
+ EOC
94
+ execute_tasks do
95
+ ssh(cmd.runnable)
96
+ end
97
+ end
98
+
99
+ def scp_string(src,dest,opts={})
100
+ end
101
+
102
+ def scp_basic_config_files
103
+ cmd=<<-EOC
104
+ mkdir -p /etc/ha.d/resource.d
105
+ mkdir -p /etc/monit
106
+ mkdir -p /etc/monit.d
107
+ EOC
108
+ execute_tasks do
109
+ scp(Application.heartbeat_authkeys_config_file, "/etc/ha.d", :dir => "/etc/ha.d/resource.d")
110
+ scp(conf_file("cloud_master_takeover"), "/etc/ha.d/resource.d/cloud_master_takeover", :dir => "/etc/ha.d/resource.d/")
111
+
112
+ scp(Application.config_file, "~/.config") if Application.config_file
113
+ Dir["#{root_dir}/config/resource.d/*"].each do |file|
114
+ scp(file, "/etc/ha.d/resource.d/#{File.basename(file)}")
115
+ end
116
+ scp(Application.monit_config_file, "/etc/monit/monitrc", :dir => "/etc/monit")
117
+ Dir["#{root_dir}/config/monit.d/*"].each do |file|
118
+ scp(file, "/etc/monit.d/#{File.basename(file)}")
119
+ end
120
+
121
+ `mkdir -p tmp/`
122
+ File.open("tmp/pool-party-haproxy.cfg", 'w') {|f| f.write(Master.build_haproxy_file) }
123
+ scp("tmp/pool-party-haproxy.cfg", "/etc/haproxy.cfg")
124
+ end
125
+ end
126
+ def scp_specific_config_files
127
+ execute_tasks({:dont_set_hosts => true}) do
128
+ if Master.requires_heartbeat?
129
+ hafile = "tmp/#{name}-pool-party-ha.cf"
130
+ File.open(hafile, 'w') {|f| f.write(Master.build_heartbeat_config_file_for(self)) }
131
+ single_scp(hafile, "/etc/ha.d/ha.cf")
132
+
133
+ haresources_file = "tmp/#{name}-pool-party-haresources"
134
+ File.open(haresources_file, 'w') {|f| f.write(Master.build_heartbeat_resources_file_for(self)) }
135
+ single_scp(haresources_file, "/etc/ha/haresources", :dir => "/etc/ha")
136
+ end
137
+ hosts_file = "tmp/#{name}-pool-party-hosts"
138
+ File.open(hosts_file, 'w') {|f| f.write(Master.build_hosts_file_for(self)) }
139
+ single_scp(hosts_file, "/etc/hosts")
140
+ end
141
+ end
142
+
143
+ # Installs with one commandline and an scp, rather than 10
144
+ def install
145
+ # unless stack_installed?
146
+ # execute_tasks do
147
+ # # scp(base_install_script, "~/base_install.sh")
148
+ # ssh("chmod +x base_install.sh && /bin/sh base_install.sh && rm base_install.sh && echo 'installed!' ")
149
+ # end
150
+ # PoolParty.message "After install execute_tasks"
151
+ # mark_installed
152
+ # end
153
+ end
154
+ # Login to store the authenticity
155
+ def login_once
156
+ run_now "ls -l"
157
+ end
158
+ # Associate a public ip if it is set and this is the master
159
+ def associate_public_ip
160
+ associate_address_with(Application.public_ip, @instance_id) if master? && Application.public_ip && !Application.public_ip.empty?
161
+ end
162
+ # Become the new master
163
+ def become_master
164
+ @master = Master.new
165
+ @number = 0
166
+ @master.nodes[0] = self
167
+ configure
168
+ end
169
+ def update_plugin_string
170
+ reset!
171
+ str = "mkdir -p #{Application.plugin_dir} && cd #{Application.plugin_dir}\n"
172
+ installed_plugins.each do |plugin_source|
173
+ str << "git clone #{plugin_source}\n"
174
+ end
175
+ end
176
+ def update_plugins(c)
177
+ ssh(c.update_plugin_string)
178
+ end
179
+ after :configure, :update_plugins
180
+ # Is this the master and if not, is the master running?
181
+ def is_not_master_and_master_is_not_running?
182
+ !master? && !Master.is_master_responding?
183
+ end
184
+ # User conf file if it exists, or default one
185
+ def conf_file(name)
186
+ user_conf = File.join(PoolParty.user_dir, "config", name)
187
+ if File.file?(user_conf)
188
+ File.join(PoolParty.user_dir, "config", name)
189
+ else
190
+ File.join(PoolParty.root_dir, "config", name)
191
+ end
192
+ end
193
+
194
+ # Description in the rake task
195
+ def description
196
+ case @status
197
+ when "running"
198
+ "#{@number}: INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
199
+ when "shutting-down"
200
+ "(terminating) INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
201
+ when "pending"
202
+ "(booting) INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
203
+ end
204
+ end
205
+ def stack_installed?
206
+ @stack_installed ||= (run_now('echo ~/.installed') == "installed")
207
+ end
208
+ def mark_installed(caller=nil)
209
+ run_now("echo 'installed' > ~/.installed")
210
+ @stack_installed = true
211
+ end
212
+ def base_install_script
213
+ "#{root_dir}/config/installers/#{Application.os.downcase}_install.sh"
214
+ end
215
+ end
216
+ end