poolparty 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. data/CHANGELOG +4 -0
  2. data/Manifest +55 -0
  3. data/README.txt +113 -0
  4. data/Rakefile +18 -0
  5. data/bin/instance +54 -0
  6. data/bin/pool +33 -0
  7. data/config/config.yml +23 -0
  8. data/config/create_proxy_ami.sh +582 -0
  9. data/config/haproxy.conf +29 -0
  10. data/config/heartbeat.conf +9 -0
  11. data/config/heartbeat_authkeys.conf +2 -0
  12. data/config/monit/haproxy.monit.conf +7 -0
  13. data/config/monit/nginx.monit.conf +0 -0
  14. data/config/monit.conf +8 -0
  15. data/config/nginx.conf +24 -0
  16. data/lib/core/array.rb +10 -0
  17. data/lib/core/exception.rb +9 -0
  18. data/lib/core/kernel.rb +9 -0
  19. data/lib/core/module.rb +22 -0
  20. data/lib/core/object.rb +14 -0
  21. data/lib/core/string.rb +49 -0
  22. data/lib/core/time.rb +41 -0
  23. data/lib/modules/callback.rb +55 -0
  24. data/lib/modules/ec2_wrapper.rb +74 -0
  25. data/lib/modules/safe_instance.rb +31 -0
  26. data/lib/pool_party/application.rb +133 -0
  27. data/lib/pool_party/init.rb +4 -0
  28. data/lib/pool_party/master.rb +189 -0
  29. data/lib/pool_party/monitors/cpu.rb +18 -0
  30. data/lib/pool_party/monitors/memory.rb +21 -0
  31. data/lib/pool_party/monitors/web.rb +18 -0
  32. data/lib/pool_party/monitors.rb +13 -0
  33. data/lib/pool_party/optioner.rb +16 -0
  34. data/lib/pool_party/os/ubuntu.rb +78 -0
  35. data/lib/pool_party/os.rb +11 -0
  36. data/lib/pool_party/remote_instance.rb +180 -0
  37. data/lib/pool_party/remoting.rb +112 -0
  38. data/lib/pool_party/scheduler.rb +93 -0
  39. data/lib/pool_party/tasks.rb +220 -0
  40. data/lib/pool_party.rb +69 -0
  41. data/lib/s3/s3_object_store_folders.rb +44 -0
  42. data/poolparty.gemspec +55 -0
  43. data/spec/application_spec.rb +32 -0
  44. data/spec/callback_spec.rb +65 -0
  45. data/spec/helpers/ec2_mock.rb +56 -0
  46. data/spec/helpers/remote_instance_mock.rb +11 -0
  47. data/spec/kernel_spec.rb +11 -0
  48. data/spec/master_spec.rb +147 -0
  49. data/spec/monitor_spec.rb +16 -0
  50. data/spec/optioner_spec.rb +22 -0
  51. data/spec/poolparty_spec.rb +8 -0
  52. data/spec/remote_instance_spec.rb +29 -0
  53. data/spec/remoting_spec.rb +75 -0
  54. data/spec/spec_helper.rb +38 -0
  55. data/spec/string_spec.rb +28 -0
  56. data/test/test_pool_party.rb +0 -0
  57. metadata +171 -0
@@ -0,0 +1,582 @@
1
+ #!/bin/bash
2
+ #
3
+ # ec2ubuntu-build-ami
4
+ #
5
+ # This script builds, bundles, and uploads an Ubuntu base install AMI
6
+ # for Amazon EC2. This can be run on any of the following Fedora Core
7
+ # AMIs, depending on what type of new Ubuntu AMI you wish to create:
8
+ #
9
+ # 32-bit, 2.6.21 kernel: ami-f51aff9c
10
+ # 32-bit, 2.6.16 kernel: ami-20b65349
11
+ # 64-bit, 2.6.16 kernel: ami-36ff1a5f
12
+ # 64-bit, 2.6.21 kernel: ami-f21aff9b
13
+ #
14
+ # Command line options:
15
+ #
16
+ # --bucket NAME - REQUIRED
17
+ # --prefix PREFIX - Defaults to a reasonable manifest name.
18
+ # --user ID - Defaults to $AWS_USER_ID
19
+ # --access-key ID - Defaults to $AWS_access_key
20
+ # --secret-key ID - Defaults to $AWS_SECRET_access_key
21
+ # --private-key PATH - Defaults to $EC2_PRIVATE_KEY
22
+ # --cert PATH - Defaults to $EC2_CERT
23
+ # --release VERSION - One of: 6.06 6.10 7.04 7.10 8.04
24
+ # --codename NAME - Only needed if release not recognized.
25
+ # --timezone ZONE - Defaults to US/Pacific
26
+ # --lang LANG - Defaults to en_US.UTF-8
27
+ # --size MB - Root partition size in megabytes
28
+ # --package NAME - Additional Ubuntu package to install
29
+ # --script FILE - External script to run before bundle
30
+ # --desktop nx - Takes longer and makes AMI much bigger
31
+ #
32
+ # It is ok to specify multiple --package and --script options.
33
+ # You can also use "--desktop yes" to install the desktop packages
34
+ # without the NX software.
35
+ #
36
+ # For Amazon EC2 AMIs built using this script:
37
+ #
38
+ # http://alestic.com
39
+ #
40
+ # For updated versions of this script:
41
+ #
42
+ # http://code.google.com/p/ec2ubuntu/
43
+ #
44
+ # For general Ubuntu on EC2 support:
45
+ #
46
+ # http://groups.google.com/group/ec2ubuntu
47
+ #
48
+ # History:
49
+ #
50
+ # 2008-05-16 Eric Hammond <ehammond@thinksome.com>
51
+ # - Run the instance user-data if it looks like a script (starts with #!)
52
+ # - Wait for the network (DHCP) to come up before grabbing instance data.
53
+ #
54
+ # 2008-05-14 Eric Hammond <ehammond@thinksome.com>
55
+ # - Create new ssh host keys on first boot.
56
+ # - Disable apparmor as we don't have the kernel module installed yet.
57
+ # - Don't claim every AMI was built by Eric Hammond in /etc/motd
58
+ # - Create /tmp earlier in the boot process to avoid warnings.
59
+ # - Implemented following suggestions from Hans Omli:
60
+ # - Disable useless CMOS hwclock at boot to avoid error and save seconds.
61
+ # - Avoid grep warning about missing authorized_keys file on first boot.
62
+ #
63
+ # 2008-05-13 Thomas Shealy <thomas.shealy@gmail.com>
64
+ # - Add --retry to ec2-upload-bundle.
65
+ #
66
+ # 2008-05-12 Thomas Shealy <thomas.shealy@gmail.com>
67
+ # - Support 64-bit desktop AMIs.
68
+ # - Upgrade to NX 3.2.
69
+ #
70
+ # 2008-04-29 Eric Hammond <ehammond@thinksome.com>
71
+ # - Support command line options with reasonable defaults.
72
+ # - Support building Ubuntu 8.04 Hardy Heron.
73
+ #
74
+ # 2008-04-22 Vincent Desjardins
75
+ # - Add a variable to override the default size of the image created
76
+ # - Add a optional call to an external script before bundling the AMI
77
+ #
78
+ # 2008-04-19 Eric Hammond <ehammond@thinksome.com>
79
+ # - Support 2.6.21 kernel with 64-bit builds.
80
+ #
81
+ # 2008-04-18 Eric Hammond <ehammond@thinksome.com>
82
+ # - Fix performance problem with 2.6.21 initrd (with killall nash-hotplug).
83
+ #
84
+ # 2008-04-11 Eric Hammond <ehammond@thinksome.com>
85
+ # - Install kernel modules for both 2.6.16-xenU and 2.6.21.7-2.fc8xen
86
+ #
87
+ # 2008-04-05 Eric Hammond <ehammond@thinksome.com>
88
+ # - Add support for desktop build with NoMachine NX Free.
89
+ #
90
+ # 2008-04-03 Eric Hammond <ehammond@thinksome.com>
91
+ # - Upgrade to latest AMI tools 1.3-20041 20071010 (obviates several patches)
92
+ # - Add new Ubuntu patches for new version of AMI tools
93
+ # - Switch from "uname -i" to "uname -m" to enable building on Ubuntu 64-bit
94
+ # - Merge Dapper, Edgy, Feisty, Gutsy, Hardy scripts (Hardy doesn't work yet)
95
+ #
96
+ # 2008-03-13 Eric Hammond <ehammond@thinksome.com>
97
+ # - Prevent apt-get from running newly installed daemons
98
+ #
99
+ # 2008-03-09 Eric Hammond <ehammond@thinksome.com>
100
+ # - Upgrade to kernel modules 2.6.16.60
101
+ # - Upgrade fuse kernel module to 2.7.3
102
+ # - Upgrade to latest AMI tools 1.3-19365 (obviates one patch)
103
+ #
104
+ # 2008-02-05 Eric Hammond <ehammond@thinksome.com>
105
+ # - Patch AMI tools to work with new Ruby 1.8.6
106
+ #
107
+ # 2008-02-03 Eric Hammond <ehammond@thinksome.com>
108
+ # - Install rsync without lutimes support (as it's not in the EC2 kernel)
109
+ #
110
+ # 2008-01-17 Eric Hammond <ehammond@thinksome.com>
111
+ # - Upgrade to debootstrap 1.0.8
112
+ #
113
+ # 2007-12-25 Eric Hammond <ehammond@thinksome.com>
114
+ # - Install fuse kernel module (32-bit)
115
+ # - Upgrade to debootstrap 1.0.7
116
+ #
117
+ # 2007-12-02 Eric Hammond <ehammond@thinksome.com>
118
+ # - Use architecture "amd64" instead of "i386" for debootstrap on 64-bit
119
+ # - Add ia32-libs compatability package for 64-bit
120
+ #
121
+ # 2007-12-01 Eric Hammond <ehammond@thinksome.com>
122
+ # - Add support for building on 64-bit kernel (large, extra large instances)
123
+ #
124
+ # 2007-11-23 Eric Hammond <ehammond@thinksome.com>
125
+ # - ssh credentials retrieved from instance parameters or ephemeral storage.
126
+ # - Patch ec2-unbundle to work on Ubuntu
127
+ # - Also add locale to /etc/default/locale
128
+ #
129
+ # 2007-11-22 Eric Hammond <ehammond@thinksome.com>
130
+ # - Upgrade Ubuntu AMI tools patch to match new AMI tools source.
131
+ # - Install ca-certificates to better support ec2-upload-bundle per:
132
+ # http://developer.amazonwebservices.com/connect/thread.jspa?threadID=16543&tstart=0
133
+ # - ec2-bundle-vol excludes /etc/udev/rules.d/70-persistent-net.rules
134
+ # so that the network works on a rebundled instance, per:
135
+ # http://developer.amazonwebservices.com/connect/message.jspa?messageID=70873
136
+ #
137
+ # 2007-11-18 Eric Hammond <ehammond@thinksome.com>
138
+ # - Original put together based on code, tricks, and fixes from many
139
+ # others.
140
+ #
141
+
142
+ export AWS_USER_ID=161964561164
143
+ export AWS_access_key=1XCTNEK1CC5BQPA3EE02
144
+ export ACCESS_KEY=$AWS_access_key
145
+ export AWS_SECRET_ACCESS_KEY=Q2qJHP0S2iOKikn9glB+KZcF/aYf4huS/GdHvqEZ
146
+ export SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
147
+ export bucket=a-instances
148
+ export prefix=ubuntu-proxy-base-$(date +%Y%m%d)
149
+
150
+ packages="ubuntu-standard openssh-server rsync ruby openssl curl ca-certificates libopenssl-ruby1.8 patch alien"
151
+
152
+ while [ $# -gt 0 ]; do
153
+ case $1 in
154
+ --release) release=$2; shift 2 ;;
155
+ --codename) codename=$2; shift 2 ;;
156
+ --tag) tag=$2; shift 2 ;;
157
+ --bucket) bucket=$2; shift 2 ;;
158
+ --prefix) prefix=$2; shift 2 ;;
159
+ --user) AWS_USER_ID=$2; shift 2 ;;
160
+ --access-key) AWS_access_key=$2; shift 2 ;;
161
+ --secret-key) AWS_SECRET_ACCESS_KEY=$2; shift 2 ;;
162
+ --private-key) EC2_PRIVATE_KEY=$2; shift 2 ;;
163
+ --cert) EC2_CERT=$2; shift 2 ;;
164
+ --timezone) timezone=$2; shift 2 ;;
165
+ --lang) LANG=$2; shift 2 ;;
166
+ --size) size=$2; shift 2 ;;
167
+ --script) scripts="$scripts $2"; shift 2 ;;
168
+ --package) packages="$packages $2"; shift 2 ;;
169
+ --desktop) desktop=$2
170
+ packages="$packages ubuntu-desktop user-setup"
171
+ shift 2 ;;
172
+ *) echo "$0: Unrecognized option: $1" >&2; exit 1;
173
+ esac
174
+ done
175
+
176
+ true ${release:=8.04}
177
+ if [ "$codename" = "" ]; then
178
+ case $release in
179
+ 6.06) codename=dapper ;;
180
+ 6.10) codename=edgy ;;
181
+ 7.04) codename=feisty ;;
182
+ 7.10) codename=gutsy ;;
183
+ 8.04) codename=hardy ;;
184
+ *) echo "$0: Unrecognized release: $release" >&2; exit 1;
185
+ esac
186
+ fi
187
+
188
+ # Required and default parameters
189
+ true ${AWS_USER_ID:?} ${AWS_access_key:?} ${AWS_SECRET_ACCESS_KEY:?} \
190
+ ${bucket:?} \
191
+ ${EC2_CERT:=$(echo /mnt/cert-*.pem)} \
192
+ ${EC2_PRIVATE_KEY:=$(echo /mnt/pk-*.pem)} \
193
+ ${timezone:=US/Pacific} ${LANG:=en_US.UTF-8} \
194
+ ${tag:=custom} ${prefix:=ubuntu-$release-$codename-$tag-$(date +%Y%m%d)}
195
+
196
+ if [ "$codename" == "dapper" ]; then
197
+ true ${size:=3072}
198
+ else
199
+ true ${size:=4096}
200
+ fi
201
+
202
+ echo Building AMI for Ubuntu $release $codename
203
+ echo timezone: $timezone
204
+ echo LANG: $LANG
205
+ echo Image size: ${size} MB
206
+ echo Uploading to: $bucket/$prefix
207
+
208
+ set -e
209
+ set -x
210
+
211
+ # The good stuff starts here.
212
+
213
+ mkdir -p /mnt/build
214
+ cd /mnt/build
215
+ mkdir -p ubuntu
216
+
217
+ if which apt-get >/dev/null 2>/dev/null; then
218
+ # Ubuntu / Debian
219
+ apt-get install -y binutils
220
+ else
221
+ # Fedora Core / Red Hat / CentOS
222
+ yum install -y binutils
223
+ fi
224
+
225
+ if [ $(uname -m) = 'x86_64' ]; then
226
+ modules="http://s3.amazonaws.com/ec2-downloads/ec2-modules-2.6.16.33-xenU-x86_64.tgz http://alestic-downloads.s3.amazonaws.com/ec2-kernel-modules-2.6.21.x86_64.tar.gz"
227
+ bundlearch="x86_64"
228
+ bsarch="amd64"
229
+ export notlongext="64"
230
+ else
231
+ # http://groups.google.com/group/ec2ubuntu/web/compiling-kernel-modules-from-source-for-amazon-ec2
232
+ modules="http://alestic-downloads.s3.amazonaws.com/ec2-kernel-modules-2.6.16-xenU.tgz http://alestic-downloads.s3.amazonaws.com/ec2-kernel-modules-2.6.21.7-2.fc8xen.tar.gz"
233
+ bundlearch="i386"
234
+ bsarch="i386"
235
+ export notlongext=""
236
+ fi
237
+
238
+ if [ "$desktop" = "nx" ]; then
239
+ notlongext="$notlongext-desktop"
240
+ fi
241
+
242
+ # Don't launch daemons on apt-get install
243
+ mkdir -p ubuntu/usr/sbin/
244
+ cat << EOF > ubuntu/usr/sbin/policy-rc.d
245
+ #!/bin/sh
246
+ exit 101
247
+ EOF
248
+ chmod 755 ubuntu/usr/sbin/policy-rc.d
249
+
250
+ # Bootstrap Ubuntu
251
+ BSURL=http://archive.ubuntu.com/ubuntu/pool/main/d/debootstrap
252
+ BSVER=debootstrap_1.0.8
253
+ BSDIR=debootstrap
254
+ curl -s $BSURL/$BSVER.tar.gz |
255
+ tar xz
256
+ curl -s $BSURL/${BSVER}_all.deb > /tmp/${BSVER}_all.deb
257
+ ar p /tmp/${BSVER}_all.deb data.tar.gz |
258
+ tar xvzOf - ./usr/share/debootstrap/devices.tar.gz > $BSDIR/devices.tar.gz
259
+ if [ "$codename" = "hardy" ]; then
260
+ ln -s gutsy $BSDIR/scripts/ubuntu/$codename
261
+ fi
262
+ ln -s ubuntu/$codename $BSDIR/scripts/$codename
263
+ export DEBOOTSTRAP_DIR=$BSDIR
264
+ perl -pi.bak -e 'print "set -x\n" if $.==2' $BSDIR/debootstrap
265
+ $BSDIR/debootstrap --arch $bsarch $codename ubuntu http://us.archive.ubuntu.com/ubuntu
266
+
267
+ chroot ubuntu mount -t proc none /proc
268
+ chroot ubuntu mkdir -p /dev/pts
269
+ chroot ubuntu mount -t devpts none /dev/pts
270
+ chroot ubuntu mknod --mode 666 /dev/ptmx c 5 2
271
+
272
+ chroot ubuntu apt-get -f install -y
273
+
274
+ # Change these to your locale and timezone
275
+ chroot ubuntu localedef -i en_US -c -f UTF-8 en_US.UTF-8
276
+ echo $timezone >ubuntu/etc/timezone
277
+ echo "LANG=\"$LANG\"" >ubuntu/etc/default/locale
278
+ /bin/cp -f ubuntu/usr/share/zoneinfo/$timezone ubuntu/etc/localtime
279
+
280
+ # Basic sources.list
281
+ mv ubuntu/etc/apt/sources.list ubuntu/etc/apt/sources.list.orig || true
282
+ cat <<EOF >ubuntu/etc/apt/sources.list
283
+ deb http://us.archive.ubuntu.com/ubuntu $codename main restricted universe multiverse
284
+ deb-src http://us.archive.ubuntu.com/ubuntu $codename main restricted universe multiverse
285
+
286
+ deb http://us.archive.ubuntu.com/ubuntu $codename-updates main restricted universe multiverse
287
+ deb-src http://us.archive.ubuntu.com/ubuntu $codename-updates main restricted universe multiverse
288
+
289
+ deb http://security.ubuntu.com/ubuntu $codename-security main restricted universe multiverse
290
+ deb-src http://security.ubuntu.com/ubuntu $codename-security main restricted universe multiverse
291
+ EOF
292
+
293
+ # Update package list
294
+ chroot ubuntu apt-get update
295
+
296
+ # Architecture/release specific instructions
297
+ if [ "$bundlearch" = "i386" ]; then
298
+ if [ "$codename" = "edgy" -o \
299
+ "$codename" = "feisty" -o \
300
+ "$codename" = "gutsy" ]; then
301
+ chroot ubuntu apt-get install -y libc6-xen
302
+ fi
303
+ if [ "$codename" = "hardy" ]; then
304
+ # tls seems to have a bug on hardy: perl -e 'glob("xxx*")'
305
+ mv ubuntu/lib/tls ubuntu/lib/tls.disabled
306
+ fi
307
+ else
308
+ chroot ubuntu apt-get install -y ia32-libs
309
+ fi
310
+
311
+ # MAKEDEV is expected in /dev by some packages.
312
+ ln -s /sbin/MAKEDEV ubuntu/dev/MAKEDEV
313
+
314
+ # Upgrade/install packages
315
+ chroot ubuntu apt-get -y upgrade
316
+ chroot ubuntu apt-get install -y $packages
317
+
318
+ # EC2 kernel modules
319
+ for module in $modules; do
320
+ curl -s $module | tar xzC ubuntu
321
+ done
322
+ chroot ubuntu depmod -a
323
+
324
+ # Xen expects a single tty1
325
+ /bin/rm -f ubuntu/etc/event.d/tty[2-6]
326
+
327
+ # Security
328
+ chroot ubuntu shadowconfig on
329
+ chroot ubuntu passwd -d root
330
+
331
+ # Basic networking
332
+ cat <<'EOF' >ubuntu/etc/network/interfaces
333
+ auto lo
334
+ iface lo inet loopback
335
+
336
+ auto eth0
337
+ iface eth0 inet dhcp
338
+ EOF
339
+
340
+ cat <<'EOF' >ubuntu/etc/hosts
341
+ 127.0.0.1 localhost.localdomain localhost
342
+
343
+ # The following lines are desirable for IPv6 capable hosts
344
+ ::1 ip6-localhost ip6-loopback
345
+ fe00::0 ip6-localnet
346
+ ff00::0 ip6-mcastprefix
347
+ ff02::1 ip6-allnodes
348
+ ff02::2 ip6-allrouters
349
+ ff02::3 ip6-allhosts
350
+ EOF
351
+
352
+ cat <<'EOF' >>ubuntu/etc/ssh/sshd_config
353
+ UseDNS no
354
+ EOF
355
+
356
+ # Fedora 6/8 initrd starts nash-hotplug which uses 100% CPU and prevents
357
+ # udev from starting, not to mention slowing down everything else.
358
+ cat <<EOF >ubuntu/etc/init.d/ec2-killall-nash-hotplug
359
+ #!/bin/sh
360
+ /usr/bin/killall nash-hotplug
361
+ EOF
362
+ chmod 755 ubuntu/etc/init.d/ec2-killall-nash-hotplug
363
+ ln -s ../init.d/ec2-killall-nash-hotplug ubuntu/etc/rcS.d/S00ec2-killall-nash-hotplug
364
+
365
+ # motd
366
+ cat <<EOF >ubuntu/etc/rc.local
367
+ #!/bin/sh -e
368
+
369
+ # Get ssh credentials from instance parameters or ephemeral storage.
370
+ /usr/local/sbin/ec2-get-credentials
371
+
372
+ # If instance user-data starts with "#!" then run it on first boot.
373
+ /usr/local/sbin/ec2-run-user-data
374
+ EOF
375
+
376
+ # Script to run user-data if it looks like a script
377
+ curl -s -o ubuntu/usr/local/sbin/ec2-run-user-data \
378
+ http://ec2ubuntu.googlecode.com/svn/trunk/bin/ec2-run-user-data
379
+ chmod 755 ubuntu/usr/local/sbin/ec2-run-user-data
380
+
381
+ # ec2-get-credentials
382
+ cat <<'EOF' >ubuntu/usr/local/sbin/ec2-get-credentials
383
+ #!/bin/bash
384
+ # Retrieve the ssh credentials and add to authorized_keys file.
385
+ # Based on /usr/local/sbin/ec2-get-credentials from ami-20b65349
386
+ public_key_url=http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
387
+ public_key_file=/tmp/openssh_id.pub
388
+ public_key_ephemeral=/mnt/openssh_id.pub
389
+ authorized_keys=/root/.ssh/authorized_keys
390
+ test -d /root/.ssh || mkdir -p -m 700 /root/.ssh
391
+ perl -MIO::Socket::INET -e '
392
+ until(new IO::Socket::INET("169.254.169.254:80")){print"Waiting for network...\n";sleep 1}
393
+ ' | logger -t "ec2"
394
+ curl --retry 3 --silent --fail -o $public_key_file $public_key_url
395
+ if [ $? -eq 0 -a -e $public_key_file ] ; then
396
+ if ! grep -s -q -f $public_key_file $authorized_keys
397
+ then
398
+ cat $public_key_file >> $authorized_keys
399
+ echo "New ssh key added to $authorized_keys from $public_key_url" |
400
+ logger -t "ec2"
401
+ fi
402
+ chmod 600 $authorized_keys
403
+ rm -f $public_key_file
404
+ elif [ -e $public_key_ephemeral ] ; then
405
+ if ! grep -s -q -f $public_key_ephemeral $authorized_keys
406
+ then
407
+ cat $public_key_ephemeral >> $authorized_keys
408
+ echo "New ssh key added to $authorized_keys from $public_key_ephemeral" |
409
+ logger -t "ec2"
410
+ fi
411
+ chmod 600 $authorized_keys
412
+ chmod 600 $public_key_ephemeral
413
+ fi
414
+ EOF
415
+ chmod 755 ubuntu/usr/local/sbin/ec2-get-credentials
416
+
417
+ # Install and patch Amazon AMI tools
418
+ curl -s http://s3.amazonaws.com/ec2-downloads/ec2-ami-tools.noarch.rpm \
419
+ > ubuntu/tmp/ec2-ami-tools.noarch.rpm
420
+ chroot ubuntu alien -i --scripts /tmp/ec2-ami-tools.noarch.rpm
421
+
422
+ ln -s /usr/lib/site_ruby/aes ubuntu/usr/local/lib/site_ruby/1.8/aes
423
+ ln -s /usr/lib/site_ruby/ec2 ubuntu/usr/local/lib/site_ruby/1.8/ec2
424
+
425
+ chroot ubuntu patch -d /usr/lib/site_ruby/aes/amiutil <<'EOF'
426
+ --- /usr/lib/site_ruby/aes/amiutil/http.rb.orig 2008-04-02 22:27:01.000000000 -0400
427
+ +++ /usr/lib/site_ruby/aes/amiutil/http.rb 2008-04-02 22:28:05.000000000 -0400
428
+ @@ -58,7 +58,7 @@
429
+ tf.close(false)
430
+
431
+ begin
432
+ - cmd_line = "curl -f #{curl_arguments} #{url} 2> #{tf.path} | tee #{path} | openssl sha1; exit ${PIPESTATUS[0]}"
433
+ + cmd_line = "/bin/bash -c 'curl -f #{curl_arguments} #{url} 2> #{tf.path} | tee #{path} | openssl sha1; exit ${PIPESTATUS[0]}'"
434
+ calculated_digest = IO.popen( cmd_line ) { |io| io.readline.chomp }
435
+
436
+ unless $?.exitstatus == 0
437
+ --- /usr/lib/site_ruby/aes/amiutil/bundlevol.rb.orig 2008-04-03 00:57:26.000000000 -0400
438
+ +++ /usr/lib/site_ruby/aes/amiutil/bundlevol.rb 2008-04-03 00:58:42.000000000 -0400
439
+ @@ -148,6 +148,10 @@
440
+ if image_file.index( volume ) == 0
441
+ exclude << image_file
442
+ end
443
+ +
444
+ + # UGLY HACK for Ubuntu: Don't save the MAC address as that prevents
445
+ + # networking from working on rebundled AMIs
446
+ + exclude << "/etc/udev/rules.d/70-persistent-net.rules"
447
+
448
+ # If we are inheriting instance data but can't access it we want to fail early
449
+ if p.inherit && !AES::AmiUtils::InstanceData.new.instance_data_accessible
450
+ EOF
451
+
452
+ chroot ubuntu patch -d /usr/lib/site_ruby/ec2/platform/base <<'EOF'
453
+ --- /usr/lib/site_ruby/ec2/platform/base/pipeline.rb.orig 2008-04-02 19:41:54.000000000 -0700
454
+ +++ /usr/lib/site_ruby/ec2/platform/base/pipeline.rb 2008-04-02 19:47:17.000000000 -0700
455
+ @@ -105,7 +105,8 @@
456
+ @results = []
457
+ create_tempfiles
458
+
459
+ - invocation = command
460
+ + # UGLY HACK for Ubuntu: Assumes no single quotes (') in command.
461
+ + invocation = "/bin/bash -c '" + command + "'"
462
+
463
+ # Execute the pipeline invocation
464
+ STDERR.puts("Pipeline.execute: command = [#{invocation}]") if verbose
465
+ EOF
466
+
467
+ cat <<EOF >ubuntu/etc/init.d/ec2-mkdir-tmp
468
+ #!/bin/sh
469
+ # Create /tmp if missing (as it's nice to bundle without it).
470
+ test -d /tmp || mkdir /tmp
471
+ chmod 01777 /tmp
472
+ EOF
473
+ chmod +x ubuntu/etc/init.d/ec2-mkdir-tmp
474
+ ln -s ../init.d/ec2-mkdir-tmp ubuntu/etc/rcS.d/S36ec2-mkdir-tmp
475
+
476
+
477
+ # Generate new ssh host keys on first boot
478
+ # Otherwise everybody knows our secret host key which makes it non-secret
479
+ cat <<EOF >ubuntu/etc/init.d/ec2-ssh-host-key-gen
480
+ #!/bin/sh
481
+ rm -f /etc/ssh/ssh_host_*_key*
482
+ ssh-keygen -f /etc/ssh/ssh_host_rsa_key -t rsa -C 'host' -N '' | logger -t \$0
483
+ ssh-keygen -f /etc/ssh/ssh_host_dsa_key -t dsa -C 'host' -N '' | logger -t \$0
484
+ rm -f /etc/rcS.d/S50ec2-ssh-host-key-gen
485
+ EOF
486
+ chmod +x ubuntu/etc/init.d/ec2-ssh-host-key-gen
487
+ ln -s ../init.d/ec2-ssh-host-key-gen ubuntu/etc/rcS.d/S50ec2-ssh-host-key-gen
488
+
489
+ # Inside Xen, CMOS clock is irrelevant, so save seconds at boot
490
+ /bin/rm -f ubuntu/etc/rc?.d/*hwclock*
491
+
492
+ # We're missing the apparmor kernel module for now so avoid boot warnings
493
+ /bin/rm -f ubuntu/etc/rc?.d/*apparmor
494
+
495
+ # Release-specific
496
+ if [ "$codename" = "gutsy" ]; then
497
+ # Install a copy of rsync with no lutimes support, as the Amazon EC2
498
+ # kernel does not support this newer feature to change symlink timestamps.
499
+ curl -s -o ubuntu/tmp/rsync_2.6.9-5ubuntu1_$bsarch.deb \
500
+ https://level22.s3.amazonaws.com/20080203-rsync-no-lutimes/rsync_2.6.9-5ubuntu1_$bsarch.deb
501
+ chroot ubuntu dpkg --install /tmp/rsync_2.6.9-5ubuntu1_$bsarch.deb
502
+ fi
503
+
504
+ # Are we installing the desktop with NoMachine NX?
505
+ if [ "$desktop" = "nx" ]; then
506
+ if [ "$bundlearch" = "i386" ]; then
507
+ nxclient_pkg="nxclient_3.2.0-9_i386.deb"
508
+ nxnode_pkg="nxnode_3.2.0-5_i386.deb"
509
+ nxserver_pkg="nxserver_3.2.0-7_i386.deb"
510
+ else
511
+ nxclient_pkg="nxclient_3.2.0-9_x86_64.deb"
512
+ nxnode_pkg="nxnode_3.2.0-5_x86_64.deb"
513
+ nxserver_pkg="nxserver_3.2.0-7_x86_64.deb"
514
+ fi
515
+
516
+ curl -o ubuntu/tmp/$nxclient_pkg \
517
+ http://64.34.161.181/download/3.2.0/Linux/$nxclient_pkg
518
+ curl -o ubuntu/tmp/$nxnode_pkg \
519
+ http://64.34.161.181/download/3.2.0/Linux/$nxnode_pkg
520
+ curl -o ubuntu/tmp/$nxserver_pkg \
521
+ http://64.34.161.181/download/3.2.0/Linux/FE/$nxserver_pkg
522
+
523
+ chroot ubuntu dpkg -i /tmp/$nxclient_pkg
524
+ chroot ubuntu dpkg -i /tmp/$nxnode_pkg
525
+ chroot ubuntu dpkg -i /tmp/$nxserver_pkg
526
+ fi
527
+
528
+ # Call external script if requested
529
+ for script in $scripts; do
530
+ if [ -f "$script" ]; then
531
+ if [ -e "$script" ]; then
532
+ "$script"
533
+ else
534
+ /bin/bash "$script"
535
+ fi
536
+ fi
537
+ done
538
+
539
+ ############################## CUSTOM INSTALLS ####################################
540
+ # BUILD AMI HERE
541
+ ###################################################################################
542
+
543
+ # cleanup
544
+ if [ "$codename" != "dapper" ]; then
545
+ chroot ubuntu apt-get -y autoremove --purge patch alien
546
+ fi
547
+ chroot ubuntu apt-get clean
548
+ rm -f ubuntu/usr/sbin/policy-rc.d
549
+ rm -rf ubuntu/tmp/* ubuntu/root/.bash_history
550
+
551
+ # Bundle & upload to S3
552
+ cp $EC2_PRIVATE_KEY ubuntu/tmp/pk.pem
553
+ cp $EC2_CERT ubuntu/tmp/cert.pem
554
+
555
+ chroot ubuntu ec2-bundle-vol \
556
+ -r $bundlearch \
557
+ -d /tmp \
558
+ -p $prefix \
559
+ -u $AWS_USER_ID \
560
+ -k /tmp/pk.pem \
561
+ -c /tmp/cert.pem \
562
+ -s $size \
563
+ -e /tmp
564
+
565
+ ec2-upload-bundle \
566
+ --retry \
567
+ -b $bucket \
568
+ -m ubuntu/tmp/$prefix.manifest.xml \
569
+ -a $AWS_access_key \
570
+ -s $AWS_SECRET_ACCESS_KEY
571
+
572
+ umount ubuntu/dev/pts
573
+ umount ubuntu/proc
574
+
575
+ set +x
576
+
577
+ cat <<EOF
578
+
579
+ Now you might want to run this command:
580
+ ec2-register $bucket/$prefix.manifest.xml
581
+
582
+ EOF
@@ -0,0 +1,29 @@
1
+ global
2
+ daemon
3
+ maxconn 4096 # total max connections (dependent on ulimit)
4
+ nbproc 1 # number of processing cores
5
+ user haproxy
6
+ chroot /var/run/haproxy
7
+ log 127.0.0.1 local0 debug
8
+ pidfile /var/run/haproxy.pid
9
+
10
+ defaults
11
+ mode http
12
+ log global # Log to the global log
13
+ clitimeout 150000
14
+ srvtimeout 150000
15
+ contimeout 30000
16
+
17
+ option httplog
18
+ option httpclose # disable keepalive (HAProxy does not yet support the HTTP keep-alive mode)
19
+ option forwardfor # enable insert of X-Forwarded-For headers
20
+ retries 3
21
+
22
+ balance roundrobin # each server is used in turns, according to assigned weight
23
+
24
+
25
+ stats uri /haproxy
26
+ stats realm Statistics\ for\ PoolParty
27
+
28
+ listen web_proxy 0.0.0.0::host_port
29
+ :servers
@@ -0,0 +1,9 @@
1
+ keepalive 2 # Time inbetween heartbeats
2
+ deadtime 32 # wait 32 seconds before declaring dead
3
+ warntime 16 # issue a warning halfway through
4
+ initdead 60 # Don't boot instances between 60 seconds of each other
5
+ baud 19200 #
6
+ auto_failback on
7
+ use_logd yes
8
+
9
+ :nodes
@@ -0,0 +1,2 @@
1
+ auth 1
2
+ 1 md5 PasswordForPoolPartyClusterApplicationButItIsOnlyADefault
@@ -0,0 +1,7 @@
1
+ check process haproxy with pidfile /var/run/haproxy.pid
2
+ start program = "/usr/sbin/haproxy -f /etc/haproxy.cfg -p /var/run/haproxy.pid"
3
+ stop program = "/usr/bin/killall -9 haproxy"
4
+ if totalmem is greater than 100.0 MB for 4 cycles then restart
5
+ if cpu is greater than 50% for 2 cycles then alert
6
+ if cpu is greater than 80% for 3 cycles then restart
7
+ if loadavg(5min) greater than 10 for 8 cycles then restart
File without changes
data/config/monit.conf ADDED
@@ -0,0 +1,8 @@
1
+ # Configuration for Monit, a monitoring tool
2
+ set daemon 20 # Perform a check every 20 seconds
3
+ set logfile /var/log/monit.log
4
+
5
+ set httpd port 2812 and
6
+ use address 0.0.0.0
7
+
8
+ include /etc/monit.d/*
data/config/nginx.conf ADDED
@@ -0,0 +1,24 @@
1
+ user usr usr;
2
+ worker_processes 2;
3
+
4
+ http {
5
+ sendfile on;
6
+ tcp_nopush on;
7
+
8
+ keepalive_timeout 65;
9
+ tcp_nodelay on;
10
+
11
+ upstream fast_mongrels { server 127.0.0.1:8100; }
12
+
13
+ server {
14
+ listen 80;
15
+ server_name srv;
16
+ root /home/usr;
17
+
18
+ location / {
19
+ proxy_pass http://fast_mongrels;
20
+ break;
21
+ }
22
+
23
+ }
24
+ }
data/lib/core/array.rb ADDED
@@ -0,0 +1,10 @@
1
+ =begin rdoc
2
+ Array extensions
3
+ =end
4
+ require "enumerator"
5
+ class Array
6
+ # Collection with the index
7
+ def collect_with_index &block
8
+ self.enum_for(:each_with_index).collect &block
9
+ end
10
+ end
@@ -0,0 +1,9 @@
1
+ =begin rdoc
2
+ Exception overloads
3
+ =end
4
+ class Exception
5
+ # Gives us a nice_message for exceptions
6
+ def nice_message(padding="")
7
+ "#{padding}#{message}\n#{padding}" + backtrace.join("\n#{padding}")
8
+ end
9
+ end