stemcell_builder 1.0.8

data/templates/ubuntu/base-stemcell.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+set -x
+
+source _helpers.sh
+source _variables.sh
+
+PACKAGES="build-essential,libssl-dev,lsof,strace,bind9-host,dnsutils,iputils-arping,curl,wget,libcurl3,bison,libreadline6-dev,libxml2,libxml2-dev,libxslt1.1,libxslt1-dev,zip,unzip,nfs-common,flex,psmisc,iptables,sysstat,rsync,openssh-server,libncurses5-dev,quota,libaio1,psmisc,dialog"
+CHROOT=/tmp/chroot
+[ -d $CHROOT ] && rm -rf $CHROOT && mkdir -p $CHROOT
+
+## Check if stemcell_base.tar.gz already exists
+[ -f $bosh_app_dir/stemcell_base.tar.gz ] && exit 0
+
+pushd /tmp
+    ARCH=$(dpkg --print-architecture)
+    debootstrap --make-tarball=debootstrap-squeeze-tarball.tar --arch=$ARCH  --include=$PACKAGES squeeze $CHROOT
+    debootstrap --variant=minbase --unpack-tarball=/tmp/debootstrap-squeeze-tarball.tar squeeze $CHROOT
+    # Creates a pristine image to be run inside a warden container.
+    hostname="ubuntu.defaultdomain ubuntu"
+    rm -f $CHROOT/var/lib/apt/lists/{archive,security,lock}*
+
+    # Reconfigure timezone and locale
+    echo 'en_US.UTF-8 UTF-8' > $CHROOT/etc/locale.gen
+    echo 'UTC' > $CHROOT/etc/timezone
+    run_in_chroot $CHROOT "
+    dpkg-reconfigure -fnoninteractive libc6
+    dpkg-reconfigure -fnoninteractive locales
+    dpkg-reconfigure -fnoninteractive tzdata
+    "
+
+    # Fix /etc/mtab
+    run_in_chroot $CHROOT "ln -s /proc/mounts /etc/mtab"
+
+   # configure the network using the dhcp
+    cat <<EOF > $CHROOT/etc/network/interfaces
+# This file describes the network interfaces available on your system
+# and how to activate them. For more information, see interfaces(5).
+
+# The loopback network interface
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet dhcp
+EOF
+
+    # set the hostname
+    cat <<EOF > $CHROOT/etc/hostname
+$hostname
+EOF
+    # set minimal hosts
+    cat <<EOF > $CHROOT/etc/hosts
+127.0.0.1   localhost
+127.0.1.1   $hostname
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+EOF
+
+    # Install Firstboot script
+    cat <<EOF > $CHROOT/etc/rc.local
+#!/bin/sh -e
+#execute firstboot.sh only once
+if [ ! -e /root/firstboot_done ]; then
+    if [ -e /root/firstboot.sh ]; then
+        /root/firstboot.sh
+    fi
+    touch /root/firstboot_done
+fi
+exit 0
+EOF
+    cat <<EOF > $CHROOT/root/firstboot.sh
+#!/bin/sh
+rm /etc/resolv.conf
+touch /etc/resolv.conf
+rm /etc/ssh/ssh_host*key*
+/etc/init.d/networking restart
+dpkg-reconfigure -fnoninteractive -pcritical openssh-server
+dpkg-reconfigure -fnoninteractive sysstat
+EOF
+    chmod 0755 $CHROOT/root/firstboot.sh
+
+    # perform clean up
+    run_in_chroot $CHROOT "
+apt-get clean
+apt-get autoremove
+"
+    echo "Creating base stemcell archive at $bosh_app_dir/stemcell_base.tar.gz"
+    tar -C $CHROOT -czf $bosh_app_dir/stemcell_base.tar.gz .
+    chmod 0700 $bosh_app_dir/stemcell_base.tar.gz
+popd

data/templates/ubuntu/bosh_agent.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -x
+
+source _variables.sh
+
+mkdir -p /tmp/bosh_agent
+echo -n $infrastructure > /etc/infrastructure
+
+pushd /tmp/bosh_agent
+    cp $SRC_DIR/_bosh_agent.tar /tmp/bosh_agent
+    tar xmvf _bosh_agent.tar
+    $bosh_dir/bin/gem install *.gem --force --no-ri --no-rdoc
+    chmod +x $bosh_dir/bin/bosh_agent
+
+    # configure bosh agent
+    mkdir -p /etc/sv/agent/log
+    mkdir -p /var/vcap/bosh/log
+
+    echo '#!/bin/bash
+    export PATH=/var/vcap/bosh/bin:$PATH
+    exec 2>&1
+    exec /var/vcap/bosh/bin/bosh_agent --configure --infrastructure=`cat /etc/infrastructure` --platform=ubuntu
+    ' > /etc/sv/agent/run
+
+    echo '#!/bin/bash
+    svlogd -tt /var/vcap/bosh/log
+    ' > /etc/sv/agent/log/run
+
+    # runit
+    chmod +x /etc/sv/agent/run /etc/sv/agent/log/run
+
+    ln -s /etc/sv/agent /etc/service/agent
+
+    cp $SRC_DIR/_empty_state.yml $bosh_dir/state.yml
+
+    # The bosh agent installs a config that rotates on size
+    mv /etc/cron.daily/logrotate /etc/cron.hourly/logrotate
+
+popd

data/templates/ubuntu/definition.rb.erb

@@ -0,0 +1,63 @@
+Veewee::Definition.declare({
+                               :cpu_count => '1', :memory_size => '512',
+                               :disk_size => '10140', :disk_format => 'VDI', :hostiocache => 'off',
+                               :virtualbox => { :vm_options => ["pae" => "on", "ioapic" => "on"]},
+                               :use_sata => false,
+                               :os_type_id => 'Ubuntu_64',
+                               :iso_file => "<%= @iso_filename %>",
+                               :iso_src => "<%= @iso %>",
+                               :iso_md5 => "<%= @iso_md5 %>",
+                               :iso_download_timeout => "1000",
+                               :boot_wait => "5", :boot_cmd_sequence => [
+        '<Esc><Esc><Enter>',
+        '/install/vmlinuz ',
+        'initrd=/install/initrd.gz ',
+        'noapic ',
+        'fb=false ', # don't bother using a framebuffer
+        'locale=en_US ', # Start installer in English
+        'console-setup/ask_detect=false ', # Don't ask to detect keyboard
+        'keyboard-configuration/layout=USA ', # set it to US qwerty
+        'keyboard-configuration/variant=USA ',
+        'hostname=%NAME% ', # Set the hostname
+        'preseed/url=http://%IP%:%PORT%/preseed.cfg ', # Fetch the rest from here
+        'auto ',
+        'debconf/frontend=noninteractive ',
+        'debian-installer=en_US ',
+        'kbd-chooser/method=us ',
+        '-- <Enter>'
+    ],
+
+                               :kickstart_port => "7122", :kickstart_timeout => "10000", :kickstart_file => "preseed.cfg",
+                               :ssh_login_timeout => "10000", :ssh_user => "vcap", :ssh_password => "c1oudc0w", :ssh_key => "",
+                               :ssh_host_port => "<%= @ssh_port %>", :ssh_guest_port => "22",
+                               :sudo_cmd => "echo '%p'|sudo -S /bin/bash '%f'",
+                               :shutdown_cmd => "shutdown -P now",
+                               :postinstall_files => [
+                                   # Files with a leading _ are not executed
+                                   "_60-bosh-sysctl.conf",
+                                   "_monitrc",
+                                   "_ntpdate",
+                                   "_sysstat",
+                                   "_empty_state.yml",
+                                   "_variables.sh",
+                                   "_helpers.sh",
+                                   "_runonce",
+                                   "_bosh_agent.tar",
+                                   # The following scripts are run on the target vm one by one
+                                   
+                                   "apt-upgrade.sh",
+                                   "timestamp.sh",
+                                   "sudo.sh",
+                                   "setup-bosh.sh",
+                                   "base-stemcell.sh",
+                                   "monit.sh",
+                                   "ruby.sh",
+                                   "bosh_agent.sh",
+                                   "vmware-tools.sh",
+                                   "network-cleanup.sh",
+                                   "zero-disk.sh",
+                                   "harden.sh",
+                                   "postinstall.sh",
+                               ],
+                               :postinstall_timeout => "10000"
+                           })

data/templates/ubuntu/harden.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+
+### stage bosh_harden
+# remove setuid binaries - except su/sudo (sudoedit is hardlinked)
+find / -xdev -perm +6000 -a -type f \
+  -a -not \( -name sudo -o -name su -o -name sudoedit \) \
+  -exec chmod ug-s {} \;
+

data/templates/ubuntu/micro.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -x
+
+bosh_app_dir=/var/vcap
+bosh_dir=${bosh_app_dir}/bosh
+infrastructure="vsphere"
+SRC_DIR=`pwd`
+
+blobstore_path=${bosh_app_dir}/micro_bosh/data/cache
+agent_host=localhost
+agent_port=6969
+agent_uri=http://vcap:vcap@${agent_host}:${agent_port}
+export PATH=${bosh_app_dir}/bosh/bin:$PATH
+
+# Packages
+apt-get -y install genisoimage libpq-dev nc g++ libboost-program-options-dev libboost-serialization-dev libpqclient-dev libmysqlclient-dev libsqlite3-dev
+
+# Install package compiler
+if [ ! -f "$bosh_dir/bin/package_compiler" ]
+then
+    mkdir -p /tmp/package_compiler
+    pushd /tmp/package_compiler
+        cp $SRC_DIR/_package_compiler.tar .
+        tar -xvf _package_compiler.tar
+        $bosh_dir/bin/gem install *.gem --no-ri --no-rdoc --local
+    popd
+fi
+
+mkdir -p ${bosh_app_dir}/bosh/blob
+mkdir -p ${blobstore_path}
+
+echo "Starting micro bosh compilation"
+
+# Start agent
+$bosh_dir/bin/bosh_agent -I ${infrastructure} -n ${agent_uri} -s ${blobstore_path} -p local &
+agent_pid=$!
+
+# Wait for agent to come up
+for i in {1..10}
+  do
+    nc -z ${agent_host} ${agent_port} && break
+    sleep 1
+  done
+
+# Start compiler
+$bosh_dir/bin/package_compiler --cpi ${infrastructure} compile $SRC_DIR/_release.yml $SRC_DIR/_release.tgz ${blobstore_path} ${agent_uri}
+
+kill -15 $agent_pid
+
+# Wait for agent
+for i in {1..5}
+do
+  kill -0 $agent_pid && break
+  sleep 1
+done
+# Force kill if required
+kill -0 $agent_pid || kill -9 $agent_pid

data/templates/ubuntu/monit.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -x
+
+source _variables.sh
+
+apt-get -y install libpam-dev
+
+### stage bosh_monit
+mkdir -p $bosh_dir/etc
+cp $SRC_DIR/_monitrc $bosh_dir/etc/monitrc
+chmod 0700 $bosh_dir/etc/monitrc
+
+pushd /tmp
+    [ ! -f "monit-5.5.tar.gz" ] && wget http://mmonit.com/monit/dist/monit-5.5.tar.gz
+    tar xzvf monit-5.5.tar.gz
+    cd monit-5.5
+    ./configure --prefix=$bosh_dir --without-ssl
+    make && make install
+    # monit refuses to start without an include file present
+    mkdir -p $bosh_app_dir/monit
+    touch /$bosh_app_dir/monit/empty.monitrc
+popd

data/templates/ubuntu/network-cleanup.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -x
+
+# Removing leftover leases and persistent rules
+echo "cleaning up dhcp leases"
+rm /var/lib/dhcp3/*
+
+# Make sure Udev doesn't block our network
+# http://6.ptmc.org/?p=164
+echo "cleaning up udev rules"
+rm /etc/udev/rules.d/70-persistent-net.rules
+mkdir /etc/udev/rules.d/70-persistent-net.rules
+rm -rf /dev/.udev/
+rm /lib/udev/rules.d/75-persistent-net-generator.rules
+
+echo "Adding a 2 sec delay to the interface up, to make the dhclient happy"
+echo "pre-up sleep 2" >> /etc/network/interfaces

data/templates/ubuntu/postinstall.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -x
+
+#import helpers scripts
+source _variables.sh
+
+### stage bosh_sysstat
+cp $SRC_DIR/_sysstat /etc/default/sysstat
+
+### stage bosh_sysctl
+cp $SRC_DIR/_60-bosh-sysctl.conf /etc/sysctl.d/60-bosh-sysctl.conf
+chmod 0644 /etc/sysctl.d/60-bosh-sysctl.conf
+
+### stage bosh_ntpdate
+# setup crontab for root to use ntpdate every 15 minutes
+mkdir -p $bosh_dir/log
+cp $SRC_DIR/_ntpdate $bosh_dir/bin/ntpdate
+chmod 0755 $bosh_dir/bin/ntpdate
+echo "0,15,30,45 * * * * ${bosh_dir}/bin/ntpdate" > /tmp/ntpdate.cron
+crontab -u root /tmp/ntpdate.cron
+rm /tmp/ntpdate.cron
+
+### stage micro_bosh ???
+
+sed -i -e 's/^\(timeout=.*\)$/timeout=0/g' /boot/grub/menu.lst
+sed -i -e 's/^\(timeout=.*\)$/timeout=0/g' /boot/grub/grub.conf
+
+
+### stage bosh_dpkg_list ??? How do we get back a list of things
+# Create list of installed packages -- legal requirement
+dpkg -l > $bosh_dir/stemcell_dpkg_l.out
+
+# Clean out all the scripts
+rm -f *.iso *.gem *.tar *.tgz
+
+# install runonce
+mkdir -p /etc/local/runonce.d/ran
+cp $SRC_DIR/_runonce /usr/local/bin/runonce
+chmod +x /usr/local/bin/runonce
+
+# Do some firstboot clean up
+# Regenerate ssh keys
+/usr/local/bin/runonce "rm -f /etc/ssh/ssh_host_*"
+/usr/local/bin/runonce "dpkg-reconfigure -fnoninteractive -pcritical openssh-server"
+

data/templates/ubuntu/preseed.cfg

@@ -0,0 +1,87 @@
+## Options to set on the command line
+d-i debian-installer/locale string en_US.utf8
+d-i console-setup/ask_detect boolean false
+d-i console-setup/layout string USA
+
+#d-i netcfg/get_hostname string dummy
+d-i netcfg/get_hostname string unassigned-hostname
+d-i netcfg/get_domain string unassigned-domain
+
+# Continue without a default route
+# Not working , specify a dummy in the DHCP
+#d-i netcfg/no_default_route boolean
+
+d-i time/zone string UTC
+d-i clock-setup/utc-auto boolean true
+d-i clock-setup/utc boolean true
+
+d-i kbd-chooser/method	select	American English
+
+d-i netcfg/wireless_wep string
+
+d-i base-installer/kernel/override-image string linux-virtual
+#d-i base-installer/kernel/override-image string linux-image-2.6.32-21-generic
+
+# Choices: Dialog, Readline, Gnome, Kde, Editor, Noninteractive
+d-i debconf debconf/frontend select Noninteractive
+
+d-i pkgsel/install-language-support boolean false
+tasksel tasksel/first multiselect standard, ubuntu-server
+
+#d-i partman-auto/method string regular
+d-i partman-auto/method string lvm
+#d-i partman-auto/purge_lvm_from_device boolean true
+
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman-auto/choose_recipe select atomic
+
+d-i partman/confirm_write_new_label boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+
+#http://ubuntu-virginia.ubuntuforums.org/showthread.php?p=9626883
+#Message: "write the changes to disk and configure lvm preseed"
+#http://serverfault.com/questions/189328/ubuntu-kickstart-installation-using-lvm-waits-for-input
+#preseed partman-lvm/confirm_nooverwrite boolean true
+
+# Write the changes to disks and configure LVM?
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+d-i partman-auto-lvm/guided_size string max
+
+## Default user, we can get away with a recipe to change this
+d-i passwd/user-fullname string vcap
+d-i passwd/username string vcap
+d-i passwd/user-password password c1oudc0w
+d-i passwd/user-password-again password c1oudc0w
+d-i user-setup/encrypt-home boolean false
+d-i user-setup/allow-password-weak boolean true
+
+## minimum is puppet and ssh and ntp
+# Individual additional packages to install
+d-i pkgsel/include string openssh-server ntp
+
+# Whether to upgrade packages after debootstrap.
+# Allowed values: none, safe-upgrade, full-upgrade
+d-i pkgsel/upgrade select full-upgrade
+
+d-i grub-installer/only_debian boolean true
+d-i grub-installer/with_other_os boolean true
+d-i finish-install/reboot_in_progress note
+
+#For the update
+d-i pkgsel/update-policy select none
+
+# debconf-get-selections --install
+#Use mirror
+#d-i apt-setup/use_mirror boolean true
+#d-i     mirror/country          string manual
+#choose-mirror-bin mirror/protocol	string http
+#choose-mirror-bin mirror/http/hostname	string 192.168.4.150
+#choose-mirror-bin mirror/http/directory	string /ubuntu
+#choose-mirror-bin mirror/suite	select maverick
+#d-i debian-installer/allow_unauthenticated	string true
+
+choose-mirror-bin mirror/http/proxy string

data/templates/ubuntu/ruby.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -x
+
+source _variables.sh
+
+### stage bosh_ruby
+apt-get -y update
+apt-get -y install build-essential zlib1g-dev libssl-dev libxml2-dev libxslt-dev libreadline6-dev libyaml-dev
+
+# install libyaml
+pushd /tmp
+	[ ! -f "yaml-0.1.4.tar.gz" ] && wget http://pyyaml.org/download/libyaml/yaml-0.1.4.tar.gz
+    tar zxf yaml-0.1.4.tar.gz
+    cd yaml-0.1.4
+    ./configure
+    make
+    make install
+    echo /usr/local/lib >> /etc/ld.so.conf
+    ldconfig
+popd
+
+# install ruby
+pushd /tmp
+	if [ ! -f "$bosh_dir/bin/ruby" ]
+	then
+        [ ! -f "ruby-1.9.3-p374.tar.gz" ] && wget http://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.3-p374.tar.gz
+        tar zxf ruby-1.9.3-p374.tar.gz
+        cd ruby-1.9.3-p374
+        ./configure --prefix=$bosh_dir --disable-install-doc
+        make
+        make install
+	fi
+popd
+# Install rubygems
+pushd /tmp
+	if [ ! -f "$bosh_dir/bin/gem" ]
+	then
+		[ ! -f "rubygems-1.8.24.tgz" ] && wget http://production.cf.rubygems.org/rubygems/rubygems-1.8.24.tgz
+		tar zxf rubygems-1.8.24.tgz
+		cd rubygems-1.8.24
+		$bosh_dir/bin/ruby setup.rb --no-format-executable
+	fi
+popd
+
+echo "PATH=$bosh_dir/bin:$PATH" > /etc/environment
+source /etc/environment
+
+$bosh_dir/bin/gem update --system --no-ri --no-rdoc
+
+mkdir -p $bosh_dir/etc
+echo "gem: --no-rdoc --no-ri" >> $bosh_dir/etc/gemrc
+
+#Install gems
+$bosh_dir/bin/gem install bundler --no-ri --no-rdoc