bosh-gen 0.16.2 → 0.17.0

data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/bin/s3test_ctl

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# Setup env vars and folders for the webapp_ctl script
+source /var/vcap/jobs/s3test/helpers/ctl_setup.sh 's3test'
+
+export PORT=${PORT:-5000}
+export LANG=en_US.UTF-8
+
+case $1 in
+
+  start)
+    pid_guard $PIDFILE $JOB_NAME
+
+    # TODO: Run some process
+    exec chpst -u vcap:vcap TODO \
+         >>$LOG_DIR/$JOB_NAME.stdout.log \
+         2>>$LOG_DIR/$JOB_NAME.stderr.log
+
+    # store pid in $PIDFILE
+    echo $! > $PIDFILE
+    ;;
+
+  stop)
+    kill_and_wait $PIDFILE
+
+    ;;
+  *)
+    echo "Usage: s3test_ctl {start|stop}"
+
+    ;;
+
+esac
+exit 0

data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/config/s3test.conf.erb

@@ -0,0 +1,5 @@
+# Example of how to do one thing when job is master/leader, and another thing if its follower/slave
+# See templates/infrastructure-warden.yml & templates/jobs.yml for where this value originates
+<% if_p("s3test.leader_address") do |leader_address| %>
+leader: <%= leader_address %>
+<% end %>

data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/data/properties.sh.erb

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# job template binding variables
+
+# job name & index of this VM within cluster
+# e.g. JOB_NAME=redis, JOB_INDEX=0
+export NAME='<%= name %>'
+export JOB_INDEX=<%= index %>
+# full job name, like redis/0 or webapp/3
+export JOB_FULL="$NAME/$JOB_INDEX"

data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/helpers/ctl_setup.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+# Setup env vars and folders for the ctl script
+# This helps keep the ctl script as readable
+# as possible
+
+# Usage options:
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh JOB_NAME OUTPUT_LABEL
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar nginx
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+JOB_NAME=$1
+output_label=${1:-JOB_NAME}
+
+export JOB_DIR=/var/vcap/jobs/$JOB_NAME
+chmod 755 $JOB_DIR # to access file via symlink
+
+# Load some bosh deployment properties into env vars
+# Try to put all ERb into data/properties.sh.erb
+# incl $NAME, $JOB_INDEX, $WEBAPP_DIR
+source $JOB_DIR/data/properties.sh
+
+source $JOB_DIR/helpers/ctl_utils.sh
+redirect_output ${output_label}
+
+export HOME=${HOME:-/home/vcap}
+
+# Add all packages' /bin & /sbin into $PATH
+for package_bin_dir in $(ls -d /var/vcap/packages/*/*bin)
+do
+  export PATH=${package_bin_dir}:$PATH
+done
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-''} # default to empty
+for package_bin_dir in $(ls -d /var/vcap/packages/*/lib)
+do
+  export LD_LIBRARY_PATH=${package_bin_dir}:$LD_LIBRARY_PATH
+done
+
+# Setup log, run and tmp folders
+
+export RUN_DIR=/var/vcap/sys/run/$JOB_NAME
+export LOG_DIR=/var/vcap/sys/log/$JOB_NAME
+export TMP_DIR=/var/vcap/sys/tmp/$JOB_NAME
+export STORE_DIR=/var/vcap/store/$JOB_NAME
+for dir in $RUN_DIR $LOG_DIR $TMP_DIR $STORE_DIR
+do
+  mkdir -p ${dir}
+  chown vcap:vcap ${dir}
+  chmod 775 ${dir}
+done
+export TMPDIR=$TMP_DIR
+
+export C_INCLUDE_PATH=/var/vcap/packages/mysqlclient/include/mysql:/var/vcap/packages/sqlite/include:/var/vcap/packages/libpq/include
+export LIBRARY_PATH=/var/vcap/packages/mysqlclient/lib/mysql:/var/vcap/packages/sqlite/lib:/var/vcap/packages/libpq/lib
+
+# consistent place for vendoring python libraries within package
+if [[ -d ${WEBAPP_DIR:-/xxxx} ]]
+then
+  export PYTHONPATH=$WEBAPP_DIR/vendor/lib/python
+fi
+
+if [[ -d /var/vcap/packages/java7 ]]
+then
+  export JAVA_HOME="/var/vcap/packages/java7"
+fi
+
+# setup CLASSPATH for all jars/ folders within packages
+export CLASSPATH=${CLASSPATH:-''} # default to empty
+for java_jar in $(ls -d /var/vcap/packages/*/*/*.jar)
+do
+  export CLASSPATH=${java_jar}:$CLASSPATH
+done
+
+PIDFILE=$RUN_DIR/$JOB_NAME.pid
+
+echo '$PATH' $PATH

data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/helpers/ctl_utils.sh

@@ -0,0 +1,156 @@
+# Helper functions used by ctl scripts
+
+# links a job file (probably a config file) into a package
+# Example usage:
+# link_job_file_to_package config/redis.yml [config/redis.yml]
+# link_job_file_to_package config/wp-config.php wp-config.php
+link_job_file_to_package() {
+  source_job_file=$1
+  target_package_file=${2:-$source_job_file}
+  full_package_file=$WEBAPP_DIR/${target_package_file}
+
+  link_job_file ${source_job_file} ${full_package_file}
+}
+
+# links a job file (probably a config file) somewhere
+# Example usage:
+# link_job_file config/bashrc /home/vcap/.bashrc
+link_job_file() {
+  source_job_file=$1
+  target_file=$2
+  full_job_file=$JOB_DIR/${source_job_file}
+
+  echo link_job_file ${full_job_file} ${target_file}
+  if [[ ! -f ${full_job_file} ]]
+  then
+    echo "file to link ${full_job_file} does not exist"
+  else
+    # Create/recreate the symlink to current job file
+    # If another process is using the file, it won't be
+    # deleted, so don't attempt to create the symlink
+    mkdir -p $(dirname ${target_file})
+    ln -nfs ${full_job_file} ${target_file}
+  fi
+}
+
+# If loaded within monit ctl scripts then pipe output
+# If loaded from 'source ../utils.sh' then normal STDOUT
+redirect_output() {
+  SCRIPT=$1
+  mkdir -p /var/vcap/sys/log/monit
+  exec 1>> /var/vcap/sys/log/monit/$SCRIPT.log
+  exec 2>> /var/vcap/sys/log/monit/$SCRIPT.err.log
+}
+
+pid_guard() {
+  pidfile=$1
+  name=$2
+
+  if [ -f "$pidfile" ]; then
+    pid=$(head -1 "$pidfile")
+
+    if [ -n "$pid" ] && [ -e /proc/$pid ]; then
+      echo "$name is already running, please stop it first"
+      exit 1
+    fi
+
+    echo "Removing stale pidfile..."
+    rm $pidfile
+  fi
+}
+
+wait_pid() {
+  pid=$1
+  try_kill=$2
+  timeout=${3:-0}
+  force=${4:-0}
+  countdown=$(( $timeout * 10 ))
+
+  echo wait_pid $pid $try_kill $timeout $force $countdown
+  if [ -e /proc/$pid ]; then
+    if [ "$try_kill" = "1" ]; then
+      echo "Killing $pidfile: $pid "
+      kill $pid
+    fi
+    while [ -e /proc/$pid ]; do
+      sleep 0.1
+      [ "$countdown" != '0' -a $(( $countdown % 10 )) = '0' ] && echo -n .
+      if [ $timeout -gt 0 ]; then
+        if [ $countdown -eq 0 ]; then
+          if [ "$force" = "1" ]; then
+            echo -ne "\nKill timed out, using kill -9 on $pid... "
+            kill -9 $pid
+            sleep 0.5
+          fi
+          break
+        else
+          countdown=$(( $countdown - 1 ))
+        fi
+      fi
+    done
+    if [ -e /proc/$pid ]; then
+      echo "Timed Out"
+    else
+      echo "Stopped"
+    fi
+  else
+    echo "Process $pid is not running"
+    echo "Attempting to kill pid anyway..."
+    kill $pid
+  fi
+}
+
+wait_pidfile() {
+  pidfile=$1
+  try_kill=$2
+  timeout=${3:-0}
+  force=${4:-0}
+  countdown=$(( $timeout * 10 ))
+
+  if [ -f "$pidfile" ]; then
+    pid=$(head -1 "$pidfile")
+    if [ -z "$pid" ]; then
+      echo "Unable to get pid from $pidfile"
+      exit 1
+    fi
+
+    wait_pid $pid $try_kill $timeout $force
+
+    rm -f $pidfile
+  else
+    echo "Pidfile $pidfile doesn't exist"
+  fi
+}
+
+kill_and_wait() {
+  pidfile=$1
+  # Monit default timeout for start/stop is 30s
+  # Append 'with timeout {n} seconds' to monit start/stop program configs
+  timeout=${2:-25}
+  force=${3:-1}
+  if [[ -f ${pidfile} ]]
+  then
+    wait_pidfile $pidfile 1 $timeout $force
+  else
+    # TODO assume $1 is something to grep from 'ps ax'
+    pid="$(ps auwwx | grep "$1" | awk '{print $2}')"
+    wait_pid $pid 1 $timeout $force
+  fi
+}
+
+check_nfs_mount() {
+  opts=$1
+  exports=$2
+  mount_point=$3
+
+  if grep -qs $mount_point /proc/mounts; then
+    echo "Found NFS mount $mount_point"
+  else
+    echo "Mounting NFS..."
+    mount $opts $exports $mount_point
+    if [ $? != 0 ]; then
+      echo "Cannot mount NFS from $exports to $mount_point, exiting..."
+      exit 1
+    fi
+  fi
+}

data/spec/fixtures/releases/s3test-boshrelease/templates/deployment.yml

@@ -0,0 +1,35 @@
+meta:
+  environment: (( merge ))
+  stemcell: (( merge ))
+
+name: (( meta.environment ))
+
+director_uuid: (( merge ))
+
+releases: (( merge ))
+
+networks: (( merge ))
+
+jobs: (( merge ))
+
+properties: (( merge ))
+
+compilation:
+  workers: 6
+  network: s3test1
+  reuse_compilation_vms: true
+  cloud_properties: (( merge ))
+
+update:
+  canaries: 1
+  max_in_flight: 1
+  canary_watch_time: 30000-60000
+  update_watch_time: 5000-60000
+  serial: false
+
+resource_pools:
+  - name: small_z1
+    network: s3test1
+    size: (( auto ))
+    stemcell: (( meta.stemcell ))
+    cloud_properties: (( merge ))

data/spec/fixtures/releases/s3test-boshrelease/templates/infrastructure-aws-ec2.yml

@@ -0,0 +1,40 @@
+meta:
+  environment: (( merge ))
+  dns_root: (( merge ))
+  security_groups: (( merge ))
+  network_name: (( merge ))
+
+  stemcell:
+    name: bosh-aws-xen-ubuntu
+    version: latest
+
+jobs:
+  - name: s3test_leader_z1
+    instances: 1
+    networks:
+      - name: (( meta.network_name ))
+  - name: s3test_z1
+    instances: 2
+    networks:
+      - name: (( meta.network_name ))
+    properties:
+      s3test:
+        leader_address: (( "0.s3test-leader-z1.s3test1." meta.environment "." meta.dns_root ))
+
+compilation:
+  cloud_properties:
+    instance_type: m1.small
+
+resource_pools:
+  - name: small_z1
+    cloud_properties:
+      instance_type: m1.small
+
+networks:
+  - name: floating
+    type: vip
+    cloud_properties: {}
+  - name: (( meta.network_name ))
+    type: dynamic
+    cloud_properties:
+      security_groups: (( meta.security_groups ))

data/spec/fixtures/releases/s3test-boshrelease/templates/infrastructure-warden.yml

@@ -0,0 +1,107 @@
+meta:
+  environment: s3test-warden
+  network_name: (( merge ))
+
+  stemcell:
+    name: bosh-warden-boshlite-ubuntu
+    version: latest
+
+update:
+  canaries: 1
+  max_in_flight: 50
+  canary_watch_time: 1000-30000
+  update_watch_time: 1000-30000
+
+jobs:
+  - name: s3test_leader_z1
+    instances: 1
+    networks:
+      - name: (( meta.network_name ))
+        static_ips: (( static_ips(0) ))
+  - name: s3test_z1
+    instances: 2
+    networks:
+      - name: (( meta.network_name ))
+        static_ips: ~
+    properties:
+      s3test:
+        leader_address: (( jobs.s3test_leader_z1.networks.s3test1.static_ips.[0] ))
+
+compilation:
+  cloud_properties:
+    name: random
+
+resource_pools:
+  - name: small_z1
+    cloud_properties:
+      name: random
+
+networks:
+- name: (( meta.network_name ))
+  # Assumes up to 5 VMs, including 1 static and 4 dynamic.
+  # Plus 5 (double the size) unused IPs, due to BOSH bug/quirk.
+  subnets:
+  - cloud_properties:
+      name: random
+    range: 10.244.2.0/30
+    reserved:
+    - 10.244.2.1
+    static:
+    - 10.244.2.2
+
+  - cloud_properties:
+      name: random
+    range: 10.244.2.4/30
+    reserved:
+    - 10.244.2.5
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.8/30
+    reserved:
+    - 10.244.2.9
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.12/30
+    reserved:
+    - 10.244.2.13
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.16/30
+    reserved:
+    - 10.244.2.17
+    static: []
+
+  # Bonus double-sized network required due to BOSH oddity
+  - cloud_properties:
+      name: random
+    range: 10.244.2.20/30
+    reserved:
+    - 10.244.2.21
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.24/30
+    reserved:
+    - 10.244.2.25
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.28/30
+    reserved:
+    - 10.244.2.29
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.32/30
+    reserved:
+    - 10.244.2.33
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.36/30
+    reserved:
+    - 10.244.2.37
+    static: []