bosh-gen 0.8.2 → 0.9.0

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_nginx_rack/templates/config/database.yml.erb

@@ -0,0 +1,17 @@
+<% if properties.postgres %>
+production:
+  adapter: postgresql
+  encoding: unicode
+  database: '<%= properties.postgres.database %>'
+  host:     '<%= properties.postgres.host %>'
+  port:     '<%= properties.postgres.port || 5432 %>'
+  username: '<%= properties.postgres.user %>'
+  password: '<%= properties.postgres.password %>'
+  pool: 5
+<% else %>
+production:
+  adapter: sqlite3
+  database: db/production.sqlite3
+  pool: 5
+  timeout: 5000
+<% end %>

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_nginx_rack/templates/config/mime.types

@@ -0,0 +1,73 @@
+types {
+    text/html                             html htm shtml;
+    text/css                              css;
+    text/xml                              xml;
+    image/gif                             gif;
+    image/jpeg                            jpeg jpg;
+    application/x-javascript              js;
+    application/atom+xml                  atom;
+    application/rss+xml                   rss;
+
+    text/mathml                           mml;
+    text/plain                            txt;
+    text/vnd.sun.j2me.app-descriptor      jad;
+    text/vnd.wap.wml                      wml;
+    text/x-component                      htc;
+
+    image/png                             png;
+    image/tiff                            tif tiff;
+    image/vnd.wap.wbmp                    wbmp;
+    image/x-icon                          ico;
+    image/x-jng                           jng;
+    image/x-ms-bmp                        bmp;
+    image/svg+xml                         svg;
+
+    application/java-archive              jar war ear;
+    application/mac-binhex40              hqx;
+    application/msword                    doc;
+    application/pdf                       pdf;
+    application/postscript                ps eps ai;
+    application/rtf                       rtf;
+    application/vnd.ms-excel              xls;
+    application/vnd.ms-powerpoint         ppt;
+    application/vnd.wap.wmlc              wmlc;
+    application/vnd.wap.xhtml+xml         xhtml;
+    application/vnd.google-earth.kml+xml  kml;
+    application/vnd.google-earth.kmz      kmz;
+    application/x-7z-compressed           7z;
+    application/x-cocoa                   cco;
+    application/x-java-archive-diff       jardiff;
+    application/x-java-jnlp-file          jnlp;
+    application/x-makeself                run;
+    application/x-perl                    pl pm;
+    application/x-pilot                   prc pdb;
+    application/x-rar-compressed          rar;
+    application/x-redhat-package-manager  rpm;
+    application/x-sea                     sea;
+    application/x-shockwave-flash         swf;
+    application/x-stuffit                 sit;
+    application/x-tcl                     tcl tk;
+    application/x-x509-ca-cert            der pem crt;
+    application/x-xpinstall               xpi;
+    application/zip                       zip;
+
+    application/octet-stream              bin exe dll;
+    application/octet-stream              deb;
+    application/octet-stream              dmg;
+    application/octet-stream              eot;
+    application/octet-stream              iso img;
+    application/octet-stream              msi msp msm;
+
+    audio/midi                            mid midi kar;
+    audio/mpeg                            mp3;
+    audio/x-realaudio                     ra;
+
+    video/3gpp                            3gpp 3gp;
+    video/mpeg                            mpeg mpg;
+    video/quicktime                       mov;
+    video/x-flv                           flv;
+    video/x-mng                           mng;
+    video/x-ms-asf                        asx asf;
+    video/x-ms-wmv                        wmv;
+    video/x-msvideo                       avi;
+}

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_nginx_rack/templates/config/nginx.conf.erb

@@ -0,0 +1,150 @@
+#deployment webapp nginx.conf
+<% require 'ostruct' %>
+<% properties.webapp ||= OpenStruct.new %>
+<% properties.webapp.nginx ||= OpenStruct.new %>
+<% package_name = properties.webapp.package || 'rackapp' %>
+<% webapp_port_start = 5000 %>
+
+user vcap vcap;
+worker_processes <%= properties.webapp.nginx.worker_processes || 1 %>;
+pid /var/vcap/sys/run/webapp/nginx.pid;
+
+events {
+  worker_connections  8192;
+  use epoll;
+}
+
+
+http {
+  include       mime.types;
+  default_type  text/html;
+  server_tokens off;
+
+  log_format main  '$host - [$time_local] '
+                   '"$request" $status $bytes_sent '
+                   '"$http_referer" "$http_user_agent" '
+                   '$remote_addr response_time:$upstream_response_time';
+
+  sendfile             on;  #enable use of sendfile()
+  tcp_nopush           on;
+  tcp_nodelay          on;  #disable nagel's algorithm
+
+  server_names_hash_bucket_size  128;
+  if_modified_since before;
+
+  gzip              on;
+  gzip_http_version 1.0;
+  gzip_comp_level   2;
+  gzip_proxied      any;
+  gzip_buffers      16 8k;
+  gzip_types        application/json text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+  upstream <%= package_name %>_upstream {
+    server 127.0.0.1:<%= webapp_port_start %>;
+  }
+
+  server {
+    #
+    # The default port to listen on.
+    #
+    # Note that this may be also specified as IP:Port combinations.
+    #
+    # By default Nginx will listen for traffic on the specified port for all IP
+    # addresses assigned to the host.
+    #
+    listen       <%= properties.webapp.nginx.port || 80 %>;
+
+    #
+    # Server Names
+    #
+    # This is the list of IP's and/or domain names with any wildcards or
+    # subdomains. The special value '_' is used to match all domains and IPs and
+    # generally denotes the default server for the entire Nginx process.
+    #
+    # Note that if you specify any domains with wild cards, the very first domain
+    # listed must be without a wild card. For example:
+    #
+    # server_name www.mydomain.org *.mydomain.org;
+    #
+    # It is good practice to include wildcard subdomains so that Nginx will not
+    # require a restart when new services are added for an application.
+    #
+    server_name  <%= properties.webapp.nginx.server_name || '_' %>;
+    server_name_in_redirect off;
+
+    #
+    # This assigns the maximum accepted body size of client request. It is placed
+    # here as a precaution to limit against certain kinds of attacks. Be aware that
+    # it is here in case the application accepts large file uploads. In such a case
+    # this value will need to be adjusted.
+    #
+    client_max_body_size 256M;
+
+    #
+    # The application root which is inherited for all location blocks below.
+    #
+    root /var/vcap/packages/<%= package_name %>/public;
+
+    #
+    # Logging
+    #
+    # Log files are stored within the /var/vcap/sys/log/webapp/ directory.
+    #
+    access_log  /var/vcap/sys/log/webapp/<%= package_name %>.access.log  main;
+    error_log   /var/vcap/sys/log/webapp/<%= package_name %>.error.log notice;
+
+    #
+    # Expire header on assets. For more information on the reasoning behind
+    # this please browse http://developer.yahoo.com/performance/rules.html#expires
+    #
+    # set Expire header on assets: see http://developer.yahoo.com/performance/rules.html#expires
+    location ~ ^/(images|assets|javascripts|stylesheets)/ {
+      try_files  $uri $uri/index.html /last_assets/$uri /last_assets/$uri.html @app_<%= package_name %>;
+      expires 10y;
+    }
+
+    # For Puma, simply proxy
+    location @app_<%= package_name %> {
+      include /var/vcap/jobs/webapp/config/nginx_proxy.conf;
+      proxy_pass http://<%= package_name %>_upstream;
+    }
+
+    #
+    # HTTP Error handling.
+    #
+    # 404 - Resource Not found.
+    error_page 404 /404.html;
+    # 50X - Application Error(s)
+    error_page 500 502 504 /500.html;
+    # 503 - System Maintenance
+    error_page 503 @503;
+    recursive_error_pages on;
+    location @503 {
+      error_page 405 = /system/maintenance.html;
+
+      # Serve static assets if found.
+      if (-f $request_filename) {
+        break;
+      }
+
+      rewrite ^(.*)$ /system/maintenance.html break;
+    }
+    error_page 503 /system/maintenance.html;
+
+    #
+    # The default location will process the request as follows:
+    #
+    # 1. If a system maintenance page is in place, serve it.
+    # 2. If the request uri matches a resource on disk, serve it.
+    # 3. If the request uri/index.html matches a cache file on disk, serve it.
+    # 4. If the request uri.html is cached on disk, serve it.
+    # 5. Failing any caching or system maintenance, pass the request to the
+    #    application.
+    #
+    location / {
+      if (-f $document_root/system/maintenance.html) { return 503; }
+      try_files $uri $uri/index.html $uri.html @app_<%= package_name %>;
+    }
+
+  }
+}

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_nginx_rack/templates/config/nginx_proxy.conf

@@ -0,0 +1,11 @@
+index index.html index.htm;
+
+# needed to forward user's IP address to rails
+proxy_set_header  X-Real-IP         $remote_addr;
+proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
+proxy_set_header  Host              $http_host;
+# request for $start_time patch to nginx is https://cloudfoundry.atlassian.net/browse/CF-88
+# proxy_set_header  X-Queue-Start     't=$start_time';
+
+proxy_redirect off;
+proxy_max_temp_file_size 0;

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_nginx_rack/templates/config/redis.yml.erb

@@ -0,0 +1,6 @@
+---
+host: '<%= (properties.redis && properties.redis.host) || 'localhost' %>'
+port: <%= (properties.redis && properties.redis.port) || 6379 %>
+<%= (properties.webapp && properties.webapp.framework_env) || 'production' %>:
+  host: '<%= (properties.redis && properties.redis.host) || 'localhost' %>'
+  port: <%= (properties.redis && properties.redis.port) || 6379 %>

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/monit.tt

@@ -0,0 +1,5 @@
+check process <%= job_name %>
+  with pidfile /var/vcap/sys/run/<%= job_name %>/<%= job_name %>.pid
+  start program "/var/vcap/jobs/<%= job_name %>/bin/monit_debugger <%= job_name %>_ctl '/var/vcap/jobs/<%= job_name %>/bin/<%= job_name %>_ctl start'"
+  stop program "/var/vcap/jobs/<%= job_name %>/bin/monit_debugger <%= job_name %>_ctl '/var/vcap/jobs/<%= job_name %>/bin/<%= job_name %>_ctl stop'"
+  group vcap

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/templates/bin/%job_name%_ctl.tt

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# Setup env vars and folders for the webapp_ctl script
+source /var/vcap/jobs/<%= job_name %>/helpers/ctl_setup.sh '<%= job_name %>'
+
+export PORT=${PORT:-5000}
+export LANG=en_US.UTF-8
+
+case $1 in
+
+  start)
+    pid_guard $PIDFILE $JOB_NAME
+
+    # TODO: Run some process
+    exec TODO
+         >>$LOG_DIR/$JOB_NAME.stdout.log \
+         2>>$LOG_DIR/$JOB_NAME.stderr.log
+
+    # store pid in $PIDFILE
+    echo $! > $PIDFILE
+    ;;
+
+  stop)
+    kill_and_wait $PIDFILE
+
+    ;;
+  *)
+    echo "Usage: <%= job_name %>_ctl {start|stop}"
+
+    ;;
+
+esac
+exit 0

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/templates/bin/monit_debugger

@@ -0,0 +1,13 @@
+#!/bin/sh
+# USAGE monit_debugger <label> command to run
+mkdir -p /var/vcap/sys/log/monit
+{
+  echo "MONIT-DEBUG date"
+  date
+  echo "MONIT-DEBUG env"
+  env
+  echo "MONIT-DEBUG $@"
+  $2 $3 $4 $5 $6 $7
+  R=$?
+  echo "MONIT-DEBUG exit code $R"
+} >/var/vcap/sys/log/monit/monit_debugger.$1.log 2>&1

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/templates/data/properties.sh.erb

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# job template binding variables
+
+# job name & index of this VM within cluster
+# e.g. JOB_NAME=redis, JOB_INDEX=0
+export NAME='<%= name %>'
+export JOB_INDEX=<%= index %>
+# full job name, like redis/0 or webapp/3
+export JOB_FULL="$NAME/$JOB_INDEX"

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/templates/helpers/ctl_setup.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+# Setup env vars and folders for the ctl script
+# This helps keep the ctl script as readable
+# as possible
+
+# Usage options:
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh JOB_NAME OUTPUT_LABEL
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar nginx
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+JOB_NAME=$1
+output_label=${1:-JOB_NAME}
+
+export JOB_DIR=/var/vcap/jobs/$JOB_NAME
+chmod 755 $JOB_DIR # to access file via symlink
+
+# Load some bosh deployment properties into env vars
+# Try to put all ERb into data/properties.sh.erb
+# incl $NAME, $JOB_INDEX, $WEBAPP_DIR
+source $JOB_DIR/data/properties.sh
+
+source $JOB_DIR/helpers/ctl_utils.sh
+redirect_output ${output_label}
+
+export HOME=${HOME:-/home/vcap}
+
+# Add all packages' /bin & /sbin into $PATH
+for package_bin_dir in $(ls -d /var/vcap/packages/*/*bin)
+do
+  export PATH=${package_bin_dir}:$PATH
+done
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-''} # default to empty
+for package_bin_dir in $(ls -d /var/vcap/packages/*/lib)
+do
+  export LD_LIBRARY_PATH=${package_bin_dir}:$LD_LIBRARY_PATH
+done
+
+# Setup log, run and tmp folders
+
+export RUN_DIR=/var/vcap/sys/run/$JOB_NAME
+export LOG_DIR=/var/vcap/sys/log/$JOB_NAME
+export TMP_DIR=/var/vcap/sys/tmp/$JOB_NAME
+export STORE_DIR=/var/vcap/store/$JOB_NAME
+for dir in $RUN_DIR $LOG_DIR $TMP_DIR $STORE_DIR
+do
+  mkdir -p ${dir}
+  chown vcap:vcap ${dir}
+  chmod 775 ${dir}
+done
+export TMPDIR=$TMP_DIR
+
+export C_INCLUDE_PATH=/var/vcap/packages/mysqlclient/include/mysql:/var/vcap/packages/sqlite/include:/var/vcap/packages/libpq/include
+export LIBRARY_PATH=/var/vcap/packages/mysqlclient/lib/mysql:/var/vcap/packages/sqlite/lib:/var/vcap/packages/libpq/lib
+
+# consistent place for vendoring python libraries within package
+if [[ -d ${WEBAPP_DIR:-/xxxx} ]]
+then
+  export PYTHONPATH=$WEBAPP_DIR/vendor/lib/python
+fi
+
+if [[ -d /var/vcap/packages/java7 ]]
+then
+  export JAVA_HOME="/var/vcap/packages/java7"
+fi
+
+# setup CLASSPATH for all jars/ folders within packages
+export CLASSPATH=${CLASSPATH:-''} # default to empty
+for java_jar in $(ls -d /var/vcap/packages/*/*/*.jar)
+do
+  export CLASSPATH=${java_jar}:$CLASSPATH
+done
+
+PIDFILE=$RUN_DIR/$JOB_NAME.pid
+
+echo '$PATH' $PATH

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/templates/helpers/ctl_utils.sh

@@ -0,0 +1,156 @@
+# Helper functions used by ctl scripts
+
+# links a job file (probably a config file) into a package
+# Example usage:
+# link_job_file_to_package config/redis.yml [config/redis.yml]
+# link_job_file_to_package config/wp-config.php wp-config.php
+link_job_file_to_package() {
+  source_job_file=$1
+  target_package_file=${2:-$source_job_file}
+  full_package_file=$WEBAPP_DIR/${target_package_file}
+
+  link_job_file ${source_job_file} ${full_package_file}
+}
+
+# links a job file (probably a config file) somewhere
+# Example usage:
+# link_job_file config/bashrc /home/vcap/.bashrc
+link_job_file() {
+  source_job_file=$1
+  target_file=$2
+  full_job_file=$JOB_DIR/${source_job_file}
+
+  echo link_job_file ${full_job_file} ${target_file}
+  if [[ ! -f ${full_job_file} ]]
+  then
+    echo "file to link ${full_job_file} does not exist"
+  else
+    # Create/recreate the symlink to current job file
+    # If another process is using the file, it won't be
+    # deleted, so don't attempt to create the symlink
+    mkdir -p $(dirname ${target_file})
+    ln -nfs ${full_job_file} ${target_file}
+  fi
+}
+
+# If loaded within monit ctl scripts then pipe output
+# If loaded from 'source ../utils.sh' then normal STDOUT
+redirect_output() {
+  SCRIPT=$1
+  mkdir -p /var/vcap/sys/log/monit
+  exec 1>> /var/vcap/sys/log/monit/$SCRIPT.log
+  exec 2>> /var/vcap/sys/log/monit/$SCRIPT.err.log
+}
+
+pid_guard() {
+  pidfile=$1
+  name=$2
+
+  if [ -f "$pidfile" ]; then
+    pid=$(head -1 "$pidfile")
+
+    if [ -n "$pid" ] && [ -e /proc/$pid ]; then
+      echo "$name is already running, please stop it first"
+      exit 1
+    fi
+
+    echo "Removing stale pidfile..."
+    rm $pidfile
+  fi
+}
+
+wait_pid() {
+  pid=$1
+  try_kill=$2
+  timeout=${3:-0}
+  force=${4:-0}
+  countdown=$(( $timeout * 10 ))
+
+  echo wait_pid $pid $try_kill $timeout $force $countdown
+  if [ -e /proc/$pid ]; then
+    if [ "$try_kill" = "1" ]; then
+      echo "Killing $pidfile: $pid "
+      kill $pid
+    fi
+    while [ -e /proc/$pid ]; do
+      sleep 0.1
+      [ "$countdown" != '0' -a $(( $countdown % 10 )) = '0' ] && echo -n .
+      if [ $timeout -gt 0 ]; then
+        if [ $countdown -eq 0 ]; then
+          if [ "$force" = "1" ]; then
+            echo -ne "\nKill timed out, using kill -9 on $pid... "
+            kill -9 $pid
+            sleep 0.5
+          fi
+          break
+        else
+          countdown=$(( $countdown - 1 ))
+        fi
+      fi
+    done
+    if [ -e /proc/$pid ]; then
+      echo "Timed Out"
+    else
+      echo "Stopped"
+    fi
+  else
+    echo "Process $pid is not running"
+    echo "Attempting to kill pid anyway..."
+    kill $pid
+  fi
+}
+
+wait_pidfile() {
+  pidfile=$1
+  try_kill=$2
+  timeout=${3:-0}
+  force=${4:-0}
+  countdown=$(( $timeout * 10 ))
+
+  if [ -f "$pidfile" ]; then
+    pid=$(head -1 "$pidfile")
+    if [ -z "$pid" ]; then
+      echo "Unable to get pid from $pidfile"
+      exit 1
+    fi
+
+    wait_pid $pid $try_kill $timeout $force
+
+    rm -f $pidfile
+  else
+    echo "Pidfile $pidfile doesn't exist"
+  fi
+}
+
+kill_and_wait() {
+  pidfile=$1
+  # Monit default timeout for start/stop is 30s
+  # Append 'with timeout {n} seconds' to monit start/stop program configs
+  timeout=${2:-25}
+  force=${3:-1}
+  if [[ -f ${pidfile} ]]
+  then
+    wait_pidfile $pidfile 1 $timeout $force
+  else
+    # TODO assume $1 is something to grep from 'ps ax'
+    pid="$(ps auwwx | grep "$1" | awk '{print $2}')"
+    wait_pid $pid 1 $timeout $force
+  fi
+}
+
+check_nfs_mount() {
+  opts=$1
+  exports=$2
+  mount_point=$3
+
+  if grep -qs $mount_point /proc/mounts; then
+    echo "Found NFS mount $mount_point"
+  else
+    echo "Mounting NFS..."
+    mount $opts $exports $mount_point
+    if [ $? != 0 ]; then
+      echo "Cannot mount NFS from $exports to $mount_point, exiting..."
+      exit 1
+    fi
+  fi
+}