bosh-gen 0.19.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 031e74cb3b6fe9d24db20ecb0baa3a0b1edcf91a
-  data.tar.gz: 6f7525aae7f6a47717148cf8b64fdbf42d14b805
+  metadata.gz: dab52b56ac409b6a60afcbbf970bf84be3e33ecc
+  data.tar.gz: cc58ecb46c8a5f4b17d58b8210156a76cf05851b
 SHA512:
-  metadata.gz: dc468c3526c7b47b18be0034562b93701c91713d288c70d29066ba06d16f8f49048835bbd2e946ce879f568c4e27fd0d57a7957d38af94dac1ae4dc9f1b29c8d
-  data.tar.gz: 29260e1fd9349b54fec9022a35a57221edfc6fd14dbd2c7ebc63e9d564bf7d4f5e1ad9b1bd0710b619ba8692edb1eb1fc2444c30ce090308ed1085b3f580ec8e
+  metadata.gz: c4227a283a5b12eebf89ee8179d2c493b7870cfb603058cabd5c970a0aec5ca4952b6fe7c1aa9a39074880e518caad0470d4bbd364eaa5f7d630e04e24da0ca7
+  data.tar.gz: 777560c1e909c7e2e0c0cdfbbd6373929a5ff47987d2b9a0dcca629751f27f17d65b55deb5d75c2d7ac850eaa88c468ad9e1e6d11f5f3705be79e4d9323ab2f3

data/.travis.yml

@@ -1,10 +1,8 @@
 language: ruby
 script: bundle exec rake spec
 rvm:
-  - ruby-1.9.3
-  # - rbx-19mode
-  - ruby-2.0.0
-  - ruby-2.1.0
+  - ruby-2.1.6
+  - ruby-2.2.2
 notifications:
   email:
     recipients:

data/ChangeLog.md

@@ -1,11 +1,35 @@
 Change Log
 ==========
 
+v0.20.0
+-------
+
+New generator:
+
+-	`package --docker-image` - create a package that snapshots and stores a docker image
+
+Improved generators:
+
+-	`new` - spiff templates include modern `templates:` array of `{name: job, release: release}`
+-	`new` - added an Apache `LICENSE.md` file which will be included in final releases
+-	`new --apt` - Vagrant image upgraded to trusty to match trusty stemcells [thx @cyrille-leclerc]
+
+Removed generators:
+
+-	`manifest` in lieu of spiff templates and `./templates/make_manifest` helper
+-	`micro` generator as this concept is no longer needed with collocated job templates
+
+Workarounds:
+
+-	`fog` - I've had issues with `fog-aws 0.1.2` so restricting to 0.1.1 for now.
+
 v0.19.0
+-------
 
 -	`cli-plugin` - generates a BOSH CLI plugin stub (Ruby code)
 
 v0.18.0
+-------
 
 -	`errand` - generates an errand-style job template
 -	`copy_property` helper now in bosh-templates gem

data/bosh-gen.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "cyoi", "~> 0.10"
   gem.add_dependency "fog", "~> 1.11"
+  gem.add_dependency "fog-aws", "0.1.1" # 0.1.2 causes issues
   gem.add_dependency "readwritesettings", "~> 3.0"
   gem.add_dependency "activesupport", ">= 4.0", "< 5.0"
 

data/lib/bosh/gen/cli.rb

@@ -21,6 +21,8 @@ module Bosh
       desc "package NAME", "Create a new package"
       method_option :apt, :type => :boolean,
         :desc => "Create package using debian/ubuntu apt .debs"
+      method_option :docker_image, :type => :string,
+        :desc => "Create package from remote docker image"
       method_option :dependencies, :aliases => ['-d'], :type => :array,
         :desc => "List of package dependencies"
       method_option :files,        :aliases => ['-f'], :type => :array,
@@ -32,6 +34,10 @@ module Bosh
         if options[:apt]
           require 'bosh/gen/generators/package_apt_generator'
           Bosh::Gen::Generators::PackageAptGenerator.start([name, dependencies])
+        elsif options[:docker_image]
+          docker_image = options[:docker_image]
+          require 'bosh/gen/generators/package_docker_image_generator'
+          Bosh::Gen::Generators::PackageDockerImageGenerator.start([name, docker_image])
         else
           files        = options[:files] || []
           sources      = options[:src] || []
@@ -81,15 +87,6 @@ module Bosh
         Bosh::Gen::Generators::ErrandGenerator.start([name, dependencies])
       end
 
-      desc "micro [JOB]", "Create a micro job - a collection of all jobs and packages"
-      method_option :jobs, :aliases => ['-j'], :type => :array,
-        :desc => "Ordered list of jobs to include"
-      def micro(job_name = "micro")
-        specific_jobs = options[:jobs] || []
-        require 'bosh/gen/generators/micro_job_generator'
-        Bosh::Gen::Generators::MicroJobGenerator.start([job_name, specific_jobs])
-      end
-
       desc "template JOB FILE_PATH",
         "Add a Job template (example FILE_PATH: config/httpd.conf)"
       def template(job_name, file_path)
@@ -115,26 +112,6 @@ module Bosh
         Bosh::Gen::Generators::ExtractPackageGenerator.start([source_package_path])
       end
 
-      desc "manifest NAME PATH",
-        "Creates a deployment manifest based on the release located at PATH"
-      method_option :force, :type => :boolean,
-        :desc => "Force override existing target manifest file"
-      method_option :addresses, :aliases => ['-a'], :type => :array,
-        :desc => "List of IP addresses available for jobs"
-      method_option :disk, :aliases => ['-d'], :type => :string,
-        :desc => "Attach persistent disks to VMs of specific size, e.g. 8196"
-      method_option :jobs, :type => :array,
-        :desc => "Specific jobs to include in manifest [default: all]"
-      def manifest(name, release_path)
-        release_path = File.expand_path(release_path)
-        ip_addresses = options["addresses"] || []
-        job_names = options["jobs"] || []
-        flags = { :force => options["force"] || false, :disk => options[:disk] }
-        require 'bosh/gen/generators/deployment_manifest_generator'
-        Bosh::Gen::Generators::DeploymentManifestGenerator.start(
-          [name, release_path, ip_addresses, job_names, flags])
-      end
-
       desc "cli-plugin NAME", "Add a BOSH CLI plugin into this project"
       def cli_plugin(plugin_name)
         require 'bosh/gen/generators/bosh_cli_plugin_generator'

data/lib/bosh/gen/generators/new_release_generator.rb

@@ -33,6 +33,10 @@ module Bosh::Gen
         template "README.md.tt", "README.md"
       end
 
+      def license
+        template "LICENSE.md.tt", "LICENSE.md"
+      end
+
       def rakefile
         copy_file "Rakefile"
       end
@@ -256,6 +260,13 @@ module Bosh::Gen
         @s3_credentials[key] || default
       end
 
+      def year
+        Date.today.year
+      end
+
+      def author
+        `git config --get user.name`.strip
+      end
     end
   end
 end

data/lib/bosh/gen/generators/new_release_generator/templates/LICENSE.md.tt

@@ -0,0 +1,76 @@
+Apache License Version 2.0, January 2004 http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1.	Definitions.
+
+	"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
+	"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+
+	"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+	"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+	"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+	"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+	"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+
+	"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+	"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+	"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+
+2.	Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+
+3.	Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+
+4.	Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+
+	(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
+
+	(b) You must cause any modified files to carry prominent notices stating that You changed the files; and
+
+	(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+
+	(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+
+	You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+
+5.	Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+
+6.	Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+
+7.	Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+
+8.	Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+
+9.	Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+```
+  To apply the Apache License to your work, attach the following
+  boilerplate notice, with the fields enclosed by brackets "{}"
+  replaced with your own identifying information. (Don't include
+  the brackets!)  The text should be enclosed in the appropriate
+  comment syntax for the file format. We also recommend that a
+  file or class name and description of purpose be included on the
+  same "printed page" as the copyright notice for easier
+  identification within third-party archives.
+```
+
+Copyright <%= year %> <%= author %>
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+```
+   http://www.apache.org/licenses/LICENSE-2.0
+```
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

data/lib/bosh/gen/generators/new_release_generator/templates/templates/jobs.yml.tt

@@ -10,14 +10,18 @@ update:
 
 jobs:
   - name: <%= project_name_underscored %>_leader_z1
-    template: <%= project_name_hyphenated %>
+    templates:
+    - name: <%= project_name_hyphenated %>
+      release: <%= project_name_hyphenated %>
     instances: 0
     resource_pool: small_z1
     networks: (( merge ))
     persistent_disk: 0
     properties: {}
   - name: <%= project_name_underscored %>_z1
-    template: <%= project_name_hyphenated %>
+    templates:
+    - name: <%= project_name_hyphenated %>
+      release: <%= project_name_hyphenated %>
     instances: 0
     resource_pool: small_z1
     networks: (( merge ))

data/lib/bosh/gen/generators/package_apt_generator/templates/Vagrantfile

@@ -3,8 +3,7 @@
 Vagrant.configure('2') do |config|
 
   config.vm.hostname='boshrelease-builder'
-  config.vm.box = "lucid64"
-  config.vm.box_url = "http://files.vagrantup.com/lucid64.box"
+  config.vm.box = "ubuntu/trusty64"
 
   # Need NFS enabled, and hence a private network for virtualbox
   # as discussed in this project's patch

data/lib/bosh/gen/generators/package_docker_image_generator.rb

@@ -0,0 +1,102 @@
+require 'thor/group'
+
+# for the #sh helper
+require 'rake'
+require 'rake/file_utils'
+
+module Bosh::Gen
+  module Generators
+    class PackageDockerImageGenerator < Thor::Group
+      include Thor::Actions
+
+      class RakeHelper
+        extend FileUtils
+      end
+
+      argument :name
+      argument :docker_image
+
+      def self.source_root
+        File.join(File.dirname(__FILE__), "package_docker_image_generator", "templates")
+      end
+
+      def check_root_is_release
+        unless File.exist?("jobs") && File.exist?("packages")
+          raise Thor::Error.new("run inside a BOSH release project")
+        end
+      end
+
+      def check_name
+        raise Thor::Error.new("'#{name}' is not a valid BOSH id") unless name.bosh_valid_id?
+      end
+
+      def docker_save
+        FileUtils.mkdir_p("blobs/docker-images")
+        RakeHelper.sh "docker pull #{docker_image}"
+        RakeHelper.sh "docker save #{docker_image} > blobs/docker-images/#{image_filename}"
+      end
+
+      def jobs
+        directory "jobs"
+      end
+
+      def packages
+        directory "packages"
+      end
+
+      def readme
+        say "Next steps:", :green
+        say <<-README.gsub(/^        /, '')
+          1. To use this BOSH release, first upload it and the docker release to your BOSH:
+            bosh upload release https://bosh.io/releases/cloudfoundry-community/consul-docker
+            bosh upload release https://bosh.io/d/github.com/cf-platform-eng/docker-boshrelease
+
+          2. To use the docker image, your deployment job needs to start with the following:
+
+            jobs:
+            - name: some_job
+            templates:
+              # run docker daemon
+              - {name: docker, release: docker}
+              # warm docker image cache from bosh package
+              - {name: #{job_name}, release: #{project_name_hyphenated}}
+
+          3. To simply run a single container, try the 'containers' job from 'docker' release
+
+            https://github.com/cloudfoundry-community/consul-docker-boshrelease/blob/master/templates/jobs.yml#L18-L40
+
+
+        README
+      end
+
+      private
+      def root_path
+        File.expand_path(".")
+      end
+
+      def project_name
+        @project_name ||= File.basename(root_path).gsub(/-(?:boshrelease|release)$/, '')
+      end
+
+      def project_name_hyphenated
+        project_name.gsub(/[^A-Za-z0-9]+/, '-')
+      end
+
+      def package_name
+        name
+      end
+
+      def image_name
+        docker_image.gsub(/\W+/, '_')
+      end
+
+      def image_filename
+        "#{image_name}.tgz"
+      end
+
+      def job_name
+        "#{image_name}_image"
+      end
+    end
+  end
+end

data/lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/monit.tt

@@ -0,0 +1,5 @@
+check process <%= job_name %>
+  with pidfile /var/vcap/sys/run/<%= job_name %>/<%= job_name %>.pid
+  start program "/var/vcap/jobs/<%= job_name %>/bin/monit_debugger install_ctl '/var/vcap/jobs/<%= job_name %>/bin/install_ctl start'"
+  stop program "/var/vcap/jobs/<%= job_name %>/bin/monit_debugger install_ctl '/var/vcap/jobs/<%= job_name %>/bin/install_ctl stop'"
+  group vcap

data/lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/spec.tt

@@ -0,0 +1,9 @@
+---
+name: <%= job_name %>
+packages: [<%= package_name %>]
+templates:
+  bin/monit_debugger: bin/monit_debugger
+  bin/install_ctl: bin/install_ctl
+  helpers/ctl_setup.sh: helpers/ctl_setup.sh
+  helpers/ctl_utils.sh: helpers/ctl_utils.sh
+properties: {}

data/lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/templates/bin/install_ctl.tt

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# Setup env vars and folders for the webapp_ctl script
+source /var/vcap/jobs/<%= job_name %>/helpers/ctl_setup.sh '<%= job_name %>'
+
+export PORT=${PORT:-5000}
+export LANG=en_US.UTF-8
+
+case $1 in
+
+  start)
+    pid_guard $PIDFILE $JOB_NAME
+
+    # store pid in $PIDFILE
+    echo $$ > $PIDFILE
+
+    time docker \
+      --host unix:///var/vcap/sys/run/docker/docker.sock \
+      load -i /var/vcap/packages/<%= package_name %>/<%= image_filename %>
+        >>$LOG_DIR/$JOB_NAME.stdout.log \
+        2>>$LOG_DIR/$JOB_NAME.stderr.log
+
+    # do nothing forever
+    exec tail -f /dev/null
+    ;;
+
+  stop)
+    kill_and_wait $PIDFILE
+
+    ;;
+  *)
+    echo "Usage: install_ctl {start|stop}"
+
+    ;;
+
+esac
+exit 0

data/lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/templates/bin/monit_debugger

@@ -0,0 +1,13 @@
+#!/bin/sh
+# USAGE monit_debugger <label> command to run
+mkdir -p /var/vcap/sys/log/monit
+{
+  echo "MONIT-DEBUG date"
+  date
+  echo "MONIT-DEBUG env"
+  env
+  echo "MONIT-DEBUG $@"
+  $2 $3 $4 $5 $6 $7
+  R=$?
+  echo "MONIT-DEBUG exit code $R"
+} >/var/vcap/sys/log/monit/monit_debugger.$1.log 2>&1

data/lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/templates/helpers/ctl_setup.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Setup env vars and folders for the ctl script
+# This helps keep the ctl script as readable
+# as possible
+
+# Usage options:
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh JOB_NAME OUTPUT_LABEL
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar nginx
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+JOB_NAME=$1
+output_label=${1:-JOB_NAME}
+
+export JOB_DIR=/var/vcap/jobs/$JOB_NAME
+chmod 755 $JOB_DIR # to access file via symlink
+
+source $JOB_DIR/helpers/ctl_utils.sh
+redirect_output ${output_label}
+
+export HOME=${HOME:-/home/vcap}
+
+# Add all packages' /bin & /sbin into $PATH
+for package_bin_dir in $(ls -d /var/vcap/packages/*/*bin)
+do
+  export PATH=${package_bin_dir}:$PATH
+done
+
+# Setup log, run and tmp folders
+
+export RUN_DIR=/var/vcap/sys/run/$JOB_NAME
+export LOG_DIR=/var/vcap/sys/log/$JOB_NAME
+export TMP_DIR=/var/vcap/sys/tmp/$JOB_NAME
+export STORE_DIR=/var/vcap/store/$JOB_NAME
+for dir in $RUN_DIR $LOG_DIR $TMP_DIR $STORE_DIR
+do
+  mkdir -p ${dir}
+  chown vcap:vcap ${dir}
+  chmod 775 ${dir}
+done
+export TMPDIR=$TMP_DIR
+
+PIDFILE=$RUN_DIR/$JOB_NAME.pid

data/lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/templates/helpers/ctl_utils.sh

@@ -0,0 +1,156 @@
+# Helper functions used by ctl scripts
+
+# links a job file (probably a config file) into a package
+# Example usage:
+# link_job_file_to_package config/redis.yml [config/redis.yml]
+# link_job_file_to_package config/wp-config.php wp-config.php
+link_job_file_to_package() {
+  source_job_file=$1
+  target_package_file=${2:-$source_job_file}
+  full_package_file=$WEBAPP_DIR/${target_package_file}
+
+  link_job_file ${source_job_file} ${full_package_file}
+}
+
+# links a job file (probably a config file) somewhere
+# Example usage:
+# link_job_file config/bashrc /home/vcap/.bashrc
+link_job_file() {
+  source_job_file=$1
+  target_file=$2
+  full_job_file=$JOB_DIR/${source_job_file}
+
+  echo link_job_file ${full_job_file} ${target_file}
+  if [[ ! -f ${full_job_file} ]]
+  then
+    echo "file to link ${full_job_file} does not exist"
+  else
+    # Create/recreate the symlink to current job file
+    # If another process is using the file, it won't be
+    # deleted, so don't attempt to create the symlink
+    mkdir -p $(dirname ${target_file})
+    ln -nfs ${full_job_file} ${target_file}
+  fi
+}
+
+# If loaded within monit ctl scripts then pipe output
+# If loaded from 'source ../utils.sh' then normal STDOUT
+redirect_output() {
+  SCRIPT=$1
+  mkdir -p /var/vcap/sys/log/monit
+  exec 1>> /var/vcap/sys/log/monit/$SCRIPT.log
+  exec 2>> /var/vcap/sys/log/monit/$SCRIPT.err.log
+}
+
+pid_guard() {
+  pidfile=$1
+  name=$2
+
+  if [ -f "$pidfile" ]; then
+    pid=$(head -1 "$pidfile")
+
+    if [ -n "$pid" ] && [ -e /proc/$pid ]; then
+      echo "$name is already running, please stop it first"
+      exit 1
+    fi
+
+    echo "Removing stale pidfile..."
+    rm $pidfile
+  fi
+}
+
+wait_pid() {
+  pid=$1
+  try_kill=$2
+  timeout=${3:-0}
+  force=${4:-0}
+  countdown=$(( $timeout * 10 ))
+
+  echo wait_pid $pid $try_kill $timeout $force $countdown
+  if [ -e /proc/$pid ]; then
+    if [ "$try_kill" = "1" ]; then
+      echo "Killing $pidfile: $pid "
+      kill $pid
+    fi
+    while [ -e /proc/$pid ]; do
+      sleep 0.1
+      [ "$countdown" != '0' -a $(( $countdown % 10 )) = '0' ] && echo -n .
+      if [ $timeout -gt 0 ]; then
+        if [ $countdown -eq 0 ]; then
+          if [ "$force" = "1" ]; then
+            echo -ne "\nKill timed out, using kill -9 on $pid... "
+            kill -9 $pid
+            sleep 0.5
+          fi
+          break
+        else
+          countdown=$(( $countdown - 1 ))
+        fi
+      fi
+    done
+    if [ -e /proc/$pid ]; then
+      echo "Timed Out"
+    else
+      echo "Stopped"
+    fi
+  else
+    echo "Process $pid is not running"
+    echo "Attempting to kill pid anyway..."
+    kill $pid
+  fi
+}
+
+wait_pidfile() {
+  pidfile=$1
+  try_kill=$2
+  timeout=${3:-0}
+  force=${4:-0}
+  countdown=$(( $timeout * 10 ))
+
+  if [ -f "$pidfile" ]; then
+    pid=$(head -1 "$pidfile")
+    if [ -z "$pid" ]; then
+      echo "Unable to get pid from $pidfile"
+      exit 1
+    fi
+
+    wait_pid $pid $try_kill $timeout $force
+
+    rm -f $pidfile
+  else
+    echo "Pidfile $pidfile doesn't exist"
+  fi
+}
+
+kill_and_wait() {
+  pidfile=$1
+  # Monit default timeout for start/stop is 30s
+  # Append 'with timeout {n} seconds' to monit start/stop program configs
+  timeout=${2:-25}
+  force=${3:-1}
+  if [[ -f ${pidfile} ]]
+  then
+    wait_pidfile $pidfile 1 $timeout $force
+  else
+    # TODO assume $1 is something to grep from 'ps ax'
+    pid="$(ps auwwx | grep "$1" | awk '{print $2}')"
+    wait_pid $pid 1 $timeout $force
+  fi
+}
+
+check_nfs_mount() {
+  opts=$1
+  exports=$2
+  mount_point=$3
+
+  if grep -qs $mount_point /proc/mounts; then
+    echo "Found NFS mount $mount_point"
+  else
+    echo "Mounting NFS..."
+    mount $opts $exports $mount_point
+    if [ $? != 0 ]; then
+      echo "Cannot mount NFS from $exports to $mount_point, exiting..."
+      exit 1
+    fi
+  fi
+}

data/lib/bosh/gen/generators/package_docker_image_generator/templates/packages/%package_name%/packaging

@@ -0,0 +1,4 @@
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+cp -a docker-images/* $BOSH_INSTALL_TARGET

data/lib/bosh/gen/generators/package_docker_image_generator/templates/packages/%package_name%/spec.tt

@@ -0,0 +1,4 @@
+---
+name: <%= package_name %>
+dependencies: []
+files: [docker-images/<%= image_filename %>]

data/lib/bosh/gen/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Gen
-    VERSION = "0.19.0"
+    VERSION = "0.20.0"
   end
 end

data/spec/fixtures/releases/s3test-boshrelease/templates/jobs.yml

@@ -10,12 +10,16 @@ update:
 
 jobs:
   - name: s3test_leader_z1
-    template: s3test
+    templates:
+    - name: s3test
+      release: s3test
     instances: 0
     resource_pool: small_z1
     networks: (( merge ))
   - name: s3test_z1
-    template: s3test
+    templates:
+    - name: s3test
+      release: s3test
     instances: 0
     resource_pool: small_z1
     networks: (( merge ))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh-gen
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.0
 platform: ruby
 authors:
 - Dr Nic Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-07 00:00:00.000000000 Z
+date: 2015-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: fog-aws
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: readwritesettings
   requirement: !ruby/object:Gem::Requirement
@@ -180,7 +194,6 @@ files:
 - lib/bosh/gen/cli.rb
 - lib/bosh/gen/generators/bosh_cli_plugin_generator.rb
 - lib/bosh/gen/generators/bosh_cli_plugin_generator/templates/lib/bosh/cli/commands/%underscore_plugin_name%.rb.tt
-- lib/bosh/gen/generators/deployment_manifest_generator.rb
 - lib/bosh/gen/generators/errand_generator.rb
 - lib/bosh/gen/generators/errand_generator/templates/.gitkeep
 - lib/bosh/gen/generators/errand_generator/templates/jobs/%job_name%/monit.tt
@@ -216,9 +229,8 @@ files:
 - lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/templates/helpers/ctl_setup.sh
 - lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/templates/helpers/ctl_utils.sh
 - lib/bosh/gen/generators/job_template_generator.rb
-- lib/bosh/gen/generators/micro_job_generator.rb
-- lib/bosh/gen/generators/micro_job_generator/templates/jobs/micro/prepare.tt
 - lib/bosh/gen/generators/new_release_generator.rb
+- lib/bosh/gen/generators/new_release_generator/templates/LICENSE.md.tt
 - lib/bosh/gen/generators/new_release_generator/templates/README.md.tt
 - lib/bosh/gen/generators/new_release_generator/templates/Rakefile
 - lib/bosh/gen/generators/new_release_generator/templates/blobs/.gitkeep
@@ -246,6 +258,15 @@ files:
 - lib/bosh/gen/generators/package_apt_generator/templates/src/apt/%name%/aptfile.tt
 - lib/bosh/gen/generators/package_apt_generator/templates/src/apt/%name%/profile.sh.tt
 - lib/bosh/gen/generators/package_apt_generator/templates/src/apt/fetch_debs.sh
+- lib/bosh/gen/generators/package_docker_image_generator.rb
+- lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/monit.tt
+- lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/spec.tt
+- lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/templates/bin/install_ctl.tt
+- lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/templates/bin/monit_debugger
+- lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/templates/helpers/ctl_setup.sh
+- lib/bosh/gen/generators/package_docker_image_generator/templates/jobs/%job_name%/templates/helpers/ctl_utils.sh
+- lib/bosh/gen/generators/package_docker_image_generator/templates/packages/%package_name%/packaging
+- lib/bosh/gen/generators/package_docker_image_generator/templates/packages/%package_name%/spec.tt
 - lib/bosh/gen/generators/package_generator.rb
 - lib/bosh/gen/generators/package_generator/templates/.gitkeep
 - lib/bosh/gen/generators/package_source_generator.rb
@@ -323,7 +344,6 @@ files:
 - spec/fixtures/releases/some_dev_releases/dev_releases/myrelease-1.yml
 - spec/fixtures/releases/some_dev_releases/dev_releases/myrelease-10.yml
 - spec/fixtures/releases/some_dev_releases/dev_releases/myrelease-2.yml
-- spec/generators/deployment_manifest_generator_spec.rb
 - spec/generators/job_generator_spec.rb
 - spec/generators/new_release_generator_spec.rb
 - spec/models/bosh_config_spec.rb
@@ -422,7 +442,6 @@ test_files:
 - spec/fixtures/releases/some_dev_releases/dev_releases/myrelease-1.yml
 - spec/fixtures/releases/some_dev_releases/dev_releases/myrelease-10.yml
 - spec/fixtures/releases/some_dev_releases/dev_releases/myrelease-2.yml
-- spec/generators/deployment_manifest_generator_spec.rb
 - spec/generators/job_generator_spec.rb
 - spec/generators/new_release_generator_spec.rb
 - spec/models/bosh_config_spec.rb

data/lib/bosh/gen/generators/deployment_manifest_generator.rb

@@ -1,109 +0,0 @@
-require 'yaml'
-require 'thor/group'
-require 'bosh/template/property_helper'
-
-module Bosh::Gen
-  module Generators
-    class DeploymentManifestGenerator < Thor::Group
-      include Thor::Actions
-      include Bosh::Template::PropertyHelper
-
-      argument :name
-      argument :release_path
-      argument :ip_addresses, :type => :array
-      argument :requested_jobs, :type => :array
-      argument :flags, :type => :hash
-
-      def check_release_path_is_release
-        unless File.exist?(release_path)
-          raise Thor::Error.new("target path '#{release_path}' doesn't exist")
-        end
-        FileUtils.chdir(release_path) do
-          unless File.exist?("jobs") && File.exist?("packages")
-            raise Thor::Error.new("target path '#{release_path}' is not a BOSH release project")
-          end
-        end
-      end
-
-      def check_valid_requested_jobs
-        if jobs.empty?
-          raise Thor::Error.new("No valid jobs specified from #{detect_jobs.inspect}")
-        end
-      end
-
-      # Create a deployment manifest (initially for AWS only)
-      def create_deployment_manifest
-        cloud_properties = {
-          "instance_type" => "m1.small",
-        }
-        cloud_properties["persistent_disk"] = flags[:disk] if flags[:disk]
-        cloud_properties["static_ips"] = ip_addresses
-        director_uuid = Bosh::Gen::Models::BoshConfig.new.target_uuid
-        manifest = Bosh::Gen::Models::DeploymentManifest.new(
-          name, director_uuid,
-          release_properties, cloud_properties, default_properties)
-        manifest.jobs = job_manifests
-        create_file manifest_file_name, manifest.to_yaml, :force => flags[:force]
-      end
-
-      def setup_bosh_deployment_target
-        run "bosh deployment #{manifest_file_name}"
-      end
-
-      private
-      def release_detector
-        @release_detector ||= Bosh::Gen::Models::ReleaseDetection.new(release_path)
-      end
-
-      # Whether +name+ contains .yml suffix or nor, returns a .yml filename for manifest to be generated
-      def manifest_file_name
-        basename = "#{name}.yml"
-      end
-
-      def job_manifests
-        jobs.map do |job_name|
-          {
-            "name" => job_name
-          }
-        end
-      end
-
-      # Return list of job names
-      def detect_jobs
-        release_detector.latest_dev_release_job_names
-      end
-
-      def jobs
-        job_names = if !requested_jobs || requested_jobs.size == 0
-          detect_jobs
-        else
-          requested_jobs & detect_jobs
-        end
-      end
-
-      # The "release" aspect of the manifest, which has two keys: name, version
-      def release_properties
-        release_detector.latest_dev_release_properties
-      end
-
-      # Default properties for manifest, based on each job's spec's properties hash, if present
-      # For example, a job's spec may include something like:
-      #   properties:
-      #     mysql.password:
-      #       default: mypassword
-      #       description: Password for mysql server
-      def default_properties
-        properties = {}
-        jobs.each do |job_name|
-          spec = YAML.load_file(File.join(release_path, "jobs", job_name, "spec"))
-          if spec_properties = spec["properties"]
-            spec_properties.each_pair do |name, definition|
-              copy_property(properties, spec_properties, name, definition["default"])
-            end
-          end
-        end
-        properties
-      end
-    end
-  end
-end

data/lib/bosh/gen/generators/micro_job_generator.rb

@@ -1,56 +0,0 @@
-require 'yaml'
-require 'thor/group'
-
-module Bosh::Gen
-  module Generators
-    class MicroJobGenerator < Thor::Group
-      include Thor::Actions
-
-      argument :job_name
-      argument :specific_jobs, :type => :array
-
-      def self.source_root
-        File.join(File.dirname(__FILE__), "micro_job_generator", "templates")
-      end
-      
-      def check_root_is_release
-        unless File.exist?("jobs") && File.exist?("packages")
-          raise Thor::Error.new("run inside a BOSH release project")
-        end
-      end
-      
-      def create_job
-        directory "jobs/micro", "jobs/#{job_name}"
-        chmod "jobs/#{job_name}/prepare", 0755
-      end
-      
-      def prepare_spec_defaults_all_jobs
-        jobs = if specific_jobs.size > 0
-          specific_jobs
-        else
-          Dir[File.expand_path("jobs/*")].map {|job| File.basename(job) } - [job_name]
-        end
-        spec = { "jobs" => jobs }
-        create_file "jobs/#{job_name}/prepare_spec", YAML.dump(spec)
-      end
-      
-      def gitignore
-        append_file ".gitignore", <<-IGNORE.gsub(/^\s{8}/, '')
-        jobs/#{job_name}/monit
-        jobs/#{job_name}/spec
-        jobs/#{job_name}/templates/
-        IGNORE
-      end
-      
-      def readme
-        say ""
-        say "Edit "; say "jobs/#{job_name}/prepare_spec ", :yellow
-          say "with ordered list of jobs to include"
-        say "in micro job. The order of jobs implicitly specifies the order in"
-        say "which they are started."
-        say ""
-        say ""
-      end
-    end
-  end
-end

data/lib/bosh/gen/generators/micro_job_generator/templates/jobs/micro/prepare.tt

@@ -1,130 +0,0 @@
-#!/usr/bin/env ruby
-#
-require 'fileutils'
-require 'yaml'
-require 'erb'
-
-TEMPLATES = "templates"
-OVERRIDE = "override"
-
-MICRO_JOB_NAME="<%= job_name %>"
-
-def job_dir(job)
-  File.expand_path("../../#{job}", __FILE__)
-end
-
-PREPARE_SPEC = YAML.load_file(File.expand_path("../prepare_spec", __FILE__))
-
-# Order of jobs, implicitly specifies the order in which they are started
-jobs = PREPARE_SPEC["jobs"]
-
-spec = {
-  "name" => MICRO_JOB_NAME,
-  "templates" => {
-    "post_install" => "bin/post_install"
-  },
-  "packages" => []
-}
-
-monit = []
-
-# make sure we start with an empty directory, as it will stop if
-# there are any conflicts when populating the template dir
-FileUtils.rm_rf(TEMPLATES)
-FileUtils.mkdir(TEMPLATES)
-
-jobs.each do |job|
-  # concatenate monit file
-  override = File.join(OVERRIDE, "monit", job)
-  if File.exist?(override)
-    monit_file = override
-  else
-    monit_file = "#{job_dir(job)}/monit"
-  end
-  File.open(monit_file) do |file|
-    monit += file.readlines
-    monit << "\n"
-  end
-
-  # copy templates from:
-  # * templates/<template_name>
-  # * templates/<template_dir>/<template_name>
-  Dir.glob("#{job_dir(job)}/templates/**/*").each do |file|
-    next if File.directory?(file)
-    
-    template_name = File.basename(file)
-    template_dir = File.basename(File.dirname(file))
-    # check if template file is in a nested dir
-    # if 'templates' then its not
-    if template_dir == "templates"
-      override = File.join(OVERRIDE, job, template_name)
-      dest = File.join(TEMPLATES, job)
-    else
-      override = File.join(OVERRIDE, job, template_dir, template_name)
-      dest = File.join(TEMPLATES, job, template_dir)
-    end
-    FileUtils.mkdir_p(dest) unless File.directory?(dest)
-  
-    if File.exist?(File.join(dest, template_name))
-      puts "File conflict for job #{job}: #{template_name}"
-      exit 1
-    elsif File.exist?(override)
-      FileUtils.cp(override, dest)
-    else
-      FileUtils.cp(file, dest)
-    end
-  end
-
-  # build spec
-  s = YAML.load_file("#{job_dir(job)}/spec")
-  spec["packages"] += s["packages"]
-  s["templates"].each do |src, dst|
-    spec["templates"]["#{job}/#{src}"] = "#{job}/#{dst}"
-  end
-end
-
-# write out the concatenated monit file
-File.open("monit", "w") do |file|
-  file.puts monit.join
-end
-
-
-# write out spec file
-spec["packages"] = spec["packages"].uniq
-File.open("spec", "w") do |file|
-  file.puts spec.to_yaml
-end
-
-
-
-# generate post_install file with all jobs
-# it creates links from
-# jobs/micro/<job>/bin/foo to jobs/<job>/bin/foo
-# jobs/micro/<job>/config/bar to jobs/<job>/config/bar
-post_install = <<POST_INSTALL
-#!/bin/sh
-
-for DIR in /var/vcap/jobs /var/vcap/data/jobs; do
-  cd $DIR
-  for JOB in <%%= jobs.join(" ") %>; do
-    mkdir -p ${JOB}/bin ${JOB}/config ${JOB}/data ${JOB}/helpers
-    for FILE in <%%= MICRO_JOB_NAME %>/$JOB/bin/*; do
-      ln -nsf /var/vcap/jobs/$FILE $JOB/bin
-    done
-    for FILE in <%%= MICRO_JOB_NAME %>/$JOB/config/*; do
-      ln -nsf /var/vcap/jobs/$FILE $JOB/config
-    done
-    for FILE in <%%= MICRO_JOB_NAME %>/$JOB/data/*; do
-      ln -nsf /var/vcap/jobs/$FILE $JOB/data
-    done
-    for FILE in <%%= MICRO_JOB_NAME %>/$JOB/helpers/*; do
-      ln -nsf /var/vcap/jobs/$FILE $JOB/helpers
-    done
-  done
-done
-POST_INSTALL
-
-File.open("templates/post_install", "w") do |file|
-  template = ERB.new(post_install)
-  file.puts template.result
-end

data/spec/generators/deployment_manifest_generator_spec.rb

@@ -1,38 +0,0 @@
-require "spec_helper"
-require "bosh/gen/generators/deployment_manifest_generator"
-
-# in a tmp folder:
-# * run generator
-# * deployment manifest created
-
-# generates dpeloyment manifest for a target release
-describe Bosh::Gen::Generators::DeploymentManifestGenerator do
-
-  xit "creates deployment manifest with properties" do
-    setup_project_release("bosh-sample-release")
-    release_folder = File.expand_path("../../fixtures/releases/bosh-sample-release", __FILE__)
-    in_home_folder do
-      generate_manifest("wordpress", release_folder)
-      expect(File.exist?("wordpress.yml")).to eq(true)
-
-      manifest = YAML.load_file("wordpress.yml")
-      properties = manifest["properties"]
-      properties.wont_be_nil "manifest properties must be set"
-      properties["mysql"].wont_be_nil
-      properties["mysql"]["password"].must_equal 'mysqlpassword'
-    end
-  end
-
-  it "checks for numerics in filenames/properties" do
-    release_folder = File.expand_path("../../tmp", __FILE__)
-    setup_project_release("s3test-boshrelease")
-    expect(File.exist?(File.join(@active_project_folder, "jobs/s3test/spec"))).to eq true
-
-    dev = YAML.load_file(File.join(@active_project_folder, "config/dev.yml"))
-    expect(dev).not_to be_nil
-    expect(dev["dev_name"]).to eq "s3test"
-
-    deployment = YAML.load_file(File.join(@active_project_folder, "templates/deployment.yml"))
-    expect(deployment["compilation"]["network"]).to eq("s3test1")
-  end
-end