bosh-gen 0.16.2 → 0.17.0

data/lib/bosh/gen/generators/new_release_generator/templates/jobs/%project_name_hyphenated%/templates/bin/%job_name%_ctl.tt

@@ -14,13 +14,13 @@ case $1 in
   start)
     pid_guard $PIDFILE $JOB_NAME
 
-    # TODO: Run some process
-    exec chpst -u vcap:vcap TODO \
+    # store pid in $PIDFILE
+    echo $$ > $PIDFILE
+
+    exec chpst -u vcap:vcap <%= job_name %> \
          >>$LOG_DIR/$JOB_NAME.stdout.log \
          2>>$LOG_DIR/$JOB_NAME.stderr.log
 
-    # store pid in $PIDFILE
-    echo $! > $PIDFILE
     ;;
 
   stop)
@@ -33,4 +33,4 @@ case $1 in
     ;;
 
 esac
-exit 0
+exit 0

data/lib/bosh/gen/generators/new_release_generator/templates/jobs/%project_name_hyphenated%/templates/data/properties.sh.erb

@@ -8,3 +8,9 @@ export NAME='<%= name %>'
 export JOB_INDEX=<%= index %>
 # full job name, like redis/0 or webapp/3
 export JOB_FULL="$NAME/$JOB_INDEX"
+
+export DEPLOYMENT_NAME=<%= spec.deployment %>
+export DNS_ROOT=<%= spec.dns_domain_name %>
+
+# $BIND_ADDR is the IP of the first network
+export BIND_ADDR=<%= spec.networks.send(spec.networks.methods(false).first).ip %>

data/lib/bosh/gen/generators/new_release_generator/templates/templates/infrastructure-aws-ec2.yml.tt

@@ -2,7 +2,7 @@ meta:
   environment: (( merge ))
   dns_root: (( merge ))
   security_groups: (( merge ))
-  network_name: (( merge ))
+  persistent_disk: 4096
 
   stemcell:
     name: bosh-aws-xen-ubuntu
@@ -12,11 +12,13 @@ jobs:
   - name: <%= project_name_underscored %>_leader_z1
     instances: 1
     networks:
-      - name: (( meta.network_name ))
-  - name: <%= project_name %>_z1
+      - name: <%= project_name_underscored %>1
+    persistent_disk: (( meta.persistent_disk ))
+  - name: <%= project_name_underscored %>_z1
     instances: 2
     networks:
-      - name: (( meta.network_name ))
+      - name: <%= project_name_underscored %>1
+    persistent_disk: (( meta.persistent_disk ))
     properties:
       <%= project_name_underscored %>:
         leader_address: (( "0.<%= project_name_hyphenated %>-leader-z1.<%= project_name_hyphenated %>1." meta.environment "." meta.dns_root ))
@@ -34,7 +36,7 @@ networks:
   - name: floating
     type: vip
     cloud_properties: {}
-  - name: (( meta.network_name ))
+  - name: <%= project_name_underscored %>1
     type: dynamic
     cloud_properties:
       security_groups: (( meta.security_groups ))

data/lib/bosh/gen/generators/new_release_generator/templates/templates/infrastructure-warden.yml.tt

@@ -1,6 +1,5 @@
 meta:
   environment: <%= project_name_hyphenated %>-warden
-  network_name: (( merge ))
 
   stemcell:
     name: bosh-warden-boshlite-ubuntu
@@ -16,12 +15,12 @@ jobs:
   - name: <%= project_name_underscored %>_leader_z1
     instances: 1
     networks:
-      - name: (( meta.network_name ))
+      - name: <%= project_name_underscored %>1
         static_ips: (( static_ips(0) ))
   - name: <%= project_name_underscored %>_z1
     instances: 2
     networks:
-      - name: (( meta.network_name ))
+      - name: <%= project_name_underscored %>1
         static_ips: ~
     properties:
       <%= project_name_underscored %>:
@@ -37,7 +36,7 @@ resource_pools:
       name: random
 
 networks:
-- name: (( meta.network_name ))
+- name: <%= project_name_underscored %>1
   # Assumes up to 5 VMs, including 1 static and 4 dynamic.
   # Plus 5 (double the size) unused IPs, due to BOSH bug/quirk.
   subnets:

data/lib/bosh/gen/generators/new_release_generator/templates/templates/jobs.yml.tt

@@ -14,11 +14,14 @@ jobs:
     instances: 0
     resource_pool: small_z1
     networks: (( merge ))
+    persistent_disk: 0
+    properties: ~
   - name: <%= project_name_underscored %>_z1
     template: <%= project_name_hyphenated %>
     instances: 0
     resource_pool: small_z1
     networks: (( merge ))
+    persistent_disk: 0
     update:
       canaries: 10
     properties:

data/lib/bosh/gen/generators/new_release_generator/templates/templates/make_manifest.tt

@@ -2,8 +2,10 @@
 
 set -e
 
-infrastructure=$1
 template_prefix="<%= project_name %>"
+STEMCELL_OS=${STEMCELL_OS:-ubuntu}
+
+infrastructure=$1
 
 if [ "$infrastructure" != "aws-ec2" ] && \
     [ "$infrastructure" != "warden" ] ; then
@@ -33,15 +35,29 @@ if [[ $infrastructure = "aws-ec2" ]]; then
   fi
 fi
 
+function latest_uploaded_stemcell {
+  echo $(bosh stemcells | grep bosh | grep $STEMCELL_OS | awk -F'|' '{ print $2, $3 }' | sort -nr -k2 | head -n1 | awk '{ print $1 }')
+}
+
+STEMCELL=${STEMCELL:-$(latest_uploaded_stemcell)}
+if [[ "${STEMCELL}X" == "X" ]]; then
+  echo
+  echo "Uploading latest $DIRECTOR_CPI/$STEMCELL_OS stemcell..."
+  STEMCELL_URL=$(bosh public stemcells --full | grep $DIRECTOR_CPI | grep $STEMCELL_OS | sort -nr | head -n1 | awk '{ print $4 }')
+  bosh upload stemcell $STEMCELL_URL
+fi
+STEMCELL=${STEMCELL:-$(latest_uploaded_stemcell)}
+
 templates=$(dirname $0)
 release=$templates/..
 tmpdir=$release/tmp
 
 mkdir -p $tmpdir
 cp $templates/stub.yml $tmpdir/stub-with-uuid.yml
-echo $DIRECTOR_NAME $DIRECTOR_CPI $DIRECTOR_UUID
+echo $DIRECTOR_NAME $DIRECTOR_CPI $DIRECTOR_UUID $STEMCELL
 perl -pi -e "s/PLACEHOLDER-DIRECTOR-UUID/$DIRECTOR_UUID/g" $tmpdir/stub-with-uuid.yml
 perl -pi -e "s/NAME/$NAME/g" $tmpdir/stub-with-uuid.yml
+perl -pi -e "s/STEMCELL/$STEMCELL/g" $tmpdir/stub-with-uuid.yml
 
 spiff merge \
   $templates/deployment.yml \

data/lib/bosh/gen/generators/new_release_generator/templates/templates/stub.yml.tt

@@ -2,9 +2,10 @@
 meta:
   environment: NAME
   dns_root: microbosh
-  network_name: <%= project_name_underscored %>1
   security_groups:
     - <%= project_name_underscored %>
+  stemcell:
+    name: STEMCELL
 
 director_uuid: PLACEHOLDER-DIRECTOR-UUID
 

data/lib/bosh/gen/settings.rb

@@ -0,0 +1,61 @@
+require "readwritesettings"
+require "fileutils"
+
+module Bosh::Gen::Settings
+
+  # The base directory for holding the manifest settings file
+  # and private keys
+  #
+  # Defaults to ~/.bosh_inception; and can be overridden with either:
+  # * $SETTINGS - to a folder (supported method)
+  def settings_dir
+    @settings_dir ||= File.expand_path(ENV["SETTINGS"] || "~/.bosh-gen")
+  end
+
+  def settings_dir=(settings_dir)
+    @settings_dir = File.expand_path(settings_dir)
+    @settings = @settings_path = nil
+    reload_settings!
+  end
+
+  def settings_ssh_dir
+    File.join(settings_dir, "ssh")
+  end
+
+  def settings_path
+    @settings_path ||= File.join(settings_dir, "settings.yml")
+  end
+
+  def settings
+    @settings ||= begin
+      unless File.exists?(settings_path)
+        FileUtils.mkdir_p(settings_dir)
+        File.open(settings_path, "w") { |file| file << "--- {}" }
+      end
+      FileUtils.chmod(0600, settings_path)
+      FileUtils.chmod(0700, settings_ssh_dir) if File.directory?(settings_ssh_dir)
+      ReadWriteSettings.new(settings_path)
+    end
+  end
+
+  # Set a nested setting with "key1.key2.key3" notation
+  def setting(nested_key, value)
+    settings.set(nested_key, value)
+    save_settings!
+  end
+
+  # Saves current nested ReadWriteSettings into pure Hash-based YAML file
+  # Recreates accessors on ReadWriteSettings object (since something has changed)
+  def save_settings!
+    File.open(settings_path, "w") { |f| f << settings.to_nested_hash.to_yaml }
+    settings.create_accessors!
+  end
+
+  def reload_settings!
+    @settings = nil
+    settings
+  end
+
+  def migrate_old_settings
+  end
+end

data/lib/bosh/gen/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Gen
-    VERSION = "0.16.2"
+    VERSION = "0.17.0"
   end
 end

data/spec/fixtures/deployment_manifests/2_jobs_1_ip_8196_disk_with_numeric.yml

@@ -0,0 +1,64 @@
+---
+name: s3test
+director_uuid: UUID
+release:
+  name: myrelease
+  version: latest
+compilation:
+  workers: 10
+  network: default
+  reuse_compilation_vms: true
+  cloud_properties:
+    instance_type: m1.small
+update:
+  canaries: 1
+  canary_watch_time: 30000
+  update_watch_time: 30000
+  max_in_flight: 4
+  max_errors: 1
+networks:
+- name: default
+  type: dynamic
+  cloud_properties:
+    security_groups:
+    - default
+- name: vip_network
+  type: vip
+  cloud_properties:
+    security_groups:
+    - default
+resource_pools:
+- name: common
+  network: default
+  size: 2
+  stemcell:
+    name: bosh-stemcell
+    version: 0.6.4
+  cloud_properties:
+    instance_type: m1.small
+  persistent_disk: 8196
+jobs:
+- name: job-with-ips
+  template: job-with-ips
+  instances: 1
+  resource_pool: common
+  networks:
+  - name: default
+    default:
+    - dns
+    - gateway
+  - name: vip_network
+    static_ips:
+    - 4.3.2.1
+  persistent_disk: 8196
+- name: misc
+  template: misc
+  instances: 1
+  resource_pool: common
+  networks:
+  - name: default
+    default:
+    - dns
+    - gateway
+  persistent_disk: 8196
+properties: {}

data/spec/fixtures/releases/s3test-boshrelease/.gitignore

@@ -0,0 +1,18 @@
+config/dev.yml
+config/private.yml
+releases/*.tgz
+dev_releases
+blobs/*
+.blobs
+.dev_builds
+.vagrant
+.idea
+.DS_Store
+.final_builds/jobs/**/*.tgz
+.final_builds/packages/**/*.tgz
+*.swp
+*~
+*#
+#*
+tmp
+my*.yml

data/spec/fixtures/releases/s3test-boshrelease/README.md

@@ -0,0 +1,51 @@
+# BOSH Release for s3test
+
+## Usage
+
+To use this bosh release, first upload it to your bosh:
+
+```
+bosh target BOSH_HOST
+git clone https://github.com/cloudfoundry-community/s3test-boshrelease.git
+cd s3test-boshrelease
+bosh upload release releases/s3test-1.yml
+```
+
+For [bosh-lite](https://github.com/cloudfoundry/bosh-lite), you can quickly create a deployment manifest & deploy a cluster:
+
+```
+templates/make_manifest warden
+bosh -n deploy
+```
+
+For AWS EC2, create a single VM:
+
+```
+templates/make_manifest aws-ec2
+bosh -n deploy
+```
+
+### Override security groups
+
+For AWS & Openstack, the default deployment assumes there is a `default` security group. If you wish to use a different security group(s) then you can pass in additional configuration when running `make_manifest` above.
+
+Create a file `my-networking.yml`:
+
+``` yaml
+---
+networks:
+  - name: s3test1
+    type: dynamic
+    cloud_properties:
+      security_groups:
+        - s3test
+```
+
+Where `- s3test` means you wish to use an existing security group called `s3test`.
+
+You now suffix this file path to the `make_manifest` command:
+
+```
+templates/make_manifest openstack-nova my-networking.yml
+bosh -n deploy
+```

data/spec/fixtures/releases/s3test-boshrelease/Rakefile

@@ -0,0 +1,15 @@
+desc "Generates a properties file for each job based on properties.X.Y used in templates"
+task :job_properties do
+  require "fileutils"
+  Dir["jobs/*"].each do |path|
+    puts "Searching job #{File.basename(path)}..."
+    FileUtils.chdir(path) do
+      properties = []
+      Dir["templates/*.erb"].each do |template_path|
+        properties |= File.read(template_path).scan(/\bproperties\.[\w\.]*\b/)
+        puts properties.join("\n")
+        File.open("properties", "w") { |file| file << properties.join("\n") }
+      end
+    end
+  end
+end

data/spec/fixtures/releases/s3test-boshrelease/config/blobs.yml

@@ -0,0 +1 @@
+--- {}

data/spec/fixtures/releases/s3test-boshrelease/config/final.yml

@@ -0,0 +1,5 @@
+---
+blobstore:
+  provider: local
+  options:
+    blobstore_path: /tmp/blobstore

data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/monit

@@ -0,0 +1,5 @@
+check process s3test
+  with pidfile /var/vcap/sys/run/s3test/s3test.pid
+  start program "/var/vcap/jobs/s3test/bin/monit_debugger s3test_ctl '/var/vcap/jobs/s3test/bin/s3test_ctl start'"
+  stop program "/var/vcap/jobs/s3test/bin/monit_debugger s3test_ctl '/var/vcap/jobs/s3test/bin/s3test_ctl stop'"
+  group vcap

data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/spec

@@ -0,0 +1,13 @@
+---
+name: s3test
+packages: []
+templates:
+  bin/monit_debugger: bin/monit_debugger
+  bin/s3test_ctl: bin/s3test_ctl
+  config/s3test.conf.erb: config/s3test.conf
+  data/properties.sh.erb: data/properties.sh
+  helpers/ctl_setup.sh: helpers/ctl_setup.sh
+  helpers/ctl_utils.sh: helpers/ctl_utils.sh
+properties:
+  s3test.leader_address:
+    description: Hostname/IP to the leader/master of the cluster

data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/bin/monit_debugger

@@ -0,0 +1,13 @@
+#!/bin/sh
+# USAGE monit_debugger <label> command to run
+mkdir -p /var/vcap/sys/log/monit
+{
+  echo "MONIT-DEBUG date"
+  date
+  echo "MONIT-DEBUG env"
+  env
+  echo "MONIT-DEBUG $@"
+  $2 $3 $4 $5 $6 $7
+  R=$?
+  echo "MONIT-DEBUG exit code $R"
+} >/var/vcap/sys/log/monit/monit_debugger.$1.log 2>&1