bosh-gen 0.13.3 → 0.14.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDU2OTNhYTgzN2FkODM4ZTlhOWI5YWI2MTE3MjZlMmJmMDRlOTY0Mw==
+    ZGY3MjdjNjc4MzQ1Y2Q0MjNmZTM1Mzg4ODYxNDJlNzFjMjdmMGRiNQ==
   data.tar.gz: !binary |-
-    Mzg0ZTdjNjY1OGUwN2M3YTI1OGJhMzBjNmJkOTE3MGY3YmNjNDY1Nw==
+    NWE2ZWFjMGY5MmZiYWI4NTU0MjU2MmY4NGRmMDdiODc0MThkYjIxMw==
 SHA512:
   metadata.gz: !binary |-
-    ZjczMWRhOWJmN2M4OTMzNTg5NGM0N2JiN2E4NzAyZDgzNDI1Yjk5YjliMzMz
-    MWMzZGY0YmZhZGE5OWY1Y2Y5NGRiZmVlODU5Mjg4Y2VjODlkMTZiY2NlMTkz
-    ZDUxNzZmZDE0NTZiY2Y0OTg0N2Q0ZjdjZjFmYmJkYzA5YjI0YzU=
+    MjAxNTliZDc2M2Y4ZWFiNjA0ZTgwYjI5Zjg2NjI1ODY1NTc2ZWJkMDBkNDNk
+    MWM4OWE0NTE3MTFhNDY1NzE3MzZkMjQxYzliOWRiMzYyZDk4MDlkMzZhZDEx
+    MmY0OGVkNDYwMTk3MzgxYmIyOWI4NDA1MTRjNjkwNmNkZWQ0ZjY=
   data.tar.gz: !binary |-
-    NWFhZDZjN2Y4MmFlYmM0YTZmNDgyZDAwNzk4MGQ5NTVmNjcyMjc1Njk5OTVm
-    Y2RmOTAyZTU1YjFmYTIzNjY4YzhhY2NhY2RjZGIzYjQxNzE1MTYwOTExNzkw
-    NDJjMTBmNDQ3NWZlMjE3NWYwNDg3ZTMxMjY4NGZlYjQ4MWJhMGU=
+    NWJlMzBlYzY2OTlmMjEwNGE3YTA3YWEyNzk3OTMxOWRhNDBmZDYyNzZiMzg3
+    MjBjNTRkM2U2Y2I3ODZkMGNiOWVmYjE4NzM1NzUyZjg4NGVjNDJlNWVjZjgw
+    ZWY4NzlhNWY3MGRlMWYxYmJiMDhjZDRiMDVmNDE1YjY1NWI0OWY=

data/.gitignore

@@ -15,4 +15,5 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
-.DS_Store
+.DS_Store
+binstubs

data/ChangeLog.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## v0.14.0
+
+* templates/make_manifest & example spiff templates generated
+* no more static example bosh-lite.yml templates
+
 ## v0.13.0
 
 * bosh-lite example manifests generated for new releases

data/Gemfile

@@ -1,8 +1,7 @@
 source 'https://rubygems.org'
-source 'https://s3.amazonaws.com/bosh-jenkins-gems/' # for bosh 1.5.0.pre gems
 
 # Specify your gem's dependencies in bosh-gen.gemspec
 gemspec
 
-gem 'bosh_cli', '~> 1.5.0.pre'
-gem 'bosh_common', '~> 1.5.0.pre'
+gem 'bosh_cli', '~> 1.1750.0'
+gem 'bosh_common', '~> 1.1750.0'

data/bin/bosh-gen

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
-$:.push File.dirname(__FILE__) + '/../lib'
 
+$:.push File.dirname(__FILE__) + '/../lib'
 require "rubygems"
 require "bosh/gen/cli"
 

data/lib/bosh/gen/cli.rb

@@ -22,7 +22,6 @@ module Bosh
       def new(path)
         flags = { :aws => options["s3"], :atmos => options["atmos"],
                   :swift => options["swift"]}
-        
         require 'bosh/gen/generators/new_release_generator'
         Bosh::Gen::Generators::NewReleaseGenerator.start([path, flags])
       end

data/lib/bosh/gen/generators/new_release_generator.rb

@@ -28,11 +28,15 @@ module Bosh::Gen
       end
       
       def directories
-        %w[jobs packages src blobs examples].each do |dir|
+        %w[jobs packages src blobs templates].each do |dir|
           directory dir
         end
       end
-      
+
+      def executables
+        chmod "templates/make_manifest", 0755
+      end
+
       def blobs_yaml
         create_file "config/blobs.yml", YAML.dump({})
       end
@@ -149,6 +153,8 @@ module Bosh::Gen
         *~
         *#
         #*
+        tmp
+        my*.yml
         IGNORE
       end
       
@@ -179,6 +185,14 @@ module Bosh::Gen
         @project_name ||= File.basename(proposed_app_path).gsub(/-(?:boshrelease|release)$/, '')
       end
 
+      def project_name_hyphenated
+        project_name.gsub(/[^A-Za-z]+/, '-')
+      end
+
+      def project_name_underscored
+        project_name.gsub(/[^A-Za-z]+/, '_')
+      end
+
       def blobstore_type
         return :s3 if s3?
         return :atmos if atmos?

data/lib/bosh/gen/generators/new_release_generator/templates/README.md.tt

@@ -11,11 +11,48 @@ cd <%= repository_name %>
 bosh upload release releases/<%= project_name %>-1.yml
 ```
 
-For [bosh-lite](https://github.com/cloudfoundry/bosh-lite), you can quickly create a deployment manifest:
+For [bosh-lite](https://github.com/cloudfoundry/bosh-lite), you can quickly create a deployment manifest & deploy a 3 VM cluster:
 
 ```
-cp examples/bosh-lite-cluster.yml local-cluster.yml
-sed -i '' -e "s/DIRECTOR_UUID/$(bosh status | grep UUID | awk '{print $2}')/" local-cluster.yml
-bosh deployment local-cluster.yml
+templates/make_manifest warden
+bosh -n deploy
+```
+
+For Openstack (Nova Networks), create a single VM:
+
+```
+templates/make_manifest openstack-nova
+bosh -n deploy
+```
+
+For AWS EC2, create a single VM:
+
+```
+templates/make_manifest aws-ec2
+bosh -n deploy
+```
+
+### Override security groups
+
+For AWS & Openstack, the default deployment assumes there is a `default` security group. If you wish to use a different security group(s) then you can pass in additional configuration when running `make_manifest` above.
+
+Create a file `my-networking.yml`:
+
+``` yaml
+---
+networks:
+  - name: <%= project_name %>1
+    type: dynamic
+    cloud_properties:
+      security_groups:
+        - <%= project_name %>
+```
+
+Where `- <%= project_name %>` means you wish to use an existing security group called `<%= project_name %>`.
+
+You now suffix this file path to the `make_manifest` command:
+
+```
+templates/make_manifest openstack-nova my-networking.yml
 bosh -n deploy
 ```

data/lib/bosh/gen/generators/new_release_generator/templates/templates/deployment.yml.tt

@@ -0,0 +1,34 @@
+meta:
+  environment: (( merge ))
+  stemcell: (( merge ))
+
+name: (( meta.environment ))
+
+director_uuid: (( merge ))
+
+releases: (( merge ))
+
+networks: (( merge ))
+
+jobs: (( merge ))
+
+properties: (( merge ))
+
+compilation:
+  workers: 6
+  network: <%= project_name_underscored %>1
+  reuse_compilation_vms: true
+  cloud_properties: (( merge ))
+
+update:
+  canaries: 1
+  max_in_flight: 1
+  canary_watch_time: 30000-60000
+  update_watch_time: 5000-60000
+
+resource_pools:
+  - name: small_z1
+    network: <%= project_name_underscored %>1
+    size: (( auto ))
+    stemcell: (( meta.stemcell ))
+    cloud_properties: (( merge ))

data/lib/bosh/gen/generators/new_release_generator/templates/templates/infrastructure-aws-ec2.yml.tt

@@ -0,0 +1,40 @@
+meta:
+  environment: (( merge ))
+  dns_root: (( merge ))
+  security_groups: (( merge ))
+  network_name: (( merge ))
+
+  stemcell:
+    name: bosh-aws-xen-ubuntu
+    version: 1722
+
+jobs:
+  - name: <%= project_name_underscored %>_leader_z1
+    instances: 1
+    networks:
+      - name: (( meta.network_name ))
+  - name: <%= project_name %>_z1
+    instances: 2
+    networks:
+      - name: (( meta.network_name ))
+    properties:
+      etcd:
+        leader_address: (( "0.<%= project_name_hyphenated %>-leader-z1.<%= project_name_hyphenated %>1." meta.environment "." meta.dns_root ))
+
+compilation:
+  cloud_properties:
+    instance_type: m1.small
+
+resource_pools:
+  - name: small_z1
+    cloud_properties:
+      instance_type: m1.small
+
+networks:
+  - name: floating
+    type: vip
+    cloud_properties: {}
+  - name: (( meta.network_name ))
+    type: dynamic
+    cloud_properties:
+      security_groups: (( meta.security_groups ))

data/lib/bosh/gen/generators/new_release_generator/templates/templates/infrastructure-warden.yml.tt

@@ -0,0 +1,107 @@
+meta:
+  environment: <%= project_name_hyphenated %>-warden
+  network_name: (( merge ))
+
+  stemcell:
+    name: bosh-warden-boshlite-ubuntu
+    version: 24
+
+update:
+  canaries: 1
+  max_in_flight: 50
+  canary_watch_time: 1000-30000
+  update_watch_time: 1000-30000
+
+jobs:
+  - name: <%= project_name_underscored %>_leader_z1
+    instances: 1
+    networks:
+      - name: (( meta.network_name ))
+        static_ips: (( static_ips(0) ))
+  - name: <%= project_name_underscored %>_z1
+    instances: 2
+    networks:
+      - name: (( meta.network_name ))
+        static_ips: ~
+    properties:
+      <%= project_name_underscored %>:
+        master: (( jobs.<%= project_name_underscored %>_leader_z1.networks.<%= project_name_underscored %>1.static_ips.[0] ))
+
+compilation:
+  cloud_properties:
+    name: random
+
+resource_pools:
+  - name: small_z1
+    cloud_properties:
+      name: random
+
+networks:
+- name: (( meta.network_name ))
+  # Assumes up to 5 VMs, including 1 static and 4 dynamic.
+  # Plus 5 (double the size) unused IPs, due to BOSH bug/quirk.
+  subnets:
+  - cloud_properties:
+      name: random
+    range: 10.244.2.0/30
+    reserved:
+    - 10.244.2.1
+    static:
+    - 10.244.2.2
+
+  - cloud_properties:
+      name: random
+    range: 10.244.2.4/30
+    reserved:
+    - 10.244.2.5
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.8/30
+    reserved:
+    - 10.244.2.9
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.12/30
+    reserved:
+    - 10.244.2.13
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.16/30
+    reserved:
+    - 10.244.2.17
+    static: []
+
+  # Bonus double-sized network required due to BOSH oddity
+  - cloud_properties:
+      name: random
+    range: 10.244.2.20/30
+    reserved:
+    - 10.244.2.21
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.24/30
+    reserved:
+    - 10.244.2.25
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.28/30
+    reserved:
+    - 10.244.2.29
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.32/30
+    reserved:
+    - 10.244.2.33
+    static: []
+  - cloud_properties:
+      name: random
+    range: 10.244.2.36/30
+    reserved:
+    - 10.244.2.37
+    static: []

data/lib/bosh/gen/generators/new_release_generator/templates/templates/jobs.yml.tt

@@ -0,0 +1,27 @@
+meta:
+  environment: ~
+
+update:
+  canaries: 1
+  max_in_flight: 50
+  canary_watch_time: 1000-30000
+  update_watch_time: 1000-30000
+
+jobs:
+  - name: <%= project_name_underscored %>_leader_z1
+    template: <%= project_name_underscored %>
+    instances: 0
+    resource_pool: small_z1
+    networks: (( merge ))
+  - name: <%= project_name_underscored %>_z1
+    template: <%= project_name_underscored %>
+    instances: 0
+    resource_pool: small_z1
+    networks: (( merge ))
+    update:
+      canaries: 10
+    properties:
+      <%= project_name_underscored %>:
+        master: (( merge ))
+
+networks: (( merge ))

data/lib/bosh/gen/generators/new_release_generator/templates/templates/make_manifest.tt

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -e
+
+infrastructure=$1
+template_prefix="<%= project_name %>"
+
+if [ "$infrastructure" != "openstack-nova" ] && \
+    [ "$infrastructure" != "aws-ec2" ] && \
+    [ "$infrastructure" != "warden" ] ; then
+  echo "usage: ./make_manifest <openstack-nova|warden>"
+  exit 1
+fi
+
+shift
+
+BOSH_STATUS=$(bosh status)
+DIRECTOR_UUID=$(echo "$BOSH_STATUS" | grep UUID | awk '{print $2}')
+DIRECTOR_CPI=$(echo "$BOSH_STATUS" | grep CPI | awk '{print $2}')
+DIRECTOR_NAME=$(echo "$BOSH_STATUS" | grep Name | awk '{print $2}')
+NAME=$template_prefix-$infrastructure
+
+if [[ $DIRECTOR_NAME = "warden" ]]; then
+  if [[ $infrastructure != "warden" ]]; then
+    echo "Not targeting bosh-lite with warden CPI. Please use 'bosh target' before running this script."
+    exit 1
+  fi
+fi
+
+if [[ $DIRECTOR_CPI = "aws" ]]; then
+  if [[ $infrastructure = "aws-ec2" ]]; then
+    echo "Not targeting an AWS BOSH. Please use 'bosh target' before running this script."
+    exit 1
+  fi
+fi
+
+if [[ $DIRECTOR_CPI != "openstack" ]]; then
+  if [[ $infrastructure = "openstack-nova" ]]; then
+    echo "Not targeting an OpenStack BOSH. Please use 'bosh target' before running this script."
+    exit 1
+  fi
+fi
+
+templates=$(dirname $0)
+release=$templates/..
+tmpdir=$release/tmp
+
+mkdir -p $tmpdir
+cp $templates/stub.yml $tmpdir/stub-with-uuid.yml
+echo $DIRECTOR_NAME $DIRECTOR_CPI $DIRECTOR_UUID
+perl -pi -e "s/PLACEHOLDER-DIRECTOR-UUID/$DIRECTOR_UUID/g" $tmpdir/stub-with-uuid.yml
+perl -pi -e "s/NAME/$NAME/g" $tmpdir/stub-with-uuid.yml
+
+spiff merge \
+  $templates/deployment.yml \
+  $templates/jobs.yml \
+  $templates/properties.yml \
+  $templates/infrastructure-${infrastructure}.yml \
+  $tmpdir/stub-with-uuid.yml \
+  $* > $tmpdir/$NAME-manifest.yml
+
+bosh deployment $tmpdir/$NAME-manifest.yml
+bosh status

data/lib/bosh/gen/generators/new_release_generator/templates/templates/properties.yml.tt

@@ -0,0 +1,5 @@
+properties:
+  networks:
+    apps: <%= project_name_underscored %>1
+
+  <%= project_name_underscored %>: {}

data/lib/bosh/gen/generators/new_release_generator/templates/templates/stub.yml.tt

@@ -0,0 +1,13 @@
+---
+meta:
+  environment: NAME
+  dns_root: microbosh
+  network_name: <%= project_name_underscored %>1
+  security_groups:
+    - <%= project_name_underscored %>
+
+director_uuid: PLACEHOLDER-DIRECTOR-UUID
+
+releases:
+  - name: <%= project_name_underscored %>
+    version: latest

data/lib/bosh/gen/generators/package_generator.rb

@@ -42,6 +42,9 @@ module Bosh::Gen
           set -e # exit immediately if a simple command exits with a non-zero status
           set -u # report the usage of uninitialized variables
           
+          # Detect # of CPUs so make jobs can be parallelized
+          CPUS=`grep -c ^processor /proc/cpuinfo`
+
           # Available variables
           # $BOSH_COMPILE_TARGET - where this package & spec'd source files are available
           # $BOSH_INSTALL_TARGET - where you copy/install files to be included in package
@@ -60,8 +63,7 @@ module Bosh::Gen
             #{unpack_command} #{name}/#{archive_file}
             cd #{unpack_base_path}
             ./configure --prefix=${BOSH_INSTALL_TARGET}
-            make
-            make install
+            make -j${CPUS} && make install
             # Alternatively, to copy archive contents:
             # cp -a #{unpack_base_path}/* $BOSH_INSTALL_TARGET
             SHELL

data/lib/bosh/gen/generators/package_source_generator.rb

@@ -50,8 +50,7 @@ module Bosh::Gen
             (
               cd #{package_dir}
               ./configure --prefix=$BOSH_INSTALL_TARGET
-              make
-              make install
+              make -j${CPUS} && make install
             )
             
             BASH

data/lib/bosh/gen/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Gen
-    VERSION = "0.13.3"
+    VERSION = "0.14.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh-gen
 version: !ruby/object:Gem::Version
-  version: 0.13.3
+  version: 0.14.0
 platform: ruby
 authors:
 - Dr Nic Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-10 00:00:00.000000000 Z
+date: 2014-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -162,11 +162,16 @@ files:
 - lib/bosh/gen/generators/new_release_generator/templates/README.md.tt
 - lib/bosh/gen/generators/new_release_generator/templates/Rakefile
 - lib/bosh/gen/generators/new_release_generator/templates/blobs/.gitkeep
-- lib/bosh/gen/generators/new_release_generator/templates/examples/bosh-lite-cluster.yml.tt
-- lib/bosh/gen/generators/new_release_generator/templates/examples/bosh-lite-solo.yml.tt
 - lib/bosh/gen/generators/new_release_generator/templates/jobs/.gitkeep
 - lib/bosh/gen/generators/new_release_generator/templates/packages/.gitkeep
 - lib/bosh/gen/generators/new_release_generator/templates/src/.gitkeep
+- lib/bosh/gen/generators/new_release_generator/templates/templates/deployment.yml.tt
+- lib/bosh/gen/generators/new_release_generator/templates/templates/infrastructure-aws-ec2.yml.tt
+- lib/bosh/gen/generators/new_release_generator/templates/templates/infrastructure-warden.yml.tt
+- lib/bosh/gen/generators/new_release_generator/templates/templates/jobs.yml.tt
+- lib/bosh/gen/generators/new_release_generator/templates/templates/make_manifest.tt
+- lib/bosh/gen/generators/new_release_generator/templates/templates/properties.yml.tt
+- lib/bosh/gen/generators/new_release_generator/templates/templates/stub.yml.tt
 - lib/bosh/gen/generators/package_generator.rb
 - lib/bosh/gen/generators/package_generator/templates/.gitkeep
 - lib/bosh/gen/generators/package_source_generator.rb
@@ -244,7 +249,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.2
+rubygems_version: 2.1.11
 signing_key: 
 specification_version: 4
 summary: ''

data/lib/bosh/gen/generators/new_release_generator/templates/examples/bosh-lite-cluster.yml.tt

@@ -1,67 +0,0 @@
----
-name: <%= project_name %>-cluster
-director_uuid: DIRECTOR_UUID
-releases:
-- name: <%= project_name %>
-  version: latest
-compilation:
-  workers: 3
-  network: <%= project_name %>-network
-  reuse_compilation_vms: true
-  cloud_properties:
-    name: random
-update:
-  canaries: 1
-  max_in_flight: 4
-  canary_watch_time: 10000-30000
-  update_watch_time: 10000-30000
-resource_pools:
-- name: common
-  network: <%= project_name %>-network
-  size: 3
-  stemcell:
-    name: bosh-stemcell
-    version: latest
-  cloud_properties:
-    name: random
-networks:
-- name: <%= project_name %>-network
-  subnets:
-<%% (0..28).each_with_index do |i| %>
-  - range: 10.244.0.<%%= i*4 %>/30
-    reserved:
-      - 10.244.0.<%%= i*4 + 1 %>
-    static:
-      - 10.244.0.<%%= i*4 + 2 %>
-    cloud_properties:
-      name: random
-<%% end %>
-<%% (29..61).each_with_index do |i| %>
-  - range: 10.244.0.<%%= i*4 %>/30
-    reserved:
-      - 10.244.0.<%%= i*4 + 1 %>
-    cloud_properties:
-      name: random
-<%% end %>
-<%% (62..63).each_with_index do |i| %>
-  - range: 10.244.0.<%%= i*4 %>/30
-    reserved:
-      - 10.244.0.<%%= i*4 + 1 %>
-    static:
-      - 10.244.0.<%%= i*4 + 2 %>
-    cloud_properties:
-      name: random
-<%% end %>
-jobs:
-- name: <%= project_name %>
-  template: <%= project_name %>
-  instances: 3
-  resource_pool: common
-  networks:
-  - name: <%= project_name %>-network
-    static_ips:
-    - 10.244.0.6
-    - 10.244.0.10
-    - 10.244.0.14
-properties:
-  <%= project_name %>: {}

data/lib/bosh/gen/generators/new_release_generator/templates/examples/bosh-lite-solo.yml.tt

@@ -1,65 +0,0 @@
----
-name: <%= project_name %>-solo
-director_uuid: DIRECTOR_UUID
-releases:
-- name: <%= project_name %>
-  version: latest
-compilation:
-  workers: 3
-  network: <%= project_name %>-network
-  reuse_compilation_vms: true
-  cloud_properties:
-    name: random
-update:
-  canaries: 1
-  max_in_flight: 4
-  canary_watch_time: 10000-30000
-  update_watch_time: 10000-30000
-resource_pools:
-- name: common
-  network: <%= project_name %>-network
-  size: 1
-  stemcell:
-    name: bosh-stemcell
-    version: latest
-  cloud_properties:
-    name: random
-networks:
-- name: <%= project_name %>-network
-  subnets:
-<%% (0..28).each_with_index do |i| %>
-  - range: 10.244.0.<%%= i*4 %>/30
-    reserved:
-      - 10.244.0.<%%= i*4 + 1 %>
-    static:
-      - 10.244.0.<%%= i*4 + 2 %>
-    cloud_properties:
-      name: random
-<%% end %>
-<%% (29..61).each_with_index do |i| %>
-  - range: 10.244.0.<%%= i*4 %>/30
-    reserved:
-      - 10.244.0.<%%= i*4 + 1 %>
-    cloud_properties:
-      name: random
-<%% end %>
-<%% (62..63).each_with_index do |i| %>
-  - range: 10.244.0.<%%= i*4 %>/30
-    reserved:
-      - 10.244.0.<%%= i*4 + 1 %>
-    static:
-      - 10.244.0.<%%= i*4 + 2 %>
-    cloud_properties:
-      name: random
-<%% end %>
-jobs:
-- name: <%= project_name %>
-  template: <%= project_name %>
-  instances: 1
-  resource_pool: common
-  networks:
-  - name: <%= project_name %>-network
-    static_ips:
-    - 10.244.0.6
-properties:
-  <%= project_name %>: {}