bosh-gen 0.16.2 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +13 -5
  2. data/.rspec +3 -0
  3. data/.travis.yml +16 -0
  4. data/ChangeLog.md +110 -89
  5. data/Gemfile +7 -1
  6. data/Guardfile +4 -8
  7. data/README.md +52 -22
  8. data/Rakefile +10 -0
  9. data/bosh-gen.gemspec +7 -5
  10. data/lib/bosh/gen/cli.rb +20 -28
  11. data/lib/bosh/gen/generators/job_generator.rb +9 -10
  12. data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_simple/templates/bin/%job_name%_ctl.tt +5 -5
  13. data/lib/bosh/gen/generators/new_release_generator.rb +63 -68
  14. data/lib/bosh/gen/generators/new_release_generator/templates/jobs/%project_name_hyphenated%/templates/bin/%job_name%_ctl.tt +5 -5
  15. data/lib/bosh/gen/generators/new_release_generator/templates/jobs/%project_name_hyphenated%/templates/data/properties.sh.erb +6 -0
  16. data/lib/bosh/gen/generators/new_release_generator/templates/templates/infrastructure-aws-ec2.yml.tt +7 -5
  17. data/lib/bosh/gen/generators/new_release_generator/templates/templates/infrastructure-warden.yml.tt +3 -4
  18. data/lib/bosh/gen/generators/new_release_generator/templates/templates/jobs.yml.tt +3 -0
  19. data/lib/bosh/gen/generators/new_release_generator/templates/templates/make_manifest.tt +18 -2
  20. data/lib/bosh/gen/generators/new_release_generator/templates/templates/stub.yml.tt +2 -1
  21. data/lib/bosh/gen/settings.rb +61 -0
  22. data/lib/bosh/gen/version.rb +1 -1
  23. data/spec/fixtures/deployment_manifests/2_jobs_1_ip_8196_disk_with_numeric.yml +64 -0
  24. data/spec/fixtures/releases/s3test-boshrelease/.gitignore +18 -0
  25. data/spec/fixtures/releases/s3test-boshrelease/README.md +51 -0
  26. data/spec/fixtures/releases/s3test-boshrelease/Rakefile +15 -0
  27. data/spec/fixtures/releases/s3test-boshrelease/config/blobs.yml +1 -0
  28. data/spec/fixtures/releases/s3test-boshrelease/config/final.yml +5 -0
  29. data/spec/fixtures/releases/s3test-boshrelease/jobs/.gitkeep +0 -0
  30. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/monit +5 -0
  31. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/spec +13 -0
  32. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/bin/monit_debugger +13 -0
  33. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/bin/s3test_ctl +36 -0
  34. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/config/.gitkeep +0 -0
  35. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/config/s3test.conf.erb +5 -0
  36. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/data/properties.sh.erb +10 -0
  37. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/helpers/ctl_setup.sh +81 -0
  38. data/spec/fixtures/releases/s3test-boshrelease/jobs/s3test/templates/helpers/ctl_utils.sh +156 -0
  39. data/spec/fixtures/releases/s3test-boshrelease/packages/.gitkeep +0 -0
  40. data/spec/fixtures/releases/s3test-boshrelease/src/.gitkeep +0 -0
  41. data/spec/fixtures/releases/s3test-boshrelease/templates/deployment.yml +35 -0
  42. data/spec/fixtures/releases/s3test-boshrelease/templates/infrastructure-aws-ec2.yml +40 -0
  43. data/spec/fixtures/releases/s3test-boshrelease/templates/infrastructure-warden.yml +107 -0
  44. data/spec/fixtures/releases/s3test-boshrelease/templates/jobs.yml +30 -0
  45. data/spec/fixtures/releases/s3test-boshrelease/templates/make_manifest +54 -0
  46. data/spec/fixtures/releases/s3test-boshrelease/templates/stub.yml +13 -0
  47. data/spec/generators/deployment_manifest_generator_spec.rb +19 -14
  48. data/spec/generators/{jobs/webapp_job_generator_spec.rb → job_generator_spec.rb} +8 -13
  49. data/spec/generators/new_release_generator_spec.rb +70 -0
  50. data/spec/models/bosh_config_spec.rb +3 -3
  51. data/spec/models/deployment_manifest_spec.rb +16 -6
  52. data/spec/models/release_detection_spec.rb +6 -6
  53. data/spec/spec_helper.rb +27 -7
  54. data/spec/support/fog.rb +3 -0
  55. data/spec/{generators → support}/generator_spec_helper.rb +47 -6
  56. metadata +105 -36
  57. data/spec/models/extract_spec.rb +0 -10
@@ -14,13 +14,13 @@ case $1 in
14
14
  start)
15
15
  pid_guard $PIDFILE $JOB_NAME
16
16
 
17
- # TODO: Run some process
18
- exec chpst -u vcap:vcap TODO \
17
+ # store pid in $PIDFILE
18
+ echo $$ > $PIDFILE
19
+
20
+ exec chpst -u vcap:vcap <%= job_name %> \
19
21
  >>$LOG_DIR/$JOB_NAME.stdout.log \
20
22
  2>>$LOG_DIR/$JOB_NAME.stderr.log
21
23
 
22
- # store pid in $PIDFILE
23
- echo $! > $PIDFILE
24
24
  ;;
25
25
 
26
26
  stop)
@@ -33,4 +33,4 @@ case $1 in
33
33
  ;;
34
34
 
35
35
  esac
36
- exit 0
36
+ exit 0
@@ -8,3 +8,9 @@ export NAME='<%= name %>'
8
8
  export JOB_INDEX=<%= index %>
9
9
  # full job name, like redis/0 or webapp/3
10
10
  export JOB_FULL="$NAME/$JOB_INDEX"
11
+
12
+ export DEPLOYMENT_NAME=<%= spec.deployment %>
13
+ export DNS_ROOT=<%= spec.dns_domain_name %>
14
+
15
+ # $BIND_ADDR is the IP of the first network
16
+ export BIND_ADDR=<%= spec.networks.send(spec.networks.methods(false).first).ip %>
@@ -2,7 +2,7 @@ meta:
2
2
  environment: (( merge ))
3
3
  dns_root: (( merge ))
4
4
  security_groups: (( merge ))
5
- network_name: (( merge ))
5
+ persistent_disk: 4096
6
6
 
7
7
  stemcell:
8
8
  name: bosh-aws-xen-ubuntu
@@ -12,11 +12,13 @@ jobs:
12
12
  - name: <%= project_name_underscored %>_leader_z1
13
13
  instances: 1
14
14
  networks:
15
- - name: (( meta.network_name ))
16
- - name: <%= project_name %>_z1
15
+ - name: <%= project_name_underscored %>1
16
+ persistent_disk: (( meta.persistent_disk ))
17
+ - name: <%= project_name_underscored %>_z1
17
18
  instances: 2
18
19
  networks:
19
- - name: (( meta.network_name ))
20
+ - name: <%= project_name_underscored %>1
21
+ persistent_disk: (( meta.persistent_disk ))
20
22
  properties:
21
23
  <%= project_name_underscored %>:
22
24
  leader_address: (( "0.<%= project_name_hyphenated %>-leader-z1.<%= project_name_hyphenated %>1." meta.environment "." meta.dns_root ))
@@ -34,7 +36,7 @@ networks:
34
36
  - name: floating
35
37
  type: vip
36
38
  cloud_properties: {}
37
- - name: (( meta.network_name ))
39
+ - name: <%= project_name_underscored %>1
38
40
  type: dynamic
39
41
  cloud_properties:
40
42
  security_groups: (( meta.security_groups ))
@@ -1,6 +1,5 @@
1
1
  meta:
2
2
  environment: <%= project_name_hyphenated %>-warden
3
- network_name: (( merge ))
4
3
 
5
4
  stemcell:
6
5
  name: bosh-warden-boshlite-ubuntu
@@ -16,12 +15,12 @@ jobs:
16
15
  - name: <%= project_name_underscored %>_leader_z1
17
16
  instances: 1
18
17
  networks:
19
- - name: (( meta.network_name ))
18
+ - name: <%= project_name_underscored %>1
20
19
  static_ips: (( static_ips(0) ))
21
20
  - name: <%= project_name_underscored %>_z1
22
21
  instances: 2
23
22
  networks:
24
- - name: (( meta.network_name ))
23
+ - name: <%= project_name_underscored %>1
25
24
  static_ips: ~
26
25
  properties:
27
26
  <%= project_name_underscored %>:
@@ -37,7 +36,7 @@ resource_pools:
37
36
  name: random
38
37
 
39
38
  networks:
40
- - name: (( meta.network_name ))
39
+ - name: <%= project_name_underscored %>1
41
40
  # Assumes up to 5 VMs, including 1 static and 4 dynamic.
42
41
  # Plus 5 (double the size) unused IPs, due to BOSH bug/quirk.
43
42
  subnets:
@@ -14,11 +14,14 @@ jobs:
14
14
  instances: 0
15
15
  resource_pool: small_z1
16
16
  networks: (( merge ))
17
+ persistent_disk: 0
18
+ properties: ~
17
19
  - name: <%= project_name_underscored %>_z1
18
20
  template: <%= project_name_hyphenated %>
19
21
  instances: 0
20
22
  resource_pool: small_z1
21
23
  networks: (( merge ))
24
+ persistent_disk: 0
22
25
  update:
23
26
  canaries: 10
24
27
  properties:
@@ -2,8 +2,10 @@
2
2
 
3
3
  set -e
4
4
 
5
- infrastructure=$1
6
5
  template_prefix="<%= project_name %>"
6
+ STEMCELL_OS=${STEMCELL_OS:-ubuntu}
7
+
8
+ infrastructure=$1
7
9
 
8
10
  if [ "$infrastructure" != "aws-ec2" ] && \
9
11
  [ "$infrastructure" != "warden" ] ; then
@@ -33,15 +35,29 @@ if [[ $infrastructure = "aws-ec2" ]]; then
33
35
  fi
34
36
  fi
35
37
 
38
+ function latest_uploaded_stemcell {
39
+ echo $(bosh stemcells | grep bosh | grep $STEMCELL_OS | awk -F'|' '{ print $2, $3 }' | sort -nr -k2 | head -n1 | awk '{ print $1 }')
40
+ }
41
+
42
+ STEMCELL=${STEMCELL:-$(latest_uploaded_stemcell)}
43
+ if [[ "${STEMCELL}X" == "X" ]]; then
44
+ echo
45
+ echo "Uploading latest $DIRECTOR_CPI/$STEMCELL_OS stemcell..."
46
+ STEMCELL_URL=$(bosh public stemcells --full | grep $DIRECTOR_CPI | grep $STEMCELL_OS | sort -nr | head -n1 | awk '{ print $4 }')
47
+ bosh upload stemcell $STEMCELL_URL
48
+ fi
49
+ STEMCELL=${STEMCELL:-$(latest_uploaded_stemcell)}
50
+
36
51
  templates=$(dirname $0)
37
52
  release=$templates/..
38
53
  tmpdir=$release/tmp
39
54
 
40
55
  mkdir -p $tmpdir
41
56
  cp $templates/stub.yml $tmpdir/stub-with-uuid.yml
42
- echo $DIRECTOR_NAME $DIRECTOR_CPI $DIRECTOR_UUID
57
+ echo $DIRECTOR_NAME $DIRECTOR_CPI $DIRECTOR_UUID $STEMCELL
43
58
  perl -pi -e "s/PLACEHOLDER-DIRECTOR-UUID/$DIRECTOR_UUID/g" $tmpdir/stub-with-uuid.yml
44
59
  perl -pi -e "s/NAME/$NAME/g" $tmpdir/stub-with-uuid.yml
60
+ perl -pi -e "s/STEMCELL/$STEMCELL/g" $tmpdir/stub-with-uuid.yml
45
61
 
46
62
  spiff merge \
47
63
  $templates/deployment.yml \
@@ -2,9 +2,10 @@
2
2
  meta:
3
3
  environment: NAME
4
4
  dns_root: microbosh
5
- network_name: <%= project_name_underscored %>1
6
5
  security_groups:
7
6
  - <%= project_name_underscored %>
7
+ stemcell:
8
+ name: STEMCELL
8
9
 
9
10
  director_uuid: PLACEHOLDER-DIRECTOR-UUID
10
11
 
@@ -0,0 +1,61 @@
1
+ require "readwritesettings"
2
+ require "fileutils"
3
+
4
+ module Bosh::Gen::Settings
5
+
6
+ # The base directory for holding the manifest settings file
7
+ # and private keys
8
+ #
9
+ # Defaults to ~/.bosh_inception; and can be overridden with either:
10
+ # * $SETTINGS - to a folder (supported method)
11
+ def settings_dir
12
+ @settings_dir ||= File.expand_path(ENV["SETTINGS"] || "~/.bosh-gen")
13
+ end
14
+
15
+ def settings_dir=(settings_dir)
16
+ @settings_dir = File.expand_path(settings_dir)
17
+ @settings = @settings_path = nil
18
+ reload_settings!
19
+ end
20
+
21
+ def settings_ssh_dir
22
+ File.join(settings_dir, "ssh")
23
+ end
24
+
25
+ def settings_path
26
+ @settings_path ||= File.join(settings_dir, "settings.yml")
27
+ end
28
+
29
+ def settings
30
+ @settings ||= begin
31
+ unless File.exists?(settings_path)
32
+ FileUtils.mkdir_p(settings_dir)
33
+ File.open(settings_path, "w") { |file| file << "--- {}" }
34
+ end
35
+ FileUtils.chmod(0600, settings_path)
36
+ FileUtils.chmod(0700, settings_ssh_dir) if File.directory?(settings_ssh_dir)
37
+ ReadWriteSettings.new(settings_path)
38
+ end
39
+ end
40
+
41
+ # Set a nested setting with "key1.key2.key3" notation
42
+ def setting(nested_key, value)
43
+ settings.set(nested_key, value)
44
+ save_settings!
45
+ end
46
+
47
+ # Saves current nested ReadWriteSettings into pure Hash-based YAML file
48
+ # Recreates accessors on ReadWriteSettings object (since something has changed)
49
+ def save_settings!
50
+ File.open(settings_path, "w") { |f| f << settings.to_nested_hash.to_yaml }
51
+ settings.create_accessors!
52
+ end
53
+
54
+ def reload_settings!
55
+ @settings = nil
56
+ settings
57
+ end
58
+
59
+ def migrate_old_settings
60
+ end
61
+ end
@@ -1,5 +1,5 @@
1
1
  module Bosh
2
2
  module Gen
3
- VERSION = "0.16.2"
3
+ VERSION = "0.17.0"
4
4
  end
5
5
  end
@@ -0,0 +1,64 @@
1
+ ---
2
+ name: s3test
3
+ director_uuid: UUID
4
+ release:
5
+ name: myrelease
6
+ version: latest
7
+ compilation:
8
+ workers: 10
9
+ network: default
10
+ reuse_compilation_vms: true
11
+ cloud_properties:
12
+ instance_type: m1.small
13
+ update:
14
+ canaries: 1
15
+ canary_watch_time: 30000
16
+ update_watch_time: 30000
17
+ max_in_flight: 4
18
+ max_errors: 1
19
+ networks:
20
+ - name: default
21
+ type: dynamic
22
+ cloud_properties:
23
+ security_groups:
24
+ - default
25
+ - name: vip_network
26
+ type: vip
27
+ cloud_properties:
28
+ security_groups:
29
+ - default
30
+ resource_pools:
31
+ - name: common
32
+ network: default
33
+ size: 2
34
+ stemcell:
35
+ name: bosh-stemcell
36
+ version: 0.6.4
37
+ cloud_properties:
38
+ instance_type: m1.small
39
+ persistent_disk: 8196
40
+ jobs:
41
+ - name: job-with-ips
42
+ template: job-with-ips
43
+ instances: 1
44
+ resource_pool: common
45
+ networks:
46
+ - name: default
47
+ default:
48
+ - dns
49
+ - gateway
50
+ - name: vip_network
51
+ static_ips:
52
+ - 4.3.2.1
53
+ persistent_disk: 8196
54
+ - name: misc
55
+ template: misc
56
+ instances: 1
57
+ resource_pool: common
58
+ networks:
59
+ - name: default
60
+ default:
61
+ - dns
62
+ - gateway
63
+ persistent_disk: 8196
64
+ properties: {}
@@ -0,0 +1,18 @@
1
+ config/dev.yml
2
+ config/private.yml
3
+ releases/*.tgz
4
+ dev_releases
5
+ blobs/*
6
+ .blobs
7
+ .dev_builds
8
+ .vagrant
9
+ .idea
10
+ .DS_Store
11
+ .final_builds/jobs/**/*.tgz
12
+ .final_builds/packages/**/*.tgz
13
+ *.swp
14
+ *~
15
+ *#
16
+ #*
17
+ tmp
18
+ my*.yml
@@ -0,0 +1,51 @@
1
+ # BOSH Release for s3test
2
+
3
+ ## Usage
4
+
5
+ To use this bosh release, first upload it to your bosh:
6
+
7
+ ```
8
+ bosh target BOSH_HOST
9
+ git clone https://github.com/cloudfoundry-community/s3test-boshrelease.git
10
+ cd s3test-boshrelease
11
+ bosh upload release releases/s3test-1.yml
12
+ ```
13
+
14
+ For [bosh-lite](https://github.com/cloudfoundry/bosh-lite), you can quickly create a deployment manifest & deploy a cluster:
15
+
16
+ ```
17
+ templates/make_manifest warden
18
+ bosh -n deploy
19
+ ```
20
+
21
+ For AWS EC2, create a single VM:
22
+
23
+ ```
24
+ templates/make_manifest aws-ec2
25
+ bosh -n deploy
26
+ ```
27
+
28
+ ### Override security groups
29
+
30
+ For AWS & Openstack, the default deployment assumes there is a `default` security group. If you wish to use a different security group(s) then you can pass in additional configuration when running `make_manifest` above.
31
+
32
+ Create a file `my-networking.yml`:
33
+
34
+ ``` yaml
35
+ ---
36
+ networks:
37
+ - name: s3test1
38
+ type: dynamic
39
+ cloud_properties:
40
+ security_groups:
41
+ - s3test
42
+ ```
43
+
44
+ Where `- s3test` means you wish to use an existing security group called `s3test`.
45
+
46
+ You now suffix this file path to the `make_manifest` command:
47
+
48
+ ```
49
+ templates/make_manifest openstack-nova my-networking.yml
50
+ bosh -n deploy
51
+ ```
@@ -0,0 +1,15 @@
1
+ desc "Generates a properties file for each job based on properties.X.Y used in templates"
2
+ task :job_properties do
3
+ require "fileutils"
4
+ Dir["jobs/*"].each do |path|
5
+ puts "Searching job #{File.basename(path)}..."
6
+ FileUtils.chdir(path) do
7
+ properties = []
8
+ Dir["templates/*.erb"].each do |template_path|
9
+ properties |= File.read(template_path).scan(/\bproperties\.[\w\.]*\b/)
10
+ puts properties.join("\n")
11
+ File.open("properties", "w") { |file| file << properties.join("\n") }
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,5 @@
1
+ ---
2
+ blobstore:
3
+ provider: local
4
+ options:
5
+ blobstore_path: /tmp/blobstore
@@ -0,0 +1,5 @@
1
+ check process s3test
2
+ with pidfile /var/vcap/sys/run/s3test/s3test.pid
3
+ start program "/var/vcap/jobs/s3test/bin/monit_debugger s3test_ctl '/var/vcap/jobs/s3test/bin/s3test_ctl start'"
4
+ stop program "/var/vcap/jobs/s3test/bin/monit_debugger s3test_ctl '/var/vcap/jobs/s3test/bin/s3test_ctl stop'"
5
+ group vcap
@@ -0,0 +1,13 @@
1
+ ---
2
+ name: s3test
3
+ packages: []
4
+ templates:
5
+ bin/monit_debugger: bin/monit_debugger
6
+ bin/s3test_ctl: bin/s3test_ctl
7
+ config/s3test.conf.erb: config/s3test.conf
8
+ data/properties.sh.erb: data/properties.sh
9
+ helpers/ctl_setup.sh: helpers/ctl_setup.sh
10
+ helpers/ctl_utils.sh: helpers/ctl_utils.sh
11
+ properties:
12
+ s3test.leader_address:
13
+ description: Hostname/IP to the leader/master of the cluster
@@ -0,0 +1,13 @@
1
+ #!/bin/sh
2
+ # USAGE monit_debugger <label> command to run
3
+ mkdir -p /var/vcap/sys/log/monit
4
+ {
5
+ echo "MONIT-DEBUG date"
6
+ date
7
+ echo "MONIT-DEBUG env"
8
+ env
9
+ echo "MONIT-DEBUG $@"
10
+ $2 $3 $4 $5 $6 $7
11
+ R=$?
12
+ echo "MONIT-DEBUG exit code $R"
13
+ } >/var/vcap/sys/log/monit/monit_debugger.$1.log 2>&1