kite 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1350153f9c5242c19356d1c6797de0d7897b990f
-  data.tar.gz: 9036d77e615d36dba63fc56a1336e5888de7c757
+  metadata.gz: b20f3f85398a1b4a4b23cb1cb818a0f29b489617
+  data.tar.gz: 8b2fa9d43523bfbb839ddd655ee3565cc028bfc1
 SHA512:
-  metadata.gz: 22c58c7d7b01bcc7affe6bc0bac131b231ebd7bb49b56e298f5d91a6d66a273d8e4783aa5db14f6b2bf67e63d41e95aa4f9ae4ef151e253779ff757ae3588491
-  data.tar.gz: fbe6f97793d40e992194dec5f03b4f1d98624c1be3dd78a2bd6acbbe4e54a2d32ce45acc97552da3ad2f9ff898a0ce1893fedaae6c20d5724c5af04cc6efb51d
+  metadata.gz: '049614c95440425c9142eb0b25423e898480e151a648f6cf4b7f55c7d74f51ff406a25211e74fe449693b2e90fd00721c46bf609d3dd0f4417e804bc53180743'
+  data.tar.gz: d039515fb358e058c1ac49da66943cc24a9496183581f144ef7df4b5163f9f0df52d2f5dc582722031fbe226d8d947ad507411c6f7342d5060265fbe31b3996b

data/README.md

@@ -23,7 +23,15 @@ Or install it yourself as:
 
 ## Usage
 
-Kite is a command line tool, documentation is embedded
+To start using kite from scratch:
+- Create a new kite project, use `kite new`
+- Fill out the `config/cloud.yml` file with your credentials.
+- For BOSH you'll need an SSH key, to generate one, use `ssh-keygen -f *path_to_key*`
+- Generate the cloud IaC needed with `kite generate --cloud=*aws or gcp*`
+- Continue with instructions from newly generated README.md
+
+
+To list all Kite commands, use
 
 ```shell
 $> kite help

data/lib/kite.rb

@@ -4,6 +4,7 @@ require 'thor'
 
 require 'kite/version'
 require 'kite/helpers'
+require 'kite/error'
 
 require 'kite/base'
 require 'kite/core'

data/lib/kite/core.rb

@@ -17,7 +17,7 @@ module Kite
     desc "generate", "Generate Cloud IaC from configuration"
     def generate()
       say "Generating Cloud #{ options[:cloud] } IaC", :green
-      @values = YAML.load(File.read('config/cloud.yml'))
+      @values = parse_cloud_config
 
       case options[:cloud]
       when 'aws'
@@ -26,9 +26,12 @@ module Kite
         copy_file('aws/terraform/outputs.tf',              'terraform/outputs.tf')
         copy_file('aws/terraform/variables.tf',            'terraform/variables.tf')
         template('aws/terraform/terraform.tfvars.erb',     'terraform/terraform.tfvars')
-
         copy_file('aws/README.md',                         'README.md')
-        copy_file('aws/bootstrap.sh',                      'bootstrap.sh')
+
+        template('aws/bosh-install.sh.erb',                 'bin/bosh-install.sh')
+        template('aws/setup-tunnel.sh.erb',                 'bin/setup-tunnel.sh')
+        chmod('bin/bosh-install.sh', 0755)
+        chmod('bin/setup-tunnel.sh', 0755)
 
       when 'gcp'
         copy_file('gcp/terraform/main.tf',                  'terraform/main.tf')
@@ -36,8 +39,13 @@ module Kite
         copy_file('gcp/terraform/outputs.tf',               'terraform/outputs.tf')
         copy_file('gcp/terraform/variables.tf',             'terraform/variables.tf')
         template('gcp/terraform/terraform.tfvars.erb',      'terraform/terraform.tfvars')
+        copy_file('gcp/README.md',                          'README.md', force: true)
+
         template('gcp/bosh-install.sh.erb',                 'bin/bosh-install.sh')
+        template('gcp/bosh-vars.yml.erb',                   'bosh-vars.yml')
+        template('gcp/setup-tunnel.sh.erb',                 'bin/setup-tunnel.sh')
         chmod('bin/bosh-install.sh', 0755)
+        chmod('bin/setup-tunnel.sh', 0755)
 
       else
         say 'Cloud provider not specified'
@@ -45,15 +53,17 @@ module Kite
       end
     end
 
+    method_option :cloud, type: :string, desc: "Cloud provider", enum: %w{aws gcp}, required: true
     desc 'render MANIFEST', 'Render manifest file from configuration and Terraform output'
     def render(manifest)
       say "Rendering #{ manifest } manifest", :green
-      @values = YAML.load(File.read('config/cloud.yml'))
+      @values = parse_cloud_config
       @tf_output = parse_tf_state('terraform/terraform.tfstate')
 
       case manifest
       when "bosh"
-        template("aws/bosh/bosh_director.yml.erb",    "bosh_director.yml")
+        cloud = options[:cloud]
+        directory("#{cloud}/deployments", 'deployments')
 
       when "concourse"
         template("aws/concourse/aws_cloud.yml.erb",   "aws_cloud.yml")

data/lib/kite/error.rb

@@ -0,0 +1,2 @@
+class Kite::Error < Thor::Error
+end

data/lib/kite/helpers.rb

@@ -1,9 +1,31 @@
 module Kite::Helpers
+  # Check config/cloud.yml file to be complete
+  def check_cloud_config(config)
+    raise Kite::Error, 'The config/cloud.yml is not filled out!' unless config.find { |key, hash| hash.find { |k, v| v.nil? } }.nil?
+  end
+
+  # Check if Terraform IaC was applied
+  def check_terraform_applied
+    raise Kite::Error, 'Did you terraform apply? terraform.tfstate is missing!' unless File.file? "terraform/terraform.tfstate"
+  end
+
   # Parse Terraform .tfstate file, returning the output hash
   def parse_tf_state(path)
-    tf_state = YAML.load(File.open(path))
+    check_terraform_applied
+
+    tf_state = YAML.load(File.read(path))
     tf_output = tf_state["modules"].first["outputs"]
     tf_output.map { |k, v| tf_output[k] = v["value"] }
+
     tf_output
   end
+
+  # Parse config/cloud.yml, returning the output hash
+  def parse_cloud_config
+    cloud_config = YAML.load(File.read('config/cloud.yml'))
+    check_cloud_config(cloud_config)
+
+    cloud_config
+  end
+
 end

data/lib/kite/version.rb

@@ -1,3 +1,3 @@
 module Kite
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/tpl/aws/README.md

@@ -13,91 +13,17 @@ Requirements
 -----
 
 - Install [terraform](https://www.terraform.io/intro/getting-started/install.html)
-- Install [bosh-init](https://bosh.io/docs/install-bosh-init.html)
-- Install the [bosh_cli](https://bosh.io/docs/bosh-cli.html)
-
-Ensure you have created a `terraform/terraform.tfvars` file with your variables, or set suitable [environment variables](https://www.terraform.io/docs/configuration/variables.html). An example tfvars file can be found in `terraform/terraform.tfvars.example`
-
-Assumptions
------
-
-You already have:
-
-- A Route53 Zone in AWS.
-- An EC2 SSH keypair
-- An SSL certificate in AWS for your Concourse ELB
+- Install [bosh](https://bosh.io/docs/cli-v2.html#install)
 
 Usage
 -----
 
-Set your desired AWS region in `terrform/variables.tf`. Ensure terraform is in your path, then apply the configuration to prepare the IaaS for BOSH and Concourse:
-
-```
-cd terraform/
-terraform apply
-```
-Set the following environment variables:
-
-```
-$AWS_ACCESS_KEY_ID
-$AWS_SECRET_ACCESS_KEY
-$AWS_REGION
-$AWS_AZ
-$BOSH_PASSWORD
-$AWS_KEYPAIR_KEY_NAME
-$PRIVATE_KEY_PATH
-```
-
-Then create the `bosh-director.yml` manifest:
-```
-./bin/make_manifest_bosh-init.sh
-```
-
-You are ready to deploy the BOSH Director
-```
-bosh-init deploy bosh-director.yml
-```
-
-Go and make a cup of tea.
-
-Once the director is deployed, target it and apply your cloud-config for AWS.
-Remember to set your chosen AZ and the subnet-id output by terraform in `aws-cloud.yml`.
-
-```
-bosh target <your EIP address>
-bosh update cloud-config aws-cloud.yml
-```
-
-Set a database password and external URL for your deployment in these environment variables:
-
-```
-$DB_PASSWORD
-$CONCOURSE_URL
-```
-Create a new OAuth application in GitHub as described [here](http://concourse.ci/authentication.html). The manifest assumes the existance of a 'CI' team that contains your authorised users, so create that too. Then set the following environment variables:
-
-```
-$GITHUB_ORG
-$GITHUB_CLIENT_ID
-$GITHUB_CLIENT_SECRET
-```
-
-Then create a concourse manifest for a single server deployment:
-```
-./bin/make_manifest_concourse.sh
-```
-Or, create a concourse manifest for small cluster:
-```
-./bin/make_manifest_concourse-cluster.sh
-```
+To deploy a BOSH Director:
+- Apply the terraform IaC from `terraform` folder
+- Run `bin/setup_tunnel.sh` to create an SSH CLI tunnel
+- Run `kite render bosh --cloud aws` to render BOSH deployment files
+- Run `bin/bosh_setup.sh` to deploy the BOSH Director
 
-Upload the necessary stemcell & releases, then deploy concourse:
-```
-bosh upload stemcell https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent
-bosh upload release https://bosh.io/d/github.com/concourse/concourse
-bosh upload release https://bosh.io/d/github.com/cloudfoundry-incubator/garden-runc-release
-bosh deployment concourse.yml
-bosh deploy
-```
+To access BOSH Director information, use bosh -e *bosh_name* env
 
-Congratulations, you should now be able to see your new CI server at https://your-concourse-url.
+To connect to Bastion over SSH, use ssh jumpbox@*bastion ip* -i jumpbox.key

data/tpl/aws/bosh-install.sh.erb

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Create a new BOSH environment with Director
+bosh create-env deployments/bosh/bosh_director.yml \
+  --state=state.json \
+  --vars-store=creds.yml \
+  --vars-file=bosh_vars.yml \
+  --var-file private_key=<%= @values['kite']['private_key_path'] %> \
+  -o deployments/bosh/cpi.yml \
+  -o deployments/bosh/jumpbox-user.yml
+
+# Configure alias for the new environment
+bosh alias-env <%= @values['bosh']['name'] %> \
+  -e <%= @values['bosh']['static_ip'] %>  \
+  --ca-cert <(bosh int ./creds.yml --path /director_ssl/ca)
+
+# Get jumpbox user key
+bosh int creds.yml --path /jumpbox_ssh/private_key > jumpbox.key
+chmod 600 jumpbox.key
+
+# Log into the newly created Director
+export BOSH_CLIENT=admin
+export BOSH_CLIENT_SECRET=`bosh int ./creds.yml --path /admin_password`

data/tpl/aws/deployments/bosh/bosh_director.yml

@@ -0,0 +1,144 @@
+---
+name: bosh
+
+releases:
+- name: bosh
+  version: "262.3"
+  url: https://s3.amazonaws.com/bosh-compiled-release-tarballs/bosh-262.3-ubuntu-trusty-3421.9-20170706-183731-831697577-20170706183736.tgz?versionId=7GmwKfufgb5JwWhJ.cwIWLnejOtm2Hu4
+  sha1: 1eae3f06282417e54ebb199656458f9d6c38e2af
+
+resource_pools:
+- name: vms
+  network: default
+  env:
+    bosh:
+      password: '*'
+      mbus:
+        cert: ((mbus_bootstrap_ssl))
+
+disk_pools:
+- name: disks
+  disk_size: 32_768
+
+networks:
+- name: default
+  type: manual
+  subnets:
+  - range: 10.0.0.0/24
+    gateway: 10.0.0.1
+    static: [10.0.0.2]
+    dns: [8.8.8.8]
+
+instance_groups:
+- name: bosh
+  instances: 1
+  jobs:
+  - {name: nats, release: bosh}
+  - {name: postgres-9.4, release: bosh}
+  - {name: blobstore, release: bosh}
+  - {name: director, release: bosh}
+  - {name: health_monitor, release: bosh}
+  resource_pool: vms
+  persistent_disk_pool: disks
+  networks:
+  - name: default
+    static_ips: [((internal_ip))]
+  properties:
+    nats:
+      address: 127.0.0.1
+      user: nats
+      password: ((nats_password))
+    postgres: &db
+      listen_address: 127.0.0.1
+      host: 127.0.0.1
+      user: postgres
+      password: ((postgres_password))
+      database: bosh
+      adapter: postgres
+    blobstore:
+      address: 10.0.0.2
+      port: 25250
+      provider: dav
+      director:
+        user: director
+        password: ((blobstore_director_password))
+      agent:
+        user: agent
+        password: ((blobstore_agent_password))
+    director:
+      address: 127.0.0.1
+      name: ((director_name))
+      db: *db
+      flush_arp: true
+      enable_post_deploy: true
+      generate_vm_passwords: true
+      enable_dedicated_status_worker: true
+      enable_nats_delivered_templates: true
+      workers: 4
+      events:
+        record_events: true
+      ssl:
+        key: ((director_ssl.private_key))
+        cert: ((director_ssl.certificate))
+      user_management:
+        provider: local
+        local:
+          users:
+          - name: admin
+            password: ((admin_password))
+          - name: hm
+            password: ((hm_password))
+    hm:
+      director_account:
+        user: hm
+        password: ((hm_password))
+        ca_cert: ((director_ssl.ca))
+      resurrector_enabled: true
+    ntp: &ntp
+    - time1.google.com
+    - time2.google.com
+    - time3.google.com
+    - time4.google.com
+    agent:
+      mbus: nats://nats:((nats_password))@((internal_ip)):4222
+
+cloud_provider:
+  mbus: https://mbus:((mbus_bootstrap_password))@((internal_ip)):6868
+  cert: ((mbus_bootstrap_ssl))
+  properties:
+    agent: {mbus: "https://mbus:((mbus_bootstrap_password))@0.0.0.0:6868"}
+    blobstore: {provider: local, path: /var/vcap/micro_bosh/data/cache}
+    ntp: *ntp
+
+variables:
+- name: admin_password
+  type: password
+- name: blobstore_director_password
+  type: password
+- name: blobstore_agent_password
+  type: password
+- name: hm_password
+  type: password
+- name: mbus_bootstrap_password
+  type: password
+- name: nats_password
+  type: password
+- name: postgres_password
+  type: password
+- name: default_ca
+  type: certificate
+  options:
+    is_ca: true
+    common_name: ca
+- name: mbus_bootstrap_ssl
+  type: certificate
+  options:
+    ca: default_ca
+    common_name: ((internal_ip))
+    alternative_names: [((internal_ip))]
+- name: director_ssl
+  type: certificate
+  options:
+    ca: default_ca
+    common_name: ((internal_ip))
+    alternative_names: [((internal_ip))]

data/tpl/aws/deployments/bosh/bosh_vars.yml.erb

@@ -0,0 +1,11 @@
+director_name:             <%= @values['bosh']['name'] %>
+internal_cidr:             <%= @values['aws']['platform_subnet_cidr_block'] %>
+internal_gw:               10.0.0.1
+internal_ip:               <%= @values['bosh']['static_ip'] %>
+access_key_id:             <%= @values['aws']['access_key'] %>
+secret_access_key:         <%= @values['aws']['secret_key'] %>
+region:                    <%= @values['aws']['region'] %>
+az:                        <%= @values['aws']['az'] %>
+default_key_name:          <%= @values['kite']['keypair_name'] %>
+default_security_groups:   [<%= @tf_output['security_group_id'] %>]
+subnet_id:                 <%= @tf_output['platform_subnet_id'] %>

data/tpl/aws/deployments/bosh/cpi.yml

@@ -0,0 +1,98 @@
+---
+- type: replace
+  path: /releases/-
+  value:
+    name: bosh-aws-cpi
+    version: 65
+    url: https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-aws-cpi-release?v=65
+    sha1: 26b3a5c43e6f82594a373309a495660d6db26254
+
+- type: replace
+  path: /resource_pools/name=vms/stemcell?
+  value:
+    url: https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent?v=3421.9
+    sha1: 316a699d44f49d69493b1545d4addd17b78b5840
+
+# Configure AWS sizes
+- type: replace
+  path: /resource_pools/name=vms/cloud_properties?
+  value:
+    instance_type: m4.xlarge
+    ephemeral_disk: {size: 25_000, type: gp2}
+    availability_zone: ((az))
+
+- type: replace
+  path: /disk_pools/name=disks/cloud_properties?
+  value: {type: gp2}
+
+- type: replace
+  path: /networks/name=default/subnets/0/cloud_properties?
+  value: {subnet: ((subnet_id))}
+
+# Enable registry job
+- type: replace
+  path: /instance_groups/name=bosh/jobs/-
+  value:
+    name: registry
+    release: bosh
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/registry?
+  value:
+    address: ((internal_ip))
+    host: ((internal_ip))
+    db: # todo remove
+      host: 127.0.0.1
+      user: postgres
+      password: ((postgres_password))
+      database: bosh
+      adapter: postgres
+    http:
+      user: registry
+      password: ((registry_password))
+      port: 25777
+    username: registry
+    password: ((registry_password))
+    port: 25777
+
+# Add CPI job
+- type: replace
+  path: /instance_groups/name=bosh/jobs/-
+  value: &cpi_job
+    name: aws_cpi
+    release: bosh-aws-cpi
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/director/cpi_job?
+  value: aws_cpi
+
+- type: replace
+  path: /cloud_provider/template?
+  value: *cpi_job
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/aws?
+  value: &aws
+    access_key_id: ((access_key_id))
+    secret_access_key: ((secret_access_key))
+    default_key_name: ((default_key_name))
+    default_security_groups: ((default_security_groups))
+    region: ((region))
+
+- type: replace
+  path: /cloud_provider/ssh_tunnel?
+  value:
+    host: ((internal_ip))
+    port: 22
+    user: vcap
+    private_key: ((private_key))
+
+- type: replace
+  path: /cloud_provider/properties/aws?
+  value: *aws
+
+- type: replace
+  path: /variables/-
+  value:
+    name: registry_password
+    type: password

data/tpl/aws/deployments/bosh/jumpbox-user.yml

@@ -0,0 +1,27 @@
+- type: replace
+  path: /releases/name=os-conf?
+  value:
+    name: os-conf
+    version: 12
+    url: https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=12
+    sha1: af5a2c9f228b9d7ec4bd051d71fef0e712fa1549
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/director/default_ssh_options?/gateway_user
+  value: jumpbox
+
+- type: replace
+  path: /instance_groups/name=bosh/jobs/-
+  value:
+    name: user_add
+    release: os-conf
+    properties:
+      users:
+      - name: jumpbox
+        public_key: ((jumpbox_ssh.public_key))
+
+- type: replace
+  path: /variables/-
+  value:
+    name: jumpbox_ssh
+    type: ssh

data/tpl/aws/setup-tunnel.sh.erb

@@ -0,0 +1,4 @@
+BASTION_IP="$(terraform output -state=terraform/terraform.tfstate bastion_ip)"
+ssh -D 5000 -fNC ubuntu@$BASTION_IP -i <%= @values['kite']['public_key_path'] %>
+
+export BOSH_ALL_PROXY=socks5://localhost:5000

data/tpl/gcp/README.md

@@ -0,0 +1,22 @@
+## GCP Cloud
+
+### Usage
+Apply terraform code
+```
+pushd terraform && terraform init && terraform apply && popd
+```
+
+Render bosh deployment
+```
+kite render bosh --cloud=gcp
+```
+
+Setup tunnel
+```
+./bin/setup-tunnel.sh
+```
+
+Install BOSH
+```
+./bin/bosh-install.sh
+```

data/tpl/gcp/bosh-install.sh.erb

@@ -2,17 +2,11 @@
 
 set -xe
 
-bosh create-env bosh-deployment/bosh.yml \
+bosh create-env deployments/bosh/bosh.yml \
   --state=config/state.json \
   --vars-store=config/creds.yml \
-  -o bosh-deployment/gcp/cpi.yml \
-  -v director_name=bosh-director \
-  -v internal_cidr=<%= @values['gcp']['subnet_cidr'] %> \
-  -v internal_gw=<%= @values['gcp']['internal_gw'] %> \
-  -v internal_ip=<%= @values['bosh']['static_ip'] %> \
+  --vars-file=bosh-vars.yml \
   --var-file gcp_credentials_json=<%= @values['gcp']['service_account'] %> \
-  -v project_id=<%= @values['gcp']['project'] %> \
-  -v zone=<%= @values['gcp']['zone'] %> \
-  -v tags=[platform-internal] \
-  -v network=<%= @values['gcp']['vpc_name'] %> \
-  -v subnetwork=<%= @values['gcp']['subnet_name'] %>
+  -v tags='[platform-internal, no-ip]' \
+  -o deployments/bosh/cpi.yml \
+  -o deployments/bosh/jumpbox-user.yml

data/tpl/gcp/bosh-vars.yml.erb

@@ -0,0 +1,8 @@
+director_name:        <%= @values['bosh']['name'] %>
+internal_cidr:        <%= @values['gcp']['subnet_cidr'] %>
+internal_gw:          <%= @values['gcp']['internal_gw'] %>
+internal_ip:          <%= @values['bosh']['static_ip'] %>
+project_id:           <%= @values['gcp']['project'] %>
+zone:                 <%= @values['gcp']['zone'] %>
+network:              <%= @values['gcp']['vpc_name'] %>
+subnetwork:           <%= @values['gcp']['subnet_name'] %>

data/tpl/gcp/deployments/bosh/bosh.yml

@@ -0,0 +1,144 @@
+---
+name: bosh
+
+releases:
+- name: bosh
+  version: "263"
+  url: https://s3.amazonaws.com/bosh-compiled-release-tarballs/bosh-263-ubuntu-trusty-3445.7-20170901-012146-902840377-20170901012153.tgz?versionId=89a.ZxB3Jc_gl6s4YESlL41xNOfoJKrO
+  sha1: cc71c2ee6992071b1e1f6ae9f2119c03a42521c5
+
+resource_pools:
+- name: vms
+  network: default
+  env:
+    bosh:
+      password: '*'
+      mbus:
+        cert: ((mbus_bootstrap_ssl))
+
+disk_pools:
+- name: disks
+  disk_size: 32_768
+
+networks:
+- name: default
+  type: manual
+  subnets:
+  - range: ((internal_cidr))
+    gateway: ((internal_gw))
+    static: [((internal_ip))]
+    dns: [8.8.8.8]
+
+instance_groups:
+- name: bosh
+  instances: 1
+  jobs:
+  - {name: nats, release: bosh}
+  - {name: postgres-9.4, release: bosh}
+  - {name: blobstore, release: bosh}
+  - {name: director, release: bosh}
+  - {name: health_monitor, release: bosh}
+  resource_pool: vms
+  persistent_disk_pool: disks
+  networks:
+  - name: default
+    static_ips: [((internal_ip))]
+  properties:
+    nats:
+      address: 127.0.0.1
+      user: nats
+      password: ((nats_password))
+    postgres: &db
+      listen_address: 127.0.0.1
+      host: 127.0.0.1
+      user: postgres
+      password: ((postgres_password))
+      database: bosh
+      adapter: postgres
+    blobstore:
+      address: ((internal_ip))
+      port: 25250
+      provider: dav
+      director:
+        user: director
+        password: ((blobstore_director_password))
+      agent:
+        user: agent
+        password: ((blobstore_agent_password))
+    director:
+      address: 127.0.0.1
+      name: ((director_name))
+      db: *db
+      flush_arp: true
+      enable_post_deploy: true
+      generate_vm_passwords: true
+      enable_dedicated_status_worker: true
+      enable_nats_delivered_templates: true
+      workers: 4
+      events:
+        record_events: true
+      ssl:
+        key: ((director_ssl.private_key))
+        cert: ((director_ssl.certificate))
+      user_management:
+        provider: local
+        local:
+          users:
+          - name: admin
+            password: ((admin_password))
+          - name: hm
+            password: ((hm_password))
+    hm:
+      director_account:
+        user: hm
+        password: ((hm_password))
+        ca_cert: ((director_ssl.ca))
+      resurrector_enabled: true
+    ntp: &ntp
+    - time1.google.com
+    - time2.google.com
+    - time3.google.com
+    - time4.google.com
+    agent:
+      mbus: nats://nats:((nats_password))@((internal_ip)):4222
+
+cloud_provider:
+  mbus: https://mbus:((mbus_bootstrap_password))@((internal_ip)):6868
+  cert: ((mbus_bootstrap_ssl))
+  properties:
+    agent: {mbus: "https://mbus:((mbus_bootstrap_password))@0.0.0.0:6868"}
+    blobstore: {provider: local, path: /var/vcap/micro_bosh/data/cache}
+    ntp: *ntp
+
+variables:
+- name: admin_password
+  type: password
+- name: blobstore_director_password
+  type: password
+- name: blobstore_agent_password
+  type: password
+- name: hm_password
+  type: password
+- name: mbus_bootstrap_password
+  type: password
+- name: nats_password
+  type: password
+- name: postgres_password
+  type: password
+- name: default_ca
+  type: certificate
+  options:
+    is_ca: true
+    common_name: ca
+- name: mbus_bootstrap_ssl
+  type: certificate
+  options:
+    ca: default_ca
+    common_name: ((internal_ip))
+    alternative_names: [((internal_ip))]
+- name: director_ssl
+  type: certificate
+  options:
+    ca: default_ca
+    common_name: ((internal_ip))
+    alternative_names: [((internal_ip))]

data/tpl/gcp/deployments/bosh/cloud-config.yml

@@ -0,0 +1,51 @@
+azs:
+- name: z1
+  cloud_properties:
+    zone: ((zone))
+- name: z2
+  cloud_properties:
+    zone: ((zone))
+- name: z3
+  cloud_properties:
+    zone: ((zone))
+
+vm_types:
+- name: default
+  cloud_properties:
+    machine_type: n1-standard-2
+    root_disk_size_gb: 20
+    root_disk_type: pd-ssd
+- name: large
+  cloud_properties:
+    machine_type: n1-standard-2
+    root_disk_size_gb: 50
+    root_disk_type: pd-ssd
+
+disk_types:
+- name: default
+  disk_size: 3000
+- name: large
+  disk_size: 50_000
+
+networks:
+- name: default
+  type: manual
+  subnets:
+  - range: ((internal_cidr))
+    gateway: ((internal_gw))
+    azs: [z1, z2, z3]
+    dns: [8.8.8.8]
+    cloud_properties:
+      network_name: ((network))
+      subnetwork_name: ((subnetwork))
+      ephemeral_external_ip: true
+      tags: ((tags))
+- name: vip
+  type: vip
+
+compilation:
+  workers: 5
+  reuse_compilation_vms: true
+  az: z1
+  vm_type: default
+  network: default

data/tpl/gcp/deployments/bosh/cpi.yml

@@ -0,0 +1,69 @@
+---
+- type: replace
+  path: /releases/-
+  value:
+    name: bosh-google-cpi
+    version: 25.10.0
+    url: https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-google-cpi-release?v=25.10.0
+    sha1: 3a551822bff0fd040d73fd385ab34fbc17b476f5
+
+- type: replace
+  path: /resource_pools/name=vms/stemcell?
+  value:
+    url: https://bosh.io/d/stemcells/bosh-google-kvm-ubuntu-trusty-go_agent?v=3445.7
+    sha1: 4bc264aab6717c81fb3a37783e796982fe9956ca
+
+# Configure sizes
+- type: replace
+  path: /resource_pools/name=vms/cloud_properties?
+  value:
+    zone: ((zone))
+    machine_type: n1-standard-1
+    root_disk_size_gb: 40
+    root_disk_type: pd-standard
+
+- type: replace
+  path: /disk_pools/name=disks/cloud_properties?
+  value: {type: pd-standard}
+
+- type: replace
+  path: /networks/name=default/subnets/0/cloud_properties?
+  value:
+    network_name: ((network))
+    subnetwork_name: ((subnetwork))
+    ephemeral_external_ip: false
+    tags: ((tags))
+
+# Add CPI job
+- type: replace
+  path: /instance_groups/name=bosh/jobs/-
+  value: &cpi_job
+    name: google_cpi
+    release: bosh-google-cpi
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/director/cpi_job?
+  value: google_cpi
+
+- type: replace
+  path: /cloud_provider/template?
+  value: *cpi_job
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/google?
+  value: &cpi_conf
+    project: ((project_id))
+    json_key: ((gcp_credentials_json))
+
+- type: replace
+  path: /cloud_provider/properties/google?
+  value: *cpi_conf
+
+# Use GCP NTP
+- type: replace
+  path: /instance_groups/name=bosh/properties/ntp
+  value: &ntp [169.254.169.254]
+
+- type: replace
+  path: /cloud_provider/properties/ntp
+  value: *ntp

data/tpl/gcp/deployments/bosh/jumpbox-user.yml

@@ -0,0 +1,27 @@
+- type: replace
+  path: /releases/name=os-conf?
+  value:
+    name: os-conf
+    version: 12
+    url: https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=12
+    sha1: af5a2c9f228b9d7ec4bd051d71fef0e712fa1549
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/director/default_ssh_options?/gateway_user
+  value: jumpbox
+
+- type: replace
+  path: /instance_groups/name=bosh/jobs/-
+  value:
+    name: user_add
+    release: os-conf
+    properties:
+      users:
+      - name: jumpbox
+        public_key: ((jumpbox_ssh.public_key))
+
+- type: replace
+  path: /variables/-
+  value:
+    name: jumpbox_ssh
+    type: ssh

data/tpl/gcp/setup-tunnel.sh.erb

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+BASTION_IP="$(terraform output -state=terraform/terraform.tfstate bastion_ip)"
+ssh -D 5000 -fNC kite@$BASTION_IP -i <%= @values['kite']['public_key_path'] %>
+
+export BOSH_ALL_PROXY=socks5://localhost:5000

data/tpl/gcp/terraform/main.tf

@@ -39,10 +39,6 @@ resource "google_compute_instance" "bastion" {
     }
   }
 
-  metadata {
-    sshKeys = "kite:${file(var.public_key)}"
-  }
-
   network_interface {
     subnetwork = "${google_compute_subnetwork.platform_net.name}"
     access_config {
@@ -50,6 +46,18 @@ resource "google_compute_instance" "bastion" {
     }
   }
 
+  can_ip_forward = true
+
+  metadata {
+    sshKeys = "kite:${file(var.public_key)}"
+  }
+
+  metadata_startup_script = <<EOT
+#!/bin/bash
+sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
+iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+EOT
+
   service_account {
     scopes = ["userinfo-email", "compute-ro", "storage-ro"]
   }

data/tpl/gcp/terraform/network.tf

@@ -9,6 +9,17 @@ resource "google_compute_subnetwork" "platform_net" {
   network       = "${google_compute_network.platform.self_link}"
 }
 
+resource "google_compute_route" "platform-gate" {
+  name                   = "platform-gate"
+  dest_range             = "0.0.0.0/0"
+  network                = "${google_compute_network.platform.name}"
+  next_hop_instance      = "${google_compute_instance.bastion.name}"
+  next_hop_instance_zone = "${var.zone}"
+  priority               = 800
+  tags                   = ["no-ip"]
+  project                = "${var.project}"
+}
+
 # Allow open access between internal VM
 resource "google_compute_firewall" "platform_internal" {
   name    = "platform-internal"

data/tpl/skel/README.md.tt

@@ -1 +1,4 @@
-# <%=@cloud_name %>
+## Generate a new cloud
+```
+kite generate --cloud=<CLOUD_NAME>
+```

data/tpl/skel/config/cloud.yml

@@ -16,7 +16,7 @@ aws:
   ops_subnet_name: "ops_services"
 
 gcp:
-  project_id: gcp-project
+  project: gcp-project
   region: europe-west1
   zone: europe-west1-b
   service_account: "~/safe/terraform.json"
@@ -26,6 +26,7 @@ gcp:
   internal_gw: "10.0.0.1"
 
 bosh:
+  name: "bosh-director"
   static_ip: "10.0.0.10"
 
 concourse:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kite
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Louis Bellet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-01 00:00:00.000000000 Z
+date: 2017-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -90,19 +90,31 @@ files:
 - lib/kite/base.rb
 - lib/kite/cloud.rb
 - lib/kite/core.rb
+- lib/kite/error.rb
 - lib/kite/helpers.rb
 - lib/kite/version.rb
 - tpl/aws/README.md
-- tpl/aws/bootstrap.sh
-- tpl/aws/bosh/bosh_director.yml.erb
+- tpl/aws/bosh-install.sh.erb
 - tpl/aws/concourse/aws_cloud.yml.erb
 - tpl/aws/concourse/concourse.yml.erb
+- tpl/aws/deployments/bosh/bosh_director.yml
+- tpl/aws/deployments/bosh/bosh_vars.yml.erb
+- tpl/aws/deployments/bosh/cpi.yml
+- tpl/aws/deployments/bosh/jumpbox-user.yml
+- tpl/aws/setup-tunnel.sh.erb
 - tpl/aws/terraform/main.tf
 - tpl/aws/terraform/network.tf
 - tpl/aws/terraform/outputs.tf
 - tpl/aws/terraform/terraform.tfvars.erb
 - tpl/aws/terraform/variables.tf
+- tpl/gcp/README.md
 - tpl/gcp/bosh-install.sh.erb
+- tpl/gcp/bosh-vars.yml.erb
+- tpl/gcp/deployments/bosh/bosh.yml
+- tpl/gcp/deployments/bosh/cloud-config.yml
+- tpl/gcp/deployments/bosh/cpi.yml
+- tpl/gcp/deployments/bosh/jumpbox-user.yml
+- tpl/gcp/setup-tunnel.sh.erb
 - tpl/gcp/terraform/main.tf
 - tpl/gcp/terraform/network.tf
 - tpl/gcp/terraform/outputs.tf

data/tpl/aws/bootstrap.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-
-pushd terraform && terraform apply && popd
-
-kite render-manifest --manifest=bosh
-bosh-init deploy bosh_director.yml
-
-pushd terraform && BOSH_DIRECTOR_IP=$(terraform output eip) && popd
-bosh target $BOSH_DIRECTOR_IP
-
-kite render-manifest --manifest=concourse
-bosh update cloud-config aws_cloud.yml
-
-bosh upload stemcell https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent
-bosh upload release https://bosh.io/d/github.com/concourse/concourse
-bosh upload release https://bosh.io/d/github.com/cloudfoundry-incubator/garden-runc-release
-
-bosh deployment concourse.yml
-
-bosh deploy

data/tpl/aws/bosh/bosh_director.yml.erb

@@ -1,133 +0,0 @@
----
-name: bosh
-
-releases:
-- name: bosh
-  url: https://bosh.io/d/github.com/cloudfoundry/bosh?v=256.2
-  sha1: ff2f4e16e02f66b31c595196052a809100cfd5a8
-- name: bosh-aws-cpi
-  url: https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-aws-cpi-release?v=52
-  sha1: dc4a0cca3b33dce291e4fbeb9e9948b6a7be3324
-
-resource_pools:
-- name: vms
-  network: private
-  stemcell:
-    url: https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent?v=3232.3
-    sha1: 1fe87c0146ad1f3b55eeed5a80ce35c01b4eb6d9
-  cloud_properties:
-    instance_type: m3.large
-    ephemeral_disk: {size: 25_000, type: gp2}
-    availability_zone: <%= @values['aws']['az'] %>
-
-disk_pools:
-- name: disks
-  disk_size: 20_000
-  cloud_properties: {type: gp2}
-
-networks:
-- name: private
-  type: manual
-  subnets:
-  - range: 10.0.0.0/24
-    gateway: 10.0.0.1
-    dns: [10.0.0.2]
-    cloud_properties: {subnet: <%= @tf_output['default_subnet_id'] %>}
-
-jobs:
-- name: bosh
-  instances: 1
-
-  templates:
-  - {name: nats, release: bosh}
-  - {name: postgres, release: bosh}
-  - {name: blobstore, release: bosh}
-  - {name: director, release: bosh}
-  - {name: health_monitor, release: bosh}
-  - {name: registry, release: bosh}
-  - {name: aws_cpi, release: bosh-aws-cpi}
-
-  resource_pool: vms
-  persistent_disk_pool: disks
-
-  networks:
-  - name: private
-    static_ips: [10.0.0.6]
-    default: [dns, gateway]
-  - name: public
-    static_ips: [<%= @tf_output['eip'] %>]
-
-  properties:
-    nats:
-      address: 127.0.0.1
-      user: nats
-      password: <%= @values['bosh']['password'] %>
-
-    postgres: &db
-      listen_address: 127.0.0.1
-      host: 127.0.0.1
-      user: postgres
-      password: <%= @values['bosh']['password'] %>
-      database: bosh
-      adapter: postgres
-
-    registry:
-      address: 10.0.0.6
-      host: 10.0.0.6
-      db: *db
-      http: {user: admin, password: <%= @values['bosh']['password'] %>, port: 25777}
-      username: admin
-      password: <%= @values['bosh']['password'] %>
-      port: 25777
-
-    blobstore:
-      address: 10.0.0.6
-      port: 25250
-      provider: dav
-      director: {user: director, password: <%= @values['bosh']['password'] %>}
-      agent: {user: agent, password: <%= @values['bosh']['password'] %>}
-
-    director:
-      address: 127.0.0.1
-      name: eb-bosh
-      db: *db
-      cpi_job: aws_cpi
-      max_threads: 10
-      user_management:
-        provider: local
-        local:
-          users:
-          - {name: admin, password: <%= @values['bosh']['password'] %>}
-          - {name: hm, password: <%= @values['bosh']['password'] %>}
-
-    hm:
-      director_account: {user: hm, password: <%= @values['bosh']['password'] %>}
-      resurrector_enabled: true
-
-    aws: &aws
-      access_key_id: <%= @values['aws']['access_key'] %>
-      secret_access_key: <%= @values['aws']['secret_key'] %>
-      default_key_name: <%= @values['bosh']['keypair_name'] %>
-      default_security_groups: [<%= @tf_output['security_group_id'] %>]
-      region: <%= @values['aws']['region'] %>
-
-    agent: {mbus: "nats://nats:<%= @values['bosh']['password'] %>@10.0.0.6:4222"}
-
-    ntp: &ntp [0.pool.ntp.org, 1.pool.ntp.org]
-
-cloud_provider:
-  template: {name: aws_cpi, release: bosh-aws-cpi}
-
-  ssh_tunnel:
-    host: <%= @tf_output['eip'] %> # <--- Replace with your Elastic IP address
-    port: 22
-    user: vcap
-    private_key: <%= @values['bosh']['private_key_path'] %> # Path relative to this manifest file
-
-  mbus: "https://mbus:<%= @values['bosh']['password'] %>@<%= @tf_output['eip'] %>:6868" # <--- Replace with Elastic IP
-
-  properties:
-    aws: *aws
-    agent: {mbus: "https://mbus:<%= @values['bosh']['password'] %>@0.0.0.0:6868"}
-    blobstore: {provider: local, path: /var/vcap/micro_bosh/data/cache}
-    ntp: *ntp