kite 0.1.0 → 0.2.0

data/tpl/gcp/bin/base/setup-tunnel.sh.tt

@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
+
 pushd terraform
 BASTION_IP="$(terraform output bastion_ip)"
 popd
 
-ssh -D 5000 -fNC kite@$BASTION_IP -i <%= @values['kite']['private_key_path'] %>
-
-export BOSH_ALL_PROXY=socks5://localhost:5000
+if [[ -z "${BASTION_IP}" ]]; then
+  echo "Something goes wrong, please check terraform environement" 1>&2
+  false
+else
+  ssh -D 5000 -fNC kite@${BASTION_IP} -i <%= @values['kite']['private_key_path'] %>
+  export BOSH_ALL_PROXY=socks5://localhost:5000
+fi

data/tpl/gcp/bin/bosh-install.sh.tt

@@ -16,3 +16,7 @@ bosh alias-env <%= @values['bosh']['name'] %> -e <%= @values['bosh']['static_ip'
 
 echo "Please run"
 echo bosh -e <%= @values['bosh']['name'] %> ucc deployments/bosh/cloud-config.yml
+
+# Get jumpbox user key
+bosh int config/creds.yml --path /jumpbox_ssh/private_key > config/jumpbox.key
+chmod 600 config/jumpbox.key

data/tpl/gcp/bin/concourse-deploy.sh.tt

@@ -8,4 +8,7 @@ bosh -e <%= @values['bosh']['name'] %> upload-release https://github.com/concour
 bosh -e <%= @values['bosh']['name'] %> upload-release https://github.com/concourse/concourse/releases/download/v3.4.1/garden-runc-1.6.0.tgz
 
 # Deploy Concourse
-bosh -e <%= @values['bosh']['name'] %> -d concourse deploy deployments/concourse/concourse.yml -v auth_password=$1  -v db_password=$2 -v vault_token=$3
+bosh -e <%= @values['bosh']['name'] %> -d concourse \
+  deploy deployments/concourse/concourse.yml \
+  -v vault_token=$1 \
+  --vars-store config/creds.yml && ./bin/ingress-update.sh

data/tpl/gcp/bin/ingress-deploy.sh.tt

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -xe
+
+bosh -e <%= @values['bosh']['name'] %> upload-release https://github.com/cloudfoundry-community/nginx-release/releases/download/v1.12.1/nginx-1.12.1.tgz
+
+bosh -e <%= @values['bosh']['name'] %> -d ingress deploy deployments/ingress/ingress.yml

data/tpl/gcp/bin/ingress-update.sh.tt

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -xe
+
+kite render manifest ingress --cloud gcp
+
+bosh -e <%= @values['bosh']['name'] %> -d ingress deploy deployments/ingress/ingress.yml

data/tpl/gcp/bin/oauth-deploy.sh.tt

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Upload necessary stemcells and releases
+bosh -e <%= @values['bosh']['name'] %> upload-stemcell \
+  https://bosh.io/d/stemcells/bosh-google-kvm-ubuntu-trusty-go_agent \
+  --skip-if-exists
+
+bosh -e <%= @values['bosh']['name'] %> upload-release \
+  https://bosh.io/d/github.com/cloudfoundry/uaa-release
+
+# Deploy Concourse
+bosh -e <%= @values['bosh']['name'] %> -d oauth deploy \
+  deployments/oauth/oauth.yml \
+  --vars-store config/creds.yml \
+  -o config/oauth.yml \
+  && ./bin/ingress-update.sh
+

data/tpl/gcp/bin/prometheus-deploy.sh.tt

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Upload necessary stemcells and releases
+bosh -e <%= @values['bosh']['name'] %> upload-stemcell https://s3.amazonaws.com/bosh-core-stemcells/google/bosh-stemcell-3445.7-google-kvm-ubuntu-trusty-go_agent.tgz
+
+# Extract BOSH Director's SSL certificate
+bosh int ./config/creds.yml --path /director_ssl/certificate > bosh.ca
+
+# Deploy Prometheus
+bosh -e <%= @values['bosh']['name'] %> -d prometheus \
+  deploy deployments/prometheus/prometheus.yml \
+  --vars-store config/creds.yml \
+  -o deployments/prometheus/monitor-kubernetes.yml \
+  --vars-file config/bosh-vars.yml \
+  --var-file kubernetes_kubeconfig=<%= @values['k8s']['config_path'] %> \
+  -o deployments/prometheus/monitor-bosh.yml \
+  -v bosh_url=<%= @values['bosh']['static_ip'] %> \
+  -v bosh_username=admin \
+  -v bosh_password=`bosh int ./config/creds.yml --path /admin_password` \
+  --var-file bosh_ca_cert=bosh.ca \
+  -v metrics_environment=kite && ingress-update.sh

data/tpl/gcp/bin/vault-deploy.sh.tt

@@ -7,4 +7,4 @@ bosh -e <%= @values['bosh']['name'] %> upload-stemcell https://s3.amazonaws.com/
 bosh -e <%= @values['bosh']['name'] %> upload-release https://bosh.io/d/github.com/cloudfoundry-community/vault-boshrelease
 
 # Deploy Concourse
-bosh -e <%= @values['bosh']['name'] %> -d vault deploy deployments/vault/vault.yml
+bosh -e <%= @values['bosh']['name'] %> -d vault deploy deployments/vault/vault.yml && ./bin/ingress-update.sh

data/tpl/gcp/bosh-vars.yml.erb

@@ -6,3 +6,4 @@ project_id:           <%= @values['gcp']['project'] %>
 zone:                 <%= @values['gcp']['zone'] %>
 network:              <%= @values['gcp']['vpc_name'] %>
 subnetwork:           <%= @values['gcp']['subnet_name'] %>
+kubernetes_apiserver: <%= @values['k8s']['api_server_address'] %>

data/tpl/gcp/config/oauth.yml

@@ -0,0 +1,59 @@
+# Groups
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/scim?/groups?
+  value: { }
+    # sysops: For automation stuff
+
+# Users
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/scim?/users?
+  value: [ ]
+    # - email: sysops@example.com
+    #   password: changeme
+    #   name: sysops
+    #   firstName: Sys
+    #   lastName: Ops
+    #   origin: uaa
+    #   groups:
+    #     - sysops
+
+# OAuth clients
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/clients?
+  value: { }
+    # some-app:
+    #   app-icon: 0000000000000000000000000000000000000000000000000000000000000000
+    #   app-launch-url: http://myapppage.com
+    #   authorities: test_resource.test_action
+    #   authorized-grant-types: authorization_code,client_credentials,refresh_token
+    #   autoapprove:
+    #     - test_resource.test_action
+    #     - test_resource.other_action
+    #   id: app
+    #   override: true
+    #   redirect-uri: http://login.example.com
+    #   scope: test_resource.test_action,test_resource.other_action
+    #   secret: app-secret
+    #   show-on-homepage: true
+
+# UAA theming
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/login/branding?
+  value: { }
+    # banner:
+    #   backgroundColor: "#eeeeee"
+    #   link: "https://example.com/logo.png"
+    #   text: "Hello!"
+    #   textColor: "#333333"
+    # company_name: "Company Name"
+    # footer_legal_text: "© 2017. Company Name. All right reserved."
+
+# Email notifications
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/login/smtp?
+  value: { }
+    # auth: false
+    # from_address: hello@example.com
+    # host: localhost
+    # port: 2525
+    # starttls: false

data/tpl/gcp/deployments/bosh/cloud-config.yml.tt

@@ -9,12 +9,25 @@ vm_types:
     machine_type: n1-standard-2
     root_disk_size_gb: 20
     root_disk_type: pd-ssd
+    tags:
+      - no-ip
+
+- name: ingress-tiny
+  cloud_properties:
+    machine_type: g1-small
+    root_disk_size_gb: 20
+    root_disk_type: pd-ssd
+    tags:
+      - http-server
+      - https-server
 
 - name: worker
   cloud_properties:
-    machine_type: n1-standard-4
+    machine_type: n1-standard-2
     root_disk_size_gb: 100
     root_disk_type: pd-ssd
+    tags:
+      - no-ip
 
 # vm_extensions:
 # - name: concourse-lb
@@ -27,10 +40,12 @@ compilation:
   reuse_compilation_vms: true
   az: z1
   cloud_properties:
-    machine_type: n1-standard-4
+    machine_type: n1-standard-2
     root_disk_size_gb: 100
     root_disk_type: pd-ssd
     preemptible: true
+    tags:
+      - no-ip
 
 networks:
   - name: public
@@ -46,7 +61,6 @@ networks:
         subnetwork_name: <%= @values['gcp']['subnet_name'] %>
         ephemeral_external_ip: false
         tags:
-          - no-ip
           - platform-internal
           - concourse-public
           - concourse-internal

data/tpl/gcp/deployments/concourse/concourse.yml.tt

@@ -15,14 +15,14 @@ instance_groups:
   stemcell: trusty
   networks:
   - name: public
-    static_ips: [<%= @private_subnet[12] %>]
+    static_ips: <%= @static_ips_concourse %>
     default: [dns, gateway]
 
   jobs:
   - name: atc
     release: concourse
     properties:
-      bind_port: 80
+      bind_port: 8080
       external_url: <%= @values['concourse']['url'] %>
       basic_auth_username: <%= @values['concourse']['auth_username'] %>
       basic_auth_password: ((auth_password))
@@ -33,7 +33,7 @@ instance_groups:
           backend: token
           client_token: ((vault_token))
         path_prefix: /concourse
-        url: "http://<%= @private_subnet[11] %>:8200" # expecting Vault to be deployed first
+        url: "http://<%= @static_ip_vault %>:8200" # expecting Vault to be deployed first
 
       postgresql_database: &atc_db atc
 
@@ -47,7 +47,9 @@ instance_groups:
   azs: [z1]
   stemcell: trusty
   persistent_disk_type: database
-  networks: [{name: public}]
+  networks:
+  - name: public
+    default: [dns, gateway]
   jobs:
   - name: postgresql
     release: concourse
@@ -62,7 +64,9 @@ instance_groups:
   vm_type: worker
   azs: [z1]
   stemcell: trusty
-  networks: [{name: public}]
+  networks:
+  - name: public
+    default: [dns, gateway]
   jobs:
   - name: groundcrew
     release: concourse
@@ -92,3 +96,9 @@ stemcells:
 - alias: trusty
   os: ubuntu-trusty
   version: latest
+
+variables:
+- name: db_password
+  type: password
+- name: auth_password
+  type: password

data/tpl/gcp/deployments/ingress/ingress.yml.erb

@@ -0,0 +1,111 @@
+---
+name: ingress
+
+releases:
+- name: nginx
+  version: latest
+
+instance_groups:
+- name: ingress
+  instances: 1
+  vm_type: ingress-tiny
+  azs: [z1]
+  stemcell: trusty
+  networks:
+  - name: public
+    static_ips: [<%= @private_subnet[13] %>]
+    default: [dns, gateway]
+
+  - name: vip
+    static_ips: [<%= @values['ingress']['vip'] %>]
+    
+  jobs:
+  - name: nginx
+    release: nginx
+    properties:
+      nginx_conf: |
+        worker_processes  1;
+        error_log /var/vcap/sys/log/nginx/error.log   info;
+        events {
+          worker_connections  1024;
+        }
+
+        http {
+          include /var/vcap/packages/nginx/conf/mime.types;
+          default_type  application/octet-stream;
+          sendfile        on;
+          keepalive_timeout  65;
+          server_names_hash_bucket_size 64;
+
+          server {
+             listen 80;
+             return 301 https://$host$request_uri;
+          }
+<% ingress_db.each do |hostname, config| %>
+<% upstream_name = hostname.gsub('.', '-') %>
+          upstream <%= upstream_name %> {
+            <%- config[:upstreams].each do |upstream| -%>
+            server <%= upstream %>:<%= config[:port] %>;
+            <%- end -%>
+          }
+          server {
+             listen 443 ssl http2;
+             server_name <%= hostname %>;
+             ssl_certificate_key /var/vcap/jobs/nginx/etc/<%= hostname %>/key.pem;
+             ssl_certificate     /var/vcap/jobs/nginx/etc/<%= hostname %>/cert.pem;
+             ssl on;
+             ssl_session_cache  builtin:1000  shared:SSL:10m;
+             ssl_protocols  TLSv1 TLSv1.1 TLSv1.2;
+             ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
+             ssl_prefer_server_ciphers on;
+
+             access_log /var/vcap/sys/log/nginx/<%= hostname %>-access.log;
+             error_log /var/vcap/sys/log/nginx/<%= hostname %>-error.log;
+
+             location / {
+               proxy_http_version 1.1;
+
+               proxy_set_header        Host $host;
+               proxy_set_header        X-Real-IP $remote_addr;
+               proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+               proxy_set_header        X-Forwarded-Proto $scheme;
+               proxy_set_header        Upgrade $http_upgrade;
+               proxy_set_header        Connection "upgrade";
+
+               proxy_pass          <%= config[:protocol] %>://<%= upstream_name%>;
+               proxy_read_timeout  90;
+             }
+          }
+<% end %>
+        }
+
+      pre_start: |
+        #!/bin/bash
+        set -x
+        source /etc/profile
+        export HOME=/root
+        export USER=root
+        echo "Running pre_start script as ${USER} with ${SHELL} with home ${HOME}"
+        if [[ ! -f ${HOME}/.acme.sh/acme.sh.env ]]; then
+          curl -s https://get.acme.sh | sh
+        fi
+        source ${HOME}/.acme.sh/acme.sh.env
+<% ingress_db.each do |hostname, config| -%>
+        mkdir -p /var/vcap/jobs/nginx/etc/<%= hostname %>/
+        ${HOME}/.acme.sh/acme.sh --issue --tls -d <%= hostname %>
+        ${HOME}/.acme.sh/acme.sh --install-cert -d <%= hostname %> \
+          --key-file /var/vcap/jobs/nginx/etc/<%= hostname %>/key.pem \
+          --fullchain-file /var/vcap/jobs/nginx/etc/<%= hostname %>/cert.pem
+<%- end -%>
+
+stemcells:
+- alias: trusty
+  os: ubuntu-trusty
+  version: latest
+
+update:
+  canaries: 1
+  max_in_flight: 1
+  serial: false
+  canary_watch_time: 1000-60000
+  update_watch_time: 1000-60000

data/tpl/gcp/deployments/oauth/oauth.yml.tt

@@ -0,0 +1,95 @@
+---
+name: oauth
+
+releases:
+  - name: uaa
+    version: latest
+
+instance_groups:
+  - name: uaa
+    instances: 1
+    vm_type: common
+    persistent_disk_type: database
+    azs: [z1]
+    stemcell: trusty
+    networks:
+      - name: public
+        static_ips: [<%= @static_ip_oauth %>]
+        default: [dns, gateway]
+    jobs:
+      - name: uaa_postgres
+        release: uaa
+        properties:
+          postgres:
+            port: 5524
+            roles:
+              - tag: admin
+                name: uaaadmin
+                password: ((uaa_db_password))
+            databases:
+              - tag: uaa
+                name: uaadb
+                citext: true
+      - name: uaa
+        release: uaa
+        properties:
+          login:
+            saml:
+              activeKeyId: key2
+              keys:
+                key2:
+                  key: ((uaa_saml_certificate.private_key))
+                  passphrase: ((uaa_saml_passphrase))
+                  certificate: ((uaa_saml_certificate.certificate))
+          uaa:
+            url: "<%= @values['oauth']['url'] %>"
+            sslPrivateKey: ((uaa_ssl.private_key))
+            sslCertificate: ((uaa_ssl.certificate))
+            jwt:
+              policy:
+                active_key_id: key-1
+                keys:
+                  key-1:
+                    signingKey: ((uaa_saml_certificate.private_key))
+          uaadb:
+          uaadb:
+            address: 127.0.0.1
+            databases:
+              - name: uaadb
+                tag: uaa
+            db_scheme: postgresql
+            port: 5524
+            roles:
+              - name: uaaadmin
+                password: ((uaa_db_password))
+                tag: admin
+
+update:
+  canaries: 1
+  max_in_flight: 1
+  serial: false
+  canary_watch_time: 1000-60000
+  update_watch_time: 1000-60000
+
+stemcells:
+  - alias: trusty
+    name: bosh-google-kvm-ubuntu-trusty-go_agent
+    version: latest
+
+variables:
+  - name: uaa_db_password
+    type: password
+  - name: uaa_saml_passphrase
+    type: password
+  - name: uaa_ssl
+    type: certificate
+    options:
+      ca: default_ca
+      common_name: "<%= @values['oauth']['url'] %>"
+      alternative_names: ["<%= @values['oauth']['url'] %>"]
+  - name: uaa_saml_certificate
+    type: certificate
+    options:
+      ca: default_ca
+      common_name: "<%= @values['oauth']['url'] %>"
+      alternative_names: ["<%= @values['oauth']['url'] %>"]