kite 0.0.8 → 0.0.9

data/tpl/aws/bin/vault-deploy.sh.tt

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Upload necessary stemcells and releases
+bosh -e <%= @values['bosh']['name'] %> upload-stemcell https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent
+bosh -e <%= @values['bosh']['name'] %> upload-release https://bosh.io/d/github.com/cloudfoundry-community/vault-boshrelease
+
+# Deploy Vault
+bosh -e <%= @values['bosh']['name'] %> -d vault deploy deployments/vault/vault.yml

data/tpl/aws/deployments/bosh/cloud-config.yml.tt

@@ -55,7 +55,8 @@ networks:
   - az: z1
     range: <%= @values['aws']['private_subnet']['network'] %>
     gateway: <%= @values['aws']['private_subnet']['gateway'] %>
-    reserved: [<%= ip_range(@private_subnet, (1..10)) %>]
+    reserved: [<%= ip_range(@private_subnet, (1..10)) %>] # Reserved range for the gateway, BOSH Director etc
+    static: [<%= ip_range(@private_subnet, (11..13)) %>] # Static IP range for Vault, Concourse web panel, nginx etc
     dns: [<%= @private_subnet[8].to_s %>]
     cloud_properties: {subnet: <%= @tf_output['platform_subnet_id'] %>}
 - name: vip

data/tpl/aws/deployments/concourse/{concourse.yml.erb → concourse.yml.tt}

@@ -26,6 +26,13 @@ instance_groups:
       basic_auth_password: <%= @values['concourse']['auth_password'] %>
       publicly_viewable: true
 
+      vault:
+        auth:
+          backend: token
+          client_token: "your Vault root token here"
+        path_prefix: /concourse
+        url: "http://<%= @private_subnet[11] %>:8200" # assuming Vault is deployed first
+
       postgresql_database: &atc_db atc
 
   - name: tsa

data/tpl/aws/deployments/vault/vault.yml.erb

@@ -0,0 +1,38 @@
+---
+name: vault
+
+releases:
+- name: vault
+  version: latest
+
+instance_groups:
+- name: vault
+  instances: 1
+  vm_type: default
+  azs: [z1]
+  stemcell: trusty
+  networks:
+  - name: platform_net
+    static_ips: [<%= @private_subnet[11] %>]
+
+  jobs:
+  - name: vault
+    release: vault
+    properties:
+      vault:
+        ha:
+          redirect: ~
+        storage:
+          use_file: true
+
+update:
+  canaries: 1
+  max_in_flight: 1
+  serial: false
+  canary_watch_time: 1000-60000
+  update_watch_time: 1000-60000
+
+stemcells:
+- alias: trusty
+  name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent
+  version: latest

data/tpl/aws/docs/bosh.md

@@ -0,0 +1,31 @@
+#### [Back](../README.md)
+
+## BOSH
+
+### Prerequisites
+
+- Terraform IaC applied
+- [BOSH CLI v2](https://bosh.io/docs/cli-v2.html#install) installed
+
+### Setup
+
+Render bosh deployment
+```
+kite render manifest bosh --cloud=gcp
+```
+
+Setup tunnel
+```
+. bin/setup-tunnel.sh
+```
+
+Install BOSH
+```
+./bin/bosh-install.sh
+```
+
+Connect to the Director
+```
+. bin/set-env.sh
+
+```

data/tpl/aws/docs/concourse.md

@@ -0,0 +1,30 @@
+#### [Back](../README.md)
+
+## Concourse
+
+### Prerequisites
+
+- Vault [deployed and initialized](vault.md)
+
+### Setup
+
+Fill out the "token" field in `deployments/concourse/concourse.yml` with root token received from `vault init`.
+
+Deploy Concourse
+```
+./bin/concourse-deploy.sh
+```
+
+### Test
+
+To run a test Concourse job:
+
+- Go to test folder: `cd deployments/concourse/test`
+- Fill out `test-credentials.yml`
+- Add necessary secrets to your Vault(see [docs/vault.md](docs/vault.md))
+- Download the `fly` client from Concourse web panel and add it to your PATH: `mv *path_to_fly* /usr/local/bin`
+- Login to Concourse using the `fly` client: `fly -t ci --concourse-url *concourse-url*`
+- Create a test pipeline with `fly set-pipeline -t ci -c test-pipeline.yml -p test --load-vars-from test-credentials.yml -n`
+- Unpause pipeline: `fly unpause-pipeline -t ci -p test`
+- Trigger and unpause the test job: `fly trigger-job -t ci -j test/test-publish`
+- See the results on Concourse web panel or use: `fly watch -p test -j test/test-publish`

data/tpl/{gcp → aws/docs}/vault.md

@@ -1,6 +1,8 @@
-# Vault usage
+#### [Back](../README.md)
 
-## Prerequisites
+## Vault
+
+### Prerequisites
 
 Before using Vault, you should have the client installed:
 
@@ -8,19 +10,24 @@ Before using Vault, you should have the client installed:
 - Unzip it and run `chmod +x vault && sudo mv vault /usr/local/bin/vault`
 - Check if the Vault is installed by running `vault -v`
 
-## Deployment
+### Deployment
 
-To deploy Vault, use `bin/vault-deploy.sh`
+To deploy Vault, use `./bin/vault-deploy.sh`
 
-## Connection
+### Connection
 
 - Export your Vault's IP using `export VAULT_ADDR=*vault_ip*`
 - Run `vault init` to initialize the vault
 - Store the keys displayed after init
 - Unseal the vault by running `vault unseal` three times using three keys from the previous step
+- Authenticate to the vault with `vault auth` using the root token you got from `vault init`
 
 [Optional]
 - Try to store a dummy secret: `vault write secret/handshake knock=knock`
 - Read it: `vault read secret/handshake`
 
-You're good to go!
+### Usage with Concourse
+
+Before using Vault with Concourse you should mount a secrets backend with `vault mount -path=concourse kv`
+
+To add new secrets accessible for Concourse use `vault write concourse/main/*secret_name* value="*secret_value*"`

data/tpl/aws/terraform/{main.tf → main.tf.tt}

@@ -16,8 +16,7 @@ resource "aws_instance" "bastion" {
   key_name = "${var.keypair_name}"
 
   vpc_security_group_ids = ["${aws_security_group.bastion_sg.id}"]
-  subnet_id = "${aws_subnet.platform_dmz.id}"
-
+  subnet_id = <%= "\"#{conditional_subnet_id(@values)}\"" %>
   associate_public_ip_address = true
 
   tags {

data/tpl/aws/terraform/{network.tf → network.tf.tt}

@@ -1,3 +1,4 @@
+<% if @values['aws']['vpc_id'].empty? %>
 # Create a VPC to launch our instances into
 resource "aws_vpc" "platform" {
   cidr_block = "${var.vpc_cidr_block}"
@@ -8,6 +9,18 @@ resource "aws_vpc" "platform" {
   }
 }
 
+# DMZ subnet
+resource "aws_subnet" "platform_dmz" {
+  vpc_id = "${aws_vpc.platform.id}"
+  availability_zone = "${var.availability_zone}"
+  cidr_block = "${var.public_subnet_cidr}"
+  map_public_ip_on_launch = false
+  tags {
+    Name = "${var.public_subnet_name}"
+    Component = "kite-platform"
+  }
+}
+
 # Create an internet gateway to give our subnet access to the outside world
 resource "aws_internet_gateway" "platform" {
   vpc_id = "${aws_vpc.platform.id}"
@@ -17,9 +30,19 @@ resource "aws_internet_gateway" "platform" {
   }
 }
 
+# Grant the VPC internet access on its main route table
+resource "aws_route" "internet_access" {
+  route_table_id = "${aws_vpc.platform.main_route_table_id}"
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id = "${aws_internet_gateway.platform.id}"
+}
+
+<% end %>
+
+<% if !@values['aws']['vpc_id'].empty? && @values['aws']['public_subnet']['id'].empty? %>
 # DMZ subnet
 resource "aws_subnet" "platform_dmz" {
-  vpc_id = "${aws_vpc.platform.id}"
+  vpc_id = "${var.vpc_id}"
   availability_zone = "${var.availability_zone}"
   cidr_block = "${var.public_subnet_cidr}"
   map_public_ip_on_launch = false
@@ -28,10 +51,11 @@ resource "aws_subnet" "platform_dmz" {
     Component = "kite-platform"
   }
 }
+<% end %>
 
 # Private subnet
 resource "aws_subnet" "platform_net" {
-  vpc_id = "${aws_vpc.platform.id}"
+  vpc_id = <%= "\"#{conditional_vpc_id(@values)}\"" %>
   availability_zone = "${var.availability_zone}"
   cidr_block = "${var.private_subnet_cidr}"
   map_public_ip_on_launch = false
@@ -41,26 +65,9 @@ resource "aws_subnet" "platform_net" {
   }
 }
 
-# Allocate an Elastic IP for NAT gateway
-resource "aws_eip" "nat_ip" {
-}
-
-# Create a NAT gateway to forward the traffic for BOSH
-resource "aws_nat_gateway" "nat_gateway" {
-  allocation_id = "${aws_eip.nat_ip.id}"
-  subnet_id = "${aws_subnet.platform_dmz.id}"
-}
-
-# Grant the VPC internet access on its main route table
-resource "aws_route" "internet_access" {
-  route_table_id = "${aws_vpc.platform.main_route_table_id}"
-  destination_cidr_block = "0.0.0.0/0"
-  gateway_id = "${aws_internet_gateway.platform.id}"
-}
-
 # Create a custom route table for the private subnet
 resource "aws_route_table" "private_route" {
-  vpc_id = "${aws_vpc.platform.id}"
+  vpc_id = <%= "\"#{conditional_vpc_id(@values)}\"" %>
 
   route {
     cidr_block = "0.0.0.0/0"
@@ -73,6 +80,16 @@ resource "aws_route_table" "private_route" {
   }
 }
 
+# Allocate an Elastic IP for NAT gateway
+resource "aws_eip" "nat_ip" {
+}
+
+# Create a NAT gateway to forward the traffic for BOSH
+resource "aws_nat_gateway" "nat_gateway" {
+  allocation_id = "${aws_eip.nat_ip.id}"
+  subnet_id = <%= "\"#{conditional_subnet_id(@values)}\"" %>
+}
+
 # Associate custom route table with private subnet
 resource "aws_route_table_association" "private_route" {
   subnet_id      = "${aws_subnet.platform_net.id}"
@@ -83,7 +100,7 @@ resource "aws_route_table_association" "private_route" {
 resource "aws_security_group" "bastion_sg" {
   name = "bastion_sg"
   description = "Bastion security group"
-  vpc_id = "${aws_vpc.platform.id}"
+  vpc_id = <%= "\"#{conditional_vpc_id(@values)}\"" %>
   tags {
     Name = "bastion-sg"
     Component = "bosh-director"
@@ -110,7 +127,7 @@ resource "aws_security_group" "bastion_sg" {
 resource "aws_security_group" "bosh_sg" {
   name = "bosh_sg"
   description = "Default BOSH security group"
-  vpc_id = "${aws_vpc.platform.id}"
+  vpc_id = <%= "\"#{conditional_vpc_id(@values)}\"" %>
   tags {
     Name = "bosh-sg"
     Component = "bosh-director"
@@ -169,7 +186,7 @@ resource "aws_security_group" "bosh_sg" {
 resource "aws_security_group" "concourse_sg" {
   name        = "concourse-sg"
   description = "Concourse security group"
-  vpc_id      = "${aws_vpc.platform.id}"
+  vpc_id = <%= "\"#{conditional_vpc_id(@values)}\"" %>
   tags {
     Name = "concourse-sg"
     Component = "concourse"

data/tpl/aws/terraform/{terraform.tfvars.erb → terraform.tfvars.tt}

@@ -7,8 +7,10 @@ availability_zone = "<%= @values['aws']['zone'] %>"
 # Network Config
 vpc_cidr_block = "<%= @values['aws']['vpc_cidr_block'] %>"
 vpc_name = "<%= @values['aws']['vpc_name'] %>"
+vpc_id = "<%= @values['aws']['vpc_id'] %>"
 public_subnet_name = "<%= @values['aws']['public_subnet']['name'] %>"
 public_subnet_cidr = "<%= @values['aws']['public_subnet']['network'] %>"
+public_subnet_id = "<%= @values['aws']['public_subnet']['id'] %>"
 private_subnet_name = "<%= @values['aws']['private_subnet']['name'] %>"
 private_subnet_cidr = "<%= @values['aws']['private_subnet']['network'] %>"
 

data/tpl/aws/terraform/variables.tf

@@ -36,10 +36,18 @@ variable "vpc_name" {
   type = "string"
 }
 
+variable "vpc_id" {
+  type = "string"
+}
+
 variable "public_subnet_cidr" {
   type = "string"
 }
 
+variable "public_subnet_id" {
+  type = "string"
+}
+
 variable "public_subnet_name" {
   type = "string"
 }

data/tpl/gcp/README.md

@@ -1,6 +1,6 @@
 ## GCP Cloud
 
-### Usage
+### Setup
 
 Set path to your service account credentials:
 ```
@@ -12,44 +12,23 @@ Apply terraform code
 pushd terraform && terraform init && terraform apply && popd
 ```
 
-Render bosh deployment
+Render BOSH manifest and related files
 ```
-kite render manifest bosh --cloud=gcp
+kite render manifest bosh --cloud gcp
 ```
 
-Setup tunnel
-```
-. bin/setup-tunnel.sh
-```
+Prepare BOSH environment using instructions from [docs/bosh.md](docs/bosh.md)
 
-Install BOSH
-```
-./bin/bosh-install.sh
+Render Vault deployment
 ```
-
-Connect to the Director
+kite render manifest vault --cloud gcp
 ```
-. bin/set-env.sh
 
-```
+Follow instructions from [docs/vault.md](docs/vault.md) to deploy Vault
 
-Render concourse deployment
+Render Concourse manifest
 ```
-kite render manifest concourse --cloud=gcp
+kite render manifest concourse --cloud gcp
 ```
 
-Install concourse
-```
-bosh -e bosh-1 update-cloud-config  deployments/concourse/cloud-config.yml
-
-bosh -e bosh-1 upload-stemcell \
-  https://bosh.io/d/stemcells/bosh-google-kvm-ubuntu-trusty-go_agent?v=3445.7
-
-bosh -e bosh-1 upload-release \
-  https://github.com/concourse/concourse/releases/download/v3.4.1/concourse-3.4.1.tgz
-
-bosh -e bosh-1 upload-release \
-  https://github.com/concourse/concourse/releases/download/v3.4.1/garden-runc-1.6.0.tgz
-
-bosh -e bosh-1 -d concourse deploy deployments/concourse/concourse.yml
-```
+Follow instructions from [docs/concourse.md](docs/concourse.md) to deploy Concourse

data/tpl/gcp/bin/base/bootstrap.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Apply Terraform IaC
+pushd terraform
+
+terraform init
+terraform apply
+
+popd
+
+# Set up an SSH tunnel to Bastion
+. bin/setup-tunnel.sh
+
+# Render BOSH manifest and related files
+kite render manifest bosh --cloud gcp
+
+# Deploy BOSH Director
+./bin/bosh-install.sh
+
+# Set the needed environment variables
+. bin/set-env.sh
+
+# Render Vault manifest and related files
+kite render manifest vault --cloud gcp
+
+# Deploy Vault
+./bin/vault-deploy.sh
+
+# Render Concourse manifest
+kite render manifest concourse --cloud gcp
+
+# Deploy Concourse
+./bin/concourse-deploy.sh