kite 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b20f3f85398a1b4a4b23cb1cb818a0f29b489617
-  data.tar.gz: 8b2fa9d43523bfbb839ddd655ee3565cc028bfc1
+  metadata.gz: 3084fa0faffd40bc3ffad4f326ea1501cb6eb31a
+  data.tar.gz: dcee9c104c6d840a0465b209aa391872b12cb629
 SHA512:
-  metadata.gz: '049614c95440425c9142eb0b25423e898480e151a648f6cf4b7f55c7d74f51ff406a25211e74fe449693b2e90fd00721c46bf609d3dd0f4417e804bc53180743'
-  data.tar.gz: d039515fb358e058c1ac49da66943cc24a9496183581f144ef7df4b5163f9f0df52d2f5dc582722031fbe226d8d947ad507411c6f7342d5060265fbe31b3996b
+  metadata.gz: c2affb89e3e7fb09feddecf3151c0042ea1708319ccd5f56fc5d896562e385aadb4dd79c32858de2e60a28f77b0830a482384a95062ca0c982afd4c0374717bf
+  data.tar.gz: 11fc96ffbe8cde21ed4bc3adb09c10acaafce7faebbe1badffc06ca3919913769f4e459f4cde23779a47f7649f41eeca1fc7cdd5dc3419b55ae1f794bdb1b19e

data/Dockerfile

@@ -0,0 +1,16 @@
+FROM ruby:2.4.1
+
+RUN apt-get update && apt-get install -y zip
+
+# Install Terraform
+RUN curl https://releases.hashicorp.com/terraform/0.10.5/terraform_0.10.5_linux_amd64.zip?_ga=2.49593953.619315674.1505216069-1504763789.1498760046 -o terraform.zip
+RUN unzip terraform -d /usr/bin/terraform && chmod +x /usr/bin/terraform
+
+# Install BOSH v2
+RUN curl https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-2.0.31-linux-amd64 -o /usr/bin/bosh && chmod +x /usr/bin/bosh
+
+# Copy kite source, build and install the gem , egnerate a test cloud skeleton
+COPY . /kite
+WORKDIR /kite
+RUN bundle && rake build && gem install pkg/kite-*
+RUN kite new test

data/Makefile

@@ -0,0 +1,12 @@
+IMAGE ?= kaigara/kitebox
+TAG   ?= $(shell git describe --tags --abbrev=0 2>/dev/null || echo "1.0.0")
+
+.PHONY: build
+
+build:
+	echo "Building $(IMAGE):$(TAG)"
+	docker build -t "$(IMAGE):$(TAG)" .
+start: build
+	docker run -d --name="kitebox" $(IMAGE):$(TAG)
+clean:
+	docker rm $(docker stop {kitebox})

data/lib/kite.rb

@@ -7,5 +7,6 @@ require 'kite/helpers'
 require 'kite/error'
 
 require 'kite/base'
+require 'kite/render'
 require 'kite/core'
 require 'kite/cloud'

data/lib/kite/base.rb

@@ -1,5 +1,10 @@
+# Base class including all Thor-related configuration
 class Kite::Base < Thor
 
   include Thor::Actions
 
+  def self.source_root
+    File.expand_path(File.join(File.dirname(__FILE__), "../../tpl"))
+  end
+
 end

data/lib/kite/cloud.rb

@@ -1,3 +1,4 @@
+# Class responsible for creating a new cloud infrastructure skeleton
 class Kite::Cloud
 
   def initialize(core, cloud_name)

data/lib/kite/core.rb

@@ -3,11 +3,8 @@ module Kite
 
     include Kite::Helpers
 
-    def self.source_root
-      File.expand_path(File.join(File.dirname(__FILE__), "../../tpl"))
-    end
-
     desc "new CLOUD_PATH", "Generate Cloud infrastructure skeleton from configuration"
+    # Creates a cloud infrastructure skeleton with a given name
     def new(cloud_name)
       target = Kite::Cloud.new(self, cloud_name)
       target.prepare
@@ -15,6 +12,7 @@ module Kite
 
     method_option :cloud, type: :string, desc: "Cloud provider", enum: %w{aws gcp}, required: true
     desc "generate", "Generate Cloud IaC from configuration"
+    # Generates Infrastructure as Code and setup scripts for the given cloud using values from <b>config/cloud.yml</b>
     def generate()
       say "Generating Cloud #{ options[:cloud] } IaC", :green
       @values = parse_cloud_config
@@ -30,7 +28,10 @@ module Kite
 
         template('aws/bosh-install.sh.erb',                 'bin/bosh-install.sh')
         template('aws/setup-tunnel.sh.erb',                 'bin/setup-tunnel.sh')
+        template('aws/concourse-deploy.sh.erb',             'bin/concourse-deploy.sh')
+        template('aws/set-env.sh.erb',                      'bin/set-env.sh')
         chmod('bin/bosh-install.sh', 0755)
+        chmod('bin/concourse-deploy.sh', 0755)
         chmod('bin/setup-tunnel.sh', 0755)
 
       when 'gcp'
@@ -44,6 +45,7 @@ module Kite
         template('gcp/bosh-install.sh.erb',                 'bin/bosh-install.sh')
         template('gcp/bosh-vars.yml.erb',                   'bosh-vars.yml')
         template('gcp/setup-tunnel.sh.erb',                 'bin/setup-tunnel.sh')
+        template('gcp/set-env.sh.erb',                      'bin/set-env.sh')
         chmod('bin/bosh-install.sh', 0755)
         chmod('bin/setup-tunnel.sh', 0755)
 
@@ -53,29 +55,11 @@ module Kite
       end
     end
 
-    method_option :cloud, type: :string, desc: "Cloud provider", enum: %w{aws gcp}, required: true
     desc 'render MANIFEST', 'Render manifest file from configuration and Terraform output'
-    def render(manifest)
-      say "Rendering #{ manifest } manifest", :green
-      @values = parse_cloud_config
-      @tf_output = parse_tf_state('terraform/terraform.tfstate')
-
-      case manifest
-      when "bosh"
-        cloud = options[:cloud]
-        directory("#{cloud}/deployments", 'deployments')
-
-      when "concourse"
-        template("aws/concourse/aws_cloud.yml.erb",   "aws_cloud.yml")
-        template("aws/concourse/concourse.yml.erb",   "concourse.yml")
-
-      else
-        say "Manifest type not specified"
-
-      end
-    end
+    subcommand "render", Kite::Render
 
     desc "version", "Return kite version"
+    # Return kite version
     def version
       say "v#{ Kite::VERSION }"
     end

data/lib/kite/error.rb

@@ -1,2 +1,3 @@
+# Error raised after invalid files or arguments are used
 class Kite::Error < Thor::Error
 end

data/lib/kite/render.rb

@@ -0,0 +1,30 @@
+module Kite
+  # Subcommand for rendering manifests, deployments etc.
+  class Render < Base
+
+    include Kite::Helpers
+
+    desc "manifest <type>", "Renders a manifest of selected type"
+    method_option :cloud, type: :string, desc: "Cloud provider", enum: %w{aws gcp}, required: true
+    # Render a manifest of selected type based on <b>config/cloud.yml</b> and <b>terraform apply</b> results
+    def manifest(type)
+      say "Rendering #{type} manifest", :green
+      @values = parse_cloud_config
+      @tf_output = parse_tf_state('terraform/terraform.tfstate')
+
+      case type
+      when "bosh"
+        cloud = options[:cloud]
+        directory("#{cloud}/deployments",                    'deployments')
+
+      when "concourse"
+        template("#{options[:cloud]}/deployments/concourse/cloud-config.yml.erb", "deployments/concourse/cloud-config.yml")
+        template("#{options[:cloud]}/deployments/concourse/concourse.yml.erb", "deployments/concourse/concourse.yml")
+
+      else
+        say "Manifest type not specified"
+
+      end
+    end
+  end
+end

data/lib/kite/version.rb

@@ -1,3 +1,3 @@
 module Kite
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/tpl/aws/README.md

@@ -20,9 +20,9 @@ Usage
 
 To deploy a BOSH Director:
 - Apply the terraform IaC from `terraform` folder
-- Run `bin/setup_tunnel.sh` to create an SSH CLI tunnel
-- Run `kite render bosh --cloud aws` to render BOSH deployment files
-- Run `bin/bosh_setup.sh` to deploy the BOSH Director
+- Run `source bin/setup-tunnel.sh` to create an SSH CLI tunnel
+- Run `kite render manifest bosh --cloud=aws` to render BOSH deployment files
+- Run `bin/bosh-install.sh` to deploy the BOSH Director
 
 To access BOSH Director information, use bosh -e *bosh_name* env
 

data/tpl/aws/bosh-install.sh.erb

@@ -3,10 +3,10 @@
 set -xe
 
 # Create a new BOSH environment with Director
-bosh create-env deployments/bosh/bosh_director.yml \
-  --state=state.json \
-  --vars-store=creds.yml \
-  --vars-file=bosh_vars.yml \
+bosh create-env deployments/bosh/bosh.yml \
+  --state=config/state.json \
+  --vars-store=config/creds.yml \
+  --vars-file=deployments/bosh/bosh_vars.yml \
   --var-file private_key=<%= @values['kite']['private_key_path'] %> \
   -o deployments/bosh/cpi.yml \
   -o deployments/bosh/jumpbox-user.yml
@@ -14,12 +14,8 @@ bosh create-env deployments/bosh/bosh_director.yml \
 # Configure alias for the new environment
 bosh alias-env <%= @values['bosh']['name'] %> \
   -e <%= @values['bosh']['static_ip'] %>  \
-  --ca-cert <(bosh int ./creds.yml --path /director_ssl/ca)
+  --ca-cert <(bosh int ./config/creds.yml --path /director_ssl/ca)
 
 # Get jumpbox user key
-bosh int creds.yml --path /jumpbox_ssh/private_key > jumpbox.key
-chmod 600 jumpbox.key
-
-# Log into the newly created Director
-export BOSH_CLIENT=admin
-export BOSH_CLIENT_SECRET=`bosh int ./creds.yml --path /admin_password`
+bosh int config/creds.yml --path /jumpbox_ssh/private_key > config/jumpbox.key
+chmod 600 config/jumpbox.key

data/tpl/aws/concourse-deploy.sh.erb

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Render Concourse-related files
+kite render manifest concourse --cloud aws
+
+# Update BOSH director cloud configuration
+bosh update-cloud-config deployments/concourse/cloud-config.yml
+
+# Upload necessary stemcells and releases
+bosh -e <%= @values['bosh']['name'] %> upload-stemcell https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent
+bosh -e <%= @values['bosh']['name'] %> upload-release https://bosh.io/d/github.com/concourse/concourse
+bosh -e <%= @values['bosh']['name'] %> upload-release https://bosh.io/d/github.com/cloudfoundry-incubator/garden-runc-release
+
+# Deploy Concourse
+bosh -e <%= @values['bosh']['name'] %> -d concourse deploy deployments/concourse/concourse.yml

data/tpl/aws/deployments/bosh/{bosh_director.yml → bosh.yml}

@@ -3,9 +3,9 @@ name: bosh
 
 releases:
 - name: bosh
-  version: "262.3"
-  url: https://s3.amazonaws.com/bosh-compiled-release-tarballs/bosh-262.3-ubuntu-trusty-3421.9-20170706-183731-831697577-20170706183736.tgz?versionId=7GmwKfufgb5JwWhJ.cwIWLnejOtm2Hu4
-  sha1: 1eae3f06282417e54ebb199656458f9d6c38e2af
+  version: "263"
+  url: https://s3.amazonaws.com/bosh-compiled-release-tarballs/bosh-263-ubuntu-trusty-3445.7-20170901-012146-902840377-20170901012153.tgz?versionId=89a.ZxB3Jc_gl6s4YESlL41xNOfoJKrO
+  sha1: cc71c2ee6992071b1e1f6ae9f2119c03a42521c5
 
 resource_pools:
 - name: vms
@@ -24,9 +24,9 @@ networks:
 - name: default
   type: manual
   subnets:
-  - range: 10.0.0.0/24
-    gateway: 10.0.0.1
-    static: [10.0.0.2]
+  - range: ((internal_cidr))
+    gateway: ((internal_gw))
+    static: [((internal_ip))]
     dns: [8.8.8.8]
 
 instance_groups:
@@ -56,7 +56,7 @@ instance_groups:
       database: bosh
       adapter: postgres
     blobstore:
-      address: 10.0.0.2
+      address: ((internal_ip))
       port: 25250
       provider: dav
       director:

data/tpl/aws/deployments/bosh/{bosh_vars.yml.erb → bosh_vars.yml.tt}

@@ -1,11 +1,11 @@
 director_name:             <%= @values['bosh']['name'] %>
-internal_cidr:             <%= @values['aws']['platform_subnet_cidr_block'] %>
-internal_gw:               10.0.0.1
+internal_cidr:             <%= @values['aws']['private_subnet']['network'] %>
+internal_gw:               <%= @values['aws']['private_subnet']['gateway'] %>
 internal_ip:               <%= @values['bosh']['static_ip'] %>
 access_key_id:             <%= @values['aws']['access_key'] %>
 secret_access_key:         <%= @values['aws']['secret_key'] %>
 region:                    <%= @values['aws']['region'] %>
-az:                        <%= @values['aws']['az'] %>
+az:                        <%= @values['aws']['zone'] %>
 default_key_name:          <%= @values['kite']['keypair_name'] %>
 default_security_groups:   [<%= @tf_output['security_group_id'] %>]
 subnet_id:                 <%= @tf_output['platform_subnet_id'] %>

data/tpl/aws/{concourse/aws_cloud.yml.erb → deployments/concourse/cloud-config.yml.erb}

@@ -1,46 +1,44 @@
 ---
 azs:
 - name: z1
-  cloud_properties: {availability_zone: <%= @values['aws']['az'] %>}
+  cloud_properties: {availability_zone: <%= @values['aws']['zone'] %>}
 
 vm_types:
 - name: concourse_standalone
   cloud_properties:
     instance_type: m3.large
     ephemeral_disk: {size: 5000, type: gp2}
-    elbs: [concourse-elb]
-    security_groups: [concourse-sg, boshdefault]
+    security_groups: [concourse-sg, bosh_sg]
 - name: concourse_web
   cloud_properties:
     instance_type: m3.medium
     ephemeral_disk: {size: 3000, type: gp2}
-    elbs: [concourse-elb]
-    security_groups: [concourse-sg, boshdefault]
+    security_groups: [concourse-sg, bosh_sg]
 - name: concourse_db
   cloud_properties:
     instance_type: m3.medium
     ephemeral_disk: {size: 3000, type: gp2}
-    security_groups: [boshdefault]
+    security_groups: [bosh_sg]
 - name: concourse_worker
   cloud_properties:
     instance_type: m3.large
     ephemeral_disk: {size: 30000, type: gp2}
-    security_groups: [boshdefault]
+    security_groups: [bosh_sg]
 - name: default
   cloud_properties:
     instance_type: t2.micro
     ephemeral_disk: {size: 3000, type: gp2}
-    security_groups: [boshdefault]
+    security_groups: [bosh_sg]
 - name: large
   cloud_properties:
     instance_type: m3.large
     ephemeral_disk: {size: 5000, type: gp2}
-    security_groups: [boshdefault]
+    security_groups: [bosh_sg]
 - name: vault-default
   cloud_properties:
     instance_type: t2.micro
     ephemeral_disk: {size: 3000, type: gp2}
-    security_groups: [vault-sg, boshdefault]
+    security_groups: [vault-sg, bosh_sg]
 
 disk_types:
 - name: default
@@ -51,25 +49,15 @@ disk_types:
   cloud_properties: {type: gp2}
 
 networks:
-- name: default
-  type: manual
-  subnets:
-  - range: 10.0.0.0/24
-    gateway: 10.0.0.1
-    az: z1
-    static: [10.0.0.6]
-    reserved: [10.0.0.1-10.0.0.5]
-    dns: [10.0.0.2]
-    cloud_properties: {subnet: <%= @tf_output['default_subnet_id'] %>}
-- name: ops_services
+- name: platform_net
   type: manual
   subnets:
-  - range: 10.0.10.0/24
-    gateway: 10.0.10.1
-    az: z1
-    reserved: [10.0.10.1-10.0.10.5]
-    dns: [10.0.0.2]
-    cloud_properties: {subnet: <%= @tf_output['ops_services_subnet_id'] %>}
+  - az: z1
+    range: <%= @values['aws']['private_subnet']['network'] %>
+    gateway: <%= @values['aws']['private_subnet']['gateway'] %>
+    reserved: [10.0.20.1-10.0.20.10]
+    dns: [10.0.20.8]
+    cloud_properties: {subnet: <%= @tf_output['platform_subnet_id'] %>}
 - name: vip
   type: vip
 
@@ -78,4 +66,4 @@ compilation:
   reuse_compilation_vms: true
   az: z1
   vm_type: large
-  network: default
+  network: platform_net

data/tpl/aws/{concourse → deployments/concourse}/concourse.yml.erb

@@ -1,37 +1,33 @@
----
 name: concourse
 
-director_uuid: <%= %x(bosh status --uuid) %>
-
 releases:
 - name: concourse
   version: latest
 - name: garden-runc
   version: latest
 
-stemcells:
-- alias: trusty
-  os: ubuntu-trusty
-  version: latest
-
 instance_groups:
 - name: web
   instances: 1
   vm_type: concourse_web
-  stemcell: trusty
   azs: [z1]
-  networks: [{name: ops_services}]
+  # vm_extensions: [concourse-lb]
+  stemcell: trusty
+  networks:
+  - name: platform_net
+    default: [dns, gateway]
   jobs:
   - name: atc
     release: concourse
     properties:
-      # replace with your CI's externally reachable URL e.g https://blah
+      bind_port: 80
       external_url: <%= @values['concourse']['url'] %>
-
       basic_auth_username: <%= @values['concourse']['auth_username'] %>
       basic_auth_password: <%= @values['concourse']['auth_password'] %>
+      publicly_viewable: true
 
       postgresql_database: &atc_db atc
+
   - name: tsa
     release: concourse
     properties: {}
@@ -39,30 +35,32 @@ instance_groups:
 - name: db
   instances: 1
   vm_type: concourse_db
-  stemcell: trusty
-  persistent_disk_type: default
   azs: [z1]
-  networks: [{name: ops_services}]
+  stemcell: trusty
+  persistent_disk_type: large
+  networks: [{name: platform_net}]
   jobs:
   - name: postgresql
     release: concourse
     properties:
       databases:
       - name: *atc_db
-        # make up a role and password
-        role: dbrole
+        role: admin
         password: <%= @values['concourse']['db_password'] %>
 
 - name: worker
   instances: 1
   vm_type: concourse_worker
-  stemcell: trusty
   azs: [z1]
-  networks: [{name: ops_services}]
+  stemcell: trusty
+  networks: [{name: platform_net}]
   jobs:
   - name: groundcrew
     release: concourse
-    properties: {}
+    properties:
+      additional_resource_types:
+      - type: gcs-resource
+        image: docker:///frodenas/gcs-resource
   - name: baggageclaim
     release: concourse
     properties: {}
@@ -72,6 +70,7 @@ instance_groups:
       garden:
         listen_network: tcp
         listen_address: 0.0.0.0:7777
+        network_mtu: 1432
 
 update:
   canaries: 1
@@ -79,3 +78,8 @@ update:
   serial: false
   canary_watch_time: 1000-60000
   update_watch_time: 1000-60000
+
+stemcells:
+- alias: trusty
+  os: ubuntu-trusty
+  version: latest

data/tpl/aws/set-env.sh.erb

@@ -0,0 +1,7 @@
+
+BASTION_IP="$(terraform output -state=terraform/terraform.tfstate bastion_ip)"
+
+export BASTION_IP
+export BOSH_ALL_PROXY=socks5://localhost:5000
+export BOSH_CLIENT=admin
+export BOSH_CLIENT_SECRET=`bosh int ./config/creds.yml --path /admin_password`

data/tpl/aws/setup-tunnel.sh.erb

@@ -1,4 +1,4 @@
 BASTION_IP="$(terraform output -state=terraform/terraform.tfstate bastion_ip)"
-ssh -D 5000 -fNC ubuntu@$BASTION_IP -i <%= @values['kite']['public_key_path'] %>
+ssh -4 -D 5000 -fNC ubuntu@$BASTION_IP -i <%= @values['kite']['private_key_path'] %>
 
 export BOSH_ALL_PROXY=socks5://localhost:5000

data/tpl/aws/terraform/main.tf

@@ -1,8 +1,8 @@
 # Specify the provider and access details
 provider "aws" {
-  region = "${var.aws_region}"
-  access_key = "${var.aws_access_key}"
-  secret_key = "${var.aws_secret_key}"
+  region = "${var.region}"
+  access_key = "${var.access_key}"
+  secret_key = "${var.secret_key}"
 }
 
 resource "aws_key_pair" "platform_key" {
@@ -11,12 +11,12 @@ resource "aws_key_pair" "platform_key" {
 }
 
 resource "aws_instance" "bastion" {
-  ami = "${lookup(var.aws_amis, var.aws_region)}"
+  ami = "${lookup(var.aws_amis, var.region)}"
   instance_type = "t2.small"
   key_name = "${var.keypair_name}"
 
-  vpc_security_group_ids = ["${aws_security_group.bosh_sg.id}"]
-  subnet_id = "${aws_subnet.platform.id}"
+  vpc_security_group_ids = ["${aws_security_group.bastion_sg.id}"]
+  subnet_id = "${aws_subnet.platform_dmz.id}"
 
   associate_public_ip_address = true
 

data/tpl/aws/terraform/network.tf

@@ -1,9 +1,9 @@
 # Create a VPC to launch our instances into
 resource "aws_vpc" "platform" {
-  cidr_block = "${var.aws_vpc_cidr_block}"
+  cidr_block = "${var.vpc_cidr_block}"
 
   tags {
-    Name = "${var.aws_vpc_name}"
+    Name = "${var.vpc_name}"
     Component = "kite-platform"
   }
 }
@@ -17,6 +17,40 @@ resource "aws_internet_gateway" "platform" {
   }
 }
 
+# DMZ subnet
+resource "aws_subnet" "platform_dmz" {
+  vpc_id = "${aws_vpc.platform.id}"
+  availability_zone = "${var.availability_zone}"
+  cidr_block = "${var.public_subnet_cidr}"
+  map_public_ip_on_launch = false
+  tags {
+    Name = "${var.public_subnet_name}"
+    Component = "kite-platform"
+  }
+}
+
+# Private subnet
+resource "aws_subnet" "platform_net" {
+  vpc_id = "${aws_vpc.platform.id}"
+  availability_zone = "${var.availability_zone}"
+  cidr_block = "${var.private_subnet_cidr}"
+  map_public_ip_on_launch = false
+  tags {
+    Name = "${var.private_subnet_name}"
+    Component = "kite-platform"
+  }
+}
+
+# Allocate an Elastic IP for NAT gateway
+resource "aws_eip" "nat_ip" {
+}
+
+# Create a NAT gateway to forward the traffic for BOSH
+resource "aws_nat_gateway" "nat_gateway" {
+  allocation_id = "${aws_eip.nat_ip.id}"
+  subnet_id = "${aws_subnet.platform_dmz.id}"
+}
+
 # Grant the VPC internet access on its main route table
 resource "aws_route" "internet_access" {
   route_table_id = "${aws_vpc.platform.main_route_table_id}"
@@ -24,27 +58,51 @@ resource "aws_route" "internet_access" {
   gateway_id = "${aws_internet_gateway.platform.id}"
 }
 
-# Create a subnet to launch our instances into
-resource "aws_subnet" "platform" {
+# Create a custom route table for the private subnet
+resource "aws_route_table" "private_route" {
   vpc_id = "${aws_vpc.platform.id}"
-  availability_zone = "${var.aws_availability_zone}"
-  cidr_block = "${var.aws_platform_subnet_cidr_block}"
-  map_public_ip_on_launch = false
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    nat_gateway_id = "${aws_nat_gateway.nat_gateway.id}"
+  }
+
   tags {
-    Name = "${var.aws_platform_subnet_name}"
+    Name = "platform-route"
     Component = "kite-platform"
   }
 }
 
-# Create an ops_services subnet
-resource "aws_subnet" "ops_services" {
+# Associate custom route table with private subnet
+resource "aws_route_table_association" "private_route" {
+  subnet_id      = "${aws_subnet.platform_net.id}"
+  route_table_id = "${aws_route_table.private_route.id}"
+}
+
+# The default security group
+resource "aws_security_group" "bastion_sg" {
+  name = "bastion_sg"
+  description = "Bastion security group"
   vpc_id = "${aws_vpc.platform.id}"
-  availability_zone = "${var.aws_availability_zone}"
-  cidr_block = "${var.aws_ops_subnet_cidr_block}"
-  map_public_ip_on_launch = false
   tags {
-    Name = "${var.aws_ops_subnet_name}"
-    Component = "ops-services"
+    Name = "bastion-sg"
+    Component = "bosh-director"
+  }
+
+  ingress {
+    from_port = 22
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = [
+      "0.0.0.0/0"]
   }
 }
 
@@ -54,7 +112,7 @@ resource "aws_security_group" "bosh_sg" {
   description = "Default BOSH security group"
   vpc_id = "${aws_vpc.platform.id}"
   tags {
-    Name = "bosh-sq"
+    Name = "bosh-sg"
     Component = "bosh-director"
   }
 
@@ -148,30 +206,3 @@ resource "aws_security_group" "concourse_sg" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 }
-
-# Create a Vault security group
-resource "aws_security_group" "vault_sg" {
-  name        = "vault-sg"
-  description = "Vault security group"
-  vpc_id      = "${aws_vpc.platform.id}"
-  tags {
-    Name = "vault-sg"
-    Component = "vault"
-  }
-
-  # outbound internet access
-  egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-
-  # inbound http
-  ingress {
-    from_port   = 8200
-    to_port     = 8200
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-}

data/tpl/aws/terraform/outputs.tf

@@ -3,13 +3,13 @@ output "security_group_id" {
 }
 
 output "platform_subnet_id" {
-    value = "${aws_subnet.platform.id}"
-}
-
-output "ops_services_subnet_id" {
-    value = "${aws_subnet.ops_services.id}"
+    value = "${aws_subnet.platform_net.id}"
 }
 
 output "bastion_ip" {
     value = "${aws_instance.bastion.public_ip}"
 }
+
+output "gateway_ip" {
+    value = "${aws_nat_gateway.nat_gateway.private_ip}"
+}

data/tpl/aws/terraform/terraform.tfvars.erb

@@ -1,16 +1,16 @@
 # Credentials
-aws_access_key = "<%= @values['aws']['access_key'] %>"
-aws_secret_key = "<%= @values['aws']['secret_key'] %>"
-aws_region = "<%= @values['aws']['region'] %>"
-aws_availability_zone = "<%= @values['aws']['az'] %>"
+access_key = "<%= @values['aws']['access_key'] %>"
+secret_key = "<%= @values['aws']['secret_key'] %>"
+region = "<%= @values['aws']['region'] %>"
+availability_zone = "<%= @values['aws']['zone'] %>"
 
 # Network Config
-aws_vpc_cidr_block = "<%= @values['aws']['vpc_cidr_block'] %>"
-aws_vpc_name = "<%= @values['aws']['vpc_name'] %>"
-aws_platform_subnet_cidr_block = "<%= @values['aws']['platform_subnet_cidr_block'] %>"
-aws_platform_subnet_name = "<%= @values['aws']['platform_subnet_name'] %>"
-aws_ops_subnet_cidr_block = "<%= @values['aws']['ops_subnet_cidr_block'] %>"
-aws_ops_subnet_name = "<%= @values['aws']['ops_subnet_name'] %>"
+vpc_cidr_block = "<%= @values['aws']['vpc_cidr_block'] %>"
+vpc_name = "<%= @values['aws']['vpc_name'] %>"
+public_subnet_name = "<%= @values['aws']['public_subnet']['name'] %>"
+public_subnet_cidr = "<%= @values['aws']['public_subnet']['network'] %>"
+private_subnet_name = "<%= @values['aws']['private_subnet']['name'] %>"
+private_subnet_cidr = "<%= @values['aws']['private_subnet']['network'] %>"
 
 # Kite config
 keypair_name = "<%= @values['kite']['keypair_name'] %>"

data/tpl/aws/terraform/variables.tf

@@ -1,8 +1,8 @@
-variable "aws_access_key" {
+variable "access_key" {
   type = "string"
 }
 
-variable "aws_secret_key" {
+variable "secret_key" {
   type = "string"
 }
 
@@ -18,37 +18,37 @@ variable "keypair_name" {
   type = "string"
 }
 
-variable "aws_region" {
+variable "region" {
   type = "string"
   default =  "eu-central-1"
 }
 
-variable "aws_availability_zone" {
+variable "availability_zone" {
   type = "string"
   default =  "eu-central-1a"
 }
 
-variable "aws_vpc_cidr_block" {
+variable "vpc_cidr_block" {
   type = "string"
 }
 
-variable "aws_vpc_name" {
+variable "vpc_name" {
   type = "string"
 }
 
-variable "aws_platform_subnet_cidr_block" {
+variable "public_subnet_cidr" {
   type = "string"
 }
 
-variable "aws_platform_subnet_name" {
+variable "public_subnet_name" {
   type = "string"
 }
 
-variable "aws_ops_subnet_cidr_block" {
+variable "private_subnet_cidr" {
   type = "string"
 }
 
-variable "aws_ops_subnet_name" {
+variable "private_subnet_name" {
   type = "string"
 }
 
@@ -56,5 +56,6 @@ variable "aws_amis" {
   default = {
     us-east-1 = "ami-1d4e7a66"
     eu-central-1 = "ami-958128fa"
+    eu-west-1 = "ami-785db401"
   }
 }

data/tpl/gcp/README.md

@@ -8,15 +8,44 @@ pushd terraform && terraform init && terraform apply && popd
 
 Render bosh deployment
 ```
-kite render bosh --cloud=gcp
+kite render manifest bosh --cloud=gcp
 ```
 
 Setup tunnel
 ```
-./bin/setup-tunnel.sh
+. bin/setup-tunnel.sh
 ```
 
 Install BOSH
 ```
 ./bin/bosh-install.sh
 ```
+
+Connect to the Director
+```
+. bin/set-env.sh
+
+bosh alias-env bosh-1 -e 10.0.0.10 --ca-cert \
+  <(bosh int ./config/creds.yml --path /director_ssl/ca)
+```
+
+Render concourse deployment
+```
+kite render manifest concourse --cloud=gcp
+```
+
+Install concourse
+```
+bosh -e bosh-1 update-cloud-config  deployments/concourse/cloud-config.yml
+
+bosh -e bosh-1 upload-stemcell \
+  https://bosh.io/d/stemcells/bosh-google-kvm-ubuntu-trusty-go_agent?v=3445.7
+
+bosh -e bosh-1 upload-release \
+  https://github.com/concourse/concourse/releases/download/v3.4.1/concourse-3.4.1.tgz
+
+bosh -e bosh-1 upload-release \
+  https://github.com/concourse/concourse/releases/download/v3.4.1/garden-runc-1.6.0.tgz
+
+bosh -e bosh-1 -d concourse deploy deployments/concourse/concourse.yml
+```

data/tpl/gcp/deployments/concourse/cloud-config.yml.erb

@@ -0,0 +1,56 @@
+azs:
+- name: z1
+  cloud_properties:
+    zone: <%= @values['gcp']['zone'] %>
+
+vm_types:
+- name: common
+  cloud_properties:
+    machine_type: n1-standard-2
+    root_disk_size_gb: 20
+    root_disk_type: pd-ssd
+
+- name: worker
+  cloud_properties:
+    machine_type: n1-standard-4
+    root_disk_size_gb: 100
+    root_disk_type: pd-ssd
+
+# vm_extensions:
+# - name: concourse-lb
+#   cloud_properties:
+#     target_pool: concourse-target-pool
+
+compilation:
+  workers: 2
+  network: public
+  reuse_compilation_vms: true
+  az: z1
+  cloud_properties:
+    machine_type: n1-standard-4
+    root_disk_size_gb: 100
+    root_disk_type: pd-ssd
+    preemptible: true
+
+networks:
+  - name: public
+    type: manual
+    subnets:
+    - az: z1
+      range: <%= @values['gcp']['subnet_cidr'] %>
+      gateway: <%= @values['gcp']['internal_gw'] %>
+      static: [10.0.0.2, 10.0.0.10]
+      cloud_properties:
+        network_name: <%= @values['gcp']['vpc_name'] %>
+        subnetwork_name: <%= @values['gcp']['subnet_name'] %>
+        ephemeral_external_ip: true
+        tags:
+          - concourse-public
+          - concourse-internal
+
+  - name: vip
+    type: vip
+
+disk_types:
+- name: database
+  disk_size: 10240

data/tpl/gcp/deployments/concourse/concourse.yml.erb

@@ -0,0 +1,86 @@
+name: concourse
+
+releases:
+- name: concourse
+  version: 3.4.1
+- name: garden-runc
+  version: 1.6.0
+
+instance_groups:
+- name: web
+  instances: 1
+  vm_type: common
+  azs: [z1]
+  # vm_extensions: [concourse-lb]
+  stemcell: trusty
+  networks:
+  - name: public
+    default: [dns, gateway]
+
+  jobs:
+  - name: atc
+    release: concourse
+    properties:
+      bind_port: 80
+      external_url: <%= @values['concourse']['url'] %>
+      basic_auth_username: <%= @values['concourse']['auth_username'] %>
+      basic_auth_password: <%= @values['concourse']['auth_password'] %>
+      publicly_viewable: true
+
+      postgresql_database: &atc_db atc
+
+  - name: tsa
+    release: concourse
+    properties: {}
+
+- name: db
+  instances: 1
+  vm_type: common
+  azs: [z1]
+  stemcell: trusty
+  persistent_disk_type: database
+  networks: [{name: public}]
+  jobs:
+  - name: postgresql
+    release: concourse
+    properties:
+      databases:
+      - name: *atc_db
+        role: admin
+        password: <%= @values['concourse']['db_password'] %>
+
+- name: worker
+  instances: 1
+  vm_type: worker
+  azs: [z1]
+  stemcell: trusty
+  networks: [{name: public}]
+  jobs:
+  - name: groundcrew
+    release: concourse
+    properties:
+      additional_resource_types:
+      - type: gcs-resource
+        image: docker:///frodenas/gcs-resource
+  - name: baggageclaim
+    release: concourse
+    properties: {}
+  - name: garden
+    release: garden-runc
+    properties:
+      garden:
+        listen_network: tcp
+        listen_address: 0.0.0.0:7777
+        network_mtu: 1432
+
+update:
+  canaries: 1
+  max_in_flight: 1
+  serial: false
+  canary_watch_time: 1000-60000
+  update_watch_time: 1000-60000
+
+stemcells:
+- alias: trusty
+  os: ubuntu-trusty
+  version: latest

data/tpl/gcp/set-env.sh.erb

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+BASTION_IP="$(terraform output -state=terraform/terraform.tfstate bastion_ip)"
+
+export BASTION_IP
+export BOSH_ALL_PROXY=socks5://localhost:5000
+export BOSH_CLIENT=admin
+export BOSH_CLIENT_SECRET=`bosh int ./config/creds.yml --path /admin_password`

data/tpl/gcp/setup-tunnel.sh.erb

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
 BASTION_IP="$(terraform output -state=terraform/terraform.tfstate bastion_ip)"
-ssh -D 5000 -fNC kite@$BASTION_IP -i <%= @values['kite']['public_key_path'] %>
+ssh -D 5000 -fNC kite@$BASTION_IP -i <%= @values['kite']['private_key_path'] %>
 
 export BOSH_ALL_PROXY=socks5://localhost:5000

data/tpl/gcp/terraform/network.tf

@@ -39,3 +39,14 @@ resource "google_compute_firewall" "platform_internal" {
   target_tags = ["platform-internal"]
   source_tags = ["platform-internal"]
 }
+
+# Allow concourse
+resource "google_compute_firewall" "allow_concourse" {
+  name    = "allow-concourse"
+  network = "${google_compute_network.platform.name}"
+
+  allow {
+    protocol = "all"
+  }
+
+}

data/tpl/skel/config/cloud.yml

@@ -7,31 +7,34 @@ aws:
   access_key: "enter your amazon key"
   secret_key: "enter your secret key"
   region: "eu-central-1"
-  az: "eu-central-1a"
-  vpc_cidr_block: "10.0.0.0/16"
+  zone: "eu-central-1a"
   vpc_name: "platform-tools"
-  platform_subnet_cidr_block: "10.0.0.0/24"
-  platform_subnet_name: "platform_net"
-  ops_subnet_cidr_block: "10.0.10.0/24"
-  ops_subnet_name: "ops_services"
+  vpc_cidr_block: "10.0.0.0/16"
+  public_subnet:
+    name: "platform-dmz"
+    network: "10.0.10.0/26"
+  private_subnet:
+    name: "platform-net"
+    gateway: "10.0.20.1"
+    network: "10.0.20.0/26"
 
 gcp:
-  project: gcp-project
-  region: europe-west1
-  zone: europe-west1-b
+  project: "gcp-project"
+  region: "europe-west1"
+  zone: "europe-west1-b"
   service_account: "~/safe/terraform.json"
   vpc_name: "platform-tools"
   subnet_name: "platform-net"
   subnet_cidr: "10.0.0.0/24"
-  internal_gw: "10.0.0.1"
+  internal_gw: "10.0.0.2"
 
 bosh:
   name: "bosh-director"
-  static_ip: "10.0.0.10"
+  static_ip: "10.0.20.10"
 
 concourse:
-  hostname: "ci.domain.io"
-  url: "http://ci.example.com"
+  hostname: "concourse.example.com"
+  url: "http://concourse.example.com"
   auth_username: "concourse"
   auth_password: "concourse"
-  db_password: "changeme"
+  db_password: "concourse"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kite
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Louis Bellet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-13 00:00:00.000000000 Z
+date: 2017-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -79,7 +79,9 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- Dockerfile
 - Gemfile
+- Makefile
 - README.md
 - Rakefile
 - bin/kite
@@ -92,15 +94,18 @@ files:
 - lib/kite/core.rb
 - lib/kite/error.rb
 - lib/kite/helpers.rb
+- lib/kite/render.rb
 - lib/kite/version.rb
 - tpl/aws/README.md
 - tpl/aws/bosh-install.sh.erb
-- tpl/aws/concourse/aws_cloud.yml.erb
-- tpl/aws/concourse/concourse.yml.erb
-- tpl/aws/deployments/bosh/bosh_director.yml
-- tpl/aws/deployments/bosh/bosh_vars.yml.erb
+- tpl/aws/concourse-deploy.sh.erb
+- tpl/aws/deployments/bosh/bosh.yml
+- tpl/aws/deployments/bosh/bosh_vars.yml.tt
 - tpl/aws/deployments/bosh/cpi.yml
 - tpl/aws/deployments/bosh/jumpbox-user.yml
+- tpl/aws/deployments/concourse/cloud-config.yml.erb
+- tpl/aws/deployments/concourse/concourse.yml.erb
+- tpl/aws/set-env.sh.erb
 - tpl/aws/setup-tunnel.sh.erb
 - tpl/aws/terraform/main.tf
 - tpl/aws/terraform/network.tf
@@ -114,6 +119,9 @@ files:
 - tpl/gcp/deployments/bosh/cloud-config.yml
 - tpl/gcp/deployments/bosh/cpi.yml
 - tpl/gcp/deployments/bosh/jumpbox-user.yml
+- tpl/gcp/deployments/concourse/cloud-config.yml.erb
+- tpl/gcp/deployments/concourse/concourse.yml.erb
+- tpl/gcp/set-env.sh.erb
 - tpl/gcp/setup-tunnel.sh.erb
 - tpl/gcp/terraform/main.tf
 - tpl/gcp/terraform/network.tf