kite 0.0.8 → 0.0.9

data/tpl/gcp/bin/base/cleanup.sh.tt

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Remove existing BOSH deployments
+bosh -e <%= @values['bosh']['name'] %> -d vault deld
+
+# Destroy BOSH director
+bosh delete-env deployments/bosh/bosh.yml \
+  --state=config/state.json \
+  --vars-store=config/creds.yml \
+  --vars-file=config/bosh-vars.yml \
+  --var-file gcp_credentials_json=<%= @values['gcp']['service_account'] %> \
+  -v tags='[platform-internal, no-ip]' \
+  -o deployments/bosh/cpi.yml \
+  -o deployments/bosh/jumpbox-user.yml
+
+# Destroy Terraform-generated infrastructure
+pushd terraform && terraform destroy && popd
+
+# Remove files generated by kite
+rm -rf terraform deployments config/{creds.yml,bosh_vars.yml,jumpbox.key} bin/*.sh

data/tpl/gcp/{set-env.sh.erb → bin/base/set-env.sh.tt}

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
-BASTION_IP="$(terraform output -state=terraform/terraform.tfstate bastion_ip)"
+pushd terraform
+BASTION_IP="$(terraform output bastion_ip)"
+popd
 
 export BASTION_IP
 export BOSH_ALL_PROXY=socks5://localhost:5000

data/tpl/gcp/{setup-tunnel.sh.erb → bin/base/setup-tunnel.sh.tt}

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
+pushd terraform
+BASTION_IP="$(terraform output bastion_ip)"
+popd
 
-BASTION_IP="$(terraform output -state=terraform/terraform.tfstate bastion_ip)"
 ssh -D 5000 -fNC kite@$BASTION_IP -i <%= @values['kite']['private_key_path'] %>
 
 export BOSH_ALL_PROXY=socks5://localhost:5000

data/tpl/gcp/{bosh-install.sh.erb → bin/bosh-install.sh.tt}

@@ -5,7 +5,7 @@ set -xe
 bosh create-env deployments/bosh/bosh.yml \
   --state=config/state.json \
   --vars-store=config/creds.yml \
-  --vars-file=bosh-vars.yml \
+  --vars-file=config/bosh-vars.yml \
   --var-file gcp_credentials_json=<%= @values['gcp']['service_account'] %> \
   -v tags='[platform-internal, no-ip]' \
   -o deployments/bosh/cpi.yml \
@@ -14,4 +14,5 @@ bosh create-env deployments/bosh/bosh.yml \
 bosh alias-env <%= @values['bosh']['name'] %> -e <%= @values['bosh']['static_ip'] %> --ca-cert \
   <(bosh int ./config/creds.yml --path /director_ssl/ca)
 
-bosh -e <%= @values['bosh']['name'] %> ucc deployments/bosh/cloud_config.yml
+echo "Please run"
+echo bosh -e <%= @values['bosh']['name'] %> ucc deployments/bosh/cloud-config.yml

data/tpl/gcp/bin/concourse-deploy.sh.tt

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Upload necessary stemcells and releases
+bosh -e <%= @values['bosh']['name'] %> upload-stemcell https://bosh.io/d/stemcells/bosh-google-kvm-ubuntu-trusty-go_agent
+bosh -e <%= @values['bosh']['name'] %> upload-release https://github.com/concourse/concourse/releases/download/v3.4.1/concourse-3.4.1.tgz
+bosh -e <%= @values['bosh']['name'] %> upload-release https://github.com/concourse/concourse/releases/download/v3.4.1/garden-runc-1.6.0.tgz
+
+# Deploy Concourse
+bosh -e <%= @values['bosh']['name'] %> -d concourse deploy deployments/concourse/concourse.yml

data/tpl/gcp/{vault-deploy.sh.erb → bin/vault-deploy.sh.tt}

@@ -3,7 +3,7 @@
 set -xe
 
 # Render Concourse-related files
-kite render manifest concourse --cloud aws
+kite render manifest vault --cloud gcp
 
 # Upload necessary stemcells and releases
 bosh -e <%= @values['bosh']['name'] %> upload-stemcell https://s3.amazonaws.com/bosh-core-stemcells/google/bosh-stemcell-3445.7-google-kvm-ubuntu-trusty-go_agent.tgz

data/tpl/gcp/deployments/bosh/cloud-config.yml.tt

@@ -39,11 +39,12 @@ networks:
     - az: z1
       range: <%= @values['gcp']['subnet_cidr'] %>
       gateway: <%= @values['gcp']['internal_gw'] %>
-      reserved: [<%= ip_range(@private_subnet, (1..10)) %>]
+      reserved: [<%= ip_range(@private_subnet, (1..10)) %>] # Reserved range for the gateway, BOSH Director etc
+      static: [<%= ip_range(@private_subnet, (11..13)) %>] # Static IP range for Vault, Concourse web panel, nginx etc
       cloud_properties:
         network_name: <%= @values['gcp']['vpc_name'] %>
         subnetwork_name: <%= @values['gcp']['subnet_name'] %>
-        ephemeral_external_ip: true
+        ephemeral_external_ip: false
         tags:
           - concourse-public
           - concourse-internal

data/tpl/gcp/deployments/concourse/{concourse.yml.erb → concourse.yml.tt}

@@ -15,6 +15,7 @@ instance_groups:
   stemcell: trusty
   networks:
   - name: public
+    static_ips: [<%= @private_subnet[12] %>]
     default: [dns, gateway]
 
   jobs:
@@ -27,6 +28,13 @@ instance_groups:
       basic_auth_password: <%= @values['concourse']['auth_password'] %>
       publicly_viewable: true
 
+      vault:
+        auth:
+          backend: token
+          client_token: "your Vault root token here"
+        path_prefix: /concourse
+        url: "http://<%= @private_subnet[11] %>:8200" # expecting Vault to be deployed first
+
       postgresql_database: &atc_db atc
 
   - name: tsa

data/tpl/gcp/deployments/concourse/test/test-credentials.yml

@@ -0,0 +1,3 @@
+dockerhub-email: "vshatravenko@heliostech.fr"
+dockerhub-repo:  "valshatravenko/piwik"
+git-source-uri:  "https://github.com/vshatravenko/piwik"

data/tpl/gcp/deployments/concourse/test/test-pipeline.yml

@@ -0,0 +1,24 @@
+---
+resources:
+  - name: test-image
+    type: docker-image
+    source:
+      email:      {{dockerhub-email}}
+      username:   ((dockerhub_username))
+      password:   ((dockerhub_password))
+      repository: {{dockerhub-repo}}
+  - name: test-src
+    type: git
+    source:
+      uri: {{git-source-uri}}
+
+jobs:
+  - name: test-publish
+    public: true
+    serial: true
+    plan:
+      - get: test-src
+        trigger: true
+      - put: test-image
+        params:
+          build: test-src

data/tpl/gcp/deployments/nginx/nginx.yml.erb

@@ -0,0 +1,62 @@
+---
+name: nginx
+
+releases:
+- name: nginx
+  version: latest
+
+
+instance_groups:
+- name: nginx
+  instances: 1
+  vm_type: default
+  azs: [z1]
+  stemcell: trusty
+  networks:
+  - name: public
+    static_ips: [<%= @private_subnet[13] %>]
+    default: [dns, gateway]
+
+  jobs:
+  - name: nginx
+    release: nginx
+    properties:
+      nginx_conf: |
+        worker_processes  1;
+        error_log /var/vcap/sys/log/nginx/error.log   info;
+        events {
+          worker_connections  1024;
+        }
+
+        http {
+          include /var/vcap/packages/nginx/conf/mime.types;
+          default_type  application/octet-stream;
+          sendfile        on;
+          keepalive_timeout  65;
+          server_names_hash_bucket_size 64;
+          server {
+            server_name kite-nginx;
+
+            location / {
+              proxy_pass http://<%= @private_subnet[13] %>; # Concourse web panel IP
+              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+             proxy_set_header Host $http_host;
+             proxy_redirect off;
+            }
+
+            access_log /var/vcap/sys/log/nginx/kite-access.log;
+            error_log /var/vcap/sys/log/nginx/kite-error.log;
+          }
+        }
+
+stemcells:
+- alias: trusty
+  os: ubuntu-trusty
+  version: latest
+
+update:
+  canaries: 1
+  max_in_flight: 1
+  serial: false
+  canary_watch_time: 1000-60000
+  update_watch_time: 1000-60000

data/tpl/gcp/deployments/vault/{vault.yml → vault.yml.erb}

@@ -13,7 +13,7 @@ instance_groups:
   stemcell: trusty
   networks:
   - name: public
-
+    static_ips: [<%= @private_subnet[11] %>]
   jobs:
   - name: vault
     release: vault

data/tpl/gcp/docs/bosh.md

@@ -0,0 +1,31 @@
+#### [Back](../README.md)
+
+## BOSH
+
+### Prerequisites
+
+- Terraform IaC applied
+- [BOSH CLI v2](https://bosh.io/docs/cli-v2.html#install) installed
+
+### Setup
+
+Render bosh deployment
+```
+kite render manifest bosh --cloud=gcp
+```
+
+Setup tunnel
+```
+. bin/setup-tunnel.sh
+```
+
+Install BOSH
+```
+./bin/bosh-install.sh
+```
+
+Connect to the Director
+```
+. bin/set-env.sh
+
+```

data/tpl/gcp/docs/concourse.md

@@ -0,0 +1,30 @@
+#### [Back](../README.md)
+
+## Concourse
+
+### Prerequisites
+
+- Vault [deployed and initialized](vault.md)
+
+### Setup
+
+Fill out the "token" field in `deployments/concourse/concourse.yml` with root token received from `vault init`.
+
+Deploy Concourse
+```
+./bin/concourse-deploy.sh
+```
+
+### Test
+
+To run a test Concourse job:
+
+- Go to test folder: `cd deployments/concourse/test`
+- Fill out `test-credentials.yml`
+- Add necessary secrets to your Vault(see [docs/vault.md](docs/vault.md))
+- Download the `fly` client from Concourse web panel and add it to your PATH: `mv *path_to_fly* /usr/local/bin`
+- Login to Concourse using the `fly` client: `fly -t ci --concourse-url *concourse-url*`
+- Create a test pipeline with `fly set-pipeline -t ci -c test-pipeline.yml -p test --load-vars-from test-credentials.yml -n`
+- Unpause pipeline: `fly unpause-pipeline -t ci -p test`
+- Trigger and unpause the test job: `fly trigger-job -t ci -j test/test-publish`
+- See the results on Concourse web panel or use: `fly watch -p test -j test/test-publish`

data/tpl/gcp/docs/vault.md

@@ -0,0 +1,33 @@
+#### [Back](../README.md)
+
+## Vault
+
+### Prerequisites
+
+Before using Vault, you should have the client installed:
+
+- Download the binary for your OS
+- Unzip it and run `chmod +x vault && sudo mv vault /usr/local/bin/vault`
+- Check if the Vault is installed by running `vault -v`
+
+### Deployment
+
+To deploy Vault, use `./bin/vault-deploy.sh`
+
+### Connection
+
+- Export your Vault's IP using `export VAULT_ADDR=*vault_ip*`
+- Run `vault init` to initialize the vault
+- Store the keys displayed after init
+- Unseal the vault by running `vault unseal` three times using three keys from the previous step
+- Authenticate to the vault with `vault auth` using the root token you got from `vault init`
+
+[Optional]
+- Try to store a dummy secret: `vault write secret/handshake knock=knock`
+- Read it: `vault read secret/handshake`
+
+### Usage with Concourse
+
+Before using Vault with Concourse you should mount a secrets backend with `vault mount -path=concourse kv`
+
+To add new secrets accessible for Concourse use `vault write concourse/main/*secret_name* value="*secret_value*"`

data/tpl/skel/config/cloud.yml

@@ -11,9 +11,11 @@ aws:
   zone: "eu-central-1a"
   vpc_name: "platform-tools"
   vpc_cidr_block: "10.0.0.0/16"
+  vpc_id: "" # submit vpc id if you want to use an existing vpc
   public_subnet:
     name: "platform-dmz"
     network: "10.0.10.0/26"
+    id: "" # submit id if you want to use an existing public subnet
   private_subnet:
     name: "platform-net"
     gateway: "10.0.20.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kite
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Louis Bellet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-22 00:00:00.000000000 Z
+date: 2017-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -66,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Kite is a bootstraping tool for your cloud provider and long term administration.
 email:
 - lbellet@heliostech.fr
@@ -94,45 +122,60 @@ files:
 - lib/kite/cloud.rb
 - lib/kite/core.rb
 - lib/kite/error.rb
+- lib/kite/generate.rb
 - lib/kite/helpers.rb
 - lib/kite/render.rb
 - lib/kite/version.rb
 - tpl/aws/README.md
-- tpl/aws/bosh-install.sh.erb
-- tpl/aws/concourse-deploy.sh.erb
+- tpl/aws/bin/base/bootstrap.sh
+- tpl/aws/bin/base/cleanup.sh.tt
+- tpl/aws/bin/base/set-env.sh.tt
+- tpl/aws/bin/base/setup-tunnel.sh.tt
+- tpl/aws/bin/bosh-install.sh.tt
+- tpl/aws/bin/concourse-deploy.sh.tt
+- tpl/aws/bin/vault-deploy.sh.tt
+- tpl/aws/bosh-vars.yml.erb
 - tpl/aws/deployments/bosh/bosh.yml
-- tpl/aws/deployments/bosh/bosh_vars.yml.tt
 - tpl/aws/deployments/bosh/cloud-config.yml.tt
 - tpl/aws/deployments/bosh/cpi.yml
 - tpl/aws/deployments/bosh/jumpbox-user.yml
-- tpl/aws/deployments/concourse/concourse.yml.erb
-- tpl/aws/set-env.sh.erb
-- tpl/aws/setup-tunnel.sh.erb
-- tpl/aws/terraform/main.tf
-- tpl/aws/terraform/network.tf
+- tpl/aws/deployments/concourse/concourse.yml.tt
+- tpl/aws/deployments/vault/vault.yml.erb
+- tpl/aws/docs/bosh.md
+- tpl/aws/docs/concourse.md
+- tpl/aws/docs/vault.md
+- tpl/aws/terraform/main.tf.tt
+- tpl/aws/terraform/network.tf.tt
 - tpl/aws/terraform/outputs.tf
-- tpl/aws/terraform/terraform.tfvars.erb
+- tpl/aws/terraform/terraform.tfvars.tt
 - tpl/aws/terraform/variables.tf
 - tpl/gcp/README.md
-- tpl/gcp/bosh-install.sh.erb
+- tpl/gcp/bin/base/bootstrap.sh
+- tpl/gcp/bin/base/cleanup.sh.tt
+- tpl/gcp/bin/base/set-env.sh.tt
+- tpl/gcp/bin/base/setup-tunnel.sh.tt
+- tpl/gcp/bin/bosh-install.sh.tt
+- tpl/gcp/bin/concourse-deploy.sh.tt
+- tpl/gcp/bin/vault-deploy.sh.tt
 - tpl/gcp/bosh-vars.yml.erb
-- tpl/gcp/concourse-deploy.sh.erb
 - tpl/gcp/deployments/bosh/bosh.yml
 - tpl/gcp/deployments/bosh/cloud-config.yml.tt
 - tpl/gcp/deployments/bosh/cpi.yml
 - tpl/gcp/deployments/bosh/jumpbox-user.yml
-- tpl/gcp/deployments/concourse/concourse.yml.erb
-- tpl/gcp/deployments/vault/vault.yml
-- tpl/gcp/set-env.sh.erb
-- tpl/gcp/setup-tunnel.sh.erb
+- tpl/gcp/deployments/concourse/concourse.yml.tt
+- tpl/gcp/deployments/concourse/test/test-credentials.yml
+- tpl/gcp/deployments/concourse/test/test-pipeline.yml
+- tpl/gcp/deployments/nginx/nginx.yml.erb
+- tpl/gcp/deployments/vault/vault.yml.erb
+- tpl/gcp/docs/bosh.md
+- tpl/gcp/docs/concourse.md
+- tpl/gcp/docs/vault.md
 - tpl/gcp/terraform/gcs.tf.tt
 - tpl/gcp/terraform/main.tf
 - tpl/gcp/terraform/network.tf
 - tpl/gcp/terraform/outputs.tf
 - tpl/gcp/terraform/terraform.tfvars.tt
 - tpl/gcp/terraform/variables.tf
-- tpl/gcp/vault-deploy.sh.erb
-- tpl/gcp/vault.md
 - tpl/skel/Gemfile.tt
 - tpl/skel/README.md.tt
 - tpl/skel/bin/kite