kite 0.1.0 → 0.2.0

data/tpl/gcp/deployments/prometheus/monitor-kubernetes.yml

@@ -0,0 +1,30 @@
+# This file assumes bosh_exporter based Service Discovery is being used: ./monitor-bosh.yml
+
+# Exporter jobs
+- type: replace
+  path: /instance_groups/name=prometheus/jobs/-
+  value:
+    name: kube_state_metrics_exporter
+    release: prometheus
+    properties:
+      kube_state_metrics_exporter:
+        apiserver: "((kubernetes_apiserver))"
+        kubeconfig: ((kubernetes_kubeconfig))
+
+# Prometheus Alerts
+- type: replace
+  path: /instance_groups/name=prometheus/jobs/name=kubernetes_alerts?/release
+  value: prometheus
+
+- type: replace
+  path: /instance_groups/name=prometheus/jobs/name=prometheus/properties/prometheus/rule_files/-
+  value: /var/vcap/jobs/kubernetes_alerts/*.alerts
+
+# Grafana Dashboards
+- type: replace
+  path: /instance_groups/name=grafana/jobs/name=kubernetes_dashboards?/release
+  value: prometheus
+
+- type: replace
+  path: /instance_groups/name=grafana/jobs/name=grafana/properties/grafana/prometheus/dashboard_files/-
+  value: /var/vcap/jobs/kubernetes_dashboards/*.json

data/tpl/gcp/deployments/prometheus/prometheus.yml

@@ -0,0 +1,183 @@
+name: prometheus
+
+instance_groups:
+  - name: alertmanager
+    azs:
+      - z1
+    instances: 1
+    vm_type: common
+    persistent_disk_type: common
+    stemcell: default
+    networks:
+      - name: public
+        static_ips: [<%= @private_subnet[14] %>]
+    jobs:
+      - name: alertmanager
+        release: prometheus
+        properties:
+          alertmanager:
+            mesh:
+              password: ((alertmanager_mesh_password))
+            route:
+              receiver: default
+            receivers:
+              - name: default
+            test_alert:
+              daily: true
+
+  - name: prometheus
+    azs:
+      - z1
+    instances: 1
+    vm_type: common
+    persistent_disk_type: database
+    stemcell: default
+    networks:
+      - name: public
+        static_ips: [<%= @private_subnet[16] %>]
+    jobs:
+      - name: prometheus
+        release: prometheus
+        properties:
+          prometheus:
+            rule_files:
+              - /var/vcap/jobs/postgres_alerts/*.alerts
+              - /var/vcap/jobs/prometheus_alerts/*.alerts
+            scrape_configs:
+              - job_name: prometheus
+                static_configs:
+                - targets:
+                  - localhost:9090
+      - name: postgres_alerts
+        release: prometheus
+      - name: prometheus_alerts
+        release: prometheus
+
+  - name: database
+    azs:
+      - z1
+    instances: 1
+    vm_type: common
+    persistent_disk_type: database
+    stemcell: default
+    networks:
+      - name: public
+    jobs:
+      - name: postgres
+        release: postgres
+        properties:
+          databases:
+            port: 5432
+            databases:
+              - name: grafana
+                citext: true
+            roles:
+              - name: grafana
+                password: ((postgres_grafana_password))
+      - name: postgres_exporter
+        release: prometheus
+        properties:
+          postgres_exporter:
+            datasource_name: postgresql://grafana:((postgres_grafana_password))@127.0.0.1:5432/?sslmode=disable
+
+  - name: grafana
+    azs:
+      - z1
+    instances: 1
+    vm_type: common
+    persistent_disk_type: database
+    stemcell: default
+    networks:
+      - name: public
+        static_ips: [<%= @private_subnet[15] %>]
+    jobs:
+      - name: grafana
+        release: prometheus
+        properties:
+          grafana:
+            database:
+              type: postgres
+              port: 5432
+              name: grafana
+              user: grafana
+              password: ((postgres_grafana_password))
+            session:
+              provider: postgres
+              provider_port: 5432
+              provider_name: grafana
+              provider_user: grafana
+              provider_password: ((postgres_grafana_password))
+            security:
+              admin_user: admin
+              admin_password: ((grafana_password))
+              secret_key: ((grafana_secret_key))
+            dashboards:
+              json:
+                enabled: true
+            prometheus:
+              dashboard_files:
+                - /var/vcap/jobs/grafana_dashboards/*.json
+                - /var/vcap/jobs/postgres_dashboards/*.json
+                - /var/vcap/jobs/prometheus_dashboards/*.json
+      - name: grafana_dashboards
+        release: prometheus
+      - name: postgres_dashboards
+        release: prometheus
+      - name: prometheus_dashboards
+        release: prometheus
+
+  - name: nginx
+    azs:
+      - z1
+    instances: 1
+    vm_type: common
+    stemcell: default
+    networks:
+      - name: public
+    jobs:
+      - name: nginx
+        release: prometheus
+        properties:
+          nginx:
+            alertmanager:
+              auth_username: admin
+              auth_password: ((alertmanager_password))
+            prometheus:
+              auth_username: admin
+              auth_password: ((prometheus_password))
+
+variables:
+  - name: alertmanager_password
+    type: password
+  - name: alertmanager_mesh_password
+    type: password
+  - name: prometheus_password
+    type: password
+  - name: postgres_grafana_password
+    type: password
+  - name: grafana_password
+    type: password
+  - name: grafana_secret_key
+    type: password
+
+update:
+  canaries: 1
+  max_in_flight: 32
+  canary_watch_time: 1000-100000
+  update_watch_time: 1000-100000
+  serial: false
+
+stemcells:
+  - alias: default
+    os: ubuntu-trusty
+    version: latest
+
+releases:
+- name: postgres
+  version: "20"
+  url: https://bosh.io/d/github.com/cloudfoundry/postgres-release?v=20
+  sha1: 3f378bcab294e20316171d4e656636df88763664
+- name: prometheus
+  version: 18.6.2
+  url: https://github.com/cloudfoundry-community/prometheus-boshrelease/releases/download/v18.6.2/prometheus-18.6.2.tgz
+  sha1: f6b7ed381a28ce8fef99017a89e1122b718d5556

data/tpl/gcp/docs/bosh.md

@@ -29,3 +29,8 @@ Connect to the Director
 . bin/set-env.sh
 
 ```
+
+Update the cloud configuration
+```
+bosh -e bosh-director ucc deployments/bosh/cloud-config.yml
+```

data/tpl/gcp/docs/concourse.md

@@ -10,9 +10,9 @@
 
 Fill out the "token" field in `deployments/concourse/concourse.yml` with root token received from `vault init`.
 
-Deploy Concourse by running the script with the required arguments
+Deploy Concourse by running the script with the Vault token as argument(strong passwords for Concourse auth and db will be generated automatically)
 ```
-./bin/concourse-deploy.sh *concourse_auth_password* *concourse_db_password* *vault_token*
+./bin/concourse-deploy.sh *vault_token*
 ```
 
 ### Connect GitHub oAuth
@@ -34,7 +34,7 @@ To run a test Concourse job:
 - Fill out `test-credentials.yml`
 - Add necessary secrets to your Vault(see [docs/vault.md](docs/vault.md))
 - Download the `fly` client from Concourse web panel and add it to your PATH: `mv *path_to_fly* /usr/local/bin`
-- Login to Concourse using the `fly` client: `fly -t ci --concourse-url *concourse-url*`
+- Login to Concourse using the `fly` client: `fly login -t ci --concourse-url *concourse-url*`
 - Create a test pipeline with `fly set-pipeline -t ci -c test-pipeline.yml -p test --load-vars-from test-credentials.yml -n`
 - Unpause pipeline: `fly unpause-pipeline -t ci -p test`
 - Trigger and unpause the test job: `fly trigger-job -t ci -j test/test-publish`

data/tpl/gcp/docs/ingress.md

@@ -0,0 +1,12 @@
+#### [Back](../README.md)
+
+## Ingress
+
+### Prerequisites
+
+- BOSH environment [ready](bosh.md)
+- All hostnames resolve to the VIP configured in cloud.yml (this is mandatory to issue SSL certificates)
+
+### Deployment
+
+To deploy Ingress, use `./bin/ingress-deploy.sh`

data/tpl/gcp/docs/oauth.md

@@ -0,0 +1,24 @@
+#### [Back](../README.md)
+
+## OAuth (UAA)
+
+### Configuration
+
+If you want to add initial groups and users, change oauth look,
+configure mail, etc. - you should edit `config/oauth.yml`.
+
+Here are links to uaa config documentation:
+
+* __users:__ [uaa.scim.users](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=uaa.scim.users)
+* __groups:__ [uaa.scim.groups](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=uaa.scim.groups)
+* __oauth clients:__ [uaa.clients](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=uaa.clients)
+* __theming:__ [login.branding](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=login.branding)
+* __email notifications:__ [login.smtp](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=login.smtp)
+
+### Deployment
+
+After editing config, run `./bin/oauth-deploy.sh`
+
+### Usage
+
+To check if OAuth works, visit [<%= @values['oauth']['hostname'] %>](<%= @values['oauth']['url'] %>).

data/tpl/gcp/docs/prometheus.md

@@ -0,0 +1,27 @@
+#### [Back](../README.md)
+
+## Prometheus
+
+### Prerequisites
+
+- BOSH environment [ready](bosh.md)
+- Kops cluster [deployed](kops.md)
+
+### Setup
+
+Enter path to your Kubernetes config in `config/cloud.yml` and add the Kubernetes API server address to `config/bosh_vars.yml`.
+
+Afterwards, deploy Prometheus
+```
+./bin/prometheus-deploy.sh
+```
+
+### Access
+
+After the deployment process is done, you can reach each Prometheus' component's web UI at:
+
+- AlertManager: http://10.0.0.14:9093
+- Grafana:      http://10.0.0.15:3000
+- Prometheus:   http://10.0.0.16:9090
+
+You can find related credentials in `config/creds.yml`

data/tpl/gcp/docs/vault.md

@@ -16,7 +16,8 @@ To deploy Vault, use `./bin/vault-deploy.sh`
 
 ### Connection
 
-- Export your Vault's IP using `export VAULT_ADDR=http://*vault_ip*:8200`
+- You can now deploy the ingress to access vault
+- Export your Vault's address using `export VAULT_ADDR=https://*vault_host*`
 - Run `vault init` to initialize the vault
 - Store the keys displayed after init
 - Unseal the vault by running `vault unseal` three times using three keys from the previous step

data/tpl/gcp/terraform/main.tf

@@ -29,7 +29,7 @@ resource "google_compute_address" "bastion" {
 
 resource "google_compute_instance" "bastion" {
   name         = "bastion"
-  machine_type = "n1-standard-1"
+  machine_type = "g1-small"
   zone         = "${var.zone}"
 
   tags = ["bastion", "platform-internal"]
@@ -63,3 +63,8 @@ EOT
     scopes = ["userinfo-email", "compute-ro", "storage-ro"]
   }
 }
+
+# Ingress
+resource "google_compute_address" "ingress" {
+  name = "ingress-ip"
+}

data/tpl/gcp/terraform/outputs.tf

@@ -1,3 +1,7 @@
 output "bastion_ip" {
   value = "${google_compute_address.bastion.address}"
 }
+
+output "ingress_ip" {
+  value = "${google_compute_address.ingress.address}"
+}

data/tpl/service/%output_path%/charts/%app_name%/Chart.yaml.tt

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: A <%= @title %> chart
+name: <%= @name %>
+version: <%= @chart_version %>

data/tpl/service/%output_path%/charts/%app_name%/templates/NOTES.txt.tt

@@ -0,0 +1,19 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range .Values.ingress.hosts }}
+  http://{{ . }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get svc -w {{ template "fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.externalPort }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }}
+{{- end }}

data/tpl/service/%output_path%/charts/%app_name%/templates/_helpers.tpl

@@ -0,0 +1,16 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "fullname" -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

data/tpl/service/%output_path%/charts/%app_name%/templates/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ template "fullname" . }}
+  labels:
+    app: {{ template "name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - containerPort: {{ .Values.service.internalPort }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.service.internalPort }}
+          readinessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.service.internalPort }}
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+    {{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+    {{- end }}

data/tpl/service/%output_path%/charts/%app_name%/templates/ingress.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.ingress.enabled -}}
+{{- $serviceName := include "fullname" . -}}
+{{- $servicePort := .Values.service.externalPort -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ template "fullname" . }}
+  labels:
+    app: {{ template "name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    {{- range $key, $value := .Values.ingress.annotations }}
+      {{ $key }}: {{ $value | quote }}
+    {{- end }}
+spec:
+  rules:
+    {{- range $host := .Values.ingress.hosts }}
+    - host: {{ $host }}
+      http:
+        paths:
+          - path: /
+            backend:
+              serviceName: {{ $serviceName }}
+              servicePort: {{ $servicePort }}
+    {{- end -}}
+  {{- if .Values.ingress.tls }}
+  tls:
+{{ toYaml .Values.ingress.tls | indent 4 }}
+  {{- end -}}
+{{- end -}}