kite 0.1.0 → 0.2.0

data/tpl/aws/deployments/prometheus/monitor-kubernetes.yml

@@ -0,0 +1,30 @@
+# This file assumes bosh_exporter based Service Discovery is being used: ./monitor-bosh.yml
+
+# Exporter jobs
+- type: replace
+  path: /instance_groups/name=prometheus/jobs/-
+  value:
+    name: kube_state_metrics_exporter
+    release: prometheus
+    properties:
+      kube_state_metrics_exporter:
+        apiserver: "((kubernetes_apiserver))"
+        kubeconfig: ((kubernetes_kubeconfig))
+
+# Prometheus Alerts
+- type: replace
+  path: /instance_groups/name=prometheus/jobs/name=kubernetes_alerts?/release
+  value: prometheus
+
+- type: replace
+  path: /instance_groups/name=prometheus/jobs/name=prometheus/properties/prometheus/rule_files/-
+  value: /var/vcap/jobs/kubernetes_alerts/*.alerts
+
+# Grafana Dashboards
+- type: replace
+  path: /instance_groups/name=grafana/jobs/name=kubernetes_dashboards?/release
+  value: prometheus
+
+- type: replace
+  path: /instance_groups/name=grafana/jobs/name=grafana/properties/grafana/prometheus/dashboard_files/-
+  value: /var/vcap/jobs/kubernetes_dashboards/*.json

data/tpl/aws/deployments/prometheus/prometheus.yml.tt

@@ -0,0 +1,184 @@
+name: prometheus
+
+instance_groups:
+  - name: alertmanager
+    azs:
+      - z1
+    instances: 1
+    vm_type: default
+    persistent_disk_type: default
+    stemcell: default
+    networks:
+      - name: platform_net
+        static_ips: [<%= @private_subnet[15] %>]
+    jobs:
+      - name: alertmanager
+        release: prometheus
+        properties:
+          alertmanager:
+            mesh:
+              password: ((alertmanager_mesh_password))
+            route:
+              receiver: default
+            receivers:
+              - name: default
+            test_alert:
+              daily: true
+
+  - name: prometheus
+    azs:
+      - z1
+    instances: 1
+    vm_type: default
+    persistent_disk_type: default
+    stemcell: default
+    networks:
+      - name: platform_net
+        static_ips: [<%= @private_subnet[16] %>]
+    jobs:
+      - name: prometheus
+        release: prometheus
+        properties:
+          prometheus:
+            rule_files:
+              - /var/vcap/jobs/postgres_alerts/*.alerts
+              - /var/vcap/jobs/prometheus_alerts/*.alerts
+            scrape_configs:
+              - job_name: prometheus
+                static_configs:
+                - targets:
+                  - localhost:9090
+      - name: postgres_alerts
+        release: prometheus
+      - name: prometheus_alerts
+        release: prometheus
+
+  - name: database
+    azs:
+      - z1
+    instances: 1
+    vm_type: default
+    persistent_disk_type: default
+    stemcell: default
+    networks:
+      - name: platform_net
+    jobs:
+      - name: postgres
+        release: postgres
+        properties:
+          databases:
+            port: 5432
+            databases:
+              - name: grafana
+                citext: true
+            roles:
+              - name: grafana
+                password: ((postgres_grafana_password))
+      - name: postgres_exporter
+        release: prometheus
+        properties:
+          postgres_exporter:
+            datasource_name: postgresql://grafana:((postgres_grafana_password))@127.0.0.1:5432/?sslmode=disable
+
+  - name: grafana
+    azs:
+      - z1
+    instances: 1
+    vm_type: default
+    persistent_disk_type: default
+    stemcell: default
+    networks:
+      - name: platform_net
+        static_ips: [<%= @private_subnet[17] %>]
+    jobs:
+      - name: grafana
+        release: prometheus
+        properties:
+          grafana:
+            database:
+              type: postgres
+              port: 5432
+              name: grafana
+              user: grafana
+              password: ((postgres_grafana_password))
+            session:
+              provider: postgres
+              provider_port: 5432
+              provider_name: grafana
+              provider_user: grafana
+              provider_password: ((postgres_grafana_password))
+            security:
+              admin_user: admin
+              admin_password: ((grafana_password))
+              secret_key: ((grafana_secret_key))
+            dashboards:
+              json:
+                enabled: true
+            prometheus:
+              dashboard_files:
+                - /var/vcap/jobs/grafana_dashboards/*.json
+                - /var/vcap/jobs/postgres_dashboards/*.json
+                - /var/vcap/jobs/prometheus_dashboards/*.json
+      - name: grafana_dashboards
+        release: prometheus
+      - name: postgres_dashboards
+        release: prometheus
+      - name: prometheus_dashboards
+        release: prometheus
+
+  - name: nginx
+    azs:
+      - z1
+    instances: 1
+    vm_type: default
+    stemcell: default
+    networks:
+      - name: platform_net
+        static_ips: [<%= @private_subnet[18] %>]
+    jobs:
+      - name: nginx
+        release: prometheus
+        properties:
+          nginx:
+            alertmanager:
+              auth_username: admin
+              auth_password: ((alertmanager_password))
+            prometheus:
+              auth_username: admin
+              auth_password: ((prometheus_password))
+
+variables:
+  - name: alertmanager_password
+    type: password
+  - name: alertmanager_mesh_password
+    type: password
+  - name: prometheus_password
+    type: password
+  - name: postgres_grafana_password
+    type: password
+  - name: grafana_password
+    type: password
+  - name: grafana_secret_key
+    type: password
+
+update:
+  canaries: 1
+  max_in_flight: 32
+  canary_watch_time: 1000-100000
+  update_watch_time: 1000-100000
+  serial: false
+
+stemcells:
+  - alias: default
+    os: ubuntu-trusty
+    version: latest
+
+releases:
+- name: postgres
+  version: "20"
+  url: https://bosh.io/d/github.com/cloudfoundry/postgres-release?v=20
+  sha1: 3f378bcab294e20316171d4e656636df88763664
+- name: prometheus
+  version: 18.6.2
+  url: https://github.com/cloudfoundry-community/prometheus-boshrelease/releases/download/v18.6.2/prometheus-18.6.2.tgz
+  sha1: f6b7ed381a28ce8fef99017a89e1122b718d5556

data/tpl/aws/docs/concourse.md

@@ -10,9 +10,9 @@
 
 Fill out the "token" field in `deployments/concourse/concourse.yml` with root token received from `vault init`.
 
-Deploy Concourse by running the script with the required arguments
+Deploy Concourse by running the script with the Vault token as argument(strong passwords for Concourse auth and db will be generated automatically)
 ```
-./bin/concourse-deploy.sh *concourse_auth_password* *concourse_db_password* *vault_token*
+./bin/concourse-deploy.sh *vault_token*
 ```
 
 ### Connect GitHub oAuth

data/tpl/aws/docs/ingress.md

@@ -0,0 +1,14 @@
+#### [Back](../README.md)
+
+## Ingress
+
+### Prerequisites
+
+- BOSH environment [ready](bosh.md)
+- All hostnames resolve to the VIP configured in cloud.yml (this is mandatory to issue SSL certificates)
+
+### Deployment
+
+To deploy Ingress, use `./bin/ingress-deploy.sh`
+
+After each new component deployed, run `./bin/ingress-update`

data/tpl/aws/docs/kops.md

@@ -22,17 +22,14 @@ export AWS_ACCESS_KEY_ID=<access key>
 export AWS_SECRET_ACCESS_KEY=<secret key>
 ```
 
-Create cluster configuration
+Deploy the `kops` cluster
 ```
-kops create cluster --name *kops.example.com* --state "s3://kops-example-state-store" --zones *eu-central-1b* --ssh-public-key *path to SSH key*
+./bin/kops-deploy.sh
 ```
 
-Review and edit cluster configuration if needed
-```
-kops edit cluster --name *kops.example.com* --state "s3://kops-example-state-store"
-```
+### Teardown
 
-Build the cluster
+To tear down the kops cluster you've created, just run
 ```
-kops update cluster --name *kops.example.com* --state "s3://kops-example-state-store" --yes
+./bin/kops-delete.sh
 ```

data/tpl/aws/docs/oauth.md

@@ -0,0 +1,24 @@
+#### [Back](../README.md)
+
+## OAuth (UAA)
+
+### Configuration
+
+If you want to add initial groups and users, change oauth look,
+configure mail, etc. - you should edit `config/oauth.yml`.
+
+Here are links to uaa config documentation:
+
+* __users:__ [uaa.scim.users](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=uaa.scim.users)
+* __groups:__ [uaa.scim.groups](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=uaa.scim.groups)
+* __oauth clients:__ [uaa.clients](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=uaa.clients)
+* __theming:__ [login.branding](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=login.branding)
+* __email notifications:__ [login.smtp](https://bosh.io/jobs/uaa?source=github.com/cloudfoundry/uaa-release&version=52#p=login.smtp)
+
+### Deployment
+
+After editing config, run `./bin/oauth-deploy.sh`
+
+### Usage
+
+To check if OAuth works, visit [<%= @values['oauth']['hostname'] %>](<%= @values['oauth']['url'] %>).

data/tpl/aws/docs/prometheus.md

@@ -0,0 +1,31 @@
+#### [Back](../README.md)
+
+## Prometheus
+
+### Prerequisites
+
+- BOSH environment [ready](bosh.md)
+- Kops cluster [deployed](kops.md)
+
+### Setup
+
+Enter path to your Kubernetes config in `config/cloud.yml` and add the Kubernetes API server address to `config/bosh_vars.yml`.
+
+Afterwards, deploy Prometheus
+```
+./bin/prometheus-deploy.sh
+```
+
+### Access
+
+After the deployment process is done, you can reach each Prometheus' component's web UI at:
+
+If you have [Ingress](ingress.md) deployed and DNS record created, each Prometheus stack component should be accessible by its respective address.
+
+Without Ingress:
+
+- AlertManager: http://10.0.0.18:9093
+- Grafana:      http://10.0.0.18:3000
+- Prometheus:   http://10.0.0.18:9090
+
+You can find related credentials in `config/creds.yml`

data/tpl/aws/terraform/kite_bucket.tf

@@ -0,0 +1,8 @@
+resource "aws_s3_bucket" "kite_bucket" {
+  bucket = "${var.bucket_name}"
+
+  tags {
+    Name = "${var.bucket_name}"
+    Component = "kite-platform"
+  }
+}

data/tpl/aws/terraform/network.tf.tt

@@ -182,6 +182,33 @@ resource "aws_security_group" "bosh_sg" {
   }
 }
 
+# Create an Ingress security group
+resource "aws_security_group" "ingress_sg" {
+  name        = "ingress-sg"
+  description = "Ingress security group"
+  vpc_id = <%= "\"#{conditional_vpc_id(@values)}\"" %>
+  tags {
+    Name = "ingress-sg"
+    Component = "ingress"
+  }
+
+  # outbound internet access
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  # inbound HTTP access
+  ingress {
+    from_port = 80
+    to_port = 80
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
 # Create a Concourse security group
 resource "aws_security_group" "concourse_sg" {
   name        = "concourse-sg"

data/tpl/aws/terraform/outputs.tf

@@ -6,6 +6,10 @@ output "platform_subnet_id" {
     value = "${aws_subnet.platform_net.id}"
 }
 
+output "dmz_subnet_id" {
+    value = "${aws_subnet.platform_dmz.id}"
+}
+
 output "bastion_ip" {
     value = "${aws_instance.bastion.public_ip}"
 }

data/tpl/aws/terraform/terraform.tfvars.tt

@@ -16,5 +16,6 @@ private_subnet_cidr = "<%= @values['aws']['private_subnet']['network'] %>"
 
 # Kite config
 keypair_name = "<%= @values['kite']['keypair_name'] %>"
+bucket_name = "<%= @values['kite']['bucket_name'] %>"
 public_key = "<%= @values['kite']['public_key_path'] %>"
 private_key = "<%= @values['kite']['private_key_path'] %>"

data/tpl/aws/terraform/variables.tf

@@ -18,6 +18,10 @@ variable "keypair_name" {
   type = "string"
 }
 
+variable "bucket_name" {
+  type = "string"
+}
+
 variable "region" {
   type = "string"
   default =  "eu-central-1"

data/tpl/gcp/README.md

@@ -1,13 +1,14 @@
-## GCP Cloud
+# GCP Cloud
 
-### Setup
+## Setup
 
+### Prerequisites
 Set path to your service account credentials:
 ```
 export GOOGLE_CREDENTIALS=*~/credentials/service-account.json*
 ```
 
-
+### Setup the basic infrastructure and bastion
 Apply terraform code
 ```
 pushd terraform && terraform init && terraform apply && popd
@@ -16,6 +17,7 @@ pushd terraform && terraform init && terraform apply && popd
 [Note]
 To destroy Bastion later, use `terraform destroy -target google_compute_instance.bastion`
 
+### Setup BOSH
 Render BOSH manifest and related files
 ```
 kite render manifest bosh --cloud gcp
@@ -23,6 +25,16 @@ kite render manifest bosh --cloud gcp
 
 Prepare BOSH environment using instructions from [docs/bosh.md](docs/bosh.md)
 
+### Setup INGRESS
+Render Ingress manifest and related files
+```
+kite render manifest ingress --cloud gcp
+```
+
+Follow instructions from [docs/ingress.md](docs/ingress.md) to deploy Ingress
+
+
+### Setup VAULT
 Render Vault deployment
 ```
 kite render manifest vault --cloud gcp
@@ -30,6 +42,10 @@ kite render manifest vault --cloud gcp
 
 Follow instructions from [docs/vault.md](docs/vault.md) to deploy Vault
 
+### Setup CONCOURSE
+[Note]
+To expose concourse publicly, you must create first (manually) a virtual IP in GCP and create a DNS A entry for the hostname for this IP. Set the IP into config/cloud.yml (concourse.vip).
+
 Render Concourse manifest
 ```
 kite render manifest concourse --cloud gcp