kite 0.1.0 → 0.2.0

data/kite.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
     f.match(%r{^(test|spec|features)/})
   end
   spec.bindir        = "bin"
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = ["kite"]
   spec.require_paths = ["lib"]
 
   spec.add_dependency "thor"

data/lib/kite/generate.rb

@@ -17,9 +17,13 @@ module Kite
           copy_file('aws/docs/kops.md',                       'docs/kops.md')
 
           directory('aws/bin/base',                           'bin')
+          template('aws/bin/kops-deploy.sh.erb',              'bin/kops-deploy.sh')
+          template('aws/bin/kops-delete.sh.erb',              'bin/kops-delete.sh')
           chmod('bin/bootstrap.sh', 0755)
           chmod('bin/cleanup.sh', 0755)
           chmod('bin/setup-tunnel.sh', 0755)
+          chmod('bin/kops-deploy.sh', 0755)
+          chmod('bin/kops-delete.sh', 0755)
 
         when 'gcp'
           directory('gcp/terraform',                          'terraform')
@@ -42,5 +46,40 @@ module Kite
     def task()
       say "Generating task #{ options[:name] } IaC", :green
     end
+
+    method_option :git, type: :string, desc: "Git repository", required: true
+    method_option :name, type: :string, desc: "Name of the service", required: false
+    method_option :image, type: :string, desc: "Docker image full name", required: true
+    method_option :output, type: :string, desc: "Config output sub-directory", default: "config"
+    method_option :slack, type: :string, desc: "Slack notifications", requied: false, default: nil
+    method_option :provider, type: :string, desc: "Cloud provider", enum: %w{aws gcp}, required: false, default: nil
+    method_option :image_version, type: :string, desc: "Docker image tag", required: false, default: '0.1.0'
+    method_option :chart_version, type: :string, desc: "Chart version", required: false, default: '0.1.0'
+    desc "service NAME", "Generate new micro-service pipeline"
+    def service(path)
+      @name     = options[:name] || File.basename(File.expand_path(path))
+      @title    = @name.split(/\W/).map(&:capitalize).join(' ')
+      @git      = options[:git]
+      @image    = options[:image]
+      @provider = options[:provider]
+      @output   = options[:output]
+      @slack    = options[:slack]
+      @image_version = options[:image_version]
+      @chart_version = options[:chart_version]
+
+      say "Generating service #{ @name }", :green
+      directory('service', path)
+    end
+
+    no_commands do
+      def output_path
+        @output ||= "config"
+      end
+
+      def app_name
+        @name ||= "app-name"
+      end
+    end
+
   end
 end

data/lib/kite/helpers/concourse.rb

@@ -0,0 +1,36 @@
+require 'json'
+require 'open3'
+
+module Kite
+  module Helpers
+    # Helpers for concourse scripts
+    module Concourse
+      def self.params(data)
+        JSON.parse data
+      end
+
+      def self.log(msg)
+        msg.split("\n").each { |line| $stderr.puts("[LOG] --- #{line}") }
+      end
+
+      def self.respond(data)
+        # keep only valid concourse values
+        data.select! { |k, _| k.to_s =~ /(version|metadata)/ }
+        puts JSON.dump(data)
+      end
+
+      def self.fatal(message)
+        respond(version: { status: 'error' }, metadata: [message])
+        exit 1
+      end
+
+      def self.execute(command, env = {})
+        log("+ #{ command }")
+        Open3.popen2e(env, command) do |stdin, stdout, wait_thr|
+          ::Kite::Helpers::Concourse.log(stdout.read)
+          return wait_thr.value.exitstatus.zero?
+        end
+      end
+    end
+  end
+end

data/lib/kite/render.rb

@@ -4,20 +4,62 @@ module Kite
 
     include Kite::Helpers
 
+    no_commands do
+      def ingress_db_file
+        "config/ingress.yml"
+      end
+
+      def ingress_db
+        @db ||= YAML.load(File.read(ingress_db_file)) rescue {}
+      end
+
+      def ingress_db_save!
+        create_file ingress_db_file, YAML.dump(ingress_db), force: true
+      end
+
+      def ingress_add_entry(hostname, upstreams, args = {})
+        raise "upstreams argument should be an array" unless upstreams.is_a?(Array)
+        args[:port] ||= 80
+        args[:protocol] ||= "http"
+        ingress_db[hostname] = {
+          upstreams: upstreams,
+          port: args[:port],
+          protocol: args[:protocol],
+        }
+        ingress_db_save!
+      end
+    end
+
     desc "manifest <type>", "Renders a manifest of selected type"
+    long_desc <<-LONGDESC
+      Available types:
+      \x5  BOSH        Render Bosh environement
+      \x5  CONCOURSE   Render Concourse deployment
+      \x5  VAULT       Render Vault deployment
+      \x5  INGRESS     Render Ingress deployment
+      \x5  PROMETHEUS  Render Prometheus deployment
+      \x5  OAUTH       Render OAuth (UAA) deployment
+    LONGDESC
     method_option :cloud, type: :string, desc: "Cloud provider", enum: %w{aws gcp}, required: true
     # Render a manifest of selected type based on <b>config/cloud.yml</b> and <b>terraform apply</b> results
     def manifest(type)
+      type = type.downcase
       say "Rendering #{type} manifest", :green
       @values = parse_cloud_config
       @tf_output = parse_tf_state('terraform/terraform.tfstate') if options[:cloud] == 'aws'
 
       if options[:cloud] == 'aws'
         @private_subnet = IPAddr.new(@values['aws']['private_subnet']['network']).to_range.to_a
+        @public_subnet = IPAddr.new(@values['aws']['public_subnet']['network']).to_range.to_a
       else
         @private_subnet = IPAddr.new(@values['gcp']['subnet_cidr']).to_range.to_a
       end
 
+      @static_ip_vault            = @private_subnet[11].to_s
+      @static_ips_concourse       = [@private_subnet[12]].map(&:to_s)
+      @static_ip_prometheus_stack = @private_subnet[18].to_s
+      @static_ip_oauth            = @private_subnet[23].to_s
+
       case type
       when "bosh"
         directory("#{options[:cloud]}/deployments/bosh",                          'deployments/bosh')
@@ -31,15 +73,39 @@ module Kite
         copy_file("#{options[:cloud]}/docs/concourse.md",                         "docs/concourse.md")
         template("#{options[:cloud]}/bin/concourse-deploy.sh.tt",                 "bin/concourse-deploy.sh")
         chmod('bin/concourse-deploy.sh', 0755)
+        ingress_add_entry(@values['concourse']['hostname'], @static_ips_concourse, port: 8080)
 
       when "vault"
         template("#{options[:cloud]}/deployments/vault/vault.yml.erb",            "deployments/vault/vault.yml")
         copy_file("#{options[:cloud]}/docs/vault.md",                             "docs/vault.md")
         template("#{options[:cloud]}/bin/vault-deploy.sh.tt",                     "bin/vault-deploy.sh")
         chmod('bin/vault-deploy.sh', 0755)
+        ingress_add_entry(@values['vault']['hostname'], [@static_ip_vault], port: 8200)
+
+      when "ingress"
+        template("#{options[:cloud]}/deployments/ingress/ingress.yml.erb",        "deployments/ingress/ingress.yml")
+        copy_file("#{options[:cloud]}/docs/ingress.md",                           "docs/ingress.md")
+        template("#{options[:cloud]}/bin/ingress-deploy.sh.tt",                   "bin/ingress-deploy.sh")
+        template("#{options[:cloud]}/bin/ingress-update.sh.tt",                   "bin/ingress-update.sh")
+        chmod('bin/ingress-deploy.sh', 0755)
+        chmod('bin/ingress-update.sh', 0755)
+
+      when "prometheus"
+        directory("#{options[:cloud]}/deployments/prometheus",                    "deployments/prometheus")
+        copy_file("#{options[:cloud]}/docs/prometheus.md",                        "docs/prometheus.md")
+        template("#{options[:cloud]}/bin/prometheus-deploy.sh.tt",                "bin/prometheus-deploy.sh")
+        chmod('bin/prometheus-deploy.sh', 0755)
+        ingress_add_entry(@values['alertmanager']['hostname'], [@static_ip_prometheus_stack], port: 9093)
+        ingress_add_entry(@values['grafana']['hostname'], [@static_ip_prometheus_stack], port: 3000)
+        ingress_add_entry(@values['prometheus']['hostname'], [@static_ip_prometheus_stack], port: 9090)
 
-      when "nginx"
-        template("#{options[:cloud]}/deployments/nginx/nginx.yml.erb",            "deployments/nginx/nginx.yml")
+      when "oauth"
+        directory("#{options[:cloud]}/deployments/oauth",                         "deployments/oauth")
+        copy_file("#{options[:cloud]}/config/oauth.yml",                          "config/oauth.yml")
+        template("#{options[:cloud]}/docs/oauth.md",                              "docs/oauth.md")
+        template("#{options[:cloud]}/bin/oauth-deploy.sh.tt",                     "bin/oauth-deploy.sh")
+        chmod('bin/oauth-deploy.sh', 0755)
+        ingress_add_entry(@values['oauth']['hostname'], [@static_ip_oauth], port: 8080)
 
       else
         say "Manifest type not specified"

data/lib/kite/version.rb

@@ -1,3 +1,3 @@
 module Kite
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/tpl/aws/README.md

@@ -17,6 +17,26 @@ kite render manifest bosh --cloud aws
 
 Prepare BOSH environment using instructions from [docs/bosh.md](docs/bosh.md)
 
+[Optional]
+If you want to access components outside of your VPC, use the Ingress deployment:
+
+Render Ingress deployment files
+```
+kite render manifest prometheus --cloud aws
+```
+
+Follow instructions from [docs/prometheus.md](docs/prometheus.md) to deploy Prometheus
+
+[Note]
+If you're using Ingress, create CNAME DNS records for each deployment as listed in `config/cloud.yml`(e.g. vault.example.com pointing to ingress.example.com)
+
+Render Prometheus deployment files
+```
+kite render manifest prometheus --cloud aws
+```
+
+Follow instructions from [docs/prometheus.md](docs/prometheus.md) to deploy Prometheus
+
 Render Vault deployment
 ```
 kite render manifest vault --cloud aws

data/tpl/aws/bin/concourse-deploy.sh.tt

@@ -8,4 +8,7 @@ bosh -e <%= @values['bosh']['name'] %> upload-release https://bosh.io/d/github.c
 bosh -e <%= @values['bosh']['name'] %> upload-release https://bosh.io/d/github.com/cloudfoundry-incubator/garden-runc-release
 
 # Deploy Concourse
-bosh -e <%= @values['bosh']['name'] %> -d concourse deploy deployments/concourse/concourse.yml -v vault_token=$1
+bosh -e <%= @values['bosh']['name'] %> -d concourse \
+  deploy deployments/concourse/concourse.yml \
+  -v vault_token=$1 \
+  --vars-store config/creds.yml

data/tpl/aws/bin/ingress-deploy.sh.tt

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -xe
+
+bosh -e <%= @values['bosh']['name'] %> upload-release https://github.com/cloudfoundry-community/nginx-release/releases/download/v1.12.1/nginx-1.12.1.tgz
+
+bosh -e <%= @values['bosh']['name'] %> -d ingress deploy deployments/ingress/ingress.yml

data/tpl/aws/bin/ingress-update.sh.tt

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -xe
+
+kite render manifest ingress --cloud aws
+
+bosh -e <%= @values['bosh']['name'] %> -d ingress deploy deployments/ingress/ingress.yml

data/tpl/aws/bin/kops-delete.sh.erb

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+export KOPS_STATE_STORE=s3://<%= @values['kite']['bucket_name'] %>
+
+kops delete cluster <%= @values['aws']['kops_address'] %> --yes

data/tpl/aws/bin/kops-deploy.sh.erb

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set NAME=<%= @values['k8s']['cluster_address'] %>
+set KOPS_STATE_STORE=s3://<%= @values['kite']['bucket_name'] %>
+
+kops create cluster \
+  --zones <%= @values['aws']['zone'] %> \
+  --ssh-public-key <%= @values['kite']['public_key_path'] %> \
+  $NAME
+
+kops update cluster $NAME --yes

data/tpl/aws/bin/oauth-deploy.sh.tt

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Upload necessary stemcells and releases
+bosh -e <%= @values['bosh']['name'] %> upload-stemcell \
+  https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent
+
+bosh -e <%= @values['bosh']['name'] %> upload-release \
+  https://bosh.io/d/github.com/cloudfoundry/uaa-release
+
+# Deploy Concourse
+bosh -e <%= @values['bosh']['name'] %> -d oauth deploy \
+  deployments/oauth/oauth.yml \
+  --vars-store config/creds.yml \
+  -o config/oauth.yml \
+  && ./bin/ingress-update.sh

data/tpl/aws/bin/prometheus-deploy.sh.tt

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -xe
+
+# Upload necessary stemcells and releases
+bosh -e <%= @values['bosh']['name'] %> upload-stemcell https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent
+
+# Extract BOSH Director's SSL certificate
+bosh int ./config/creds.yml --path /director_ssl/certificate > bosh.ca
+
+# Deploy Prometheus
+bosh -e <%= @values['bosh']['name'] %> -d prometheus \
+  deploy deployments/prometheus/prometheus.yml \
+  --vars-store config/creds.yml \
+  -o deployments/prometheus/monitor-kubernetes.yml \
+  --vars-file config/bosh-vars.yml \
+  --var-file kubernetes_kubeconfig=<%= @values['k8s']['config_path'] %> \
+  -o deployments/prometheus/monitor-bosh.yml \
+  -v bosh_url=<%= @values['bosh']['static_ip'] %> \
+  -v bosh_username=admin \
+  -v bosh_password=`bosh int ./config/creds.yml --path /admin_password` \
+  --var-file bosh_ca_cert=bosh.ca \
+  -v metrics_environment=kite

data/tpl/aws/bosh-vars.yml.erb

@@ -9,3 +9,4 @@ az:                        <%= @values['aws']['zone'] %>
 default_key_name:          <%= @values['kite']['keypair_name'] %>
 default_security_groups:   [<%= @tf_output['security_group_id'] %>]
 subnet_id:                 <%= @tf_output['platform_subnet_id'] %>
+kubernetes_apiserver:      <%= @values['aws']['kops_api_server_address'] %>

data/tpl/aws/config/oauth.yml

@@ -0,0 +1,59 @@
+# Groups
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/scim?/groups?
+  value: { }
+    # sysops: For automation stuff
+
+# Users
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/scim?/users?
+  value: [ ]
+    # - email: sysops@example.com
+    #   password: changeme
+    #   name: sysops
+    #   firstName: Sys
+    #   lastName: Ops
+    #   origin: uaa
+    #   groups:
+    #     - sysops
+
+# OAuth clients
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/clients?
+  value: { }
+    # some-app:
+    #   app-icon: 0000000000000000000000000000000000000000000000000000000000000000
+    #   app-launch-url: http://myapppage.com
+    #   authorities: test_resource.test_action
+    #   authorized-grant-types: authorization_code,client_credentials,refresh_token
+    #   autoapprove:
+    #     - test_resource.test_action
+    #     - test_resource.other_action
+    #   id: app
+    #   override: true
+    #   redirect-uri: http://login.example.com
+    #   scope: test_resource.test_action,test_resource.other_action
+    #   secret: app-secret
+    #   show-on-homepage: true
+
+# UAA theming
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/login/branding?
+  value: { }
+    # banner:
+    #   backgroundColor: "#eeeeee"
+    #   link: "https://example.com/logo.png"
+    #   text: "Hello!"
+    #   textColor: "#333333"
+    # company_name: "Company Name"
+    # footer_legal_text: "© 2017. Company Name. All right reserved."
+
+# Email notifications
+- type: replace
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/login/smtp?
+  value: { }
+    # auth: false
+    # from_address: hello@example.com
+    # host: localhost
+    # port: 2525
+    # starttls: false

data/tpl/aws/deployments/bosh/cloud-config.yml.tt

@@ -39,6 +39,11 @@ vm_types:
     instance_type: t2.micro
     ephemeral_disk: {size: 3000, type: gp2}
     security_groups: [vault-sg, bosh_sg]
+- name: ingress_default
+  cloud_properties:
+    instance_type: t2.micro
+    ephemeral_disk: {size: 3000, type: gp2}
+    security_groups: [ingress-sg, bosh_sg]
 
 disk_types:
 - name: default
@@ -49,6 +54,17 @@ disk_types:
   cloud_properties: {type: gp2}
 
 networks:
+- name: platform_dmz
+  type: manual
+  subnets:
+  - az: z1
+    range: <%= @values['aws']['public_subnet']['network'] %>
+    gateway: <%= @values['aws']['public_subnet']['gateway'] %>
+    reserved: [<%= ip_range(@public_subnet, (1..10)) %>] # Reserved range for the gateway, BOSH Director etc
+    # static: [<%= ip_range(@public_subnet, (11..16)) %>] # Static IP range for Vault, Concourse web panel, nginx etc
+    dns: [<%= @public_subnet[8].to_s %>]
+    cloud_properties: {subnet: <%= @tf_output['dmz_subnet_id'] %>}
+
 - name: platform_net
   type: manual
   subnets:
@@ -56,7 +72,7 @@ networks:
     range: <%= @values['aws']['private_subnet']['network'] %>
     gateway: <%= @values['aws']['private_subnet']['gateway'] %>
     reserved: [<%= ip_range(@private_subnet, (1..10)) %>] # Reserved range for the gateway, BOSH Director etc
-    static: [<%= ip_range(@private_subnet, (11..13)) %>] # Static IP range for Vault, Concourse web panel, nginx etc
+    static: [<%= ip_range(@private_subnet, (11..20)) %>] # Static IP range for Vault, Concourse web panel, nginx etc
     dns: [<%= @private_subnet[8].to_s %>]
     cloud_properties: {subnet: <%= @tf_output['platform_subnet_id'] %>}
 - name: vip

data/tpl/aws/deployments/concourse/concourse.yml.tt

@@ -90,3 +90,9 @@ stemcells:
 - alias: trusty
   os: ubuntu-trusty
   version: latest
+
+variables:
+- name: db_password
+  type: password
+- name: auth_password
+  type: password