kite 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a4676c9a2793922ae3134375354a14353523d773
-  data.tar.gz: 26bbcfc505d24b57af425afdcbe287afad7bee74
+  metadata.gz: 90bb5e8944368e0fe0576c0c954ccc1a80c72e0d
+  data.tar.gz: 174dfc7e5123c6ee0eadef4ff11db6dfce68fd07
 SHA512:
-  metadata.gz: c242dc22c331f1f18036b679afa2b033286d0bc7242d3f7af1cedb2488d9bef169a748d6bbafde57764a33915650300af678e08af6e7608ed4d7f1d5816536f1
-  data.tar.gz: de50ab0014b70949fef89ad15396951b9edc241094fa2c847c1cce795ec1cc2e3291e28a99444a15ae4c8909b37f615e00282e4b307a94af499e8140f8bff3c6
+  metadata.gz: 70e301d6a0bcdbf445a29542e7c4142c0cada31588b8d1191dc15828a5c30d27064816c87066ca1046c725b36b14f9a698888569e28c44e9609087a5211bc08d
+  data.tar.gz: 81638c0f37000c26497ea98b8a8fe0e7f67f497b9b098b57f2e1d8536e0ae84638176f660b0e656d6bb5eb12dbc29160bb025d383dd39f1bd6727ef18b59d639

data/README.md

@@ -1,5 +1,7 @@
 # Kite
 
+[![Build Status](https://travis-ci.org/helios-technologies/kite.svg?branch=master)](https://travis-ci.org/helios-technologies/kite)
+
 Kite is a CLI for scaffolding and managing devops modules
 The main purpose is templating of various tools for devops around terraform, bosh, ansible
 

data/bin/kite

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+require "thor"
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+require 'kite'
+
+Kite::Commands.start

data/kite.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
   spec.add_dependency "thor"

data/lib/kite.rb

@@ -1,5 +1,7 @@
-require "kite/version"
+require "erb"
+require "yaml"
+require "thor"
 
-module Kite
-  # Your code goes here...
-end
+require "kite/version"
+require "kite/helpers"
+require "kite/commands"

data/lib/kite/commands.rb

@@ -0,0 +1,46 @@
+class Kite::Commands < Thor
+  include Thor::Actions
+  include Kite::Helpers
+
+  def self.source_root
+    File.expand_path(File.join(File.dirname(__FILE__), "../.."))
+  end
+
+  method_option :values, type: :string, default: "defaults.yml", required: true
+  method_option :cloud, type: :string, desc: "Cloud supplier", enum: %w{aws gcp}, required: true
+  desc "new CLOUDNAME", "Generate Cloud IaC from configuration"
+  def new(cloud_name)
+    say "Generating Cloud #{ cloud_name } IaC", :green
+    @values = YAML.load(File.read(options[:values]))
+
+    case options[:cloud]
+    when "aws"
+      copy_file("tpl/aws/bin/make_cloud_config.sh",                 "#{cloud_name}/bin/make_cloud_config.sh")
+      copy_file("tpl/aws/bin/make_manifest_bosh-init.sh",           "#{cloud_name}/bin/make_manifest_bosh-init.sh")
+      copy_file("tpl/aws/bin/make_manifest_concourse-cluster.sh",   "#{cloud_name}/bin/make_manifest_concourse-cluster.sh")
+
+      copy_file("tpl/aws/terraform/aws-concourse.tf",               "#{cloud_name}/terraform/aws-concourse.tf")
+      copy_file("tpl/aws/terraform/aws-vault.tf",                   "#{cloud_name}/terraform/aws-vault.tf")
+      copy_file("tpl/aws/terraform/bosh-aws-base.tf",               "#{cloud_name}/terraform/bosh-aws-base.tf")
+      copy_file("tpl/aws/terraform/outputs.tf",                     "#{cloud_name}/terraform/outputs.tf")
+      template("tpl/aws/terraform/terraform.tfvars.erb",            "#{cloud_name}/terraform/terraform.tfvars")
+      copy_file("tpl/aws/terraform/variables.tf",                   "#{cloud_name}/terraform/variables.tf")
+      copy_file("tpl/aws/terraform/variables.tf",                   "#{cloud_name}/terraform/variables.tf")
+
+      template("tpl/aws/env.example.erb",                           "#{cloud_name}/.env")
+      copy_file("tpl/aws/README.md",                                "#{cloud_name}/README.md")
+      copy_file("tpl/aws/bootstrap.sh",                             "#{cloud_name}/bootstrap.sh")
+
+    when "gcp"
+      template("tpl/gcp/manifest.yml.erb",            "#{cloud_name}/manifest.yml")
+      template("tpl/gcp/cloud-config.yml.erb",        "#{cloud_name}/cloud-config.yml")
+      copy_file("tpl/gcp/concourse.yml.erb",          "#{cloud_name}/concourse.yml")
+      copy_file("tpl/gcp/README.md",                  "#{cloud_name}/README.md")
+      directory("tpl/gcp/scripts",                    "#{cloud_name}/scripts")
+      copy_file("tpl/gcp/INSTALL.md",                 "#{cloud_name}/INSTALL.md")
+      template("tpl/gcp/env.example.erb",             "#{cloud_name}/.env")
+      copy_file("tpl/gcp/main.tf",                    "#{cloud_name}/main.tf")
+      copy_file("tpl/gcp/concourse.tf",               "#{cloud_name}/concourse.tf")
+    end
+  end
+end

data/lib/kite/helpers.rb

@@ -0,0 +1,2 @@
+module Kite::Helpers
+end

data/lib/kite/version.rb

@@ -1,3 +1,3 @@
 module Kite
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/tpl/aws/README.md

@@ -0,0 +1,103 @@
+BOSH Director & Concourse Bootstrap
+===================================
+
+This project achieves the following:
+
+- Preparation of an AWS environment for BOSH & Concourse
+- Deployment of a new BOSH Director using bosh-init
+- Deployment of a new Concourse cluster, or standalone server
+
+Terraform is used to setup the base network and security infrastructure, including an ELB for Concourse.
+
+Requirements
+-----
+
+- Install [terraform](https://www.terraform.io/intro/getting-started/install.html)
+- Install [bosh-init](https://bosh.io/docs/install-bosh-init.html)
+- Install the [bosh_cli](https://bosh.io/docs/bosh-cli.html)
+
+Ensure you have created a `terraform/terraform.tfvars` file with your variables, or set suitable [environment variables](https://www.terraform.io/docs/configuration/variables.html). An example tfvars file can be found in `terraform/terraform.tfvars.example`
+
+Assumptions
+-----
+
+You already have:
+
+- A Route53 Zone in AWS.
+- An EC2 SSH keypair
+- An SSL certificate in AWS for your Concourse ELB
+
+Usage
+-----
+
+Set your desired AWS region in `terrform/variables.tf`. Ensure terraform is in your path, then apply the configuration to prepare the IaaS for BOSH and Concourse:
+
+```
+cd terraform/
+terraform apply
+```
+Set the following environment variables:
+
+```
+$AWS_ACCESS_KEY_ID
+$AWS_SECRET_ACCESS_KEY
+$AWS_REGION
+$AWS_AZ
+$BOSH_PASSWORD
+$AWS_KEYPAIR_KEY_NAME
+$PRIVATE_KEY_PATH
+```
+
+Then create the `bosh-director.yml` manifest:
+```
+./bin/make_manifest_bosh-init.sh
+```
+
+You are ready to deploy the BOSH Director
+```
+bosh-init deploy bosh-director.yml
+```
+
+Go and make a cup of tea.
+
+Once the director is deployed, target it and apply your cloud-config for AWS.
+Remember to set your chosen AZ and the subnet-id output by terraform in `aws-cloud.yml`.
+
+```
+bosh target <your EIP address>
+bosh update cloud-config aws-cloud.yml
+```
+
+Set a database password and external URL for your deployment in these environment variables:
+
+```
+$DB_PASSWORD
+$CONCOURSE_URL
+```
+Create a new OAuth application in GitHub as described [here](http://concourse.ci/authentication.html). The manifest assumes the existance of a 'CI' team that contains your authorised users, so create that too. Then set the following environment variables:
+
+```
+$GITHUB_ORG
+$GITHUB_CLIENT_ID
+$GITHUB_CLIENT_SECRET
+```
+
+Then create a concourse manifest for a single server deployment:
+```
+./bin/make_manifest_concourse.sh
+```
+Or, create a concourse manifest for small cluster:
+```
+./bin/make_manifest_concourse-cluster.sh
+```
+
+Upload the necessary stemcell & releases, then deploy concourse:
+```
+bosh upload stemcell https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent
+bosh upload release https://bosh.io/d/github.com/concourse/concourse
+bosh upload release https://bosh.io/d/github.com/cloudfoundry-incubator/garden-runc-release
+bosh deployment concourse.yml
+bosh deploy
+```
+
+Congratulations, you should now be able to see your new CI server at https://your-concourse-url.

data/tpl/aws/bin/make_cloud_config.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+#
+#  Please set the following environment variables:
+#  $AWS_AZ
+
+
+function getvars() {
+  cd terraform/
+  DEFAULT_SUBNET=$(terraform output default_subnet_id)
+  OPS_SUBNET=$(terraform output ops_services_subnet_id)
+  cd ../
+}
+
+getvars
+
+echo "Default Subnet = $DEFAULT_SUBNET"
+echo "Ops Services Subnet = $OPS_SUBNET"
+echo "AWS AZ" = $AWS_AZ
+
+
+cat >aws-cloud.yml <<YAML
+---
+azs:
+- name: z1
+  cloud_properties: {availability_zone: $AWS_AZ}
+
+vm_types:
+- name: concourse_standalone
+  cloud_properties:
+    instance_type: m3.large
+    ephemeral_disk: {size: 5000, type: gp2}
+    elbs: [concourse-elb]
+    security_groups: [concourse-sg, boshdefault]
+- name: concourse_web
+  cloud_properties:
+    instance_type: m3.medium
+    ephemeral_disk: {size: 3000, type: gp2}
+    elbs: [concourse-elb]
+    security_groups: [concourse-sg, boshdefault]
+- name: concourse_db
+  cloud_properties:
+    instance_type: m3.medium
+    ephemeral_disk: {size: 3000, type: gp2}
+    security_groups: [boshdefault]
+- name: concourse_worker
+  cloud_properties:
+    instance_type: m3.large
+    ephemeral_disk: {size: 30000, type: gp2}
+    security_groups: [boshdefault]
+- name: default
+  cloud_properties:
+    instance_type: t2.micro
+    ephemeral_disk: {size: 3000, type: gp2}
+    security_groups: [boshdefault]
+- name: large
+  cloud_properties:
+    instance_type: m3.large
+    ephemeral_disk: {size: 5000, type: gp2}
+    security_groups: [boshdefault]
+- name: vault-default
+  cloud_properties:
+    instance_type: t2.micro
+    ephemeral_disk: {size: 3000, type: gp2}
+    security_groups: [vault-sg, boshdefault]
+
+disk_types:
+- name: default
+  disk_size: 3000
+  cloud_properties: {type: gp2}
+- name: large
+  disk_size: 50_000
+  cloud_properties: {type: gp2}
+
+networks:
+- name: default
+  type: manual
+  subnets:
+  - range: 10.0.0.0/24
+    gateway: 10.0.0.1
+    az: z1
+    static: [10.0.0.6]
+    reserved: [10.0.0.1-10.0.0.5]
+    dns: [10.0.0.2]
+    cloud_properties: {subnet: $DEFAULT_SUBNET}
+- name: ops_services
+  type: manual
+  subnets:
+  - range: 10.0.10.0/24
+    gateway: 10.0.10.1
+    az: z1
+    reserved: [10.0.10.1-10.0.10.5]
+    dns: [10.0.0.2]
+    cloud_properties: {subnet: $OPS_SUBNET}
+- name: vip
+  type: vip
+
+compilation:
+  workers: 5
+  reuse_compilation_vms: true
+  az: z1
+  vm_type: large
+  network: default
+
+YAML

data/tpl/aws/bin/make_manifest_bosh-init.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+#
+#  Please set the following environment variables:
+#  $AWS_ACCESS_KEY_ID
+#  $AWS_SECRET_ACCESS_KEY
+#  $AWS_REGION
+#  $AWS_AZ
+#  $BOSH_PASSWORD
+#  $AWS_KEYPAIR_KEY_NAME
+#  $PRIVATE_KEY_PATH
+
+function getvars() {
+  cd terraform/
+  EIP=$(terraform output eip)
+  SUBNET=$(terraform output default_subnet_id)
+  SECURITY_GROUP=$(terraform output security_group_id)
+  cd ../
+}
+
+getvars
+
+echo "Subnet = $SUBNET"
+echo "Security Group = $SECURITY_GROUP"
+echo "EIP = $EIP"
+echo "AWS REGION = $AWS_REGION"
+echo "AWS AZ = $AWS_AZ"
+
+cat >bosh-director.yml <<YAML
+---
+name: bosh
+
+releases:
+- name: bosh
+  url: https://bosh.io/d/github.com/cloudfoundry/bosh?v=256.2
+  sha1: ff2f4e16e02f66b31c595196052a809100cfd5a8
+- name: bosh-aws-cpi
+  url: https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-aws-cpi-release?v=52
+  sha1: dc4a0cca3b33dce291e4fbeb9e9948b6a7be3324
+
+resource_pools:
+- name: vms
+  network: private
+  stemcell:
+    url: https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent?v=3232.3
+    sha1: 1fe87c0146ad1f3b55eeed5a80ce35c01b4eb6d9
+  cloud_properties:
+    instance_type: m3.large
+    ephemeral_disk: {size: 25_000, type: gp2}
+    availability_zone: $AWS_AZ
+
+disk_pools:
+- name: disks
+  disk_size: 20_000
+  cloud_properties: {type: gp2}
+
+networks:
+- name: private
+  type: manual
+  subnets:
+  - range: 10.0.0.0/24
+    gateway: 10.0.0.1
+    dns: [10.0.0.2]
+    cloud_properties: {subnet: $SUBNET}
+- name: public
+  type: vip
+
+jobs:
+- name: bosh
+  instances: 1
+
+  templates:
+  - {name: nats, release: bosh}
+  - {name: postgres, release: bosh}
+  - {name: blobstore, release: bosh}
+  - {name: director, release: bosh}
+  - {name: health_monitor, release: bosh}
+  - {name: registry, release: bosh}
+  - {name: aws_cpi, release: bosh-aws-cpi}
+
+  resource_pool: vms
+  persistent_disk_pool: disks
+
+  networks:
+  - name: private
+    static_ips: [10.0.0.6]
+    default: [dns, gateway]
+  - name: public
+    static_ips: [$EIP]
+
+  properties:
+    nats:
+      address: 127.0.0.1
+      user: nats
+      password: $BOSH_PASSWORD
+
+    postgres: &db
+      listen_address: 127.0.0.1
+      host: 127.0.0.1
+      user: postgres
+      password: $BOSH_PASSWORD
+      database: bosh
+      adapter: postgres
+
+    registry:
+      address: 10.0.0.6
+      host: 10.0.0.6
+      db: *db
+      http: {user: admin, password: $BOSH_PASSWORD, port: 25777}
+      username: admin
+      password: $BOSH_PASSWORD
+      port: 25777
+
+    blobstore:
+      address: 10.0.0.6
+      port: 25250
+      provider: dav
+      director: {user: director, password: $BOSH_PASSWORD}
+      agent: {user: agent, password: $BOSH_PASSWORD}
+
+    director:
+      address: 127.0.0.1
+      name: eb-bosh
+      db: *db
+      cpi_job: aws_cpi
+      max_threads: 10
+      user_management:
+        provider: local
+        local:
+          users:
+          - {name: admin, password: $BOSH_PASSWORD}
+          - {name: hm, password: $BOSH_PASSWORD}
+
+    hm:
+      director_account: {user: hm, password: $BOSH_PASSWORD}
+      resurrector_enabled: true
+
+    aws: &aws
+      access_key_id: $AWS_ACCESS_KEY_ID
+      secret_access_key: $AWS_SECRET_ACCESS_KEY
+      default_key_name: $AWS_KEYPAIR_KEY_NAME
+      default_security_groups: [$SECURITY_GROUP]
+      region: $AWS_REGION
+
+    agent: {mbus: "nats://nats:$BOSH_PASSWORD@10.0.0.6:4222"}
+
+    ntp: &ntp [0.pool.ntp.org, 1.pool.ntp.org]
+
+cloud_provider:
+  template: {name: aws_cpi, release: bosh-aws-cpi}
+
+  ssh_tunnel:
+    host: $EIP # <--- Replace with your Elastic IP address
+    port: 22
+    user: vcap
+    private_key: $PRIVATE_KEY_PATH # Path relative to this manifest file
+
+  mbus: "https://mbus:$BOSH_PASSWORD@$EIP:6868" # <--- Replace with Elastic IP
+
+  properties:
+    aws: *aws
+    agent: {mbus: "https://mbus:$BOSH_PASSWORD@0.0.0.0:6868"}
+    blobstore: {provider: local, path: /var/vcap/micro_bosh/data/cache}
+    ntp: *ntp
+YAML