kite 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35907f92cd177fd33f70eb0bc63657862c640bc7
-  data.tar.gz: 06e20e23873eafdc2c5f9def0d1877a8bbfbb243
+  metadata.gz: 1350153f9c5242c19356d1c6797de0d7897b990f
+  data.tar.gz: 9036d77e615d36dba63fc56a1336e5888de7c757
 SHA512:
-  metadata.gz: e39367feaa741376f65874369ff4ba2e1a2fd25ea412fcee9e32b47ee153506ca04eb309b45fc33439f0cfa7cf97e16895424a933ae578875e13ea50439d80cb
-  data.tar.gz: a710b0698232d8110005af89d5cec87d9ecc5b87088227118bd9c7507bfa72033a528a5dddb0d38d91c52ff7007d0cb135e326ea69aec3d446433c168249cff5
+  metadata.gz: 22c58c7d7b01bcc7affe6bc0bac131b231ebd7bb49b56e298f5d91a6d66a273d8e4783aa5db14f6b2bf67e63d41e95aa4f9ae4ef151e253779ff757ae3588491
+  data.tar.gz: fbe6f97793d40e992194dec5f03b4f1d98624c1be3dd78a2bd6acbbe4e54a2d32ce45acc97552da3ad2f9ff898a0ce1893fedaae6c20d5724c5af04cc6efb51d

data/lib/kite/core.rb

@@ -7,7 +7,6 @@ module Kite
       File.expand_path(File.join(File.dirname(__FILE__), "../../tpl"))
     end
 
-    method_option :cloud, type: :string, desc: "Cloud provider", enum: %w{aws gcp}
     desc "new CLOUD_PATH", "Generate Cloud infrastructure skeleton from configuration"
     def new(cloud_name)
       target = Kite::Cloud.new(self, cloud_name)
@@ -21,36 +20,54 @@ module Kite
       @values = YAML.load(File.read('config/cloud.yml'))
 
       case options[:cloud]
-      when "aws"
-        copy_file("aws/bin/make_cloud_config.sh",                 "bin/make_cloud_config.sh")
-        copy_file("aws/bin/make_manifest_bosh-init.sh",           "bin/make_manifest_bosh-init.sh")
-        copy_file("aws/bin/make_manifest_concourse-cluster.sh",   "bin/make_manifest_concourse-cluster.sh")
-
-        copy_file("aws/terraform/aws-concourse.tf",               "terraform/aws-concourse.tf")
-        copy_file("aws/terraform/aws-vault.tf",                   "terraform/aws-vault.tf")
-        copy_file("aws/terraform/bosh-aws-base.tf",               "terraform/bosh-aws-base.tf")
-        copy_file("aws/terraform/outputs.tf",                     "terraform/outputs.tf")
-        copy_file("aws/terraform/variables.tf",                   "terraform/variables.tf")
-        copy_file("aws/terraform/variables.tf",                   "terraform/variables.tf")
-
-        template("aws/env.example.erb",                           ".env")
-        copy_file("aws/README.md",                                "README.md")
-        copy_file("aws/bootstrap.sh",                             "bootstrap.sh")
-
-      when "gcp"
-        template("gcp/manifest.yml.erb",            "manifest.yml")
-        template("gcp/cloud-config.yml.erb",        "cloud-config.yml")
-        copy_file("gcp/concourse.yml.erb",          "concourse.yml")
-        copy_file("gcp/README.md",                  "README.md")
-        directory("gcp/scripts",                    "scripts")
-        copy_file("gcp/INSTALL.md",                 "INSTALL.md")
-        template("gcp/env.example.erb",             ".env")
-        copy_file("gcp/main.tf",                    "main.tf")
-        copy_file("gcp/concourse.tf",               "concourse.tf")
+      when 'aws'
+        copy_file('aws/terraform/main.tf',                 'terraform/main.tf')
+        copy_file('aws/terraform/network.tf',              'terraform/network.tf')
+        copy_file('aws/terraform/outputs.tf',              'terraform/outputs.tf')
+        copy_file('aws/terraform/variables.tf',            'terraform/variables.tf')
+        template('aws/terraform/terraform.tfvars.erb',     'terraform/terraform.tfvars')
+
+        copy_file('aws/README.md',                         'README.md')
+        copy_file('aws/bootstrap.sh',                      'bootstrap.sh')
+
+      when 'gcp'
+        copy_file('gcp/terraform/main.tf',                  'terraform/main.tf')
+        copy_file('gcp/terraform/network.tf',               'terraform/network.tf')
+        copy_file('gcp/terraform/outputs.tf',               'terraform/outputs.tf')
+        copy_file('gcp/terraform/variables.tf',             'terraform/variables.tf')
+        template('gcp/terraform/terraform.tfvars.erb',      'terraform/terraform.tfvars')
+        template('gcp/bosh-install.sh.erb',                 'bin/bosh-install.sh')
+        chmod('bin/bosh-install.sh', 0755)
+
+      else
+        say 'Cloud provider not specified'
+
+      end
+    end
+
+    desc 'render MANIFEST', 'Render manifest file from configuration and Terraform output'
+    def render(manifest)
+      say "Rendering #{ manifest } manifest", :green
+      @values = YAML.load(File.read('config/cloud.yml'))
+      @tf_output = parse_tf_state('terraform/terraform.tfstate')
+
+      case manifest
+      when "bosh"
+        template("aws/bosh/bosh_director.yml.erb",    "bosh_director.yml")
+
+      when "concourse"
+        template("aws/concourse/aws_cloud.yml.erb",   "aws_cloud.yml")
+        template("aws/concourse/concourse.yml.erb",   "concourse.yml")
+
       else
-        say "Cloud provider not specified"
+        say "Manifest type not specified"
 
       end
     end
+
+    desc "version", "Return kite version"
+    def version
+      say "v#{ Kite::VERSION }"
+    end
   end
 end

data/lib/kite/helpers.rb

@@ -1,2 +1,9 @@
 module Kite::Helpers
+  # Parse Terraform .tfstate file, returning the output hash
+  def parse_tf_state(path)
+    tf_state = YAML.load(File.open(path))
+    tf_output = tf_state["modules"].first["outputs"]
+    tf_output.map { |k, v| tf_output[k] = v["value"] }
+    tf_output
+  end
 end

data/lib/kite/version.rb

@@ -1,3 +1,3 @@
 module Kite
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/tpl/aws/bootstrap.sh

@@ -1,24 +1,21 @@
 #!/usr/bin/env bash
 
-source ./.env
 
 pushd terraform && terraform apply && popd
 
-bash ./bin/make_manifest_bosh-init.sh
-bosh-init deploy bosh-director.yml
+kite render-manifest --manifest=bosh
+bosh-init deploy bosh_director.yml
 
-read -p "Enter bosh director ip: " bosh_director_ip
 pushd terraform && BOSH_DIRECTOR_IP=$(terraform output eip) && popd
 bosh target $BOSH_DIRECTOR_IP
 
-bash ./bin/make_cloud_config.sh
-bosh update cloud-config aws-cloud.yml
+kite render-manifest --manifest=concourse
+bosh update cloud-config aws_cloud.yml
 
 bosh upload stemcell https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent
 bosh upload release https://bosh.io/d/github.com/concourse/concourse
 bosh upload release https://bosh.io/d/github.com/cloudfoundry-incubator/garden-runc-release
 
-bash ./bin/make_manifest_concourse-cluster.sh
 bosh deployment concourse.yml
 
 bosh deploy

data/tpl/aws/{bin/make_manifest_bosh-init.sh → bosh/bosh_director.yml.erb}

@@ -1,31 +1,3 @@
-#!/bin/bash
-#
-#  Please set the following environment variables:
-#  $AWS_ACCESS_KEY_ID
-#  $AWS_SECRET_ACCESS_KEY
-#  $AWS_REGION
-#  $AWS_AZ
-#  $BOSH_PASSWORD
-#  $AWS_KEYPAIR_KEY_NAME
-#  $PRIVATE_KEY_PATH
-
-function getvars() {
-  cd terraform/
-  EIP=$(terraform output eip)
-  SUBNET=$(terraform output default_subnet_id)
-  SECURITY_GROUP=$(terraform output security_group_id)
-  cd ../
-}
-
-getvars
-
-echo "Subnet = $SUBNET"
-echo "Security Group = $SECURITY_GROUP"
-echo "EIP = $EIP"
-echo "AWS REGION = $AWS_REGION"
-echo "AWS AZ = $AWS_AZ"
-
-cat >bosh-director.yml <<YAML
 ---
 name: bosh
 
@@ -46,7 +18,7 @@ resource_pools:
   cloud_properties:
     instance_type: m3.large
     ephemeral_disk: {size: 25_000, type: gp2}
-    availability_zone: $AWS_AZ
+    availability_zone: <%= @values['aws']['az'] %>
 
 disk_pools:
 - name: disks
@@ -60,9 +32,7 @@ networks:
   - range: 10.0.0.0/24
     gateway: 10.0.0.1
     dns: [10.0.0.2]
-    cloud_properties: {subnet: $SUBNET}
-- name: public
-  type: vip
+    cloud_properties: {subnet: <%= @tf_output['default_subnet_id'] %>}
 
 jobs:
 - name: bosh
@@ -85,19 +55,19 @@ jobs:
     static_ips: [10.0.0.6]
     default: [dns, gateway]
   - name: public
-    static_ips: [$EIP]
+    static_ips: [<%= @tf_output['eip'] %>]
 
   properties:
     nats:
       address: 127.0.0.1
       user: nats
-      password: $BOSH_PASSWORD
+      password: <%= @values['bosh']['password'] %>
 
     postgres: &db
       listen_address: 127.0.0.1
       host: 127.0.0.1
       user: postgres
-      password: $BOSH_PASSWORD
+      password: <%= @values['bosh']['password'] %>
       database: bosh
       adapter: postgres
 
@@ -105,17 +75,17 @@ jobs:
       address: 10.0.0.6
       host: 10.0.0.6
       db: *db
-      http: {user: admin, password: $BOSH_PASSWORD, port: 25777}
+      http: {user: admin, password: <%= @values['bosh']['password'] %>, port: 25777}
       username: admin
-      password: $BOSH_PASSWORD
+      password: <%= @values['bosh']['password'] %>
       port: 25777
 
     blobstore:
       address: 10.0.0.6
       port: 25250
       provider: dav
-      director: {user: director, password: $BOSH_PASSWORD}
-      agent: {user: agent, password: $BOSH_PASSWORD}
+      director: {user: director, password: <%= @values['bosh']['password'] %>}
+      agent: {user: agent, password: <%= @values['bosh']['password'] %>}
 
     director:
       address: 127.0.0.1
@@ -127,21 +97,21 @@ jobs:
         provider: local
         local:
           users:
-          - {name: admin, password: $BOSH_PASSWORD}
-          - {name: hm, password: $BOSH_PASSWORD}
+          - {name: admin, password: <%= @values['bosh']['password'] %>}
+          - {name: hm, password: <%= @values['bosh']['password'] %>}
 
     hm:
-      director_account: {user: hm, password: $BOSH_PASSWORD}
+      director_account: {user: hm, password: <%= @values['bosh']['password'] %>}
       resurrector_enabled: true
 
     aws: &aws
-      access_key_id: $AWS_ACCESS_KEY_ID
-      secret_access_key: $AWS_SECRET_ACCESS_KEY
-      default_key_name: $AWS_KEYPAIR_KEY_NAME
-      default_security_groups: [$SECURITY_GROUP]
-      region: $AWS_REGION
+      access_key_id: <%= @values['aws']['access_key'] %>
+      secret_access_key: <%= @values['aws']['secret_key'] %>
+      default_key_name: <%= @values['bosh']['keypair_name'] %>
+      default_security_groups: [<%= @tf_output['security_group_id'] %>]
+      region: <%= @values['aws']['region'] %>
 
-    agent: {mbus: "nats://nats:$BOSH_PASSWORD@10.0.0.6:4222"}
+    agent: {mbus: "nats://nats:<%= @values['bosh']['password'] %>@10.0.0.6:4222"}
 
     ntp: &ntp [0.pool.ntp.org, 1.pool.ntp.org]
 
@@ -149,16 +119,15 @@ cloud_provider:
   template: {name: aws_cpi, release: bosh-aws-cpi}
 
   ssh_tunnel:
-    host: $EIP # <--- Replace with your Elastic IP address
+    host: <%= @tf_output['eip'] %> # <--- Replace with your Elastic IP address
     port: 22
     user: vcap
-    private_key: $PRIVATE_KEY_PATH # Path relative to this manifest file
+    private_key: <%= @values['bosh']['private_key_path'] %> # Path relative to this manifest file
 
-  mbus: "https://mbus:$BOSH_PASSWORD@$EIP:6868" # <--- Replace with Elastic IP
+  mbus: "https://mbus:<%= @values['bosh']['password'] %>@<%= @tf_output['eip'] %>:6868" # <--- Replace with Elastic IP
 
   properties:
     aws: *aws
-    agent: {mbus: "https://mbus:$BOSH_PASSWORD@0.0.0.0:6868"}
+    agent: {mbus: "https://mbus:<%= @values['bosh']['password'] %>@0.0.0.0:6868"}
     blobstore: {provider: local, path: /var/vcap/micro_bosh/data/cache}
     ntp: *ntp
-YAML

data/tpl/aws/{bin/make_cloud_config.sh → concourse/aws_cloud.yml.erb}

@@ -1,28 +1,7 @@
-#!/bin/bash
-#
-#  Please set the following environment variables:
-#  $AWS_AZ
-
-
-function getvars() {
-  cd terraform/
-  DEFAULT_SUBNET=$(terraform output default_subnet_id)
-  OPS_SUBNET=$(terraform output ops_services_subnet_id)
-  cd ../
-}
-
-getvars
-
-echo "Default Subnet = $DEFAULT_SUBNET"
-echo "Ops Services Subnet = $OPS_SUBNET"
-echo "AWS AZ" = $AWS_AZ
-
-
-cat >aws-cloud.yml <<YAML
 ---
 azs:
 - name: z1
-  cloud_properties: {availability_zone: $AWS_AZ}
+  cloud_properties: {availability_zone: <%= @values['aws']['az'] %>}
 
 vm_types:
 - name: concourse_standalone
@@ -81,7 +60,7 @@ networks:
     static: [10.0.0.6]
     reserved: [10.0.0.1-10.0.0.5]
     dns: [10.0.0.2]
-    cloud_properties: {subnet: $DEFAULT_SUBNET}
+    cloud_properties: {subnet: <%= @tf_output['default_subnet_id'] %>}
 - name: ops_services
   type: manual
   subnets:
@@ -90,7 +69,7 @@ networks:
     az: z1
     reserved: [10.0.10.1-10.0.10.5]
     dns: [10.0.0.2]
-    cloud_properties: {subnet: $OPS_SUBNET}
+    cloud_properties: {subnet: <%= @tf_output['ops_services_subnet_id'] %>}
 - name: vip
   type: vip
 
@@ -100,5 +79,3 @@ compilation:
   az: z1
   vm_type: large
   network: default
-
-YAML

data/tpl/aws/{bin/make_manifest_concourse-cluster.sh → concourse/concourse.yml.erb}

@@ -1,21 +1,7 @@
-#!/bin/bash
-#
-#  Please set the following environment variables:
-#  $DB_PASSWORD
-#  $CONCOURSE_URL
-#  $CONCOURSE_AUTH_USERNAME
-#  $CONCOURSE_AUTH_PASSWORD
-
-DIRECTOR_UUID=`bosh status --uuid`
-
-echo "director_uuid = $DIRECTOR_UUID"
-echo "concourse url = $CONCOURSE_URL"
-
-cat >concourse.yml <<YAML
 ---
 name: concourse
 
-director_uuid: $DIRECTOR_UUID
+director_uuid: <%= %x(bosh status --uuid) %>
 
 releases:
 - name: concourse
@@ -40,10 +26,10 @@ instance_groups:
     release: concourse
     properties:
       # replace with your CI's externally reachable URL e.g https://blah
-      external_url: $CONCOURSE_URL
+      external_url: <%= @values['concourse']['url'] %>
 
-      basic_auth_username: $CONCOURSE_AUTH_USERNAME
-      basic_auth_password: $CONCOURSE_AUTH_PASSWORD
+      basic_auth_username: <%= @values['concourse']['auth_username'] %>
+      basic_auth_password: <%= @values['concourse']['auth_password'] %>
 
       postgresql_database: &atc_db atc
   - name: tsa
@@ -65,7 +51,7 @@ instance_groups:
       - name: *atc_db
         # make up a role and password
         role: dbrole
-        password: $DB_PASSWORD
+        password: <%= @values['concourse']['db_password'] %>
 
 - name: worker
   instances: 1
@@ -93,4 +79,3 @@ update:
   serial: false
   canary_watch_time: 1000-60000
   update_watch_time: 1000-60000
-YAML

data/tpl/aws/terraform/main.tf

@@ -0,0 +1,37 @@
+# Specify the provider and access details
+provider "aws" {
+  region = "${var.aws_region}"
+  access_key = "${var.aws_access_key}"
+  secret_key = "${var.aws_secret_key}"
+}
+
+resource "aws_key_pair" "platform_key" {
+  key_name   = "${var.keypair_name}"
+  public_key = "${file("${var.public_key}")}"
+}
+
+resource "aws_instance" "bastion" {
+  ami = "${lookup(var.aws_amis, var.aws_region)}"
+  instance_type = "t2.small"
+  key_name = "${var.keypair_name}"
+
+  vpc_security_group_ids = ["${aws_security_group.bosh_sg.id}"]
+  subnet_id = "${aws_subnet.platform.id}"
+
+  associate_public_ip_address = true
+
+  tags {
+    Name = "bastion"
+  }
+
+  connection {
+    user = "ubuntu"
+    private_key = "${file(var.private_key)}"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "curl -fsSL get.docker.com | sh"
+    ]
+  }
+}

data/tpl/aws/terraform/network.tf

@@ -0,0 +1,177 @@
+# Create a VPC to launch our instances into
+resource "aws_vpc" "platform" {
+  cidr_block = "${var.aws_vpc_cidr_block}"
+
+  tags {
+    Name = "${var.aws_vpc_name}"
+    Component = "kite-platform"
+  }
+}
+
+# Create an internet gateway to give our subnet access to the outside world
+resource "aws_internet_gateway" "platform" {
+  vpc_id = "${aws_vpc.platform.id}"
+  tags {
+    Name = "platform-gateway"
+    Component = "kite-platform"
+  }
+}
+
+# Grant the VPC internet access on its main route table
+resource "aws_route" "internet_access" {
+  route_table_id = "${aws_vpc.platform.main_route_table_id}"
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id = "${aws_internet_gateway.platform.id}"
+}
+
+# Create a subnet to launch our instances into
+resource "aws_subnet" "platform" {
+  vpc_id = "${aws_vpc.platform.id}"
+  availability_zone = "${var.aws_availability_zone}"
+  cidr_block = "${var.aws_platform_subnet_cidr_block}"
+  map_public_ip_on_launch = false
+  tags {
+    Name = "${var.aws_platform_subnet_name}"
+    Component = "kite-platform"
+  }
+}
+
+# Create an ops_services subnet
+resource "aws_subnet" "ops_services" {
+  vpc_id = "${aws_vpc.platform.id}"
+  availability_zone = "${var.aws_availability_zone}"
+  cidr_block = "${var.aws_ops_subnet_cidr_block}"
+  map_public_ip_on_launch = false
+  tags {
+    Name = "${var.aws_ops_subnet_name}"
+    Component = "ops-services"
+  }
+}
+
+# The default security group
+resource "aws_security_group" "bosh_sg" {
+  name = "bosh_sg"
+  description = "Default BOSH security group"
+  vpc_id = "${aws_vpc.platform.id}"
+  tags {
+    Name = "bosh-sq"
+    Component = "bosh-director"
+  }
+
+  # inbound access rules
+  ingress {
+    from_port = 6868
+    to_port = 6868
+    protocol = "tcp"
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port = 25555
+    to_port = 25555
+    protocol = "tcp"
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port = 22
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port = 0
+    to_port = 65535
+    protocol = "tcp"
+    self = true
+  }
+
+  ingress {
+    from_port = 0
+    to_port = 65535
+    protocol = "udp"
+    self = true
+  }
+
+  # outbound internet access
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+}
+
+# Create a Concourse security group
+resource "aws_security_group" "concourse_sg" {
+  name        = "concourse-sg"
+  description = "Concourse security group"
+  vpc_id      = "${aws_vpc.platform.id}"
+  tags {
+    Name = "concourse-sg"
+    Component = "concourse"
+  }
+
+  # outbound internet access
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  # inbound connections from ELB
+  ingress {
+    from_port   = 8080
+    to_port     = 8080
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port = 8080
+    to_port = 8080
+    protocol = "tcp"
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port   = 2222
+    to_port     = 2222
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# Create a Vault security group
+resource "aws_security_group" "vault_sg" {
+  name        = "vault-sg"
+  description = "Vault security group"
+  vpc_id      = "${aws_vpc.platform.id}"
+  tags {
+    Name = "vault-sg"
+    Component = "vault"
+  }
+
+  # outbound internet access
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  # inbound http
+  ingress {
+    from_port   = 8200
+    to_port     = 8200
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}