concourse-deployer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e35d5f547ba50ab86ab55fa4e7767cf17ac17e051e7fc2908f192e5633528da2
-  data.tar.gz: bfd4742437ff9911aca9b9d7302a3df8ebb3c2e8cf83651120ad779da8ac2c7d
+  metadata.gz: 25c2a4453ab7efcd1e0866deea1dff9ff2a343c7977e0ac62496409eeb8cf99e
+  data.tar.gz: d8400270b0a80d7238ecd774596c451cd6b4e8f0f88e5ad44f7f6f36eb2c5329
 SHA512:
-  metadata.gz: ff1ce32001402f065b54fe410a0f690b729a6d1a4e5ca1a73065c7daadbde7e913c040b462556b1481407781d10aeb3986d342276e623327dce915cf645713db
-  data.tar.gz: 24a6f65a6bd1d0facd44948f89a88c5df47b3969ab6f36e68e897883334b5e18dc48739acf8591e73a3b5711e85348c1b871b5fd811e83c5a1982a0d197548c0
+  metadata.gz: a2c7aa9c926366bca72b17d43184ca815cd4cfd23e642a7f87c033e61a4ba116d0d9647853713b45471944c3b09d2d156775f92ab61c1d4a2d55db4d3541aaa3
+  data.tar.gz: cd7b31dce5e2c977d503c4b36cba068536b622b618993ca26fd5b791b7848de298da05b342cd4134d12c42e3a6627b127ee8dfa77fecd15a43dfc207fa6ad547

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog for `concourse-deployer`
 
+## v0.2.0 / 2019-02-10
+
+Features:
+
+- Use Caddy (via caddy-bosh-release) for managing LetsEncrypt certificates.
+- `scale-vars.yml` is now `deployment-vars.yml` and presents additional customizable variables.
+- New task `db:connect` for getting a postgres commandline prompt.
+- New task `bosh:interpolate` for examining the final BOSH manifest
+
+
 ## v0.1.0 / 2019-01-04
 
 First release.

data/README.md

@@ -4,7 +4,7 @@ Provides easy installation and maintenance of an opinionated [Concourse](https:/
 
 - external Postgres database
 - Github auth integration
-- LetsEncrypt integration for SSL cert management
+- LetsEncrypt integration, via [caddy](https://caddyserver.com/) and [caddy-bosh-release](https://github.com/dpb587/caddy-bosh-release)
 - Windows™ workers
 
 Today this only supports deployment to GCP.
@@ -22,12 +22,8 @@ rake bosh:update
 rake bosh:deploy
 ```
 
-You can create and deploy a LetsEncrypt SSL cert:
+During `bbl:gcp:init` and `bosh:init` you'll be prompted interactively for any necessary information. Note that you need a DNS domain name in order for Caddy to create and manage your SSL certs.
 
-``` sh
-rake letsencrypt:create letsencrypt:backup letsencrypt:import
-rake bosh:deploy
-```
 
 ## Requirements
 
@@ -65,17 +61,15 @@ Concourse::Deployer.new.create_tasks!
 Available tasks:
 
 ``` sh
-rake bbl:gcp:init[gcp_project_id]  # initialize bosh-bootloader for GCP
-rake bbl:gcp:up                    # terraform your environment and deploy the bosh director
-rake bosh:deploy                   # deploy concourse
-rake bosh:init                     # prepare the concourse bosh deployment
-rake bosh:update                   # upload stemcells and releases to the director
-rake bosh:update:ubuntu_stemcell   # upload ubuntu stemcell to the director
-rake letsencrypt:backup            # backup web:/etc/letsencrypt to local disk
-rake letsencrypt:create            # create a cert
-rake letsencrypt:import            # import letsencrypt keys into `secrets.yml` from backup
-rake letsencrypt:renew             # renew the certificate
-rake letsencrypt:restore           # restore web:/etc/letsencrypt from backup
+rake bbl:gcp:init[gcp_project_id]      # initialize bosh-bootloader for GCP
+rake bbl:gcp:up                        # terraform your environment and deploy the bosh director
+rake bosh:deploy                       # deploy concourse
+rake bosh:init                         # prepare the concourse bosh deployment
+rake bosh:interpolate                  # view interpolated manifest
+rake bosh:update                       # macro task for all `update` subtasks
+rake bosh:update:concourse_deployment  # update the git submodule for concourse-bosh-deployment
+rake bosh:update:ubuntu_stemcell       # upload ubuntu stemcell to the director
+rake db:connect                        # connect to the postgres database
 ```
 
 See full instructions below.
@@ -92,7 +86,6 @@ Files which contain sensitive data:
 * `secrets.yml`
 * `cluster-creds.yml`
 * the `vars` subdirectory
-* `letsencrypt.tar.gz` (if you're using the letsencrypt SSL cert functionality)
 
 You will see these files listed in `.gitattributes` invoking git-crypt for them.
 
@@ -212,33 +205,40 @@ __NOTE:__ This task is idempotent! Yay Bosh.
 
 ### Scale your Concourse deployment
 
-Your first deployment will spin up one (1) web VM, and two (2) Linux worker VMs. But you can scale these numbers up as needed by editing the file `scale-vars.yml`, whose default contents looks like:
+Your first deployment will spin up one (1) web VM, and two (2) Linux worker VMs. But you can scale these numbers up as needed by editing the file `deployment-vars.yml`, whose default contents include the values:
 
 ```yaml
 ---
 web_instances: 1
 worker_instances: 2
+web_vm_type: default
+worker_vm_type: default
+worker_ephemeral_disk: 50GB_ephemeral_disk
 ```
 
 Edit this file as appropriate for your needs, and re-run `rake bosh:deploy`.
 
 
-### Manage your letsencrypt SSL cert
+### Custom bosh ops files
+
+If you want to perform any custom operations on the manifest, put them in a file named `operations.yml` and they'll be pulled in as the __final__ ops file during deployment.
+
+
+### Connect to the database
+
+If you ever need to connect to the database, here's how:
 
 ``` sh
-$ rake letsencrypt:backup
-$ rake letsencrypt:create
-$ rake letsencrypt:restore
-$ rake letsencrypt:import
-$ rake letsencrypt:renew
+rake db:connect
 ```
 
-__NOTE:__ These tasks will create and use `letsencrypt.tar.gz` which contains sensitive data.
-
+This will:
 
-### Custom bosh ops files
+* securely write your SSL cert, key, and CA cert to disk
+* run `psql` and connect to the database
+* clean up the cert and key files
 
-If you want to perform any custom operations on the manifest, put them in a file named `operations.yml` and they'll be pulled in as the __final__ ops file during deployment.
+Note that you will need to type in your database password; this is located in `secrets.yml`.
 
 
 ## Upgrading `bbl`
@@ -286,8 +286,8 @@ The gem is available as open source under the terms of the [MIT License](http://
 - [x] +      x_frame_options: "SAMEORIGIN"
 - [x] +      container_placement_strategy: random
 - [ ] enable encryption https://concourse.ci/encryption.html
-- [ ] allow scaling up/down by locally setting number of VMs (currently hardcoded in gem)
-- [ ] start using https://github.com/dpb587/caddy-bosh-release instead of the letsencrypt rake tasks
+- [x] allow scaling up/down by locally setting number of VMs (currently hardcoded in gem)
+- [x] start using https://github.com/dpb587/caddy-bosh-release instead of the letsencrypt rake tasks
 
 
 Things to follow up on:

data/lib/concourse/deployer.rb

@@ -5,26 +5,25 @@ require "open-uri"
 require "nokogiri"
 require "yaml"
 require "rake"
+require "tempfile"
 
 module Concourse
   class Deployer
     include Rake::DSL
     include Concourse::Deployer::Utils
 
-    GCP_SERVICE_ACCOUNT_FILE = "service-account.key.json"
-    ENVRC_FILE               = ".envrc"
+    GCP_SERVICE_ACCOUNT_FILE  = "service-account.key.json"
+    ENVRC_FILE                = ".envrc"
 
-    BBL_STATE_FILE           = "bbl-state.json"
-    BBL_VARS_DIR             = "vars"
+    BBL_STATE_FILE            = "bbl-state.json"
+    BBL_VARS_DIR              = "vars"
 
-    BOSH_DEPLOYMENT          = "concourse"
-    BOSH_SECRETS             = "secrets.yml"
-    BOSH_VARS_STORE          = "cluster-creds.yml"
-    BOSH_OPERATIONS          = "operations.yml"
+    BOSH_DEPLOYMENT           = "concourse"
+    BOSH_SECRETS              = "secrets.yml"
+    BOSH_VARS_STORE           = "cluster-creds.yml"
+    BOSH_OPERATIONS           = "operations.yml"
 
-    CONCOURSE_SCALE_VARS     = "scale-vars.yml"
-
-    LETSENCRYPT_BACKUP_FILE  = "letsencrypt.tar.gz"
+    CONCOURSE_DEPLOYMENT_VARS = "deployment-vars.yml"
 
     def bbl_init
       unless_which "bbl", "https://github.com/cloudfoundry/bosh-bootloader/releases"
@@ -118,6 +117,16 @@ module Concourse
           end
         end
       end
+
+      ensure_file CONCOURSE_DEPLOYMENT_VARS do |f|
+        f.write({
+                  "web_instances" => 1,
+                  "worker_instances" => 2, # 1
+                  "web_vm_type" => "default",
+                  "worker_vm_type" => "default", # "n1-standard-2"
+                  "worker_ephemeral_disk" => "50GB_ephemeral_disk",
+                }.to_yaml)
+      end
     end
 
     def bosh_update_concourse_deployment
@@ -146,33 +155,31 @@ module Concourse
     #   bosh_update_release "cloudfoundry-incubator/windows-utilities-release"
     # end
 
-    def bosh_deploy
+    def bosh_deploy command: "deploy"
       unless File.exists?(BOSH_SECRETS)
         error "File #{BOSH_SECRETS} does not exist. Please run `rake bosh:init` first."
       end
 
+      unless File.exists?(CONCOURSE_DEPLOYMENT_VARS)
+        error "File #{CONCOURSE_DEPLOYMENT_VARS} does not exist. Please run `rake bosh:init` first."
+      end
+
       ensure_in_gitcrypt BOSH_SECRETS
       ensure_in_gitcrypt BOSH_VARS_STORE
 
-      ensure_file CONCOURSE_SCALE_VARS do |f|
-        f.write({"web_instances" => 1, "worker_instances" => 2}.to_yaml)
-      end
-
       external_dns_name = bosh_secrets['external_dns_name']
       external_url = "https://#{external_dns_name}"
 
+      ops_files = Dir[File.join(File.dirname(__FILE__), "deployer", "operations", "*.yml")]
+
       # command will be run in the bosh deployment submodule's cluster directory
       command = [].tap do |c|
-        c << "bosh deploy concourse.yml"
+        c << "bosh #{command} concourse.yml"
         # c << "--no-redact" # DEBUG
         c << "-l ../versions.yml"
         c << "-l ../../#{BOSH_SECRETS}"
         c << "--vars-store ../../#{BOSH_VARS_STORE}"
         c << "-o operations/basic-auth.yml"
-        c << "-o operations/privileged-http.yml"
-        c << "-o operations/privileged-https.yml"
-        c << "-o operations/tls.yml"
-        c << "-o operations/tls-vars.yml"
         c << "-o operations/web-network-extension.yml"
         c << "-o operations/external-postgres.yml"
         c << "-o operations/external-postgres-tls.yml"
@@ -186,13 +193,13 @@ module Concourse
         c << "--var network_name=default"
         c << "--var external_host='#{external_dns_name}'"
         c << "--var external_url='#{external_url}'"
-        c << "--var web_vm_type=default"
-        c << "--var worker_vm_type=default"
-        c << "--var worker_ephemeral_disk=50GB_ephemeral_disk"
         c << "--var deployment_name=#{BOSH_DEPLOYMENT}"
         c << "--var web_network_name=private"
         c << "--var web_network_vm_extension=lb"
-        c << "-l ../../#{CONCOURSE_SCALE_VARS}"
+        c << "-l ../../#{CONCOURSE_DEPLOYMENT_VARS}"
+        ops_files.each do |ops_file|
+          c << "-o #{ops_file}"
+        end
       end.join(" ")
 
       Dir.chdir("concourse-bosh-deployment/cluster") do
@@ -200,66 +207,26 @@ module Concourse
       end
     end
 
-    def letsencrypt_create
-      external_dns_name = bosh_secrets['external_dns_name']
-      if external_dns_name == bbl_external_ip
-        error "Please set your external DNS name in #{BOSH_SECRETS}"
-      end
-
-      sh "bosh ssh web -c 'sudo chmod 777 /tmp'"
-      sh "bosh ssh web -c 'sudo add-apt-repository -y ppa:certbot/certbot'"
-      sh "bosh ssh web -c 'sudo apt-get update'"
-      sh "bosh ssh web -c 'sudo apt-get install -y certbot'"
+    def db_connect
+      tempfile_cert = Tempfile.new
+      tempfile_key = Tempfile.new
+      tempfile_ca = Tempfile.new
       begin
-        sh "bosh stop web"
-        note "logging you into the web server. run this command: sudo certbot certonly --standalone -d \"#{external_dns_name}\""
-        sh "bosh ssh web"
-      ensure
-        sh "bosh start web"
-      end
-    end
+        tempfile_cert.write bosh_secrets['postgres_client_cert']['certificate']
+        tempfile_key.write bosh_secrets['postgres_client_cert']['private_key']
+        tempfile_ca.write bosh_secrets['postgres_ca_cert']['certificate']
 
-    def letsencrypt_backup
-      ensure_in_gitcrypt LETSENCRYPT_BACKUP_FILE
-      sh %Q{bosh ssh web -c 'sudo tar -zcvf /var/tmp/#{LETSENCRYPT_BACKUP_FILE} -C /etc letsencrypt'}
-      sh %Q{bosh scp web:/var/tmp/#{LETSENCRYPT_BACKUP_FILE} .}
-    end
+        tempfile_cert.close
+        tempfile_key.close
+        tempfile_ca.close
 
-    def letsencrypt_import
-      ensure_in_gitcrypt LETSENCRYPT_BACKUP_FILE
-      external_dns_name = bosh_secrets['external_dns_name']
+        command = %Q{psql "sslmode=verify-ca sslrootcert=#{tempfile_ca.path} sslcert=#{tempfile_cert.path} sslkey=#{tempfile_key.path} hostaddr=#{bosh_secrets['postgres_host']} user=#{bosh_secrets['postgres_role']} dbname=atc"}
 
-      begin
-        sh "tar -zxf #{LETSENCRYPT_BACKUP_FILE}"
-        note "importing certificate and private key for #{external_dns_name} ..."
-        bosh_secrets do |v|
-          v["atc_tls"] ||= {}
-          v["atc_tls"]["certificate"] = File.read "letsencrypt/live/#{external_dns_name}/fullchain.pem"
-          v["atc_tls"]["private_key"] = File.read "letsencrypt/live/#{external_dns_name}/privkey.pem"
-        end
-      ensure
-        sh "rm -rf letsencrypt"
-      end
-    end
-
-    def letsencrypt_restore
-      ensure_in_gitcrypt LETSENCRYPT_BACKUP_FILE
-      sh "bosh ssh web -c 'sudo rm -rf /etc/letsencrypt /var/tmp/#{LETSENCRYPT_BACKUP_FILE}'"
-      sh "bosh scp #{LETSENCRYPT_BACKUP_FILE} web:/var/tmp"
-      sh "bosh ssh web -c 'sudo tar -zxvf /var/tmp/#{LETSENCRYPT_BACKUP_FILE} -C /etc'"
-      sh "bosh ssh web -c 'sudo chown -R root:root /etc/letsencrypt'"
-    end
-
-    def letsencrypt_renew
-      sh "bosh ssh web -c 'sudo chmod 1777 /tmp'" # see https://github.com/cloudfoundry/bosh-linux-stemcell-builder/issues/39
-      sh "bosh ssh web -c 'sudo add-apt-repository -y ppa:certbot/certbot'"
-      sh "bosh ssh web -c 'sudo apt-get update'"
-      sh "bosh ssh web -c 'sudo apt-get install -y certbot'"
-      begin
-        sh "bosh stop web"
-        sh "bosh ssh web -c 'sudo certbot renew'"
+        sh command
       ensure
-        sh "bosh start web"
+        tempfile_cert.unlink
+        tempfile_key.unlink
+        tempfile_ca.unlink
       end
     end
 
@@ -330,32 +297,17 @@ module Concourse
         task "deploy" do
           bosh_deploy
         end
-      end
-
-      namespace "letsencrypt" do
-        desc "create a cert"
-        task "create" do
-          letsencrypt_create
-        end
-
-        desc "backup web:/etc/letsencrypt to local disk"
-        task "backup" do
-          letsencrypt_backup
-        end
-
-        desc "import letsencrypt keys into `#{BOSH_SECRETS}` from backup"
-        task "import" do
-          letsencrypt_import
-        end
 
-        desc "restore web:/etc/letsencrypt from backup"
-        task "restore" do
-          letsencrypt_restore
+        desc "view interpolated manifest"
+        task "interpolate" do
+          bosh_deploy command: "interpolate"
         end
+      end
 
-        desc "renew the certificate"
-        task "renew" do
-          letsencrypt_renew
+      namespace "db" do
+        desc "connect to the postgres database"
+        task "connect" do
+          db_connect
         end
       end
     end

data/lib/concourse/deployer/operations/caddy.yml

@@ -0,0 +1,28 @@
+- path: /releases/name=caddy?
+  type: replace
+  value:
+    name: "caddy"
+    version: "0.4.1"
+    url: "https://bosh.io/d/github.com/dpb587/caddy-bosh-release?v=0.4.1"
+    sha1: "7d9ca0c3e0bed5a68a5a202f864084108a41b47e"
+
+# see https://github.com/dpb587/caddy-bosh-release/blob/master/manifests/caddy.yml
+- path: /instance_groups/name=web/persistent_disk?
+  type: replace
+  value: 1024
+
+# see https://github.com/dpb587/caddy-bosh-release/blob/master/manifests/examples/concourse-ops.yml
+- path: /instance_groups/name=web/jobs/name=caddy?
+  type: replace
+  value:
+    name: "caddy"
+    release: "caddy"
+    properties:
+      caddyfile: |
+        ((external_host)) {
+          gzip
+          proxy / localhost:8080 {
+            transparent
+            websocket
+          }
+        }

data/lib/concourse/deployer/version.rb

@@ -1,5 +1,5 @@
 module Concourse
   class Deployer
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: concourse-deployer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Mike Dalessio
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-04 00:00:00.000000000 Z
+date: 2019-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
@@ -100,6 +100,7 @@ files:
 - bin/setup
 - concourse-deployer.gemspec
 - lib/concourse/deployer.rb
+- lib/concourse/deployer/operations/caddy.yml
 - lib/concourse/deployer/utils.rb
 - lib/concourse/deployer/version.rb
 homepage: https://github.com/flavorjones/concourse-deployer