mina-kubernetes 2.0.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f69cc691a91d27b5cefa7d4a79056184cfad126ffd833ce2c11c8c85438516ef
-  data.tar.gz: 15e3ee21f521d47f1cc4d22ab70edeb7ace125244b301a571fcb88a3ea01537d
+  metadata.gz: 7be64670afaedf5f60cb172d47331af1df66e3e843d21cdeee54851f2aafb5fa
+  data.tar.gz: 9eea0c759fd4823586b551413b7a456b780baeb8af24a0464ea962683212023e
 SHA512:
-  metadata.gz: 4e3d514d8ee7e171f28d646f2142dc1fad2acca1416ecb30baa77742c1f046e19cd92990d88e34016d196f62fbaf7492f99778158a69504365af7c3844096a70
-  data.tar.gz: '01066097331bdac2aeba3b2228fc4d78f68511accef2ea59d111478d379bf37551b55673842dbf77b9cc1d0012222d6378328bb7538dde1c9ea46335c3121be9'
+  metadata.gz: 59927afc8c3504e3b381b4c749b5fb390fc2c7f20c143674811bbede57f6c2e89d6852dc9b4ecd534fd84a7bf1ef6b3aa35144c1704ec36569f157868feca7e3
+  data.tar.gz: 5b96dc9e7d993063be539d294656f2a3337290adbfaef1e104a5877c9f09736b7e3bb8ee35ee6407812ce7072d047f95c84fe04b3194b2ef0d446150b11eaf82

data/CHANGELOG.md

@@ -1,3 +1,51 @@
+## 2.5.0
+
+*Enhancements*
+
+- `kubernetes:command` starts pod with identifiable name and allows session reconnection
+- `kubernetes:command` accepts a `kubectl_pod_overrides` option
+
+## 2.4.1
+
+*Fixes*
+
+- Security: update rake dependency
+
+## 2.4.0
+
+*Enhancements*
+
+- Use `secrets.ejson` if present
+
+## 2.3.0
+
+*Enhancements*
+
+- Allow using a proxy to connect to a Kubernetes cluster
+
+## 2.2.4
+
+*Fixes*
+
+- run custom command within given namespace instead of `default` 
+
+## 2.2.1 to 2.2.3
+
+*Fixes*
+
+- handle nil/undefined options passed to `krane`
+
+## 2.2.0
+
+*Enhancements*
+
+- Using `krane` 1.0.0 (previously `kubernetes-deploy`)
+- Allow passing of options to `krane`
+
+## 2.1.0
+
+Yanked release.
+
 ## 2.0.0
 
 *Breaking*
@@ -7,4 +55,4 @@
 
 *Fixes*
 
-- Not overriding $KUBE_CONFIG environment variable anymore
+- Not overriding $KUBE_CONFIG environment variable anymore

data/README.md

@@ -1,16 +1,16 @@
 # mina-kubernetes
-Plugin for the [mina](https://github.com/mina-deploy/mina) deployment tool to streamline deployment of resources to Kubernetes cluster, using the [kubernetes-deploy](https://github.com/Shopify/kubernetes-deploy) gem and [mina-multistage](https://github.com/endoze/mina-multistage) plugin.
+ mina-kubernetes is a plugin for the [mina](https://github.com/mina-deploy/mina) deployment tool to streamline deployment of resources to Kubernetes clusters, using the [krane](https://github.com/Shopify/krane) gem with the [mina-multistage](https://github.com/endoze/mina-multistage) plugin.
 
-Requires local Docker and [kubectl](https://cloud.google.com/kubernetes-engine/docs/quickstart) with authentication set up to connect to the destination Kubernetes cluster.
+It requires local Docker and [kubectl](https://cloud.google.com/kubernetes-engine/docs/quickstart) with local authentication set up to connect to the destination Kubernetes cluster as context in your local KUBE_CONFIG. See https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl#generate_kubeconfig_entry for example with Google Kubernetes Engine.
 
-NB: `docker manifest inspect` is used to check the image is available. This requires experimental features to be enabled in your local Docker config by adding `"experimental": "enabled"` to `~/.docker/config.json`.
-If the image repository is not public authentication will need to be set up for your local Docker, for instance see https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud_as_a_docker_credential_helper for images hosted on the Google Cloud Registry
+NB: `docker manifest inspect` is used to check whether the Docker image with requested tag is available. At the time of writing this is still an experimental feature that needs to be enabled in your local Docker config by adding `"experimental": "enabled"` to `~/.docker/config.json`.
+If the image to deploy is in a private repository authentication will have to be set up for your local Docker, for instance see https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud_as_a_docker_credential_helper for images hosted on the Google Cloud Registry.
 
 ## Usage
 
-Add `mina-kubernetes` to your local Gemfile. 
+Add `mina-kubernetes` to your local Gemfile.
 
-Create a configuration file for mina in `config/deploy.rb` like the one below:
+Create a configuration file for mina in `config/deploy.rb` similar to the one below (which replaces the default deploy task):
 ```ruby
 require "mina/default"
 require "mina/multistage"
@@ -28,17 +28,40 @@ set :image_repo, "gcr.io/project-id/myapp"
 set :kubernetes_context, "kubernetes_context_name"
 ```
 
-If `set :image_tag, "my_image_tag"` is also defined, it'll be used to deploy the image tagged with this tag on the repository. Otherwise you'll be prompted to pick a branch from current working Git repository and the image to deploy will be assumed to be tagged with the Git commit hash, i.e. `gcr.io/project-123456/my_app:abcd1234`.
+If `set :image_tag, "my_image_tag"` is also defined, it'll be used to deploy the image tagged with this tag on the repository. Otherwise you'll be prompted to pick a branch from the current Git repository and the image to deploy will be assumed to be tagged with the commit hash of that branch, i.e. `gcr.io/project-id/myapp:abcd1234`.
 
-Then add `*.yml.erb` Kubernetes resource definition files in the stage folder, i.e. `config/deploy/production/app.yml.erb`. Occurences of `<%= image_repo %>` and `<%= current_sha %>` in these files will be dynamically replaced on deploy by the image repository URL and the latest commit hash of the selected branch on its git origin.
+Then add `*.yml.erb` Kubernetes resource definition files in the stage folder, for instance `config/deploy/production/webserver.yml.erb` and `config/deploy/production/backgroundjobs.yml.erb`. Occurences of `<%= image_repo %>` and `<%= current_sha %>` in these files will be dynamically replaced on deploy by the image repository URL and the latest commit hash of the selected branch on its git origin.
 
-When you run `mina production deploy`, a namespace labelled `my_app-production` will be created on the Kubernetes cluster and set as a local kubectl context. Then the resources are applied to the cluster after checking/waiting for the image to be available on the repository.
+You can also get the RAILS_MASTER_KEY for encrypted credentials deployed as a Kubernetes secrets by adding a secrets.yml.erb like below:
+```yml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: secrets
+data:
+  RAILS_MASTER_KEY: <%= Base64.strict_encode64(File.read("#{Dir.pwd}/config/credentials/production.key").strip) %>
+```
+
+When running `mina production deploy`, it'll check the image is available on the repository and then call the `krane` executable to fill in the variables in the resource templates and apply them all to the cluster under the given namespace (see https://github.com/Shopify/krane#deploy-walkthrough for more details)
+
+### EJSON Encrypted secrets
+
+Krane can dynamically generate Kubernetes secrets from an encrypted EJSON file, see: https://github.com/Shopify/krane#deploying-kubernetes-secrets-from-ejson. As per current Krane documentation "The ejson file must be included in the resources passed to --filenames, it can not be read through stdin.", so
+following convention-over-configuration principles `mina-kubernetes` checks for the presence of a file named `secrets.ejson` in the stage folder and uses it if available.
+
+### Passing custom options to krane
+
+```ruby
+  invoke :"kubernetes:deploy", "--no-prune"
+```
 
-### Tasks available
+Refer to https://github.com/Shopify/krane#usage for a complete set of options
+
+## Tasks available
 
 #### `kubernetes:deploy`
 
-Creates namespace on cluster and assigns it to a local kubectl context, prompts for git branch if no image tag specified, applies all resources to cluster after checking tagged image is available.
+Creates the namespace on cluster if it doesn't exist, prompts for a git branch if no image tag is already specified in stage file, then applies all resources to cluster after checking tagged image is available.
 
 #### `kubernetes:bash`
 
@@ -46,8 +69,26 @@ Prompts for branch unless image tag is set, then spins up a temporary pod with t
 
 #### `kubernetes:command`
 
-Prompts for branch unless image tag is set, then spins up a temporary pod with the image and run command given by task variable `command`, for instance with `set :command, "rails console"`. Environment variables can also be given by defining`env_hash`, i.e. `set :env_hash, {"RAILS_ENV" => "production", "MY_VAR" => "abcd123"}`
+Prompts for branch unless image tag is set, then spins up a temporary pod with the image and runs the command given in the task variable `command`, for instance with `set :command, "rails console"`. Environment variables can also be passed by defining`env_hash`, i.e. `set :env_hash, {"RAILS_ENV" => "production", "MY_VAR" => "abcd123"}`
+
+The pod will be named `command-username-branch`, and can be reattached/killed in case of disconnection.
+
+A `kubectl_pod_overrides` task option is available to pass a value to the `overrides` option of the `kubectl run` command.
 
 #### `kubernetes:delete`
 
-Confirms and delete all resources on cluster under namespace. 
+Confirms and delete all resources on cluster under namespace.
+
+## Example use: run rails console on non-preemptible GKE node
+
+Add the following to your `deploy.rb`
+``` ruby
+task :console do
+  set :command, "rails console"
+  set :env_hash, "RAILS_ENV" => fetch(:stage), "RAILS_MASTER_KEY" => File.read("#{Dir.pwd}/config/credentials/#{fetch(:stage)}.key").strip
+  set :kubectl_pod_overrides, '{"spec": {"affinity": {"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": {"nodeSelectorTerms": [{"matchExpressions": [{"key": "cloud.google.com/gke-preemptible", "operator": "DoesNotExist"} ] } ] } } } } }'
+
+  invoke :'kubernetes:command'
+end
+```
+You can now run `mina production console` to open a rails console in production environment with the image of your choice!

data/lib/mina/kubernetes.rb

@@ -1,34 +1,34 @@
 require "tty-prompt"
 require "tty-spinner"
-require "securerandom"
 require "json"
-require "base64"
+require "time"
 
 # required by mina
 set :execution_mode, :pretty
 
 namespace :kubernetes do
+  set :proxy, nil
 
-  task :deploy do
+  task :deploy, [:options] do |task, args|
     desc "Set image tag to be latest commit of prompted branch (unless provided) then applies resources to cluster"
     set_tag_from_branch_commit unless fetch(:image_tag)
     wait_until_image_ready(fetch(:image_tag))
     create_namespace_on_cluster
-    apply_kubernetes_resources
+    apply_kubernetes_resources(args[:options])
   end
 
   task :bash do
     desc "Spins up temporary pod with image and opens remote interactive bash"
     set_tag_from_branch_commit unless fetch(:image_tag)
     wait_until_image_ready(fetch(:image_tag))
-    run_terminal_command("bash")
+    run_command("bash")
   end
 
   task :command do
+    set :skip_report_time, true
     desc "Spins up temporary pod with image and runs given command in interactive shell, passing given environment variable"
     set_tag_from_branch_commit unless fetch(:image_tag)
-    wait_until_image_ready(fetch(:image_tag))
-    run_terminal_command(fetch(:command), env_hash_arg)
+    run_command(fetch(:command), env_hash_arg)
   end
 
   task :delete do
@@ -51,7 +51,7 @@ end
 
 def set_tag_from_branch_commit
   run :local do
-    comment "Updating Git branches..."
+    comment "Refreshing Git branches..."
   end
   remote_branches = `git fetch --prune && git branch -r --no-merged master --sort=-committerdate | grep origin`.split("\n").collect { |b| b.strip.gsub("origin/", "") }.reject { |b| b == "master" }
   set :branch, TTY::Prompt.new.select("Which branch?", ["master"].concat(remote_branches))
@@ -61,13 +61,14 @@ end
 def create_namespace_on_cluster
   run :local do
     comment "Create/update namespace on Kubernetes cluster..."
-    command "kubectl create namespace #{fetch(:namespace)} --dry-run -o yaml | kubectl apply -f - --context=#{fetch(:kubernetes_context)}"
+    proxy_env = "HTTPS_PROXY=#{fetch(:proxy)}" if fetch(:proxy)
+    command "kubectl create namespace #{fetch(:namespace)} --dry-run -o yaml | #{proxy_env} kubectl apply -f - --context=#{fetch(:kubernetes_context)}"
   end
 end
 
 def wait_until_image_ready(commit)
   run :local do
-    comment "Check image #{fetch(:image_repo)}:#{commit} is available..."
+    comment "Checking image #{fetch(:image_repo)}:#{commit} is available..."
   end
   spinner = TTY::Spinner.new
   spinner.auto_spin
@@ -81,16 +82,50 @@ def image_available?(commit)
   system("docker manifest inspect #{fetch(:image_repo)}:#{commit} > /dev/null") == true
 end
 
-def run_terminal_command(command, env_hash = {})
+def run_command(command, env_hash = {})
   env = env_hash.collect{|k,v| "--env #{k}=#{v}" }.join(" ")
-  label = command.downcase.gsub(" ", "-").gsub(":", "-")
-  # using system instead of mina's command so tty opens successfully
-  system "kubectl run #{label}-#{SecureRandom.hex(4)} --rm -i --tty --restart=Never --context=#{fetch(:kubernetes_context)} --image #{fetch(:image_repo)}:#{fetch(:image_tag)} #{env} -- #{command}"
+  label = command.downcase.gsub(" ", "-").gsub(":", "-")+ "-#{`whoami`}".strip + "-#{fetch(:branch)}" 
+  proxy_env = "HTTPS_PROXY=#{fetch(:proxy)}" if fetch(:proxy)
+
+  run :local do
+    comment "Lauching Pod #{color(label, 36)} to run #{color(command, 36)}"
+  end
+
+  pod_description = `#{proxy_env} kubectl get pod #{label} -o json --ignore-not-found --context=#{fetch(:kubernetes_context)} --namespace=#{fetch(:namespace)}`
+
+  if pod_description.empty?
+    wait_until_image_ready(fetch(:image_tag))
+    run_command = "#{proxy_env} kubectl run #{label} --rm -i --tty --restart=Never --overrides='#{fetch(:kubectl_pod_overrides)}' --context=#{fetch(:kubernetes_context)} --namespace=#{fetch(:namespace)} --image #{fetch(:image_repo)}:#{fetch(:image_tag)} #{env}"
+    system "#{run_command} -- #{command}"
+  else
+    started_at = Time.parse(JSON.parse(pod_description)["status"]["startTime"]).strftime('%b %e, %H:%M')
+    choice = TTY::Prompt.new.select("Pod already exists, running since #{started_at} UTC, what would you like to do?", {"Reattach session" => 1, "Kill it" => 0})
+    
+    attach_command = "#{proxy_env} kubectl attach #{label} -i --tty -c #{label} --context=#{fetch(:kubernetes_context)} --namespace=#{fetch(:namespace)}"
+    delete_command = "#{proxy_env} kubectl delete pod #{label} --context=#{fetch(:kubernetes_context)} --namespace=#{fetch(:namespace)}"
+    
+    if choice == 1
+      system "#{attach_command} && #{delete_command}"
+    else
+      system delete_command
+    end
+  end
 end
 
-def apply_kubernetes_resources
+def apply_kubernetes_resources(options)
   run :local do
-    comment "Apply all Kubernetes resources..."
-    command "REVISION=#{fetch(:image_tag)} kubernetes-deploy --template-dir=config/deploy/#{fetch(:stage)} --bindings=image_repo=#{fetch(:image_repo)},image_tag=#{fetch(:image_tag)},namespace=#{fetch(:namespace)} #{fetch(:namespace)} #{fetch(:kubernetes_context)}"
+    comment "Applying all Kubernetes resources..."
+
+    proxy_env = "HTTPS_PROXY=#{fetch(:proxy)}" if fetch(:proxy)
+    filepaths = options&.[](:filepaths) || "config/deploy/#{fetch(:stage)}"
+
+    render_cmd = "#{proxy_env} krane render --bindings=image_repo=#{fetch(:image_repo)},image_tag=#{fetch(:image_tag)},namespace=#{fetch(:namespace)} --current_sha #{fetch(:image_tag)} -f #{filepaths}"
+    deploy_cmd = "#{proxy_env} krane deploy #{fetch(:namespace)} #{fetch(:kubernetes_context)} --stdin "
+    deploy_cmd += options[:deployment_options] if options&.[](:deployment_options)
+
+    ejson_secrets_path = "#{filepaths}/secrets.ejson"
+    deploy_cmd += " --filenames #{ejson_secrets_path}" if File.exists?(ejson_secrets_path)
+
+    command "#{render_cmd} | #{deploy_cmd}"
   end
 end

data/lib/mina/kubernetes/version.rb

@@ -1,5 +1,5 @@
 module Mina
   module Kubernetes
-    VERSION = "2.0.0"
+    VERSION = "2.5.0"
   end
-end
+end

data/mina-kubernetes.gemspec

@@ -20,11 +20,11 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency 'bundler', '~> 1.11'
-  spec.add_development_dependency 'rake', '~> 10.0'
-  
+  spec.add_development_dependency 'rake', '>= 12.3.3'
+
   spec.add_runtime_dependency 'mina', '~> 1.0'
   spec.add_runtime_dependency 'mina-multistage', '~> 1.0'
-  spec.add_runtime_dependency 'kubernetes-deploy'
+  spec.add_runtime_dependency 'krane', '~> 1.0'
   spec.add_runtime_dependency 'tty-prompt'
   spec.add_runtime_dependency 'tty-spinner'
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mina-kubernetes
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Antoine Sabourin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-28 00:00:00.000000000 Z
+date: 2020-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,16 +28,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: mina
   requirement: !ruby/object:Gem::Requirement
@@ -67,19 +67,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: kubernetes-deploy
+  name: krane
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: tty-prompt
   requirement: !ruby/object:Gem::Requirement