krane 1.0.0

data/Rakefile

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+desc("Run integration tests that can be run in parallel")
+Rake::TestTask.new(:integration_test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/integration/**/*_test.rb']
+end
+
+desc("Run integration tests that CANNOT be run in parallel")
+Rake::TestTask.new(:serial_integration_test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/integration-serial/**/*_test.rb']
+end
+
+desc("Run unit tests")
+Rake::TestTask.new(:unit_test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/unit/**/*_test.rb']
+end
+
+desc("Run cli tests")
+Rake::TestTask.new(:cli_test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/exe/**/*_test.rb']
+end
+
+desc("Run all tests")
+task(test: %w(unit_test serial_integration_test integration_test cli_test))
+
+task(default: :test)

data/bin/ci

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ -n "${DEBUG:+set}" ]]; then
+  set -x
+fi
+
+docker run --rm \
+    --net=host \
+    -v "$HOME/.kube":"/root/.kube" \
+    -v "$HOME/.minikube":"$HOME/.minikube" \
+    -v "$PWD":/usr/src/app \
+    -v "/usr/bin/kubectl":"/usr/bin/kubectl" \
+    -e CI=1 \
+    -e CODECOV_TOKEN=$CODECOV_TOKEN \
+    -e COVERAGE=1 \
+    -e VERBOSE=1 \
+    -e PARALLELISM=$PARALLELISM \
+    -w /usr/src/app \
+    ruby:2.4 \
+    bin/test

data/bin/setup

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+if [ ! -x "$(which minikube)" ]; then
+  echo -e "\n\033[0;33mIf you're going to run the tests, please follow the minikube setup instructions for your operating system:\nhttps://kubernetes.io/docs/getting-started-guides/minikube/#installation\033[0m"
+fi
+
+if [ ! -x "$(which kubectl)" ]; then
+  echo -e "\n\033[0;33mPlease install kubectl version 1.11.0 or higher:\nhttps://kubernetes.io/docs/user-guide/prereqs/\033[0m"
+else
+  KUBECTL_VERSION=$(kubectl version --short --client | grep -oe "v[[:digit:]\.]\+")
+  echo -e "\n\033[0;32mKubectl version $KUBECTL_VERSION is already installed. This gem requires version v1.11.0 or greater.\033[0m"
+fi

data/bin/test

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -uo pipefail
+
+err=0
+trap 'err=1' ERR
+trap 'exit 1' SIGINT
+
+function print_header() {
+  if [[ ${CI:="0"} == "1" ]]; then
+    printf "+++ :kubernetes: %s\n" "${1}"
+  else
+    printf "\n\n\033[0;35m⎈   %s\033[0m\n" "${1}"
+  fi
+}
+
+if [[ ${PARALLELISM:=0} -lt 1 ]]; then
+  if [[ $(uname) == "Darwin" ]]; then
+    num_cpus=$(sysctl -n hw.ncpu)
+  else
+    num_cpus=$(nproc --all)
+  fi
+
+  if [[ $num_cpus -le 2 ]]; then
+    PARALLELISM=1
+  else
+    (( PARALLELISM=num_cpus/2 ))
+  fi
+fi
+
+if [[ ${CI:="0"} == "1" ]]; then
+  echo "--- :ruby: Bundle Install"
+  bundle install --jobs 4
+fi
+
+print_header "Run CLI Tests"
+bundle exec rake cli_test
+
+print_header "Run Unit Tests"
+bundle exec rake unit_test
+
+print_header "Run Non-Parallel Integration Tests"
+bundle exec rake serial_integration_test
+
+print_header "Run Parallel Integration Tests (N=$PARALLELISM)"
+PARALLELIZE_ME=1 N=$PARALLELISM bundle exec rake integration_test
+
+test $err -eq 0

data/dev.yml

@@ -0,0 +1,28 @@
+---
+name: kubernetes-deploy
+up:
+  - ruby: 2.4.6 # Matches gemspec
+  - bundler
+  - homebrew:
+    - Caskroom/cask/minikube
+  - custom:
+      name: Install the minikube fork of driver-hyperkit
+      met?: command -v docker-machine-driver-hyperkit
+      meet: curl -LO https://storage.googleapis.com/minikube/releases/latest/docker-machine-driver-hyperkit && sudo install -o root -g wheel -m 4755 docker-machine-driver-hyperkit /usr/local/bin/ && rm ./docker-machine-driver-hyperkit
+  - custom:
+      name: Minikube Cluster
+      met?: test $(minikube status | grep Running | wc -l) -ge 2 && $(minikube status | grep -q 'Correctly Configured')
+      meet: minikube start --kubernetes-version=v1.11.6 --vm-driver=hyperkit
+      down: minikube stop
+commands:
+  reset-minikube: minikube delete && rm -rf ~/.minikube
+  test:
+    run: bin/test
+  tophat:
+    run: PRINT_LOGS=1 bundle exec ruby -I test test/integration/krane_deploy_test.rb -n/${1}/
+    desc: Tophat a change by running a test scenario with logging output enabled.
+    syntax:
+      optional:
+        argument: TEST_REGEX
+  doc:
+    run: bundle exec yard doc

data/dev/flamegraph-from-tests

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+SVG_FILENAME="dev/flamegraph.svg"
+PROFILE_FILENAME="dev/profile"
+FLAMEGRAPH_PL="dev/flamegraph.pl"
+
+if [[ -z $1 ]]; then
+  echo "Usage: `basename "$0"` TEST_NAME_OR_REGEX"
+  exit 1
+fi
+
+if ! [[ -s $FLAMEGRAPH_PL ]]; then
+  echo "Downloading flamegraph.pl"
+  curl -Lo $FLAMEGRAPH_PL https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl
+fi
+
+if [[ -f $SVG_FILENAME ]]; then
+  rm $SVG_FILENAME
+fi
+
+if [[ -f $PROFILE_FILENAME ]]; then
+  rm $PROFILE_FILENAME
+fi
+
+echo "Running test(s) with profiling"
+PROFILE=1 bundle exec ruby -I test test/integration/krane_deploy_test.rb -n /$1/ > /dev/null
+
+echo "Processing profile"
+cat $PROFILE_FILENAME | perl -w $FLAMEGRAPH_PL --countname=ms --width=1500 --title=$1 > $SVG_FILENAME
+
+if [[ -f $SVG_FILENAME ]]; then
+  echo "Done. Opening ${SVG_FILENAME}"
+  open $SVG_FILENAME
+else
+  echo "Done, but ${SVG_FILENAME} does not exist. Something went wrong."
+fi

data/exe/krane

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'krane/cli/krane'
+Krane::CLI::Krane.start(ARGV)

data/krane.gemspec

@@ -0,0 +1,44 @@
+# coding: utf-8
+# frozen_string_literal: true
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'krane/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "krane"
+  spec.version       = Krane::VERSION
+  spec.authors       = ["Katrina Verey", "Daniel Turner", "Kir Shatrov"]
+  spec.email         = ["ops-accounts+shipit@shopify.com"]
+
+  spec.summary       = 'A command line tool that helps you ship changes to a Kubernetes' \
+    ' namespace and understand the result'
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/Shopify/krane"
+  spec.license       = "MIT"
+
+  spec.files         = %x(git ls-files -z).split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = %w(lib)
+
+  spec.required_ruby_version = '>= 2.4.0'
+  spec.add_dependency("activesupport", ">= 5.0")
+  spec.add_dependency("kubeclient", "~> 4.3")
+  spec.add_dependency("googleauth", "~> 0.8.0")
+  spec.add_dependency("ejson", "~> 1.0")
+  spec.add_dependency("colorize", "~> 0.8")
+  spec.add_dependency("statsd-instrument", ['>= 2.8', "< 3.1"])
+  spec.add_dependency("oj", "~> 3.0")
+  spec.add_dependency("concurrent-ruby", "~> 1.1")
+  spec.add_dependency("jsonpath", "~> 0.9.6")
+  spec.add_dependency("thor", "~>  0.20.3")
+
+  spec.add_development_dependency("bundler")
+  spec.add_development_dependency("rake", "~> 10.0")
+  spec.add_development_dependency("minitest", "~> 5.0")
+  spec.add_development_dependency("minitest-stub-const", "~> 0.6")
+  spec.add_development_dependency("webmock", "~> 3.0")
+  spec.add_development_dependency("mocha", "~> 1.5")
+end

data/lib/krane.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'krane/deploy_task'
+require 'krane/render_task'
+require 'krane/restart_task'
+require 'krane/runner_task'
+require 'krane/global_deploy_task'

data/lib/krane/bindings_parser.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+require 'json'
+require 'yaml'
+require 'csv'
+
+module Krane
+  class BindingsParser
+    def self.parse(string)
+      new(string).parse
+    end
+
+    def initialize(initial_string = nil)
+      @raw_bindings = Array(initial_string)
+    end
+
+    def add(string)
+      @raw_bindings << string
+    end
+
+    def parse
+      result = {}
+      @raw_bindings.each do |string|
+        bindings = parse_file(string) || parse_json(string) || parse_csv(string)
+        unless bindings
+          raise ArgumentError, "Failed to parse bindings."
+        end
+        result.deep_merge!(bindings)
+      end
+      result
+    end
+
+    private
+
+    def parse_file(string)
+      return unless string =~ /\A@/
+
+      begin
+        file_path = string.gsub(/\A@/, '')
+
+        case File.extname(file_path)
+        when '.json'
+          bindings = parse_json(File.read(file_path))
+        when '.yaml', '.yml'
+          bindings = YAML.safe_load(File.read(file_path), [], [], true, file_path)
+        else
+          raise ArgumentError, "Supplied file does not appear to be JSON or YAML"
+        end
+
+        bindings
+      rescue Errno::ENOENT
+        raise ArgumentError, "Supplied file does not exist: #{string}"
+      end
+    end
+
+    def parse_json(string)
+      bindings = JSON.parse(string)
+
+      unless bindings.is_a?(Hash)
+        raise ArgumentError, "Expected JSON data to be a hash."
+      end
+
+      bindings
+    rescue JSON::ParserError
+      nil
+    end
+
+    def parse_csv(string)
+      lines = CSV.parse(string)
+      bindings = {}
+
+      lines.each do |line|
+        line.each do |binding|
+          key, value = binding.split('=', 2)
+
+          if key.blank?
+            raise ArgumentError, "key is blank"
+          end
+
+          bindings[key] = value
+        end
+      end
+
+      bindings
+    rescue CSV::MalformedCSVError
+      nil
+    end
+  end
+end

data/lib/krane/cli/deploy_command.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Krane
+  module CLI
+    class DeployCommand
+      DEFAULT_DEPLOY_TIMEOUT = '300s'
+      PROTECTED_NAMESPACES = %w(
+        default
+        kube-system
+        kube-public
+      )
+      OPTIONS = {
+        "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
+                         aliases: :f, required: false, default: [],
+                         desc: "Directories and files that contains the configuration to apply" },
+        "stdin" => { type: :boolean, default: false,
+                     desc: "Read resources from stdin" },
+        "global-timeout" => { type: :string, banner: "duration", default: DEFAULT_DEPLOY_TIMEOUT,
+                              desc: "Max duration to monitor workloads correctly deployed" },
+        "protected-namespaces" => { type: :array, banner: "namespace1 namespace2 namespaceN",
+                                    desc: "Enable deploys to a list of selected namespaces; set to an empty string "\
+                                      "to disable",
+                                    default: PROTECTED_NAMESPACES },
+        "prune" => { type: :boolean, desc: "Enable deletion of resources that do not appear in the template dir",
+                     default: true },
+        "selector" => { type: :string, banner: "'label=value'",
+                        desc: "Select workloads by selector(s)" },
+        "verbose-log-prefix" => { type: :boolean, desc: "Add [context][namespace] to the log prefix",
+                                  default: true },
+        "verify-result" => { type: :boolean, default: true,
+                             desc: "Verify workloads correctly deployed" },
+      }
+
+      def self.from_options(namespace, context, options)
+        require 'krane/deploy_task'
+        require 'krane/options_helper'
+        require 'krane/label_selector'
+
+        selector = ::Krane::LabelSelector.parse(options[:selector]) if options[:selector]
+
+        logger = ::Krane::FormattedLogger.build(namespace, context,
+          verbose_prefix: options['verbose-log-prefix'])
+
+        protected_namespaces = options['protected-namespaces']
+        if options['protected-namespaces'].size == 1 && %w('' "").include?(options['protected-namespaces'][0])
+          protected_namespaces = []
+        end
+
+        # never mutate options directly
+        filenames = options[:filenames].dup
+        filenames << "-" if options[:stdin]
+        if filenames.empty?
+          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+        end
+
+        ::Krane::OptionsHelper.with_processed_template_paths(filenames) do |paths|
+          deploy = ::Krane::DeployTask.new(
+            namespace: namespace,
+            context: context,
+            filenames: paths,
+            logger: logger,
+            global_timeout: ::Krane::DurationParser.new(options["global-timeout"]).parse!.to_i,
+            selector: selector,
+            protected_namespaces: protected_namespaces,
+          )
+
+          deploy.run!(
+            verify_result: options["verify-result"],
+            prune: options[:prune]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/krane/cli/global_deploy_command.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Krane
+  module CLI
+    class GlobalDeployCommand
+      DEFAULT_DEPLOY_TIMEOUT = '300s'
+      OPTIONS = {
+        "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
+                         aliases: :f, required: false, default: [],
+                         desc: "Directories and files that contains the configuration to apply" },
+        "stdin" => { type: :boolean, default: false, desc: "Read resources from stdin" },
+        "global-timeout" => { type: :string, banner: "duration", default: DEFAULT_DEPLOY_TIMEOUT,
+                              desc: "Max duration to monitor workloads correctly deployed" },
+        "verify-result" => { type: :boolean, default: true,
+                             desc: "Verify workloads correctly deployed" },
+        "selector" => { type: :string, banner: "'label=value'", required: true,
+                        desc: "Select workloads owned by selector(s)" },
+        "prune" => { type: :boolean, desc: "Enable deletion of resources that match"\
+                     " the provided selector and do not appear in the provided templates",
+                     default: true },
+      }
+
+      def self.from_options(context, options)
+        require 'krane/global_deploy_task'
+        require 'krane/options_helper'
+        require 'krane/label_selector'
+        require 'krane/duration_parser'
+
+        selector = ::Krane::LabelSelector.parse(options[:selector])
+
+        # never mutate options directly
+        filenames = options[:filenames].dup
+        filenames << "-" if options[:stdin]
+        if filenames.empty?
+          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+        end
+
+        ::Krane::OptionsHelper.with_processed_template_paths(filenames) do |paths|
+          deploy = ::Krane::GlobalDeployTask.new(
+            context: context,
+            filenames: paths,
+            global_timeout: ::Krane::DurationParser.new(options["global-timeout"]).parse!.to_i,
+            selector: selector,
+          )
+
+          deploy.run!(
+            verify_result: options["verify-result"],
+            prune: options[:prune],
+          )
+        end
+      end
+    end
+  end
+end