kube_deploy_tools 3.0.5

data/bin/publish

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+require 'kube_deploy_tools/publish'
+require 'kube_deploy_tools/publish/options'
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+
+def options
+  $options ||= begin
+    parser = KubeDeployTools::Publish::Optparser.new
+    parser.parse(ARGV)
+  end
+end
+
+KubeDeployTools::Logger.logger = KubeDeployTools::FormattedLogger.build
+KubeDeployTools::Shellrunner.shellrunner = KubeDeployTools::Shellrunner.new
+
+config = KubeDeployTools::DeployConfigFile.new(options.manifest_file)
+artifact_registry = config.artifact_registries[config.artifact_registry]
+
+KubeDeployTools::Publish.new(
+  manifest: options.manifest_file,
+  artifact_registry: artifact_registry,
+  output_dir: options.output_path,
+).publish

data/bin/push

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+# Utility for templating Kubernetes manifests with ERB.
+#
+# Example usage:
+#
+# push \
+# --registry aws \
+# --local-prefix local-registry/ \
+# --tag my-custom-tag \
+# image1 image2 image3
+#
+# Will tag local-registry/image1:latest, local-registry/image2:latest etc. as
+# 123456.dkr.ecr.us-west-2.amazonaws.com/image1:my-custom-tag.
+# Default values for every option are probably what you want to use most of the time.
+
+require 'optparse'
+
+require 'kube_deploy_tools/push'
+require 'kube_deploy_tools/push/options'
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+
+def options
+  $options ||= begin
+    parser = KubeDeployTools::Push::Optparser.new
+    parser.parse(ARGV)
+  end
+end
+
+def images
+  # Make sure options have been parsed
+  options
+
+  # The rest of ARGV is now the list of images to retag
+  if ARGV.empty?
+    msg = %q(Must supply at least one image
+
+Example usage:
+
+push \
+--local-prefix local-registry/ \
+--tag my-custom-tag \
+image1 image2 image3
+    )
+    puts msg
+    exit(1)
+  end
+
+  ARGV
+end
+
+KubeDeployTools::Logger.logger = KubeDeployTools::FormattedLogger.build
+KubeDeployTools::Shellrunner.shellrunner = KubeDeployTools::Shellrunner.new
+config = KubeDeployTools::DeployConfigFile.new(options.manifest_file)
+
+KubeDeployTools::Push.new(config, options.local_prefix, options.registries, images, options.tag).publish

data/bin/render_deploys_hook

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+# Default rendering hook. Uses built in `templater` to render out all files
+# underneath kubernetes/ directory, recursively.
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/render_deploys_hook'
+
+KubeDeployTools::Logger.logger = KubeDeployTools::FormattedLogger.build
+KubeDeployTools::Logger.warn "This subcommand will be disappearing in kdt v3.0."
+
+def main(argv)
+  config = argv[0]
+  input_dir = argv[1]
+  output_root = argv[2]
+  KubeDeployTools::RenderDeploysHook.render_deploys(config, input_dir, output_root)
+end
+
+main(ARGV)

data/bin/templater

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+# Utility for templating Kubernetes manifests with ERB.
+#
+# Example usage:
+#
+# templater \
+# --template kubernetes/ds-datadog.yaml.erb \
+# --set environment=staging \
+# --set tag=$(git rev-parse --short=7 HEAD) \
+# --values staging.yaml \
+# > build/kubernetes/ds-datadog.yaml
+#
+# Values can be passed to templater to expose variables available in
+# the ERB templates, such as a git tag or the environment name.
+# Values can be passed with:
+# * 1 or more key-value pairs e.g. --set git_tag=123456 --set environment=local
+# * a YAML file e.g. --values production.yaml
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/templater'
+require 'kube_deploy_tools/templater/options'
+
+KubeDeployTools::Logger.logger = KubeDeployTools::FormattedLogger.build
+KubeDeployTools::Logger.warn "This subcommand will be disappearing in kdt v3.0."
+
+def options
+  parser = KubeDeployTools::Templater::Optparser.new
+  ops = parser.parse(ARGV)
+  ops.merge_values
+  ops
+end
+
+templater = KubeDeployTools::Templater.new
+templater.template_to_file(options.template, options.values, options.output)

data/bin/upgrade

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+require 'kube_deploy_tools/deploy_config_file'
+
+KubeDeployTools::Logger.logger = KubeDeployTools::FormattedLogger.build
+KubeDeployTools::Shellrunner.shellrunner = KubeDeployTools::Shellrunner.new
+
+options = {
+  :file => 'deploy.yml',
+}
+OptionParser.new do |opts|
+  opts.on('-f', '--file file', 'Config file path') do |file|
+    options[:file] = file
+  end
+end.parse!
+
+config_file_path = options[:file] || ARGV[0]
+
+KubeDeployTools::DeployConfigFile.new(config_file_path).upgrade!

data/lib/kube_deploy_tools.rb

@@ -0,0 +1,17 @@
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/kubectl'
+require 'kube_deploy_tools/shellrunner'
+
+require 'kube_deploy_tools/deploy'
+require 'kube_deploy_tools/deploy/options'
+
+require 'kube_deploy_tools/templater'
+require 'kube_deploy_tools/templater/options'
+
+require 'kube_deploy_tools/generate'
+require 'kube_deploy_tools/generate/options'
+
+module KubeDeployTools
+end
+

data/lib/kube_deploy_tools/artifact_registry.rb

@@ -0,0 +1,30 @@
+require 'kube_deploy_tools/artifact_registry/driver'
+require 'kube_deploy_tools/formatted_logger'
+
+module KubeDeployTools
+  # Read-only model for artifact_registries[] array element in KDT deploy.yaml
+  # configuration file.
+  class ArtifactRegistry
+    attr_accessor :name, :driver_name, :config, :driver
+
+    def initialize(h)
+      @name = h['name']
+      @driver_name = h['driver']
+      @config = h['config']
+
+      if !ArtifactRegistry::Driver::MAPPINGS.key?(@driver_name)
+        Logger.warn("Unsupported .artifact_registries.driver: #{@driver_name}")
+      else
+        @driver = ArtifactRegistry::Driver::MAPPINGS
+          .fetch(@driver_name)
+          .new(config: @config)
+      end
+    end
+
+    def ==(o)
+      @name == o.name
+      @driver == o.driver
+      @config == o.config
+    end
+  end
+end

data/lib/kube_deploy_tools/artifact_registry/driver.rb

@@ -0,0 +1,13 @@
+require_relative 'driver_artifactory'
+require_relative 'driver_gcs'
+
+module KubeDeployTools
+  class ArtifactRegistry
+    module Driver
+      MAPPINGS = {
+        'artifactory' => Artifactory,
+        'gcs' => GCS,
+      }
+    end
+  end
+end

data/lib/kube_deploy_tools/artifact_registry/driver_artifactory.rb

@@ -0,0 +1,155 @@
+require_relative 'driver_base'
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+require 'kube_deploy_tools/object'
+
+require 'artifactory'
+require 'fileutils'
+require 'uri'
+
+
+EXT_TAR_GZ = ".tar.gz"
+
+module KubeDeployTools
+  class ArtifactRegistry::Driver::Artifactory < ArtifactRegistry::Driver::Base
+    # Artifactory configuration is configurable by environment variables
+    # by default:
+    # export ARTIFACTORY_ENDPOINT=http://my.storage.server/artifactory
+    # export ARTIFACTORY_USERNAME=admin
+    # export ARTIFACTORY_PASSWORD=password
+    # See https://github.com/chef/artifactory-client#create-a-connection.
+
+    def initialize(config:)
+      @config = config
+
+      Artifactory.endpoint = @config.fetch('endpoint', '')
+      @repo = @config.fetch('repo', '')
+    end
+
+
+    def get_local_artifact_path(name:, flavor:, local_dir:)
+      artifact_name = get_artifact_name(name: name, flavor: flavor)
+
+      local_artifact_path = File.join(local_dir, artifact_name)
+
+      local_artifact_path
+    end
+
+    def get_registry_artifact_path(name:, flavor:, project:, build_number:)
+      # NOTE(joshk): If the naming format changes, it represents a breaking
+      # change where all past clients will not be able to download new builds and
+      # new clients will not be able to download old builds. Change with caution.
+      "#{project}/#{build_number}/#{get_artifact_name(name: name, flavor: flavor)}"
+    end
+
+    def upload(local_dir:, name:, flavor:, project:, build_number:)
+      # Pack up contents of each flavor_dir to a correctly named artifact.
+      flavor_dir = File.join(local_dir, "#{name}_#{flavor}")
+
+      package(
+        name: name,
+        flavor: flavor,
+        input_dir: flavor_dir,
+        output_dir: local_dir,
+      )
+
+      local_artifact_path = get_local_artifact_path(
+        local_dir: local_dir,
+        name: name,
+        flavor: flavor,
+      )
+
+      registry_artifact_path = get_registry_artifact_path(
+        project: project,
+        name: name,
+        flavor: flavor,
+        build_number: build_number,
+      )
+      artifactory_url = "#{Artifactory.endpoint}/#{@repo}/#{registry_artifact_path}"
+      Logger.info("Uploading #{local_artifact_path} to #{artifactory_url}")
+      artifact = Artifactory::Resource::Artifact.new(local_path: local_artifact_path)
+      artifact.upload(@repo, registry_artifact_path)
+    end
+
+    def get_artifact_name(name:, flavor:)
+      "manifests_#{name}_#{flavor}#{EXT_TAR_GZ}"
+    end
+
+    def package(name:, flavor:, input_dir:, output_dir:)
+      local_artifact_path = get_local_artifact_path(name: name, flavor: flavor, local_dir: output_dir)
+
+      Shellrunner.check_call('tar', '-C', input_dir, '-czf', local_artifact_path, '.')
+
+      local_artifact_path
+    end
+
+    def download(project:, build_number:, flavor:, name:, pre_apply_hook:, output_dir:)
+      registry_artifact_path = get_registry_artifact_path(
+        name: name, flavor: flavor, project: project, build_number: build_number)
+
+      registry_artifact_full_path = [
+        Artifactory.endpoint,
+        @repo,
+        registry_artifact_path,
+      ].join('/')
+
+      local_artifact_path = download_artifact(registry_artifact_full_path, output_dir)
+      local_artifact_path = uncompress_artifact(local_artifact_path, output_dir)
+
+      if pre_apply_hook
+        out, err, status = Shellrunner.run_call(pre_apply_hook, local_artifact_path)
+        if !status.success?
+          raise "Failed to run post download hook #{pre_apply_hook}"
+        end
+      end
+
+      local_artifact_path
+    end
+
+    def get_latest_build_number(project)
+      project_url = [
+        Artifactory.endpoint,
+        @repo,
+        "#{project}/"
+      ].join('/')
+      project_builds_html = Shellrunner.run_call('curl', project_url).first
+      # store build entries string from html into an array
+      build_links_pattern = /(?<=">).+(?=\s{4})/
+      build_entries = project_builds_html.scan(build_links_pattern) # example of element: 10/</a>    13-Nov-2017 13:51
+      build_number_pattern = /^\d+/
+      build_number = build_entries.
+        map { |x| x.match(build_number_pattern).to_s.to_i }.
+        max.
+        to_s
+      if build_number.empty?
+        raise "Failed to find a valid build number. Project URL: #{project_url}"
+      end
+      build_number
+    end
+
+    def download_artifact(input_path, output_dir_path)
+      uri = URI.parse(input_path)
+      filename = File.basename(uri.path)
+      output_path = File.join(output_dir_path, filename)
+      out, err, status = Shellrunner.run_call('curl', '-o', output_path, input_path, '--silent', '--fail')
+      if !status.success?
+        raise "Failed to download remote deploy artifact #{uri}"
+      end
+
+      output_path
+    end
+
+    def uncompress_artifact(input_path, output_dir_path)
+      dirname = File.basename(input_path).chomp(EXT_TAR_GZ)
+      output_path = File.join(output_dir_path, dirname)
+      FileUtils.mkdir_p(output_path)
+      out, err, status = Shellrunner.run_call('tar', '-xzf', input_path, '-C', output_path)
+      if !status.success?
+        raise "Failed to uncompress deploy artifact #{input_path}"
+      end
+
+      output_path
+    end
+  end
+end

data/lib/kube_deploy_tools/artifact_registry/driver_base.rb

@@ -0,0 +1,37 @@
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+
+# Abstract Driver class that specific implementations inherit
+module KubeDeployTools
+  class ArtifactRegistry
+    module Driver
+      class Base
+        def initialize(config:)
+          @config = config
+        end
+
+        # upload should publish the artifact identified by the given name and flavor
+        # in the input directory to the corresponding location in the artifact
+        # registry. The project and build number should be included in the
+        # namespace of the artifact registry path for this artifact.
+        def upload(local_dir:, name:, flavor:, project:, build_number:)
+          raise "#{self.class}#publish not implemented"
+        end
+
+        # download should retrieve the artifact namespaced with the given
+        # project and build number and identified by the name and flavor.
+        # The artifact should be put into the output directory.
+        # An optional pre-apply hook will process each artifact at the end.
+        def download(project:, build_number:, flavor:, name:, pre_apply_hook:, output_dir:)
+          raise "#{self.class}#download not implemented"
+        end
+
+        # get_latest_build_number should find the artifact from the most recent
+        # build
+        def get_latest_build_number(project)
+          raise "#{self.class}#get_latest_build_number not implemented"
+        end
+      end
+    end
+  end
+end

data/lib/kube_deploy_tools/artifact_registry/driver_gcs.rb

@@ -0,0 +1,120 @@
+require_relative 'driver_base'
+
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/shellrunner'
+require 'kube_deploy_tools/object'
+
+require 'fileutils'
+require 'uri'
+require 'find'
+
+
+module KubeDeployTools
+  class ArtifactRegistry::Driver::GCS < ArtifactRegistry::Driver::Base
+    def initialize(config:)
+      @config = config
+
+      @bucket = @config.fetch('bucket')
+      prefix = @config.fetch('prefix', '')
+      if !prefix.empty?
+        @bucket = "#{@bucket}/#{prefix}"
+      end
+    end
+
+    def get_local_artifact_path(name:, flavor:, local_dir:)
+      artifact_name = get_artifact_name(name: name, flavor: flavor)
+
+      local_artifact_path = File.join(local_dir, artifact_name)
+
+      local_artifact_path
+    end
+
+    def get_registry_artifact_path(name:, flavor:, project:, build_number:)
+      # NOTE(joshk): If the naming format changes, it represents a breaking
+      # change where all past clients will not be able to download new builds and
+      # new clients will not be able to download old builds. Change with caution.
+      #
+      "#{@bucket}/project/#{project}/build/#{build_number}/artifact/#{get_artifact_name(name: name, flavor: flavor)}"
+    end
+
+    def get_artifact_name(name:, flavor:)
+      "manifests_#{name}_#{flavor}.yaml"
+    end
+
+    def package(name:, flavor:, input_dir:, output_dir:)
+      local_artifact_path = get_local_artifact_path(name: name, flavor: flavor, local_dir: output_dir)
+      File.open(local_artifact_path, 'w') do |merged|
+        Find.find(input_dir).
+          select { |path| path =~ /.*\.yaml$/ }.
+          each do |e|
+            contents = File.open(e, 'r').read
+            contents.each_line do |line|
+              merged << line
+            end
+          end
+      end
+      local_artifact_path
+    end
+
+    def download(project:, build_number:, flavor:, name:, pre_apply_hook:, output_dir:)
+      registry_artifact_path = get_registry_artifact_path(
+        name: name, flavor: flavor, project: project, build_number: build_number)
+
+      local_artifact_path = download_artifact(registry_artifact_path, output_dir)
+
+      if pre_apply_hook
+        out, err, status = Shellrunner.run_call(pre_apply_hook, local_artifact_path)
+        if !status.success?
+          raise "Failed to run post download hook #{pre_apply_hook}"
+        end
+      end
+
+      local_artifact_path
+    end
+
+    def download_artifact(input_path, output_dir_path)
+      filename = File.basename(input_path)
+      output_path = File.join(output_dir_path, filename)
+      out, err, status = Shellrunner.run_call('gsutil', 'cp', input_path, output_path)
+
+      if !status.success?
+        raise "Failed to download remote deploy artifact #{input_path}"
+      end
+
+      output_path
+    end
+
+    def upload(local_dir:, name:, flavor:, project:, build_number:)
+      # Pack up contents of each flavor_dir to a correctly named artifact.
+      flavor_dir = File.join(local_dir, "#{name}_#{flavor}")
+
+      package(
+        name: name,
+        flavor: flavor,
+        input_dir: flavor_dir,
+        output_dir: local_dir,
+      )
+
+      local_artifact_path = get_local_artifact_path(
+        local_dir: local_dir,
+        name: name,
+        flavor: flavor,
+      )
+
+      registry_artifact_path = get_registry_artifact_path(
+        project: project,
+        name: name,
+        flavor: flavor,
+        build_number: build_number,
+      )
+
+      Logger.info("Uploading #{local_artifact_path} to #{registry_artifact_path}")
+      out, err, status = Shellrunner.run_call('gsutil',  '-m', 'cp', local_artifact_path, registry_artifact_path)
+      if !status.success?
+        raise "Failed to upload remote deploy artifact from #{local_artifact_path} to #{registry_artifact_path}"
+      end
+
+      registry_artifact_path
+    end
+  end
+end