kube_deploy_tools 3.0.5

data/lib/kube_deploy_tools/make_configmap.rb

@@ -0,0 +1,51 @@
+require 'kube_deploy_tools/object'
+require 'optparse'
+
+module KubeDeployTools
+
+  class ConfigMap
+    def initialize(name, from_file, namespace = 'default', labels = nil)
+      @name = name
+      @namespace = namespace
+      @labels = labels
+      @from_file = from_file
+    end
+
+    def base
+      {
+        'apiVersion' => 'v1',
+        'kind' => 'ConfigMap',
+        'metadata' => {},
+        'data' => {}
+      }
+    end
+
+    def target_hash
+      res = base
+      res['metadata']['name'] = @name
+      res['metadata']['namespace'] = @namespace
+      res['metadata']['labels'] = @labels if @labels
+      @from_file.each do |from_file|
+        if from_file.include? '='
+          # e.g. --from-file=config.yml=/path/to/configs/production.yml
+          configmap_key, filepath = from_file.split("=", 2)
+          res['data'][configmap_key] = File.read(filepath)
+        elsif File.file?(from_file)
+          # e.g. --from-file=/path/to/configs/production.yml
+          configmap_key = File.basename(from_file)
+          filepath = from_file
+          res['data'][configmap_key] = File.read(filepath)
+        elsif File.directory?(from_file)
+          # e.g. --from-file=/path/to/configs/
+          Dir[File.join(from_file, '*')].each do |filepath|
+            # NOTE(jmodes): Multiple levels of directories are not supported.
+            next if File.directory?(filepath)
+            configmap_key = File.basename(filepath)
+            res['data'][configmap_key] = File.read(filepath)
+          end
+        end
+      end
+      res
+    end
+  end
+end

data/lib/kube_deploy_tools/make_configmap/options.rb

@@ -0,0 +1,39 @@
+require 'optparse'
+require 'yaml'
+
+module KubeDeployTools
+  class Options
+    def options
+      $options ||= begin
+        res = {from_file: [], labels: {}}
+        OptionParser.new do |opts|
+          opts.banner = "Usage: #{opts.program_name}.rb [options]. Outputs YAML to STDOUT"
+
+          opts.on("--name [NAME]", "ConfigMap name") do |v|
+            res[:name] = v
+          end
+
+          res[:namespace] = "default"
+          opts.on("--namespace [NAMESPACE]", "ConfigMap namespace") do |v|
+            res[:namespace] = v
+          end
+
+          opts.on("--label [NAME=VALUE]", "ConfigMap metadata label") do |v|
+            res[:labels].store(*v.split('=', 2))
+          end
+
+          opts.on("--from-file [KEYFILE]", "Key file can be specified using its file path, in which case file basename will be used as
+    configmap key, or optionally with a key and file path, in which case the given key will be used.  Specifying a directory
+    will iterate each named file in the directory whose basename is a valid configmap key.") do |v|
+            res[:from_file] << v
+          end
+        end.parse!
+
+        raise "no name given" unless res[:name].strip != ''
+        raise "no files given" if res[:from_file].empty?
+
+        res
+      end
+    end
+  end
+end

data/lib/kube_deploy_tools/object.rb

@@ -0,0 +1,11 @@
+
+class Object
+  def present?
+    self && to_s.strip != ''
+  end
+
+  def blank?
+    !present?
+  end
+end
+

data/lib/kube_deploy_tools/publish.rb

@@ -0,0 +1,40 @@
+require 'uri'
+require 'yaml'
+
+require 'kube_deploy_tools/built_artifacts_file'
+require 'kube_deploy_tools/deploy_config_file'
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/object'
+
+module KubeDeployTools
+  class Publish
+    def initialize(manifest:, artifact_registry:, output_dir:)
+      @config = DeployConfigFile.new(manifest)
+      @output_dir = output_dir
+
+      @project = KubeDeployTools::PROJECT
+      @build_number = KubeDeployTools::BUILD_NUMBER
+
+      @artifact_registry = artifact_registry.driver
+    end
+
+    def publish()
+      @config.artifacts.each do |c|
+        name = c.fetch('name')
+
+        # Allow deploy.yaml to gate certain flavors to certain targets.
+        cluster_flavors = @config.flavors.select { |key, value| c['flavors'].nil? || c['flavors'].include?(key) }
+
+        cluster_flavors.each do |flavor, _|
+          @artifact_registry.upload(
+            local_dir: @output_dir,
+            name: name,
+            flavor: flavor,
+            project: @project,
+            build_number: @build_number,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/kube_deploy_tools/publish/options.rb

@@ -0,0 +1,34 @@
+require 'optparse'
+
+module KubeDeployTools
+  class Publish::Optparser
+    class Options
+      attr_accessor :manifest_file, :output_path
+
+      def initialize
+        self.output_path = File.join('build', 'kubernetes')
+      end
+
+      def define_options(parser)
+        parser.on('-mMANIFEST', '--manifest MANIFEST', 'The configuration MANIFEST to render deploys with.') do |f|
+          self.manifest_file = f
+        end
+
+        parser.on('-oPATH', '--output-path PATH', 'Path where rendered manifests are written.') do |p|
+          self.output_path = p
+        end
+
+        parser.on('-')
+      end
+    end
+
+    def parse(args)
+      @options = Options.new
+      OptionParser.new do |parser|
+        @options.define_options(parser)
+        parser.parse(args)
+      end
+      @options
+    end
+  end
+end

data/lib/kube_deploy_tools/push.rb

@@ -0,0 +1,129 @@
+require 'fileutils'
+
+require 'kube_deploy_tools/shellrunner'
+require 'kube_deploy_tools/built_artifacts_file'
+require 'kube_deploy_tools/deploy_config_file'
+
+require 'kube_deploy_tools/image_registry/driver'
+require 'kube_deploy_tools/image_registry/image'
+
+BUILT_ARTIFACTS_FILE = 'build/kubernetes/images.yaml'.freeze
+
+module KubeDeployTools
+  class Push
+    def initialize(config, local_prefix, registries, images, tag)
+      @config = config
+      @local_prefix = local_prefix
+
+      # If |registries| is empty, assume we will push them all,
+      # if not, subtract unwanted ones from |config.image_registries|
+      if registries.empty?
+        to_instantiate = config.image_registries.values
+      else
+        to_instantiate = []
+        config.image_registries.each do |name, registry|
+          if registries.member? name
+            to_instantiate.push registry
+          end
+        end
+      end
+
+      @drivers = to_instantiate.map do |registry|
+        driver_class = ImageRegistry::Driver::MAPPINGS.fetch(registry.driver)
+        [registry, driver_class.new(registry: registry)]
+      end.to_h
+      @base_image_names = images
+      @tag = tag
+    end
+
+    def publish
+      dirname = File.dirname(BUILT_ARTIFACTS_FILE)
+      FileUtils.mkdir_p(dirname)
+
+      driver_images = []
+
+      @drivers.each_pair do |registry, driver|
+        driver_images.unshift [driver, tag_images(registry, @base_image_names)]
+        # Does whatever is necessary to authorize against this registry
+        driver.authorize
+      end
+
+      # Push first images to each registry in parallel
+      driver_images.map do |driver, all_images|
+        Thread.new { driver.push_image all_images[0] }
+      end.each(&:join)
+
+      # Push the rest of the images to each registry in parallel
+      driver_images.map do |driver, all_images|
+        _, *remaining_images = all_images
+        remaining_images.map do |i|
+          Thread.new { driver.push_image i }
+        end
+      end.flatten.each(&:join)
+
+      # Can't lock the file if it doesn't exist. Create the file as a
+      # placeholder until more content is loaded
+      File.open(BUILT_ARTIFACTS_FILE, File::CREAT|File::RDWR) do |file|
+        flock(file, File::LOCK_EX) do |file|
+          driver_images.each do |_, all_images|
+            update_built_artifacts(all_images, file)
+          end
+        end
+      end
+
+    # Clean registry authorization in the end no matter what
+    ensure
+      Logger.info "Removing registry authorizations"
+      @drivers.each_pair do |registry, driver|
+        driver.unauthorize
+      end
+    end
+
+    private
+
+    def tag_images(r, base_image_names)
+      base_image_names.map do |i|
+        local = Image.new(@local_prefix, i, 'latest')
+        remote = Image.new(r.prefix, i, @tag)
+        Shellrunner.check_call('docker', 'tag', local.full_tag, remote.full_tag)
+        remote
+      end
+    end
+
+    def update_built_artifacts(images_to_push, file)
+      artifacts = KubeDeployTools::BuiltArtifactsFile.new(file)
+      build_id = ENV.fetch('BUILD_ID', 'LOCAL')
+
+      if !artifacts.build_id.nil? && artifacts.build_id != build_id
+        # Clear the images as this is a fresh build.
+        artifacts.images = Set.new
+        # Truncate the file so it will generate a new file
+        # and remove any old builds
+        file.truncate(0)
+      end
+
+      # Add new images to the output list.
+      artifacts.build_id = build_id
+      images_to_push.each do |image|
+        artifacts.images.add image.full_tag
+      end
+
+      # Write the config list.
+      artifacts.write file
+    end
+
+    # Method used to protect reads and writes. From:
+    # https://www.safaribooksonline.com/library/view/ruby-cookbook/0596523696/ch06s13.html
+    def flock(file, mode)
+      success = file.flock(mode)
+      if success
+        begin
+          yield file
+        ensure
+          file.flock(File::LOCK_UN)
+        end
+      end
+      return success
+    end
+  end
+end

data/lib/kube_deploy_tools/push/options.rb

@@ -0,0 +1,46 @@
+require 'optparse'
+require 'set'
+require 'kube_deploy_tools/tag'
+
+module KubeDeployTools
+  class Push::Optparser
+
+    class PushOptions
+      attr_accessor :local_prefix, :manifest_file, :registries, :images, :tag
+
+      def initialize
+        self.local_prefix = 'local-registry'
+        self.registries = Set.new
+        self.tag = KubeDeployTools::tag_from_local_env
+      end
+
+      def define_options(parser)
+        parser.on('-lPREFIX', '--local-prefix PREFIX', 'The local Docker prefix to strip to get to the base image name') do |f|
+          self.local_prefix = f
+        end
+
+        parser.on('-tTAG', '--tag TAG', 'Tag Docker images with TAG') do |t|
+          self.tag = t
+        end
+
+        parser.on('-mMANIFEST', '--manifest MANIFEST', 'The configuration MANIFEST to render deploys with.') do |f|
+          self.manifest_file = f
+        end
+
+        parser.on('-rPREFIX', '--registry REGISTRY', 'The remote Docker registry to push to (can specify multiple times). If not specified, then will push to all registries in config.') do |r|
+          self.registries.add r
+        end
+      end
+
+    end
+
+    def parse(args)
+      @options = PushOptions.new
+      OptionParser.new do |parser|
+        @options.define_options(parser)
+        parser.parse!(args)
+      end
+      @options
+    end
+  end
+end

data/lib/kube_deploy_tools/render_deploys_hook.rb

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+# Default rendering hook. Uses built in `templater` to render out all files
+# underneath kubernetes/ directory, recursively.
+
+require 'fileutils'
+require 'json'
+require 'yaml'
+require 'kube_deploy_tools/formatted_logger'
+require 'kube_deploy_tools/templater'
+require 'kube_deploy_tools/file_filter'
+
+module KubeDeployTools
+  module RenderDeploysHook
+    TEMPLATING_SUFFIX = '.erb'
+
+    def self.render_deploys(config_file, input_dir, output_root, file_filters)
+      # Parse config into a struct.
+      config = YAML.load_file(config_file)
+      t = KubeDeployTools::Templater.new
+
+      get_valid_files(file_filters, input_dir).each do |yml|
+        # PREFIX/b/c/foo.yml.in -> foo.yml
+        output_base = File.basename(yml, TEMPLATING_SUFFIX)
+
+        # PREFIX/b/c/foo.yml.in -> b/c
+        subdir = File.dirname(yml[input_dir.size..-1])
+
+        # PREFIX/b/c/foo.yml.in -> output/b/c/foo.yml
+        if subdir == '.'
+          # If |subdir| is '.', joining it with output_root results in
+          # output_root/. , which is not concise for human readability.
+          # Handle this case explicitly.
+          output_dir = output_root
+        else
+          output_dir = File.join(output_root, subdir)
+        end
+        output_file = File.join(output_dir, output_base)
+
+        if yml.end_with? TEMPLATING_SUFFIX
+          # File needs to be templated with templater.
+          Logger.debug("Generating #{output_file} from #{yml}")
+          t.template_to_file(yml, config, output_file)
+        else
+          # File is not templatable, and is copied verbatim.
+          Logger.debug("Copying #{output_file} from #{yml}")
+          FileUtils.mkdir_p output_dir
+          FileUtils.copy(yml, output_file)
+        end
+
+        # Bonus: YAML validate the output.
+        # * Must be valid YAML
+        # * If .kind is a type that takes .metadata.namespace, then require
+        #   that .metadata.namespace is present.
+        begin
+          if File.file?(output_file)
+            yaml = []
+            YAML.load_stream(File.read(output_file)) { |doc| yaml << doc }
+            yaml = yaml.
+              compact.
+              map do |data|
+              # XXX(joshk): Non-exhaustive list.
+              must_have_ns = [
+                'ConfigMap', 'CronJob', 'DaemonSet', 'Deployment', 'Endpoints', 'HorizontalPodAutoscaler',
+                'Ingress', 'PersistentVolumeClaim', 'PodDisruptionBudget', 'ServiceAccount', 'Secret', 'Service'
+              ]
+              if must_have_ns.member?(data.fetch('kind'))
+                raise "Rendered Kubernetes template missing a .metadata.namespace: #{yml}" if data.fetch('metadata', {}).fetch('namespace', '').empty?
+              end
+              # annotation added to each manifest
+              annotations = data['metadata']['annotations'] || {}
+              if config['git_commit']
+                annotations['git_commit'] = config['git_commit']
+              end
+
+              if config['git_project']
+                annotations['git_project'] = config['git_project']
+              end
+
+              data['metadata']['annotations'] = annotations
+              data
+            end
+            File.open(output_file, 'w') { |f| f << YAML.dump_stream(*yaml) }
+          end
+        rescue => e
+          raise "Failed to YAML validate #{output_file} (generated from #{yml}): #{e}"
+        end
+      end
+    end
+
+    def self.get_valid_files(file_filters, input_dir)
+      filtered_files = FileFilter.filter_files(filters: file_filters, files_path: input_dir)
+      filtered_files.select { |f| f =~ /\.y.?ml[^\/]*$/ }
+    end
+  end
+end