hippo-cli 1.0.0

data/lib/hippo/cli_steps.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'hippo/error'
+require 'hippo/recipe'
+
+module Hippo
+  class CLISteps
+    attr_reader :recipe
+    attr_reader :stage
+
+    def initialize(recipe, stage)
+      @recipe = recipe
+      @stage = stage
+    end
+
+    # Prepare the repository for this build by getting the latest
+    # version from the remote and checking out the branch.
+    def prepare_repository(fetch: true)
+      info "Using repository #{@recipe.repository.url}"
+      if fetch
+        if @recipe.repository.cloned?
+          info 'Repository is already cloned'
+          action 'Fetching the latest repository data...'
+          @recipe.repository.fetch
+        else
+          info 'Repository is not yet cloned.'
+          action 'Cloning repository...'
+          @recipe.repository.clone
+        end
+
+      elsif !fetch && !@recipe.repository.cloned?
+        raise Error, 'Repository is not cloned yet so cannot continue'
+      else
+        info 'Not fetching latest repository, using cached copy'
+      end
+
+      action "Checking out '#{@stage.branch}' branch..."
+      @recipe.repository.checkout(@stage.branch)
+      @commit = @recipe.repository.commit
+      info "Latest commit on branch is #{@commit.objectish}"
+      info "Message: #{@commit.message.split("\n").first}"
+      @commit
+    end
+
+    def build
+      if @commit.nil?
+        raise Error, 'You cannot build without first preparing the repository'
+      end
+
+      Dir.chdir(@recipe.repository.path) do
+        @recipe.build_specs.each do |_, build_spec|
+          if build_spec.image_name.nil?
+            raise Error, "No image-name has been specified for build #{build_spec.name}"
+          end
+
+          image_name = build_spec.image_name_for_commit(@commit)
+          action "Building #{build_spec.name} with tag #{image_name}"
+
+          command = [
+            'docker', 'build', '.',
+            '-f', build_spec.dockerfile,
+            '-t', image_name
+          ]
+          external_command do
+            if system(*command)
+              @built_commit = @commit
+              success "Successfully built image #{build_spec.image_name} for #{build_spec.name}"
+            else
+              raise Error, "Image for #{build_spec.name} did not succeed. Check output and try again."
+            end
+          end
+        end
+      end
+    end
+
+    def publish
+      if @built_commit.nil?
+        raise Error, 'You cannot publish without first building the image'
+      end
+
+      Dir.chdir(@recipe.repository.path) do
+        @recipe.build_specs.each do |_, build_spec|
+          if build_spec.image_name.nil?
+            raise Error, "No image-name has been specified for build #{build_spec.name}"
+          end
+
+          image_name = build_spec.image_name_for_commit(@built_commit.objectish)
+          action "Publishing #{build_spec.name} with tag #{image_name}"
+
+          command = ['docker', 'push', image_name]
+          external_command do
+            if system(*command)
+              success "Successfully published image #{image_name} for #{build_spec.name}"
+            else
+              raise Error, "Image for #{build_spec.name} was not published successfully. Check output and try again."
+            end
+          end
+        end
+      end
+    end
+
+    def run_install_jobs
+      run_jobs('install')
+    end
+
+    def run_deploy_jobs
+      run_jobs('deploy')
+    end
+
+    def deploy
+      action 'Applying deployments'
+      deployments = @recipe.kubernetes.objects('deployments', @stage, @commit)
+
+      if deployments.nil?
+        info 'No deployments file configured. Not applying any deployments'
+      end
+
+      external_command do
+        @recipe.kubernetes.apply_with_kubectl(deployments)
+      end
+      success 'Deployments applied successfully'
+      puts 'You can watch the deployment progressing using the command below:'
+      puts
+      puts "  ⏰  #{@stage.kubectl('get pods --watch')}"
+      deployments.each do |deployment|
+        puts '  👩🏼‍💻  ' + @stage.kubectl("describe deployment #{deployment['metadata']['name']}")
+      end
+      puts
+    end
+
+    def apply_services
+      action 'Applying services'
+      objects = @recipe.kubernetes.objects('services', @stage, @commit)
+      if objects.empty?
+        info 'No services have been defined'
+      else
+        external_command do
+          @recipe.kubernetes.apply_with_kubectl(objects)
+        end
+        success 'Services applied successfully'
+      end
+    end
+
+    def apply_namespace
+      action 'Applying namespace'
+      external_command { @recipe.kubernetes.apply_namespace(@stage) }
+      success 'Namespace applied successfully'
+    end
+
+    def apply_config
+      action 'Applying configuration'
+      objects = @recipe.kubernetes.objects('config', @stage, @commit)
+      if objects.empty?
+        info 'No configuration files have been defined'
+      else
+        external_command do
+          @recipe.kubernetes.apply_with_kubectl(objects)
+        end
+        success 'Configuration applied successfully'
+      end
+    end
+
+    def apply_secrets
+      require 'hippo/secret_manager'
+      action 'Applying secrets'
+      manager = SecretManager.new(@recipe, @stage)
+      unless manager.key_available?
+        error 'No secret encryption key was available. Not applying secrets.'
+        return
+      end
+
+      yamls = manager.secrets.map(&:to_secret_yaml).join("---\n")
+      external_command do
+        @recipe.kubernetes.apply_with_kubectl(yamls)
+      end
+      success 'Secrets applicated successfully'
+    end
+
+    private
+
+    def info(text)
+      puts text
+    end
+
+    def success(text)
+      puts "\e[32m#{text}\e[0m"
+    end
+
+    def action(text)
+      puts "\e[33m#{text}\e[0m"
+    end
+
+    def error(text)
+      puts "\e[31m#{text}\e[0m"
+    end
+
+    def external_command
+      $stdout.print "\e[37m"
+      yield
+    ensure
+      $stdout.print "\e[0m"
+    end
+
+    def run_jobs(type)
+      objects = @recipe.kubernetes.objects("jobs/#{type}", @stage, @commit)
+      if objects.empty?
+        info "No #{type} jobs exist so not applying anything"
+        return true
+      end
+
+      action "Applying #{type} job objects objects to Kubernetes"
+
+      result = nil
+      external_command do
+        # Remove any previous jobs that might have been running before
+        objects.each do |job|
+          @recipe.kubernetes.delete_job(@stage, job['metadata']['name'])
+        end
+
+        result = @recipe.kubernetes.apply_with_kubectl(objects)
+      end
+
+      puts 'Waiting for all scheduled jobs to finish...'
+      timeout, jobs = @recipe.kubernetes.wait_for_jobs(@stage, result.keys)
+      success_jobs = []
+      failed_jobs = []
+      jobs.each do |job|
+        if job['status']['succeeded']
+          success_jobs << job
+        else
+          failed_jobs << job
+        end
+      end
+
+      if success_jobs.size == jobs.size
+        success 'All jobs completed successfully.'
+        puts 'You can review the logs for these by running the commands below.'
+        puts
+        result = true
+      else
+        error 'Not all install jobs completed successfully.'
+        puts 'You should review the logs for these using the commands below.'
+        puts
+        result = false
+      end
+
+      jobs.each do |job|
+        icon = if job['status']['succeeded']
+                 '✅'
+               else
+                 '❌'
+               end
+        puts "  #{icon}  " + @stage.kubectl("logs job/#{job['metadata']['name']}")
+      end
+      puts
+
+      result
+    end
+
+    class << self
+      def setup(context)
+        recipe = Hippo::Recipe.load_from_file(context.options[:hippofile] || './Hippofile')
+
+        stage = recipe.stages[CURRENT_STAGE]
+        if stage.nil?
+          raise Error, "Invalid stage name `#{CURRENT_STAGE}`. Check this has been defined in in your stages directory with a matching name?"
+        end
+
+        new(recipe, stage)
+      end
+    end
+  end
+end

data/lib/hippo/error.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Hippo
+  class Error < StandardError
+  end
+
+  class RepositoryAlreadyClonedError < Error
+  end
+
+  class RepositoryCloneError < Error
+  end
+
+  class RepositoryFetchError < Error
+  end
+
+  class RepositoryCheckoutError < Error
+  end
+end

data/lib/hippo/kubernetes.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require 'liquid'
+require 'hippo/util'
+
+module Hippo
+  class Kubernetes
+    OBJECT_DIRECTORY_NAMES = %w[config deployments jobs/install jobs/deploy services].freeze
+
+    include Hippo::Util
+
+    def initialize(recipe, options)
+      @recipe = recipe
+      @options = options
+    end
+
+    # Load and return a set of objects from a given path.
+    # Parse them through the templating and return them in the appropriate
+    # context.
+    #
+    # @param stage [Hippo::Stage]
+    # @param commit [String] the commit ref
+    # @param path [String]
+    # @return
+    def objects(path, stage, commit)
+      time = Time.now
+
+      yamls = load_yaml_from_directory(path)
+      yamls |= load_yaml_from_directory(File.join(path, stage.name))
+
+      yamls.map do |yaml_part|
+        object = yaml_part.parse(@recipe, stage, commit)
+
+        # Unless a namespace has been specified in the metadata we will
+        # want to add the namespace for the current stage.
+        if object['metadata'].nil? || object['metadata']['namespace'].nil?
+          object['metadata'] ||= {}
+          object['metadata']['namespace'] = stage.namespace
+        end
+
+        # Add our own details to the metadata of all objets created by us so
+        # we know where they came from.
+        object['metadata']['annotations'] ||= {}
+        object['metadata']['annotations']['hippo.adam.ac/builtAt'] ||= time.to_s
+        object['metadata']['annotations']['hippo.adam.ac/builtBy'] ||= ENV['USER'] || 'unknown'
+
+        add_default_labels(object, stage)
+
+        # Add some information to Deployments to reflect the latest
+        # information about this deployment.
+        if object['kind'] == 'Deployment'
+          object['metadata']['annotations']['hippo.adam.ac/deployID'] ||= time.to_i.to_s
+          if commit
+            object['metadata']['annotations']['hippo.adam.ac/commitRef'] ||= commit.objectish
+            object['metadata']['annotations']['hippo.adam.ac/commitMessage'] ||= commit.message
+          end
+
+          if pod_metadata = object.dig('spec', 'template', 'metadata')
+            pod_metadata['annotations'] ||= {}
+            pod_metadata['annotations']['hippo.adam.ac/deployID'] ||= time.to_i.to_s
+            if commit
+              pod_metadata['annotations']['hippo.adam.ac/commitRef'] ||= commit.objectish
+            end
+          end
+        end
+
+        object
+      end
+    end
+
+    def apply_namespace(stage)
+      namespace = {
+        'kind' => 'Namespace',
+        'apiVersion' => 'v1',
+        'metadata' => {
+          'name' => stage.namespace
+        }
+      }
+      add_default_labels(namespace, stage)
+      apply_with_kubectl(namespace.to_yaml)
+    end
+
+    # Apply the given configuration with kubectl
+    #
+    # @param config [Array<Hippo::YAMLPart>, String]
+    # @return [void]
+    def apply_with_kubectl(yaml_parts)
+      unless yaml_parts.is_a?(String)
+        yaml_parts = [yaml_parts] unless yaml_parts.is_a?(Array)
+        yaml_parts = yaml_parts.map { |yp| yp.hash.to_yaml }.join("\n---\n")
+      end
+
+      Open3.popen3('kubectl apply -f -') do |stdin, stdout, stderr, wt|
+        stdin.puts yaml_parts
+        stdin.close
+
+        stdout = stdout.read.strip
+        stderr = stderr.read.strip
+
+        if wt.value.success?
+          puts stdout
+          stdout.split("\n").each_with_object({}) do |line, hash|
+            if line =~ %r{\A([\w\/\-\.]+) (\w+)\z}
+              hash[Regexp.last_match(1)] = Regexp.last_match(2)
+            end
+          end
+        else
+          raise Error, "[kubectl] #{stderr}"
+        end
+      end
+    end
+
+    # Get details of objects using kubectl.
+    #
+    # @param stage [Hippo::Stage]
+    # @param names [Array<String>]
+    # @raises [Hippo::Error]
+    # @return [Array<Hash>]
+    def get_with_kubectl(stage, *names)
+      command = [
+        'kubectl',
+        '-n', stage.namespace,
+        'get',
+        names,
+        '-o', 'yaml'
+      ].flatten.reject(&:nil?)
+
+      Open3.popen3(*command) do |_, stdout, stderr, wt|
+        if wt.value.success?
+          yaml = YAML.safe_load(stdout.read, permitted_classes: [Time])
+          yaml['items'] || [yaml]
+        else
+          raise Error, "[kutectl] #{stderr.read}"
+        end
+      end
+    end
+
+    # Delete a named job from the cluster
+    #
+    # @param stage [Hippo::Stage]
+    # @param name [String]
+    # @raises [Hippo::Error]
+    # @return [void]
+    def delete_job(stage, name)
+      command = [
+        'kubectl',
+        '-n', stage.namespace,
+        'delete',
+        'job',
+        name
+      ]
+
+      Open3.popen3(*command) do |_, stdout, stderr, wt|
+        if wt.value.success?
+          puts stdout.read
+          true
+        else
+          stderr = stderr.read
+          if stderr =~ /\"#{name}\" not found/
+            false
+          else
+            raise Error, "[kutectl] #{stderr.read}"
+          end
+        end
+      end
+    end
+
+    # Poll the named jobs and return them when all are complete
+    # or the number of checks is exceeded.
+    #
+    # @param stage [Hippo::Stage]
+    # @param names [Array<String>]
+    # @param times [Integer]
+    # @return [Array<Boolean, Array<Hash>]
+    def wait_for_jobs(stage, names, times = 120)
+      jobs = nil
+      times.times do
+        jobs = get_with_kubectl(stage, *names)
+
+        # Are all the jobs completed?
+        if jobs.all? { |j| j['status']['active'].nil? }
+          return [false, jobs]
+        else
+          sleep 2
+        end
+      end
+      [true, jobs]
+    end
+
+    private
+
+    def add_default_labels(object, stage)
+      object['metadata']['labels'] ||= {}
+      object['metadata']['labels']['app.kubernetes.io/name'] = @recipe.name
+      object['metadata']['labels']['app.kubernetes.io/instance'] = stage.name
+      object['metadata']['labels']['app.kubernetes.io/managed-by'] = 'hippo'
+      object
+    end
+  end
+end

data/lib/hippo/recipe.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'hippo/build_spec'
+require 'hippo/error'
+require 'hippo/kubernetes'
+require 'hippo/repository'
+require 'hippo/stage'
+require 'hippo/util'
+
+module Hippo
+  class Recipe
+    include Util
+
+    class RecipeNotFound < Error
+    end
+
+    class << self
+      # Load a new Recipe class from a given file.
+      #
+      # @param path [String] path to recipe file.
+      # @return [Hippo::Recipe]
+      def load_from_file(path)
+        unless File.file?(path)
+          raise RecipeNotFound, "No recipe file found at #{path}"
+        end
+
+        hash = YAML.load_file(path)
+        new(hash, path)
+      end
+    end
+
+    # @param hash [Hash] the raw hash from the underlying yaml file
+    def initialize(hash, hippofile_path = nil)
+      @hash = hash
+      @hippofile_path = hippofile_path
+    end
+
+    attr_reader :path
+
+    # Return the root directory where the Hippofile is located
+    #
+    def root
+      File.dirname(@hippofile_path)
+    end
+
+    # Return the repository that this manifest should be working with
+    #
+    # @return [Hippo::Repository]
+    def repository
+      return unless @hash['repository']
+
+      @repository ||= Repository.new(@hash['repository'])
+    end
+
+    # Return the app name
+    #
+    # @return [String, nil]
+    def name
+      @hash['name']
+    end
+
+    # Return kubernetes configuration
+    #
+    # @return [Hippo::Kubernetes]
+    def kubernetes
+      @kubernetes ||= Kubernetes.new(self, @hash['kubernetes'] || {})
+    end
+
+    # Return the stages for this recipe
+    #
+    # @return [Hash<Hippo::Stage>]
+    def stages
+      @stages ||= begin
+        yamls = load_yaml_from_directory(File.join(root, 'stages'))
+        yamls.each_with_object({}) do |yaml, hash|
+          stage = Stage.new(yaml)
+          hash[stage.name] = stage
+        end
+      end
+    end
+
+    # Return the builds for this recipe
+    #
+    # @return [Hash<Hippo::BuildSpec>]
+    def build_specs
+      @build_specs ||= @hash['builds'].each_with_object({}) do |(key, options), hash|
+        hash[key] = BuildSpec.new(self, key, options)
+      end
+    end
+
+    # Return configuration
+    #
+    # @return []
+    def console
+      @hash['console']
+    end
+
+    # Return the template variables that should be exposed
+    #
+    # @return [Hash]
+    def template_vars
+      {
+        'repository' => repository ? repository.template_vars : nil,
+        'builds' => build_specs.each_with_object({}) { |(_, bs), h| h[bs.name] = bs.template_vars }
+      }
+    end
+
+    # Parse a string through the template parser
+    #
+    # @param string [String]
+    # @return [String]
+    def parse(stage, commit, string)
+      template = Liquid::Template.parse(string)
+      template_variables = template_vars
+      template_variables['stage'] = stage.template_vars
+      if commit
+        template_variables['commit'] = {
+          'ref' => commit.objectish,
+          'message' => commit.message
+        }
+      end
+      template.render(template_variables)
+    end
+  end
+end