k8s-harness 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ace0cae7e7c7b384c5420e91e29bc2ddac484160f23772b9fc0b58e49a73e16e
+  data.tar.gz: d595a4aff27c2b8a14acf89d40b9d3a18d9482debded7894328e8a91350ec511
+SHA512:
+  metadata.gz: 7754badb5e231b29e20c81c466feb865d52e9444dc904ab7be7fa285dc2ec4a8476f91c5862971a07071e696eaff8b58734beacd41d9bc59f299e6177b1b75f4
+  data.tar.gz: a946ffb779de5a8a433f1844d3d1b95d038ea81a6f580e1c5e4d7f2840708d2875393d348742dbed7803735f8311fbcc23d0f26894ab24c46b30136fc32a5a44

data/bin/k8s-harness

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'k8s_harness'
+
+KubernetesHarness::CLI.parse(ARGV)

data/conf/required_software.yaml

@@ -0,0 +1,5 @@
+---
+- name: vagrant
+  version_check: vagrant --version
+- name: ansible-playbook
+  version_check: ansible-playbook --version

data/include/Vagrantfile

@@ -0,0 +1,39 @@
+# vim: set ft=ruby:
+num_nodes = ENV["K3S_NUMBER_OF_NODES"] || 2
+memory_per_node = ENV["K3S_NODE_MEMORY_GB"] || 1024
+ssh_pub_key = File.read("#{ENV['VAGRANT_CWD']}/ssh_key.pub").gsub("\n","")
+install_ansible_command = <<-COMMAND
+  apk update
+  if ! apk add ansible
+  then
+    echo "ERROR: Failed to install Ansible on this machine."
+    exit 1
+  fi
+COMMAND
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "maier/alpine-3.6-x86_64"
+  config.vm.provider "virtualbox" do |vb|
+    vb.customize [ "modifyvm", :id, "--memory", memory_per_node ]
+  end
+
+  config.vm.define "k3s-registry" do |node|
+    node.vm.hostname  = "k3s-registry"
+    node.vm.network "private_network", ip: "192.168.50.200"
+    node.vm.network "forwarded_port", guest: 5000, host: 5000
+    node.vm.provision "shell",
+      inline: "echo '#{ssh_pub_key}' >> /home/vagrant/.ssh/authorized_keys"
+    node.vm.provision "shell", inline: install_ansible_command
+  end
+
+  num_nodes.times do |node_id|
+    config.vm.define "k3s-node-#{node_id}" do |node|
+      node.vm.hostname  = "k3s-node-#{node_id}"
+      node.vm.network "private_network", ip: "192.168.50.#{node_id+2}"
+      node.vm.network "forwarded_port", guest: 6443, host: 6443 if node_id == 0
+      node.vm.provision "shell",
+        inline: "echo '#{ssh_pub_key}' >> /home/vagrant/.ssh/authorized_keys"
+      node.vm.provision "shell", inline: install_ansible_command
+    end
+  end
+end

data/include/inventory

@@ -0,0 +1,10 @@
+# These IP addresses are statically allocated.
+# See scripts/deploy_k3s for more info.
+[master]
+192.168.50.2
+
+[worker]
+192.168.50.3
+
+[registry]
+192.168.50.200

data/include/site.yml

@@ -0,0 +1,207 @@
+---
+- hosts: master
+  become: true
+  gather_facts: no
+  tasks:
+    - name: Get this host's IP address
+      shell: "echo $(ip -4 -o addr show eth1 | awk '{print $4}' | cut -f1 -d '/')"
+      register: result
+
+    - set_fact:
+        ip_address: "{{ result.stdout }}"
+
+    - name: Create directories
+      file:
+        path: "{{ item }}"
+        state: directory
+      with_items:
+        - /etc/rancher/k3s
+        - /etc/docker
+
+    - name: Create registry files
+      file:
+        path: "{{ item }}"
+        state: touch
+      with_items:
+        - /etc/rancher/k3s/registries.yaml
+        - /etc/docker/daemon.json
+
+    - name: Configure insecure registries for k3s
+      blockinfile:
+        path: /etc/rancher/k3s/registries.yaml
+        block: |
+          mirrors:
+            "10.0.2.2:5000":
+              endpoint:
+                - "http://10.0.2.2:5000"
+
+    - name: Configure insecure regsitries for containerd
+      block:
+        - name: Create the daemon file
+          blockinfile:
+            path: /etc/docker/daemon.json
+            marker: ""
+            block: |
+              { "insecure-registries": [ "10.0.2.2:5000" ] }
+
+        - name: Remove blank lines
+          lineinfile:
+            path: /etc/docker/daemon.json
+            state: absent
+            regexp: '^$'
+
+    - name: Install Rancher k3s
+      shell: curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--node-ip={{ ip_address }} --flannel-iface=eth1" K3S_TOKEN={{ k3s_token }} sh -
+
+    - name: Check if extlinux updated
+      shell: "grep -q cgroup_enable=cpuset /etc/update-extlinux.conf"
+      register: extlinux_enabled_result
+      ignore_errors: true
+
+    - name: Update extlinux per documentation
+      shell: "echo 'default_kernel_opts=\"...  cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\" >> /etc/update-extlinux.conf'"
+      when: extlinux_enabled_result.rc != 0
+
+    - name: Apply extlinux updates
+      shell: update-extlinux
+      when: extlinux_enabled_result.rc != 0
+
+    - name: Reboot
+      shell: /sbin/reboot
+      when: extlinux_enabled_result.rc != 0
+
+    - name: "Wait for machine"
+      become: false
+      register: wait_result
+      local_action: wait_for host={{ ip_address }} port=22 timeout=300 connect_timeout=300
+
+- hosts: worker
+  become: true
+  tasks:
+    - name: Get this host's IP address
+      shell: "echo $(ip -4 -o addr show eth1 | awk '{print $4}' | cut -f1 -d '/')"
+      register: result
+
+    - set_fact:
+        ip_address: "{{ result.stdout }}"
+
+    - name: Wait for master to become available
+      register: wait_result
+      wait_for:
+        timeout: 300
+        connect_timeout: 300
+        host: 192.168.50.2
+        port: 6443
+
+
+    - name: Create directories
+      file:
+        path: "{{ item }}"
+        state: directory
+      with_items:
+        - /etc/rancher/k3s
+        - /etc/docker
+
+    - name: Create registry files
+      file:
+        path: "{{ item }}"
+        state: touch
+      with_items:
+        - /etc/rancher/k3s/registries.yaml
+        - /etc/docker/daemon.json
+
+    - name: Configure insecure registries for k3s
+      blockinfile:
+        path: /etc/rancher/k3s/registries.yaml
+        block: |
+          mirrors:
+            "10.0.2.2:5000":
+              endpoint:
+                - "http://10.0.2.2:5000"
+
+    - name: Configure insecure regsitries for containerd
+      block:
+        - name: Create the daemon file
+          blockinfile:
+            path: /etc/docker/daemon.json
+            marker: ""
+            block: |
+              { "insecure-registries": [ "10.0.2.2:5000" ] }
+
+        - name: Remove blank lines
+          lineinfile:
+            path: /etc/docker/daemon.json
+            state: absent
+            regexp: '^$'
+
+    - name: Install k3s as worker
+      shell: curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--node-ip={{ ip_address }} --flannel-iface=eth1" K3S_URL=https://192.168.50.2:6443 K3S_TOKEN={{ k3s_token }} sh -
+
+    - name: Check if extlinux updated
+      shell: "grep -q cgroup_enable=cpuset /etc/update-extlinux.conf"
+      register: extlinux_enabled_result
+      ignore_errors: true
+
+    - name: Update extlinux per documentation
+      shell: "echo 'default_kernel_opts=\"...  cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\" >> /etc/update-extlinux.conf'"
+      when: extlinux_enabled_result.rc != 0
+
+    - name: Apply extlinux updates
+      shell: update-extlinux
+      when: extlinux_enabled_result.rc != 0
+
+    - name: Reboot
+      shell: /sbin/reboot
+      when: extlinux_enabled_result.rc != 0
+
+    - name: "Wait for machine"
+      become: false
+      register: wait_result
+      local_action: wait_for host={{ ip_address }} port=22 timeout=300 connect_timeout=300
+
+
+- hosts: registry
+  become: true
+  tasks:
+    - name: Get this host's IP address
+      shell: "echo $(ip -4 -o addr show eth1 | awk '{print $4}' | cut -f1 -d '/')"
+      register: result
+
+    - set_fact:
+        ip_address: "{{ result.stdout }}"
+
+
+    - name: Install Docker
+      apk:
+        name:
+        - docker
+
+    - name: Add docker as service
+      shell: "rc-update add docker boot"
+
+    - name: Reboot
+      shell: /sbin/reboot
+
+    - name: "Wait for machine"
+      become: false
+      register: wait_result
+      local_action: wait_for host={{ ip_address }} port=22 timeout=300 connect_timeout=300
+
+    - name: Start docker daemon
+      shell: "service docker start"
+      retries: 5
+      delay: 2
+
+    - name: Confirm Docker available
+      shell: "docker run --rm hello-world"
+      retries: 5
+      delay: 2
+
+    - name: Check for instances of registry
+      shell: "sudo docker ps | grep -q registry"
+      register: result
+      ignore_errors: true
+
+    - name: Start Docker Registry
+      shell: "sudo docker run -d --restart=always -p 5000:5000 --name registry registry:2"
+      when: result.rc != 0

data/lib/k8s_harness.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'k8s_harness/cli'
+require 'k8s_harness/clusters'
+require 'k8s_harness/logging'

data/lib/k8s_harness/cli.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require 'k8s_harness/subcommand'
+
+# KubernetesHarness
+module KubernetesHarness
+  # This module contains everything CLI-related.
+  # We're using it as the entry-point for k8s-harness.
+  module CLI
+    @options = {
+      base: {}
+    }
+    @subcommands = {
+      run: {
+        description: 'Runs tests',
+        option_parser: OptionParser.new do |opts|
+          opts.banner = 'Usage: k8s-harness run [options]'
+          opts.separator 'Runs tests'
+          opts.separator ''
+          opts.separator 'Commands:'
+          opts.on('-h', '--help', 'Displays this help message') do
+            add_option(options: { show_usage: true }, subcommand: :run)
+            puts opts
+          end
+          opts.on('--disable-teardown', 'Keeps the cluster up for local testing') do
+            add_option(options: { disable_teardown: true }, subcommand: :run)
+          end
+        end
+      },
+      validate: {
+        description: 'Validates .k8sharness files',
+        option_parser: OptionParser.new do |opts|
+          opts.banner = 'Usage: k8s-harness validate [options]'
+          opts.separator 'Validates that a .k8sharness file is correct'
+          opts.separator ''
+          opts.separator 'Commands:'
+          opts.on('-h', '--help', 'Displays this help message') do
+            add_option(options: { show_usage: true }, subcommand: :validate)
+            puts opts
+          end
+        end
+      },
+      destroy: {
+        description: 'Deletes a live cluster provisioned by k8s-harness WITHOUT WARNING.',
+        option_parser: OptionParser.new do |opts|
+          opts.banner = 'Usage: k8s-harness destroy [options]'
+          opts.separator 'Deletes live clusters provisioned by k8s-harness WITHOUT WARNING'
+          opts.separator ''
+          opts.separator 'Commands:'
+          opts.on('-h', '--help', 'Displays this help message') do
+            add_option(options: { show_usage: true }, subcommand: :destroy)
+            puts opts
+          end
+        end
+      }
+    }
+
+    @base_command = OptionParser.new do |opts|
+      opts.banner = 'Usage: k8s-harness [subcommand] [options]'
+      opts.separator 'Test your apps in disposable Kubernetes clusters'
+      opts.separator ''
+      opts.separator 'Sub-commands:'
+      opts.separator ''
+      @subcommands.each_key do |subcommand|
+        opts.separator "    #{subcommand.to_s.ljust(20)} #{@subcommands[subcommand][:description]}"
+      end
+      opts.separator ''
+      opts.separator 'See k8s-harness [subcommand] --help for more specific options.'
+      opts.separator ''
+      opts.separator 'Global options:'
+      opts.on('-d', '--debug', 'Show debug output') do
+        add_option(options: { enable_debug_logging: true })
+      end
+      opts.on('-h', '--help', 'Displays this help message') do
+        add_option(options: { help: opts.help })
+      end
+    end
+
+    def self.parse(args)
+      args.push('-h') if args.empty? || subcommands_missing?(args)
+      @base_command.order!(args)
+      subcommand = args.shift
+      if subcommand.nil?
+        puts @options[:base][:help]
+      else
+        enable_debug_logging_if_present
+        @subcommands[subcommand.to_sym][:option_parser].order!(args)
+        call_entrypoint(subcommand)
+      end
+    end
+
+    def self.enable_debug_logging_if_present
+      KubernetesHarness::Logging.enable_debug_logging if @options[:base][:enable_debug_logging]
+    end
+
+    def self.subcommands_missing?(args)
+      args.select { |arg| arg.match?(/^[a-z]/) }.empty?
+    end
+
+    def self.add_option(options:, subcommand: nil)
+      if subcommand.nil?
+        @options[:base].merge!(options)
+      else
+        @options[subcommand] = {} unless @options.key subcommand
+        @options[subcommand].merge!(options)
+      end
+    end
+
+    def self.call_entrypoint(subcommand)
+      KubernetesHarness::Subcommand.method(subcommand.to_sym).call(@options[subcommand.to_sym])
+    end
+
+    private_class_method :call_entrypoint, :add_option, :subcommands_missing?
+  end
+end

data/lib/k8s_harness/clusters.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require 'k8s_harness/clusters/ansible'
+require 'k8s_harness/clusters/constants'
+require 'k8s_harness/clusters/cluster_info'
+require 'k8s_harness/clusters/metadata'
+require 'k8s_harness/clusters/required_software'
+require 'k8s_harness/clusters/vagrant'
+require 'k8s_harness/shell_command'
+
+module KubernetesHarness
+  # Handles bring up and deletion of disposable clusters.
+  module Clusters
+    def self.create!
+      RequiredSoftware.ensure_installed_or_exit!
+      Metadata.initialize!
+      create_ssh_key!
+      vagrant_up_disposable_cluster_or_exit!
+      cluster = ClusterInfo.new(master_ip_address_command: master_ip_address_command,
+                                worker_ip_addresses_command: worker_ip_addresses_command,
+                                docker_registry_command: docker_registry_command,
+                                kubeconfig_path: 'not_yet',
+                                ssh_key_path: cluster_ssh_key)
+      Metadata.write!('cluster.yaml', cluster.to_yaml)
+      cluster
+    end
+
+    def self.create_ssh_key!
+      ssh_key_fp = File.join(Metadata.default_dir, 'ssh_key')
+      return if File.exist? ssh_key_fp
+
+      KubernetesHarness.nice_logger.info 'Creating a new SSH key for the cluster.'
+      ssh_key_command = ShellCommand.new(
+        "ssh-keygen -t rsa -f '#{ssh_key_fp}' -q -N ''"
+      )
+      raise 'Unable to create a SSH key for the cluster' unless ssh_key_command.execute!
+    end
+
+    def self.provision!(cluster_info)
+      all_results = provision_nodes_in_parallel!(cluster_info)
+      failures = all_results.filter { |thread| !thread.success? }
+      raise failed_cluster_error(failures) unless failures.empty?
+
+      cluster_info.kubeconfig_path = cluster_kubeconfig
+      true
+    end
+
+    # TODO: tests missing
+    def self.teardown!
+      destroy_nodes_in_parallel!
+
+      true
+    end
+
+    # TODO: tests missing
+    def self.destroy_existing!
+      destroy_nodes_in_parallel!
+    end
+
+    def self.destroy_nodes_in_parallel!
+      if cluster_running?
+        KubernetesHarness.logger.debug('🚨 Deleting all nodes! 🚨')
+        vagrant_threads = []
+        Constants::ALL_NODES.each do |node|
+          KubernetesHarness.logger.debug("Starting thread for node #{node}")
+          vagrant_threads << Thread.new do
+            vagrant_command = Vagrant.new_command('destroy', ['-f', node])
+            vagrant_command.execute!
+            vagrant_command
+          end
+        end
+        results = vagrant_threads.each(&:join).map(&:value)
+        failures = results.filter { |result| !result.success? }
+        raise failed_cluster_destroy_error(failures) unless failures.empty?
+
+        delete_cluster_yaml_and_ssh_key!
+      else
+        KubernetesHarness.nice_logger.info('No clusters found to destroy. Stopping.')
+      end
+    end
+
+    def self.provision_nodes_in_parallel!(cluster_info)
+      ansible_threads = []
+      ssh_key_path = cluster_info.ssh_key_path
+      [cluster_info.master_ip_address,
+       cluster_info.worker_ip_addresses,
+       cluster_info.docker_registry_address].flatten.each do |addr|
+        ansible_threads << Thread.new do
+          command = Ansible::Playbook.create_run_against_single_host(
+            playbook_fp: playbook_path,
+            ssh_key_path: ssh_key_path,
+            inventory_fp: inventory_path,
+            ip_address: addr,
+            extra_vars: ["k3s_token=#{cluster_info.kubernetes_cluster_token}"]
+          )
+          command.execute!
+          command
+        end
+      end
+      ansible_threads.each(&:join).map(&:value)
+    end
+
+    def self.worker_ip_addresses_command
+      Constants::WORKER_NODE_NAMES.map do |node|
+        Vagrant.create_and_execute_new_ssh_command(node, Constants::IP_ETH1_COMMAND)
+      end
+    end
+
+    def self.master_ip_address_command
+      Vagrant.create_and_execute_new_ssh_command(Constants::MASTER_NODE_NAME,
+                                                 Constants::IP_ETH1_COMMAND)
+    end
+
+    def self.docker_registry_command
+      Vagrant.create_and_execute_new_ssh_command(Constants::DOCKER_REGISTRY_NAME,
+                                                 Constants::IP_ETH1_COMMAND)
+    end
+
+    def self.cluster_kubeconfig
+      args = [
+        '-c',
+        '"sudo cat /etc/rancher/k3s/k3s.yaml"',
+        Constants::MASTER_NODE_NAME.to_s
+      ]
+      command = Vagrant.new_command('ssh', args)
+      command.execute!
+      if command.stdout.empty?
+        KubernetesHarness.logger.warn('No kubeconfig created!')
+        return
+      end
+      Metadata.write!('kubeconfig', command.stdout)
+      File.join Metadata.default_dir, 'kubeconfig'
+    end
+
+    def self.cluster_ssh_key
+      File.join KubernetesHarness::Clusters::Metadata.default_dir, '/ssh_key'
+    end
+
+    def self.playbook_path
+      File.join Metadata.default_dir, 'site.yml'
+    end
+
+    def self.inventory_path
+      File.join Metadata.default_dir, 'inventory'
+    end
+
+    def self.vagrant_up_disposable_cluster_or_exit!
+      KubernetesHarness.logger.debug('🚀 Creating node new disposable cluster 🚀')
+      vagrant_threads = []
+      Constants::ALL_NODES.each do |node|
+        KubernetesHarness.logger.debug("Starting thread for node #{node}")
+        vagrant_threads << Thread.new do
+          vagrant_command = Vagrant.new_command('up', [node])
+          vagrant_command.execute!
+          vagrant_command
+        end
+      end
+      results = vagrant_threads.each(&:join).map(&:value)
+      failures = results.filter { |result| !result.success? }
+      raise failed_cluster_error(failures) unless failures.empty?
+    end
+
+    def self.generate_err_msg(cmd)
+      header = "From command '#{cmd.command}':"
+      separator = '-' * (header.length + 4)
+
+      <<~MESSAGE
+        #{header}
+        #{separator}
+
+        Output:
+        #{cmd.stdout}
+
+        Errors:
+        #{cmd.stderr}
+      MESSAGE
+    end
+
+    def self.failed_cluster_error(command)
+      stderr = if command.is_a? Array
+                 command.map do |cmd|
+                   generate_err_msg(cmd)
+                 end.flatten.join("\n\n")
+               else
+                 generate_err_msg(command)
+               end
+      raise "Failed to start Kubernetes cluster. Here's why:\n\n#{stderr}"
+    end
+
+    def self.failed_cluster_destroy_error(command)
+      stderr = if command.is_a? Array
+                 command.map(&:stderr).uniq!.join("\n")
+               else
+                 command.stderr
+               end
+      raise "Failed to delete Kubernetes cluster. Here's why:\n\n#{stderr}"
+    end
+
+    def self.delete_cluster_yaml_and_ssh_key!
+      ['ssh_key', 'ssh_key.pub', 'cluster.yaml'].each do |file|
+        Metadata.delete!(file)
+      end
+    end
+
+    def self.cluster_running?
+      vagrant_status_command = Vagrant.new_command('global-status')
+      vagrant_status_command.execute!
+      vagrant_status_command.stdout.match?(/k3s-/)
+    end
+  end
+end

data/lib/k8s_harness/clusters/ansible.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'k8s_harness/paths'
+
+module KubernetesHarness
+  module Clusters
+    # Simple module for interacting with Vagrant.
+    module Ansible
+      # for ansible-playbook
+      module Playbook
+        def self.create_run_against_single_host(playbook_fp:,
+                                                inventory_fp:,
+                                                ssh_key_path:,
+                                                ip_address:,
+                                                extra_vars:)
+          log_new_run(playbook_fp, inventory_fp, ssh_key_path, ip_address, extra_vars)
+          command_env = {
+            ANSIBLE_HOST_KEY_CHECKING: 'no',
+            ANSIBLE_SSH_ARGS: '-o IdentitiesOnly=true',
+            ANSIBLE_COMMAND_WARNINGS: 'False',
+            ANSIBLE_PYTHON_INTERPRETER: '/usr/bin/python'
+          }
+          KubernetesHarness::ShellCommand.new(
+            command(playbook_fp, inventory_fp, ssh_key_path, ip_address, extra_vars),
+            environment: command_env
+          )
+        end
+
+        def self.command(playbook_fp, inventory_fp, ssh_key_path, ip_address, extra_vars)
+          [
+            'ansible-playbook',
+            "-i #{inventory_fp}",
+            "-e \"ansible_ssh_user=\\\"#{ENV['ANSIBLE_SSH_USER'] || 'vagrant'}\\\"\"",
+            extra_vars.map { |var| "-e \"#{var}\"" },
+            "-l #{ip_address}",
+            "--private-key #{ssh_key_path}",
+            playbook_fp
+          ].flatten.join(' ')
+        end
+
+        def self.log_new_run(playbook_fp, inventory_fp, ssh_key_path, ip_address = '', extra_vars)
+          KubernetesHarness.logger.info(
+            <<~MESSAGE.strip
+              Creating a new single-host Ansible Playbook run! \
+              playbook: #{playbook_fp}, \
+              inventory: #{inventory_fp}, \
+              ssh_key: #{ssh_key_path}, \
+              ip_address: #{ip_address}, \
+              extra_vars: #{extra_vars}
+            MESSAGE
+          )
+        end
+      end
+    end
+  end
+end

data/lib/k8s_harness/clusters/cluster_info.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'digest/md5'
+
+module KubernetesHarness
+  module Clusters
+    # This class provides a handy set of information that might be useful for k8s-harness
+    # users after creating their clusters.
+    class ClusterInfo
+      attr_reader :master_ip_address,
+                  :worker_ip_addresses,
+                  :docker_registry_address,
+                  :ssh_key_path,
+                  :kubernetes_cluster_token
+      attr_accessor :kubeconfig_path
+
+      IP_ADDRESS_REGEX = /
+        \b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.
+        (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.
+        (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.
+        (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b
+      /x.freeze
+
+      def initialize(master_ip_address_command:,
+                     worker_ip_addresses_command:,
+                     docker_registry_command:,
+                     kubeconfig_path:,
+                     ssh_key_path:)
+        @kubeconfig_path = kubeconfig_path
+        @ssh_key_path = ssh_key_path
+        @master_ip_address = get_ip_addresses_from_command(master_ip_address_command).first
+        @docker_registry_address = get_ip_addresses_from_command(docker_registry_command).first
+        @worker_ip_addresses =
+          worker_ip_addresses_command.map do |command|
+            get_ip_addresses_from_command(command)
+          end.flatten
+        @kubernetes_cluster_token = generate_k8s_token(
+          @master_ip_address,
+          @worker_ip_addresses,
+          @docker_registry_address
+        )
+      end
+
+      def to_yaml
+        YAML.dump({
+                    master_ip_address: @master_ip_address,
+                    worker_ip_addresses: @worker_ip_addresses,
+                    docker_registry_address: @docker_registry_address,
+                    kubeconfig_path: @kubeconfig_path,
+                    ssh_key_path: @ssh_key_path,
+                    kubernetes_cluster_token: @kubernetes_cluster_token
+                  })
+      end
+
+      private
+
+      def generate_k8s_token(master_ip, worker_ip, docker_registry)
+        combined_addresses = [master_ip, worker_ip, docker_registry].flatten.join('')
+        Digest::MD5.hexdigest(combined_addresses)
+      end
+
+      def get_ip_addresses_from_command(command)
+        command.execute!
+        command.stdout.split("\n").select { |line| line.match? IP_ADDRESS_REGEX }
+      end
+    end
+  end
+end

data/lib/k8s_harness/clusters/constants.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module KubernetesHarness
+  module Clusters
+    # Just constants.
+    module Constants
+      MASTER_NODE_NAME = 'k3s-node-0'
+      WORKER_NODE_NAMES = ['k3s-node-1'].freeze
+      DOCKER_REGISTRY_NAME = 'k3s-registry'
+      IP_ETH1_COMMAND =
+        "\"ip addr show dev eth1 | grep \'\\<inet\\>\' | awk \'{print \\$2}\' | cut -f1 -d \'/\'\""
+      ALL_NODES = [MASTER_NODE_NAME, WORKER_NODE_NAMES, DOCKER_REGISTRY_NAME].flatten
+    end
+  end
+end

data/lib/k8s_harness/clusters/metadata.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'k8s_harness/paths'
+
+module KubernetesHarness
+  # This module is for everything around CRUD'ing disposable clusters.
+  module Clusters
+    # k8s-harness relies on storing things like Ansible playbooks for our
+    # disposable cluster and extra files that users might use.
+    # This module handles all of that.
+    module Metadata
+      def self.default_dir
+        "#{ENV['PWD']}/.k8sharness_data"
+      end
+
+      def self.create_dir!
+        ::FileUtils.mkdir_p default_dir unless Dir.exist? default_dir
+      end
+
+      def self.initialize!
+        create_dir!
+        FileUtils.cp_r("#{KubernetesHarness::Paths.include_dir}/.", default_dir)
+      end
+
+      def self.write!(file_name, content)
+        KubernetesHarness.logger.debug "Creating new metadata: #{file_name}"
+        fp = File.join default_dir, file_name
+        File.write(fp, content)
+      end
+
+      def self.delete!(file_name)
+        KubernetesHarness.logger.debug "Deleting from metadata: #{file_name}"
+        fp = File.join default_dir, file_name
+        FileUtils.rm(fp)
+      end
+    end
+  end
+end

data/lib/k8s_harness/clusters/required_software.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'k8s_harness/paths'
+require 'k8s_harness/shell_command'
+
+module KubernetesHarness
+  module Clusters
+    # This module ensures that we have the software we need to run k8s-harness
+    # on the user's machine.
+    module RequiredSoftware
+      def self.software
+        YAML.safe_load(
+          File.read(File.join(KubernetesHarness::Paths.conf_dir, 'required_software.yaml')),
+          symbolize_names: true
+        )
+      end
+
+      def self.ensure_installed_or_exit!
+        missing = []
+        software.each do |app_data|
+          name = app_data[:name]
+          version_check = app_data[:version_check]
+          KubernetesHarness.logger.debug("Checking that this is installed: #{name}")
+          command_string = "sh -c '#{version_check}; exit $?'"
+          command = KubernetesHarness::ShellCommand.new(command_string)
+          command.execute!
+          missing.push name unless command.success?
+        end
+
+        raise show_missing_software_message(missing) unless missing.empty?
+      end
+
+      def self.show_missing_software_message(apps)
+        <<~MESSAGE.strip
+          You are missing the following software:
+
+          #{apps.map { |app| "- #{app}" }.join("\n")}
+
+          Please consult the README to learn what you'll need to install before using k8s-harness.
+        MESSAGE
+      end
+
+      private_class_method :show_missing_software_message
+    end
+  end
+end

data/lib/k8s_harness/clusters/vagrant.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module KubernetesHarness
+  module Clusters
+    # Simple module for interacting with Vagrant.
+    module Vagrant
+      def self.new_command(command, args = nil)
+        command_env = {
+          VAGRANT_CWD: Metadata.default_dir
+        }
+        command = "vagrant #{command}"
+        command = "#{command} #{[args].flatten.join(' ')}" unless args.nil?
+        KubernetesHarness::ShellCommand.new(command, environment: command_env)
+      end
+
+      def self.create_and_execute_new_ssh_command(node_name, command)
+        args = ['-c', command, node_name]
+        command = Vagrant.new_command('ssh', args)
+        command.execute!
+        command
+      end
+    end
+  end
+end

data/lib/k8s_harness/harness_file.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'shellwords'
+require 'yaml'
+require 'k8s_harness/shell_command'
+
+module KubernetesHarness
+  # This module handles reading and validating .k8sharness files.
+  module HarnessFile
+    def self.execute_setup!(options)
+      exec_command!(options, :setup, 'Setting up your tests.')
+    end
+
+    # TODO: Tests missing (but execute_setup! has a test and implementation is the same.)
+    def self.execute_tests!(options)
+      exec_command!(options, :test, 'Running your tests.')
+    end
+
+    # TODO: Tests missing (but execute_setup! has a test and implementation is the same.)
+    def self.execute_teardown!(options)
+      exec_command!(options, :teardown, 'Tearing down your test bench.')
+    end
+
+    def self.exec_command!(options, key, message)
+      rendered = render(options)
+      raise 'No tests found' if (key == :test) && !rendered.key?(:test)
+
+      KubernetesHarness.logger.debug "Checking for: #{key}"
+      return nil unless rendered.key? key
+
+      KubernetesHarness.nice_logger.info message
+      command = KubernetesHarness::ShellCommand.new(rendered[key])
+      command.execute!
+      KubernetesHarness.logger.error command.stderr unless command.stderr.empty?
+      puts command.stdout
+    end
+
+    def self.test_present?(options)
+      harness_file(options).key? :test
+    end
+
+    def self.convert_to_commands(options)
+      # TODO: Currently, we are assuming that the steps provided in the .k8sharness
+      # will always be invoked in a shell.
+      # First, we shouldn't assume that the user will want to use `sh` for these commands.
+      # Second, we should allow users to invoke code in the language of their preference to
+      # maximize codebase homogeneity.
+      rendered = harness_file(options)
+      rendered.each_key do |key|
+        if rendered[key].match?(/.(sh|bash|zsh)$/)
+          rendered[key] = "sh #{rendered[key]}"
+        else
+          rendered[key] = "sh -c '#{Shellwords.escape(rendered[key])}'" \
+            unless rendered[key].match?(/^(sh|bash|zsh) -c/)
+        end
+      end
+    end
+
+    def self.render(options = {})
+      fp = harness_file_path(options)
+      raise "k8s-harness file not found at: #{fp}" unless File.exist? fp
+      return convert_to_commands(options) if test_present?(options)
+
+      raise KeyError, <<~MESSAGE.strip
+        It appears that your test isn't defined in #{fp}. Ensure that \
+        a key called 'test' is in #{fp}. See .k8sharness.example for \
+        an example of what a valid .k8sharness looks like.
+      MESSAGE
+    end
+
+    def self.validate(options)
+      puts YAML.dump(render(options.to_h))
+    end
+
+    def self.default_harness_file_path
+      "#{Dir.pwd}/.k8sharness"
+    end
+
+    def self.harness_file_path(options)
+      if !options.nil? && options.key?(:alternate_harnessfile)
+        options[:alternate_harnessfile]
+      else
+        default_harness_file_path
+      end
+    end
+
+    def self.harness_file(options)
+      YAML.safe_load(File.read(harness_file_path(options)), symbolize_names: true)
+    end
+  end
+end

data/lib/k8s_harness/logging.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+# KubernetesHarness.
+module KubernetesHarness
+  @logger = Logger.new($stdout)
+  @logger.level = ENV['LOG_LEVEL'] || Logger::WARN
+  @nice_logger = Logger.new($stdout)
+  @nice_logger.formatter = proc do |_sev, datetime, _app, message|
+    if @logger.level == Logger::DEBUG
+      "--> [#{datetime.strftime('%F %T %z')}] #{message}\n"
+    else
+      "--> #{message}\n"
+    end
+  end
+  def self.logger
+    @logger
+  end
+
+  def self.nice_logger
+    @nice_logger
+  end
+
+  # Functions to manipulate log control.
+  module Logging
+    def self.enable_debug_logging
+      KubernetesHarness.logger.level = Logger::DEBUG
+    end
+  end
+end

data/lib/k8s_harness/paths.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module KubernetesHarness
+  # The canonical source of all toplevel paths
+  module Paths
+    def self.root_dir
+      File.expand_path '../..', __dir__
+    end
+
+    def self.include_dir
+      File.join root_dir, 'include'
+    end
+
+    def self.conf_dir
+      File.join root_dir, 'conf'
+    end
+  end
+end

data/lib/k8s_harness/shell_command.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'English'
+require 'open3'
+
+module KubernetesHarness
+  # Handles all interactions with shells
+  class ShellCommand
+    attr_accessor :command, :stdout, :stderr
+
+    # Ruby 2.7 deprecated keyword arguments in favor of passing in Hashes.
+    # TODO: Refactor to account for this.
+    def initialize(command, environment: {})
+      KubernetesHarness.logger.debug("Creating new command #{command} with env #{environment}")
+      @command = command
+      @environment = environment
+      @exitcode = nil
+      @stdout = nil
+      @stderr = nil
+    end
+
+    def execute!
+      @stdout, @stderr, @exitcode = read_output_in_chunks(@environment)
+
+      show_debug_command_output
+    end
+
+    def success?(exit_code: 0)
+      @exitcode == exit_code
+    end
+
+    private
+
+    def read_output_in_chunks(environment = {})
+      # Courtesy of: https://gist.github.com/chrisn/7450808
+      def all_eof(files)
+        files.find { |f| !f.eof }.nil?
+      end
+      block_size = 1024
+      final_stdout = ''
+      final_stderr = ''
+      final_process = nil
+      KubernetesHarness.logger.debug("Running #{@command} with env #{environment}")
+      Open3.popen3(environment.transform_keys(&:to_s), @command) do |stdin, stdout, stderr, thread|
+        stdin.close_write
+
+        begin
+          files = [stdout, stderr]
+          until all_eof(files)
+            ready = IO.select(files)
+            next unless ready
+
+            readable = ready[0]
+            readable.each do |f|
+              data = f.read_nonblock(block_size)
+              stdout_chunk = f == stdout ? data : ''
+              stderr_chunk = f == stderr ? data : ''
+              if f == stdout
+                final_stdout += stdout_chunk
+              else
+                final_stderr += stderr_chunk
+              end
+              KubernetesHarness.logger.debug("command: #{@command}, stdout_chunk: #{stdout_chunk}")
+              KubernetesHarness.logger.debug("command: #{@command}, stderr_chunk: #{stderr_chunk}")
+            rescue EOFError
+              KubernetesHarness.logger.debug("command: #{@command}, stream has EOF'ed")
+            end
+          end
+        rescue IOError => e
+          puts "IOError: #{e}"
+        end
+        final_process = thread.value.exitstatus
+      end
+
+      [final_stdout, final_stderr, final_process]
+    end
+
+    def show_debug_command_output
+      message = <<~MESSAGE.strip
+        Running #{@command} done, \
+        rc = #{@exitcode}, \
+        stdout = '#{@stdout}', \
+        stderr = '#{@stderr}'
+      MESSAGE
+      KubernetesHarness.logger.debug message
+    end
+  end
+end

data/lib/k8s_harness/subcommand.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'k8s_harness/clusters'
+require 'k8s_harness/clusters/cluster_info'
+require 'k8s_harness/clusters/metadata'
+require 'k8s_harness/harness_file'
+
+module KubernetesHarness
+  # All entrypoints for our subcommands live here.
+  module Subcommand
+    def self.run(options = {})
+      fail_if_validate_fails!(options)
+      disable_teardown = !options.nil? && options[:disable_teardown]
+      return true if !options.nil? && options[:show_usage]
+
+      print_warning_if_teardown_disabled(disable_teardown)
+      cluster_info = create!
+      provision!(cluster_info)
+      print_post_create_message(cluster_info)
+      setup!(options)
+      run_tests!(options)
+      teardown!(options)
+      destroy_cluster!(disable_teardown)
+    end
+
+    def self.validate(options = {})
+      return true if options.to_h[:show_usage]
+
+      KubernetesHarness::HarnessFile.validate(options)
+    end
+
+    def self.destroy(options = {})
+      return true if options.to_h[:show_usage]
+
+      KubernetesHarness.nice_logger.info('Destroying your cluster (if any found).')
+      KubernetesHarness::Clusters.destroy_existing!
+    end
+
+    def self.print_warning_if_teardown_disabled(teardown_flag)
+      return unless teardown_flag
+
+      KubernetesHarness.nice_logger.warn(
+        <<~MESSAGE.strip
+          Teardown is disabled. Your cluster will stay up until you run \
+          'k8s-harness destroy'.
+        MESSAGE
+      )
+    end
+
+    def self.create!
+      KubernetesHarness.nice_logger.info(
+        'Creating your cluster now. Provisioning will occur in a few minutes.'
+      )
+      KubernetesHarness::Clusters.create!
+    end
+
+    def self.provision!(cluster_info)
+      KubernetesHarness.nice_logger.info('Provisioning the cluster. This will take a few minutes.')
+      KubernetesHarness::Clusters.provision!(cluster_info)
+    end
+
+    def self.setup!(options)
+      KubernetesHarness::HarnessFile.execute_setup!(options)
+    end
+
+    def self.run_tests!(options)
+      KubernetesHarness.nice_logger.info('Running your tests.')
+      KubernetesHarness::HarnessFile.execute_tests!(options)
+    end
+
+    def self.teardown!(options)
+      KubernetesHarness::HarnessFile.execute_teardown!(options)
+    end
+
+    def self.destroy_cluster!(disable_teardown)
+      KubernetesHarness.nice_logger.info('Done. Tearing down the cluster.')
+      KubernetesHarness::Clusters.teardown! unless disable_teardown
+    end
+
+    def self.fail_if_validate_fails!(options)
+      _ = KubernetesHarness::HarnessFile.render(options)
+    end
+
+    def self.print_post_create_message(cluster_info)
+      # TODO: Make this not hardcoded.
+      cluster_info_yaml_path = File.join Clusters::Metadata.default_dir, 'cluster.yaml'
+      KubernetesHarness.nice_logger.info(
+        <<~MESSAGE.strip
+          Cluster has been created. Details are below and in YAML at #{cluster_info_yaml_path}:
+
+              * Master address: '#{cluster_info.master_ip_address}'
+              * Worker addresses: #{cluster_info.worker_ip_addresses}
+              * Docker registry address: '#{cluster_info.docker_registry_address}'
+              * Kubeconfig path: #{cluster_info.kubeconfig_path}
+              * SSH key path: #{cluster_info.ssh_key_path}
+        MESSAGE
+      )
+    end
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: k8s-harness
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Carlos Nunez
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-10-28 00:00:00.000000000 Z
+dependencies: []
+description: Please visit the README in the Github repo linked to this gem for more
+  info.
+email: dev@carlosnunez.me
+executables:
+- k8s-harness
+extensions: []
+extra_rdoc_files: []
+files:
+- "./conf/required_software.yaml"
+- "./include/Vagrantfile"
+- "./include/inventory"
+- "./include/site.yml"
+- "./lib/k8s_harness.rb"
+- "./lib/k8s_harness/cli.rb"
+- "./lib/k8s_harness/clusters.rb"
+- "./lib/k8s_harness/clusters/ansible.rb"
+- "./lib/k8s_harness/clusters/cluster_info.rb"
+- "./lib/k8s_harness/clusters/constants.rb"
+- "./lib/k8s_harness/clusters/metadata.rb"
+- "./lib/k8s_harness/clusters/required_software.rb"
+- "./lib/k8s_harness/clusters/vagrant.rb"
+- "./lib/k8s_harness/harness_file.rb"
+- "./lib/k8s_harness/logging.rb"
+- "./lib/k8s_harness/paths.rb"
+- "./lib/k8s_harness/shell_command.rb"
+- "./lib/k8s_harness/subcommand.rb"
+- bin/k8s-harness
+homepage: https://github.com/carlosonunez/k8s-harness
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Test your apps in disposable, prod-like Kubernetes clusters
+test_files: []