hub-clusters-creator 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e112445bd7b1d2f431a97502a2ca6fb52b0baaff096340aa3ab8fef52a0e6e37
+  data.tar.gz: e0d971cf87376aa8885c9b25a5c7b0645af4204487438f32c71d490e668ebd61
+SHA512:
+  metadata.gz: 6ee5df2fa3bb45b1852ff7c206cd5ddb0dbe154f429f09bb26bd5c53a53f5438109cb72163620114de79a2721d1df4de8a550e175644173d8ad58ec60a931a85
+  data.tar.gz: fe053f1117639caaecb6e537d23d51db9f7fd015a8386ebb07fbea101ffc989ef91b227b34b767ae8a0c424dd8dd8435a25bace004b0b833aea567afa005a8f9

data/.gitignore

@@ -0,0 +1,5 @@
+*.swp
+*.gem
+*.orig
+account.json
+dev.env

data/Makefile

@@ -0,0 +1,28 @@
+#
+#  vim:ts=2:sw=2:et
+#
+NAME=hub-bootstrap
+AUTHOR ?= appvia
+REGISTRY ?= quay.io
+VERSION ?= latest
+
+.PHONY: build test docker
+
+default: build
+
+build:
+	@echo "--> Building the GEM"
+	gem build hub-clusters-creator.gemspec
+
+docker:
+	@echo "--> Building the docker image: ${REGISTRY}/${AUTHOR}/${NAME}:${VERSION}"
+	@(cd docker && docker build -t ${REGISTRY}/${AUTHOR}/${NAME}:${VERSION} .)
+
+push:
+	@echo "--> Pushing the image to respository"
+	docker push ${REGISTRY}/${AUTHOR}/${NAME}:${VERSION}
+
+clean:
+	@echo "--> Performing a cleanup"
+	@docker rmi -f ${REGISTRY}/${AUTHOR}/${NAME}:${VERSION} 2>/dev/null
+	@rm -f *.gem

data/bin/aks.rb

@@ -0,0 +1,42 @@
+#!/bin/ruby
+# frozen_string_literal: true
+
+# Copyright (C) 2019  Rohith Jayawardene <gambol99@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '.', '../lib')
+
+require 'hub-clusters-creator'
+
+creator = HubClustersCreator.new(
+  client_id: ENV['AKS_CLIENT_ID'],
+  client_secret: ENV['AKS_CLIENT_SECRET'],
+  provider: 'aks',
+  region: 'uksouth',
+  subscription: ENV['AKS_SUBSCRIPTION'],
+  tenant: ENV['AKS_TENANT']
+)
+puts creator.provision(
+  description: 'just a test',
+  domain: 'akslearning.appvia.io',
+  github_client_id: ENV['GITHUB_CLIENT_ID'],
+  github_client_secret: ENV['GITHUB_CLIENT_SECRET'],
+  github_organization: ENV['GITHUB_ORG'],
+  grafana_hostname: 'grafana.akslearning.appvia.io',
+  size: 3,
+  ssh_key: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAB/RZqb2UuuXmJbwml/G1InXVwgNe1tgz+Xd6HTfPlS00GSHTynjyLNKWbDBZhzj2jsM/40TyCZLLKai8hV4Lc72ijMsEAbSJ8gou0Kq6P25GxY7uQcCeACPT6AbQCZjiyzPCFPy+QC56W64QOtS+SLRefY8g4uGAz01ZfhChK0J2Mev1oiPXRAlOmzGz9fPpx4J8YE0R/OkEoWQaWdKVOV6nq2nz3vbwvLfbdK0EK/KAivwv9mlgalwSr3bgAXwRS2nXq7vyITvpjgDYcRu85fWSE9yeMyw4S10ya5/ALi+p4HkSbDaz2YGwksSup1lBeRflSKKghWba4+dzf4iO87cwYCN8erVxATznuPoB0TWYYOXrUwc3yGadOV5GkzQLLDhIzJ8BmMb/iT5jHBdHs1bK1lTThwPllmgkRANFIWMC9jyA3BgJp2vtHcLyOXOsFFaXqpjXZ2tvjaBVjaaDUY+CE2rbJMBquUMbvtxwBtkhVVQ4COofUsXhKOcwjOyo1Hw1hcJ9ig6DiuzYhiPr/JZPUk2l8ovAKkGwFjEui7jltcWzGrmRzFp8+Xdq9wl/BEeglPv+eZuuwXgAWRge992kepda1f5OlBUl8JrohbxtVcv+zvmVaiZqcrKpyNbtN0ELMA8e4K6wCllQ0oBr17oTG7JwMVgfvTCgsI+U/ jest@starfury',
+  name: 'dev',
+  version: '1.14.3'
+)

data/bin/gke.rb

@@ -0,0 +1,48 @@
+#!/bin/ruby
+# frozen_string_literal: true
+
+# Copyright (C) 2019  Rohith Jayawardene <gambol99@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '.', '../lib')
+
+require 'hub-clusters-creator'
+require 'pp'
+
+account = File.read('account.json')
+region = 'europe-west2'
+project = 'gke-learning-242311'
+
+#creator = HubClustersCreator.new(
+#  account: account,
+#  provider: 'gke',
+#  project: project,
+#  region: region
+#)
+#puts creator.provision(
+#  description: 'just a test',
+#  domain: 'gkelearning.appvia.io',
+#  github_client_id: ENV['GITHUB_CLIENT_ID'],
+#  github_client_secret: ENV['GITHUB_CLIENT_SECRET'],
+#  github_organization: ENV['GITHUB_ORG'],
+#  grafana_hostname: 'grafana.gkelearning.appvia.io',
+#  grafana_ingress: true,
+#  logging: true,
+#  machine_type: 'n1-standard-1',
+#  name: 'test',
+#  version: '1.13.7-gke.8'
+#)
+
+puts HubClustersCreator.schema.to_json

data/docker/Dockerfile

@@ -0,0 +1,20 @@
+FROM fedora:30
+MAINTAINER Rohith Jayawardene <gambol99@gmail.com>
+
+ENV ARCH="amd64" \
+    HELM_VERSION="2.14.1" \
+    KUBECTL_VERSION="1.14.0"
+
+RUN dnf install -y bash curl jq
+
+RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl -o /usr/bin/kubectl && \
+    curl -sL https://get.helm.sh/helm-v${HELM_VERSION}-linux-${ARCH}.tar.gz | tar -C /tmp -xzf - && mv /tmp/linux-${ARCH}/helm /usr/bin/helm && \
+    chmod +x /usr/bin/kubectl /usr/bin/helm && \
+    rm -rf /tmp/linux-${ARCH}
+
+RUN kubectl version --client && \
+    helm version --client
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]

data/docker/entrypoint.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Copyright (C) 2019  Rohith Jayawardene <gambol99@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+log()      { (2>/dev/null echo -e "$@"); }
+info()     { log "[$(date)][info] $@";   }
+failed()   { log "[$(date)][fail] $@";   }
+error()    { log "[$(date)][error] $@";  }
+
+CONFIG_DIR="${CONFIG_DIR:-"/config"}"
+HELM_DIR="${CONFIG_DIR}/bundles"
+HELM_REPOS="${HELM_DIR}/repositories"
+HELM_BUNDLES="${HELM_DIR}/charts"
+KUBE_DIR="${CONFIG_DIR}/manifests"
+
+# deploy-manifests deploys all the files in the manifests directory
+deploy-manifests() {
+  info "deploying the kubernetes manifests from: ${KUBE_DIR}"
+  [[ -d "${KUBE_DIR}" ]] || return 0
+
+  ret=0
+  for filename in ${KUBE_DIR}/*; do
+    [[ -f "${filename}"          ]] || continue
+    [[ ${filename} =~ ^.*\.ya?ml ]] || continue
+
+    info "deploying the manifest: ${filename}"
+    if ! kubectl apply -f ${manifest}; then
+      error "failed to deploy the manifest: ${filename}"
+    fi
+  done
+  return $ret
+}
+
+# deploy-bundles is responsible for deploying charts into the cluster
+# loki,bundles/loki,overrides/loki.yaml
+deploy-bundles() {
+  info "installing helm tiller service"
+  helm init --wait --service-account=sysadmin >/dev/null || return 1
+
+  if [[ -f ${HELM_REPOS} ]]; then
+    info "installing any repository requirements"
+    while IFS=',' read name repository; do
+      info "adding the helm repository: ${repository}"
+      helm repo add ${name} ${repository} || return 1
+    done < <(cat ${HELM_REPOS})
+
+    info "updating the repositories cache"
+    helm repo update || return 1
+  fi
+
+  if [[ -f ${HELM_BUNDLES} ]]; then
+    info "installing the helm charts"
+    while IFS=',' read chart namespace options; do
+      namespace=${namespace:-"default"}
+      name=${chart%%/*}
+      if helm ls -q | grep ^${name}; then
+        info "installing chart: ${chart}, namespace: ${namespace}, options: ${options}"
+        helm upgrade --wait ${chart} --namespace ${namespace} ${options} || return 1
+      else
+        info "upgrading chart: ${chart}, namespace: ${namespace}, options: ${options}"
+        helm install --wait ${chart} --namespace ${namespace} ${options} || return 1
+      fi
+    done < <(cat ${HELM_BUNDLES})
+  fi
+}
+
+deploy-manifests || {
+  error "failed to deploy the kubernetes manifests";
+  exit 1;
+}
+deploy-bundles || {
+  error "failed to deploy the software manifests";
+  exit 1;
+}
+

data/hub-clusters-creator.gemspec

@@ -0,0 +1,34 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/LineLength
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '.', 'lib/hub-clusters-creator')
+
+require 'version'
+
+Gem::Specification.new do |s|
+  s.name        = 'hub-clusters-creator'
+  s.version     = HubClustersCreator::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.date        = '2019-08-02'
+  s.authors     = ['Rohith Jayawardene']
+  s.email       = 'gambol99@gmail.com'
+  s.homepage    = 'http://rubygems.org/gems/hub-clusters-creator'
+  s.summary     = 'An agent used to provision GKE clusters for the Appvia Hub'
+  s.description = 'An agent used to provision GKE clusters for the Appvia Hub '
+  s.license     = 'GPL-2.0'
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+
+  s.add_dependency('azure_mgmt_container_service', '~> 0.18.5')
+  s.add_dependency('azure_mgmt_dns', '~> 0.18.5')
+  s.add_dependency('azure_mgmt_resources', '~> 0.18.5')
+  s.add_dependency('deep_merge', '~> 1.2.1')
+  s.add_dependency('google-api-client', '~> 0.30')
+  s.add_dependency('googleauth', '~> 0.7')
+  s.add_dependency('json_schema', '~> 0.20.4')
+  s.add_dependency('k8s-client', '~> 0.10')
+  s.add_dependency('stringio', '~> 0.0.2')
+end
+# rubocop:enable Metrics/LineLength

data/lib/hub-clusters-creator.rb

@@ -0,0 +1,49 @@
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+#
+# rubocop:enable Naming/FileName
+# Copyright (C) 2019  Rohith Jayawardene <gambol99@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative 'hub-clusters-creator/agent'
+require_relative 'hub-clusters-creator/version'
+
+# Clusters providers the wrapper to the providers
+module HubClustersCreator
+  def self.version
+    HubClustersCreator::VERSION
+  end
+
+  def self.new(name)
+    HubClustersCreator::Agent.new(name)
+  end
+
+  def self.defaults(name)
+    HubClustersCreator::Agent.defaults(name)
+  end
+
+  def self.schema
+    o = []
+    HubClustersCreator::Agent.providers.each do |x|
+      o.push(
+        id: x,
+        init_options: HubClustersCreator::Agent.provider_schema(x),
+        provision_options: HubClustersCreator::Agent.cluster_schema(x)
+      )
+    end
+    o
+  end
+end

data/lib/hub-clusters-creator/agent.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2019  Rohith Jayawardene <gambol99@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+require 'deep_merge'
+require 'json'
+require 'json_schema'
+
+require 'hub-clusters-creator/errors'
+require 'hub-clusters-creator/logging'
+require 'hub-clusters-creator/providers/aks/azure.rb'
+require 'hub-clusters-creator/providers/gke/gke.rb'
+
+# rubocop:disable Metrics/MethodLength,Metrics/LineLength
+module HubClustersCreator
+  # Agent is the main agent class
+  class Agent
+    include Errors
+    include Logging
+
+    # rubocop:disable Metrics/AbcSize
+    def initialize(provider)
+      @provider_name = provider[:provider]
+
+      # @step: validate the provider configuration
+      JsonSchema.parse!(HubClustersCreator::Agent.provider_schema(@provider_name)).validate(provider)
+
+      # @step: create and return a provider instance
+      case @provider_name
+      when 'gke'
+        @provider = HubClustersCreator::Providers::GKE.new(
+          account: provider[:account],
+          project: provider[:project],
+          region: provider[:region]
+        )
+      when 'aks'
+        @provider = HubClustersCreator::Providers::AKS.new(
+          client_id: provider[:client_id],
+          client_secret: provider[:client_secret],
+          region: provider[:region],
+          subscription: provider[:subscription],
+          tenant: provider[:tenant]
+        )
+      else
+        raise ArgumentError, "cloud provider: #{@provider_name} not supported"
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    # providers provides a list of providers
+    def self.providers
+      %w[aks gke]
+    end
+
+    # defaults builds the default from the schema
+    def self.defaults(name)
+      values = {}
+      cluster_schema(name)['properties'].reject { |x, _v| x == 'authorized_master_cidrs' }.each do |k, v|
+        values[k.to_sym] = v['default']
+      end
+      # @TODO find a better way of doing this
+      unless values[:authorized_master_cidrs]
+        values[:authorized_master_cidrs] = [{ name: 'any', cidr: '0.0.0.0/0' }]
+      end
+      values
+    end
+
+    # provider_schema returns the provider schema
+    def self.provider_schema(name)
+      schemas(name).first
+    end
+
+    # cluster_schema returns a cluster schema for a specific provider
+    def self.cluster_schema(name)
+      schemas(name).last
+    end
+
+    # schemas returns the json schemais defining the providers configuration schema and the
+    # cluster schema for tha cloud provider
+    def self.schemas(name)
+      file = "#{__dir__}/providers/#{name}/schema.yaml"
+      raise ArgumentError, "provider: '#{name}' is not supported" unless File.exist?(file)
+
+      # loads and parses both the provider and cluster schema
+      provider_schemas = YAML.load_stream(File.read(file))
+      # load and parse the base schema which is used across all providers
+      provider_base = YAML.safe_load(File.read("#{__dir__}/providers/schema.yaml"))
+      # we deep merge the provider with the defaults
+      provider_schemas.last.deep_merge(provider_base)
+
+      provider_schemas
+    end
+
+    # destroy is responsible is tearing down the cluster
+    def destroy(name, options)
+      @provider.destroy(name, options)
+    end
+
+    # provision is responsible for provisioning the cluster
+    # rubocop:disable Lint/RescueException, Metrics/AbcSize
+    def provision(options)
+      name = options[:name]
+      config = HubClustersCreator.defaults(@provider_name).merge(options)
+
+      # @step: provision the cluster if not already there
+      begin
+        schema = HubClustersCreator::Agent.provider_schema(@provider_name)
+        # verify the options
+        JsonSchema.parse!(schema).validate(config)
+        # provision the cluster
+        @provider.create(name, config)
+      rescue InfrastructureError => e
+        error "failed to provision the infrastructure: #{name}, error: #{e}"
+        raise e
+      rescue ConfigurationError => e
+        error "invalid configuration for cluster: #{name}, error: #{e}"
+        raise e
+      rescue InitializerError => e
+        error "failed to initialize cluster: #{name}, error: #{e}"
+        raise e
+      rescue Exception => e
+        error "failed to provision the cluster: #{name}, error: #{e}"
+        raise e
+      end
+    end
+    # rubocop:enable Lint/RescueException, Metrics/AbcSize
+  end
+end
+# rubocop:enable Metrics/MethodLength,Metrics/LineLength