kubernetes_helper 0.3.2 → 1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ef481cc4f5afcb7c2bca9605b7ecc72649ac2e2522400a58e2dc964802f22bd
-  data.tar.gz: 2b2aa0863511f4aa192a1332d87f12c80ce1c30a435248ade3f5437192e83384
+  metadata.gz: a680149c9a36ebc58fc0f7560a1257bc48bedfbf8a12ed93cfed112b12799407
+  data.tar.gz: b0b0dafefba5594194bed8bf990ace04f04297939f2861575d79b84c716aacca
 SHA512:
-  metadata.gz: 7c0250a176f8acc04262bceab8cf550306006b540a4df43590b94f5bbb81e69f768fa023bcf0e88ffbd84dbd1d15964f374cf1950a6ddfc23211b1413487c380
-  data.tar.gz: bddf79fea09a01c9329592482866f552744ba0113952500cb3d032bf619ae2855613f8b9acc5567a87587c38f780c92e983c20338c68e5ec0289c3b455f1cbfb
+  metadata.gz: 2b3829818e7e37a552fe645686b954a27ac7779ab6097aaba16fe3b6b91538c471c4501b9c9d1346230737553772058ae0c4cc5183b4fe61ca1a289cf5422d24
+  data.tar.gz: c98898c74f76a1d796387953450360a5d167b070a23461a58f3ae20b93a840906920fc191b196354132ff124925d6c877bd1baf28a60a5d6f766fd7c0af89e60

data/README.md

@@ -1,37 +1,61 @@
 # KubernetesHelper
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/kubernetes_helper`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+This gem is a helper to manage easily Kubernetes settings for GCloud (easy customization for other cloud services) where configuring and deploying a new application can be done in a couple of minutes.
+Configuration and customization can be done for multiple environments and at any level which permits to deploy simple and complex applications.
 
 ## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'kubernetes_helper'
+```bash
+cd my_app/
+gem install kubernetes_helper -v '~> 1.0'
+kubernetes_helper generate_templates
 ```
-
-And then execute:
-
-    $ bundle install
-
-Or install it yourself as:
-
-    $ gem install kubernetes_helper
-
-## Usage
+Note: Requires ruby 1.7+      
+
+## Deployment
+Once you generated the basic templates, it comes with the corresponding [readme.md](/lib/templates/README.md) which includes all the steps to deploy your application.
+
+## API
+- Run any kubernetes document    
+  `DEPLOY_ENV=<env name> kubernetes_helper run_deployment "<document name>" "<bash command>"`    
+  Evaluates the kubernetes document with the following details:
+  - Supports for `- documents` to include multiple documents in a file and share yml variables between them (Sample: `lib/templates/deployment.yml#1`)
+  - Replaces all setting values based on `DEPLOY_ENV`
+  - Supports for secrets auto importer using `import_secrets: ['secrets.yml', '<%=secrets.name%>']` (Sample: `lib/templates/deployment.yml#29`)
+  - Supports for template including by `include_template 'template_name.yml.erb'`    
+  Sample: `DEPLOY_ENV=beta kubernetes_helper run_deployment "deployment.yml" "kubectl create"`
+   
+- Run kubernetes commands    
+  `DEPLOY_ENV=<env name> rake kubernetes_helper:run_command "<bash or k8s commands>"`           
+  Replaces all setting variables inside command based on `DEPLOY_ENV` and performs it as a normal bash command.             
+  Sample: `DEPLOY_ENV=beta rake kubernetes_helper:run_command "gcloud compute addresses create \#{ingress.ip_name} --global"'`    
+  
+- Run kubernetes bash scripts     
+  `DEPLOY_ENV=<env name> kubernetes_helper run_script "<script name>"`    
+  Performs the script name located inside `.kubernetes` folder or kubernetes_helper template as the second option.
+  All setting variables inside the script will be replaced based on `DEPLOY_ENV`.      
+  Sample: `DEPLOY_ENV=beta kubernetes_helper run_script "cd.sh"`
+
+- Generate templates    
+  `DEPLOY_ENV=<env name> kubernetes_helper generate_templates "<mode_or_template_name>"`     
+  Copy files based on mode (`basic|advanced`) or a specific file from templates.     
+  Sample: `DEPLOY_ENV=beta kubernetes_helper generate_templates "basic"`    
+  Sample: `DEPLOY_ENV=beta kubernetes_helper generate_templates "ingress.yml"`    
+
+When performing a script it looks first for file inside .kubernetes folder, if not exist, 
+it looks for the file inside kubernetes_helper template folder.    
+
+## Templating
+When performing a command or script, the setting variables are replaced based on `DEPLOY_ENV`. 
+All these setting variable values are configured in `.kubernetes/settings.rb` which defines the values based on `DEPLOY_ENV`.     
+These setting variables use [erb](https://github.com/ruby/erb) template gem to define variable replacement and conditional blocks, and so on.
+Note: Setting variable values are referenced as an object format instead of a hash format for simplicity.
+  
+
+## Sample
+https://owen2345.github.io/kubernetes_helper/
 
 ## TODO
-- Documentation
-    - Include conditional blocks
-    - Include hardcoded env values
-    - Tasks
-    - Add docs to include partials using `include_template 'sample.yml.erb'`
-- Rake verify files
 - Add one_step_configuration.sh
-- Fix if/end_if for blocks
-- Ability to copy specific template
 
 ## Contributing
 

data/exe/kubernetes_helper

@@ -9,8 +9,8 @@ case ARGV[0]
 when 'run_command'
   KubernetesHelper::Core.new(ENV['DEPLOY_ENV']).run_command(ARGV[1])
 # Run the deployment script.
-#   Sample: DEPLOY_ENV=beta kubernetes_helper run_deployment "cd_gcloud.sh"
-when 'run_deployment'
+#   Sample: DEPLOY_ENV=beta kubernetes_helper run_deployment "cd.sh"
+when 'run_deployment', 'run_script'
   script_path = KubernetesHelper.settings_path(ARGV[1], use_template: true)
   KubernetesHelper::Core.new(ENV['DEPLOY_ENV']).run_script(script_path)
 # Parses kubernetes yml files (supporting multiple documents, Config variables replacement, include secrets).
@@ -22,6 +22,7 @@ when 'run_yml'
     .parse_yml_file(KubernetesHelper.settings_path(ARGV[1], use_template: true), output_path)
   KubernetesHelper.run_cmd("#{ARGV[2]} -f #{output_path}")
 # Generate template files
+#   Sample: DEPLOY_ENV=beta kubernetes_helper generate_templates "basic"
 when 'generate_templates' #  Sample: kubernetes_helper generate basic
   mode = ARGV[1] || 'basic'
   KubernetesHelper.copy_templates(mode)

data/lib/kubernetes_helper.rb

@@ -39,11 +39,14 @@ module KubernetesHelper
     file_name ? File.join(path, file_name) : path
   end
 
-  # @param mode (basic, advanced)
-  def self.copy_templates(mode)
+  # @param mode_or_file (basic, advanced, String) mode name or any specific template name
+  def self.copy_templates(mode_or_file)
     FileUtils.mkdir(settings_path) unless Dir.exist?(settings_path)
+    template_path = templates_path(mode_or_file)
+    return FileUtils.cp(template_path, settings_path(mode_or_file)) if File.exist?(template_path)
+
     files = %w[README.md secrets.yml settings.rb]
-    files += %w[deployment.yml cd.sh ingress.yml service.yml] if mode == 'advanced'
+    files += %w[deployment.yml cd.sh ingress.yml service.yml] if mode_or_file == 'advanced'
     files.each do |name|
       path = settings_path(name)
       FileUtils.cp(templates_path(name), path) unless File.exist?(path)

data/lib/kubernetes_helper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KubernetesHelper
-  VERSION = '0.3.2'
+  VERSION = '1.2'
 end

data/lib/templates/README.md

@@ -2,54 +2,56 @@
 
 ## Configure a new application environment
 - Create the project on Gcloud
-- Set the project where to work on
+- Set the project where to work on     
     `gcloud config set project my-project`
     
-- Create the cluster (Only if not exist)
+- Create the cluster (Only if not exist)    
     `gcloud container clusters create my-cluster`
     `# gcloud container clusters list --region europe-west4-a # to list clusters`
+    Note: `--preemptible` recommended for non production environments   
   
-- Use the cluster/project as default
+- Use the cluster/project as default    
     `gcloud container clusters get-credentials my-cluster --zone europe-west4-a`
   
-- Install helper for the next commands
+- Install helper for the next commands    
   `gem install kubernetes_helper`
   
-- Verify or update k8s settings in .kubernetes/settings.rb
+- Verify or update k8s settings in .kubernetes/settings.rb        
+  Note: Please do not include sensitive values in this file, ENV values are recommended instead.
 
-- Register shared cloudsql proxy configuration (only if not exists)
+- Register shared cloudsql proxy configuration (only if not exists)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_command "kubectl create secret generic <%=deployment.cloud_secret_name%> --from-file=credentials.json=<path-to-downloaded/credentials.json>"
     ```
     
-- Register manually env vars (values must be encrypted using base64) 
+- Register manually env vars (values must be encrypted using base64)     
     Open and register secret values in `.kubernetes/secrets.yml`     
-    Note: Enter base64 encoded values
+    Note: Enter base64 encoded values      
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'secrets.yml' 'kubectl create'
     # kubectl get secrets # to list all secrets registered
     ```
 
-- Create deployment to run application
+- Create deployment to run application    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'deployment.yml' 'kubectl create'
     # kubectl get deployment # to list deployments
     ```
 
-- Create service to connect pods and ingress
+- Create service to connect pods and ingress    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'service.yml' 'kubectl create'
     # kubectl get services # to list all registered services
     ```
 
-- Create the public ip address (only if static ip is required)
+- Create the public ip address (only if static ip is required)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_command "gcloud compute addresses create <%=ingress.ip_name%> --global"
     # gcloud compute addresses list # to list static ips generated
     # Copy new external ip generated by the previous command and point your domain to it
     ```
   
-- Register ingress to receive external http calls (includes ssl certificates if defined)
+- Register ingress to receive external http calls (includes ssl certificates if defined)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'ingress.yml' 'kubectl create'
     # kubectl get ingress # to list all registered ingresses
@@ -60,18 +62,18 @@
     ```
 
 ## Apply any k8s setting changes
-- Secrets
+- Secrets    
   Open kubernetes secrets and add/edit/remove values and then save it    
   `kubectl edit secret ...`
   Once secrets were updated, then restart all related pods, see: https://medium.com/devops-dudes/how-to-propagate-a-change-in-kubernetes-secrets-by-restarting-dependent-pods-b71231827656
   
-- Other settings
+- Other settings    
   ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'deployment.yml' 'kubectl apply'
   ```
 
 ## Configure continuous deployment for github actions
-* Go to github repository settings
+* Go to github repository settings    
 * Register a new secret variable with content downloaded from https://console.cloud.google.com/iam-admin/serviceaccounts 
     (Make sure to attach a "Editor", "Storage Admin" and "Kubernetes engine cluster admin" role to the service account)
   ```bash
@@ -79,14 +81,14 @@
     production: PROD_CLOUD_TOKEN=<secret content here>
   ```
   
-* Add action to run deployment:
+* Add action to run deployment:    
   ```bash
     env:
       KB_AUTH_TOKEN: secrets.BETA_CLOUD_TOKEN
     run: DEPLOY_ENV=beta kubernetes_helper run_deployment 'cd.sh'
   ``` 
   
-* Sample:
+* Sample:    
   ```yml
   - run: sudo gem install kubernetes_helper
   - name: Staging deployment

data/lib/templates/cd.sh

@@ -5,33 +5,37 @@ set -e
 SCRIPT_DIR=`dirname "$(realpath -s "$0")"` # app_dir/.kubernetes/
 cd "$SCRIPT_DIR/../" # project directory
 
-DEPLOYMENTS="<%=[deployment.name, deployment.job_name].join(',')%>"
+DEPLOYMENTS="<%=[deployment.job_name, deployment.name].join(',')%>"
 IMAGE_NAME="<%=continuous_deployment.image_name%>"
 CLUSTER_NAME="<%=continuous_deployment.cluster_name%>"
 PROJECT_NAME="<%=continuous_deployment.project_name%>"
 CLUSTER_REGION="<%=continuous_deployment.cluster_region%>"
 DOCKER_BUILD_CMD="<%=continuous_deployment.docker_build_cmd || 'build -f Dockerfile'%>"
 
-CI_COMMIT_SHA=$(git rev-parse --verify HEAD)
+CI_COMMIT_SHA=$(git rev-parse --verify HEAD || :)
+CI_COMMIT_SHA=${CI_COMMIT_SHA:-$(date +%s) }
 DEPLOY_NAME="${IMAGE_NAME}:${CI_COMMIT_SHA}"
 LATEST_NAME="${IMAGE_NAME}:latest"
-AUTH_PATH="$SCRIPT_DIR/k8s-auth-token.json"
 
-rm -f -- $AUTH_PATH
-echo $KB_AUTH_TOKEN >> $AUTH_PATH
-
-## ***** GOOGLE CONNECTOR
-# Download and install Google Cloud SDK
-if [ -z "$(which gcloud)" ]; then
-  export CLOUDSDK_CORE_DISABLE_PROMPTS=1; curl https://sdk.cloud.google.com | bash && source /home/runner/google-cloud-sdk/path.bash.inc &&  gcloud --quiet components update kubectl
+if [ ! -z "$KB_AUTH_TOKEN" ]
+then
+  AUTH_PATH="$SCRIPT_DIR/k8s-auth-token.json"
+  rm -f -- $AUTH_PATH
+  echo $KB_AUTH_TOKEN >> $AUTH_PATH
+
+  ## ***** GOOGLE CONNECTOR
+  # Download and install Google Cloud SDK
+  if [ -z "$(which gcloud)" ]; then
+    export CLOUDSDK_CORE_DISABLE_PROMPTS=1; curl https://sdk.cloud.google.com | bash && source /home/runner/google-cloud-sdk/path.bash.inc &&  gcloud --quiet components update kubectl
+  fi
+
+  # Connect to cluster
+  gcloud auth activate-service-account --key-file $AUTH_PATH --project $PROJECT_NAME
+  gcloud docker --authorize-only --project $PROJECT_NAME
+  gcloud container clusters get-credentials $CLUSTER_NAME --region $CLUSTER_REGION
+  ## ***** END GOOGLE CONNECTOR
 fi
 
-# Connect to cluster
-gcloud auth activate-service-account --key-file $AUTH_PATH --project $PROJECT_NAME
-gcloud docker --authorize-only --project $PROJECT_NAME
-gcloud container clusters get-credentials $CLUSTER_NAME --region $CLUSTER_REGION
-## ***** END GOOGLE CONNECTOR
-
 
 ## Build and push containers
 docker $DOCKER_BUILD_CMD -t $DEPLOY_NAME .

data/lib/templates/certificate.yml

@@ -0,0 +1,7 @@
+apiVersion: networking.gke.io/v1beta1
+kind: ManagedCertificate
+metadata:
+  name: '<%=ingress.certificate_name%>'
+spec:
+  domains: # does not support for willcard domains
+    - '<%=ingress.domain_name%>'

data/lib/templates/deployment.yml

@@ -47,29 +47,41 @@ documents:
                   name: applog
                   mountPath: /app/log
 
+                <% if deployment.cloud_secret_name %>
+                - &cloud_credentials_volume
+                  name: gcloud-creds
+                  mountPath: /secrets/gcloud
+                  readOnly: true
+                <% end %>
+
+            <% if deployment.cloud_secret_name %>
             - &cloudsql_container
               image: gcr.io/cloudsql-docker/gce-proxy:1.09 # Cloud sql proxy
               name: cloudsql-proxy
               command: ["/cloud_sql_proxy", "--dir=/cloudsql",
                         "-instances=<%=deployment.cloud_sql_instance%>",
-                        "-credential_file=/secrets/cloudsql/credentials.json"]
+                        "-credential_file=/secrets/gcloud/credentials.json"]
+              env: []
               volumeMounts:
-                - name: cloudsql-creds
-                  mountPath: /secrets/cloudsql
-                  readOnly: true
+                - *cloud_credentials_volume
+            <% end %>
 
             - &logs_container # print to stdout all log files
               name: print-logs
               image: busybox
-              command: [/bin/sh, -c, 'until find log/*.log; do sleep 1; done; tail -n+1 -f log/*.log']
+              command: [/bin/sh, -c, 'until find /app/log/*.log; do sleep 1; done; tail -n+1 -f /app/log/*.log']
+              env: []
               volumeMounts:
                 - *log_volume
 
           terminationGracePeriodSeconds: 120
           volumes:
-            - name: cloudsql-creds
+            <% if deployment.cloud_secret_name %>
+            - name: gcloud-creds
               secret:
                 secretName: '<%=deployment.cloud_secret_name%>'
+            <% end %>
+
             - name: applog
               emptyDir: {}
 
@@ -98,7 +110,7 @@ documents:
               <% end %>
 
             <% if deployment.job_sidekiq_alive_gem %>
-              <%= include_template "_sidekiq_alive_gem.yml.erb" %>
+              <%= include_template "_sidekiq_alive_gem.yml" %>
             <% else %>
               ports: [ ]
               <% if (deployment.job_services || []).any? %>
@@ -114,10 +126,13 @@ documents:
               <% end %>
             <% end %>
 
+            <% if deployment.cloud_secret_name %>
             - *cloudsql_container
+            <% end %>
+
             - <<: *logs_container
   <% end %>
 
   <% if deployment.replicas_range %>
-  <%= include_template "_replicas.yml.erb" %>
+  <%= include_template "_replicas.yml" %>
   <% end %>

data/lib/templates/ingress.yml

@@ -1,11 +1,5 @@
 <% if ingress.domain_name %>
-apiVersion: networking.gke.io/v1beta1
-kind: ManagedCertificate
-metadata:
-  name: '<%=ingress.certificate_name%>'
-spec:
-  domains: # does not support for willcard domains
-    - '<%=ingress.domain_name%>'
+<%= include_template "certificate.yml" %>
 <% end %>
 
 ---

data/lib/templates/settings.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 is_production = ENV['DEPLOY_ENV'] == 'production'
-app_name = is_production ? 'my_app' : 'my_beta_app'
+app_name = is_production ? 'my-app' : 'my-beta-app' # underscore not accepted
 settings = {
   deployment: {
     name: app_name,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes_helper
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: '1.2'
 platform: ruby
 authors:
 - owen2345
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-04 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erb
@@ -40,9 +40,10 @@ files:
 - lib/kubernetes_helper/railtie.rb
 - lib/kubernetes_helper/version.rb
 - lib/templates/README.md
-- lib/templates/_replicas.yml.erb
-- lib/templates/_sidekiq_alive_gem.yml.erb
+- lib/templates/_replicas.yml
+- lib/templates/_sidekiq_alive_gem.yml
 - lib/templates/cd.sh
+- lib/templates/certificate.yml
 - lib/templates/deployment.yml
 - lib/templates/ingress.yml
 - lib/templates/secrets.yml