kubernetes_helper 0.3.1 → 1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d90e5910eaf66ffcc3df7cde769ca54136f106993afe0452f003a0997ab8b802
-  data.tar.gz: edd494736fad99801f3ce1b35e7c9c57c760a3854a6e5e213d9b502ca7fee03a
+  metadata.gz: 3bed0753481fee12fa822da715a6836b55fb561deaca02e8763e59d05b3edc6b
+  data.tar.gz: 28d56d40d55c667e479ee26126fc7014415afec77ee884eb58daac2c4e60809e
 SHA512:
-  metadata.gz: 31415960ef3adbd3168fe0c76ba6343b85f0dbf0ef8718056b7468cf17f0db5909c8ed9cabb216d2e429171f63cd911fbb57a367de970e2656e7f820639cb338
-  data.tar.gz: 5d94999264d596f7a3c0c483262c7b5b186b64674ad552e66607193c36c9ca72817a0cd31db1cc10d50f430d1ed66e785f7ca6876482c00989caa878c7de0dc0
+  metadata.gz: d4d8b5606a4d3d9649de1add4341dcea076e2892e2d908d24f1cce5c14fb206b9bd779da7546544634b0341806163f3949f98c24bdd9a0a16dbf4149fa597117
+  data.tar.gz: 331f4263fb0ad4fd29bda318f079e36e0954c58bd8a50dc4507e49d0e88043afcfe03a8bad22390d5776042bd07837d4232c5b7879a06074870369274866537d

data/README.md

@@ -1,37 +1,61 @@
 # KubernetesHelper
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/kubernetes_helper`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+This gem is a helper to manage easily Kubernetes settings for GCloud (easy customization for other cloud services) where configuring and deploying a new application can be done in a couple of minutes.
+Configuration and customization can be done for multiple environments and at any level which permits to deploy simple and complex applications.
 
 ## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'kubernetes_helper'
+```bash
+cd my_app/
+gem install kubernetes_helper -v '~> 1.0'
+kubernetes_helper generate_templates
 ```
-
-And then execute:
-
-    $ bundle install
-
-Or install it yourself as:
-
-    $ gem install kubernetes_helper
-
-## Usage
+Note: Requires ruby 1.7+      
+
+## Deployment
+Once you generated the basic templates, it comes with the corresponding [readme.md](/lib/templates/README.md) which includes all the steps to deploy your application.
+
+## API
+- Run any kubernetes document    
+  `DEPLOY_ENV=<env name> kubernetes_helper run_deployment "<document name>" "<bash command>"`    
+  Evaluates the kubernetes document with the following details:
+  - Supports for `- documents` to include multiple documents in a file and share yml variables between them (Sample: `lib/templates/deployment.yml#1`)
+  - Replaces all setting values based on `DEPLOY_ENV`
+  - Supports for secrets auto importer using `import_secrets: ['secrets.yml', '<%=secrets.name%>']` (Sample: `lib/templates/deployment.yml#29`)
+  - Supports for template including by `include_template 'template_name.yml.erb'`    
+  Sample: `DEPLOY_ENV=beta kubernetes_helper run_deployment "deployment.yml" "kubectl create"`
+   
+- Run kubernetes commands    
+  `DEPLOY_ENV=<env name> rake kubernetes_helper:run_command "<bash or k8s commands>"`           
+  Replaces all setting variables inside command based on `DEPLOY_ENV` and performs it as a normal bash command.             
+  Sample: `DEPLOY_ENV=beta rake kubernetes_helper:run_command "gcloud compute addresses create \#{ingress.ip_name} --global"'`    
+  
+- Run kubernetes bash scripts     
+  `DEPLOY_ENV=<env name> kubernetes_helper run_script "<script name>"`    
+  Performs the script name located inside `.kubernetes` folder or kubernetes_helper template as the second option.
+  All setting variables inside the script will be replaced based on `DEPLOY_ENV`.      
+  Sample: `DEPLOY_ENV=beta kubernetes_helper run_script "cd.sh"`
+
+- Generate templates    
+  `DEPLOY_ENV=<env name> kubernetes_helper generate_templates "<mode_or_template_name>"`     
+  Copy files based on mode (`basic|advanced`) or a specific file from templates.     
+  Sample: `DEPLOY_ENV=beta kubernetes_helper generate_templates "basic"`    
+  Sample: `DEPLOY_ENV=beta kubernetes_helper generate_templates "ingress.yml"`    
+
+When performing a script it looks first for file inside .kubernetes folder, if not exist, 
+it looks for the file inside kubernetes_helper template folder.    
+
+## Templating
+When performing a command or script, the setting variables are replaced based on `DEPLOY_ENV`. 
+All these setting variable values are configured in `.kubernetes/settings.rb` which defines the values based on `DEPLOY_ENV`.     
+These setting variables use [erb](https://github.com/ruby/erb) template gem to define variable replacement and conditional blocks, and so on.
+Note: Setting variable values are referenced as an object format instead of a hash format for simplicity.
+  
+
+## Sample
+https://owen2345.github.io/kubernetes_helper/
 
 ## TODO
-- Documentation
-    - Include conditional blocks
-    - Include hardcoded env values
-    - Tasks
-    - Add docs to include partials using `include_template 'sample.yml.erb'`
-- Rake verify files
 - Add one_step_configuration.sh
-- Fix if/end_if for blocks
-- Ability to copy specific template
 
 ## Contributing
 

data/exe/kubernetes_helper

@@ -9,8 +9,8 @@ case ARGV[0]
 when 'run_command'
   KubernetesHelper::Core.new(ENV['DEPLOY_ENV']).run_command(ARGV[1])
 # Run the deployment script.
-#   Sample: DEPLOY_ENV=beta kubernetes_helper run_deployment "cd_gcloud.sh"
-when 'run_deployment'
+#   Sample: DEPLOY_ENV=beta kubernetes_helper run_deployment "cd.sh"
+when 'run_deployment', 'run_script'
   script_path = KubernetesHelper.settings_path(ARGV[1], use_template: true)
   KubernetesHelper::Core.new(ENV['DEPLOY_ENV']).run_script(script_path)
 # Parses kubernetes yml files (supporting multiple documents, Config variables replacement, include secrets).
@@ -22,6 +22,7 @@ when 'run_yml'
     .parse_yml_file(KubernetesHelper.settings_path(ARGV[1], use_template: true), output_path)
   KubernetesHelper.run_cmd("#{ARGV[2]} -f #{output_path}")
 # Generate template files
+#   Sample: DEPLOY_ENV=beta kubernetes_helper generate_templates "basic"
 when 'generate_templates' #  Sample: kubernetes_helper generate basic
   mode = ARGV[1] || 'basic'
   KubernetesHelper.copy_templates(mode)

data/lib/kubernetes_helper.rb

@@ -39,11 +39,14 @@ module KubernetesHelper
     file_name ? File.join(path, file_name) : path
   end
 
-  # @param mode (basic, advanced)
-  def self.copy_templates(mode)
+  # @param mode_or_file (basic, advanced, String) mode name or any specific template name
+  def self.copy_templates(mode_or_file)
     FileUtils.mkdir(settings_path) unless Dir.exist?(settings_path)
+    template_path = templates_path(mode_or_file)
+    return FileUtils.cp(template_path, settings_path(mode_or_file)) if File.exist?(template_path)
+
     files = %w[README.md secrets.yml settings.rb]
-    files += %w[deployment.yml cd.sh ingress.yml service.yml] if mode == 'advanced'
+    files += %w[deployment.yml cd.sh ingress.yml service.yml] if mode_or_file == 'advanced'
     files.each do |name|
       path = settings_path(name)
       FileUtils.cp(templates_path(name), path) unless File.exist?(path)

data/lib/kubernetes_helper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KubernetesHelper
-  VERSION = '0.3.1'
+  VERSION = '1.1'
 end

data/lib/templates/README.md

@@ -2,54 +2,56 @@
 
 ## Configure a new application environment
 - Create the project on Gcloud
-- Set the project where to work on
+- Set the project where to work on     
     `gcloud config set project my-project`
     
-- Create the cluster (Only if not exist)
+- Create the cluster (Only if not exist)    
     `gcloud container clusters create my-cluster`
     `# gcloud container clusters list --region europe-west4-a # to list clusters`
+    Note: `--preemptible` recommended for non production environments   
   
-- Use the cluster/project as default
+- Use the cluster/project as default    
     `gcloud container clusters get-credentials my-cluster --zone europe-west4-a`
   
-- Install helper for the next commands
+- Install helper for the next commands    
   `gem install kubernetes_helper`
   
-- Verify or update k8s settings in .kubernetes/settings.rb
+- Verify or update k8s settings in .kubernetes/settings.rb        
+  Note: Please do not include sensitive values in this file, ENV values are recommended instead.
 
-- Register shared cloudsql proxy configuration (only if not exists)
+- Register shared cloudsql proxy configuration (only if not exists)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_command "kubectl create secret generic <%=deployment.cloud_secret_name%> --from-file=credentials.json=<path-to-downloaded/credentials.json>"
     ```
     
-- Register manually env vars (values must be encrypted using base64) 
+- Register manually env vars (values must be encrypted using base64)     
     Open and register secret values in `.kubernetes/secrets.yml`     
-    Note: Enter base64 encoded values
+    Note: Enter base64 encoded values      
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'secrets.yml' 'kubectl create'
     # kubectl get secrets # to list all secrets registered
     ```
 
-- Create deployment to run application
+- Create deployment to run application    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'deployment.yml' 'kubectl create'
     # kubectl get deployment # to list deployments
     ```
 
-- Create service to connect pods and ingress
+- Create service to connect pods and ingress    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'service.yml' 'kubectl create'
     # kubectl get services # to list all registered services
     ```
 
-- Create the public ip address (only if static ip is required)
+- Create the public ip address (only if static ip is required)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_command "gcloud compute addresses create <%=ingress.ip_name%> --global"
     # gcloud compute addresses list # to list static ips generated
     # Copy new external ip generated by the previous command and point your domain to it
     ```
   
-- Register ingress to receive external http calls (includes ssl certificates if defined)
+- Register ingress to receive external http calls (includes ssl certificates if defined)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'ingress.yml' 'kubectl create'
     # kubectl get ingress # to list all registered ingresses
@@ -60,18 +62,18 @@
     ```
 
 ## Apply any k8s setting changes
-- Secrets
+- Secrets    
   Open kubernetes secrets and add/edit/remove values and then save it    
   `kubectl edit secret ...`
   Once secrets were updated, then restart all related pods, see: https://medium.com/devops-dudes/how-to-propagate-a-change-in-kubernetes-secrets-by-restarting-dependent-pods-b71231827656
   
-- Other settings
+- Other settings    
   ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'deployment.yml' 'kubectl apply'
   ```
 
 ## Configure continuous deployment for github actions
-* Go to github repository settings
+* Go to github repository settings    
 * Register a new secret variable with content downloaded from https://console.cloud.google.com/iam-admin/serviceaccounts 
     (Make sure to attach a "Editor", "Storage Admin" and "Kubernetes engine cluster admin" role to the service account)
   ```bash
@@ -79,14 +81,14 @@
     production: PROD_CLOUD_TOKEN=<secret content here>
   ```
   
-* Add action to run deployment:
+* Add action to run deployment:    
   ```bash
     env:
       KB_AUTH_TOKEN: secrets.BETA_CLOUD_TOKEN
     run: DEPLOY_ENV=beta kubernetes_helper run_deployment 'cd.sh'
   ``` 
   
-* Sample:
+* Sample:    
   ```yml
   - run: sudo gem install kubernetes_helper
   - name: Staging deployment

data/lib/templates/_replicas.yml.erb

@@ -0,0 +1,12 @@
+  - apiVersion: autoscaling/v1
+    kind: HorizontalPodAutoscaler
+    metadata:
+      name: <%= "#{deployment.name}-replicas" %>
+    spec:
+      scaleTargetRef:
+        apiVersion: apps/v1
+        kind: Deployment
+        name: <%= deployment.name %>
+      minReplicas: <%= deployment.replicas_range[0] %>
+      maxReplicas: <%= deployment.replicas_range[1] %>
+      targetCPUUtilizationPercentage: 50

data/lib/templates/cd.sh

@@ -5,31 +5,37 @@ set -e
 SCRIPT_DIR=`dirname "$(realpath -s "$0")"` # app_dir/.kubernetes/
 cd "$SCRIPT_DIR/../" # project directory
 
-DEPLOYMENTS="<%=[deployment.name, deployment.job_name].join(',')%>"
+DEPLOYMENTS="<%=[deployment.job_name, deployment.name].join(',')%>"
 IMAGE_NAME="<%=continuous_deployment.image_name%>"
 CLUSTER_NAME="<%=continuous_deployment.cluster_name%>"
 PROJECT_NAME="<%=continuous_deployment.project_name%>"
 CLUSTER_REGION="<%=continuous_deployment.cluster_region%>"
 DOCKER_BUILD_CMD="<%=continuous_deployment.docker_build_cmd || 'build -f Dockerfile'%>"
 
-CI_COMMIT_SHA=$(git rev-parse --verify HEAD)
+CI_COMMIT_SHA=$(git rev-parse --verify HEAD || :)
+CI_COMMIT_SHA=${CI_COMMIT_SHA:-$(date +%s) }
 DEPLOY_NAME="${IMAGE_NAME}:${CI_COMMIT_SHA}"
 LATEST_NAME="${IMAGE_NAME}:latest"
-AUTH_PATH="$SCRIPT_DIR/k8s-auth-token.json"
-echo $KB_AUTH_TOKEN >> $AUTH_PATH
 
-## ***** GOOGLE CONNECTOR
-# Download and install Google Cloud SDK
-if [ -z "$(which gcloud)" ]; then
-  export CLOUDSDK_CORE_DISABLE_PROMPTS=1; curl https://sdk.cloud.google.com | bash && source /home/runner/google-cloud-sdk/path.bash.inc &&  gcloud --quiet components update kubectl
+if [ ! -z "$KB_AUTH_TOKEN" ]
+then
+  AUTH_PATH="$SCRIPT_DIR/k8s-auth-token.json"
+  rm -f -- $AUTH_PATH
+  echo $KB_AUTH_TOKEN >> $AUTH_PATH
+
+  ## ***** GOOGLE CONNECTOR
+  # Download and install Google Cloud SDK
+  if [ -z "$(which gcloud)" ]; then
+    export CLOUDSDK_CORE_DISABLE_PROMPTS=1; curl https://sdk.cloud.google.com | bash && source /home/runner/google-cloud-sdk/path.bash.inc &&  gcloud --quiet components update kubectl
+  fi
+
+  # Connect to cluster
+  gcloud auth activate-service-account --key-file $AUTH_PATH --project $PROJECT_NAME
+  gcloud docker --authorize-only --project $PROJECT_NAME
+  gcloud container clusters get-credentials $CLUSTER_NAME --region $CLUSTER_REGION
+  ## ***** END GOOGLE CONNECTOR
 fi
 
-# Connect to cluster
-gcloud auth activate-service-account --key-file $AUTH_PATH --project $PROJECT_NAME
-gcloud docker --authorize-only --project $PROJECT_NAME
-gcloud container clusters get-credentials $CLUSTER_NAME --region $CLUSTER_REGION
-## ***** END GOOGLE CONNECTOR
-
 
 ## Build and push containers
 docker $DOCKER_BUILD_CMD -t $DEPLOY_NAME .
@@ -43,5 +49,5 @@ for deployment in "${deployments[@]}"; do
   [ -z "$deployment" ] && continue # if empty value
 
   kubectl set image deployment/$deployment $deployment=$DEPLOY_NAME
-  [ "$deployment" = "${deployments[0]}" ] && kubectl rollout status deployment/$deployment
+  [ "$deployment" = "${deployments[0]}" ] && kubectl rollout status deployment/$deployment || true
 done

data/lib/templates/deployment.yml

@@ -47,16 +47,23 @@ documents:
                   name: applog
                   mountPath: /app/log
 
+                <% if deployment.cloud_secret_name %>
+                - &cloud_credentials_volume
+                  name: gcloud-creds
+                  mountPath: /secrets/gcloud
+                  readOnly: true
+                <% end %>
+
+            <% if deployment.cloud_secret_name %>
             - &cloudsql_container
               image: gcr.io/cloudsql-docker/gce-proxy:1.09 # Cloud sql proxy
               name: cloudsql-proxy
               command: ["/cloud_sql_proxy", "--dir=/cloudsql",
                         "-instances=<%=deployment.cloud_sql_instance%>",
-                        "-credential_file=/secrets/cloudsql/credentials.json"]
+                        "-credential_file=/secrets/gcloud/credentials.json"]
               volumeMounts:
-                - name: cloudsql-creds
-                  mountPath: /secrets/cloudsql
-                  readOnly: true
+                - *cloud_credentials_volume
+            <% end %>
 
             - &logs_container # print to stdout all log files
               name: print-logs
@@ -65,10 +72,14 @@ documents:
               volumeMounts:
                 - *log_volume
 
+          terminationGracePeriodSeconds: 120
           volumes:
-            - name: cloudsql-creds
+            <% if deployment.cloud_secret_name %>
+            - name: gcloud-creds
               secret:
                 secretName: '<%=deployment.cloud_secret_name%>'
+            <% end %>
+
             - name: applog
               emptyDir: {}
 
@@ -113,6 +124,13 @@ documents:
               <% end %>
             <% end %>
 
+            <% if deployment.cloud_secret_name %>
             - *cloudsql_container
+            <% end %>
+
             - <<: *logs_container
   <% end %>
+
+  <% if deployment.replicas_range %>
+  <%= include_template "_replicas.yml.erb" %>
+  <% end %>

data/lib/templates/settings.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-is_beta = ENV['DEPLOY_ENV'] == 'beta'
-app_name = is_beta ? 'my_beta_app' : 'my_app'
+is_production = ENV['DEPLOY_ENV'] == 'production'
+app_name = is_production ? 'my-app' : 'my-beta-app' # underscore not accepted
 settings = {
   deployment: {
     name: app_name,
-    replicas: is_beta ? 1 : 2,
-    cloud_secret_name: "#{is_beta ? 'beta' : 'production'}-cloud-secret",
+    replicas: is_production ? 2 : 1,
+    cloud_secret_name: "#{is_production ? 'production' : 'beta'}-cloud-secret",
     cloud_sql_instance: 'xxx:xxx:xxx=tcp:5432', # 5432 => postgres, 3306 => mysql
     env_vars: {}, # Sample: { 'CUSTOM_VAR' => 'value' }
     # command: '', # custom container command (default empty to be managed by Dockerfile)
@@ -19,7 +19,7 @@ settings = {
     name: "#{app_name}-ingress",
     ip_name: "#{app_name}-static-ip", # nil if static ip is not necessary
     certificate_name: "#{app_name}-lets-encrypt", # nil if ssl is not required
-    domain_name: is_beta ? 'beta.myapp.com' : 'myapp.com' # nil if domain is not required
+    domain_name: is_production ? 'myapp.com' : 'beta.myapp.com' # nil if domain is not required
   },
   continuous_deployment: {
     image_name: "gcr.io/my-account/#{app_name}",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes_helper
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: '1.1'
 platform: ruby
 authors:
 - owen2345
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-03 00:00:00.000000000 Z
+date: 2021-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erb
@@ -40,6 +40,7 @@ files:
 - lib/kubernetes_helper/railtie.rb
 - lib/kubernetes_helper/version.rb
 - lib/templates/README.md
+- lib/templates/_replicas.yml.erb
 - lib/templates/_sidekiq_alive_gem.yml.erb
 - lib/templates/cd.sh
 - lib/templates/deployment.yml