kubernetes_helper 0.3.0 → 1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c95b3029857b4b9793c337abb65f8d983376db3055bd411b8982907cce9b5a1d
-  data.tar.gz: '08a11127589a60bd3fc340c11a2ddecd635a44347dd77506686d932093a53f4f'
+  metadata.gz: 528e61587a3011330c7e474b523ce04079c8e49483bf894d5eb1806927436888
+  data.tar.gz: 9df4e7aab70108b75831077d2f4390067e5807664050f70e1557f504835144bd
 SHA512:
-  metadata.gz: 05e61bc79a77cbce7894430788ccfdddfceb3ac17041453f55dda0809ba42b8f50ab970dc72a566726f5548b1df3aa9bff068ef98adfd21e867d729b0bed316f
-  data.tar.gz: 5464337442b676f531ff14f294652ebe43270110f054dcba401045e5e75b92352ac1432ebb7b281bf84cd91c87c0f1ee9984ed0578d535287926b6751f1e446c
+  metadata.gz: 9ed371a62084598f9bb4755df54ca059ffbeb729731555e4a6a6cf58dceecc0c347741921b5c46fc11408a937c914ea2d3fddaf3da94bc801dd6cd7f9ecc1ed8
+  data.tar.gz: 6248035be5a30995594a39ad56b73b811935b3b55988d067173a530ee0522294f6afb0e6361f664d4f696e7b370ce7b816ed74a53cb3f416c584e12c76ff023d

data/README.md

@@ -1,37 +1,61 @@
 # KubernetesHelper
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/kubernetes_helper`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+This gem is a helper to manage easily Kubernetes settings for GCloud (easy customization for other cloud services) where configuring and deploying a new application can be done in a couple of minutes.
+Configuration and customization can be done for multiple environments and at any level which permits to deploy simple and complex applications.
 
 ## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'kubernetes_helper'
+```bash
+cd my_app/
+gem install kubernetes_helper -v '~> 1.0'
+kubernetes_helper generate_templates
 ```
-
-And then execute:
-
-    $ bundle install
-
-Or install it yourself as:
-
-    $ gem install kubernetes_helper
+Note: Requires ruby 1.7+      
 
 ## Usage
+- Run any kubernetes document    
+  `DEPLOY_ENV=<env name> kubernetes_helper run_deployment "<document name>" "<bash command>"`    
+  Evaluates the kubernetes document with the following details:
+  - Supports for `- documents` to include multiple documents in a file and share yml variables between them (Sample: `lib/templates/deployment.yml#1`)
+  - Replaces all setting values based on `DEPLOY_ENV`
+  - Supports for secrets auto importer using `import_secrets: ['secrets.yml', '<%=secrets.name%>']` (Sample: `lib/templates/deployment.yml#29`)
+  - Supports for template including by `include_template 'template_name.yml.erb'`    
+  Sample: `DEPLOY_ENV=beta kubernetes_helper run_deployment "deployment.yml" "kubectl create"`
+   
+- Run kubernetes commands    
+  `DEPLOY_ENV=<env name> rake kubernetes_helper:run_command "<bash or k8s commands>"`           
+  Replaces all setting variables inside command based on `DEPLOY_ENV` and performs it as a normal bash command.             
+  Sample: `DEPLOY_ENV=beta rake kubernetes_helper:run_command "gcloud compute addresses create \#{ingress.ip_name} --global"'`    
+  
+- Run kubernetes bash scripts     
+  `DEPLOY_ENV=<env name> kubernetes_helper run_script "<script name>"`    
+  Performs the script name located inside `.kubernetes` folder or kubernetes_helper template as the second option.
+  All setting variables inside the script will be replaced based on `DEPLOY_ENV`.      
+  Sample: `DEPLOY_ENV=beta kubernetes_helper run_script "cd.sh"`
+
+- Generate templates    
+  `DEPLOY_ENV=<env name> kubernetes_helper generate_templates "<mode_or_template_name>"`     
+  Copy files based on mode (`basic|advanced`) or a specific file from templates.     
+  Sample: `DEPLOY_ENV=beta kubernetes_helper generate_templates "basic"`    
+  Sample: `DEPLOY_ENV=beta kubernetes_helper generate_templates "ingress.yml"`    
+
+When performing a script it looks first for file inside .kubernetes folder, if not exist, 
+it looks for the file inside kubernetes_helper template folder.    
+
+## Templating
+When performing a command or script, the setting variables are replaced based on `DEPLOY_ENV`. 
+All these setting variable values are configured in `.kubernetes/settings.rb` which defines the values based on `DEPLOY_ENV`.     
+These setting variables use [erb](https://github.com/ruby/erb) template gem to define variable replacement and conditional blocks, and so on.
+Note: Setting variable values are referenced as an object format instead of a hash format for simplicity.
+
+
+## Deployment
+Once you generated the basic templates, it comes with the corresponding [readme.md](/lib/templates/README.md) which includes all the steps to deploy your application.  
+
+## Sample
+https://owen2345.github.io/kubernetes_helper/
 
 ## TODO
-- Documentation
-    - Include conditional blocks
-    - Include hardcoded env values
-    - Tasks
-    - Add docs to include partials using `include_template 'sample.yml.erb'`
-- Rake verify files
 - Add one_step_configuration.sh
-- Fix if/end_if for blocks
-- Ability to copy specific template
 
 ## Contributing
 

data/exe/kubernetes_helper

@@ -9,8 +9,8 @@ case ARGV[0]
 when 'run_command'
   KubernetesHelper::Core.new(ENV['DEPLOY_ENV']).run_command(ARGV[1])
 # Run the deployment script.
-#   Sample: DEPLOY_ENV=beta kubernetes_helper run_deployment "cd_gcloud.sh"
-when 'run_deployment'
+#   Sample: DEPLOY_ENV=beta kubernetes_helper run_deployment "cd.sh"
+when 'run_deployment', 'run_script'
   script_path = KubernetesHelper.settings_path(ARGV[1], use_template: true)
   KubernetesHelper::Core.new(ENV['DEPLOY_ENV']).run_script(script_path)
 # Parses kubernetes yml files (supporting multiple documents, Config variables replacement, include secrets).
@@ -22,6 +22,7 @@ when 'run_yml'
     .parse_yml_file(KubernetesHelper.settings_path(ARGV[1], use_template: true), output_path)
   KubernetesHelper.run_cmd("#{ARGV[2]} -f #{output_path}")
 # Generate template files
+#   Sample: DEPLOY_ENV=beta kubernetes_helper generate_templates "basic"
 when 'generate_templates' #  Sample: kubernetes_helper generate basic
   mode = ARGV[1] || 'basic'
   KubernetesHelper.copy_templates(mode)

data/lib/kubernetes_helper.rb

@@ -39,11 +39,14 @@ module KubernetesHelper
     file_name ? File.join(path, file_name) : path
   end
 
-  # @param mode (basic, advanced)
-  def self.copy_templates(mode)
+  # @param mode_or_file (basic, advanced, String) mode name or any specific template name
+  def self.copy_templates(mode_or_file)
     FileUtils.mkdir(settings_path) unless Dir.exist?(settings_path)
+    template_path = templates_path(mode_or_file)
+    return FileUtils.cp(template_path, settings_path(mode_or_file)) if File.exist?(template_path)
+
     files = %w[README.md secrets.yml settings.rb]
-    files += %w[deployment.yml cd.sh ingress.yml service.yml] if mode == 'advanced'
+    files += %w[deployment.yml cd.sh ingress.yml service.yml] if mode_or_file == 'advanced'
     files.each do |name|
       path = settings_path(name)
       FileUtils.cp(templates_path(name), path) unless File.exist?(path)

data/lib/kubernetes_helper/core.rb

@@ -57,7 +57,7 @@ module KubernetesHelper
       File.write(tmp_file, content)
       KubernetesHelper.run_cmd("chmod +x #{tmp_file}")
       KubernetesHelper.run_cmd(tmp_file)
-      File.delete(tmp_file)
+      # File.delete(tmp_file) # keep tmp script for analysis purpose
     end
 
     private

data/lib/kubernetes_helper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KubernetesHelper
-  VERSION = '0.3.0'
+  VERSION = '1.0'
 end

data/lib/templates/README.md

@@ -2,54 +2,56 @@
 
 ## Configure a new application environment
 - Create the project on Gcloud
-- Set the project where to work on
+- Set the project where to work on     
     `gcloud config set project my-project`
     
-- Create the cluster (Only if not exist)
+- Create the cluster (Only if not exist)    
     `gcloud container clusters create my-cluster`
     `# gcloud container clusters list --region europe-west4-a # to list clusters`
+    Note: `--preemptible` recommended for non production environments   
   
-- Use the cluster/project as default
+- Use the cluster/project as default    
     `gcloud container clusters get-credentials my-cluster --zone europe-west4-a`
   
-- Install helper for the next commands
+- Install helper for the next commands    
   `gem install kubernetes_helper`
   
-- Verify or update k8s settings in .kubernetes/settings.rb
+- Verify or update k8s settings in .kubernetes/settings.rb        
+  Note: Please do not include sensitive values in this file, ENV values are recommended instead.
 
-- Register shared cloudsql proxy configuration (only if not exists)
+- Register shared cloudsql proxy configuration (only if not exists)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_command "kubectl create secret generic <%=deployment.cloud_secret_name%> --from-file=credentials.json=<path-to-downloaded/credentials.json>"
     ```
     
-- Register manually env vars (values must be encrypted using base64) 
+- Register manually env vars (values must be encrypted using base64)     
     Open and register secret values in `.kubernetes/secrets.yml`     
-    Note: Enter base64 encoded values
+    Note: Enter base64 encoded values      
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'secrets.yml' 'kubectl create'
     # kubectl get secrets # to list all secrets registered
     ```
 
-- Create deployment to run application
+- Create deployment to run application    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'deployment.yml' 'kubectl create'
     # kubectl get deployment # to list deployments
     ```
 
-- Create service to connect pods and ingress
+- Create service to connect pods and ingress    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'service.yml' 'kubectl create'
     # kubectl get services # to list all registered services
     ```
 
-- Create the public ip address (only if static ip is required)
+- Create the public ip address (only if static ip is required)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_command "gcloud compute addresses create <%=ingress.ip_name%> --global"
     # gcloud compute addresses list # to list static ips generated
     # Copy new external ip generated by the previous command and point your domain to it
     ```
   
-- Register ingress to receive external http calls (includes ssl certificates if defined)
+- Register ingress to receive external http calls (includes ssl certificates if defined)    
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'ingress.yml' 'kubectl create'
     # kubectl get ingress # to list all registered ingresses
@@ -60,18 +62,18 @@
     ```
 
 ## Apply any k8s setting changes
-- Secrets
+- Secrets    
   Open kubernetes secrets and add/edit/remove values and then save it    
   `kubectl edit secret ...`
   Once secrets were updated, then restart all related pods, see: https://medium.com/devops-dudes/how-to-propagate-a-change-in-kubernetes-secrets-by-restarting-dependent-pods-b71231827656
   
-- Other settings
+- Other settings    
   ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'deployment.yml' 'kubectl apply'
   ```
 
 ## Configure continuous deployment for github actions
-* Go to github repository settings
+* Go to github repository settings    
 * Register a new secret variable with content downloaded from https://console.cloud.google.com/iam-admin/serviceaccounts 
     (Make sure to attach a "Editor", "Storage Admin" and "Kubernetes engine cluster admin" role to the service account)
   ```bash
@@ -79,15 +81,16 @@
     production: PROD_CLOUD_TOKEN=<secret content here>
   ```
   
-* Add action to run deployment:
+* Add action to run deployment:    
   ```bash
     env:
       KB_AUTH_TOKEN: secrets.BETA_CLOUD_TOKEN
     run: DEPLOY_ENV=beta kubernetes_helper run_deployment 'cd.sh'
   ``` 
   
-* Sample:
+* Sample:    
   ```yml
+  - run: sudo gem install kubernetes_helper
   - name: Staging deployment
     env: # Env variable saved in github that contains gcloud credential (json format)
       KB_AUTH_TOKEN: ${{ secrets.BETA_GOOGLE_AUTH }}

data/lib/templates/_replicas.yml.erb

@@ -0,0 +1,12 @@
+  - apiVersion: autoscaling/v1
+    kind: HorizontalPodAutoscaler
+    metadata:
+      name: <%= "#{deployment.name}-replicas" %>
+    spec:
+      scaleTargetRef:
+        apiVersion: apps/v1
+        kind: Deployment
+        name: <%= deployment.name %>
+      minReplicas: <%= deployment.replicas_range[0] %>
+      maxReplicas: <%= deployment.replicas_range[1] %>
+      targetCPUUtilizationPercentage: 50

data/lib/templates/cd.sh

@@ -2,19 +2,22 @@
 set -e
 # expected ENV VAR "KB_AUTH_TOKEN"
 
-SCRIPT_DIR=`dirname "$(realpath -s "$0")"`
+SCRIPT_DIR=`dirname "$(realpath -s "$0")"` # app_dir/.kubernetes/
+cd "$SCRIPT_DIR/../" # project directory
 
-DEPLOYMENTS="<%=[deployment.name, deployment.job_name].join(',')%>"
+DEPLOYMENTS="<%=[deployment.job_name, deployment.name].join(',')%>"
 IMAGE_NAME="<%=continuous_deployment.image_name%>"
 CLUSTER_NAME="<%=continuous_deployment.cluster_name%>"
 PROJECT_NAME="<%=continuous_deployment.project_name%>"
 CLUSTER_REGION="<%=continuous_deployment.cluster_region%>"
-DOCKER_BUILD_CMD="<%=continuous_deployment.docker_build_cmd%>"
+DOCKER_BUILD_CMD="<%=continuous_deployment.docker_build_cmd || 'build -f Dockerfile'%>"
 
 CI_COMMIT_SHA=$(git rev-parse --verify HEAD)
 DEPLOY_NAME="${IMAGE_NAME}:${CI_COMMIT_SHA}"
 LATEST_NAME="${IMAGE_NAME}:latest"
 AUTH_PATH="$SCRIPT_DIR/k8s-auth-token.json"
+
+rm -f -- $AUTH_PATH
 echo $KB_AUTH_TOKEN >> $AUTH_PATH
 
 ## ***** GOOGLE CONNECTOR
@@ -42,5 +45,5 @@ for deployment in "${deployments[@]}"; do
   [ -z "$deployment" ] && continue # if empty value
 
   kubectl set image deployment/$deployment $deployment=$DEPLOY_NAME
-  [ "$deployment" = "${deployments[0]}" ] && kubectl rollout status deployment/$deployment
+  [ "$deployment" = "${deployments[0]}" ] && kubectl rollout status deployment/$deployment || true
 done

data/lib/templates/deployment.yml

@@ -46,17 +46,19 @@ documents:
                 - &log_volume
                   name: applog
                   mountPath: /app/log
+                - &cloud_credentials_volume
+                  name: gcloud-creds
+                  mountPath: /secrets/gcloud
+                  readOnly: true
 
             - &cloudsql_container
               image: gcr.io/cloudsql-docker/gce-proxy:1.09 # Cloud sql proxy
               name: cloudsql-proxy
               command: ["/cloud_sql_proxy", "--dir=/cloudsql",
                         "-instances=<%=deployment.cloud_sql_instance%>",
-                        "-credential_file=/secrets/cloudsql/credentials.json"]
+                        "-credential_file=/secrets/gcloud/credentials.json"]
               volumeMounts:
-                - name: cloudsql-creds
-                  mountPath: /secrets/cloudsql
-                  readOnly: true
+                - *cloud_credentials_volume
 
             - &logs_container # print to stdout all log files
               name: print-logs
@@ -65,8 +67,9 @@ documents:
               volumeMounts:
                 - *log_volume
 
+          terminationGracePeriodSeconds: 120
           volumes:
-            - name: cloudsql-creds
+            - name: gcloud-creds
               secret:
                 secretName: '<%=deployment.cloud_secret_name%>'
             - name: applog
@@ -116,3 +119,7 @@ documents:
             - *cloudsql_container
             - <<: *logs_container
   <% end %>
+
+  <% if deployment.replicas_range %>
+  <%= include_template "_replicas.yml.erb" %>
+  <% end %>

data/lib/templates/settings.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-is_beta = ENV['DEPLOY_ENV'] == 'beta'
-app_name = is_beta ? 'my_beta_app' : 'my_app'
+is_production = ENV['DEPLOY_ENV'] == 'production'
+app_name = is_production ? 'my_app' : 'my_beta_app'
 settings = {
   deployment: {
     name: app_name,
-    replicas: is_beta ? 1 : 2,
-    cloud_secret_name: "#{is_beta ? 'beta' : 'production'}-cloud-secret",
+    replicas: is_production ? 2 : 1,
+    cloud_secret_name: "#{is_production ? 'production' : 'beta'}-cloud-secret",
     cloud_sql_instance: 'xxx:xxx:xxx=tcp:5432', # 5432 => postgres, 3306 => mysql
     env_vars: {}, # Sample: { 'CUSTOM_VAR' => 'value' }
     # command: '', # custom container command (default empty to be managed by Dockerfile)
@@ -19,13 +19,14 @@ settings = {
     name: "#{app_name}-ingress",
     ip_name: "#{app_name}-static-ip", # nil if static ip is not necessary
     certificate_name: "#{app_name}-lets-encrypt", # nil if ssl is not required
-    domain_name: is_beta ? 'beta.myapp.com' : 'myapp.com' # nil if domain is not required
+    domain_name: is_production ? 'myapp.com' : 'beta.myapp.com' # nil if domain is not required
   },
   continuous_deployment: {
     image_name: "gcr.io/my-account/#{app_name}",
     project_name: 'my-project-name',
     cluster_name: 'my-cluster-name',
-    cluster_region: 'europe-west4-a'
+    cluster_region: 'europe-west4-a',
+    docker_build_cmd: 'build -f Dockerfile'
   },
   secrets: {
     name: "#{app_name}-secrets"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes_helper
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: '1.0'
 platform: ruby
 authors:
 - owen2345
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-03 00:00:00.000000000 Z
+date: 2021-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erb
@@ -40,6 +40,7 @@ files:
 - lib/kubernetes_helper/railtie.rb
 - lib/kubernetes_helper/version.rb
 - lib/templates/README.md
+- lib/templates/_replicas.yml.erb
 - lib/templates/_sidekiq_alive_gem.yml.erb
 - lib/templates/cd.sh
 - lib/templates/deployment.yml