kubernetes_helper 1.6.1 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59945c95287cd19d19577b1e96f5f6b9853ce4e40a82b022de362ba676ad5bf9
-  data.tar.gz: 0b5360c14ea7b0f14300b87112225e9421090299ae099f60cf07b48a22ecebde
+  metadata.gz: ad11b73ac23d45b86c9b5789ae815af2ee565d6360462368e96c4b15dfcc3301
+  data.tar.gz: 61df31315a5b137c9e1c4575002b875f87175d72d5319cf0e1439d8c01afaee6
 SHA512:
-  metadata.gz: 8057efca5943cccfa92bc1a8a3a696f5ca2f3b3b21415da3923794c4674848ba7797098e5d718f2b7da376c9d05634182185ea45572500cc9af9481b1246f107
-  data.tar.gz: 2dd11b252b27be829f8d1de244409c37eda5104f479dda45839b555e9f92423fc0142122e536c3824add5c73e91eba79ca0a5779f26dc9f78ba6bc95441ad942
+  metadata.gz: f1a177f81e8bf2a9337d233369c39dd7cc42e2baf0c7dca7fdba25984d86014837db2207bcab10a1b9f8b8bc2060b55dee8c0dc0054074ac4846592856c8d778
+  data.tar.gz: 8be87c0b86cc7e4a7711d20d6360e96d5b99d7655ea3a3bed0a8291dec59cc552a9a326911355d2427f0e5548c0976045d7d28204af1c9dcee2c5fd1bc2af1f9

data/README.md

@@ -9,7 +9,15 @@ cd my_app/
 gem install kubernetes_helper -v '~> 1.0'
 kubernetes_helper generate_templates
 ```
-Note: Requires ruby 1.7+      
+Note: Requires ruby 1.7+     
+
+## Configuration
+- Edit `.kubernetes/settings.rb` and enter or replace all settings with the valid ones
+- For special applications where default configurations are not enough, you can do the following: 
+    - Download the corresponding template from [lib/templates](lib/templates)
+    - Put it inside `.kubernetes` folder
+    - Customize based on your needs (You can add or use your custom variables from `settings.rb`)    
+    Note: The local template will be used instead of the default.
 
 ## Deployment
 Once you generated the basic templates, it comes with the corresponding [readme.md](/lib/templates/README.md) which includes all the steps to deploy your application.
@@ -21,7 +29,7 @@ Once you generated the basic templates, it comes with the corresponding [readme.
   - Supports for `- documents` to include multiple documents in a file and share yml variables between them (Sample: `lib/templates/deployment.yml#1`)
   - Replaces all setting values based on `DEPLOY_ENV`
   - Supports for secrets auto importer using `import_secrets: ['secrets.yml', '<%=secrets.name%>']` (Sample: `lib/templates/deployment.yml#29`)
-  - Supports for template including by `include_template 'template_name.yml.erb'`    
+  - Supports for sub templates by `include_template 'template_name.yml.erb', { my_local_var: 10 }`    
   Sample: `DEPLOY_ENV=beta kubernetes_helper run_deployment "deployment.yml" "kubectl create"`
    
 - Run kubernetes commands    
@@ -44,6 +52,67 @@ Once you generated the basic templates, it comes with the corresponding [readme.
 When performing a script it looks first for file inside .kubernetes folder, if not exist, 
 it looks for the file inside kubernetes_helper template folder.    
 
+## Settings API
+Below settings are used when running Continuous Deployment
+- `continuous_deployment.image_name` (String, Optional): Partial docker image url where `:latest` will be automatically added. Sample: `gcr.io/my-account/my_app_name`
+- `continuous_deployment.image` (String, Optional, is mandatory if `image_name` is empty): Full docker image url. Sample: `gcr.io/my-account/my_app_name:latest`
+- `continuous_deployment.project_name`: Cloud project name. Sample: `my-project-name`
+- `continuous_deployment.cluster_name`: Cluster cluster name. Sample: `my-cluster-name`
+- `continuous_deployment.cluster_region`: Cluster region name. Sample: `europe-west4-a`
+- `continuous_deployment.docker_build_cmd`: Docker command to build the corresponding image. Sample: `build --target production -f Dockerfile `
+- `continuous_deployment.update_deployment` (Boolean, default: false): If true permits to re-generate and update the k8s deployment(s) before applying the new version (new docker image) 
+
+Below settings are used when configuring the application in the k8s environment
+- `deployment.name` (String): Web deployment name (Note: Underscores are not accepted). Sample: `my-app`  
+- `deployment.replicas` (Integer): Quantity of replicas. Sample: `1`
+- `deployment.replicas_range` (Array<min, max, cpu_percentage>, Optional): Defines the minimum and the maximum number of pods that could automatically be created when `CPUUtilizationPercentage` is above than defined. Sample: `[1, 3, 50]`
+- `deployment.cloud_secret_name` (String, Optional): K8s credentials name where cloud secrets will be saved (includes permission like DB). Sample: `my-app-cloud-secret`
+- `deployment.cloud_sql_instance` (String, Optional): Cloud sql instance name. Sample: `my-project:europe-west1:my-instance-name=tcp:5432` (5432 => postgres, 3306 => mysql)
+- `deployment.env_vars` (Hash, optional): List of static env variables (Note: Not recommended for sensitive values). Sample: `{ 'RAILS_ENV' => 'production' }`
+- `deployment.command` (String, Optional): Bash command to be used for web containers. Sample: `rails s -b 0.0.0.0`
+- `deployment.liveness_path` (String, Optional): Relative path to be used for readiness and liveness checker of the web app. Sample: `/check_liveness`
+- `deployment.custom_volumes` (Hash<name: path>, Optional): Custom volumes to be mounted. Sample: `{ my_volume: { kind: 'hostPath', mount_path: '/', settings: { path: '..', type: 'Directory' } }  }`
+
+- `deployment.job_name` (String, optional): Job deployment name (Note: Underscores are not accepted). Sample: `my-app-job`. Note: This deployment is created only if this value is present
+- `deployment.job_command` (String, optional): Bash command to be used for job container. Sample: `bundle exec sidekiq`
+- `deployment.job_sidekiq_alive_gem` (Boolean, default false): If true will add liveness checker settings using `sidekiq_alive_gem` (`sidekiq_alive` gem needs to be present in your Gemfile)
+- `deployment.job_services` (Array, Optional, only `job_sidekiq_alive_gem` or `job_services` is allowed): List of linux service names that are required for a healthy job container. Sample: `['sidekiq', 'cron']` 
+
+
+- `secrets.name` (String): K8s secrets name where env vars will be saved and fetched from. Sample: `my-app-secrets`
+
+- `service.name`: K8s service name. Sample: `my-app-service`
+- `service.port_name`: Http port name to connect between k8s ingress and service. Sample: `http-port`. Note: max 15 characters
+- `service.backend_port_name` (String): Web backend port name to be connected between k8s service and web deployments. Sample: `b-port`. Note: max 15 characters
+
+- `ingress.name`: Name of k8s ingress for the app: Sample: `my-app-ingress`
+- `ingress.ip_name` (Optional): Static ip address is not created nor assigned if empty value. Sample: `my-app-static-ip`
+- `ingress.certificate_name` (Optional): Ssl certificate is not created nor assigned if empty value. Sample: `my-app-lets-encrypt`. Note: requires `certificate_domain` 
+- `ingress.certificate_domain` (Optional): Domain name for the certificate. Sample: `myapp.com`. Note: does not support for willcard domains
+
+### Partials
+- `_container_extra_settings.yml` Partial template to add custom container settings. Receives `pod` as local variable (`web` | `job` | `cloudsql` | `logs`). Sample:
+  ```yaml
+               <% if locals[:pod] == 'job' %>
+               resources:
+                 requests:
+                   cpu: 50m
+                   memory: 256Mi
+                 limits:
+                   cpu: 500m
+                   memory: 1Gi
+               <% end %>
+  ``` 
+- `_custom_containers.yml` Partial template to add extra containers (Receives `pod` as local variable: `web` | `job`). Sample:
+```yaml
+           <% if locals[:pod] == 'job' %>
+           - name: scraper
+             image: owencio/easy_scraper
+             ...
+           <% end %>
+```
+- `_cd_apply_images.sh` Partial template to customize the process to apply the new version (new docker image)
+
 ## Templating
 When performing a command or script, the setting variables are replaced based on `DEPLOY_ENV`. 
 All these setting variable values are configured in `.kubernetes/settings.rb` which defines the values based on `DEPLOY_ENV`.     

data/lib/kubernetes_helper/core.rb

@@ -11,8 +11,8 @@ module KubernetesHelper
       binding
     end
 
-    def include_template(name)
-      render_template.call(name)
+    def include_template(name, locals = {})
+      render_template.call(name, locals)
     end
   end
 
@@ -36,8 +36,8 @@ module KubernetesHelper
 
     # @param text (String)
     # Sample: replicas: '#{deployment.replicas}'
-    def replace_config_variables(text)
-      values = config_values.map do |key, value| # rubocop:disable Style/HashTransformValues
+    def replace_config_variables(text, locals = {})
+      values = config_values.merge(locals: locals).map do |key, value| # rubocop:disable Style/HashTransformValues
         [key, value.is_a?(Hash) ? OpenStruct.new(value) : value]
       end.to_h
       values[:render_template] = method(:render_template)
@@ -75,10 +75,10 @@ module KubernetesHelper
       end
     end
 
-    def render_template(template_name)
+    def render_template(template_name, locals = {})
       path = KubernetesHelper.settings_path(template_name, use_template: true)
       text = "\n#{File.read(path)}"
-      replace_config_variables(text)
+      replace_config_variables(text, locals)
     end
 
     def static_env_vars
@@ -91,10 +91,11 @@ module KubernetesHelper
     end
 
     # parse secrets auto importer
-    def parse_import_secrets(document)
+    def parse_import_secrets(document) # rubocop:disable Metrics/AbcSize
       containers = document.dig('spec', 'template', 'spec', 'containers') || []
       containers.each do |container|
-        container['env'] = (container['env'] || []) + static_env_vars
+        container['env'] = (container['env'] || [])
+        container['env'] = container['env'] + static_env_vars if container.delete('static_env')
         if container['import_secrets']
           container['env'] = container['env'] + import_secrets(*container['import_secrets'])
           container.delete('import_secrets')

data/lib/kubernetes_helper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KubernetesHelper
-  VERSION = '1.6.1'
+  VERSION = '1.10.0'
 end

data/lib/templates/_container_extra_settings.yml

@@ -0,0 +1,9 @@
+#              <% if locals[:pod] == 'job' %>
+#              resources:
+#                requests:
+#                  cpu: 50m
+#                  memory: 256Mi
+#                limits:
+#                  cpu: 500m
+#                  memory: 1Gi
+#              <% end %>

data/lib/templates/_custom_containers.yml

@@ -0,0 +1,5 @@
+#            <% if locals[:pod] == 'job' %>
+#            - name: scraper
+#              image: owencio/easy_scraper
+#              ...
+#            <% end %>

data/lib/templates/_custom_volumes.yml

@@ -0,0 +1,3 @@
+#            - name: dshm
+#              emptyDir:
+#                medium: Memory

data/lib/templates/_job_liveness.yml

@@ -0,0 +1,12 @@
+              ports: [ ]
+              <% if (deployment.job_services || []).any? %>
+              livenessProbe: &liveness_probe
+                exec:
+                   command: [ /bin/sh, -c,
+                     'if [ $(ps -ef | grep "<%= deployment.job_services.join("\|") %>" | grep -v "grep" | wc -l) -lt <%= deployment.job_services.count %> ]; then
+                               echo "Some required services are not running"; exit 1;
+                           fi' ]
+                initialDelaySeconds: 120
+                periodSeconds: 30
+              readinessProbe: *liveness_probe
+              <% end %>

data/lib/templates/_replicas.yml

@@ -9,4 +9,4 @@
         name: <%= deployment.name %>
       minReplicas: <%= deployment.replicas_range[0] %>
       maxReplicas: <%= deployment.replicas_range[1] %>
-      targetCPUUtilizationPercentage: 50
+      targetCPUUtilizationPercentage: <%= deployment.replicas_range[2] || 50 %>

data/lib/templates/_volumes.yml

@@ -0,0 +1,20 @@
+            <% if deployment.cloud_secret_name %>
+            - name: gcloud-creds
+              secret:
+                secretName: '<%=deployment.cloud_secret_name%>'
+            <% end %>
+
+            - name: applog
+              emptyDir: {}
+
+            # Custom volumes
+            # { my_volume: { kind: 'hostPath', mount_path: '/', settings: { path: '..', type: 'Directory' } }  }
+            <% (deployment.custom_volumes || {}).each do |name, vol| %>
+            - name: <%= name %>
+              <%= vol[:kind] %>:
+                <% vol[:settings].each do |k, v| %>
+                <%= k %>: <%= v %>
+                <% end %>
+              <% end %>
+
+            <%= include_template "_custom_volumes.yml", { pod: locals[:pod] } %>

data/lib/templates/cd.sh

@@ -43,14 +43,16 @@ docker tag $DEPLOY_NAME $LATEST_NAME
 docker push $DEPLOY_NAME
 docker push $LATEST_NAME
 
+## Update new secrets defined in secrets.yml as ENV vars for deployments
+<% if continuous_deployment.update_deployment %>
+  kubernetes_helper run_yml 'deployment.yml' 'kubectl apply'
+<% end %>
+
 ## Apply deployments
 IFS=',' read -r -a deployments <<< "$DEPLOYMENTS"
 for deployment in "${deployments[@]}"; do
   [ -z "$deployment" ] && continue # if empty value
 
   <%= include_template "_cd_apply_images.sh" %>
-
-  <% if continuous_deployment.update_deployment %>
-    kubernetes_helper run_yml 'deployment.yml' 'kubectl apply'
-  <% end %>
 done
+

data/lib/templates/certificate.yml

@@ -4,4 +4,4 @@ metadata:
   name: '<%=ingress.certificate_name%>'
 spec:
   domains: # does not support for willcard domains
-    - '<%=ingress.domain_name%>'
+    - '<%= ingress.domain_name || ingress.certificate_domain %>'

data/lib/templates/deployment.yml

@@ -21,11 +21,17 @@ documents:
         spec: &template_spec
           containers:
             - &app_container
-              image: '<%=continuous_deployment.image_name%>:latest'
+              <% if continuous_deployment.image_name %>
+              image: '<%= continuous_deployment.image_name %>:latest'
+              <% else %>
+              image: '<%= continuous_deployment.image %>'
+              <% end %>
+
               <% if deployment.command %>
               command: ["/bin/bash", "-c", "<%= deployment.command %>"]
               <% end %>
               name: *app_name
+              static_env: true
               import_secrets: ['secrets.yml', '<%=secrets.name%>']
               ports:
                 - containerPort: &port 3000
@@ -41,6 +47,7 @@ documents:
                 periodSeconds: 15
               readinessProbe: *liveness_probe
               <% end %>
+              <%= include_template "_container_extra_settings.yml", { pod: 'web' } %>
 
               volumeMounts:
                 - &log_volume
@@ -69,6 +76,7 @@ documents:
               env: []
               volumeMounts:
                 - *cloud_credentials_volume
+              <%= include_template "_container_extra_settings.yml", { pod: 'cloudsql' } %>
             <% end %>
 
             - &logs_container # print to stdout all log files
@@ -78,27 +86,13 @@ documents:
               env: []
               volumeMounts:
                 - *log_volume
+              <%= include_template "_container_extra_settings.yml", { pod: 'logs' } %>
+
+            <%= include_template "_custom_containers.yml", { pod: 'web' } %>
 
           terminationGracePeriodSeconds: 120
           volumes:
-            <% if deployment.cloud_secret_name %>
-            - name: gcloud-creds
-              secret:
-                secretName: '<%=deployment.cloud_secret_name%>'
-            <% end %>
-
-            - name: applog
-              emptyDir: {}
-
-            # Custom volumes
-            # { my_volume: { kind: 'hostPath', mount_path: '/', settings: { path: '..', type: 'Directory' } }  }
-            <% (deployment.custom_volumes || {}).each do |name, vol| %>
-            - name: <%= name %>
-              <%= vol[:kind] %>:
-                <% vol[:settings].each do |k, v| %>
-                <%= k %>: <%= v %>
-                <% end %>
-            <% end %>
+            <%= include_template "_volumes.yml", { pod: 'web' } %>
 
 
   <% if deployment.job_name %>
@@ -124,22 +118,12 @@ documents:
               <% if deployment.job_command %>
               command: [ "/bin/bash", "-c", "<%= deployment.job_command %>" ]
               <% end %>
+              <%= include_template "_container_extra_settings.yml", { pod: 'job' } %>
 
             <% if deployment.job_sidekiq_alive_gem %>
               <%= include_template "_sidekiq_alive_gem.yml" %>
             <% else %>
-              ports: [ ]
-              <% if (deployment.job_services || []).any? %>
-              livenessProbe: &liveness_probe
-                exec:
-                  command: [ /bin/sh, -c,
-                      'if [ $(ps -ef | grep "<%= deployment.job_services.join("\|") %>" | grep -v "grep" | wc -l) -lt <%= deployment.job_services.count %> ]; then
-                           echo "Some required services are not running"; exit 1;
-                       fi' ]
-                initialDelaySeconds: 120
-                periodSeconds: 30
-              readinessProbe: *liveness_probe
-              <% end %>
+              <%= include_template "_job_liveness.yml" %>
             <% end %>
 
             <% if deployment.cloud_secret_name %>
@@ -147,6 +131,11 @@ documents:
             <% end %>
 
             - <<: *logs_container
+
+            <%= include_template "_custom_containers.yml", { pod: 'job' } %>
+
+          volumes:
+            <%= include_template "_volumes.yml", { pod: 'job' } %>
   <% end %>
 
   <% if deployment.replicas_range %>

data/lib/templates/ingress.yml

@@ -1,4 +1,4 @@
-<% if ingress.domain_name %>
+<% if ingress.certificate_name %>
 <%= include_template "certificate.yml" %>
 <% end %>
 
@@ -17,7 +17,7 @@ metadata:
     kubernetes.io/ingress.global-static-ip-name: "<%=ingress.ip_name%>"
     <% end %>
 
-    <% if ingress.domain_name %>
+    <% if ingress.certificate_name %>
     networking.gke.io/managed-certificates: '<%=ingress.certificate_name%>'
     <% end %>
 

data/lib/templates/service.yml

@@ -4,7 +4,7 @@
 apiVersion: cloud.google.com/v1beta1
 kind: BackendConfig
 metadata:
-  name: '<%=service.config_name%>'
+  name: '<%=service.name%>-bconfig'
 spec:
   timeoutSec: 1800
 ---
@@ -16,7 +16,7 @@ metadata:
   name: '<%=service.name%>'
   annotations:
     <% if cloud.name == 'gcloud' %>
-    beta.cloud.google.com/backend-config: '{"ports": {"80":"<%=service.config_name%>"}}'
+    beta.cloud.google.com/backend-config: '{"ports": {"80":"<%=service.name%>-bconfig"}}'
     <% end %>
 spec:
   selector:

data/lib/templates/settings.rb

@@ -6,6 +6,7 @@ settings = {
   deployment: {
     name: app_name,
     replicas: is_production ? 2 : 1,
+    replicas_range: nil, # [min, max] or nil to ignore, sample: is_production ? [1, 2] : nil
     cloud_secret_name: "#{is_production ? 'production' : 'beta'}-cloud-secret",
     cloud_sql_instance: 'xxx:xxx:xxx=tcp:5432', # 5432 => postgres, 3306 => mysql
     env_vars: {}, # Sample: { 'CUSTOM_VAR' => 'value' }
@@ -20,10 +21,10 @@ settings = {
     name: "#{app_name}-ingress",
     ip_name: "#{app_name}-static-ip", # nil if static ip is not necessary
     certificate_name: "#{app_name}-lets-encrypt", # nil if ssl is not required
-    domain_name: is_production ? 'myapp.com' : 'beta.myapp.com' # nil if domain is not required
+    certificate_domain: is_production ? 'myapp.com' : 'beta.myapp.com' # nil if domain is not required
   },
   continuous_deployment: {
-    image_name: "gcr.io/my-account/#{app_name}",
+    image: "gcr.io/my-account/#{app_name}:latest",
     project_name: 'my-project-name',
     cluster_name: 'my-cluster-name',
     cluster_region: 'europe-west4-a',
@@ -36,8 +37,7 @@ settings = {
   service: {
     name: app_name,
     port_name: 'http-port', # max 15 characters
-    backend_port_name: 'b-port', # max 15 characters
-    config_name: "#{app_name}-backend-config"
+    backend_port_name: 'b-port' # max 15 characters
   }
 }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes_helper
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.10.0
 platform: ruby
 authors:
 - owen2345
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-06 00:00:00.000000000 Z
+date: 2022-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erb
@@ -41,8 +41,13 @@ files:
 - lib/kubernetes_helper/version.rb
 - lib/templates/README.md
 - lib/templates/_cd_apply_images.sh
+- lib/templates/_container_extra_settings.yml
+- lib/templates/_custom_containers.yml
+- lib/templates/_custom_volumes.yml
+- lib/templates/_job_liveness.yml
 - lib/templates/_replicas.yml
 - lib/templates/_sidekiq_alive_gem.yml
+- lib/templates/_volumes.yml
 - lib/templates/cd.sh
 - lib/templates/certificate.yml
 - lib/templates/deployment.yml