kubernetes_helper 1.16.0 → 1.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd1b1fbd529561166b5e90b3ece98cd5f9abb81d1d652dcd52d62179daddbd14
-  data.tar.gz: a6bb05e8553de13d43d7b9bc9c46ea5aa1dc03e1adca3fc286cb7b7ed73609ff
+  metadata.gz: 3d37444885845a0d47f9824c6b9e220f278ae7db62de9cc7d3222d3ea91da753
+  data.tar.gz: 64834c2950efa2394c032adfacdd13b8dcb392d7a5346a6101fb7c7c04e30c92
 SHA512:
-  metadata.gz: 5fde4038a234dafaa435aa6aca47348a090bb79d53a3ba3357a3d632b1f85a849322b70851da89674412abe5d679a72feeb30ac0b86b2de83f4dd9bc3224fb8d
-  data.tar.gz: 89ca63330472ad00a1b06004be34c43b7de2582a509247c75d605c5aa6743b64fd653d51fe3459eb1a9d24071faf18a1632663946d274c2ff7dd425a5dc7f220
+  metadata.gz: aa17228790e2b026d284272ae7312d9a241d00646be1fb904b863fec037c266d95e853932d3d4c62147db54e9b167d30a89191f50d8d7ae5da74605b963f7100
+  data.tar.gz: 65a3991a40379500a37789b37af86c258d472d627a48e95f108366050ff9300f7cc4851981e3723092074dbe161d2366935022df629447117a6cce0e7d137609

data/README.md

@@ -22,7 +22,7 @@ Configuration and customization can be done for multiple environments and at any
       Note: The local template will be used instead of the default one.
 
 3. Install/setup the application on kubernetes    
-  Open [.kubernetes/README.md](lib/templates/README.md) to see the instructions (customize the file according to your project and keep it updated)
+  Open [.kubernetes/README.md](lib/templates/README.md) to see the instructions (customize the file according to your project and keep it updated in your repository)
 
 
 ## Settings API
@@ -49,15 +49,24 @@ Configuration and customization can be done for multiple environments and at any
 - `deployment.logs_resources` (Hash, optional): Configure depending on the app requirements. Default: `{ cpu: { max: '200m', min: '50m' }, mem: { max: '200Mi', min: '50Mi' } }`
 
 ### Application deployment.yml for jobs or services without internet interaction (Optional)
-- `deployment.job_name` (String, optional): Job deployment name (Note: Underscores are not accepted). Sample: `my-app-job`. Note: This deployment is created only if this value is present
-- `deployment.job_command` (String, optional): Bash command to be used for job container. Sample: `bundle exec sidekiq`
-- `deployment.job_sidekiq_alive_gem` (Boolean, default false): If true will add liveness checker settings using `sidekiq_alive_gem` (`sidekiq_alive` gem needs to be present in your Gemfile)
-- `deployment.job_services` (Array, Optional, only `job_sidekiq_alive_gem` or `job_services` is allowed): List of linux service names that are required for a healthy job container. Sample: `['sidekiq', 'cron']` 
-- `deployment.job_resources` (Hash, optional): Configure depending on the job app requirements. Sample: `{ cpu: { max: '1', min: '500m' }, mem: { max: '1Gi', min: '500Mi' } }`
+- `deployment.job_apps[].name` (String, optional): Job deployment name (Note: Underscores are not accepted). Sample: `my-app-job`. Note: This deployment is created only if this value is present
+- `deployment.job_apps[].command` (String, optional): Bash command to be used for job container. Sample: `bundle exec sidekiq`
+- `deployment.job_apps[].sidekiq_alive_gem` (Boolean, default false): If true will add liveness checker settings using `sidekiq_alive_gem` (`sidekiq_alive` gem needs to be present in your Gemfile)
+- `deployment.job_apps[].services` (Array, Optional): List of linux service names that are required for a healthy job container. Sample: `['sidekiq', 'cron']`. Note: This will be ignored if `sidekiq_alive_gem` was defined.     
+- `deployment.job_apps[].resources` (Hash, optional): Configure depending on the job app requirements. Sample: `{ cpu: { max: '1', min: '500m' }, mem: { max: '1Gi', min: '500Mi' } }`
+
+### Required settings for Cronjob apps (Note: Cronjobs do not support `sidekiq_alive_gem` and `services`)
+- `deployment.job_apps[].schedule` (String): Cron schedule. Sample: `*/5 * * * *`
+- `deployment.job_apps[].kind` (String, default `Deployment`): Kind of job application [`Deployment` or `CronJob`]
+- `deployment.job_apps[].concurrency_policy` (String, default `Forbid`): [Documentation](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#concurrency-policy)
+- `deployment.job_apps[].suspend` (String, default `false`): If `true` then marks as finished the job application (stops creating new pods).
 
 ### Applications secrets.yml (Optional)
 - `secrets.name` (String): K8s secrets name where env vars will be saved and fetched from. Sample: `my-app-secrets`
-
+- `secrets.import_all_secrets` (Boolean, default false): 
+  - `true`: Allows k8s to auto import all secrets from `secrets.yml` as env values for the apps (No longer needed to update `deployment.yml` everytime there is a new env var)
+  - `false`: Permits to the gem to auto include each secret from `secrets.yml -> data` as env value for the apps (base64 encoded values. Requires to update `deployment.yml` everytime there is a new env var)
+    
 ### Application service.yml (Optional)
 - `service.name`: K8s service name. Sample: `my-app-service`
 - `service.port_name` (String, default `http-port`): Http port name to connect between k8s ingress and service. Sample: `http-port`. Note: max 15 characters
@@ -88,7 +97,7 @@ Configuration and customization can be done for multiple environments and at any
 - `continuous_deployment.update_deployment` (Boolean, default: false): If true permits to re-generate and update the k8s deployment(s) before applying the new version (new docker image)
 
 ### Gem templating partials
-- `_container_extra_settings.yml` Partial template to add custom container settings. Receives `pod` as local variable (`web` | `job` | `cloudsql` | `logs`). Sample:
+- `_container_extra_settings.yml` Partial template to add custom container settings. Receives `pod` as local variable (`web` | `job` | `cloudsql` | `logs`) and `pod_name`. Sample:
   ```yaml
                <% if locals[:pod] == 'job' %>
                resources:
@@ -100,7 +109,7 @@ Configuration and customization can be done for multiple environments and at any
                    memory: 1Gi
                <% end %>
   ``` 
-- `_custom_containers.yml` Partial template to add extra containers (Receives `pod` as local variable: `web` | `job`). Sample:
+- `_custom_containers.yml` Partial template to add extra containers (Receives `pod` as local variable: `web` | `job`) and `pod_name`. Sample:
 ```yaml
            <% if locals[:pod] == 'job' %>
            - name: scraper

data/lib/kubernetes_helper/core.rb

@@ -67,7 +67,7 @@ module KubernetesHelper
     def import_secrets(path, secrets_name)
       path = KubernetesHelper.settings_path(path)
       data = YAML.load(File.read(path)) # rubocop:disable Security/YAMLLoad
-      data['data'].keys.map do |secret|
+      (data['data'] || {}).keys.map do |secret|
         {
           'name' => secret.upcase,
           'valueFrom' => { 'secretKeyRef' => { 'name' => secrets_name, 'key' => secret } }
@@ -78,6 +78,7 @@ module KubernetesHelper
     def render_template(template_name, locals = {})
       path = KubernetesHelper.settings_path(template_name, use_template: true)
       text = "\n#{File.read(path)}"
+      text = text.gsub("\n", "\n#{'  ' * locals[:tab]}") if locals[:tab]
       replace_config_variables(text, locals)
     end
 
@@ -94,12 +95,13 @@ module KubernetesHelper
 
     # parse secrets auto importer
     def parse_import_secrets(document) # rubocop:disable Metrics/AbcSize
-      containers = document.dig('spec', 'template', 'spec', 'containers') || []
+      cronjob_containers = document.dig('spec', 'jobTemplate', 'spec', 'template', 'spec', 'containers')
+      containers = document.dig('spec', 'template', 'spec', 'containers') || cronjob_containers || []
       containers.each do |container|
         container['env'] = (container['env'] || [])
-        container['env'] = container['env'] + static_env_vars if container.delete('static_env')
+        container['env'] = (container['env'] + static_env_vars).uniq if container.delete('static_env')
         if container['import_secrets']
-          container['env'] = container['env'] + import_secrets(*container['import_secrets'])
+          container['env'] = (container['env'] + import_secrets(*container['import_secrets'])).uniq
           container.delete('import_secrets')
         end
       end

data/lib/kubernetes_helper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KubernetesHelper
-  VERSION = '1.16.0'
+  VERSION = '1.19.0'
 end

data/lib/kubernetes_helper.rb

@@ -25,7 +25,8 @@ module KubernetesHelper
       deployment: {
         log_container: true,
         log_folder: '/app/log',
-        external_secrets: {}
+        external_secrets: {},
+        job_apps: settings[:job_apps] || job_apps_from_old_settings(settings)
       },
       service: {
         port_name: 'http-port',
@@ -75,4 +76,18 @@ module KubernetesHelper
       FileUtils.cp(templates_path(name), path) unless File.exist?(path)
     end
   end
+
+  def self.job_apps_from_old_settings(settings)
+    return [] unless settings[:deployment][:job_name]
+
+    [
+      {
+        name: settings[:deployment][:job_name],
+        command: settings[:deployment][:job_command],
+        services: settings[:deployment][:job_services],
+        resources: settings[:deployment][:job_resources],
+        sidekiq_alive_gem: settings[:deployment][:job_sidekiq_alive_gem]
+      }
+    ]
+  end
 end

data/lib/templates/README.md

@@ -25,9 +25,8 @@
     DEPLOY_ENV=beta kubernetes_helper run_command "kubectl create secret generic <%=deployment.cloud_secret_name%> --from-file=credentials.json=<path-to-downloaded/credentials.json>"
     ```
     
-- Register manually env vars (values must be encrypted using base64)     
+- Register manually env vars     
     Open and register secret values in `.kubernetes/secrets.yml`     
-    Note: Enter base64 encoded values      
     ```bash
     DEPLOY_ENV=beta kubernetes_helper run_yml 'secrets.yml' 'kubectl create'
     # kubectl get secrets # to list all secrets registered

data/lib/templates/_cronjobs_pod.yml

@@ -0,0 +1,40 @@
+  - apiVersion: batch/v1
+    kind: CronJob
+    metadata:
+      name: &cronjob_name <%= locals[:job_app][:name] %>
+    spec:
+      schedule: "<%= locals[:job_app][:schedule] %>"
+      concurrencyPolicy: "<%= locals[:job_app][:concurrency_policy] || 'Forbid' %>"
+      suspend: <%= locals[:job_app][:suspend] || false %>
+      # startingDeadlineSeconds: 200
+      jobTemplate:
+        spec:
+          template:
+            spec:
+              <<: *template_spec
+              restartPolicy: OnFailure
+              containers:
+                - <<: *app_container
+                  name: *cronjob_name
+                  <% if locals[:job_app][:command] %>
+                  command: [ "/bin/bash", "-c", "<%= locals[:job_app][:command] %>" ]
+                  <% end %>
+                  <% if locals[:job_app][:resources] %>
+                  <%= include_template "_resources.yml", locals[:job_app][:resources].merge(tab: 2) %>
+                  <% end %>
+                  livenessProbe: null
+                  readinessProbe: null
+                  <%= include_template "_container_extra_settings.yml", { pod: 'job', pod_name: locals[:job_app][:name], tab: 2 } %>
+
+                <% if deployment.cloud_secret_name %>
+                - *cloudsql_container
+                <% end %>
+
+                <% if deployment.log_container %>
+                - <<: *logs_container
+                <% end %>
+
+              <%= include_template "_custom_containers.yml", { pod: 'job', pod_name: locals[:job_app][:name], tab: 2 } %>
+
+              volumes:
+                <%= include_template "_volumes.yml", { pod: 'job', pod_name: locals[:job_app][:name], tab: 2 } %>

data/lib/templates/_jobs_pod.yml

@@ -0,0 +1,47 @@
+  - apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: &job_app_name <%= locals[:job_app][:name] %>
+    spec:
+      replicas: 1
+      selector:
+        matchLabels:
+          name: *job_app_name
+      strategy:
+        type: Recreate
+      minReadySeconds: 10
+      template:
+        metadata:
+          labels:
+            name: *job_app_name
+        spec:
+          <<: *template_spec
+          containers:
+            - <<: *app_container
+              name: *job_app_name
+              <% if locals[:job_app][:command] %>
+              command: [ "/bin/bash", "-c", "<%= locals[:job_app][:command] %>" ]
+              <% end %>
+              <% if locals[:job_app][:resources] %>
+              <%= include_template "_resources.yml", locals[:job_app][:resources] %>
+              <% end %>
+            <%= include_template "_container_extra_settings.yml", { pod: 'job', pod_name: locals[:job_app][:name] } %>
+
+            <% if locals[:job_app][:sidekiq_alive_gem] %>
+            <%= include_template "_sidekiq_alive_gem.yml" %>
+            <% else %>
+            <%= include_template "_job_liveness.yml" %>
+            <% end %>
+
+            <% if deployment.cloud_secret_name %>
+            - *cloudsql_container
+            <% end %>
+
+            <% if deployment.log_container %>
+            - <<: *logs_container
+            <% end %>
+
+          <%= include_template "_custom_containers.yml", { pod: 'job', pod_name: locals[:job_app][:name] } %>
+
+          volumes:
+            <%= include_template "_volumes.yml", { pod: 'job', pod_name: locals[:job_app][:name] } %>

data/lib/templates/_volumes.yml

@@ -18,4 +18,4 @@
                 <% end %>
               <% end %>
 
-            <%= include_template "_custom_volumes.yml", { pod: locals[:pod] } %>
+            <%= include_template "_custom_volumes.yml", { pod: locals[:pod], tab: locals[:tab] } %>

data/lib/templates/cd.sh

@@ -5,7 +5,7 @@ set -e
 SCRIPT_DIR=`dirname "$(realpath -s "$0")"` # app_dir/.kubernetes/
 cd "$SCRIPT_DIR/../" # project directory
 
-DEPLOYMENTS="<%=[deployment.job_name, deployment.name].join(',')%>"
+DEPLOYMENTS="<%=(deployment.job_apps.map { |a| a[:name] } + [deployment.name]).join(',')%>"
 IMAGE_NAME="<%=continuous_deployment.image_name%>"
 CLUSTER_NAME="<%=continuous_deployment.cluster_name%>"
 PROJECT_NAME="<%=continuous_deployment.project_name%>"
@@ -21,7 +21,7 @@ LATEST_NAME="${IMAGE_NAME}:<%= continuous_deployment.image_tag || 'latest' %>"
 
 ## Update new secrets defined in secrets.yml as ENV vars for deployments
 <% if continuous_deployment.update_deployment %>
-  kubernetes_helper run_yml 'deployment.yml' 'kubectl apply'
+  DEPLOY_IMAGE_TAG=$CI_COMMIT_SHA kubernetes_helper run_yml 'deployment.yml' 'kubectl apply'
 <% end %>
 
 ## Apply deployments

data/lib/templates/deployment.yml

@@ -3,7 +3,7 @@ documents:
     kind: Deployment
     metadata:
       name: &app_name <%=deployment.name%>
-    spec: &default_spec
+    spec:
       replicas: <%=deployment.replicas%>
       selector:
         matchLabels:
@@ -12,8 +12,8 @@ documents:
         type: RollingUpdate
         rollingUpdate:
           maxSurge: 1
-          maxUnavailable: 1
-      minReadySeconds: 5
+          maxUnavailable: 0
+      minReadySeconds: 10
       template:
         metadata:
           labels:
@@ -21,13 +21,21 @@ documents:
         spec: &template_spec
           containers:
             - &app_container
-              image: '<%= continuous_deployment.image_name %>:<%= continuous_deployment.image_tag || "latest" %>'
+              image: '<%= continuous_deployment.image_name %>:<%= ENV['DEPLOY_IMAGE_TAG'] || continuous_deployment.image_tag || "latest" %>'
               <% if deployment.command %>
               command: ["/bin/bash", "-c", "<%= deployment.command %>"]
               <% end %>
               name: *app_name
+
               static_env: true
+              <% if secrets.import_all_secrets %>
+              envFrom:
+              - secretRef:
+                  name: <%= secrets.name %>
+              <% else %>
               import_secrets: ['secrets.yml', '<%=secrets.name%>']
+              <% end %>
+
               ports:
                 - containerPort: &port <%= deployment.app_port || 3000 %>
                   name: '<%=service.backend_port_name || 'b-port'%>'
@@ -98,53 +106,9 @@ documents:
           volumes:
             <%= include_template "_volumes.yml", { pod: 'web' } %>
 
-
-  <% if deployment.job_name %>
-  - apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: &job_app_name <%=deployment.job_name%>
-    spec:
-      <<: *default_spec
-      replicas: 1
-      selector:
-        matchLabels:
-          name: *job_app_name
-      template:
-        metadata:
-          labels:
-            name: *job_app_name
-        spec:
-          <<: *template_spec
-          containers:
-            - <<: *app_container
-              name: *job_app_name
-              <% if deployment.job_command %>
-              command: [ "/bin/bash", "-c", "<%= deployment.job_command %>" ]
-              <% end %>
-              <% if deployment.job_resources %>
-              <%= include_template "_resources.yml", deployment.job_resources %>
-              <% end %>
-              <%= include_template "_container_extra_settings.yml", { pod: 'job' } %>
-
-            <% if deployment.job_sidekiq_alive_gem %>
-              <%= include_template "_sidekiq_alive_gem.yml" %>
-            <% else %>
-              <%= include_template "_job_liveness.yml" %>
-            <% end %>
-
-            <% if deployment.cloud_secret_name %>
-            - *cloudsql_container
-            <% end %>
-
-            <% if deployment.log_container %>
-            - <<: *logs_container
-            <% end %>
-
-            <%= include_template "_custom_containers.yml", { pod: 'job' } %>
-
-          volumes:
-            <%= include_template "_volumes.yml", { pod: 'job' } %>
+  <% deployment.job_apps.each do |job_app| %>
+  <%= include_template '_jobs_pod.yml', job_app: job_app if job_app[:kind] != 'CronJob' %>
+  <%= include_template '_cronjobs_pod.yml', job_app: job_app if job_app[:kind] == 'CronJob' %>
   <% end %>
 
   <% if deployment.replicas_range %>

data/lib/templates/secrets.yml

@@ -1,4 +1,3 @@
-# Every Value has to be base64 encoded
 # IMPORTANT: For security reason, never ever commit secret values, only keys
 
 apiVersion: v1
@@ -6,6 +5,6 @@ kind: Secret
 metadata:
   name: '<%=secrets.name%>'
 type: Opaque
-data:
-  my_key1: "based_64_value_encoded"
-  my_key2: "based_64_value_encoded"
+stringData:
+  MY_KEY1: "my value1"
+  MY_KEY2: "my value2"

data/lib/templates/settings.rb

@@ -12,13 +12,18 @@ settings = {
     env_vars: {}, # Sample: { 'CUSTOM_VAR' => 'value' }
     # command: '', # custom container command (default empty to be managed by Dockerfile)
     # liveness_path: '/check_liveness', # nil if not exist
-    # job_name: "#{app_name}-job", # enable if there is any background service
-    # job_command: 'bundle exec sidekiq -C config/sidekiq.yml',
-    # job_services: ['sidekiq', 'cron'] # list of linux services needed.
     # custom_volumes: { my_volume: { kind: 'hostPath', mount_path: '/', settings: { path: '..', type: 'Directory' } }  }
+    job_apps: [
+      # {
+      #   name: "#{app_name}-job", # enable if there is any background service
+      #   command: 'bundle exec sidekiq -C config/sidekiq.yml',
+      #   services: ['sidekiq', 'cron'] # list of linux services needed.
+      # }
+    ]
   },
   secrets: {
-    name: "#{app_name}-secrets"
+    name: "#{app_name}-secrets",
+    import_all_secrets: true
   },
   service: {
     name: app_name,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes_helper
 version: !ruby/object:Gem::Version
-  version: 1.16.0
+  version: 1.19.0
 platform: ruby
 authors:
 - owen2345
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-28 00:00:00.000000000 Z
+date: 2022-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erb
@@ -44,9 +44,11 @@ files:
 - lib/templates/_cd_digital.sh
 - lib/templates/_cd_google.sh
 - lib/templates/_container_extra_settings.yml
+- lib/templates/_cronjobs_pod.yml
 - lib/templates/_custom_containers.yml
 - lib/templates/_custom_volumes.yml
 - lib/templates/_job_liveness.yml
+- lib/templates/_jobs_pod.yml
 - lib/templates/_replicas.yml
 - lib/templates/_resources.yml
 - lib/templates/_sidekiq_alive_gem.yml