@flowfuse/driver-kubernetes 2.22.2-d3f9800-202510171249.0 → 2.23.0

package/.github/workflows/release-publish.yml

@@ -9,7 +9,7 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-            - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
+            - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
               with:
                   node-version: 18
             - run: npm ci

package/CHANGELOG.md

@@ -1,3 +1,12 @@
+#### 2.23.0: Release
+
+ - Bump actions/setup-node from 5.0.0 to 6.0.0 (#251)
+ - Use new cache system for in-memory state (#243) @hardillb
+ - feat: Improve projects ingress annotations logic (#242) @ppawlowski
+ - feat: Add possibility to configure container security context (#250) @ppawlowski
+ - feat: Add possibility to configure probes (#248) @ppawlowski
+ - Update container name validator (#247) @hardillb
+
 #### 2.22.1: Release
 
  - Bump JS-DevTools/npm-publish from 4.0.1 to 4.1.1 (#240)

package/README.md

@@ -65,6 +65,7 @@ AWS EKS specific annotation for ALB Ingress. or `openshift` to allow running on
 - `storage.storageClassEFSTag` Used instead of `storage.storageClass` when needing to shard across multiple EFS file systems (default not set)
 - `storage.size` Size of the volume to request (default not set)
 - `podSecurityContext` Settings linked to the [security context of the pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)
+- `containerSecurityContext` Settings linked to the [security context of the container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)
 - `service.type` Type of service to create for the editor (allowed `ClusterIP` or `NodePort`, default `ClusterIP`)
 
 Expects to pick up K8s credentials from the environment

package/kubernetes.js

@@ -194,6 +194,14 @@ const createDeployment = async (project, options) => {
         this._app.log.info('[k8s] OpenShift, removing PodSecurityContext')
     }
 
+    if (this._app.config.driver.options?.containerSecurityContext) {
+        localPod.spec.containers[0].securityContext = this._app.config.driver.options.containerSecurityContext
+        this._app.log.info(`[k8s] Using custom ContainerSecurityContext ${JSON.stringify(this._app.config.driver.options.containerSecurityContext)}`)
+    } else if (this._app.license.active() && this._cloudProvider === 'openshift') {
+        localPod.spec.containers[0].securityContext = {}
+        this._app.log.info('[k8s] OpenShift, removing ContainerSecurityContext')
+    }
+
     if (stack.memory && stack.cpu) {
         localPod.spec.containers[0].resources.requests.memory = `${stack.memory}Mi`
         // increase limit to give npm more room to run in
@@ -271,8 +279,37 @@ const createIngress = async (project, options) => {
 
     const localIngress = JSON.parse(JSON.stringify(ingressTemplate))
 
+    let addIngressTls = false
+
     if (this._certManagerIssuer) {
         localIngress.metadata.annotations['cert-manager.io/cluster-issuer'] = this._certManagerIssuer
+        addIngressTls = true
+
+        // Add non-cert-manager annotations from projectIngressAnnotations if they exist
+        if (this._projectIngressAnnotations) {
+            Object.keys(this._projectIngressAnnotations).forEach((key) => {
+                if (!key.startsWith('cert-manager.io/')) {
+                    localIngress.metadata.annotations[key] = this._projectIngressAnnotations[key]
+                }
+            })
+        }
+    } else if (this._projectIngressAnnotations) {
+        const hasCertManagerAnnotation = Object.keys(this._projectIngressAnnotations).some(key =>
+            key.startsWith('cert-manager.io/')
+        )
+
+        if (hasCertManagerAnnotation) {
+            addIngressTls = true
+        }
+
+        // Add all annotations from projectIngressAnnotations
+        Object.keys(this._projectIngressAnnotations).forEach((key) => {
+            localIngress.metadata.annotations[key] = this._projectIngressAnnotations[key]
+        })
+    }
+
+    // Add TLS configuration if needed
+    if (addIngressTls) {
         localIngress.spec.tls = [
             {
                 hosts: [
@@ -319,8 +356,37 @@ const createCustomIngress = async (project, hostname, options) => {
     customIngress.spec.rules[0].host = hostname
     customIngress.spec.rules[0].http.paths[0].backend.service.name = `${prefix}${project.safeName}`
 
+    let addCustomIngressTls = false
+
     if (this._customHostname?.certManagerIssuer) {
         customIngress.metadata.annotations['cert-manager.io/cluster-issuer'] = this._customHostname.certManagerIssuer
+        addCustomIngressTls = true
+
+        // Add non-cert-manager annotations from projectIngressAnnotations if they exist
+        if (this._customHostname?.ingressAnnotations) {
+            Object.keys(this._customHostname?.ingressAnnotations).forEach((key) => {
+                if (!key.startsWith('cert-manager.io/')) {
+                    customIngress.metadata.annotations[key] = this._customHostname?.ingressAnnotations[key]
+                }
+            })
+        }
+    } else if (this._customHostname?.ingressAnnotations) {
+        const hasCertManagerAnnotation = Object.keys(this._customHostname?.ingressAnnotations).some(key =>
+            key.startsWith('cert-manager.io/')
+        )
+
+        if (hasCertManagerAnnotation) {
+            addCustomIngressTls = true
+        }
+
+        // Add all annotations from projectIngressAnnotations
+        Object.keys(this._customHostname?.ingressAnnotations).forEach((key) => {
+            customIngress.metadata.annotations[key] = this._customHostname?.ingressAnnotations[key]
+        })
+    }
+
+    // Add TLS configuration if needed
+    if (addCustomIngressTls) {
         customIngress.spec.tls = [
             {
                 hosts: [
@@ -535,7 +601,9 @@ const createProject = async (project, options) => {
     project.state = 'running'
     await project.save()
 
-    this._projects[project.id].state = 'starting'
+    const cachedProject = await this._projects.get(project.id)
+    cachedProject.state = 'starting'
+    await this._projects.set(project.id, cachedProject)
 }
 
 const getEndpoints = async (project) => {
@@ -631,13 +699,14 @@ module.exports = {
             throw Error('Failed to load Kubernetes node client', { cause: err })
         }
         this._app = app
-        this._projects = {}
+        this._projects = app.caches.getCache('driver-k8s-projects') // {}
         this._options = options
 
         this._namespace = this._app.config.driver.options?.projectNamespace || 'flowforge'
         this._k8sDelay = this._app.config.driver.options?.k8sDelay || 1000
         this._k8sRetries = this._app.config.driver.options?.k8sRetries || 10
         this._certManagerIssuer = this._app.config.driver.options?.certManagerIssuer
+        this._projectIngressAnnotations = this._app.config.driver.options?.projectIngressAnnotations
         this._logPassthrough = this._app.config.driver.options?.logPassthrough || false
         this._cloudProvider = this._app.config.driver.options?.cloudProvider
         if (this._app.config.driver.options?.customHostname?.enabled) {
@@ -673,22 +742,25 @@ module.exports = {
                 'TeamId'
             ]
         })
-        projects.forEach(async (project) => {
-            if (this._projects[project.id] === undefined) {
-                this._projects[project.id] = {
+        for (const project of projects) {
+            if (await this._projects.get(project.id) === undefined) {
+                await this._projects.set(project.id, {
                     state: 'unknown'
-                }
+                })
             }
-        })
+        }
 
         this._initialCheckTimeout = setTimeout(async () => {
             this._app.log.debug('[k8s] Restarting projects')
             const namespace = this._namespace
-            projects.forEach(async (project) => {
+            for (const project of projects) {
                 try {
                     if (project.state === 'suspended') {
                         // Do not restart suspended projects
-                        return
+                        const cachedProject = await this._projects.get(project.id)
+                        cachedProject.state = 'suspened'
+                        await this._projects.set(project.id, cachedProject)
+                        continue
                     }
 
                     // need to upgrade bare pods to deployments
@@ -742,7 +814,7 @@ module.exports = {
                 } catch (err) {
                     this._app.log.error(`[k8s] Instance ${project.id} - error resuming project: ${err.stack}`)
                 }
-            })
+            }
 
             // get list of all MQTTBrokers
             if (this._app.db.models.BrokerCredentials) {
@@ -751,7 +823,7 @@ module.exports = {
                 })
 
                 // Check restarting MQTT-Schema-Agent
-                brokers.forEach(async (broker) => {
+                for (const broker of brokers) {
                     const agent = broker.constructor.name === 'TeamBrokerAgent'
                     if (broker.Team && broker.state === 'running') {
                         try {
@@ -765,9 +837,9 @@ module.exports = {
                             await createMQTTTopicAgent(broker)
                         }
                     }
-                })
+                }
             }
-        }, 1000)
+        }, Math.floor(1000 + (Math.random() * 5))) // space this out so if 2 instances running they shouldn't run at the same time
 
         // need to work out what we can expose for K8s
         return {
@@ -802,9 +874,9 @@ module.exports = {
      * @return {forge.containers.Project}
      */
     start: async (project) => {
-        this._projects[project.id] = {
+        await this._projects.set(project.id, {
             state: 'starting'
-        }
+        })
 
         // Rather than await this promise, we return it. That allows the wrapper
         // to respond to the create request much quicker and the create can happen
@@ -824,7 +896,9 @@ module.exports = {
      */
     stop: async (project) => {
         // Stop the project
-        this._projects[project.id].state = 'stopping'
+        const cachedProject = await this._projects.get(project.id)
+        cachedProject.state = 'stopping'
+        await this._projects.set(project.id, cachedProject)
 
         try {
             await this._k8sNetApi.deleteNamespacedIngress({ name: project.safeName, namespace: this._namespace })
@@ -838,6 +912,17 @@ module.exports = {
             } catch (err) {
                 this._app.log.error(`[k8s] Instance ${project.id} - error deleting tls secret: ${err.toString()} ${err.stack}`)
             }
+        } else if (this._projectIngressAnnotations) {
+            const hasCertManagerAnnotation = Object.keys(this._projectIngressAnnotations).some(key =>
+                key.startsWith('cert-manager.io/')
+            )
+            if (hasCertManagerAnnotation) {
+                try {
+                    await this._k8sApi.deleteNamespacedSecret({ name: project.safeName, namespace: this._namespace })
+                } catch (err) {
+                    this._app.log.error(`[k8s] Instance ${project.id} - error deleting tls secret: ${err.toString()} ${err.stack}`)
+                }
+            }
         }
 
         if (this._customHostname?.enabled) {
@@ -928,7 +1013,8 @@ module.exports = {
         //     }
         // }
 
-        this._projects[project.id].state = 'suspended'
+        cachedProject.state = 'suspended'
+        await this._projects.set(project.id, cachedProject)
         return new Promise((resolve, reject) => {
             let counter = 0
             const pollInterval = setInterval(async () => {
@@ -969,6 +1055,17 @@ module.exports = {
             } catch (err) {
                 this._app.log.error(`[k8s] Instance ${project.id} - error deleting tls secret: ${err.toString()}`)
             }
+        } else if (this._projectIngressAnnotations) {
+            const hasCertManagerAnnotation = Object.keys(this._projectIngressAnnotations).some(key =>
+                key.startsWith('cert-manager.io/')
+            )
+            if (hasCertManagerAnnotation) {
+                try {
+                    await this._k8sApi.deleteNamespacedSecret({ name: project.safeName, namespace: this._namespace })
+                } catch (err) {
+                    this._app.log.error(`[k8s] Instance ${project.id} - error deleting tls secret: ${err.toString()}`)
+                }
+            }
         }
         if (this._customHostname?.enabled) {
             try {
@@ -976,7 +1073,7 @@ module.exports = {
             } catch (err) {
                 this._app.log.error(`[k8s] Instance ${project.id} - error deleting custom ingress: ${err.toString()}`)
             }
-            if (this._customHostname?.certManagerIssuer) {
+            if (this._customHostname?.certManagerIssuer || this._customHostname?.certManagerAnnotations) {
                 try {
                     await this._k8sApi.deleteNamespacedSecret({ name: `${project.safeName}-custom`, namespace: this._namespace })
                 } catch (err) {
@@ -1019,7 +1116,7 @@ module.exports = {
                 // console.log(err)
             }
         }
-        delete this._projects[project.id]
+        await this._projects.del(project.id)
     },
     /**
      * Retrieves details of a project's container
@@ -1027,14 +1124,15 @@ module.exports = {
      * @return {Object}
      */
     details: async (project) => {
-        if (this._projects[project.id] === undefined) {
+        const cachedProject = await this._projects.get(project.id)
+        if (cachedProject === undefined) {
             return { state: 'unknown' }
         }
-        if (this._projects[project.id].state === 'suspended') {
+        if (cachedProject.state === 'suspended') {
             // We should only poll the launcher if we think it is running.
             // Otherwise, return our cached state
             return {
-                state: this._projects[project.id].state
+                state: cachedProject.state
             }
         }
         const prefix = project.safeName.match(/^[0-9]/) ? 'srv-' : ''
@@ -1048,7 +1146,8 @@ module.exports = {
                 details = await this._k8sAppApi.readNamespacedDeployment({ name: project.safeName, namespace: this._namespace })
                 if (details.status?.conditions[0].status === 'False') {
                     // return "starting" status until pod it running
-                    this._projects[project.id].state = 'starting'
+                    cachedProject.state = 'starting'
+                    await this._projects.set(project.id, cachedProject)
                     return {
                         id: project.id,
                         state: 'starting',
@@ -1062,7 +1161,8 @@ module.exports = {
                     const infoURL = `http://${prefix}${project.safeName}.${this._namespace}:2880/flowforge/info`
                     try {
                         const info = JSON.parse((await got.get(infoURL, { timeout: { request: 1000 } })).body)
-                        this._projects[project.id].state = info.state
+                        cachedProject.state = info.state
+                        await this._projects.set(project.id, cachedProject)
                         return info
                     } catch (err) {
                         this._app.log.debug(`error getting state from instance ${project.id}: ${err}`)
@@ -1084,7 +1184,8 @@ module.exports = {
                 details = await this._k8sApi.readNamespacedPodStatus({ name: project.safeName, namespace: this._namespace })
                 if (details.status?.phase === 'Pending') {
                     // return "starting" status until pod it running
-                    this._projects[project.id].state = 'starting'
+                    cachedProject.state = 'starting'
+                    this._projects.set(project.id, cachedProject)
                     return {
                         id: project.id,
                         state: 'starting',
@@ -1095,7 +1196,8 @@ module.exports = {
                     const infoURL = `http://${prefix}${project.safeName}.${this._namespace}:2880/flowforge/info`
                     try {
                         const info = JSON.parse((await got.get(infoURL, { timeout: { request: 1000 } })).body)
-                        this._projects[project.id].state = info.state
+                        cachedProject.state = info.state
+                        await this._projects.set(project.id, cachedProject)
                         return info
                     } catch (err) {
                         this._app.log.debug(`error getting state from instance ${project.id}: ${err}`)
@@ -1145,7 +1247,8 @@ module.exports = {
      * @return {forge.Status}
      */
     startFlows: async (project) => {
-        if (this._projects[project.id] === undefined) {
+        const cachedProject = await this._projects.get(project.id)
+        if (cachedProject === undefined) {
             return { state: 'unknown' }
         }
         const endpoints = await getEndpoints(project)
@@ -1167,7 +1270,8 @@ module.exports = {
      * @return {forge.Status}
      */
     stopFlows: async (project) => {
-        if (this._projects[project.id] === undefined) {
+        const cachedProject = await this._projects.get(project.id)
+        if (cachedProject === undefined) {
             return { state: 'unknown' }
         }
         const endpoints = await getEndpoints(project)
@@ -1189,7 +1293,8 @@ module.exports = {
      * @return {array} logs
      */
     logs: async (project) => {
-        if (this._projects[project.id] === undefined) {
+        const cachedProject = await this._projects.get(project.id)
+        if (cachedProject === undefined) {
             return { state: 'unknown' }
         }
         if (await project.getSetting('ha')) {
@@ -1215,7 +1320,8 @@ module.exports = {
      * @return {forge.Status}
      */
     restartFlows: async (project) => {
-        if (this._projects[project.id] === undefined) {
+        const cachedProject = await this._projects.get(project.id)
+        if (cachedProject === undefined) {
             return { state: 'unknown' }
         }
         const endpoints = await getEndpoints(project)
@@ -1370,7 +1476,8 @@ module.exports = {
 
     // Resouces api
     resources: async (project) => {
-        if (this._projects[project.id] === undefined) {
+        const cachedProject = await this._projects.get(project.id)
+        if (cachedProject === undefined) {
             return { state: 'unknown' }
         }
         if (await project.getSetting('ha')) {
@@ -1403,7 +1510,8 @@ module.exports = {
         }
     },
     resourcesStream: async (project, socket) => {
-        if (this._projects[project.id] === undefined) {
+        const cachedProject = await this._projects.get(project.id)
+        if (cachedProject === undefined) {
             throw new Error('Cannot get instance resources')
         }
         if (await project.getSetting('ha')) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@flowfuse/driver-kubernetes",
-    "version": "2.22.2-d3f9800-202510171249.0",
+    "version": "2.23.0",
     "description": "Kubernetes driver for FlowFuse",
     "main": "kubernetes.js",
     "scripts": {