@flowfuse/driver-docker 2.14.1 → 2.16.0

package/.github/workflows/build.yml

@@ -13,9 +13,9 @@ jobs:
             matrix:
                 node-version: [16.x]
         steps:
-            - uses: actions/checkout@v4
+            - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
             - name: Use Node.js ${{ matrix.node-version }}
-              uses: actions/setup-node@v4
+              uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4.3.0
               with:
                   node-version: ${{ matrix.node-version }}
             - name: Install Dependencies

package/.github/workflows/release-publish.yml

@@ -8,11 +8,11 @@ jobs:
     publish:
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v4
-            - uses: actions/setup-node@v4
+            - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+            - uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4.3.0
               with:
                   node-version: 18
             - run: npm ci --omit dev
-            - uses: JS-DevTools/npm-publish@v3
+            - uses: JS-DevTools/npm-publish@19c28f1ef146469e409470805ea4279d47c3d35c # v3.1.1
               with:
                   token: ${{ secrets.NPM_PUBLISH_TOKEN }}

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+#### 2.16.0: Release
+
+ - chore: fix lint script (#128) @ppawlowski
+ - Bump tar-fs and dockerode (#127) @app/dependabot
+ - chore: Pin external actions to commit hash (#126) @ppawlowski
+
+#### 2.15.0: Release
+
+ - docs: Remove unused configuration keyfrom README.md (#124) @ppawlowski
+ - Mqtt agent support for Docker (#121) @hardillb
+ - feat: Use docker volume as persistent storage backend for project instances (#123) @ppawlowski
+
 #### 2.14.1: Release
 
 

package/README.md

@@ -17,15 +17,13 @@ driver:
     logPassthrough: true
     storage:
       enabled: true
-      path: /opt/flowfuse/storage
 ```
 
  - `registry` is the Docker Registry to load Stack Containers from (default: Docker Hub)
  - `socket` is the path to the docker unix domain socket (default: /var/run/docker.sock)
  - `privateCA`: is the fully qualified path to a pem file containing trusted CA cert chain (default: not set)
  - `logPassthrough` Have Node-RED logs printed in JSON format to container stdout (default: false)
- - `storage.enabled` enables mounting a directory into each Node-RED instance as persistence storage (default: false)
- - `storage.path` is the fully qualified path to the root directory for the storage on the host (default: not set)
+ - `storage.enabled` enables volume-based persistent storage for Node-RED instance (default: false)
 
 ### Configuration via environment variables
 

package/docker.js

@@ -1,8 +1,6 @@
 const got = require('got')
 const FormData = require('form-data')
 const Docker = require('dockerode')
-const path = require('path')
-const { chownSync, mkdirSync, rmSync } = require('fs')
 
 const createContainer = async (project, domain) => {
     const stack = project.ProjectStack.properties
@@ -105,22 +103,26 @@ const createContainer = async (project, domain) => {
         contOptions.Env.push('NODE_EXTRA_CA_CERTS=/usr/local/ssl-certs/chain.pem')
     }
 
-    if (this._app.config.driver.options?.storage?.enabled && this._app.config.driver.options?.storage?.path) {
+    if (this._app.config.driver.options?.storage?.enabled) {
         try {
-            const localPath = path.join('/opt/persistent-storage', project.id)
-            console.log(`Creating dir in container ${localPath}`)
-            mkdirSync(localPath)
-            chownSync(localPath, 1000, 1000)
+            const volumeName = `flowfuse-${project.id}-storage`
+            await this._docker.createVolume({
+                Name: volumeName,
+                Labels: {
+                    flowfuse: 'project-storage',
+                    'project-id': project.id
+                }
+            })
+
+            if (Array.isArray(contOptions.HostConfig?.Binds)) {
+                contOptions.HostConfig.Binds.push(`${volumeName}:/data/storage`)
+            } else {
+                contOptions.HostConfig.Binds = [
+                    `${volumeName}:/data/storage`
+                ]
+            }
         } catch (err) {
-            this._app.log.info(`[docker] problem creating persistent storage for ${project.id}`)
-        }
-        const projectPath = path.join(this._app.config.driver.options?.storage?.path, project.id)
-        if (Array.isArray(contOptions.HostConfig?.Binds)) {
-            contOptions.HostConfig.Binds.push(`${projectPath}:/data/storage`)
-        } else {
-            contOptions.HostConfig.Binds = [
-                `${projectPath}:/data/storage`
-            ]
+            this._app.log.error(`[docker] problem creating storage volume for ${project.id}: ${err.message}`)
         }
     }
 
@@ -178,6 +180,74 @@ const getStaticFileUrl = async (instance, filePath) => {
     return `http://${instance.id}:2880/flowforge/files/_/${encodeURIComponent(filePath)}`
 }
 
+const createMQttTopicAgent = async (broker) => {
+    const image = this._app.config.driver.options?.mqttSchemaContainer || `${this._app.config.driver.options?.registry ? this._app.config.driver.options.registry + '/' : ''}flowfuse/mqtt-schema-agent`
+    const name = `mqtt-schema-agent-${broker.Team.hashid.toLowerCase()}-${broker.hashid.toLowerCase()}`
+    const contOptions = {
+        Image: image,
+        name,
+        Env: [],
+        Labels: {
+            flowforge: 'mqtt-agent'
+        },
+        AttachStdin: false,
+        AttachStdout: false,
+        AttachStderr: false,
+        HostConfig: {
+            NetworkMode: this._network,
+            RestartPolicy: {
+                Name: 'unless-stopped'
+            },
+            NanoCpus: ((10 / 100) * (10 ** 9)), // 10%
+            Memory: (100 * 1024 * 1024) // 100mb
+        }
+    }
+
+    const { token } = await broker.refreshAuthTokens()
+    contOptions.Env.push(`FORGE_TEAM_TOKEN=${token}`)
+    contOptions.Env.push(`FORGE_URL=${this._app.config.api_url}`)
+    contOptions.Env.push(`FORGE_BROKER_ID=${broker.hashid}`)
+    contOptions.Env.push(`FORGE_TEAM_ID=${broker.Team.hashid}`)
+
+    const containerList = await this._docker.listImages()
+    let containerFound = false
+    let stackName = image
+    if (stackName.indexOf(':') === -1) {
+        stackName = stackName + ':latest'
+    }
+    for (const cont of containerList) {
+        if (cont.RepoTags.includes(stackName)) {
+            containerFound = true
+            break
+        }
+    }
+    if (!containerFound) {
+        this._app.log.info(`Container for MQTT Schema Agent not found, pulling ${stackName}`)
+        try {
+            await new Promise((resolve, reject) => {
+                this._docker.pull(stackName, (err, stream) => {
+                    if (!err) {
+                        this._docker.modem.followProgress(stream, onFinished)
+                        function onFinished (err, output) {
+                            if (!err) {
+                                resolve(true)
+                                return
+                            }
+                            reject(err)
+                        }
+                    } else {
+                        reject(err)
+                    }
+                })
+            })
+        } catch (err) {
+            this._app.log.debug(`Error pulling image ${stackName} ${err.message}`)
+        }
+    }
+    const container = await this._docker.createContainer(contOptions)
+    await container.start()
+}
+
 /**
  * Docker Container driver
  *
@@ -254,7 +324,7 @@ module.exports = {
             }
         })
 
-        this._initialCheckTimeout = setTimeout(() => {
+        this._initialCheckTimeout = setTimeout(async () => {
             this._app.log.debug('[docker] Restarting projects')
             projects.forEach(async (project) => {
                 try {
@@ -301,6 +371,50 @@ module.exports = {
                     this._app.log.error(`[docker] Project ${project.id} - error resuming project: ${err.stack}`)
                 }
             })
+
+            if (this._app.db.models.BrokerCredentials) {
+                const brokers = await this._app.db.models.BrokerCredentials.findAll({
+                    include: [{ model: this._app.db.models.Team }]
+                })
+
+                brokers.forEach(async (broker) => {
+                    if (broker.Team) {
+                        if (broker.state === 'running') {
+                            const name = `mqtt-schema-agent-${broker.Team.hashid.toLowerCase()}-${broker.hashid.toLowerCase()}`
+                            this._app.log.info(`[docker] Testing MQTT Agent ${name} container exists`)
+                            this._app.log.debug(`${name}`)
+                            let container
+                            try {
+                                container = await this._docker.listContainers({
+                                    all: true,
+                                    filters: {
+                                        name: [name]
+                                    }
+                                })
+                                if (container[0]) {
+                                    container = await this._docker.getContainer(container[0].Id)
+                                } else {
+                                    container = undefined
+                                }
+                                if (container) {
+                                    const state = await container.inspect()
+                                    if (!state.State.Running) {
+                                        this._app.log.info(`[docker] MQTT Agent ${name} - restarting container [${container.id.substring(0, 12)}]`)
+                                        await container.start()
+                                    } else {
+                                        this._app.log.info(`[docker] MQTT Agent ${name} - already running container [${container.id.substring(0, 12)}]`)
+                                    }
+                                } else {
+                                    this._app.log.info(`[docker] MQTT Agent ${name} - recreating container`)
+                                    createMQttTopicAgent(broker)
+                                }
+                            } catch (err) {
+                                console.log(err)
+                            }
+                        }
+                    }
+                })
+            }
         }, 1000)
 
         return {
@@ -372,13 +486,12 @@ module.exports = {
             } catch (err) {}
         }
         if (this._app.config.driver.options?.storage?.enabled) {
-            // need to be sure we have permission to delete the dir and it's contents?
             try {
-                // This is better and assumes that directory is mounted on `/opt/storage`
-                const projectPersistentPath = path.join('/opt/persistent-storage', project.id)
-                rmSync(projectPersistentPath, { recursive: true, force: true })
+                const volumeName = `flowfuse-${project.id}-storage`
+                const volume = this._docker.getVolume(volumeName)
+                await volume.remove()
             } catch (err) {
-                this._app.log.error(`[docker] Project ${project.id} - error deleting persistent storage: ${err.stack}`)
+                this._app.log.error(`[docker] Project ${project.id} - error removing storage volume: ${err.stack}`)
             }
         }
         delete this._projects[project.id]
@@ -612,5 +725,45 @@ module.exports = {
             err.statusCode = err.response.statusCode
             throw err
         }
+    },
+
+    // Broker Agent
+    startBrokerAgent: async (broker) => {
+        createMQttTopicAgent(broker)
+    },
+    stopBrokerAgent: async (broker) => {
+        const name = `mqtt-schema-agent-${broker.Team.hashid.toLowerCase()}-${broker.hashid.toLowerCase()}`
+        try {
+            const container = await this._docker.getContainer(name)
+            await container.stop()
+            await container.remove()
+        } catch (err) {
+            console.log(err)
+        }
+    },
+    getBrokerAgentState: async (broker) => {
+        const name = `mqtt-schema-agent-${broker.Team.hashid.toLowerCase()}-${broker.hashid.toLowerCase()}`
+        try {
+            const status = await got.get(`http://${name}:3500/api/v1/status`).json()
+            return status
+        } catch (err) {
+            return { error: 'error_getting_status', message: err.toString() }
+        }
+    },
+    sendBrokerAgentCommand: async (broker, command) => {
+        const name = `mqtt-schema-agent-${broker.Team.hashid.toLowerCase()}-${broker.hashid.toLowerCase()}`
+        if (command === 'start' || command === 'restart') {
+            try {
+                await got.post(`http://${name}:3500/api/v1/commands/start`)
+            } catch (err) {
+
+            }
+        } else if (command === 'stop') {
+            try {
+                await got.post(`http://${name}:3500/api/v1/commands/stop`)
+            } catch (err) {
+
+            }
+        }
     }
 }

package/package.json

@@ -1,12 +1,12 @@
 {
     "name": "@flowfuse/driver-docker",
-    "version": "2.14.1",
+    "version": "2.16.0",
     "description": "Docker driver for FlowFuse",
     "main": "docker.js",
     "scripts": {
         "test": "echo \"Error: no test specified\" && exit 1",
-        "lint": "eslint -c .eslintrc *.js",
-        "lint:fix": "eslint -c .eslintrc *.js --fix"
+        "lint": "eslint -c .eslintrc \"*.js\"",
+        "lint:fix": "eslint -c .eslintrc \"*.js\" --fix"
     },
     "repository": {
         "type": "git",
@@ -21,7 +21,7 @@
     },
     "license": "Apache-2.0",
     "dependencies": {
-        "dockerode": "^3.3.1",
+        "dockerode": "^4.0.5",
         "form-data": "^4.0.0",
         "got": "^11.8.0"
     },