mikser-io-whitebox 4.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Almero Digital Marketing
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,50 @@
+# mikser-io-whitebox
+
+WhiteBox integration for [Mikser](https://github.com/almero-digital-marketing/mikser-io). Pushes processed entities to a WhiteBox `feed` service and synchronises a watched folder with a WhiteBox `storage` service.
+
+## Install
+
+```bash
+npm install mikser-io-whitebox
+```
+
+## Usage
+
+```js
+// mikser.config.js
+export default {
+  plugins: ['whitebox'],
+  whitebox: {
+    context: 'my-project',
+    services: {
+      feed: {
+        url: 'https://feed.example.com',
+        token: 'FEED_TOKEN',
+        expire: '10 days',
+        match: (entity) => entity.type === 'document'
+      },
+      storage: {
+        url: 'https://storage.example.com',
+        token: 'STORAGE_TOKEN',
+        storageFolder: 'storage',
+        expire: '10 days',
+        match: (entity) => entity.id.startsWith('/storage/')
+      }
+    }
+  }
+}
+```
+
+Either service is optional — omit `services.feed` or `services.storage` to disable that half. When `context` is not set, the plugin falls back to a per-machine id (`machineId_hostname_username`).
+
+### Feed
+
+Catalogs entities matching `feed.match` (default: `entity.type === 'document'`) into the WhiteBox feed on every `processed` phase, and expires/clears the cache after each run.
+
+### Storage
+
+Watches `storageFolder` (default `storage/`) and uploads matching entities — by source on `processed`, and by render output on `finalize`. `storage.match` defaults to `entity.id` containing `/storage/`. Imports existing files on startup; with `--clear` the remote storage is wiped first.
+
+## License
+
+MIT

package/index.js

@@ -0,0 +1,49 @@
+import { hostname, userInfo } from 'os'
+import axios from 'axios'
+import MID from 'node-machine-id'
+
+import feed from './src/feed.js'
+import storage from './src/storage.js'
+
+export default (core) => {
+    const { runtime, useLogger, onLoaded } = core
+
+    async function whiteboxApi(service, route, data) {
+        const logger = useLogger()
+        const { services } = runtime.config.whitebox
+        const { url, token } = services[service]
+        if (!url || !token) return
+
+        try {
+            const response = await axios.post(url + route + '?stamp=' + Date.now(), data, {
+                headers: {
+                    Authorization: 'Bearer ' + token,
+                }
+            })
+            if (response.data.success) return response.data
+        } catch (err) {
+            logger.error(err, 'WhiteBox system error: %s %o', route, data)
+        }
+    }
+
+    let machineId
+    async function useMachineId() {
+        if (!machineId) {
+            machineId = await MID.machineId() + '_' + hostname() + '_' + userInfo().username
+        }
+        return machineId
+    }
+
+    onLoaded(() => {
+        const logger = useLogger()
+        logger.info('WhiteBox context: %s', runtime.config.whitebox?.context)
+    })
+
+    return {
+        whiteboxApi,
+        useMachineId,
+        ...feed({ ...core, whiteboxApi, useMachineId }),
+        ...storage({ ...core, whiteboxApi, useMachineId }),
+    }
+
+}

package/package.json

@@ -0,0 +1,33 @@
+{
+	"name": "mikser-io-whitebox",
+	"version": "4.0.0",
+	"description": "",
+	"main": "index.js",
+	"type": "module",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/almero-digital-marketing/mikser-io-whitebox.git"
+	},
+	"author": "",
+	"license": "MIT",
+	"bugs": {
+		"url": "https://github.com/almero-digital-marketing/mikser-io-whitebox/issues"
+	},
+	"homepage": "https://github.com/almero-digital-marketing/mikser-io-whitebox#readme",
+	"peerDependencies": {
+		"mikser-io": "^6.0.0"
+	},
+	"dependencies": {
+		"aguid": "^2.0.0",
+		"axios": "^1.16.0",
+		"form-data": "^4.0.5",
+		"globby": "^16.2.0",
+		"lodash": "^4.18.1",
+		"node-machine-id": "^1.1.12",
+		"p-map": "^7.0.4",
+		"uuid": "^14.0.0"
+	}
+}

package/src/feed.js

@@ -0,0 +1,108 @@
+import pMap from 'p-map'
+import { v1 as uuidv1 } from 'uuid'
+import aguid from 'aguid'
+import _ from 'lodash'
+
+export default ({
+    runtime,
+    onProcessed,
+    useLogger,
+    useJournal,
+    onLoaded,
+    whiteboxApi,
+    useMachineId,
+    constants: { OPERATION },
+}) => {
+    let types = new Set()
+
+    async function expireCatalog() {
+        const { context } = runtime.config.whitebox
+        for (let type of types) {
+            await whiteboxApi('feed', '/api/catalog/expire', {
+                context: context || await useMachineId(),
+                stamp: runtime.stamp,
+                type
+            })
+        }
+    }
+
+    async function clearCache() {
+        const logger = useLogger()
+        logger.debug('WhiteBox feed %s: %s', 'clear', 'cache')
+        const { context } = runtime.config.whitebox
+        let data = {
+            context: context || await useMachineId()
+        }
+        return whiteboxApi('feed', '/api/catalog/clear/cache', data)
+    }
+
+    onLoaded(async () => {
+        const logger = useLogger()
+        if (runtime.options.clear) {
+            const { context } = runtime.config.whitebox
+            const data = {
+                context: context || await useMachineId()
+            }
+
+            logger.debug('WhiteBox feed %s: %s', 'clear', 'catalog')
+            await whiteboxApi('feed', '/api/catalog/clear', data)
+            await clearCache()
+        }
+    })
+
+    onProcessed(async (signal) => {
+        const logger = useLogger()
+        const { context, services: { feed } } = runtime.config.whitebox || { services: {} }
+        if (!feed) return
+
+        let added = 0
+        let deleted = 0
+        await pMap(useJournal('WhiteBox feed', [OPERATION.CREATE, OPERATION.UPDATE, OPERATION.DELETE], signal), async ({ entity, operation }) => {
+            if (entity.meta && (feed.match && feed.match(entity) || !feed.match && entity.type == 'document')) {
+                switch (operation) {
+                    case OPERATION.CREATE:
+                    case OPERATION.UPDATE:
+                        added++
+                        if (!entity.name || !entity.id) {
+                            logger.warn(entity, 'WhiteBox feed skipping')
+                            return
+                        }
+                        logger.trace('WhiteBox feed: %s', entity.id)
+                        const keepData = {
+                            passportId: uuidv1(),
+                            vaultId: aguid(entity.id),
+                            refId: '/' + entity.name.replace('index', ''),
+                            type: 'mikser.' + (entity.meta.type || entity.type),
+                            data: _.pick(entity, ['meta', 'stamp', 'content', 'type', 'collection', 'format', 'id', 'uri']),
+                            date: new Date(entity.time),
+                            vaults: entity.meta.vaults,
+                            context: context || await useMachineId(),
+                            expire: feed.expire === false ? false : feed.expire || '10 days'
+                        }
+                        types.add(keepData.type)
+
+                        logger.debug('WhiteBox feed %s: %s %s', 'keep', entity.type, keepData.refId)
+                        await whiteboxApi('feed', '/api/catalog/keep/one', keepData)
+                        break
+                    case OPERATION.DELETE:
+                        deleted++
+                        const removeData = {
+                            vaultId: aguid(entity.id),
+                            context: context || await useMachineId()
+                        }
+
+                        if (!runtime.options.clear) {
+                            logger.debug('WhiteBox feed %s: %s %s', 'remove', entity.type, entity.id)
+                            return whiteboxApi('feed', '/api/catalog/remove', removeData)
+                        }
+                        break
+                }
+            }
+        }, { concurrency: 4, signal })
+        logger.debug('WhiteBox feed %s: %s', 'keep', added)
+        logger.debug('WhiteBox feed %s: %s', 'remove', deleted)
+
+        await expireCatalog()
+        await clearCache()
+    })
+}

package/src/storage.js

@@ -0,0 +1,292 @@
+import pMap from 'p-map'
+import fs from 'fs/promises'
+import { createReadStream } from 'node:fs'
+import axios from 'axios'
+import FormData from 'form-data'
+import path from 'path'
+import { globby } from 'globby'
+import { setTimeout } from 'timers/promises'
+
+export default ({
+    runtime,
+    useLogger,
+    onLoaded,
+    onImport,
+    onSync,
+    onProcessed,
+    onFinalize,
+    useJournal,
+    useMachineId,
+    watch,
+    checksum,
+    matchEntity,
+    findEntity,
+    createEntity,
+    updateEntity,
+    deleteEntity,
+    constants: { ACTION, OPERATION },
+}) => {
+    const collection = 'storage'
+    const type = 'file'
+
+    const pendingUploads = new Set()
+    const history = new Map()
+
+    async function upload(fileName, uploadName, uploadChecksum) {
+        const logger = useLogger()
+
+        if (pendingUploads.has(fileName)) return
+        pendingUploads.add(fileName)
+        if (history.get(uploadName) == uploadChecksum) return
+
+        const uploadWhenReady = async () => {
+            try {
+                const fh = await fs.open(fileName, 0x10000000)
+                try {
+                    const { context, services: { storage } } = runtime.config.whitebox
+                    let data = {
+                        file: uploadName,
+                        context: context || await useMachineId()
+                    }
+                    const responseHash = await axios.post(storage.url + '/' + storage.token + '/checksum', data)
+                    const matchedHash = responseHash.data.success && uploadChecksum == responseHash.data.hash
+                    logger.debug('WhiteBox storage %s: %s %s', 'checksum', fileName, matchedHash)
+                    if (!matchedHash) {
+                        const uploadHeaders = {
+                            expire: storage.expire === false ? false : storage.expire || '10 days',
+                            context: data.context
+                        }
+                        let form = new FormData()
+                        form.append(uploadName, createReadStream(fileName))
+                        let formHeaders = form.getHeaders()
+                        try {
+                            const responseUpload = await axios
+                                .post(storage.url + '/upload', form, {
+                                    headers: {
+                                        Authorization: 'Bearer ' + storage.token,
+                                        ...formHeaders,
+                                        ...uploadHeaders,
+                                    },
+                                    maxContentLength: Infinity,
+                                    maxBodyLength: Infinity
+                                })
+                            if (responseUpload.data.uploads) {
+                                for (let file in responseUpload.data.uploads) {
+                                    logger.debug('WhiteBox storage %s: %s', 'upload', uploadName)
+                                    logger.debug('WhiteBox storage %s: %s', 'link', responseUpload.data.uploads[file])
+                                    history.set(uploadName, uploadChecksum)
+                                }
+                            }
+                        } catch (err) {
+                            logger.error('WhiteBox storage upload error: %s', err.message)
+                        }
+                    } else {
+                        history.set(uploadName, uploadChecksum)
+                        logger.debug('WhiteBox storage %s: %s', 'skip', uploadName)
+                    }
+                } catch (err) {
+                    logger.error('WhiteBox storage error: %s', err.message)
+                }
+                fh.close()
+            } catch (err) {
+                logger.trace(err, 'WhiteBox storage postponed: %s', uploadName)
+                await setTimeout(3000)
+                await uploadWhenReady()
+            }
+        }
+        await uploadWhenReady()
+        pendingUploads.delete(fileName)
+    }
+
+    async function link(uploadName) {
+        const logger = useLogger()
+        const { context, services: { storage } } = runtime.config.whitebox
+        let data = {
+            file: uploadName,
+            context: context || await useMachineId()
+        }
+        try {
+            const response = await axios.post(storage.url + '/' + storage.token + '/link', data)
+            logger.debug('WhiteBox storage %s: %s', 'link', response.data?.link)
+            return response.data?.link
+        } catch (err) {
+            logger.trace('WhiteBox storage error: %s', err)
+        }
+    }
+
+    async function unlink(uploadName) {
+        const logger = useLogger()
+        const { context, services: { storage } } = runtime.config.whitebox
+        let data = {
+            file: uploadName,
+            context: context || await useMachineId()
+        }
+        try {
+            await axios.post(storage.url + '/' + storage.token + '/unlink', data)
+            logger.debug('WhiteBox storage: %s %s', 'unlink', uploadName)
+        } catch (err) {
+            logger.trace('WhiteBox storage error: %s', err)
+        }
+    }
+
+    onImport(async () => {
+        const logger = useLogger()
+        const paths = await globby('**/*', { cwd: runtime.options.storageFolder })
+        logger.info('Importing whitebox storage: %d', paths.length)
+
+        return Promise.all(paths.map(async relativePath => {
+            const source = path.join(runtime.options.storageFolder, relativePath)
+            const uploadName = source.replace(runtime.options.workingFolder, '')
+
+            await createEntity({
+                id: path.join(`/${collection}`, relativePath),
+                uri: uploadName,
+                collection,
+                type,
+                format: path.extname(relativePath).substring(1).toLowerCase(),
+                name: relativePath,
+                source,
+                checksum: await checksum(source),
+                link: await link(uploadName)
+            })
+        }))
+    })
+
+    onProcessed(async (signal) => {
+        const logger = useLogger()
+        const { services: { storage } } = runtime.config.whitebox || { services: {} }
+        if (!storage) return
+
+        let added = 0
+        let deleted = 0
+        await pMap(useJournal('WhiteBox storage processing', [OPERATION.CREATE, OPERATION.UPDATE, OPERATION.DELETE], signal), async ({ entity, operation }) => {
+            const match = storage.match || ((entity) => entity.id.indexOf('/storage/') != -1)
+            if (matchEntity(entity, match)) {
+                const uploadName = entity.source.replace(runtime.options.workingFolder, '')
+                switch (operation) {
+                    case OPERATION.CREATE:
+                    case OPERATION.UPDATE:
+                        added++
+                        await upload(entity.source, uploadName, entity.checksum)
+                        break
+                    case OPERATION.DELETE:
+                        deleted++
+                        await unlink(uploadName)
+                        break
+                }
+            }
+        }, { concurrency: 4, signal })
+
+        logger.debug('WhiteBox storage %s: %s', 'upload', added)
+        logger.debug('WhiteBox storage %s: %s', 'unlink', deleted)
+    })
+
+    onFinalize(async (signal) => {
+        const logger = useLogger()
+        const { services: { storage } } = runtime.config.whitebox || { services: {} }
+        if (!storage) return
+
+        await pMap(useJournal('WhiteBox storage output', [OPERATION.RENDER], signal), async ({ entity, output }) => {
+            if (output?.success) {
+                if (storage.match && storage.match(entity) || !storage.match && entity.id.indexOf('/storage/') != -1) {
+                    const uploadName = entity.destination.replace(runtime.options.outputFolder, '').replace(runtime.options.workingFolder, '')
+                    try {
+                        const uploadChecksum = await checksum(entity.destination)
+                        await upload(entity.destination, uploadName, uploadChecksum)
+                    } catch (err) {
+                        if (err.code == 'ENOENT') {
+                            logger.error('Output is missing: %s', entity.destination)
+                        } else {
+                            logger.error('WhiteBox storage error: %s', err.message)
+                        }
+                    }
+                }
+            }
+        }, { concurrency: 4, signal })
+    })
+
+    onLoaded(async () => {
+        const logger = useLogger()
+        const { context, services: { storage } } = runtime.config.whitebox
+        if (!storage) return
+
+        runtime.options.storage = storage?.storageFolder || collection
+        runtime.options.storageFolder = path.join(runtime.options.workingFolder, runtime.options.storage)
+
+        if (runtime.options.clear) {
+            const data = {
+                context: context || await useMachineId()
+            }
+            try {
+                logger.info('WhiteBox storage: %s', 'clear')
+                await axios.post(storage.url + '/' + storage.token + '/clear', data)
+            } catch (err) {
+                logger.error('WhiteBox storage error: %s', err.message)
+            }
+        }
+
+        logger.info('WhiteBox storage folder: %s', runtime.options.storageFolder)
+        await fs.mkdir(runtime.options.storageFolder, { recursive: true })
+
+        watch(collection, runtime.options.storageFolder)
+    })
+
+    onSync(collection, async ({ action, context }) => {
+        if (!context.relativePath) return false
+        const { relativePath } = context
+
+        const source = path.join(runtime.options.storageFolder, relativePath)
+        const format = path.extname(relativePath).substring(1).toLowerCase()
+        const id = path.join(`/${collection}`, relativePath)
+        const uploadName = source.replace(runtime.options.workingFolder, '')
+
+        let synced = true
+        switch (action) {
+            case ACTION.CREATE:
+                await createEntity({
+                    id,
+                    uri: uploadName,
+                    name: relativePath,
+                    collection,
+                    type,
+                    format,
+                    source,
+                    checksum: await checksum(source),
+                    link: await link(uploadName)
+                })
+                break
+            case ACTION.UPDATE:
+                const current = await findEntity({ id })
+                if (current?.checksum != checksum) {
+                    await updateEntity({
+                        id,
+                        uri: uploadName,
+                        name: relativePath,
+                        collection,
+                        type,
+                        format,
+                        source,
+                        checksum: await checksum(source),
+                        link: await link(uploadName)
+                    })
+                } else {
+                    synced = false
+                }
+                break
+            case ACTION.DELETE:
+                await unlink(uploadName)
+                await deleteEntity({
+                    id,
+                    collection,
+                    type,
+                })
+                break
+        }
+        return synced
+    })
+
+    return {
+        collection,
+        type
+    }
+}