mikser-io 6.15.0 → 6.21.2

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "mikser-io",
-	"version": "6.15.0",
+	"version": "6.21.2",
 	"description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
 	"main": "index.js",
 	"scripts": {
@@ -60,6 +60,7 @@
 		"mikser-io-render-eta": "file:../mikser-io-render-eta",
 		"mikser-io-render-liquid": "file:../mikser-io-render-liquid",
 		"mikser-io-render-markdown": "file:../mikser-io-render-markdown",
+		"mikser-io-vector": "file:../mikser-io-vector",
 		"sharp": "^0.34.5"
 	},
 	"directories": {

package/src/engine.js

@@ -6,7 +6,7 @@ import { existsSync } from 'fs'
 import _ from 'lodash'
 import Piscina from 'piscina'
 import runtime from './runtime.js'
-import { onInitialize, onInitialized, onRender, onCancel, onCancelled, onFinalized, onLoaded, onAfterRender, onBeforePostprocess, onPostprocess, postprocessEntities } from './lifecycle.js'
+import { onInitialize, onInitialized, onLoad, onRender, onCancel, onCancelled, onFinalized, onLoaded, onAfterRender, onBeforePostprocess, onPostprocess, postprocessEntities } from './lifecycle.js'
 import { useJournal, updateEntry } from './journal.js'
 import { globby } from 'globby'
 import { OPERATION, TASKS } from './constants.js'
@@ -46,6 +46,7 @@ export async function setup(options) {
             .option('-d --debug', 'display debug statements')
             .option('-t --trace', 'display trace statements')
             .option('-e --runtime-folder <folder>', 'set mikser runtime folder relative to working folder', 'runtime')
+            .option('-s --server [port]', 'start an Express server on the given port (defaults to 3001)')
 
         Object.assign(runtime.options, options || runtime.engine.commander.parse(process.argv).opts())
         runtime.options.info = true
@@ -84,6 +85,73 @@ export async function setup(options) {
             }
         }
         await mkdir(runtime.options.runtimeFolder, { recursive: true })
+
+        // Server bring-up: two paths, controlled by which of these are set.
+        //
+        //   runtime.options.app   — pre-supplied Express app (e.g. mikser
+        //                           embedded inside an existing service).
+        //                           The caller owns the listen lifecycle
+        //                           and the static-route policy. Engine
+        //                           stays out of routing/listening so it
+        //                           doesn't clobber their setup; plugins
+        //                           still mount their own routers on it.
+        //
+        //   --server [port]       — engine creates the Express app, mounts
+        //   (runtime.options.server)  static for the output folder, and
+        //                           listens on the port. The actual
+        //                           listen() is deferred via the onLoad
+        //                           hook below so it runs LAST in the
+        //                           onLoaded phase — after every plugin
+        //                           has had a chance to register routes.
+        //
+        // If both are present, the externally-supplied app wins; --server
+        // becomes a no-op (the caller is in charge).
+        if (runtime.options.app) {
+            logger.info('Using externally-supplied Express app on runtime.options.app')
+        } else if (runtime.options.server) {
+            const { default: express } = await import('express').catch(() => {
+                throw new Error('Express is required for --server. Run: npm install express')
+            })
+            runtime.options.app = express()
+            runtime.options.port = runtime.options.server === true
+                ? 3001
+                : Number(runtime.options.server) || 3001
+            logger.info('Server starting on port %d', runtime.options.port)
+        }
+    })
+
+    // Registered here (inside setup) so it runs AFTER plugins.js's onLoad
+    // (which is registered at module-import time). That ordering matters
+    // because plugins.js loads user plugins during its onLoad — each plugin
+    // factory may register onLoaded handlers that mount routes on
+    // runtime.options.app. We want our listen() to run LAST in the
+    // onLoaded phase, which is why we register it from inside another
+    // onLoad — by then plugins have already appended their handlers.
+    onLoad(() => {
+        // Only auto-mount static + auto-listen when the engine owns the
+        // app (created via --server). When an external app was supplied,
+        // `port` is unset and we stay out of the way — the caller manages
+        // both routing decisions and the listen lifecycle themselves.
+        if (!runtime.options.app || runtime.options.port == null) return
+        onLoaded(async () => {
+            const logger = useLogger()
+            const { default: express } = await import('express')
+
+            // Serve the output folder as the catch-all static route.
+            // Mounted LAST in the middleware chain so plugin routes
+            // (e.g. /api/*) match first; anything that didn't match a
+            // plugin's router falls through to the static handler and
+            // gets served from <outputFolder>.
+            runtime.options.app.use(express.static(runtime.options.outputFolder))
+            logger.info('Serving %s as /', runtime.options.outputFolder.replace(runtime.options.workingFolder + '/', ''))
+
+            await new Promise(resolve => {
+                runtime.options.app.listen(runtime.options.port, () => {
+                    logger.info('Server listening on port %d', runtime.options.port)
+                    resolve()
+                })
+            })
+        })
     })
 
     onLoaded(async () => {

package/src/journal.js

@@ -70,12 +70,12 @@ export async function clearJournal(aborted) {
     await journal('operations').del()
     if (!aborted) {
         // Tear down the sqlite connection only when this is genuinely a
-        // one-shot run that's about to exit. Watch mode and any plugin
-        // that keeps the process alive (e.g. the API plugin's HTTP server)
-        // sets `persistent` so subsequent cycles can still write to the
-        // journal — without this, the second /render request would crash
-        // with "Unable to acquire a connection".
-        if (runtime.options.watch !== true && runtime.options.persistent !== true) {
+        // one-shot run that's about to exit. The two ways mikser stays
+        // alive across cycles are watch mode (chokidar handles keep the
+        // event loop ref'd) and an HTTP server (`runtime.options.app`,
+        // set either by --server or by a caller passing setup({ app }));
+        // both need the journal to survive subsequent cycles.
+        if (runtime.options.watch !== true && !runtime.options.app) {
             journal.destroy()
         }
     }

package/src/plugins/api.js

@@ -73,13 +73,23 @@ export default ({
     onLoaded(async () => {
         const logger = useLogger()
 
+        // The api plugin no longer creates its own Express app. It mounts
+        // onto an existing app provided either by --server (engine
+        // creates one) or by a caller that programmatically passed
+        // setup({ app: ... }). Without an app there's nowhere to mount
+        // routes, so fail fast with a message that points at the fix.
+        const app = runtime.options.app
+        if (!app) {
+            throw new Error(
+                'API plugin requires runtime.options.app — run mikser with --server, ' +
+                'or pass { app: yourExpressInstance } to setup() before loading the api plugin'
+            )
+        }
+
         const { default: express } = await import('express').catch(() => {
-            throw new Error('express is required for the api plugin — run: npm install express')
+            throw new Error('Express is required for the api plugin — run: npm install express')
         })
 
-        const ownApp = !runtime.options.app
-        const app = runtime.options.app ?? express()
-
         const router = express.Router()
         router.use(express.json())
 
@@ -167,22 +177,8 @@ export default ({
             }
         })
 
-        // The HTTP server keeps the process alive across many process()
-        // cycles. Tell the journal layer not to tear down the sqlite
-        // connection at the end of each cycle.
-        runtime.options.persistent = true
-
         const base = runtime.config.api?.base ?? '/api'
         app.use(base, router)
-
-        if (ownApp) {
-            const port = runtime.config.api?.port ?? 3001
-            const url = runtime.config.api?.url ?? 'http://localhost'
-            app.listen(port, () => {
-                logger.info('Api listening: %s:%d%s', url, port, base)
-            })
-        } else {
-            logger.info('Api mounted: %s', base)
-        }
+        logger.info('Api mounted: %s', base)
     })
 }

package/src/plugins/data.js

@@ -36,9 +36,18 @@ export default ({
             }
         }
         for (let entitiesName in entitiesConfig) {
+            // Per-config namespacing token. When set, this entity-writer's
+            // files land under <dataFolder>/<token>/ instead of straight
+            // in <dataFolder>. Each named entity config can declare its
+            // own token, so they can target different namespaces (useful
+            // for multi-tenant exports into shared storage).
+            const token = entitiesConfig[entitiesName].token
+            const targetFolder = token
+                ? path.join(runtime.options.dataFolder, token)
+                : runtime.options.dataFolder
             const {
                 query,
-                map,
+                map: mapEntity = entity => entity,
                 pick,
                 save: saveEntity = async entity => {
                     if (!entity.name) {
@@ -46,12 +55,12 @@ export default ({
                         return
                     }
                     const dump = JSON.stringify(normalize(entity))
-                    const entityFile = path.join(runtime.options.dataFolder, `${entity.name}.${entitiesName}.json`)
+                    const entityFile = path.join(targetFolder, `${entity.name}.${entitiesName}.json`)
                     await mkdir(path.dirname(entityFile), { recursive: true })
                     await writeFile(entityFile, dump, 'utf8')
                 },
                 delete: deleteEntity = async entity => {
-                    const entityFile = path.join(runtime.options.dataFolder, `${entity.name}.json`)
+                    const entityFile = path.join(targetFolder, `${entity.name}.json`)
                     await unlink(entityFile)
                 }
             } = entitiesConfig[entitiesName]
@@ -62,12 +71,12 @@ export default ({
                         case OPERATION.CREATE:
                         case OPERATION.UPDATE:
                             logger.debug('Data export entity %s %s: %s', entity.collection, operation, entity.id)
-                            await saveEntity(map ? await map(entity) : {
+                            await saveEntity(({
                                 refId: ('/' + entity.name.replaceAll('\\', '/')).replace(/\/index$/g, '/'),
                                 name: entity.name,
                                 date: new Date(entity.time),
-                                data: _.pick(entity, pick || ['collection', 'format', 'type', 'destination', 'stamp', 'meta', 'id',])
-                            })
+                                data: _.pick(await mapEntity(entity), pick || ['collection', 'format', 'type', 'destination', 'stamp', 'meta', 'id',])
+                            }))
                             break
                         case OPERATION.DELETE:
                             await deleteEntity(entity)
@@ -90,14 +99,19 @@ export default ({
             }
         }
         for (let contextName in contextConfig) {
+            // Per-config namespacing token — see the entities loop above.
+            const token = contextConfig[contextName].token
+            const targetFolder = token
+                ? path.join(runtime.options.dataFolder, token)
+                : runtime.options.dataFolder
             const {
                 query,
-                map,
+                map: mapEntityContext = (entity, context) => context,
                 pick,
                 save: saveConext = async (entity, context) => {
                     if (context?.data) {
                         const entityName = entity.name
-                        const contextFile = path.join(runtime.options.dataFolder, `${entityName}.${contextName}.json`)
+                        const contextFile = path.join(targetFolder, `${entityName}.${contextName}.json`)
                         await mkdir(path.dirname(contextFile), { recursive: true })
                         await writeFile(contextFile, JSON.stringify(context), 'utf8')
                     }
@@ -107,7 +121,7 @@ export default ({
             for await (let { entity, context } of useJournal('Data context', [OPERATION.RENDER])) {
                 if (query(entity)) {
                     logger.debug('Data export context: %s', entity.name)
-                    await saveConext(entity, map ? await map(entity, context) : _.pick(context, pick || ['data']))
+                    await saveConext(entity, _.pick(await mapEntityContext(entity, context), pick || ['data']))
                 }
             }
         }
@@ -116,23 +130,29 @@ export default ({
     onFinalize(async () => {
         const logger = useLogger()
         for (let catalogName in runtime.config.data?.catalog || {}) {
+            // Per-config namespacing token — see the entities loop above.
+            const token = runtime.config.data?.catalog[catalogName].token
+            const targetFolder = token
+                ? path.join(runtime.options.dataFolder, token)
+                : runtime.options.dataFolder
             const {
                 query: queryEntities = entity => entity.type == 'document',
-                map,
+                map: mapEntity = entity => entity,
                 pick,
                 save: saveEntities = async entities => {
-                    const entitiesFile = path.join(runtime.options.dataFolder, `${catalogName}.json`)
+                    const entitiesFile = path.join(targetFolder, `${catalogName}.json`)
+                    await mkdir(path.dirname(entitiesFile), { recursive: true })
                     logger.debug('Data export catalog %s %s: %s', catalogName, entities.length, entitiesFile)
                     await writeFile(entitiesFile, JSON.stringify(entities), 'utf8')
                 }
             } = runtime.config.data?.catalog[catalogName]
             const entities = await findEntities(queryEntities)
-            await saveEntities(await pMap(entities, async entity => map ? await map(entity) : {
+            await saveEntities(await pMap(entities, async entity => ({
                 refId: ('/' + entity.name.replaceAll('\\', '/')).replace(/\/index$/g, '/'),
                 name: entity.name,
                 date: new Date(entity.time),
-                data: _.pick(entity, pick || ['collection', 'format', 'type', 'destination', 'stamp', 'meta', 'id',])
-            }))
+                data: _.pick(await mapEntity(entity), pick || ['collection', 'format', 'type', 'destination', 'stamp', 'meta', 'id',])
+            })))
         }
     })
 }

package/src/plugins/post/pdf.js

@@ -14,7 +14,7 @@ export async function setup({ config, logger }) {
     }
 
     const { default: puppeteer } = await import('puppeteer').catch(() => {
-        throw new Error('puppeteer is required for the pdf postprocessor — run: npm install puppeteer')
+        throw new Error('Puppeteer is required for the pdf postprocessor — run: npm install puppeteer')
     })
     browser = await puppeteer.launch({
         headless: true,

package/src/plugins/resources.js

@@ -24,6 +24,7 @@ export default ({
     trackProgress,
     updateProgress,
     updateEntry,
+    matchEntity,
     constants: { OPERATION },
 }) => {
     const collection = 'resources'
@@ -65,8 +66,7 @@ export default ({
                 _.eachDeep(entity.meta, resource => {
                     if (typeof resource == 'string') {
                         for (let library in resourceLib) {
-                            const match = new RegExp(library)
-                            if (resource.match(match)) {
+                            if (matchEntity(resource, library)) {
                                 resourceMap[entity.id].push({ library, resource, entity })
                             }
                         }

package/src/runtime.js

@@ -77,7 +77,7 @@ const runtime = {
         else if (this.abortController) {
             await this.cancel()
         }
-        this.mutex.use(async () => {
+        await this.mutex.use(async () => {
             try {
                 this.abortController = new AbortController()
                 const { signal } = this.abortController

package/src/utils.js

@@ -49,13 +49,13 @@ export function matchEntity(entity, match) {
     if (typeof match == 'function') return match(entity)
     else if (typeof match == 'string') {
         if (match.substring(0, 2) == '@/') {
-            return minimatch(entity.name, match.substring(2))
+            return minimatch(typeof entity == 'string' ? entity : entity.name, match.substring(2))
         } else {
-            return minimatch(entity.id, match)
+            return minimatch(typeof entity == 'string' ? entity : entity.id, match)
         }
     }
     else if (typeof match == 'object') return _.isMatch(entity, match)
-    throw new Error('Ivalid match type')
+    throw new Error('Invalid match type')
 }
 
 export function changeExtension(file, format) {