mikser-io 6.3.1 → 6.9.0

package/index.js

@@ -8,4 +8,5 @@ export * from './src/config.js'
 export * from './src/plugins.js'
 export * from './src/manager.js'
 export * from './src/tracking.js'
-export * from './src/engine.js'
+export * from './src/engine.js'
+export * from './src/api.js'

package/package.json

@@ -1,10 +1,13 @@
 {
 	"name": "mikser-io",
-	"version": "6.3.1",
+	"version": "6.9.0",
 	"description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
 	"main": "index.js",
 	"scripts": {
-		"test": "node --no-warnings app.js --debug --working-folder test"
+		"debug": "node --no-warnings app.js --debug --working-folder test/fixture",
+		"test:unit": "node --test --test-reporter=spec 'test/unit/**/*.test.js'",
+		"test:smoke": "node --no-warnings app.js --working-folder test/fixture",
+		"test": "npm run test:unit && npm run test:smoke"
 	},
 	"bin": {
 		"mikser": "app.js"

package/src/api.js

@@ -0,0 +1,165 @@
+// Public, transport-agnostic primitives that the REST plugin's HTTP
+// endpoints are thin wrappers over. Library users embedding mikser
+// programmatically can import these directly.
+
+import { randomUUID } from 'node:crypto'
+import { writeFile, unlink, mkdir } from 'node:fs/promises'
+import path from 'node:path'
+import { updateEntity as defaultUpdateEntity } from './lifecycle.js'
+
+/**
+ * Bind to the runtime and return an on-demand renderer that pipelines
+ * concurrent calls into the minimum number of `runtime.process()` cycles.
+ * The returned binding is stateful — each call to useRenderer() owns its
+ * own pending queue and `completed`-hook lifecycle. Mount once per
+ * consumer (the REST plugin mounts one; a library service mounts its own).
+ *
+ * @example
+ *   const { render } = useRenderer(runtime)
+ *   const { output, entity } = await render(entityShape)
+ *
+ * @param {object} runtime                  - the mikser runtime singleton
+ * @param {object} [opts]
+ * @param {Function} [opts.updateEntity]    - override lifecycle.updateEntity (mostly for testing)
+ * @param {number}   [opts.defaultTimeout]  - per-render timeout in ms (default 30_000)
+ * @returns {{ render: (entity, { timeout? }?) => Promise<{ output, entity }> }}
+ */
+export function useRenderer(runtime, {
+    updateEntity = defaultUpdateEntity,
+    defaultTimeout = 30_000,
+} = {}) {
+    let pending = []
+    let cycleRunning = false
+
+    async function runBatch() {
+        if (cycleRunning || pending.length === 0) return
+        cycleRunning = true
+
+        const batch = pending
+        pending = []
+
+        const remaining = new Map(batch.map(b => [b.correlationId, b]))
+        const completedHooks = runtime.hooks.completed
+        const hook = async (entry) => {
+            const cid = entry.entity?._correlationId
+            if (!cid) return
+            const item = remaining.get(cid)
+            if (!item) return
+            remaining.delete(cid)
+            clearTimeout(item.timer)
+            item.resolve({ output: entry.output, entity: entry.entity })
+        }
+        completedHooks.push(hook)
+
+        for (const item of batch) {
+            item.timer = setTimeout(() => {
+                if (remaining.delete(item.correlationId)) {
+                    item.reject(new Error(`Render timeout for ${item.entity.id}`))
+                }
+            }, item.timeout)
+        }
+
+        try {
+            for (const item of batch) {
+                await updateEntity(item.entity).catch(item.reject)
+            }
+            await runtime.process()
+        } catch (err) {
+            for (const item of remaining.values()) {
+                clearTimeout(item.timer)
+                item.reject(err)
+            }
+            remaining.clear()
+        } finally {
+            for (const item of remaining.values()) {
+                clearTimeout(item.timer)
+                item.reject(new Error(`Render did not complete for ${item.entity.id}`))
+            }
+            const idx = completedHooks.indexOf(hook)
+            if (idx >= 0) completedHooks.splice(idx, 1)
+            cycleRunning = false
+            if (pending.length) setImmediate(runBatch)
+        }
+    }
+
+    /**
+     * Submit an entity for rendering. Resolves with `{ output, entity }`
+     * where `output.result` is whatever the renderer/postprocessor returned
+     * (a string for HTML/text outputs, a Buffer for PDFs, etc.).
+     *
+     * Requests arriving concurrently are coalesced into the next available
+     * `runtime.process()` cycle — within that cycle, mikser's worker pool
+     * renders the batch in parallel.
+     *
+     * @param {object} entity                - any entity-shaped object
+     * @param {object} [opts]
+     * @param {number} [opts.timeout]        - override the default timeout
+     * @returns {Promise<{output, entity}>}
+     */
+    function render(entity, { timeout = defaultTimeout } = {}) {
+        return new Promise((resolve, reject) => {
+            const correlationId = randomUUID()
+            pending.push({
+                entity: { ...entity, _correlationId: correlationId },
+                correlationId,
+                timeout,
+                resolve,
+                reject,
+                timer: null,
+            })
+            if (!cycleRunning) setImmediate(runBatch)
+        })
+    }
+
+    return { render }
+}
+
+/**
+ * Bind to a single collection's source folder and return file-level
+ * `write` / `remove` operations against it. Each collection plugin sets
+ * `runtime.options.<name>Folder` during its onLoaded hook; this looks
+ * that up lazily, so it's safe to call useCollection() anywhere after
+ * `runtime.start()`.
+ *
+ * Distinct from `lifecycle.updateEntity` / `lifecycle.deleteEntity` —
+ * those write journal entries. These write actual files; in watch mode
+ * the resulting fs change is what kicks the next sync→process cycle.
+ *
+ * @example
+ *   const documents = useCollection(runtime, 'documents')
+ *   await documents.write('en/draft.md', '# Hi')
+ *   await documents.remove('en/old.md')
+ *
+ * @param {object} runtime         - the mikser runtime singleton
+ * @param {string} name            - collection name (e.g. 'documents')
+ * @returns {{
+ *   name: string,
+ *   folder: string,
+ *   write(relativePath: string, content?: string): Promise<string>,
+ *   remove(relativePath: string): Promise<void>,
+ * }}
+ */
+export function useCollection(runtime, name) {
+    function resolveFolder() {
+        const folder = runtime?.options?.[`${name}Folder`]
+        if (!folder) throw new Error(`Unknown collection: ${name}`)
+        return folder
+    }
+
+    return {
+        name,
+        get folder() { return resolveFolder() },
+
+        async write(relativePath, content = '') {
+            const uri = path.join(resolveFolder(), relativePath)
+            await mkdir(path.dirname(uri), { recursive: true })
+            await writeFile(uri, content, 'utf8')
+            return uri
+        },
+
+        async remove(relativePath) {
+            const uri = path.join(resolveFolder(), relativePath)
+            await unlink(uri)
+        },
+    }
+}

package/src/plugins/layouts.js

@@ -21,6 +21,7 @@ export default ({
     onSync,
     matchEntity,
     changeExtension,
+    getFormatInfo,
     findEntity,
     findEntities,
     constants: { ACTION, OPERATION, TASKS },
@@ -28,15 +29,6 @@ export default ({
     const collection = 'layouts'
     const type = 'layout'
 
-    function getFormatInfo(relativePath) {
-        const template = path.extname(relativePath).substring(1).toLowerCase()
-        const withoutTemplate = relativePath.replace(path.extname(relativePath), '')
-        const formatExt = path.extname(withoutTemplate).substring(1).toLowerCase()
-        const [format, postprocessor] = formatExt.split('-')
-        const name = formatExt ? withoutTemplate.replace(path.extname(withoutTemplate), '') : withoutTemplate
-        return { name, format: format || 'html', template, postprocessor }
-    }
-
     function addToSitemap(entity) {
         const logger = useLogger()
         const { sitemap } = runtime.state.layouts

package/src/plugins/rest.js

@@ -1,12 +1,75 @@
 import path from 'node:path'
-import { writeFile, unlink, mkdir, access } from 'node:fs/promises'
-import { updateEntity } from '../lifecycle.js'
-import { findEntities } from '../catalog.js'
+import { access } from 'node:fs/promises'
+import { useRenderer, useCollection } from '../api.js'
+
+// MIME type lookup used when streaming a postprocessor's output back over
+// HTTP. The renderer's output extension lives on entity.destination
+// (assigned by the layouts plugin), so we use it as the source of truth.
+const MIME_BY_EXT = {
+    pdf: 'application/pdf',
+    html: 'text/html; charset=utf-8',
+    xml: 'application/xml; charset=utf-8',
+    xhtml: 'application/xhtml+xml; charset=utf-8',
+    rss: 'application/rss+xml; charset=utf-8',
+    atom: 'application/atom+xml; charset=utf-8',
+    json: 'application/json; charset=utf-8',
+    css: 'text/css; charset=utf-8',
+    js: 'application/javascript; charset=utf-8',
+    svg: 'image/svg+xml',
+    png: 'image/png',
+    jpg: 'image/jpeg',
+    jpeg: 'image/jpeg',
+    webp: 'image/webp',
+    gif: 'image/gif',
+    mp4: 'video/mp4',
+    webm: 'video/webm',
+    txt: 'text/plain; charset=utf-8',
+    md: 'text/markdown; charset=utf-8',
+}
+
+export function mimeForEntity(entity) {
+    if (!entity?.destination) return null
+    const ext = path.extname(entity.destination).toLowerCase().replace(/^\./, '')
+    return MIME_BY_EXT[ext] ?? null
+}
+
+// Decide how to send the render output over HTTP. Exported (and pure-ish)
+// so tests can exercise the branching without spinning up a real server.
+export async function sendRenderOutput(res, output, entity) {
+    if (output == null || output.result == null) {
+        return res.status(204).send()
+    }
+    const result = output.result
+    const mime = mimeForEntity(entity)
+
+    if (Buffer.isBuffer(result)) {
+        if (mime) res.type(mime)
+        return res.send(result)
+    }
+    if (typeof result === 'string') {
+        // A few postprocessors might return an absolute path to a generated
+        // file rather than its contents. Only attempt this when the string
+        // looks plausibly path-shaped — short, starts with a slash, and the
+        // file actually exists. Otherwise treat it as content.
+        if (result.length < 4096 && (result.startsWith('/') || /^[A-Za-z]:[\\/]/.test(result))) {
+            try {
+                await access(result)
+                return res.sendFile(result)
+            } catch { /* not a path, fall through */ }
+        }
+        if (mime) res.type(mime)
+        return res.send(result)
+    }
+    // Anything else (plain object, etc.) is sent as JSON.
+    return res.json(result)
+}
 
 export default ({
     runtime,
     onLoaded,
     useLogger,
+    updateEntity,
+    findEntities,
 }) => {
     onLoaded(async () => {
         const logger = useLogger()
@@ -28,15 +91,19 @@ export default ({
             res.status(401).json({ error: 'Unauthorized' })
         }
 
-        function collectionFolder(collection) {
-            return runtime.options[`${collection}Folder`]
-        }
+        // Reuse the transport-agnostic primitives from src/api.js so the
+        // library entry point and the REST endpoints share the exact same
+        // batching/timeouts/error semantics.
+        const { render } = useRenderer(runtime, {
+            updateEntity,
+            defaultTimeout: runtime.config.rest?.renderTimeout ?? 30_000,
+        })
 
         router.get('/entities', async (req, res) => {
             try {
                 const { page: rawPage, limit: rawLimit, ...filter } = req.query
                 const page = Math.max(1, parseInt(rawPage) || 1)
-                const limit = Math.min(100, Math.max(1, parseInt(rawLimit) || runtime.config.rest?.pageSize ?? 10))
+                const limit = Math.min(100, Math.max(1, parseInt(rawLimit) || (runtime.config.rest?.pageSize ?? 10)))
                 const query = Object.keys(filter).length ? filter : undefined
 
                 const all = await findEntities(query)
@@ -62,69 +129,34 @@ export default ({
         router.put('/entities', auth, async (req, res) => {
             try {
                 const { collection, relativePath, content = '' } = req.body
-                const folder = collectionFolder(collection)
-                if (!folder) return res.status(400).json({ error: `Unknown collection: ${collection}` })
-                const uri = path.join(folder, relativePath)
-                await mkdir(path.dirname(uri), { recursive: true })
-                await writeFile(uri, content, 'utf8')
+                await useCollection(runtime, collection).write(relativePath, content)
                 res.status(202).json({ ok: true })
             } catch (err) {
                 logger.error('REST update error: %s', err.message)
-                res.status(500).json({ error: err.message })
+                res.status(/Unknown collection/.test(err.message) ? 400 : 500).json({ error: err.message })
             }
         })
 
         router.delete('/entities', auth, async (req, res) => {
             try {
                 const { collection, relativePath } = req.body
-                const folder = collectionFolder(collection)
-                if (!folder) return res.status(400).json({ error: `Unknown collection: ${collection}` })
-                const uri = path.join(folder, relativePath)
-                await unlink(uri)
+                await useCollection(runtime, collection).remove(relativePath)
                 res.status(202).json({ ok: true })
             } catch (err) {
                 logger.error('REST delete error: %s', err.message)
-                res.status(500).json({ error: err.message })
+                res.status(/Unknown collection/.test(err.message) ? 400 : 500).json({ error: err.message })
             }
         })
 
-        router.post('/render', async (req, res) => {
+        router.post('/render', auth, async (req, res) => {
             try {
-                const entity = { ...req.body, _correlationId: crypto.randomUUID() }
-                const timeout = runtime.config.rest?.renderTimeout ?? 30_000
-
-                const output = await new Promise((resolve, reject) => {
-                    const timer = setTimeout(() => {
-                        runtime.removeHook('completed', hook)
-                        reject(new Error(`Render timeout for ${entity.id}`))
-                    }, timeout)
-
-                    const hook = async (entry) => {
-                        if (entry.entity._correlationId !== entity._correlationId) return
-                        clearTimeout(timer)
-                        runtime.removeHook('completed', hook)
-                        resolve(entry.output)
-                    }
-                    runtime.addHook('completed', hook)
-
-                    updateEntity(entity)
-                        .then(() => runtime.process())
-                        .catch(reject)
-                })
-
-                if (output?.result != null) {
-                    const isFile = await access(output.result).then(() => true).catch(() => false)
-                    if (isFile) {
-                        res.sendFile(output.result)
-                    } else {
-                        res.send(output.result)
-                    }
-                } else {
-                    res.status(204).send()
-                }
+                const { output, entity } = await render(req.body)
+                await sendRenderOutput(res, output, entity)
             } catch (err) {
                 logger.error('REST render error: %s', err.message)
-                res.status(500).json({ error: err.message })
+                if (!res.headersSent) {
+                    res.status(500).json({ error: err.message })
+                }
             }
         })
 

package/src/plugins/validator.js

@@ -10,7 +10,7 @@ export default ({
     onLoad(() => {
         for (let { match, validate, operations = [OPERATION.CREATE, OPERATION.UPDATE] } of runtime.config.validator?.validators || []) {
             onValidate(operations, async entry => {
-                if (entity.meta && matchEntity(entry.entity, match)) {
+                if (entry.entity?.meta && matchEntity(entry.entity, match)) {
                     return await validate(entry.entity)
                 }
             })

package/src/render.js

@@ -3,24 +3,7 @@ import { createRequire } from 'node:module'
 import path from 'node:path'
 import _ from 'lodash'
 import { useLogger } from './engine.js'
-
-// Flatten template-helper args into a single human-readable message.
-// Handlebars helpers receive a trailing options object (it has a `.hash`
-// property) which we drop.
-function formatLogArgs(args) {
-    if (args.length && typeof args[args.length - 1] === 'object' && args[args.length - 1] !== null && 'hash' in args[args.length - 1]) {
-        args = args.slice(0, -1)
-    }
-    return args
-        .map(arg => {
-            if (arg == null) return String(arg)
-            if (typeof arg === 'object') {
-                try { return JSON.stringify(arg) } catch { return String(arg) }
-            }
-            return String(arg)
-        })
-        .join(' ')
-}
+import { formatLogArgs } from './utils.js'
 
 export default async ({ entity, options, config, context, state, logger, port }) => {
     logger = logger || {

package/src/utils.js

@@ -1,4 +1,4 @@
-import { hashFile, hashingStream } from 'hasha'
+import { hashFile, hash } from 'hasha'
 import { stat } from 'node:fs/promises'
 import TruncateStream from 'truncate-stream'
 import { createReadStream } from 'node:fs'
@@ -23,7 +23,7 @@ export async function checksum(uri) {
         const truncate = new TruncateStream({ maxBytes })
         const fileStream = createReadStream(uri)
         fileStream.pipe(truncate)
-        const checksum = size.toString() + ':' + await hashingStream(truncate, { algorithm: 'md5' })
+        const checksum = size.toString() + ':' + await hash(truncate, { algorithm: 'md5' })
         return checksum
     }
 }
@@ -48,7 +48,7 @@ export function matchEntity(entity, match) {
     if (!match) return false
     if (typeof match == 'function') return match(entity)
     else if (typeof match == 'string') {
-        if (match.substring(0, 1) == '@/') {
+        if (match.substring(0, 2) == '@/') {
             return minimatch(entity.name, match.substring(2))
         } else {
             return minimatch(entity.id, match)
@@ -64,6 +64,43 @@ export function changeExtension(file, format) {
     return result
 }
 
+// Decode a layout filename into its parts. `template` is the outer
+// extension (the renderer); `format` is the output format encoded as a
+// second extension (defaults to 'html'); `postprocessor`, if present,
+// is everything after the `-` in the format segment.
+//
+// Examples:
+//   foo.hbs                 -> { name:'foo',  format:'html', template:'hbs',  postprocessor:undefined }
+//   page.css.hbs            -> { name:'page', format:'css',  template:'hbs',  postprocessor:undefined }
+//   report.html-pdf.hbs     -> { name:'report', format:'html', template:'hbs', postprocessor:'pdf' }
+//   welcome.html-mjml.liquid-> { name:'welcome', format:'html', template:'liquid', postprocessor:'mjml' }
+export function getFormatInfo(relativePath) {
+    const template = path.extname(relativePath).substring(1).toLowerCase()
+    const withoutTemplate = relativePath.replace(path.extname(relativePath), '')
+    const formatExt = path.extname(withoutTemplate).substring(1).toLowerCase()
+    const [format, postprocessor] = formatExt.split('-')
+    const name = formatExt ? withoutTemplate.replace(path.extname(withoutTemplate), '') : withoutTemplate
+    return { name, format: format || 'html', template, postprocessor }
+}
+
+// Flatten template-helper args into a single human-readable message.
+// Handlebars helpers receive a trailing options object (it has a `.hash`
+// property) which we drop. Liquid filters and Eta calls don't.
+export function formatLogArgs(args) {
+    if (args.length && typeof args[args.length - 1] === 'object' && args[args.length - 1] !== null && 'hash' in args[args.length - 1]) {
+        args = args.slice(0, -1)
+    }
+    return args
+        .map(arg => {
+            if (arg == null) return String(arg)
+            if (typeof arg === 'object') {
+                try { return JSON.stringify(arg) } catch { return String(arg) }
+            }
+            return String(arg)
+        })
+        .join(' ')
+}
+
 // Build a compact "[layouts/foo.hbs:12:4]" suffix from whatever the
 // underlying template engine attached to its thrown error. Renderer
 // plugins are expected to set `err.layoutUri` (and optionally `err.line` /