mikser-io 11.11.1 → 11.12.0

package/app.js

@@ -2,6 +2,7 @@
 import path from 'node:path'
 
 import { setup } from './index.js'
+import runtime from './src/runtime.js'
 import { forward, isInstanceLive } from './src/instance.js'
 
 // Before anything is imported or read.
@@ -132,6 +133,43 @@ async function main() {
     }
 
     const mikser = await setup()
+    guardAgainstSilentDeath()
     await mikser.start()
 }
+
+// A long-running process must not die of an asynchronous error nobody caught.
+//
+// Node's default for an unhandled rejection is to print it and exit 1. For a
+// one-shot build that is right — the build failed, and the exit code is how a
+// deploy finds out. For a WATCHER it is the worst available behaviour: the
+// process is gone, the only evidence is an exit code, and whoever is using it
+// finds out because the site stopped updating. Measured on a live site: four
+// deaths in one afternoon, each from a file that moved between the event and
+// the read, and each time the entire signal was "the server is not there any
+// more".
+//
+// So the handler is honest about the difference. It always says what happened,
+// with the stack, because a swallowed fault is the same silence in a different
+// costume. It only keeps the process alive where staying alive is the correct
+// answer.
+//
+// The specific rejections that caused those deaths are now caught where they
+// happen (src/manager.js), which is the better fix. This is the floor under
+// the ones nobody has found yet — a timer callback that rejects, a plugin's
+// detached promise — and there is no shortage of those in an engine this size.
+function guardAgainstSilentDeath() {
+    const longRunning = Boolean(runtime.options?.watch || runtime.options?.server)
+    process.on('unhandledRejection', (reason) => {
+        const logger = runtime.engine?.logger
+        const detail = reason?.stack ?? reason?.message ?? String(reason)
+        if (!longRunning) {
+            // Same outcome as the default, with a line saying so first.
+            logger?.fatal?.('Unhandled rejection: %s', detail)
+            process.exitCode = 1
+            throw reason
+        }
+        logger?.error('Unhandled rejection (the watcher is still running): %s', detail)
+    })
+}
+
 main()

package/docs/diagnostics.md

@@ -1072,6 +1072,7 @@ distinction is the whole of [Faults](#faults) above.
 | --- | --- | --- |
 | `source-content-not-text` | warn | A `sources()` collection with `content: true` loaded a file whose bytes are not text. They are decoded as UTF-8 and stored mangled, and every consumer inherits that. Set `content: false` to catalogue the files by path instead — `entity.uri` still points at them. Once per collection. |
 | `front-matter-unreadable` | warn | A file's front-matter block is present but not valid YAML. Its attributes are not available and the text is left as written. Previously this threw and ended the whole run with a parser stack that never named the file. |
+| `source-vanished` | warn | A watched file disappeared between the event and the read. Nothing was imported for it. This is what an upload that writes a temporary file and renames it looks like, and it used to kill the process. |
 | `observer-bad-uri` | warn | An observer's `uri` is not an absolute URL, so no webhook can be routed to it. |
 | `untracked-file-read` | warn | A template read a file outside every folder mikser takes entities from, so it has no entity and changing it invalidates nothing. |
 | `progress` | info | A long phase reporting where it has got to. See [Progress](#progress). |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "11.11.1",
+  "version": "11.12.0",
   "files": [
     "app.js",
     "index.js",

package/src/manager.js

@@ -131,6 +131,44 @@ export async function deletedHook(name, context) {
 // `ignored` and a function is the one form that has stayed stable.
 const ignoreJunk = (filePath) => /[/\\]\./.test(filePath) || junkFilter()(filePath)
 
+// Wait for the bytes to stop moving before saying a file arrived.
+//
+// Without this an event fires the moment a file appears, and the tools people
+// actually upload with do not write in one go. A WebDAV client, Finder and
+// `rsync` without --inplace all write a temporary file into the watched folder
+// and rename it afterwards, so the sequence is: `add` for a name that is about
+// to stop existing, a checksum read of a file that is still growing, and a
+// rename out from under both.
+//
+// Measured on a live site during a video upload. Four server deaths in one
+// afternoon from the vanishing half, and — worse, because nothing stopped —
+// one derivative built from a file read at ZERO BYTES: its checksum note held
+// d41d8cd98f00b204e9800998ecf8427e, the md5 of nothing, where a file that size
+// should carry `size:head:tail`. The build called that current and the next
+// one agreed with it.
+//
+// 500ms is the compromise the two failures point at from opposite sides. It is
+// long enough to swallow a temp-file rename and a local tear, short enough
+// that an incremental rebuild of a saved document still feels immediate — the
+// engine's own budget for one is about 100ms. A big upload over a slow link
+// needs more, and the files plugin asks for more; anything can, through
+// `options`.
+const AWAIT_WRITE_FINISH = { stabilityThreshold: 500, pollInterval: 100 }
+
+// `interval` and `binaryInterval` are NOT here, and their absence is the fix.
+//
+// They were set to 1000 and 3000, which reads as a debounce and is not one:
+// chokidar consumes both only inside `if (opts.usePolling)` (handler.js), and
+// polling is off. They described an intention that never ran, which is the
+// most expensive kind of setting to leave lying around — somebody reads them
+// and stops looking for the debounce that is actually missing. Polling can be
+// asked for explicitly through `options`, and then they would apply.
+const WATCH_DEFAULTS = {
+    ignored: ignoreJunk,
+    ignoreInitial: true,
+    awaitWriteFinish: AWAIT_WRITE_FINISH,
+}
+
 // Watch a folder, with mikser's own settings and none of its lifecycle.
 //
 // `watch()` below turns file events into SYNC events — it is how a source
@@ -148,35 +186,65 @@ const ignoreJunk = (filePath) => /[/\\]\./.test(filePath) || junkFilter()(filePa
 export function watchFolder(folder, handler, options = {}) {
     return chokidar
         .watch(folder, {
-            interval: 1000,
-            binaryInterval: 3000,
-            ignored: ignoreJunk,
-            ignoreInitial: true,
+            ...WATCH_DEFAULTS,
             followSymlinks: true,
             ...options,
         })
         .on('all', (event, fullPath) => handler(event, fullPath))
 }
 
-export function watch(name, folder, options = { interval: 1000, binaryInterval: 3000, ignored: ignoreJunk, ignoreInitial: true }) {
+export function watch(name, folder, options = {}) {
     if (runtime.options.watch !== true) return
 
-    chokidar.watch(folder, options)
+    // Every hook is AWAITED and its failure caught here.
+    //
+    // They used to be called and dropped: `createdHook(name, …)` with no
+    // await inside an async listener, which makes a rejection nobody is
+    // holding. There is no unhandledRejection handler in a plugin's reach, so
+    // the default applied — Node printed the error and exited 1. A watcher
+    // that dies of an asynchronous error from a file that moved is the worst
+    // available failure mode, because the only evidence is an exit code: it
+    // happened four times in one afternoon on a live site and each time the
+    // symptom was "the server is gone".
+    //
+    // Caught per event, not per watcher, so one bad file costs that file and
+    // not the process.
+    const deliver = async (hook, fullPath) => {
+        const relativePath = fullPath.replace(`${folder}/`, '')
+        try {
+            await hook(name, { relativePath })
+        } catch (err) {
+            reportWatchFailure(err, fullPath)
+        }
+    }
+
+    chokidar.watch(folder, { ...WATCH_DEFAULTS, ...options })
         .on('all', () => {
             clearTimeout(runtime.engine.processTimeout)
         })
-        .on('add', async fullPath => {
-            const relativePath = fullPath.replace(`${folder}/`, '')
-            createdHook(name, { relativePath })
-        })
-        .on('change', async fullPath => {
-            const relativePath = fullPath.replace(`${folder}/`, '')
-            updatedHook(name, { relativePath })
-        })
-        .on('unlink', async fullPath => {
-            const relativePath = fullPath.replace(`${folder}/`, '')
-            deletedHook(name, { relativePath })
-        })
+        .on('add',    fullPath => deliver(createdHook, fullPath))
+        .on('change', fullPath => deliver(updatedHook, fullPath))
+        .on('unlink', fullPath => deliver(deletedHook, fullPath))
+        .on('error',  err => reportWatchFailure(err))
+}
+
+// A file that is gone is not an error; anything else is.
+//
+// The vanishing case is ordinary for a watcher — the event describes a moment
+// that has already passed — so it is a warning naming the path, and the build
+// carries on. Everything else keeps its stack, because a watcher that
+// swallows real faults is the same silence in a different costume.
+function reportWatchFailure(err, fullPath) {
+    const logger = useLogger()
+    if (err?.code === 'ENOENT') {
+        logger?.warn(
+            { code: 'source-vanished', path: err.path ?? fullPath },
+            'Watched file disappeared before it could be read: %s. Nothing was imported for it — '
+            + 'this is what an upload that writes a temporary file and renames it looks like.',
+            err.path ?? fullPath ?? 'unknown')
+        return
+    }
+    logger?.error('Watcher failed for %s: %s', fullPath ?? 'the watched folder', err?.stack ?? err?.message ?? err)
 }
 
 export function schedule(name, expression, context) {

package/src/plugins/files.js

@@ -103,7 +103,21 @@ export function files(options = {}) {
 
             let synced = true
             switch (action) {
-                case ACTION.CREATE:
+                case ACTION.CREATE: {
+                    // The checksum FIRST, and the link only once it answered.
+                    //
+                    // Linking first left a symlink pointing at nothing when
+                    // the source moved between the event and the read: the
+                    // link was made, `checksum` threw ENOENT, the entity was
+                    // never created, and out/ kept a dangling link that no
+                    // catalog row admits to. It then broke whatever walked
+                    // the output with stat — a consumer's own scope check,
+                    // twice.
+                    //
+                    // Reading first makes the vanishing case leave nothing
+                    // behind: the throw happens before anything is written,
+                    // and the watcher logs it as the ordinary event it is.
+                    const sourceChecksum = await checksum(source)
                     await ensureLink(relativePath)
                     await createEntity({
                         id,
@@ -118,10 +132,11 @@ export function files(options = {}) {
                         // $-ref to this entity expands to it; consumers read
                         // meta.url for the served location (ADR-0011).
                         meta: { url: '/' + name },
-                        checksum: await checksum(source),
+                        checksum: sourceChecksum,
                         link: await link(source)
                     })
                     break
+                }
                 case ACTION.UPDATE: {
                     const current = await findEntity({ id })
                     // `checksum` is the source-checksum FUNCTION from the
@@ -177,7 +192,19 @@ export function files(options = {}) {
             logger.debug('Files folder: %s', runtime.options.filesFolder)
             await mkdir(runtime.options.filesFolder, { recursive: true })
 
-            watch(collection, runtime.options.filesFolder)
+            // A longer settle than the engine's default, because this is the
+            // collection large binaries land in and they are the case that
+            // bit: a video uploaded over WebDAV grows for as long as the link
+            // takes, and an event fired mid-upload gets a checksum of a
+            // partial file — once, of a file read at zero bytes, which was
+            // then published as current.
+            //
+            // Three seconds of stability costs nothing here (nobody edits a
+            // 300 MB video and waits for the page) and buys the margin a slow
+            // or stalling link needs.
+            watch(collection, runtime.options.filesFolder, {
+                awaitWriteFinish: { stabilityThreshold: 3000, pollInterval: 200 },
+            })
         })
 
         onImport(async () => {