mikser-io 10.9.0 → 10.10.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.9.0",
+  "version": "10.10.1",
   "files": [
     "app.js",
     "index.js",

package/src/plugins/assets.js

@@ -118,6 +118,11 @@ export function assets(options = {}) {
     const collection = 'presets'
     const type = 'preset'
     const checksumMap = new Set()
+    // Every marker on disk, indexed by the derivative it claims. Built beside
+    // checksumMap so forgetting a derivative's markers costs a lookup rather
+    // than a scan — this runs once per entity that renders, and a directory
+    // walk per entity is quadratic on an assets tree of any size.
+    const markersByDestination = new Map()
 
     // A preset that matches nothing builds green and says nothing, which is
     // how a mistyped pattern ships. `files({ outputFolder })` prefixes `name`
@@ -387,6 +392,40 @@ export function assets(options = {}) {
         return result
     }
 
+    // A marker outliving the render it describes is the whole defect.
+    //
+    // `isPresetRendered` is keyed on the SOURCE checksum, and an interrupted
+    // render does not change the source — so a derivative that was being
+    // rewritten when the process died keeps a marker that still matches, and
+    // every later build reports nothing to do over a half-written file.
+    // Measured on a 20KB fixture: delete a derivative, interrupt the
+    // re-render, and the site serves 10KB of it indefinitely.
+    //
+    // The marker means "a completed render produced this file", so it is
+    // removed before the render starts and written again by onComplete. An
+    // interrupted render then leaves no claim behind and the next ordinary
+    // build re-derives. The FILE is deliberately left alone: a preset that
+    // fails before writing anything still has a good derivative on disk, and
+    // removing it would take a working asset off the site until the next
+    // build.
+    // Where a preset puts a given entity. One implementation, because the
+    // reuse check, the render dispatch and the unmarked-derivative scan all
+    // need it and a second copy would drift from `presetUrl` the moment a
+    // preset grew an option.
+    function presetDestination(entity, preset) {
+        const name = preset.format ? changeExtension(entity.name, preset.format) : entity.name
+        return path.join(runtime.options.assetsFolder, preset.name, name)
+    }
+
+    async function forgetPresetMarkers(entity) {
+        const destination = path.resolve(entity.destination)
+        for (const marker of markersByDestination.get(destination) ?? []) {
+            await rm(marker, { force: true })
+            checksumMap.delete(marker)
+        }
+        markersByDestination.delete(destination)
+    }
+
     async function renderPresets(entities, { rendererChanged = new Set() } = {}) {
         const { presets, assetsMap } = runtime.state.assets
 
@@ -409,11 +448,7 @@ export function assets(options = {}) {
                 const { options: configOptions } = normalizePresetConfig(
                     options.presets?.[entityPreset]
                 )
-                let destination = entity.name
-                if (entity.preset.format) {
-                    destination = changeExtension(destination, entity.preset.format)
-                }
-                entity.destination = path.join(runtime.options.assetsFolder, entityPreset, destination)
+                entity.destination = presetDestination(entity, entity.preset)
                 // Two gates sit between a scheduled entity and a render: the
                 // manifest's "its source did not change", and this plugin's
                 // marker. A forced render clears both — a marker at the
@@ -421,6 +456,9 @@ export function assets(options = {}) {
                 // disregard.
                 const forced = rendererChanged.has(entityToRender.id)
                 const ignore = forced ? false : await isPresetRendered(entity)
+                // Asked after isPresetRendered, which reads the markers this
+                // removes.
+                if (!ignore) await forgetPresetMarkers(entity)
                 tasks.push({
                     entity,
                     options: {
@@ -661,12 +699,26 @@ export function assets(options = {}) {
     })
 
     onBeforeRender(async signal => {
-        const { assetsMap } = runtime.state.assets
+        const { assetsMap, presets } = runtime.state.assets
 
         checksumMap.clear()
+        markersByDestination.clear()
         const checksumFiles = await globby('**/*.md5', { cwd: runtime.options.assetsFolder })
         for (let checksumFile of checksumFiles) {
-            checksumMap.add(path.join(runtime.options.assetsFolder, checksumFile))
+            const marker = path.join(runtime.options.assetsFolder, checksumFile)
+            checksumMap.add(marker)
+            // `<destination>.<revision>.md5` — the same shape the orphan
+            // sweep strips to get back to the derivative.
+            //
+            // Resolved on both sides of the comparison, because the other
+            // side is a manifest destination and these two are built by
+            // different subsystems. It is a no-op while assetsFolder is
+            // derived from workingFolder (see onLoaded), which is why no test
+            // can tell it apart — normalization at the boundary, not a
+            // transform anything depends on.
+            const destination = path.resolve(marker.replace(/\.[^.]+\.md5$/, ''))
+            if (!markersByDestination.has(destination)) markersByDestination.set(destination, [])
+            markersByDestination.get(destination).push(marker)
         }
 
         const entitiesToRender = new Map()
@@ -674,6 +726,80 @@ export function assets(options = {}) {
         // preset moved, or --render-presets asked for them.
         const presetMoved = new Set()
 
+        // A derivative with no marker beside it is a render that was killed
+        // before it could write one.
+        //
+        // Nothing else notices. The file is present, so the engine's
+        // missing-output path does not fire; the source did not change, so
+        // the journal never mentions it; and the marker check that WOULD
+        // catch it is only consulted for entities something else already
+        // scheduled. So the derivative sits there half-written and every
+        // build reports nothing to do — which is the whole reason the marker
+        // is now removed before a render rather than merely written after
+        // one. Removing it only helps if its absence is read.
+        //
+        // Empty on any healthy build, and the catalog walk is behind that
+        // emptiness: this costs one Set lookup per cycle until something has
+        // actually gone wrong.
+        // Asked of the MANIFEST, not of the folder.
+        //
+        // The first version walked every file under each preset folder and
+        // called anything without a marker an interrupted render. But only
+        // the destination the engine hands the preset gets a marker, so a
+        // preset that legitimately writes MORE than one file — a poster frame
+        // beside a video — left permanent evidence of a failure that never
+        // happened. It warned on every build for ever, said the derivative
+        // was "being re-derived", and re-derived nothing, because no entity's
+        // destination ever matched the extra file. A warning that cries wolf
+        // on every build is worse than the silence it replaced, and it
+        // teaches people to ignore the one that matters.
+        //
+        // A file nothing claims is an ORPHAN, which is a different fault with
+        // its own report. This asks only about outputs the manifest recorded,
+        // which is also what makes the recovery possible: a snapshot carries
+        // the entity id, so there is nothing to reverse-map and no catalog to
+        // walk.
+        const presetRoots = Object.keys(presets)
+            .map(name => path.resolve(runtime.options.assetsFolder, name) + path.sep)
+        const unfinished = new Map()
+        for (const snapshot of runtime.manifest?.all?.() ?? []) {
+            if (!snapshot.destination) continue
+            const destination = path.resolve(snapshot.destination)
+            if (!presetRoots.some(root => destination.startsWith(root))) continue
+            if (markersByDestination.has(destination)) continue
+            // Gone entirely is a different fault, and the engine's
+            // missing-output path already has it. Reporting it here as well
+            // would name the wrong cause.
+            if (outputMissing(snapshot.destination)) continue
+            unfinished.set(snapshot.id, destination)
+        }
+        if (unfinished.size) {
+            const damaged = []
+            for (const [id, destination] of unfinished) {
+                const entity = await findEntity({ id })
+                // Source gone: the orphan sweep removes the derivative at
+                // finalize. Nothing to re-derive and nothing to say.
+                if (!entity) continue
+                assetsMap[entity.id] ??= await getEntityPresets(entity)
+                // Forced past the reuse check as well as scheduled: the file
+                // on disk is exactly what must not be trusted, and
+                // isPresetRendered cannot tell half-written bytes from
+                // finished ones.
+                presetMoved.add(entity.id)
+                entitiesToRender.set(entity.id, entity)
+                damaged.push(destination)
+            }
+            // Only when something is actually being re-derived. The warning
+            // claimed the action rather than reporting it, so a build that
+            // scheduled nothing still announced a recovery.
+            if (damaged.length) {
+                useLogger().warn({ code: 'preset-unfinished', derivatives: damaged },
+                    'Assets: %d derivative(s) have no completed-render marker and are being re-derived. '
+                    + 'A preset render did not finish — the file left behind is not trustworthy.',
+                    damaged.length)
+            }
+        }
+
         // --render-presets [name]: re-derive though nothing moved.
         //
         // The escape hatch for what the incremental machinery cannot see — a

package/src/plugins/render/preset.js

@@ -1,4 +1,4 @@
-import { mkdir } from 'node:fs/promises'
+import { mkdir, rm, stat } from 'node:fs/promises'
 import path from 'node:path'
 
 // A renderer's `load` runs for EVERY entity in the cycle, not only the ones
@@ -23,9 +23,42 @@ export async function load({ entity, runtime }) {
     runtime.preset = preset.default
 }
 
+async function describe(file) {
+    try {
+        const { mtimeMs, size } = await stat(file)
+        return { mtimeMs, size }
+    } catch { return null }
+}
+
+// A render that failed leaves nothing half-written behind.
+//
+// This is the one place user code is handed a final output path and left to
+// fill it, and a half-written derivative there is not self-correcting. The
+// marker `isPresetRendered` consults is keyed on the SOURCE checksum, and an
+// interruption does not change the source; the file exists, so the engine's
+// missing-output path does not fire either. Both gates therefore say "nothing
+// to do" over a truncated file, for as long as nobody looks. Measured on a
+// 20KB fixture: interrupt one re-render and every later build serves 10KB of
+// it. Worse where the preset wraps a tool whose non-zero exit it does not
+// check — it RESOLVES, the manifest snapshots the truncated bytes, and
+// --audit-output then reads green, because it compares each output against
+// the hash its own render recorded.
+//
+// Removed only when THIS render is what changed it. A preset that fails
+// before writing anything still has a good derivative on disk, and deleting
+// that would take a working asset off the site until the next build — so the
+// file is measured before and after, and left alone if it did not move.
 export async function render({ entity, options, config, context, plugins, runtime, state, logger }) {
     await mkdir(path.dirname(entity.destination), { recursive: true })
-    await runtime.preset({ entity, options, config, context, plugins, runtime, state, logger })
+    const before = await describe(entity.destination)
+    try {
+        await runtime.preset({ entity, options, config, context, plugins, runtime, state, logger })
+    } catch (error) {
+        const after = await describe(entity.destination)
+        const touched = after && (!before || after.size !== before.size || after.mtimeMs !== before.mtimeMs)
+        if (touched) await rm(entity.destination, { force: true })
+        throw error
+    }
     return entity.destination
 }