mikser-io 10.10.0 → 10.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.10.0",
+  "version": "10.11.0",
   "files": [
     "app.js",
     "index.js",

package/src/engine.js

@@ -835,7 +835,17 @@ The full version, with what each code means: docs/diagnostics.md`)
         // creates for itself. By `import` every onLoaded has run and the
         // registry is complete. Nothing is imported, because this exits first,
         // the same way --explain and --audit-output do.
-        if (runtime.options.tools || runtime.options.tool) {
+        //
+        // --audit-output is here for the same reason, found the same way. It
+        // asks which trees hold outputs, and a plugin cannot answer until its
+        // own onLoaded has run — the assets plugin resolves the folder its
+        // derivatives live in there, because `--assets` is a plugin option
+        // and stage two of the parse is what makes it readable. Dispatched
+        // from the engine's onLoaded it ran ahead of all of them, so the one
+        // check that reports unclaimed files could not see the one tree whose
+        // files reach the site through a symlink it does not follow. It
+        // reported `0 orphaned` over any amount of debris.
+        if (runtime.options.tools || runtime.options.tool || runtime.options.auditOutput) {
             const code = await runReportOnly()
             if (code !== null) process.exit(code)
         }
@@ -877,14 +887,16 @@ The full version, with what each code means: docs/diagnostics.md`)
         // one implementation, so a forwarded --audit-output cannot disagree with a
         // local one about what it checked.
         //
-        // --tools and --tool are NOT among them: they are dispatched at `import`
-        // instead, because the tool registry is not complete until every
-        // plugin's onLoaded has run and this hook runs ahead of all of them.
+        // --tools, --tool and --audit-output are NOT among them: they are
+        // dispatched at `import` instead, because what they read is not
+        // complete until every plugin's onLoaded has run and this hook runs
+        // ahead of all of them — the tool registry for the first two, the set
+        // of trees that hold outputs for the third.
         // The import dispatch existed already and was documented as
         // load-bearing, but this call reached the same code first and exited,
         // so it never ran — and every tool a plugin registers was missing from
         // the CLI while the listing looked healthy, just short.
-        if (!runtime.options.tools && !runtime.options.tool) {
+        if (!runtime.options.tools && !runtime.options.tool && !runtime.options.auditOutput) {
             const code = await runReportOnly()
             if (code !== null) process.exit(code)
         }

package/src/manifest.js

@@ -1102,7 +1102,7 @@ export function createManifest(db) {
         // Walk the output folder against recorded snapshots, returning
         // a diff describing missing / mismatched / orphaned /
         // unverifiable. Backs `mikser --audit-output`. Pure: no mutations.
-        async auditOutput({ outputFolder } = {}) {
+        async auditOutput({ outputFolder, roots } = {}) {
             outputFolder = outputFolder || runtime.options.outputFolder
             const missing = []
             const mismatched = []
@@ -1112,16 +1112,12 @@ export function createManifest(db) {
                 const snap = rowToSnap(row)
                 if (!snap.destination) continue
                 const filePath = resolveOutputPath(snap.destination, outputFolder)
-                // Orphan detection compares against a globby walk of
-                // outputFolder, so `claimed` has to hold exactly the relative
-                // form that walk produces. A destination resolving outside
-                // outputFolder can never appear in it and is not claimable;
-                // one inside it must be claimed by its relative path, not by
-                // the raw string with its leading slashes stripped.
-                const relative = path.relative(outputFolder, filePath)
-                if (relative && !relative.startsWith('..') && !path.isAbsolute(relative)) {
-                    claimed.add(relative)
-                }
+                // Claimed by ABSOLUTE path, so the set does not depend on the
+                // relative form any particular walk produces. It was keyed on
+                // a path relative to outputFolder, which worked only while
+                // that was the one tree walked — and quietly claimed nothing
+                // for every destination outside it.
+                claimed.add(path.resolve(filePath))
                 if (!existsSync(filePath)) {
                     missing.push({ id: snap.id, destination: snap.destination })
                     continue
@@ -1150,15 +1146,47 @@ export function createManifest(db) {
                 }
             }
             const { globby } = await import('globby')
-            const onDisk = await globby('**/*', {
-                cwd: outputFolder,
-                onlyFiles: true,
-                followSymbolicLinks: false,
-            })
+            // Every tree mikser writes into, not only the output folder.
+            //
+            // Preset derivatives live at the working-folder root and reach
+            // the site through a symlink, which the walk does not follow — so
+            // a file nothing claims there was invisible to this check, which
+            // is the one place the concept of an orphan exists. Plugins
+            // declare their own trees on `runtime.options.auditRoots`,
+            // carrying their own ignore patterns, so nothing here has to know
+            // what a preset is or that a `.md5` beside a derivative is
+            // bookkeeping rather than output.
+            const declared = [{ path: outputFolder }, ...(roots ?? runtime.options.auditRoots ?? [])]
+            const walked = []
+            for (const root of declared) {
+                const resolved = path.resolve(root.path ?? root)
+                // A root inside one already walked would report every file in
+                // it twice.
+                const nested = walked.some(({ resolved: seen }) =>
+                    resolved === seen || resolved.startsWith(seen + path.sep))
+                if (nested) continue
+                walked.push({ resolved, ignore: root.ignore ?? [] })
+            }
             const orphaned = []
-            for (const rel of onDisk) {
-                if (claimed.has(rel)) continue
-                orphaned.push({ path: rel })
+            for (const { resolved, ignore } of walked) {
+                const onDisk = await globby('**/*', {
+                    cwd: resolved,
+                    onlyFiles: true,
+                    followSymbolicLinks: false,
+                    ignore,
+                })
+                for (const rel of onDisk) {
+                    if (claimed.has(path.join(resolved, rel))) continue
+                    // Relative to the output folder for the output folder, as
+                    // before; relative to the working folder for anything
+                    // else, because `web/media/x.jpg` alone does not say which
+                    // tree it is in.
+                    const isOutput = resolved === path.resolve(outputFolder)
+                    const display = isOutput
+                        ? rel
+                        : path.relative(runtime.options.workingFolder ?? resolved, path.join(resolved, rel))
+                    orphaned.push({ path: display, root: resolved })
+                }
             }
             // Reported alongside, not as a mismatch: two entities claiming one
             // destination usually produces NO mismatch at all, because each

package/src/plugins/assets.js

@@ -418,11 +418,12 @@ export function assets(options = {}) {
     }
 
     async function forgetPresetMarkers(entity) {
-        for (const marker of markersByDestination.get(entity.destination) ?? []) {
+        const destination = path.resolve(entity.destination)
+        for (const marker of markersByDestination.get(destination) ?? []) {
             await rm(marker, { force: true })
             checksumMap.delete(marker)
         }
-        markersByDestination.delete(entity.destination)
+        markersByDestination.delete(destination)
     }
 
     async function renderPresets(entities, { rendererChanged = new Set() } = {}) {
@@ -504,6 +505,7 @@ export function assets(options = {}) {
         runtime.engine ??= {}
         runtime.engine.assets = { explainMissing }
 
+
         // `??`, not `||`: an option commander did not see is undefined, and
         // falling through to the config is the point. `||` would also fall
         // through for an intentional empty string, which is a different
@@ -515,6 +517,28 @@ export function assets(options = {}) {
 
         runtime.options.assets = runtime.options.assets ?? options.assetsFolder ?? 'assets'
         runtime.options.assetsFolder = path.join(runtime.options.workingFolder, runtime.options.assets)
+        // This tree holds outputs too, and --audit-output could not see it:
+        // it walks the output folder, and derivatives reach the site through
+        // a symlink the walk does not follow. Declared rather than hardcoded
+        // in the manifest, with the ignore that only this plugin can state —
+        // a `.md5` beside a derivative is bookkeeping, and without excluding
+        // them every derivative on the site would report one orphan.
+        //
+        // `auditIgnore` is the escape hatch for a preset that writes MORE
+        // than one file. The engine records the one destination it handed
+        // over, so a poster frame written beside a video is genuinely
+        // unclaimed and genuinely reported — correct, and a permanent
+        // non-zero exit for a legitimate preset if there were no way to say
+        // "these are expected". Stated in config, next to the preset that
+        // produces them, rather than inferred.
+        const auditRoot = {
+            path: runtime.options.assetsFolder,
+            ignore: ['**/*.md5', ...(options.auditIgnore ?? [])],
+        }
+        runtime.options.auditRoots = [
+            ...(runtime.options.auditRoots ?? []).filter(root => root.path !== auditRoot.path),
+            auditRoot,
+        ]
         logger.debug('Assets folder: %s', runtime.options.assetsFolder)
 
         // --clear means "throw away what was derived and build it again", and
@@ -702,27 +726,22 @@ export function assets(options = {}) {
 
         checksumMap.clear()
         markersByDestination.clear()
-        // Scoped to the preset folders rather than the whole assets tree: the
-        // assets folder also carries what the files plugin symlinks into it,
-        // and those have no markers and never will. Unscoped, every one of
-        // them would read as an interrupted render on every cycle.
-        const presetFolders = Object.keys(runtime.state.assets.presets).map(name => `${name}/**/*`)
-        const files = presetFolders.length
-            ? await globby(presetFolders, { cwd: runtime.options.assetsFolder, onlyFiles: true })
-            : []
-        const derivatives = new Set()
-        for (let file of files) {
-            const absolute = path.join(runtime.options.assetsFolder, file)
-            if (!file.endsWith('.md5')) {
-                derivatives.add(absolute)
-                continue
-            }
-            checksumMap.add(absolute)
+        const checksumFiles = await globby('**/*.md5', { cwd: runtime.options.assetsFolder })
+        for (let checksumFile of checksumFiles) {
+            const marker = path.join(runtime.options.assetsFolder, checksumFile)
+            checksumMap.add(marker)
             // `<destination>.<revision>.md5` — the same shape the orphan
             // sweep strips to get back to the derivative.
-            const destination = absolute.replace(/\.[^.]+\.md5$/, '')
+            //
+            // Resolved on both sides of the comparison, because the other
+            // side is a manifest destination and these two are built by
+            // different subsystems. It is a no-op while assetsFolder is
+            // derived from workingFolder (see onLoaded), which is why no test
+            // can tell it apart — normalization at the boundary, not a
+            // transform anything depends on.
+            const destination = path.resolve(marker.replace(/\.[^.]+\.md5$/, ''))
             if (!markersByDestination.has(destination)) markersByDestination.set(destination, [])
-            markersByDestination.get(destination).push(absolute)
+            markersByDestination.get(destination).push(marker)
         }
 
         const entitiesToRender = new Map()
@@ -745,28 +764,63 @@ export function assets(options = {}) {
         // Empty on any healthy build, and the catalog walk is behind that
         // emptiness: this costs one Set lookup per cycle until something has
         // actually gone wrong.
-        const unmarked = new Set()
-        for (const derivative of derivatives) {
-            if (!markersByDestination.has(derivative)) unmarked.add(derivative)
+        // Asked of the MANIFEST, not of the folder.
+        //
+        // The first version walked every file under each preset folder and
+        // called anything without a marker an interrupted render. But only
+        // the destination the engine hands the preset gets a marker, so a
+        // preset that legitimately writes MORE than one file — a poster frame
+        // beside a video — left permanent evidence of a failure that never
+        // happened. It warned on every build for ever, said the derivative
+        // was "being re-derived", and re-derived nothing, because no entity's
+        // destination ever matched the extra file. A warning that cries wolf
+        // on every build is worse than the silence it replaced, and it
+        // teaches people to ignore the one that matters.
+        //
+        // A file nothing claims is an ORPHAN, which is a different fault with
+        // its own report. This asks only about outputs the manifest recorded,
+        // which is also what makes the recovery possible: a snapshot carries
+        // the entity id, so there is nothing to reverse-map and no catalog to
+        // walk.
+        const presetRoots = Object.keys(presets)
+            .map(name => path.resolve(runtime.options.assetsFolder, name) + path.sep)
+        const unfinished = new Map()
+        for (const snapshot of runtime.manifest?.all?.() ?? []) {
+            if (!snapshot.destination) continue
+            const destination = path.resolve(snapshot.destination)
+            if (!presetRoots.some(root => destination.startsWith(root))) continue
+            if (markersByDestination.has(destination)) continue
+            // Gone entirely is a different fault, and the engine's
+            // missing-output path already has it. Reporting it here as well
+            // would name the wrong cause.
+            if (outputMissing(snapshot.destination)) continue
+            unfinished.set(snapshot.id, destination)
         }
-        if (unmarked.size) {
-            for await (const candidate of iterateEntities({ collection: { $ne: collection } })) {
-                const candidatePresets = await getEntityPresets(candidate)
-                const damaged = candidatePresets.filter(name => presets[name]
-                    && unmarked.has(presetDestination(candidate, presets[name])))
-                if (!damaged.length) continue
-                assetsMap[candidate.id] ??= candidatePresets
+        if (unfinished.size) {
+            const damaged = []
+            for (const [id, destination] of unfinished) {
+                const entity = await findEntity({ id })
+                // Source gone: the orphan sweep removes the derivative at
+                // finalize. Nothing to re-derive and nothing to say.
+                if (!entity) continue
+                assetsMap[entity.id] ??= await getEntityPresets(entity)
                 // Forced past the reuse check as well as scheduled: the file
                 // on disk is exactly what must not be trusted, and
-                // isPresetRendered has no way to tell half-written bytes from
+                // isPresetRendered cannot tell half-written bytes from
                 // finished ones.
-                presetMoved.add(candidate.id)
-                entitiesToRender.set(candidate.id, candidate)
+                presetMoved.add(entity.id)
+                entitiesToRender.set(entity.id, entity)
+                damaged.push(destination)
+            }
+            // Only when something is actually being re-derived. The warning
+            // claimed the action rather than reporting it, so a build that
+            // scheduled nothing still announced a recovery.
+            if (damaged.length) {
+                useLogger().warn({ code: 'preset-unfinished', derivatives: damaged },
+                    'Assets: %d derivative(s) have no completed-render marker and are being re-derived. '
+                    + 'A preset render did not finish — the file left behind is not trustworthy.',
+                    damaged.length)
             }
-            useLogger().warn({ code: 'preset-unfinished', derivatives: [...unmarked], sources: entitiesToRender.size },
-                'Assets: %d derivative(s) have no completed-render marker and are being re-derived. '
-                + 'A preset render did not finish — the file left behind is not trustworthy.',
-                unmarked.size)
         }
 
         // --render-presets [name]: re-derive though nothing moved.