mikser-io 9.80.0 → 9.83.0

package/app.js

@@ -43,10 +43,25 @@ function locate(argv) {
         : tool                    ? { type: 'report', tool, toolArgs: value('--tool-args'), json: has('--json') }
         : explain                 ? { type: 'report', explain, json: has('--json') }
         : has('--audit-output')   ? { type: 'report', auditOutput: true, json: has('--json') }
-        : { type: 'build', clear: has('--clear'), renderPresets: has('--render-presets') ? (value('--render-presets') ?? true) : undefined }
+        : { type: 'build',
+            clear: has('--clear'),
+            // Not a flag that happens to be set — the client's OUTPUT
+            // CONTRACT. `--json` promises the document on stdout and every
+            // log line on stderr, and BOTH halves are decided by the process
+            // that does the writing. Forwarded, that is the instance, which
+            // was started without the flag. So the contract has to travel
+            // with the request or it is not honoured at all.
+            json: has('--json'),
+            renderPresets: has('--render-presets') ? (value('--render-presets') ?? true) : undefined }
 
     return {
         longRunning,
+        // The flags this process was invoked with, forwarded so the INSTANCE
+        // can reject an unknown one. This pre-parser reads the few options it
+        // needs and passes the rest along, so commander never runs on a
+        // forwarded invocation — and a typo was accepted in silence, built,
+        // and reported success.
+        argv,
         workingFolder: value('--working-folder', '-i') ?? '.',
         config: value('--config', '-c') ?? 'mikser.config.js',
         request,
@@ -74,7 +89,7 @@ async function main() {
         const code = await forward({
             workingFolder,
             config: path.resolve(where.workingFolder, where.config),
-            request: where.request,
+            request: { ...where.request, argv: where.argv },
         })
         // null means nobody was listening — carry on exactly as before.
         if (code !== null) process.exit(code)

package/docs/diagnostics.md

@@ -796,6 +796,19 @@ queue. A client that writes a file and immediately asks can beat the file
 event, and draining would then build without the change that prompted the
 request.
 
+`--json`, `--tool` and `--tools` keep their stream contract across the socket:
+the document on stdout, the log on stderr, so `mikser --json | jq` means the
+same thing whether or not something is listening. Both halves of that contract
+are decided by the process that writes, which when forwarding is the instance —
+so the flag travels with the request, and each captured chunk is replayed on
+the stream the instance actually used. The contract applies to that one request
+and is restored afterwards: asking for a document does not put the instance
+into json mode for its own output or for the next caller.
+
+`--audit-output` is the exception, and it is the same with or without an
+instance: it reports through the log and writes no document, so `--json` there
+only moves its output to stderr.
+
 ## Faults
 
 A **fault** is a subsystem saying it cannot do its job — as opposed to an
@@ -914,14 +927,39 @@ surfaces that turn silence into a statement:
   evidence:
   - **The emitted output**, read back and resolved the way a browser
     would — `src`, `href`, `poster`, `srcset` and CSS `url()` across
-    html and css. Anything resolving to no file warns under
-    `reference-broken`, naming the url and the pages carrying it. This
-    one sees paths written by hand, not just helper output.
+    html and css. This one sees paths written by hand, not just helper
+    output, and it separates three problems that share a symptom:
+    `reference-no-derivative` when the assets plugin can say why the file
+    is not there, `reference-wrong-base` when the file exists elsewhere in
+    the output (the url was built from the wrong root, and the report names
+    where the file actually is), `reference-broken` when nothing produced
+    it at all.
   - **The render track**, which records every `asset()` / `resource()`
     call and tests the destination it built. This catches a url that
     never reaches an html file at all — one emitted into a feed or a
     sitemap — and warns under `asset-missing`. Where both can see the
     same file, the output scan reports it and this one stays quiet.
+
+  A url under the assets folder gets a cause rather than a guess. Whether a
+  preset covers a file is decided by `match` against the entity id, which is
+  not visible from a url, so the assets plugin is asked and the answer is one
+  of four: the preset name is not configured; the preset does not cover this
+  file (with the `match` that decided it, and any preset that *does* cover it,
+  since the fix is then in the template); the preset covers it and the
+  derivative still is not there, so a render failed; or there is no source
+  file under that name at all. It travels into `--json` as `reason`.
+
+  This matters most where it looks least like itself. `files()` copies the
+  source into the output, so a derivative that was never produced leaves a
+  file of the same name sitting elsewhere — and the wrong-base search finds
+  it and reports a misplaced file with complete confidence. The base is
+  right; the derivative does not exist. A real answer wins over the
+  heuristic wherever there is one.
+
+  `asset()` looks nothing up. It takes a path rather than an entity and is
+  the hottest call in a render, so it stays a string operation; the question
+  is answered once per missing destination after the cycle, and only when
+  something is already wrong.
 - **A link that works only by accident** — a url with one `..` too many
   still loads, because a browser discards a climb above the origin root
   rather than failing. Reported under `reference-over-deep`, separately

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "9.80.0",
+  "version": "9.83.0",
   "files": [
     "app.js",
     "index.js",

package/src/engine.js

@@ -10,7 +10,7 @@ import { instanceControl } from './instance.js'
 import { useJournal, updateEntry } from './journal.js'
 import { globby } from 'globby'
 import { OPERATION, TASKS } from './constants.js'
-import { changeExtension, formatErrorContext, projectMeta, lookupKeys } from './utils.js'
+import { changeExtension, formatErrorContext, projectMeta, lookupKeys, siteRootFor } from './utils.js'
 import { reportRendered, reportSkipped, reportError, renderErrorCount, emitReport, finishCycle, reportAssetUse, assetUse } from './report.js'
 import { checkReferences } from './references.js'
 import { toolSchemas, invokeTool, toolResultText, toolResultFailed } from './tools.js'
@@ -116,15 +116,62 @@ async function reportBrokenReferences(logger) {
     const named = (files) =>
         files.slice(0, 3).join(', ') + (files.length > 3 ? ` and ${files.length - 3} more` : '')
 
-    for (const { url, target, files } of broken.slice(0, SHOWN)) {
-        logger.warn({ code: 'reference-broken', url, target, files },
-            'Resolves to nothing: %s (from %s) — %s', url, named(files), target)
+    // Ask the assets plugin before guessing.
+    //
+    // A derivative that was never produced has its SOURCE sitting in the
+    // output — files() copies it there — so the same-name search finds it and
+    // "the file is at media/icons/logo.svg" reads as a base that is off by a
+    // folder. It is not: the base is right and the derivative does not exist,
+    // because the preset does not cover that file. Confidently naming the
+    // wrong cause is worse than naming none, so a real answer wins over the
+    // heuristic wherever there is one.
+    const explain = runtime.state?.assets?.explainMissing
+    const reasons = new Map()
+    if (explain) {
+        for (const { target } of broken.slice(0, SHOWN)) {
+            // Site-relative, because that is what the assets folder is named
+            // relative to. A build that deploys out/<lang> as its own domain
+            // root resolves this target to `a/derived/web/...`, and a plain
+            // prefix test on the assets folder sees `a` and gives up — the
+            // check goes quiet on exactly the multi-site builds where a
+            // derivative is shared into each root.
+            const root = siteRootFor(target, siteRoots)
+            const local = root ? target.slice(root.length).replace(/^\/+/, '') : target
+            try { reasons.set(target, await explain(local)) } catch { /* never break the report */ }
+        }
+    }
+
+    for (const { url, target, files, elsewhere } of broken.slice(0, SHOWN)) {
+        const reason = reasons.get(target)
+        // Two different problems wear the same symptom. A target whose file
+        // exists elsewhere in the output is a base that is wrong, not an asset
+        // that is missing — and saying which one saves the reader the search.
+        if (reason) {
+            logger.warn({ code: 'reference-no-derivative', url, target, files, reason },
+                'No derivative was produced: %s (from %s) — %s',
+                url, named(files), reason)
+        } else if (elsewhere?.length) {
+            logger.warn({ code: 'reference-wrong-base', url, target, files, elsewhere },
+                'Points at the wrong place: %s (from %s) — nothing at %s, but the file is at %s',
+                url, named(files), target, elsewhere.join(', '))
+        } else {
+            logger.warn({ code: 'reference-broken', url, target, files },
+                'Resolves to nothing, and nothing produced it: %s (from %s) — %s',
+                url, named(files), target)
+        }
     }
     if (broken.length) {
-        logger.warn({ code: 'reference-broken-summary', broken: broken.length, checked },
-            '%d of %d reference(s) in the output resolve to nothing%s. A URL helper builds the '
-            + 'path rather than looking it up, so these are links to files nothing produced.',
-            broken.length, checked, broken.length > SHOWN ? `, ${SHOWN} shown` : '')
+        const explained = broken.slice(0, SHOWN).filter(b => reasons.get(b.target)).length
+        const misplaced = broken.filter(b => b.elsewhere?.length && !reasons.get(b.target)).length
+        logger.warn({
+            code: 'reference-broken-summary',
+            broken: broken.length, wrongBase: misplaced, noDerivative: explained, checked,
+        },
+            '%d of %d reference(s) in the output resolve to nothing%s. No derivative produced: %d. '
+            + 'Wrong base (the file exists elsewhere): %d. Never produced: %d. A URL helper builds the '
+            + 'path rather than looking it up, so none of them can fail at the point it is written.',
+            broken.length, checked, broken.length > SHOWN ? `, ${SHOWN} shown` : '',
+            explained, misplaced, broken.length - misplaced - explained)
     }
 
     // Grouped by how FAR each climbed, because a site whose every over-deep url
@@ -199,10 +246,19 @@ async function reportMissingAssets(logger, alreadyReported = new Set()) {
     // every page on the site, and a thousand lines of it buries whatever else
     // the build said.
     const SHOWN = 10
+    const explain = runtime.state?.assets?.explainMissing
     for (const [destination, ids] of missing.slice(0, SHOWN)) {
-        logger.warn({ code: 'asset-missing', destination, referencedBy: ids },
-            'Linked but not in the output: %s — referenced by %s', destination,
-            ids.slice(0, 3).join(', ') + (ids.length > 3 ? ` and ${ids.length - 3} more` : ''))
+        // Same question, same answer, wherever the symptom surfaces. This path
+        // sees urls that never reach an html file at all — a sitemap, a feed —
+        // which the output scan cannot look at.
+        let reason = null
+        if (explain) {
+            try { reason = await explain(destination) } catch { /* never break the report */ }
+        }
+        logger.warn({ code: 'asset-missing', destination, referencedBy: ids, reason },
+            'Linked but not in the output: %s — referenced by %s%s', destination,
+            ids.slice(0, 3).join(', ') + (ids.length > 3 ? ` and ${ids.length - 3} more` : ''),
+            reason ? `. ${reason[0].toUpperCase()}${reason.slice(1)}` : '')
     }
     logger.warn({ code: 'asset-missing-summary', missing: missing.length, checked: used.length },
         '%d of %d linked file(s) are not in the output%s. A URL helper builds the path rather than looking it '

package/src/instance.js

@@ -125,8 +125,11 @@ export function forward({ workingFolder, config, request }) {
         readFrames(socket, (message) => {
             if (message.type === 'log') {
                 // The instance's output for THIS request, on the stream it
-                // would have used locally.
-                process.stderr.write(message.chunk)
+                // would have used locally — which requires knowing which
+                // stream that was. An instance too old to say defaults to
+                // stderr, which is what every frame used to mean.
+                const out = message.stream === 'stdout' ? process.stdout : process.stderr
+                out.write(message.chunk)
             } else if (message.type === 'refused') {
                 answered = true
                 process.stderr.write(`mikser: ${message.reason}\n`)
@@ -197,14 +200,22 @@ let server = null
 // adding a log transport means the client sees precisely what it would have
 // seen locally, formatting and all, with no second rendering of the same
 // records to keep in step.
+// WHICH stream, not merely that something was written.
+//
+// stdout and stderr are not two ways of saying the same thing: under --json,
+// --tool and --tools, stdout carries a machine-readable document and stderr
+// carries the log, and the split is the entire value of those flags. Capturing
+// both into one undifferentiated stream throws that away, and the client can
+// only guess — it guessed stderr, so every forwarded document landed where no
+// consumer looks while the command exited 0.
 function captureOutput(onChunk) {
     const originals = [process.stdout.write, process.stderr.write]
-    const patch = (stream, original) => function (chunk, encoding, callback) {
-        try { onChunk(typeof chunk === 'string' ? chunk : chunk.toString()) } catch { /* client gone */ }
+    const patch = (stream, original, name) => function (chunk, encoding, callback) {
+        try { onChunk(typeof chunk === 'string' ? chunk : chunk.toString(), name) } catch { /* client gone */ }
         return original.call(stream, chunk, encoding, callback)
     }
-    process.stdout.write = patch(process.stdout, originals[0])
-    process.stderr.write = patch(process.stderr, originals[1])
+    process.stdout.write = patch(process.stdout, originals[0], 'stdout')
+    process.stderr.write = patch(process.stderr, originals[1], 'stderr')
     return () => {
         process.stdout.write = originals[0]
         process.stderr.write = originals[1]
@@ -264,10 +275,10 @@ async function configStale() {
 // the only process that can answer correctly. Same guards as a build: wrong
 // config refuses, drifted config refuses.
 async function serveReport(socket, request, logger) {
-    const restore = captureOutput((chunk) => frame(socket, { type: 'log', chunk }))
+    const restore = captureOutput((chunk, stream) => frame(socket, { type: 'log', chunk, stream }))
     let code = 0
     try {
-        code = await runReportOnly(request) ?? 0
+        code = await withRequestOutput(request, () => runReportOnly(request)) ?? 0
     } catch (err) {
         logger?.error('instance: forwarded report failed — %s', err.message)
         code = 3
@@ -286,6 +297,39 @@ function refuseConfig(socket, request, wrongConfig) {
     })
 }
 
+// An option the instance's own commander does not recognise.
+//
+// A forwarded invocation never reaches commander: app.js pre-parses the few
+// flags it needs and forwards the rest, so `mikser --bogus` against a running
+// watcher built normally and exited 0, while the same command with nothing
+// listening was rejected outright. Same words, two answers, and the quiet one
+// is the one that looks like it worked.
+//
+// parseOptions REPORTS unknowns without applying anything, which is what makes
+// this safe to run against the live instance's option table.
+function refuseUnknownFlags(socket, request) {
+    const argv = Array.isArray(request.argv) ? request.argv : null
+    if (!argv?.length) return false
+    const commander = runtime.engine?.commander
+    if (!commander) return false
+    let unknown = []
+    try {
+        unknown = commander.parseOptions([...argv]).unknown ?? []
+    } catch {
+        return false   // never let a probe refuse a legitimate build
+    }
+    unknown = unknown.filter(a => a.startsWith('-'))
+    if (!unknown.length) return false
+    frame(socket, {
+        type: 'refused',
+        reason: `unknown option ${unknown.map(u => `'${u}'`).join(', ')}.`,
+        detail: 'Forwarded to the instance running in this folder, which does not recognise it. '
+            + 'The same command with nothing listening would have been rejected too — this says so '
+            + 'rather than building as though the flag had been understood.',
+    })
+    return true
+}
+
 function refuseStale(socket, movedFile) {
     frame(socket, {
         type: 'refused',
@@ -295,8 +339,37 @@ function refuseStale(socket, movedFile) {
     })
 }
 
+// The client's output contract, applied to the instance for one request.
+//
+// `--json` is answered by two pieces of code that both read runtime.options:
+// the logger picks its stream per line, and emitReport() writes nothing at all
+// unless the option is set. The instance was started without the flag, so a
+// forwarded `--json` was answered under the INSTANCE's contract — a build
+// emitted no document whatsoever, and a report emitted one into the same
+// stream as the log it was supposed to be separated from.
+//
+// Tighten only, never loosen. A client asking for a document states something
+// the instance cannot know; a client not asking states nothing, and silencing
+// an instance that was itself started under `--json` would take a document
+// away from whatever is reading ITS stdout.
+//
+// Per request and restored after, like renderPresets: one client's flag does
+// not put the instance into json mode for everyone.
+async function withRequestOutput(request, run) {
+    const prior = {}
+    for (const key of ['json', 'tool', 'tools']) {
+        prior[key] = runtime.options[key]
+        if (request[key]) runtime.options[key] = request[key]
+    }
+    try {
+        return await run()
+    } finally {
+        Object.assign(runtime.options, prior)
+    }
+}
+
 async function serveBuild(socket, request, logger) {
-    const restore = captureOutput((chunk) => frame(socket, { type: 'log', chunk }))
+    const restore = captureOutput((chunk, stream) => frame(socket, { type: 'log', chunk, stream }))
     let code = 0
     try {
         // Fire the pending debounce rather than waiting it out.
@@ -323,7 +396,11 @@ async function serveBuild(socket, request, logger) {
         const priorRenderPresets = runtime.options.renderPresets
         if (request.renderPresets !== undefined) runtime.options.renderPresets = request.renderPresets
         try {
-            await runtime.rebuild()
+            // The report is emitted by the cycle itself, from inside
+            // rebuild() — the same call a one-shot makes. Nothing here
+            // re-implements it: the contract is what decides whether it
+            // writes, so setting the contract is the whole fix.
+            await withRequestOutput(request, () => runtime.rebuild())
         } finally {
             runtime.options.renderPresets = priorRenderPresets
         }
@@ -373,6 +450,7 @@ export function serveInstance() {
                     // instance's — a report against the wrong config is the
                     // original incident, and it is wrong whether or not it
                     // writes anything.
+                    if (refuseUnknownFlags(socket, request)) return
                     const wrongConfig = configMismatch(request.config)
                     if (wrongConfig) return refuseConfig(socket, request, wrongConfig)
                     const movedFile = await configStale()

package/src/plugins/assets.js

@@ -97,19 +97,100 @@ export function assets(options = {}) {
     // reported once per process at onFinalize.
     const matchTally = { evaluated: 0, matched: new Set(), reported: false }
 
+    // Which presets select this entity. No counters, no side effects.
+    //
+    // Split out because the same question gets asked twice for different
+    // reasons: once as files enter the catalog, where the answer drives the
+    // render and feeds the unmatched-preset tally, and once after the cycle to
+    // explain a derivative that is not there. The second must not move the
+    // counters the first reports on.
+    //
+    // `some`, not a push per matching pattern. Two patterns that both cover a
+    // file used to name their preset twice in the list; the second render was
+    // gated by the checksum, so nothing rendered twice — but "which presets
+    // cover this file" is a question with one answer, and the explainer below
+    // puts that answer in front of a reader.
+    function presetsSelecting(entity) {
+        const selected = []
+        for (const preset in (options.presets || {})) {
+            const { matches } = normalizePresetConfig(options.presets[preset])
+            if (matches.some(match => matchEntity(entity, match))) selected.push(preset)
+        }
+        return selected
+    }
+
     async function getEntityPresets(entity) {
-        const entityPresets = []
         matchTally.evaluated++
-        for (let preset in (options.presets || {})) {
-            const { matches } = normalizePresetConfig(options.presets[preset])
-            for (let match of matches) {
-                if (matchEntity(entity, match)) {
-                    entityPresets.push(preset)
-                    matchTally.matched.add(preset)
-                }
+        const selected = presetsSelecting(entity)
+        for (const preset of selected) matchTally.matched.add(preset)
+        return selected
+    }
+
+    // Why a linked derivative is not in the output.
+    //
+    // The engine detects the CONSEQUENCE — it knows what the output points at
+    // and what exists on disk — but it cannot name the cause, because whether
+    // a preset covers a file is decided by `match` against the entity id and
+    // none of that is visible from a url. So one sentence covered a mistyped
+    // preset name, a preset that did not run, and a file no preset was ever
+    // asked to cover. The last is the common one, the only one whose fix is in
+    // the config rather than in the template, and the one the reader is least
+    // likely to guess.
+    //
+    // Answered here rather than in the helper: `asset()` takes a path, not an
+    // entity, and it is the hottest call in a render. Nothing is looked up
+    // when the url is built. This runs at most a handful of times, after
+    // everything has settled, and only when something is already wrong.
+    let sourceIndex = null
+    let sourceIndexCycle
+    async function explainMissing(destination) {
+        const assetsName = runtime.options.assets
+        const parts = String(destination).replace(/^\/+/, '').split('/')
+        if (!assetsName || parts[0] !== assetsName || parts.length < 3) return null
+        const preset = parts[1]
+        const name = parts.slice(2).join('/')
+
+        const configured = Object.keys(options.presets || {}).sort()
+        if (!options.presets?.[preset]) {
+            return `no preset named '${preset}' is configured (configured: ${configured.join(', ') || 'none'})`
+        }
+
+        // The derivative wears the PRESET's extension, so the source is found
+        // by stem — `media/hero.webp` came from `media/hero.jpg`.
+        const stem = (value) => value.slice(0, value.length - path.extname(value).length)
+
+        // Built once per cycle and only on this path: a build with nothing
+        // broken never walks the catalog for it.
+        const cycle = runtime.state?.cycle?.id ?? null
+        if (sourceIndexCycle !== cycle) { sourceIndex = null; sourceIndexCycle = cycle }
+        if (!sourceIndex) {
+            sourceIndex = new Map()
+            for await (const candidate of iterateEntities({ collection: { $ne: collection } })) {
+                if (typeof candidate.name !== 'string') continue
+                const key = stem(candidate.name)
+                if (!sourceIndex.has(key)) sourceIndex.set(key, [])
+                sourceIndex.get(key).push(candidate)
             }
         }
-        return entityPresets
+
+        const source = (sourceIndex.get(stem(name)) ?? [])[0]
+        if (!source) {
+            return `no source file is named '${stem(name)}' — nothing would produce this derivative `
+                + 'under any preset'
+        }
+
+        const owns = presetsSelecting(source)
+        if (owns.includes(preset)) {
+            return `preset '${preset}' does cover ${source.id}, so the derivative should be there — it was `
+                + 'not produced this cycle (the preset render failed, or the preset changed and only '
+                + '--render-presets re-derives what is already in the catalog)'
+        }
+        const { matches } = normalizePresetConfig(options.presets[preset])
+        return `preset '${preset}' does not cover ${source.id} — its match is `
+            + `${matches.join(', ') || '(none)'}`
+            + (owns.length
+                ? `. Presets that do cover it: ${owns.join(', ')}`
+                : '. No configured preset covers it')
     }
 
     // Report presets that matched none of the entities this run evaluated.
@@ -315,6 +396,11 @@ export function assets(options = {}) {
         runtime.state.assets = {
             presets: {},
             assetsMap: {},
+            // The engine asks this when a linked derivative is not on disk.
+            // Published on state rather than imported, so the engine keeps
+            // knowing nothing about presets and says nothing when this plugin
+            // is not loaded.
+            explainMissing,
             assetsFolder: options.outputFolder
                 ? path.join(options.outputFolder, assetsName)
                 : assetsName,

package/src/references.js

@@ -173,5 +173,31 @@ export async function checkReferences(outputFolder, { siteRoots = [] } = {}) {
         }
     }
 
+    // "It was never written" and "it is written somewhere else" are different
+    // problems with one symptom, and they were reported with one sentence.
+    //
+    // A missing target whose FILE exists elsewhere in the output is almost
+    // always a base problem: the url was built from the wrong root, or with a
+    // segment too many. Naming where the file actually is turns "resolves to
+    // nothing" into the answer. A target that exists nowhere really was never
+    // produced — a preset that did not run, an extension nothing emits.
+    //
+    // Indexed only when something is broken, so a clean build pays nothing.
+    if (broken.size) {
+        const byName = new Map()
+        for (const file of await globby('**/*', {
+            cwd: outputFolder, followSymbolicLinks: true, onlyFiles: true, suppressErrors: true,
+        })) {
+            const name = path.basename(file)
+            if (!byName.has(name)) byName.set(name, [])
+            byName.get(name).push(file)
+        }
+        for (const entry of broken.values()) {
+            const elsewhere = (byName.get(path.basename(entry.target)) ?? [])
+                .filter(f => f !== entry.target)
+            if (elsewhere.length) entry.elsewhere = elsewhere.slice(0, 3)
+        }
+    }
+
     return { broken: [...broken.values()], overDeep: [...overDeepRefs.values()], checked }
 }