mikser-io 10.6.0 → 10.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "10.6.0",
+  "version": "10.7.0",
   "files": [
     "app.js",
     "index.js",

package/src/engine.js

@@ -1593,6 +1593,16 @@ The full version, with what each code means: docs/diagnostics.md`)
     // undefined here. The logger has no such problem — it writes during the
     // run, by which time options exist.
     const quietStdout = ['--json', '--tool', '--tools'].some(flag => process.argv.includes(flag))
+    // The banner is an info record, so a level above info must not print it.
+    //
+    // `--log silent` printed the version line and nothing else — the one line
+    // the flag most obviously promises to remove — because this runs before
+    // the level is applied. Only the FORWARDED path was ever silent, and that
+    // is the path the test exercises, so the test could not see it.
+    //
+    // argv again, for the reason quietStdout reads it: commander parses in a
+    // lifecycle hook, which runs after setup() returns.
+    if (!argvWantsInfo()) return runtime
     // Through the logger when nobody is watching, so it gets a timestamp.
     //
     // This line marks a process start, which in a supervisor's log is the
@@ -1625,6 +1635,26 @@ The full version, with what each code means: docs/diagnostics.md`)
     return runtime
 }
 
+// The level asked for on the command line, before commander has parsed.
+// Both spellings, both flags: `--log warn` and `--log=warn`, and
+// `--log-install`, which on a run with no instance to forward to sets the
+// level for this process.
+function argvLogLevel() {
+    for (const flag of ['--log', '--log-install', '-l']) {
+        const at = process.argv.indexOf(flag)
+        if (at >= 0 && process.argv[at + 1]) return process.argv[at + 1]
+        const inline = process.argv.find(arg => arg.startsWith(`${flag}=`))
+        if (inline) return inline.slice(flag.length + 1)
+    }
+    return null
+}
+
+function argvWantsInfo() {
+    const level = argvLogLevel()
+    if (!level || !LOG_LEVELS.includes(level)) return true
+    return LOG_LEVELS.indexOf(level) <= LOG_LEVELS.indexOf('info')
+}
+
 export function useLogger() {
     return runtime.engine?.logger
 }

package/src/instance.js

@@ -218,10 +218,29 @@ let server = null
 // both into one undifferentiated stream throws that away, and the client can
 // only guess — it guessed stderr, so every forwarded document landed where no
 // consumer looks while the command exited 0.
-function captureOutput(onChunk) {
+// `echoStdout: false` frames the chunk to the client and writes NOTHING
+// locally.
+//
+// The instance wears the client's output so an operator watching it sees what
+// a forwarded request did — right for log lines, wrong for a DOCUMENT. A
+// --json request put its 364-line report into the instance's own console as
+// well as onto the client's stdout, and under pm2 that is 364 lines in the out
+// log for every report an agent asks for.
+//
+// The seam is exact rather than a guess: under --json / --tool the logger
+// writes to stderr and stdout carries only the document (see logger.js), so
+// stdout chunks during a document-carrying request ARE the document.
+function captureOutput(onChunk, { echoStdout = true } = {}) {
     const originals = [process.stdout.write, process.stderr.write]
     const patch = (stream, original, name) => function (chunk, encoding, callback) {
         try { onChunk(typeof chunk === 'string' ? chunk : chunk.toString(), name) } catch { /* client gone */ }
+        if (!echoStdout && name === 'stdout') {
+            // write()'s callback is the second argument when encoding is
+            // omitted. Dropped, a caller awaiting the drain never resumes.
+            const done = typeof encoding === 'function' ? encoding : callback
+            if (typeof done === 'function') process.nextTick(done)
+            return true
+        }
         return original.call(stream, chunk, encoding, callback)
     }
     process.stdout.write = patch(process.stdout, originals[0], 'stdout')
@@ -242,6 +261,12 @@ function captureOutput(onChunk) {
 // not what tells the two apart. Different configs are different files.
 // Does this request name a level that does not exist? Pure, so the refusal
 // can be decided before anything is applied.
+// Does this request's answer go on stdout as a document?
+// The same trio logger.js gates the progress bar on, for the same reason.
+function carriesDocument(request) {
+    return Boolean(request?.json || request?.tool || request?.tools)
+}
+
 function logRequestRefusal(request) {
     for (const [flag, level] of [['--log', request?.log], ['--log-install', request?.logInstall]]) {
         if (level !== undefined && level !== null && !LOG_LEVELS.includes(level)) {
@@ -296,7 +321,9 @@ async function configStale() {
 // the only process that can answer correctly. Same guards as a build: wrong
 // config refuses, drifted config refuses.
 async function serveReport(socket, request, logger) {
-    const restore = captureOutput((chunk, stream) => frame(socket, { type: 'log', chunk, stream }))
+    const restore = captureOutput(
+        (chunk, stream) => frame(socket, { type: 'log', chunk, stream }),
+        { echoStdout: !carriesDocument(request) })
     let code = 0
     try {
         code = await withRequestOutput(request, () => runReportOnly(request)) ?? 0
@@ -501,7 +528,9 @@ async function withRequestOutput(request, run) {
 }
 
 async function serveBuild(socket, request, logger) {
-    const restore = captureOutput((chunk, stream) => frame(socket, { type: 'log', chunk, stream }))
+    const restore = captureOutput(
+        (chunk, stream) => frame(socket, { type: 'log', chunk, stream }),
+        { echoStdout: !carriesDocument(request) })
     let code = 0
     try {
         // Fire the pending debounce rather than waiting it out.

package/src/journal.js

@@ -227,9 +227,8 @@ export async function* useJournal(name, operations, signal) {
                 stopProgress()
                 throw new AbortError()
             }
-            updateProgress()
-
             const entry = rowToEntry(row)
+            updateProgress(entry.entity?.id)
             const originalEntity = row.entity   // already a JSON string
 
             yield entry

package/src/logger.js

@@ -28,6 +28,7 @@ import pino from 'pino'
 import pretty from 'pino-pretty'
 import Gauge from 'gauge'
 import { Writable } from 'node:stream'
+import path from 'node:path'
 import runtime from './runtime.js'
 import { useLogger } from './engine.js'
 import { onLoad, onFinalized } from './lifecycle.js'
@@ -482,10 +483,48 @@ onLoad(() => {
     runtime.engine.logger = createMikserLogger(level)
 })
 
-// Progress API. Surface preserved: trackProgress / updateProgress /
-// stopProgress / updateProgressDetails. Gauge-backed; no-op in non-TTY
-// contexts or when runtime.options.info is false (i.e. --debug / --trace
-// modes, where logs are voluminous and a bar would just be noise).
+// Progress API: trackProgress / updateProgress / stopProgress /
+// updateProgressDetails. A gauge where a terminal is watching, coded records
+// where one is not, and nothing at all for a phase that took no time.
+
+// Long phases only, and long means TIME.
+//
+// A record per quartile is right for 800 documents and absurd for 5. Eleven
+// of a plain build's thirteen phases carry exactly five items, so reporting
+// them the way a bar counts took a piped no-op build from 32 lines to 97 —
+// sixty-five progress records, five in the same millisecond to say a phase
+// did nothing. That is written into the log a deployment keeps, which is the
+// log the installed-level expiry exists to protect.
+//
+// A minimum TOTAL would fix the count and get the other half wrong: four PDFs
+// through Chrome is a small phase and a slow one, and it is exactly the phase
+// worth narrating. Elapsed time is what "long" means, it needs no per-phase
+// tuning, and it applies to the drawn path too — `finished: 5 0s` is the same
+// non-information in a terminal, and its own `0s` says so.
+export const PROGRESS_MIN_MS = 1000
+
+// Which item, not just how far. A phase name alone says a build is doing
+// something; the thing a stuck build needs to say is WHAT it is stuck on.
+function progressDetail(detail) {
+    if (detail === null || detail === undefined || detail === '') return null
+    const text = String(detail)
+    const root = runtime.options?.workingFolder
+    if (!root || !path.isAbsolute(text)) return text
+    // An entity id LOOKS absolute and is not a filesystem path.
+    // `/documents/p3706.md` relativised against the working folder becomes
+    // `../../../documents/p3706.md`, which names nothing anyone can act on.
+    // So shorten only what is genuinely INSIDE the folder, and leave every
+    // other string exactly as the call site handed it over.
+    const relative = path.relative(root, text)
+    return relative.startsWith('..') ? text : relative
+}
+
+function emitProgress({ name, total, value, detail }) {
+    const at = progressDetail(detail)
+    const fields = { code: 'progress', phase: name, total, value }
+    if (at) useLogger()?.info({ ...fields, detail: at }, '%s: %d/%d — %s', name, value, total, at)
+    else useLogger()?.info(fields, '%s: %d/%d', name, value, total)
+}
 
 export function trackProgress(name, total) {
     if (!name || !total) return
@@ -512,14 +551,18 @@ export function trackProgress(name, total) {
     // --json would have extended that to every machine reading the output.
     // Losing the graphics is the point; losing the information is not.
     const drawn = !carriesDocument && Boolean(process.stdout.isTTY) && Boolean(runtime.options.info)
-    currentBar = { name, total, value: 0, started: Date.now(), drawn, milestone: 0 }
+    currentBar = { name, total, value: 0, started: Date.now(), drawn, milestone: 0, detail: null }
     if (drawn) ensureGauge().show({ section: name, subsection: `0/${total}` }, 0)
-    else logger.info({ code: 'progress', phase: name, total, value: 0 }, '%s: 0/%d', name, total)
 }
 
-export function updateProgress() {
+export function updateProgress(detail) {
     if (!currentBar) return
     currentBar.value++
+    // The call site hands over whatever identifies the item it is on — a
+    // path, an id — and pays one assignment for it. Formatting happens at
+    // emit time, at most four times a phase, so naming the work costs the
+    // 14k-entity import nothing.
+    if (detail !== undefined) currentBar.detail = detail
     const { name, total, value, drawn } = currentBar
     if (drawn) {
         gauge?.show({ section: name, subsection: `${value}/${total}` }, value / total)
@@ -527,11 +570,14 @@ export function updateProgress() {
         // Quartiles, not every item: a bar redraws in place and costs one
         // line, a log record does not. 800 documents is 800 lines of noise if
         // this counts the way the bar does.
+        //
+        // The milestone advances whether or not the record is emitted. Held
+        // back, every quartile the phase passed under the threshold would fire
+        // in a burst the moment it crossed.
         const reached = Math.floor((value / total) * 4)
         if (reached > currentBar.milestone && value < total) {
             currentBar.milestone = reached
-            useLogger()?.info({ code: 'progress', phase: name, total, value },
-                '%s: %d/%d', name, value, total)
+            if (Date.now() - currentBar.started >= PROGRESS_MIN_MS) emitProgress(currentBar)
         }
     }
     if (value >= total) stopProgress()
@@ -541,14 +587,17 @@ export function stopProgress() {
     if (!currentBar) return
     const logger = useLogger()
     const { name, total, value, started } = currentBar
+    const ms = Date.now() - started
     gauge?.hide()
     // Structured either way, so a machine reading --json's stderr gets the
     // same facts a person reads off the bar.
+    //
+    // A phase that did not finish is worth a line at any duration — it is a
+    // warning, not progress, and the whole point is that it is unexpected.
     if (value < total) {
         logger.warn({ code: 'progress-unfinished', phase: name, total, value, missing: total - value },
             '%s unfinished: %d', name, total - value)
-    } else {
-        const ms = Date.now() - started
+    } else if (ms >= PROGRESS_MIN_MS) {
         logger.info({ code: 'progress-finished', phase: name, total, ms },
             '%s finished: %d %ds', name, total, Math.round(ms / 1000))
     }

package/src/plugins/files.js

@@ -228,7 +228,7 @@ export function files(options = {}) {
                 // nothing to emit. Progress ticks either way — a gated file
                 // was still looked at.
                 const newChecksum = await gateChecksum(source, id, { priorChecksums })
-                updateProgress()
+                updateProgress(id)
                 if (newChecksum === null) return
                 await createEntity({
                     id,

package/src/plugins/observer.js

@@ -100,7 +100,7 @@ export function observer(options = {}) {
                         synced++
                     }
                 }
-                updateProgress()
+                updateProgress(meta.id)
             }
 
             const entitiesToRemove = await findEntities({
@@ -114,7 +114,7 @@ export function observer(options = {}) {
             for (let entity of entitiesToRemove) {
                 deleteEntity(entity)
                 removed++
-                updateProgress()
+                updateProgress(entity.id)
             }
             if (synced || removed) {
                 logger.debug('Syncing api [%s] synced: %d, removed: %d', collection, synced, removed)

package/src/plugins/resources.js

@@ -145,7 +145,7 @@ export function resources(options = {}) {
                     logger.error('Resource error: %s %s %s', entity.id, resource, err.message)
                 }
             }
-            updateProgress()
+            updateProgress(resource)
         }
 
         const resourceFiles = await globby('**/*', { cwd: runtime.options.resourcesFolder })
@@ -226,7 +226,7 @@ export function resources(options = {}) {
                         checksum: await checksum(resource)
                     })
                 }
-                updateProgress()
+                updateProgress(resource)
             }, { concurrency: 10, signal })
             count && logger.info('Downloaded: %d', count)
         }

package/src/source.js

@@ -431,7 +431,7 @@ export function useSource(core, options) {
         // is never the bottleneck.
         await pMap(files, async (file) => {
             await registerFile(file, { logger, scanned, stats: scanStats, priorChecksums })
-            if (phase === 'import') updateProgress()
+            if (phase === 'import') updateProgress(file)
         }, { concurrency: SCAN_CONCURRENCY })
 
         scanStats.deleted = await sweepDeleted(collection, scanned, async (e) => {