@feasibleone/blong-gogo 1.35.0 → 1.37.0

package/src/load.ts

@@ -18,7 +18,7 @@ import {
     type SolutionFactory,
 } from '@feasibleone/blong/types';
 
-import {withProgress} from '@feasibleone/blong-lib';
+import {SLOW_STEP_MS, withProgress} from '@feasibleone/blong-lib';
 import {WELL_KNOWN_LAYERS} from '@feasibleone/blong-lib/layers';
 import {Type, type TSchema} from 'typebox';
 import merge from 'ut-function.merge';
@@ -216,6 +216,42 @@ async function discoverRealmTestMethods(
     return methods;
 }
 
+/**
+ * The margin a step of a load is measured against when the realm names none.
+ *
+ * Read off the log config (`log.slowMs`) so a realm or a runner can raise it for
+ * a machine that is legitimately slow, and so one number tunes both halves of a
+ * delayed start: the loader's steps and the retention store's own.
+ */
+function slowStepMs(log: unknown): number {
+    const configured = (log as {slowMs?: unknown} | undefined)?.slowMs;
+    return typeof configured === 'number' && configured >= 0 ? configured : SLOW_STEP_MS;
+}
+
+/**
+ * A logger for the steps that run before the realm's own log exists.
+ *
+ * A root load builds the log component first, and until it is up there is nothing
+ * to log through — which is exactly when a slow step is least visible and most
+ * confusing, because the process prints nothing at all for as long as it takes.
+ * This writes to stderr, where the framework's other startup messages go, and
+ * stays silent in a platform that has no process (the browser bundle). Only
+ * `warn` is answered: a step that is not slow has nothing to say, and the
+ * loader's ordinary progress belongs in the log the realm is about to have.
+ */
+function bootstrapLogger(): {warn: (...args: unknown[]) => void} {
+    return {
+        // Pino-shaped, like every other logger `withProgress` is handed: the
+        // details object first, the message second.
+        warn: (...args: unknown[]) => {
+            const [details, message] = args as [Record<string, unknown> | undefined, string];
+            const stderr = (globalThis as {process?: {stderr?: {write: (text: string) => void}}})
+                .process?.stderr;
+            stderr?.write(`warn  blong ${message} ${JSON.stringify(details ?? {})}\n`);
+        },
+    };
+}
+
 const System: symbol = Symbol('system');
 
 export function system(original: {[key: symbol]: boolean}): void {
@@ -232,12 +268,24 @@ function activeConfigs<T extends TSchema>(
     platformConfigs: string[],
 ): (boolean | object)[] {
     return (
-        (['default'] as string[])
-            .concat(configNames)
-            .concat(platformConfigs)
-            .map(name => (mod.config as unknown as Record<string, unknown>)?.[name])
-            .filter(Boolean) as (boolean | object)[]
-    ).concat({pkg: mod.pkg, children: mod.children, url: mod.url});
+        (
+            (['default'] as string[])
+                .concat(configNames)
+                .concat(platformConfigs)
+                // A CI run *is* a captured run, so the `ci` block applies whether or not the
+                // entry named the intent — the same reason `exit` is derived from `isCI()`
+                // in the synthetic config. Which entry happened to know about CI decided
+                // whether a package's tap leg wrote Allure results at all, and a package
+                // whose only runner is tap then had no report to link.
+                //
+                // Named last, which is where a caller that passes it puts it: the Playwright
+                // webServer passes `ci` after `playwright` so its colours win over the block's
+                // `log.color = false`.
+                .concat(isCI() ? ['ci'] : [])
+                .map(name => (mod.config as unknown as Record<string, unknown>)?.[name])
+                .filter(Boolean) as (boolean | object)[]
+        ).concat({pkg: mod.pkg, children: mod.children, url: mod.url})
+    );
 }
 
 /** A thenable value that can be externally resolved/rejected. */
@@ -449,7 +497,15 @@ export default async function loadRealm<T extends TSchema>(
     // always false, and every framework realm is silently absent: the gateway then
     // refuses every unauthenticated route (no login realm) and RBAC resolves nothing.
     const isPlatformRoot = api === undefined;
-    const mod = await def({type: Type, manifest});
+    const bootstrap = bootstrapLogger();
+    // A factory may answer with the module or with a promise for it, so the
+    // measurement wraps whatever it answered rather than assuming the shape.
+    const mod = await withProgress(
+        bootstrap,
+        `suite ${name}`,
+        Promise.resolve(def({type: Type, manifest})),
+        {slowMs: SLOW_STEP_MS},
+    );
     // Record the realm's own file in the tree's set, so a realm reached twice -
     // once as a suite's child and once by the framework's `frameworkRealms` - is
     // recognised the second time. A realm answers with its *factory*, so its url
@@ -489,6 +545,26 @@ export default async function loadRealm<T extends TSchema>(
                     default: {
                         watch: {
                             test: [],
+                            /**
+                             * Allure reporting for handler tests. Off by default: a report is
+                             * something a run is asked for, which is what `--watch.allure.enabled=true`
+                             * on the blong CLI does for a dev run and what the `ci` block below does
+                             * for a captured one. `blong-allure` is a dependency of this package, so
+                             * the CLI that turns the results into HTML is the one they were written
+                             * for rather than whatever is on the runner's PATH.
+                             *
+                             * The results have a directory of their own because a package can report
+                             * to Allure from two producers — the browser tests of a realm and its
+                             * handler tests — and each clears the directory it writes to before a run.
+                             * `blong-dev` merges every producer's directory into the one report a
+                             * package publishes.
+                             */
+                            allure: {
+                                enabled: false,
+                                outputDir: 'allure-results-tap',
+                                historyPath: '.allure/history.jsonl',
+                                generateOnEnd: false,
+                            },
                         },
                         // The server's log implementation. The browser platform
                         // keeps its own logger regardless: the loader checks the
@@ -522,7 +598,7 @@ export default async function loadRealm<T extends TSchema>(
                             // The semantic implementation's store, and the pino
                             // implementation's cache beside it: they share one
                             // directory and one key space, so a
-                            // `semantic-log://record/<id>` reference resolves
+                            // `semlog://r/<id>` reference resolves
                             // whichever implementation wrote the entry.
                             cache: {
                                 dir: '~/.blong/log-cache',
@@ -606,9 +682,22 @@ export default async function loadRealm<T extends TSchema>(
                     /**
                      * A run whose output is captured rather than watched. The
                      * Playwright webServer passes this last in CI, so it wins over the
-                     * `playwright` block's colours.
+                     * `playwright` block's colours. A captured run is also the run whose
+                     * report nobody is watching, so its Allure results are written and
+                     * its HTML report generated at the end.
+                     *
+                     * On CI the block applies whether or not the intent was named: the
+                     * environment already says the output is captured, and an entry that
+                     * has to remember to pass `ci` for its package to have a report is a
+                     * trap for the next package (`activeConfigs`).
                      */
-                    ci: {log: {color: false}},
+                    ci: {
+                        log: {color: false},
+                        // Results, not a report: `blong-dev` turns every producer's results
+                        // into the one report a package publishes, so generating here would
+                        // be the same work again, into a directory nobody uploads.
+                        watch: {allure: {enabled: true}},
+                    },
                     /**
                      * A deployed process never runs the service itself: the
                      * service is deployed once, beside the processes whose
@@ -806,7 +895,7 @@ export default async function loadRealm<T extends TSchema>(
         });
     }
     loadedConfigs.push(...activeConfigs(mod, configNames, platformApi.configs));
-    const {loadedConfig: mergedConfig, configRuntime} = await platformApi.loadConfig(
+    const configPromise = platformApi.loadConfig(
         {
             name,
             pkg: {name, version: '0.0.0'},
@@ -834,6 +923,14 @@ export default async function loadRealm<T extends TSchema>(
         parentConfig,
         loadedConfigs.filter(Boolean) as object[],
     );
+    // The config is what says how slow a step may be (`log.slowMs`), so it cannot
+    // itself be measured against a configured margin — only against the default.
+    const {loadedConfig: mergedConfig, configRuntime} = await withProgress(
+        bootstrap,
+        `config ${name}`,
+        configPromise,
+        {slowMs: SLOW_STEP_MS},
+    );
 
     // Populate the manifest from config values (e.g. `--manifest.gatewayPort=8080`
     // parsed by blong-config into `mergedConfig.manifest.gatewayPort`).
@@ -866,6 +963,12 @@ export default async function loadRealm<T extends TSchema>(
             context: `${defKind}`,
         },
     );
+    // From here on every step of the load is measured against the margin the
+    // config names, and a step that crosses it is reported at warn level through
+    // whichever logger exists — the realm's own, or the bootstrap one while the
+    // log component is still being built.
+    const slowMs = slowStepMs(mergedConfig.log);
+    const stepLog = () => logger ?? bootstrap;
     if (typeof parentConfig === 'string' && mergedConfig.watch)
         mergedConfig.watch.configs = mergedConfig.configs;
 
@@ -891,11 +994,11 @@ export default async function loadRealm<T extends TSchema>(
         const explicitChildren = new Set(
             children.filter(c => typeof c === 'string').map(c => platformApi.basename(c as string)),
         );
-        const discoveredFolders = await discoverLayerFolders(
-            platformApi,
-            base,
-            rootKind,
-            explicitChildren,
+        const discoveredFolders = await withProgress(
+            stepLog(),
+            `discover layers of ${name}`,
+            discoverLayerFolders(platformApi, base, rootKind, explicitChildren),
+            {slowMs},
         );
         for (const [folderName, activation] of discoveredFolders) {
             if (!(folderName in mergedConfig))
@@ -1300,12 +1403,22 @@ export default async function loadRealm<T extends TSchema>(
                 );
                 item = async () => loaded.filter(Boolean);
             }
-            const loadedModules = await (item as () => Promise<unknown[]>)();
+            // The import and the start are separate steps with separate costs: an
+            // import is the module graph, which is slow on a cold filesystem, while
+            // a start is the component's own init. A delayed start that names only
+            // "load log" would not say which of the two to look at, so the two are
+            // reported under labels that do.
+            const loadedModules = await withProgress(
+                stepLog(),
+                `import ${itemName}`,
+                (item as () => Promise<unknown[]>)(),
+                {slowMs},
+            );
             const modules = Array.isArray(loadedModules) ? loadedModules : [loadedModules];
             let loadedCount = 0;
             await withProgress(
-                logger,
-                `load ${itemName}`,
+                stepLog(),
+                `start ${itemName}`,
                 (async () => {
                     for (const module of modules) {
                         const item = await module;

package/src/mle.ts

@@ -94,7 +94,7 @@ export default fp<IConfig>(async function mlePlugin(fastify: FastifyInstance, co
                       jsonrpc?: unknown;
                       result?: Record<string, unknown>;
                       error?: Record<string, unknown>;
-                      checkpoints?: unknown;
+                      progress?: unknown;
                   },
         ) => {
             if (isPublic(request.originalUrl)) return payload;
@@ -119,7 +119,7 @@ export default fp<IConfig>(async function mlePlugin(fastify: FastifyInstance, co
                           id: undefined,
                           jsonrpc: undefined,
                           error: undefined,
-                          checkpoints: undefined,
+                          progress: undefined,
                       };
                 let result,
                     error = undefined as string | undefined;
@@ -141,7 +141,7 @@ export default fp<IConfig>(async function mlePlugin(fastify: FastifyInstance, co
                         jsonrpc: where.jsonrpc,
                         result: result && (await encrypt(Buffer.from(result))),
                         error: error && (await encrypt(Buffer.from(error))),
-                        checkpoints: where.checkpoints,
+                        progress: where.progress,
                     });
                 } catch (error) {
                     reply.code(400);

package/src/orchestrator/common/dispatch.ts

@@ -1,5 +1,5 @@
 import {orchestrator, type IMeta} from '@feasibleone/blong/types';
-import {legName, recordedCall} from '../../lib.ts';
+import {recordedCall} from '../../lib.ts';
 import {declareCall, namespaceOf} from '../../semanticContext.ts';
 
 export default orchestrator<{destination?: string; appendNamespace?: string}>(({remote}) => ({
@@ -35,10 +35,11 @@ export default orchestrator<{destination?: string; appendNamespace?: string}>(({
                 // name would credit the hop to whichever namespace happened to be served
                 // alongside it.
                 //
-                // The leg is spelled as every other leg is, `<caller>.<target>.<method>`:
-                // slash routing is the wire form only, and a `/` in an id is flattened to `-`
-                // by the leg grammar, which would hide the target the arrow names.
-                const declared = legName(forwarded);
+                // The leg is the method the forwarded call reaches its destination with — the
+                // wire name, slash and all (`db/gateway.bundle.find`): that is the method the
+                // callee strips back to, and a label that dotted it would no longer be the name
+                // the call was made by. Who made it is stated beside the method, not inside it.
+                const declared = forwarded;
                 const forward = async (): Promise<unknown[] | undefined> =>
                     (await recordedCall(this, declared, () =>
                         remote.dispatch(...params, {...$meta, method: forwarded}),

package/src/pino-cacache.ts

@@ -44,8 +44,48 @@ export interface CacacheTransportOptions {
 const RETENTION_STATE_KEY = '__blong_retention_state__';
 const ONE_DAY_MS = 24 * 60 * 60 * 1000;
 
+/**
+ * `cacache.rm.entry` with its third argument, which the published types do not
+ * declare.
+ *
+ * `removeFully` is real: `cacache`'s own `lib/rm.js` hands the options to
+ * `index.delete`, which removes the key's index file rather than appending a
+ * deletion to it. The types shipped for `cacache` 20 model only the two-argument
+ * form, so the option is declared here once instead of being cast at the call.
+ */
+const removeEntryFully = cacache.rm.entry as unknown as (
+    cachePath: string,
+    key: string,
+    opts: {removeFully?: boolean},
+) => Promise<unknown>;
+
+/**
+ * How long a retention step may take before it is reported.
+ *
+ * The transport runs in a pino worker thread with no logger of its own, so it
+ * writes to stderr — which is where the loader's own messages go, and where a
+ * reader of the process's output will see it. Matches the framework's margin for
+ * a slow step: a retention pass that took longer is the kind of delay that has
+ * cost a start its first seconds.
+ */
+const SLOW_STEP_MS = 1_000;
+
+function reportSlow(step: string, started: number, details?: Record<string, unknown>): void {
+    const elapsedMs = Date.now() - started;
+    if (elapsedMs < SLOW_STEP_MS) {
+        return;
+    }
+    process.stderr.write(
+        `warn  pino-cacache ${step} took ${elapsedMs}ms` +
+            (details ? ` ${JSON.stringify(details)}` : '') +
+            '\n',
+    );
+}
+
 async function pruneOldEntries(cachePath: string, retentionCount: number): Promise<void> {
+    const scanStarted = Date.now();
     const index = await cacache.ls(cachePath);
+    reportSlow('index scan', scanStarted, {entries: Object.keys(index).length});
 
     // Collect all real log entries (skip the retention-state entry itself)
     const entries = Object.values(index).filter(e => e.key !== RETENTION_STATE_KEY);
@@ -61,13 +101,25 @@ async function pruneOldEntries(cachePath: string, retentionCount: number): Promi
         return ta - tb;
     });
 
+    const pruneStarted = Date.now();
     const toDelete = entries.slice(0, entries.length - retentionCount);
     for (const entry of toDelete) {
-        await cacache.rm.entry(cachePath, entry.key);
+        // `removeFully` deletes the entry's index file. Without it `cacache`
+        // appends a deletion and keeps the file, so the index grows one file per
+        // entry ever written — and `cacache.ls` above, which reads all of them,
+        // becomes the slowest thing a process does. See
+        // `core/semantic-log/src/cache.ts`, which shares this directory and makes
+        // the same call.
+        await removeEntryFully(cachePath, entry.key, {removeFully: true});
     }
 
     // Garbage-collect content that is no longer referenced by any index entry
+    const verifyStarted = Date.now();
     await cacache.verify(cachePath);
+    reportSlow('retention pass', pruneStarted, {
+        pruned: toDelete.length,
+        verifyMs: Date.now() - verifyStarted,
+    });
 }
 
 async function retentionCheckRun(cachePath: string, retentionCount: number): Promise<void> {
@@ -100,8 +152,12 @@ export default async function transport(options: CacacheTransportOptions) {
     const cachePath = resolveHome(rawCachePath);
 
     // Run retention check once on transport startup (at most once per day)
-    retentionCheckRun(cachePath, retentionCount).catch(() => {
-        // Retention errors must not crash the transport
+    retentionCheckRun(cachePath, retentionCount).catch((error: unknown) => {
+        // Retention errors must not crash the transport — but a retention that
+        // fails is why a cache grows without bound, so it is reported rather than
+        // swallowed: silently skipping the pass is indistinguishable from running
+        // it and finding nothing to do.
+        process.stderr.write(`warn  pino-cacache retention failed: ${String(error)}\n`);
     });
 
     return build(async function (source) {

package/src/pino-pretty.ts

@@ -46,7 +46,7 @@ export default (options: PinoPretty.PrettyOptions) =>
                 $meta,
             } = log;
             return [
-                id && colors.dim(`semantic-log://record/${id}`),
+                id && colors.dim(`semlog://r/${id}`),
                 context && colors.greenBright(context),
                 prefix,
                 $meta?.mtid && colors.magenta($meta.mtid),

package/src/registry.lib.test.ts

@@ -0,0 +1,97 @@
+/**
+ * The `lib` a handler is built with (PRD R26).
+ *
+ * `blongLib` is spread into `lib` first, so a name the library exports cannot silently
+ * replace a handle the framework injects: `blongLib` exports `snapshot` (a chain step) and
+ * used to export `checkpoint` for it, an unrelated thing that shares a word with the
+ * runtime's milestone handle, so the spread used to sit after the injections. The order
+ * is the invariant, so it is pinned here rather than left to a comment — together with
+ * the mode-gated `assert`, the other member whose presence the mode decides.
+ */
+import * as blongLib from '@feasibleone/blong-lib';
+import type {ILib} from '@feasibleone/blong/types';
+import {vocabulary} from '@feasibleone/semantic-log/attachable';
+import t from 'tap';
+import Registry from './Registry.ts';
+
+// Minimal stand-ins: `_createHandlers` only reads the error factory, the platform's
+// timing and the api schema, and closes over the rest.
+const components = {
+    log: {logger: () => ({info() {}, error() {}, warn() {}, debug() {}})},
+    error: {register: () => () => undefined},
+    rpcServer: {},
+    remote: {remote: () => ({})},
+    gateway: {},
+    local: {},
+    resolution: {},
+    watch: {},
+    apiSchema: {},
+    platform: {context: {}, timing: () => undefined},
+};
+
+/** Run one handler through the factory and hand back the `lib` it was given. */
+async function captureLib(mode: 'test' | 'debug' | 'production'): Promise<ILib> {
+    const registry = new Registry({checkpointMode: mode}, components as never);
+    let lib: ILib | undefined;
+    await (
+        registry as unknown as {
+            _createHandlers(handlers: unknown, port: object | undefined): Promise<unknown>;
+        }
+    )._createHandlers(
+        [
+            async (params: {lib: ILib}) => {
+                lib = params.lib;
+            },
+        ],
+        undefined,
+    );
+    if (lib === undefined) {
+        throw new Error('the handler factory was not called');
+    }
+    return lib;
+}
+
+t.test('a handler is handed the framework branch helper, not a name the library owns', async t => {
+    const lib = await captureLib('test');
+    t.type(lib.decide, 'function', 'the branch helper is always injected');
+    t.equal(
+        lib.decide,
+        vocabulary.decide,
+        'and it is the attachable one, so a realm needs no dependency on the log library',
+    );
+    t.equal(lib.checkpoint, vocabulary.point, 'the milestone handle is the ambient one');
+    t.notOk(
+        'checkpoint' in blongLib,
+        'the library does not define that name, so the spread cannot replace it',
+    );
+    t.notOk('decide' in blongLib, 'nor the branch helper');
+    t.notOk('assert' in blongLib, 'nor the assertion handle');
+    t.end();
+});
+
+t.test('checkpoint follows the mode, while decide is always there', async t => {
+    t.type((await captureLib('test')).checkpoint, 'function', 'test mode reports points');
+    t.type((await captureLib('debug')).checkpoint, 'function', 'debug mode reports them too');
+    t.equal(
+        (await captureLib('production')).checkpoint,
+        undefined,
+        'and production leaves the slot empty, so `lib.checkpoint?.()` is one call',
+    );
+    t.type(
+        (await captureLib('production')).decide,
+        'function',
+        'a branch is selected in production as anywhere else',
+    );
+    t.end();
+});
+
+t.test('assert is there only where a failure may be expected', async t => {
+    t.type((await captureLib('test')).assert, 'function', 'test mode asserts');
+    t.type((await captureLib('debug')).assert, 'function', 'debug mode asserts too');
+    t.equal(
+        (await captureLib('production')).assert,
+        undefined,
+        'and production leaves the slot empty, so `lib.assert?.(…)` is one call',
+    );
+    t.end();
+});