@volter/twin-standard 1.0.148 → 1.0.150

package/dist/src/conformance.js

@@ -56,6 +56,41 @@ export function unpublishedUnitFiles(unit) {
 }
 /** Each of UNIT_FILES a unit lacks where conformance reads it: the manifest of every unit, the life at its vendor's root
  *  (lifeServed), the spec's record and the decision table of a unit with a spec (decided()). */
+const trackedJourneySources = new Map();
+/** Table ownership follows one index snapshot per checkout/run. */
+function hasJourneySource(unit) {
+    const dir = join(unit, 'journeys');
+    let checkout = resolve(unit);
+    while (!existsSync(join(checkout, '.git'))) {
+        const parent = dirname(checkout);
+        if (parent === checkout) {
+            // An unpacked published package has no Git index; its delivered files
+            // are its source. Markers and empty directories are not table inputs.
+            if (!existsSync(dir))
+                return false;
+            const holdsFile = (at) => readdirSync(at, { withFileTypes: true }).some(entry => entry.name !== '.metadata_never_index'
+                && (entry.isFile() || (entry.isDirectory() && holdsFile(join(at, entry.name)))));
+            return holdsFile(dir);
+        }
+        checkout = parent;
+    }
+    const env = { ...process.env };
+    for (const key of ['GIT_DIR', 'GIT_COMMON_DIR', 'GIT_WORK_TREE', 'GIT_PREFIX', 'GIT_INDEX_FILE'])
+        delete env[key];
+    if (process.env.VOLTER_PACK_HOOK_INDEX)
+        env.GIT_INDEX_FILE = process.env.VOLTER_PACK_HOOK_INDEX;
+    const key = `${checkout}\0${env.GIT_INDEX_FILE ?? ''}`;
+    let source = trackedJourneySources.get(key);
+    if (!source) {
+        const files = spawnSync('git', ['-C', checkout, 'ls-files', '-z', '--cached', '--', ':(glob)**/journeys/**'], { env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 16 * 1024 * 1024 });
+        if (files.status !== 0 || files.error)
+            throw new Error(`cannot read tracked journey source: ${String(files.stderr || files.error?.message)}`);
+        source = files.stdout.split('\0').filter(path => path && !path.split('/').includes('.metadata_never_index'));
+        trackedJourneySources.set(key, source);
+    }
+    const prefix = relative(checkout, dir).replaceAll('\\', '/') + '/';
+    return source.some(path => path.startsWith(prefix));
+}
 export function unitFilesMissing(unit) {
     const vendor = existsSync(join(unit, 'package.json')) ? unit : dirname(unit);
     const spec = hasOwnSurface(unit);
@@ -63,7 +98,7 @@ export function unitFilesMissing(unit) {
         'src/manifest.ts': unit,
         'spec/SOURCE.md': spec ? unit : undefined,
         'journeys/customer-life.json': vendor,
-        'journeys/decisions.json': spec ? (existsSync(join(unit, 'journeys')) ? unit : vendor) : undefined,
+        'journeys/decisions.json': spec ? (hasJourneySource(unit) ? unit : vendor) : undefined,
     };
     return UNIT_FILES.filter((f) => at[f] !== undefined && !existsSync(join(at[f], f))).map((f) => `${relative(unit, join(at[f], f))} is missing (conformance reads it)`);
 }
@@ -119,7 +154,7 @@ async function decided(pack) {
     for (const unit of unitsIn(pack).filter(hasOwnSurface)) {
         const name = nameOf(pack, unit);
         const key = unit === pack ? '' : basename(unit);
-        const journeys = existsSync(join(unit, 'journeys')) ? join(unit, 'journeys') : join(unit, '..', 'journeys');
+        const journeys = hasJourneySource(unit) ? join(unit, 'journeys') : join(unit, '..', 'journeys');
         const owners = byUnit ? byUnit[key] ?? {} : (await makeOf(unit))?.().owners?.();
         if (!owners) {
             out.push(`${name}: its fetch (${fetchName(unit)}) names no owners() to check the decision table against`);
@@ -321,6 +356,190 @@ async function standalone(pack) {
     }
     return out;
 }
+/** The request an application's first call of an operation sends when it knows nothing yet: every path parameter, every
+ *  required query parameter and every header the operation is routed by holding a placeholder, its discriminator's
+ *  values, and an empty body in the operation's encoding. What it measures is the twin's answer to a call it did not
+ *  script: a refusal in the vendor's form, or a result in the vendor's shape, and never a crash, a hang or a row. */
+function sweepRequest(op, basePath, host, credential) {
+    const fill = (name, type) => (type === 'integer' || type === 'number' ? '1' : type === 'boolean' ? 'true' : `conformance-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-')}`);
+    const path = `${op.basePath ?? basePath}${op.path}`.replace(/\{([^}+]+)\+?\}/g, (_m, name) => fill(name));
+    const url = new URL(`https://${host}${path}`);
+    for (const q of op.query ?? [])
+        if (q.required)
+            url.searchParams.set(q.name, fill(q.name, q.type));
+    for (const [name, value] of Object.entries(op.discriminator ?? {}))
+        url.searchParams.set(name, value ?? fill(name));
+    const headers = new Headers(credential);
+    for (const [name, value] of Object.entries(op.headers ?? {}))
+        headers.set(name, value ?? fill(name));
+    const method = op.method.toUpperCase();
+    let body;
+    if (op.bodyEncoding === 'form') {
+        headers.set('content-type', 'application/x-www-form-urlencoded');
+        body = '';
+    }
+    else if (op.bodyEncoding === 'xml') {
+        headers.set('content-type', 'application/xml');
+        body = '';
+    }
+    else if (op.bodyEncoding === 'json' || op.bodyEncoding === 'multipart') {
+        headers.set('content-type', 'application/json');
+        body = '{}';
+    }
+    return new Request(url, { method, headers, ...(body !== undefined && !['GET', 'HEAD'].includes(method) ? { body } : {}) });
+}
+/** Every operation the pack serves, called over HTTP as an application calls it, against its server.ts served as a World
+ *  runs it: once plainly and once with a scenario loaded (an empty one, as a World that scripts nothing loads it). Each
+ *  answers within 10 s, never with a 500 the spec does not declare, never with a field of the twin's own (`_`
+ *  bookkeeping, the kernel's `updatedAt`) and, answered successfully, in the shape the spec gives the operation; and every
+ *  refresh list a fresh account can read (one with no parameter in its path) answers it successfully, so a World rooted
+ *  at this twin reads its resources back. Escaped before: the scenario hang (a served pack given a scenario answered
+ *  nothing), leaked kernel fields in answers no life step read, and refresh scopes on operations the pack answered with
+ *  the gap. The life walks what a customer does; this calls what the pack declares it serves. */
+async function surfaceSwept(pack, scratch) {
+    const out = [];
+    const descriptor = (await manifestOf(pack)).descriptor;
+    const fetchOf = (await import(__rewriteRelativeImportExtension(join(pack, 'src', 'fetch.ts'))))[fetchName(pack)];
+    const root = fetchOf?.();
+    const byUnit = root?.ownersByUnit?.() ?? { '': root?.owners?.() ?? {} };
+    const { shapeChecker } = await import("./walk/shape.js");
+    const { leaked } = await import("./walk/behavior-journey.js");
+    const shape = await shapeChecker(pack);
+    const units = [];
+    const vendorFields = new Set();
+    for (const unit of unitsIn(pack).filter(hasOwnSurface)) {
+        const surface = readJson(join(unit, 'src', 'generated', 'surface.gen.json'));
+        for (const r of surface?.resources ?? [])
+            for (const f of r.fields)
+                vendorFields.add(f.name);
+        // a line protocol and a command table ride in no path of their own: the life's line steps and the examples read them
+        if (!surface?.operations || ['command-reply', 'command-table'].includes(String(surface.format)))
+            continue;
+        const m = (await manifestOf(unit));
+        const host = m.hosts?.find((h) => h.host)?.host ?? descriptor?.hosts?.find((h) => h.host)?.host;
+        if (!host) {
+            out.push(`${nameOf(pack, unit)}: no host is named (its manifest's hosts or the descriptor's) for an application to call it at`);
+            continue;
+        }
+        const owners = byUnit[unit === pack ? '' : basename(unit)] ?? {};
+        const refreshLists = Object.entries(m.resources ?? {}).flatMap(([resource, decl]) => (decl.refresh?.list ? [{ resource, id: decl.refresh.list }] : []));
+        units.push({ name: nameOf(pack, unit), host, basePath: surface.basePath ?? '', ops: surface.operations.filter((o) => owners[o.id] !== undefined && owners[o.id] !== 'gap'), refreshLists, ...(m.auth ? { auth: m.auth } : {}) });
+    }
+    if (!units.length)
+        return { failures: out, note: 'no unit serves an operation on a path of its own' };
+    const tally = { calls: 0, ok: 0, refused: 0 };
+    const env = globalThis.process.env;
+    const scenarioPath = join(scratch, 'surface-scenario.json');
+    writeFileSync(scenarioPath, '{"handlers":[]}');
+    for (const mode of ['plain', 'scenario']) {
+        const before = env.VOLTER_WORLD_BOOT_ID;
+        env.VOLTER_WORLD_BOOT_ID = 'conformance';
+        let server;
+        try {
+            server = await serve(pack, { root: join(scratch, `surface-${mode}`), ...(mode === 'scenario' ? { scenarioPath } : {}) });
+        }
+        finally {
+            if (before === undefined)
+                delete env.VOLTER_WORLD_BOOT_ID;
+            else
+                env.VOLTER_WORLD_BOOT_ID = before;
+        }
+        const http = servedHttpFetch(server.inspectPort ?? server.port);
+        const send = async (request) => {
+            await server.managedDatabaseUrl?.();
+            const abort = new AbortController();
+            try {
+                return await within(10_000, http(request, { signal: abort.signal }));
+            }
+            catch (error) {
+                return error;
+            }
+            finally {
+                abort.abort();
+            }
+        };
+        try {
+            // the account an application holds: the key the credential door issues, sent as the vendor reads its credential
+            let issued;
+            if (descriptor?.credentialDoor) {
+                const door = descriptor.credentialDoor;
+                const answer = await send(new Request(`https://${descriptor.hosts?.find((h) => h.host)?.host ?? units[0].host}${door.path}`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(door.body) }));
+                const fields = answer instanceof Response ? (await answer.json().catch(() => ({}))) : {};
+                issued = Object.values(door.fill).map((f) => fields[f]).find((v) => typeof v === 'string');
+            }
+            const said = mode === 'scenario' ? ' with a scenario loaded' : '';
+            for (const u of units) {
+                // a unit whose manifest declares no `auth` checks its key in its own front: the key goes as most vendors read
+                // one, a bearer token, and a refusal of it is no finding of the refresh below
+                const auth = u.auth ?? { header: 'authorization', scheme: 'Bearer' };
+                const credential = issued ? { [auth.header]: auth.scheme ? `${auth.scheme} ${issued}` : issued } : {};
+                const answered = new Map();
+                for (const op of u.ops) {
+                    const request = sweepRequest(op, u.basePath, u.host, credential);
+                    const where = `${u.name}: ${op.id} (${request.method} ${new URL(request.url).pathname})`;
+                    const res = await send(request);
+                    if (res === 'timeout') {
+                        out.push(`${where} answered nothing in 10 s${said}`);
+                        continue;
+                    }
+                    if (res instanceof Error) {
+                        out.push(`${where}: the connection failed${said} (${res.message})`);
+                        continue;
+                    }
+                    answered.set(op.id, res.status);
+                    if (mode === 'plain') {
+                        tally.calls++;
+                        if (res.status < 300)
+                            tally.ok++;
+                        else
+                            tally.refused++;
+                    }
+                    const text = await res.text();
+                    if (res.status === 500 && !op.responseMedia?.['500'] && !op.responseMedia?.['5XX']) {
+                        out.push(`${where} answered 500${said}, which the spec does not declare: ${text.slice(0, 160)}`);
+                        continue;
+                    }
+                    if (mode === 'scenario')
+                        continue;
+                    const type = res.headers.get('content-type') ?? '';
+                    let body;
+                    if (/json/.test(type)) {
+                        try {
+                            body = JSON.parse(text);
+                        }
+                        catch {
+                            body = undefined;
+                        }
+                    }
+                    for (const key of leaked(body, vendorFields, '', !op.answers || res.status >= 300))
+                        out.push(`${where} answers the twin's own field ${key}`);
+                    if (res.status >= 200 && res.status < 300 && shape) {
+                        const hit = shape.operationOf(request.method, request.url, Object.fromEntries(request.headers));
+                        if (hit)
+                            for (const detail of shape.failures(hit.op, body, type, request.headers.get('accept') ?? undefined, res.status, { method: request.method, responseText: text }))
+                                out.push(`${where}: ${detail}`);
+                    }
+                }
+                if (mode === 'plain')
+                    for (const r of u.refreshLists) {
+                        const op = u.ops.find((o) => o.id === r.id);
+                        if (!op || op.path.includes('{') || (op.basePath ?? '').includes('{'))
+                            continue;
+                        const status = answered.get(r.id);
+                        if (status !== undefined && (status === 401 || status === 403) && (!u.auth || !issued))
+                            continue;
+                        if (status !== undefined && (status < 200 || status >= 300))
+                            out.push(`${u.name}: ${r.resource}'s refresh lists by ${r.id}, which answered a fresh account ${status}: a World rooted at this twin reads nothing back`);
+                    }
+            }
+        }
+        finally {
+            await server.stop();
+        }
+    }
+    // one defect repeated over a surface reads as one: the first 40 findings, and how many more
+    return { failures: out.length > 40 ? [...out.slice(0, 40), `… ${out.length - 40} more`] : out, note: `${tally.calls} served operations called, twice each: ${tally.ok} answered successfully, ${tally.refused} refused, unscripted` };
+}
 /** A line protocol's connection over TCP, in the walk's stream shape. A line drew its replies when bytes arrived and then
  *  50 ms passed with no more (a multi-line reply arrives in pieces), or when 500 ms passed with none (a line that draws no
  *  reply: a message's body before its end); 10 s bounds a server that keeps writing. The windows are timing, not protocol:
@@ -394,7 +613,9 @@ async function lifeServed(pack) {
             socket: (url, headers) => new WebSocket(`ws://127.0.0.1:${port}${url}`, { headers }),
             // a line connection opened as an application's client opens it: TCP to the protocol listener (the primary port)
             stream: lineConnection(server.port),
-            stop: () => server.stop(),
+            // bounded: Bun's server.stop(true) never settles once a WebSocket peer has closed under it (discord's life, Bun
+            // 1.3.11: the walk ended and the stop spun at full CPU for 45 minutes), so the check waits 5 s and goes on
+            stop: async () => { await within(5000, Promise.resolve(server.stop())); },
         };
     };
     const world = inMemoryWorld(`conform-${basename(pack)}`);
@@ -430,7 +651,9 @@ async function clientCases(pack, scratch) {
     const descriptor = (await manifestOf(pack)).descriptor;
     // the first host that names one: a descriptor may lead with a suffix (a region's hosts under one domain), which no
     // request addresses; the case's client reaches whatever host its issued credentials name
-    const host = descriptor?.hosts?.find((h) => h.host)?.host;
+    // a raw protocol's pack (hostsNone: SMTP, whose relay is whatever host the operator names) is reached by its TCP
+    // connection alone; its client names a stand-in host the connection never resolves
+    const host = descriptor?.hosts?.find((h) => h.host)?.host ?? (descriptor?.hostsNone ? 'relay.world.test' : undefined);
     if (!host)
         return { failures: ['the descriptor names no host for its client to address'] };
     const scenarioPath = join(scratch, 'client-scenario.json');
@@ -453,13 +676,31 @@ async function clientCases(pack, scratch) {
         const issued = {};
         const door = descriptor?.credentialDoor;
         if (door) {
-            const answer = (await (await fetch(`http://127.0.0.1:${server.port}${door.path}`, { method: 'POST', headers: { host, 'content-type': 'application/json' }, body: JSON.stringify(door.body) })).json().catch(() => ({})));
+            // a door that answers nothing is the pack's failure, never conform's hang (a pack served with a scenario that answered
+            // nothing once held this request open for good)
+            const abort = new AbortController();
+            const asked = await within(10_000, fetch(`http://127.0.0.1:${server.port}${door.path}`, { method: 'POST', headers: { host, 'content-type': 'application/json' }, body: JSON.stringify(door.body), signal: abort.signal }).then((r) => r.json().catch(() => ({}))));
+            abort.abort();
+            if (asked === 'timeout')
+                return { failures: [`the credential door (POST ${door.path}) answered nothing in 10 s, served with a scenario loaded`] };
+            const answer = asked;
             for (const [name, field] of Object.entries(door.fill))
                 if (answer[field] !== undefined)
                     issued[name] = String(answer[field]);
         }
         const job = join(scratch, 'client-job.json');
-        writeFileSync(job, JSON.stringify({ vendor, port: server.port, host, env: issued, scenarioPath, packFacts: join(pack, 'generated', 'pack-facts.json') }));
+        // a release carries its facts (generated/pack-facts.json, `twin-standard facts`); a source checkout does not, so its
+        // facts are compiled here, as that command compiles them, for the client the case runs
+        let packFacts = join(pack, 'generated', 'pack-facts.json');
+        if (!existsSync(packFacts)) {
+            const { compilePackFacts, renderPackFacts } = await import("./pack-facts.js");
+            const compiled = await compilePackFacts([{ vendor, dir: pack }]);
+            if (compiled.undeclared.length)
+                throw new Error('Pack descriptor is missing');
+            packFacts = join(scratch, 'pack-facts.json');
+            writeFileSync(packFacts, renderPackFacts(compiled));
+        }
+        writeFileSync(job, JSON.stringify({ vendor, port: server.port, host, env: issued, scenarioPath, packFacts, ...(descriptor?.hostsNone ? { raw: true } : {}) }));
         // Node, as an application runs its SDK (asynchronous: this process serves the pack meanwhile)
         const out = await new Promise((resolve) => {
             const child = spawn('node', ['--experimental-strip-types', '--no-warnings', join(clients, `run${extension}`), job], { stdio: ['ignore', 'pipe', 'pipe'] });
@@ -479,8 +720,46 @@ async function clientCases(pack, scratch) {
         await server.stop();
     }
 }
+/** The gate over every unit of the pack: the form section of the grade, which holds the rules an author's code keeps
+ *  (p3-rules.ts: ids from ctx.mint, the twin's URL from ctx.publicBase, the kernel's crypto, bytes in ctx.blobs, secrets
+ *  drawn from ctx.secret) and the pack layout, handler, kernel and manifest contracts. A contributor's conform is the
+ *  whole of what a machine decides of the pack, as the catalog's admission reads it. */
+async function formOf(pack) {
+    const { p3Gate } = await import("./gate.js");
+    return (await p3Gate(unitsIn(pack))).flatMap((r) => r.failures.map((f) => `${nameOf(pack, r.unit)}: ${f}`));
+}
+/** The pack's records hold what a reviewer reads them for: each demand row cites the application's line it was read at
+ *  (`<path>:<line>`, or `<package>@<version>:<path>:<line>` for a call a dependency makes), the spec's record names a
+ *  vendored file that is there, the life names the session that wrote it (what makes a judge's verdict independent),
+ *  and the package says what it serves in a README. Rules every builder brief carried by hand, decided here. */
+function records(pack) {
+    const out = [];
+    if (!existsSync(join(pack, 'README.md')))
+        out.push('README.md is missing: the package says what it serves, for whom, its doors and what it leaves out');
+    const life = readJson(join(pack, 'journeys', 'customer-life.json'));
+    if (life && (typeof life.author !== 'string' || !life.author.trim()))
+        out.push('journeys/customer-life.json names no author (the session that wrote it), so no verdict on it can be shown independent');
+    for (const unit of unitsIn(pack)) {
+        const name = nameOf(pack, unit);
+        const journeys = hasJourneySource(unit) ? join(unit, 'journeys') : join(unit, '..', 'journeys');
+        const demand = unit === pack || hasJourneySource(unit) ? readJson(join(journeys, 'demand.json')) : undefined;
+        for (const app of demand?.applications ?? [])
+            for (const row of app.backend ?? []) {
+                if (typeof row.evidence !== 'string' || !/:\d+/.test(row.evidence))
+                    out.push(`${name}: ${app.name ?? 'an application'}'s call of ${row.operation ?? '(no operation)'} cites no line it was read at (evidence "<path>:<line>")`);
+            }
+        if (!hasOwnSurface(unit))
+            continue;
+        const source = existsSync(join(unit, 'spec', 'SOURCE.md')) ? readFileSync(join(unit, 'spec', 'SOURCE.md'), 'utf8') : '';
+        const file = /\*\*File:\*\*[^`\n]*`([^`]+)`/.exec(source)?.[1];
+        if (file && !existsSync(join(unit, 'spec', file)))
+            out.push(`${name}: spec/SOURCE.md names ${file} as the vendored spec, and spec/ holds no such file`);
+    }
+    return out;
+}
 /** Every check over the pack at `dir`, in order. */
-export async function conformance(dir) {
+/** `onCheck` hears each check's id as it starts, so a caller can show where a long run stands. */
+export async function conformance(dir, options = {}) {
     const pack = resolve(dir);
     if (!existsSync(join(pack, 'package.json')))
         throw new Error(`${dir} is no pack (no package.json): conformance takes a pack's directory`);
@@ -490,8 +769,10 @@ export async function conformance(dir) {
     const here = process.cwd();
     process.chdir(scratch);
     const run = async (id, asks, check) => {
+        options.onCheck?.(id);
         try {
-            return { id, asks, failures: await check() };
+            const r = await check();
+            return Array.isArray(r) ? { id, asks, failures: r } : { id, asks, failures: r.failures, note: r.note };
         }
         catch (error) {
             return { id, asks, failures: [`the check could not run: ${error.message.split('\n')[0]}`] };
@@ -506,6 +787,7 @@ export async function conformance(dir) {
             await run('registered', 'a vendor-backed unit, lane or not, reaches the registered pack\'s state system', () => registered(pack, runtime)),
             await run('allowance', 'a rate budget above the fallback rests on the vendor\'s documented allowance, cited in its manifest', () => allowance(pack, runtime)),
             await (async () => {
+                options.onCheck?.('client');
                 const asks = "driven by the vendor's own client (its official SDK), served as a World runs it, the vendor's documented behaviour holds";
                 try {
                     const r = await clientCases(pack, scratch);
@@ -517,6 +799,9 @@ export async function conformance(dir) {
             })(),
             await run('life', 'the life walked over HTTP against the pack served as a World runs it, with its scenario: every check held', () => lifeServed(pack)),
             await run('standalone', 'served by its server.ts as a World runs it, the pack answers HTTP and the boot probe, and each declared socket upgrades', () => standalone(pack)),
+            await run('surface', 'every served operation called over HTTP as an application calls it, plainly and with a scenario loaded: answered within 10 s, no undeclared 500, no field of the twin\'s own, a success in the spec\'s shape, and every refresh list a fresh account reads answering', () => surfaceSwept(pack, scratch)),
+            await run('records', 'each demand row cites the line it was read at, the spec\'s record names a vendored file that is there, the life names its author, the package has its README', () => records(pack)),
+            await run('form', 'every unit passes the form section of the grade (the gate): the layout, the handler and kernel contracts, the manifest as data, the rules an author\'s code holds', () => formOf(pack)),
         ];
     }
     finally {

package/dist/src/consumer.js

@@ -201,7 +201,11 @@ export async function runConsumerWorkflow(packDir, options) {
                 throw new Error('Resume changed recorded application state before readback');
             await run('retained-readback', ['run', '--', ...recipe.retention.readback]);
             const after = JSON.parse(await run('log-after-readback', ['log', '--json']));
-            if (JSON.stringify(before) !== JSON.stringify(after))
+            // the readback adds no write of the application's: what it adds may only be the vendor's own moves (a key's
+            // last-used time, recorded by the vendor as a request uses it: actor `system`, ctx.asVendor), never an API call's
+            const added = after.filter((entry) => !before.some((prior) => prior.id === entry.id));
+            const kept = before.every((prior) => after.some((entry) => JSON.stringify(entry) === JSON.stringify(prior)));
+            if (!kept || added.some((entry) => entry.actor?.kind !== 'system'))
                 throw new Error('Retained readback added mutations; it must only inspect existing state');
             retainedReadback = true;
         }

package/dist/src/create-pack.d.ts

@@ -39,4 +39,9 @@ export declare function initFiles(vendor: string): Record<string, string>;
  *  surface of its own under `<lane>/src`, keeping the vendor's one store and sharing its `src/semantics/shared.ts`, and the
  *  pack's server routing each lane's host to it. */
 export declare function laneInitFiles(vendor: string, lanes: string[]): Record<string, string>;
+/** A unit's fixed files as the manifest and the spec make them now (architecture, "Creating a pack"): a lane's fetch, and
+ *  a pack's fetch, index, server and cli. A pack with a wire of its own (`src/stream.ts`, SMTP's and Redis's byte-stream
+ *  door) writes its server and cli itself: they carry its listener beside the fetch, so they are not fixed. `--index`
+ *  writes these and the grade's `fixed-reproducible` compares them byte for byte. */
+export declare function fixedFiles(dir: string): Promise<Record<string, string>>;
 export declare function createPack(args: string[]): Promise<void>;

package/dist/src/create-pack.js

@@ -160,6 +160,9 @@ export function packFetchFile(dir, name) {
     const hasLive = existsSync(join(semantics, 'live.ts'));
     // a GraphQL API beside the REST one: its resolvers and the vendor's schema (graphql-wire.ts)
     const hasGraphql = existsSync(join(semantics, 'graphql.ts')) && existsSync(join(dir, 'src', 'generated', 'graphql-sdl.gen.json'));
+    // a stream whose documents are BSON (the MongoDB wire): the pack's own bson library, resolved from the pack as graphql's is
+    const packageJson = existsSync(join(dir, 'package.json')) ? parseExactJson(readFileSync(join(dir, 'package.json'), 'utf8')) : {};
+    const hasBson = existsSync(join(dir, 'src', 'stream.ts')) && packageJson.dependencies?.bson !== undefined;
     const screensDir = join(dir, 'src', 'screens');
     const screens = existsSync(screensDir) ? readdirSync(screensDir).filter((n) => /\.tsx?$/.test(n) && !/\.(test|uitest)\.tsx?$/.test(n) && !/^shared\.tsx?$/.test(n)).sort() : [];
     const id = (file) => file.replace(/\.tsx?$/, '');
@@ -205,6 +208,7 @@ export function packFetchFile(dir, name) {
         ...screens.map((f) => `import { screen as ${identifier(`screen_${id(f)}`)} } from './screens/${f}';`),
         ...laneImports,
         '',
+        ...(hasBson ? ["/** The pack's BSON library, resolved from the pack as graphql's loader is (architecture, the MongoDB wire). */", "export const loadBson = (): Promise<unknown> => import('bson');", ''] : []),
         `export function ${name}(options: PackFetchOptions = {}): DerivedFetch {`,
         '  return createPackFetch({',
         '    manifest,',
@@ -507,6 +511,27 @@ export function laneInitFiles(vendor, lanes) {
     }
     return files;
 }
+/** A unit's fixed files as the manifest and the spec make them now (architecture, "Creating a pack"): a lane's fetch, and
+ *  a pack's fetch, index, server and cli. A pack with a wire of its own (`src/stream.ts`, SMTP's and Redis's byte-stream
+ *  door) writes its server and cli itself: they carry its listener beside the fetch, so they are not fixed. `--index`
+ *  writes these and the grade's `fixed-reproducible` compares them byte for byte. */
+export async function fixedFiles(dir) {
+    const files = { 'src/fetch.ts': packFetchFile(dir, fetchName(dir)) };
+    if (!existsSync(join(dir, 'package.json')))
+        return files;
+    const { manifest } = await import(__rewriteRelativeImportExtension(join(dir, 'src', 'manifest.ts')));
+    const vendor = manifest?.vendor;
+    if (!vendor || !/^[a-z][a-z0-9-]*$/.test(vendor))
+        throw new Error('Pack manifest must declare its vendor identity independently of the npm package name');
+    files['src/index.ts'] = packIndexFile(dir, vendor);
+    if (!existsSync(join(dir, 'src', 'stream.ts'))) {
+        // a unit whose spec is a proto (derive-pack generated its schema) is served over gRPC: its serve seam in that form
+        const lanesOnly = !hasOwnSurface(dir) && readdirSync(dir).some((n) => existsSync(join(dir, n, 'src', 'manifest.ts')));
+        files['src/server.ts'] = serverFile(vendor, !lanesOnly && existsSync(join(dir, 'src', 'generated', 'proto.gen.json')));
+        files['src/cli.ts'] = cliFile(vendor);
+    }
+    return files;
+}
 export async function createPack(args) {
     const [unit, ...rest] = args;
     if (!unit || (!rest.includes('--index') && !rest.includes('--brief') && !rest.includes('--init'))) {
@@ -598,9 +623,12 @@ export async function createPack(args) {
             writeFileSync(path, handlerIndex(dir));
             console.log(`create-pack: wrote ${path}`);
         }
-        // the fetch, again: a door file, the clock or a screen may have been added since
-        writeFileSync(join(dir, 'src', 'fetch.ts'), packFetchFile(dir, fetchName(dir)));
-        console.log(`create-pack: wrote ${join(dir, 'src', 'fetch.ts')}`);
+        // the fixed files, again: a door file, the clock, a screen, a lane or a proto may have been added since, and a pack
+        // made by an earlier create-pack takes the current form
+        for (const [rel, text] of Object.entries(await fixedFiles(dir))) {
+            writeFileSync(join(dir, rel), text);
+            console.log(`create-pack: wrote ${join(dir, rel)}`);
+        }
         // a pack's own index (a lane's is the vendor's to name): the descriptor with what the kernel derives from the surface
         if (existsSync(join(dir, 'package.json'))) {
             const { manifest } = await import(__rewriteRelativeImportExtension(join(dir, 'src', 'manifest.ts')));
@@ -617,7 +645,13 @@ export async function createPack(args) {
             // a unit whose spec is a proto (derive-pack generated its schema) is served over gRPC: its serve seam in that form,
             // the one form a unit takes after --init (a server.ts of a wire of the pack's own, SMTP's, is never written over)
             if (!lanesOnly && existsSync(join(dir, 'src', 'generated', 'proto.gen.json'))) {
-                writeFileSync(join(dir, 'src', 'server.ts'), serverFile(vendor, true, existsSync(join(dir, 'src', 'stream.ts'))));
+                const wire = existsSync(join(dir, 'src', 'stream.ts'));
+                // a raw-tcp pack exporting one create<Name>TwinStream is taken to serve that stream on its main port, and is admitted
+                // by its TCP answer alone (world-runtime host.ts servesByteStream); this one serves gRPC there, its stream on a side port
+                if (wire && manifest?.descriptor?.transport === 'raw-tcp') {
+                    throw new Error(`create-pack: ${vendor} serves gRPC on its main port and its own wire (./stream.ts) on a side port, so its descriptor's transport is not raw-tcp: raw-tcp with a stream export says the main port is that stream (world-runtime host.ts servesByteStream).`);
+                }
+                writeFileSync(join(dir, 'src', 'server.ts'), serverFile(vendor, true, wire));
                 console.log(`create-pack: wrote ${join(dir, 'src', 'server.ts')}`);
             }
             else if (manifest?.live?.length) {

package/dist/src/derive.js

@@ -21,7 +21,7 @@ import { gunzipSync } from 'node:zlib';
 import { CRUD_CLASSES, fromOpenAPI, fromSmithy, fromSwagger2, slug } from "./spec-ir.js";
 import { fromCommandReplies } from "./spec-ir-lines.js";
 import { fromProto, protoSchema, readProtoFiles } from "./spec-ir-proto.js";
-import { fromRedisCommands } from "./spec-ir-commands.js";
+import { fromDriverCommandBuilders, fromRedisCommands } from "./spec-ir-commands.js";
 import { fromClientCalls } from "./spec-ir-client.js";
 import { discoveryMethods, fromDiscovery, isDiscovery } from "./spec-ir-discovery.js";
 import { handlerCitations, quotedPage, sourceHeld, urlsIn } from "./check-sources.js";
@@ -34,22 +34,28 @@ if (!vendor) {
     console.error('usage: twin-standard derive <pack dir> [--out <dir>]');
     process.exit(1);
 }
+// A source-declared refusal is deterministic over the pinned bytes. Unexpected
+// loader, filesystem and process failures keep status 1 and must be retried.
+const SOURCE_REFUSAL = 2;
 const PKG = unitAt(vendor).dir;
 // an OpenAPI document, an AWS Smithy JSON model (smithy.json), the RFC a line protocol is (rfcNNNN.txt), or the
 // Protocol Buffers files an RPC service is published as (spec/proto/, every file the service imports), or the calls a
 // vendor's own client makes where the vendor publishes no spec (client-ops.json, spec-ir-client.ts), or one API's several
 // OpenAPI documents (spec/openapi/, read as their union: spec-documents.ts), or a Google API Discovery document
-// (discovery.json, spec-ir-discovery.ts)
+// (discovery.json, spec-ir-discovery.ts), or the command documents a vendor's own client builds where the vendor
+// publishes no table a pack may derive from (command-builders.json, spec-ir-commands.ts)
 const rfc = existsSync(join(PKG, 'spec')) ? readdirSync(join(PKG, 'spec')).find((f) => /^rfc\d+\.txt$/.test(f)) : undefined;
-const specPath = ['openapi.json.gz', 'openapi.json', 'openapi.yaml.gz', 'openapi.yaml', 'smithy.json.gz', 'smithy.json', 'discovery.json.gz', 'discovery.json', ...(rfc ? [rfc] : []), 'openapi', 'proto', 'commands', 'client-ops.json'].map((f) => join(PKG, 'spec', f)).find((p) => existsSync(p));
+const specPath = ['openapi.json.gz', 'openapi.json', 'openapi.yaml.gz', 'openapi.yaml', 'smithy.json.gz', 'smithy.json', 'discovery.json.gz', 'discovery.json', ...(rfc ? [rfc] : []), 'openapi', 'proto', 'commands', 'command-builders.json', 'client-ops.json'].map((f) => join(PKG, 'spec', f)).find((p) => existsSync(p));
 if (!specPath) {
-    console.error(`derive-pack: no spec at ${join(PKG, 'spec')}/openapi.{json,yaml}[.gz], smithy.json[.gz], rfcNNNN.txt, proto/, commands/ or client-ops.json`);
-    process.exit(1);
+    console.error(`derive-pack: no spec at ${join(PKG, 'spec')}/openapi.{json,yaml}[.gz], smithy.json[.gz], rfcNNNN.txt, proto/, commands/, command-builders.json or client-ops.json`);
+    process.exit(SOURCE_REFUSAL);
 }
 const rpcService = basename(specPath) === 'proto';
 const severalDocuments = basename(specPath) === 'openapi';
 // a command language's table, one vendored file per command (Redis's src/commands/*.json), with its version in SOURCE.md
 const commandTable = basename(specPath) === 'commands';
+// a language whose commands are documents: the table a pack's extractor read from the vendor's client
+const commandBuilders = basename(specPath) === 'command-builders.json';
 const protoTree = (dir, at = '') => Object.fromEntries(readdirSync(join(dir, at), { withFileTypes: true }).flatMap((e) => (e.isDirectory() ? Object.entries(protoTree(dir, join(at, e.name))) : e.name.endsWith('.proto') ? [[join(at, e.name), readFileSync(join(dir, at, e.name), 'utf8')]] : [])));
 const bytes = rpcService || commandTable || severalDocuments ? Buffer.from('') : readFileSync(specPath);
 const text = (specPath.endsWith('.gz') ? gunzipSync(bytes) : bytes).toString('utf8');
@@ -164,7 +170,7 @@ if (existsSync(patchPath)) {
     for (const p of parseExactJson(readFileSync(patchPath, 'utf8')).patches) {
         if (!p.why?.trim()) {
             console.error(`${vendor}: spec patch ${p.op} ${p.path} gives no why`);
-            process.exit(1);
+            process.exit(SOURCE_REFUSAL);
         }
         {
             const s = p.source;
@@ -183,7 +189,7 @@ if (severalDocuments) {
     const { doc: merged, contradictions } = mergeDocuments(doc);
     if (contradictions.length) {
         console.error(`${vendor}: the documents contradict each other; a patch (citing what shows which holds) rules each:\n  ${contradictions.join('\n  ')}`);
-        process.exit(1);
+        process.exit(SOURCE_REFUSAL);
     }
     doc = merged;
 }
@@ -193,7 +199,9 @@ const out = outAt >= 0 && process.argv[outAt + 1] ? process.argv[outAt + 1] : jo
 mkdirSync(out, { recursive: true });
 // A line protocol's surface is its commands and the replies each may answer; its manifest's transitions
 // name commands and cite the RFC.
-if (lineProtocol || commandTable) {
+if (commandBuilders)
+    doc = fromDriverCommandBuilders(doc);
+if (lineProtocol || commandTable || commandBuilders) {
     const spec = doc;
     writeFileSync(join(out, 'surface.gen.json'), `${JSON.stringify({ generatedBy: 'scripts/derive-pack.ts', spec: basename(specPath), ...spec })}\n`);
     const manifestPath = join(PKG, 'src', 'manifest.ts');
@@ -213,7 +221,7 @@ if (lineProtocol || commandTable) {
     }
     if (evidence.length) {
         console.error(`${vendor}: ${evidence.length} citation(s) do not hold:\n  ${[...new Set(evidence)].join('\n  ')}`);
-        process.exit(1);
+        process.exit(SOURCE_REFUSAL);
     }
     console.log(`${vendor}: spec ${spec.version}, ${spec.commands.length} commands${'anyCommand' in spec ? `; any command may answer ${spec.anyCommand.join(', ')}` : ''}`);
     process.exit(0);
@@ -322,7 +330,7 @@ if (existsSync(manifestPath)) {
     }
     if (evidence.length) {
         console.error(`${vendor}: ${evidence.length} citation(s) do not hold:\n  ${[...new Set(evidence)].join('\n  ')}`);
-        process.exit(1);
+        process.exit(SOURCE_REFUSAL);
     }
     // every state candidate of a resource the pack models is ruled on: a state field, or named not one
     const unruled = ir.resources.flatMap((r) => {
@@ -331,7 +339,7 @@ if (existsSync(manifestPath)) {
     });
     if (unruled.length) {
         console.error(`${vendor}: the manifest does not rule on ${unruled.length} state candidate(s): ${unruled.join(', ')} (declare each a state field or list it in notState)`);
-        process.exit(1);
+        process.exit(SOURCE_REFUSAL);
     }
     // a resource's refresh scope ("The real-system adapters", What a pack declares) names the list that enumerates it or
     // its retrieve, or says why the vendor offers no read-back; whether every stored resource declares one is
@@ -421,14 +429,14 @@ if (existsSync(manifestPath)) {
     });
     if (scopes.length) {
         console.error(`${vendor}: ${scopes.length} refresh scope(s) do not hold:\n  ${scopes.join('\n  ')}`);
-        process.exit(1);
+        process.exit(SOURCE_REFUSAL);
     }
     // the architecture's mechanical rules over the pack's own code (./p3-rules.ts): a line breaking one records why, or
     // the build stops
     const hits = p3RuleHits(PKG);
     if (hits.length) {
         console.error(`${vendor}: ${hits.length} line(s) break a Protocol 3 rule (fix it, or record why on the line: // p3: <reason>):\n  ${hits.map((h) => `${h.file}:${h.line}: ${h.rule}\n      ${h.text}`).join('\n  ')}`);
-        process.exit(1);
+        process.exit(SOURCE_REFUSAL);
     }
 }
 const counts = new Map();

package/dist/src/gate.d.ts

@@ -7,4 +7,5 @@ export declare function laneRootFailures(dir: string): string[];
 export declare function p3Gate(units: string[], opts?: Pick<GradeOptions, 'onCriterion'>): Promise<Array<{
     unit: string;
     failures: string[];
+    cacheable?: boolean;
 }>>;