@volter/twin-standard 1.0.148 → 1.0.150

package/src/types.ts

@@ -29,6 +29,8 @@ export interface ScoreReport {
 
 export interface CriterionResult {
   pass: boolean;
+  /** An interrupted check is not a verdict about unchanged source. */
+  cacheable?: boolean;
   /** why it failed, or what it measured when it passed */
   reason: string;
 }

package/src/verdict.ts

@@ -7,7 +7,7 @@
 /** Committed attribution names people or author roles, never execution identities. */
 export function attributionName(name: string | undefined): string | undefined {
   if (name === undefined) return undefined;
-  if (/\bcx-|\bt_[0-9a-f]{8}|wt\/|\/root(?:\/|$|[: )])|\b(?:pane|session)[ -:]|[0-9a-f]{8}-[0-9a-f-]{27}|\b[0-9a-f]{16,64}\b/i.test(name)) throw new Error('attribution must use an author name, not an execution identifier');
+  if (/\bcx-|\bt_[0-9a-f]{8}|wt\/|\/root(?:\/|$|[: )])|\b(?:pane|session)[ :-]|[0-9a-f]{8}-[0-9a-f-]{27}|\b[0-9a-f]{16,64}\b/i.test(name)) throw new Error('attribution must use an author name, not an execution identifier');
   return name;
 }
 export type Verdict = { judge?: { agent?: string; name?: string; session?: string; review?: string }; authors?: string[] };

package/src/walk/behavior-journey.ts

@@ -9,7 +9,8 @@ import { attributionName } from '../verdict.ts';
 //
 // The caller names the pack repository first (usePacks).
 
-import { mysql, redis } from '@volter/world-core';
+import { mongodb, mysql, redis } from '@volter/world-core';
+import { createRequire } from 'node:module';
 import { gzipSync } from 'node:zlib';
 import { parse, visit } from 'graphql';
 import { existsSync, readdirSync, readFileSync } from 'node:fs';
@@ -17,7 +18,7 @@ import { join } from 'node:path';
 import { databaseSession, type DatabaseSession, getActivePackAssets, getActiveWorldStore, LocalPackAssets, MemoryBlobStore, MemoryWorldStore, setActiveBlobStore, setActivePackAssets, setActiveWorldStore, setDeliveryObserver, setInProcessTwins, type InProcessTwin, type TwinStream, type TwinStreamConnection } from '@volter/world-core/runtime';
 import { createHostTable, type HostFacts } from '@volter/world-core/host-routing';
 import packFactsDocument from '@volter/world-core/generated/pack-facts.json' with { type: 'json' };
-import { type Captured, type GitStep, type Journey, type JourneyStep, type LineStep, type SocketStep, type SqlStep, type RespStep, type MysqlStep, isMysql, isResp, at, captureAt, echoFailures, expandFills, expectFailures, fill, fillJson, formBody, isGit, isLine, isSql, isSocket, isWait, jwtAuthorization, oauth1Authorization, presignedPath, sigv4Headers, waitMs } from './journey-kit.ts';
+import { type Captured, type GitStep, type Journey, type JourneyStep, type LineStep, type SocketStep, type SqlStep, type RespStep, type MysqlStep, type MongoStep, isMongo, isMysql, isResp, at, captureAt, echoFailures, expandFills, expectFailures, fill, fillJson, formBody, isGit, isLine, isSql, isSocket, isWait, jwtAuthorization, oauth1Authorization, presignedPath, sigv4Headers, waitMs } from './journey-kit.ts';
 import { lanesOf, lineShape, shapeChecker, type LineConnection, type ShapeChecker } from './shape.ts';
 import { walkPackDirectory } from './pack-directory.ts';
 import { xmlMembers } from './xml-view.ts';
@@ -141,7 +142,7 @@ export type Failure = { step: string; beat: string; detail: string };
  *  reads (an event's delivery, a socket frame), the kernel's own `updatedAt`, which a raw row filtered on `_` alone
  *  carries (Slack's message events once did). The kernel's `deleted` is not read there: vendors send their own
  *  (Stripe's deleted customer's `deleted: true`). */
-function leaked(v: unknown, vendorFields: Set<string>, at = '', unshaped = false): string[] {
+export function leaked(v: unknown, vendorFields: Set<string>, at = '', unshaped = false): string[] {
   if (Array.isArray(v)) return v.flatMap((x, i) => leaked(x, vendorFields, `${at}[${i}]`, unshaped)).slice(0, 5);
   if (!v || typeof v !== 'object') return [];
   return Object.entries(v).flatMap(([k, x]) => ((k.startsWith('_') || (unshaped && k === 'updatedAt')) && !vendorFields.has(k) ? [`${at}${at ? '.' : ''}${k}`] : leaked(x, vendorFields, `${at}${at ? '.' : ''}${k}`, unshaped))).slice(0, 5);
@@ -446,6 +447,61 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
     finally { opts.hooks?.after?.(step as never, link, answered, collect.slice(before)); }
   };
 
+  // A MongoDB step speaks the pack's MongoDB wire as an application's driver does (@volter/world-core/mongodb's client), on
+  // the same byte-stream adapter the owned TCP listener serves. The document codec is BSON, the library the pack
+  // installs: the walk loads it from the pack's own dependencies, so a step is encoded by the library the pack decodes with.
+  const mongoLinks = new Map<string, { client: ReturnType<typeof mongodb.mongoClient>; database: string; close: () => void }>();
+  type Bson = { serialize(document: Record<string, unknown>): Uint8Array; deserialize(bytes: Uint8Array, options: Record<string, unknown>): Record<string, unknown>; EJSON: { deserialize(value: unknown, options: { relaxed: boolean }): Record<string, unknown>; serialize(value: unknown, options: { relaxed: boolean }): Record<string, unknown> } };
+  let bson: Bson | undefined;
+  const bareNumber = (value: unknown, at = 'mongo'): string | undefined => {
+    if (typeof value === 'number') return at;
+    if (Array.isArray(value)) { for (const [i, item] of value.entries()) { const found = bareNumber(item, `${at}[${i}]`); if (found) return found; } return undefined; }
+    if (value && typeof value === 'object') for (const [key, item] of Object.entries(value)) { const found = bareNumber(item, `${at}.${key}`); if (found) return found; }
+    return undefined;
+  };
+  const mongoStep = async (step: MongoStep): Promise<void> => {
+    const before = collect.length; let answered: { status: number; body: unknown } = { status: 0, body: undefined };
+    const link = step.link ?? 'mongo';
+    opts.hooks?.before?.(step as never, link, step.mongo);
+    try {
+      if (!bson) {
+        const pack = walkPackDirectory(packsDir(), journey.vendor, dependenciesDir());
+        if (!pack) throw new Error(`no pack directory for ${journey.vendor} to load its BSON library from`);
+        bson = createRequire(join(pack, 'package.json'))('bson') as Bson;
+      }
+      const codec: mongodb.MongoCodec = { serialize: (document) => bson!.serialize(document), deserialize: (bytes) => bson!.deserialize(bytes, { promoteValues: false, promoteLongs: false, bsonRegExp: true }) };
+      let held = mongoLinks.get(link);
+      if (!held) {
+        if (!served?.stream) streams ??= await vendorStream(packsDir(), journey.vendor, dependenciesDir());
+        const factory = served?.stream ?? streams?.(root, {}); if (!factory) throw new Error(`no create<Name>TwinStream adapter enrolled for ${journey.vendor}`);
+        let out: number[] = [];
+        const conn = factory({ write: (b) => { out.push(...b); }, end: () => {} }, '127.0.0.1');
+        const exchange: mongodb.MongoExchange = async (bytes) => { conn.data(bytes); await conn.settled(); const got = Uint8Array.from(out); out = []; return got; };
+        held = { client: mongodb.mongoClient(exchange, codec), database: 'test', close: () => conn.close() };
+        mongoLinks.set(link, held);
+      }
+      if (step.database) held.database = fill(step.database, captured);
+      const written = fillJson(step.mongo, captured);
+      const bare = bareNumber(written);
+      if (bare) throw new Error(`${bare} is a bare JSON number: write its kind ({ "$numberInt": "…" }, $numberLong or $numberDouble)`);
+      const reply = await held.client.command(held.database, bson.EJSON.deserialize(written, { relaxed: false }));
+      const body = bson.EJSON.serialize(reply, { relaxed: false });
+      const ok = (body.ok as Record<string, unknown> | undefined), failed = !ok || Number(Object.values(ok)[0]) !== 1;
+      const code = failed ? Number(Object.values((body.code as Record<string, unknown> | undefined) ?? { none: Number.NaN })[0]) : undefined;
+      answered = { status: code ?? 0, body };
+      if (opts.verbose) console.log(`  ${String(code ?? 'OK').padEnd(5)} ${link.padEnd(6)} ${String(Object.keys(step.mongo)[0] ?? '').padEnd(20)} ${step.name}`);
+      if (step.refused !== undefined ? code !== step.refused : failed) {
+        collect.push({ step: step.name, beat: 'STATUS', detail: failed ? `MongoDB refused it: ${code} ${String(body.codeName ?? '')} ${String(body.errmsg ?? '')}${step.refused ? ` (wanted ${step.refused})` : ''}` : `MongoDB completed it, wanted ${step.refused}` });
+        return;
+      }
+      for (const [name, dot] of Object.entries(step.capture ?? {})) { const value = captureAt(body, dot); if (value === undefined || value === null) collect.push({ step: step.name, beat: 'CAPTURE', detail: `nothing at "${dot}" in the answer to capture as {${name}}` }); else remember(name, value); }
+      try { for (const detail of expectFailures(step.expect ?? [], body, captured)) collect.push({ step: step.name, beat: 'EXPECT', detail }); }
+      catch (e) { collect.push({ step: step.name, beat: 'CAPTURE', detail: (e as Error).message }); }
+      if (step.close) { mongoLinks.get(link)?.close(); mongoLinks.delete(link); }
+    } catch (e) { collect.push({ step: step.name, beat: 'MONGO', detail: (e as Error).message }); }
+    finally { opts.hooks?.after?.(step as never, link, answered, collect.slice(before)); }
+  };
+
   // A git step is a git client's clone or push on the pack's smart-HTTP plane (packages/twin-standard/src/walk/git-client.ts); its answer is
   // checked as a request's is (status, captures, expectations).
   const gitStep = async (step: GitStep): Promise<void> => {
@@ -679,6 +735,7 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
   try {
   for (const step of journey.steps) {
     stepNow = step.name;
+    if (isMongo(step)) { await mongoStep(step); continue; }
     if (isMysql(step)) { await mysqlStep(step); continue; }
     if (isResp(step)) { await respStep(step); continue; }
     if (isLine(step)) { await speak(step); continue; }
@@ -766,7 +823,11 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
     let res: Answer;
     if (deferred) {
       pending.delete(step.await!);
-      try { res = await deferred.answer; } catch (error) { collect.push({ step: step.name, beat: 'PENDING', detail: String(error) }); continue; }
+      const got = await deferred.answer.then((a) => ({ a }), (error: unknown) => ({ error }));
+      // a body the vendor broke off, as the collect expects (`breaks`): its words are the answer
+      if ('error' in got) { if (step.breaks === undefined || !String(got.error).includes(step.breaks)) collect.push({ step: step.name, beat: 'PENDING', detail: String(got.error) }); continue; }
+      // over a listener, which ends a broken stream as a short body, the ordinary checks hold the part that arrived
+      res = got.a;
     } else res = await send({ ...step, body, headers, ...(formed ? { formed } : {}) }, path);
     answered = { status: res.status, body: res.body, headers: res.headers };
     if (opts.verbose) console.log(`  ${String(res.status).padEnd(4)} ${step.method.padEnd(6)} ${path.slice(0, 70).padEnd(70)} ${step.name}`);

package/src/walk/journey-kit.ts

@@ -52,6 +52,8 @@ export type JourneyStep = {
   defer?: string;
   /** Collect a previously deferred request once, with all ordinary response checks. */
   await?: string;
+  /** A collect whose answer's body breaks off with these words (a stream the vendor ends in an error), not arriving whole. */
+  breaks?: string;
   body?: unknown;
   /** A body of raw bytes (a gzip'd package, an image, an audio file), base64 in the journey and sent decoded: a
    *  string body is sent as UTF-8, which no binary format survives. A step carries `body` or `bodyBase64`, never both. */
@@ -215,7 +217,8 @@ export function expectFailures(expect: NonNullable<JourneyStep['expect']>, body:
     if (e.contains !== undefined) {
       const want = fill(e.contains, captured);
       const text = typeof v === 'string' ? v : JSON.stringify(v ?? '');
-      if (!text.includes(want)) out.push(`${e.path || 'the answer'} does not contain ${JSON.stringify(want)}`);
+      // what it holds instead, its first characters, so a failure says what the twin answered
+      if (!text.includes(want)) out.push(`${e.path || 'the answer'} does not contain ${JSON.stringify(want)} (it begins ${JSON.stringify(text.replace(/\s+/g, ' ').slice(0, 160))})`);
     }
     // `notContains`: the value does not hold the text (a list that omits what the caller may not see)
     if (e.notContains !== undefined) {
@@ -327,7 +330,7 @@ export type Journey = {
   steps: Step[];
 };
 
-export type Step = JourneyStep | WaitStep | LineStep | GitStep | SocketStep | SqlStep | RespStep | MysqlStep;
+export type Step = JourneyStep | WaitStep | LineStep | GitStep | SocketStep | SqlStep | RespStep | MysqlStep | MongoStep;
 
 /** A step on a MySQL wire (the kernel's @volter/world-core/mysql, a pack's `create<Name>TwinStream`), spoken as an
  *  application's driver speaks it. `signIn` opens a named connection (`link`, `mysql` by default) with a user and
@@ -342,6 +345,19 @@ export type MysqlStep = {
 };
 export const isMysql = (step: Step): step is MysqlStep => 'mysql' in step || 'signIn' in step;
 
+/** A step on a pack's MongoDB wire, sent as an application's driver sends it (`@volter/world-core/mongodb`'s client)
+ *  on a named `link`, opened by the first step that names it. `mongo` is ONE command document, written in
+ *  canonical extended JSON so that what a document holds is written as it is: an ObjectId as `{ "$oid": … }`, a date
+ *  as `{ "$date": … }`, and a number with its kind (`{ "$numberInt": "1" }`, `$numberLong`, `$numberDouble`); a bare
+ *  JSON number is refused, because JSON does not say which of three kinds it is. `database` is the database it is
+ *  sent to (the link's last one, else `test`). Its answer is the reply document in the same form. A reply whose
+ *  `ok` is 0 fails the step unless `refused` names its `code`. */
+export type MongoStep = {
+  name: string; mongo: Record<string, unknown>; database?: string; link?: string; close?: boolean;
+  refused?: number; capture?: JourneyStep['capture']; expect?: JourneyStep['expect']; example?: string; exampleDiffers?: Record<string, string>;
+};
+export const isMongo = (step: Step): step is MongoStep => 'mongo' in step;
+
 /** A typed request/reply stream step. `resp` is one command or a pipeline of commands, on a named connection.
  * Captures and expectations see `{ replies, last, error }`; binary arguments use `{ base64 }`.
  * `read` drains pushes or a blocking command's answer without sending bytes. */
@@ -530,7 +546,9 @@ export function fill(template: string, captured: Captured, graphqlDocument = fal
   }
   // a doubled brace is the vendor's own template syntax ({{sample.output_text}} in an OpenAI grader), not a capture; a
   // capture's name starts with a letter, so a pattern's count (`[A-Za-z0-9]{40}` in an expectation) is no capture
-  return encoded.replace(/\{\{[^{}]*\}\}|\{([A-Za-z_][A-Za-z0-9_.]*)\}/g, (m, key: string | undefined, offset: number) => {
+  // `{literal:word}` is `{word}` as the vendor's page prints it (a Redis hash tag, `{queue}`), never a capture
+  return encoded.replace(/\{literal:([^{}]*)\}|\{\{[^{}]*\}\}|\{([A-Za-z_][A-Za-z0-9_.]*)\}/g, (m, literal: string | undefined, key: string | undefined, offset: number) => {
+    if (literal !== undefined) return `{${literal}}`;
     if (selections.has(offset)) return m;
     if (key === undefined) return m;
     const v = captured[key];

package/src/walk/rendered-scalar.ts

@@ -0,0 +1,17 @@
+/** Unfilled scalar type words in schema-rendered examples. Real values and enum words remain literal. */
+const BOOLEAN = Symbol('rendered boolean');
+const INTEGER = Symbol('rendered integer');
+const NUMBER = Symbol('rendered number');
+export function renderedScalar(value: unknown, types: unknown[], enumerated: boolean): symbol | undefined {
+  if (enumerated || typeof value !== 'string') return undefined;
+  if (value === 'boolean' && types.includes('boolean')) return BOOLEAN;
+  if (value === 'integer' && types.includes('integer')) return INTEGER;
+  if (value === 'number' && types.includes('number')) return NUMBER;
+  return undefined;
+}
+export function isRenderedScalar(value: unknown): boolean { return value === BOOLEAN || value === INTEGER || value === NUMBER; }
+export function matchesRenderedScalar(marker: unknown, value: unknown): boolean {
+  if (marker === BOOLEAN) return typeof value === 'boolean';
+  if (marker === INTEGER) return typeof value === 'bigint' || (typeof value === 'number' && Number.isInteger(value));
+  return marker === NUMBER && typeof value === 'number' && Number.isFinite(value);
+}

package/src/walk/sse-example.ts

@@ -1,6 +1,7 @@
 import { parseExactJson } from '@volter/world-core/exact-json';
 /** Match a published SDK-decoded SSE sample to a real HTTP frame; preserve non-SSE answers and mismatches. */
 export function sseExampleAnswer(example: unknown, answer: unknown, matches: (frame: unknown) => boolean): unknown {
+  if (typeof example === 'string' && typeof answer === 'string') return streamSample(example, answer);
   if (typeof answer !== 'string' || !example || typeof example !== 'object') return answer;
   const wrapped = 'data' in example;
   if (!wrapped && !('type' in example) && !('object' in example)) return answer;
@@ -23,3 +24,39 @@ export function sseExampleAnswer(example: unknown, answer: unknown, matches: (fr
   const candidates = relevant.map((frame) => wrapped ? frame : frame.data);
   return candidates.find(matches) ?? candidates[0] ?? answer;
 }
+
+type Frame = { data: unknown; event?: string };
+/** A stream's frames: each block's `event:` and joined `data:` lines, or a block that is JSON alone (a page printing the
+ *  chunks without their framing); `[DONE]`, an elision (`...`, `....`) and anything else not JSON are no frame. */
+function framesOf(text: string): Frame[] {
+  const frames: Frame[] = [];
+  for (const block of text.replace(/\r\n/g, '\n').split(/\n\s*\n/)) {
+    const lines = block.split('\n').map((l) => l.trim()).filter(Boolean);
+    const framed = lines.some((l) => l.startsWith('data:'));
+    const data = framed ? lines.filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join('\n') : lines.join('\n');
+    if (!data || data === '[DONE]' || /^\.{3,}$/.test(data)) continue;
+    const event = lines.find((l) => l.startsWith('event:'))?.slice(6).trim();
+    try { frames.push({ data: parseExactJson(data), ...(event ? { event } : {}) }); } catch { /* not a JSON frame */ }
+  }
+  return frames;
+}
+const kindOf = (v: unknown): string => (v === null ? 'null' : Array.isArray(v) ? 'array' : typeof v);
+/** Whether an answer's frame holds a published frame's keys with the same kinds of value (a null published value admits
+ *  any), its type or object the same. */
+function holds(published: unknown, got: unknown): boolean {
+  if (!published || typeof published !== 'object' || Array.isArray(published)) return kindOf(published) === kindOf(got) || published === null;
+  if (!got || typeof got !== 'object' || Array.isArray(got)) return false;
+  const p = published as Record<string, unknown>, g = got as Record<string, unknown>;
+  for (const key of ['type', 'object']) if (typeof p[key] === 'string' && p[key] !== g[key]) return false;
+  return Object.entries(p).every(([k, v]) => k in g && (v === null || kindOf(v) === kindOf(g[k]) || (kindOf(v) === 'object' && holds(v, g[k]))));
+}
+/** A published stream printed as text (a page's sample of a stream, its middle elided): each frame it shows has a frame
+ *  of the answer holding it, the answer then standing for the sample; else the answer, unchanged, which the literal
+ *  comparison reports. */
+function streamSample(example: string, answer: string): unknown {
+  const published = framesOf(example);
+  if (published.length < 2) return answer;
+  const frames = framesOf(answer);
+  const held = published.every((p) => frames.some((f) => (!p.event || p.event === f.event) && holds(p.data, f.data)));
+  return held ? example : answer;
+}