@volter/twin-standard 1.0.157 → 1.0.159

package/src/walk/behavior-journey.ts

@@ -12,16 +12,19 @@ import { attributionName } from '../verdict.ts';
 import { mongodb, mysql, redis } from '@volter/world-core';
 import { createRequire } from 'node:module';
 import { gzipSync } from 'node:zlib';
-import { parse, visit } from 'graphql';
 import { existsSync, readdirSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { databaseSession, type DatabaseSession, getActivePackAssets, getActiveWorldStore, LocalPackAssets, MemoryBlobStore, MemoryWorldStore, setActiveBlobStore, setActivePackAssets, setActiveWorldStore, setDeliveryObserver, setInProcessTwins, type InProcessTwin, type TwinStream, type TwinStreamConnection } from '@volter/world-core/runtime';
+import { databaseSession, type DatabaseSession, getActivePackAssets, getActiveWorldStore, LocalPackAssets, MemoryBlobStore, MemoryWorldStore, setActiveBlobStore, setActivePackAssets, setActiveWorldStore, setDeliveryObserver, setInProcessTwins, settleEventDeliveries, worldDoorMatcher, worldNow, type InProcessTwin, type TwinStream, type TwinStreamConnection } from '@volter/world-core/runtime';
 import { createHostTable, type HostFacts } from '@volter/world-core/host-routing';
 import packFactsDocument from '@volter/world-core/generated/pack-facts.json' with { type: 'json' };
 import { type Captured, type GitStep, type Journey, type JourneyStep, type LineStep, type SocketStep, type SqlStep, type RespStep, type MysqlStep, type MongoStep, isMongo, isMysql, isResp, at, captureAt, echoFailures, expandFills, expectFailures, fill, fillJson, formBody, isGit, isLine, isSql, isSocket, isWait, jwtAuthorization, oauth1Authorization, presignedPath, sigv4Headers, waitMs } from './journey-kit.ts';
-import { lanesOf, lineShape, shapeChecker, type LineConnection, type ShapeChecker } from './shape.ts';
+import { decodeResponse, leaked, lineShape, responseProblems, shapeChecker, vendorFieldsOf, type LineConnection, type ShapeChecker } from './shape.ts';
 import { walkPackDirectory } from './pack-directory.ts';
-import { xmlMembers } from './xml-view.ts';
+import { unitsIn } from '../lanes.ts';
+import { openWebSocket, readHttpResponse } from '../clients/harness.ts';
+import { timestampOrderFailures, type ReplayStep } from './replay.ts';
+import type { ManagedDatabaseWalk } from './walk-database.ts';
+export { leaked } from './shape.ts';
 
 /** The pack repository the walk finds a vendor's pack in, set by the caller before its first walk. */
 let packsRoot: string | undefined;
@@ -134,19 +137,11 @@ const claims = (keys: string[], url: string): boolean => {
   return keys.some((key) => walkHosts[key]!(u.hostname, u.pathname));
 };
 
-type Answer = { status: number; body: unknown; text: string; headers: Record<string, string> };
+type SentRequest = { method: string; url: string; headers: Record<string, string> };
+type StepAnswer = { status: number; body: unknown; headers?: Record<string, string>; request?: SentRequest; wireBody?: unknown; sql?: ReplayStep['sql']; connection?: ReplayStep['connection'] };
+type Answer = StepAnswer & { text: string; headers: Record<string, string>; request: SentRequest; wireBody: unknown; bodyError?: { message: string; cancelled: boolean } };
 export type Failure = { step: string; beat: string; detail: string };
 
-/** What a stored row keeps for the twin that no schema of the vendor's spec names, by path, in anything the World shows a
- *  client: the pack's `_`-prefixed bookkeeping (GitHub's `_links` is the vendor's), and, in what no answer's SHAPE
- *  reads (an event's delivery, a socket frame), the kernel's own `updatedAt`, which a raw row filtered on `_` alone
- *  carries (Slack's message events once did). The kernel's `deleted` is not read there: vendors send their own
- *  (Stripe's deleted customer's `deleted: true`). */
-export function leaked(v: unknown, vendorFields: Set<string>, at = '', unshaped = false): string[] {
-  if (Array.isArray(v)) return v.flatMap((x, i) => leaked(x, vendorFields, `${at}[${i}]`, unshaped)).slice(0, 5);
-  if (!v || typeof v !== 'object') return [];
-  return Object.entries(v).flatMap(([k, x]) => ((k.startsWith('_') || (unshaped && k === 'updatedAt')) && !vendorFields.has(k) ? [`${at}${at ? '.' : ''}${k}`] : leaked(x, vendorFields, `${at}${at ? '.' : ''}${k}`, unshaped))).slice(0, 5);
-}
 /** Whether a tool_choice forbids every tool call (OpenAI's 'none', Anthropic's { type: 'none' }). */
 const forbidsTools = (choice: unknown): boolean => choice === 'none' || (!!choice && typeof choice === 'object' && (choice as { type?: unknown }).type === 'none');
 /** The tool calls an answer or frames carry, in any vendor's form: a chat message's `tool_calls`, a Responses or
@@ -160,24 +155,18 @@ function toolCallsIn(v: unknown, out: string[] = []): string[] {
   for (const [k, x] of Object.entries(o)) if (k !== 'tool_calls') toolCallsIn(x, out);
   return out;
 }
-/** Every field name the vendor's spec gives any resource (the derived surface), per vendor. */
+/** The selected vendor's shared schema and native-error inventory, loaded before walking. */
 const fieldsByVendor = new Map<string, Set<string>>();
 function specFields(vendor: string): Set<string> {
-  if (!fieldsByVendor.has(vendor)) {
-    // the pack's surface and each of its lanes' (a walk reaches a lane through the pack's router)
-    const directory = directoryFor(vendor);
-    const files = [directory, ...lanesOf(directory)].map((dir) => join(dir, 'src', 'generated', 'surface.gen.json')).filter((f) => existsSync(f));
-    const surfaces = files.map((f) => parseExactJson(readFileSync(f, 'utf8')) as { resources?: Array<{ fields: Array<{ name: string }> }> });
-    fieldsByVendor.set(vendor, new Set(surfaces.flatMap((surface) => (surface.resources ?? []).flatMap((r) => r.fields.map((f) => f.name)))));
-  }
   return fieldsByVendor.get(vendor)!;
 }
 
 /** What a caller that scores a walk observes around each step: the step, its resolved path and body,
- *  and (after) the answer and the failures the step produced. */
+ *  and (after) the answer and the failures the step produced. HTTP answers include the constructed request and
+ *  decoded wire body separately from the capture view. */
 export type StepHooks = {
   before?: (step: JourneyStep | LineStep | GitStep | RespStep, path: string, body: unknown) => void;
-  after?: (step: JourneyStep | LineStep | GitStep | RespStep, path: string, answer: { status: number; body: unknown; headers?: Record<string, string> }, failures: Failure[]) => void;
+  after?: (step: JourneyStep | LineStep | GitStep | RespStep, path: string, answer: StepAnswer, failures: Failure[]) => void;
 };
 
 /** One reply of a line protocol: its code and the text of each of its lines (`250-a`, `250 b`). */
@@ -202,7 +191,7 @@ function takeReplies(c: Connection): LineReply[] {
 /** A pack served for a walk the way a World runs it (its server.ts on a port, the World's root and scenario its options):
  *  the walk's requests go to it over HTTP, the vendor's host in their Host header, instead of to the fetch in process;
  *  its sockets are WebSockets to that port, and a line connection (`stream`) is a TCP connection to its protocol port. */
-export type ServedTwin = (root: string, options: { scenarioPath?: string; application?: import('@volter/world-core').PackFetchOptions['application'] }) => Promise<{ scheme?: 'http' | 'https'; fetch: (request: Request, options?: { credentials?: RequestCredentials }) => Promise<Response>; stop: () => void | Promise<void>; socket?: (url: string, headers: Record<string, string>) => WebSocket; stream?: TwinStream }>;
+export type ServedTwin = (root: string, options: { scenarioPath?: string; application?: import('@volter/world-core').PackFetchOptions['application'] }) => Promise<{ scheme?: 'http' | 'https'; fetch: ((request: Request, options?: { credentials?: RequestCredentials }) => Promise<Response>) & ManagedDatabaseWalk; stop: () => void | Promise<void>; socket?: (url: string, headers: Record<string, string>) => WebSocket; stream?: TwinStream }>;
 
 export async function walk(journey: Journey, root: string, collect: Failure[], opts: { beats: boolean; verbose?: boolean; hooks?: StepHooks; captured?: Captured; served?: ServedTwin }): Promise<Captured> {
   attributionName(journey.author);
@@ -220,23 +209,9 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
   const factory = await vendorFetch(journey.vendor);
   const views = await vendorAnswerViews(packsDir(), journey.vendor, dependenciesDir());
   if (!factory) { collect.push({ step: '(setup)', beat: 'ADAPTER', detail: `no create<Name>Fetch adapter resolved for ${journey.vendor}` }); return {}; }
-  // the life's companions (`companions`): the packs its vendor calls, each over this walk's root, on the same World and
-  // clock; reached in process by the vendor's ctx.vendorFetch and by a step to a URL their host rules claim
-  const companions: Array<{ vendor: string; keys: string[]; fetch: InProcessTwin }> = [];
-  for (const vendor of journey.companions ?? []) {
-    const keys = routingKeys(vendor);
-    const companion = vendor === journey.vendor ? undefined : await vendorFetch(vendor);
-    if (!companion || !keys.length) { collect.push({ step: '(setup)', beat: 'ADAPTER', detail: vendor === journey.vendor ? `the life's own vendor ${vendor} is no companion of it` : !companion ? `no create<Name>Fetch adapter for the companion ${vendor}` : `the companion ${vendor} declares no host rules (pack facts) a call could reach it by` }); return {}; }
-    companions.push({ vendor, keys, fetch: companion(root) });
-  }
-  const ownKeys = routingKeys(journey.vendor);
-  /** The companion that serves a step's URL: one whose host rules claim it where the life's own vendor's do not. */
-  const companionOf = (url: string) => (claims(ownKeys, url) ? undefined : companions.find((c) => claims(c.keys, url)));
-  // the life's scenario is the World's: written into the walk's World store, which the pack reads it from as a World's
-  // handlers/<vendor>.json is read
-  const scenarioPath = journey.scenario ? join(directoryFor(journey.vendor), 'journeys', journey.scenario) : undefined;
-  if (scenarioPath) getActiveWorldStore().write(scenarioPath, readFileSync(scenarioPath, 'utf8'));
-  // the application the vendor asks (ctx.ask) is the life's stand-in, this walk's fetch's alone: its answers, in order,
+  // what an earlier walk captured, when this one continues it (the branch round-trip sends a life's write again)
+  const captured: Captured = { ...(opts.captured ?? {}) };
+  // the application the vendors call is the life's stand-in, shared by this walk's packs: its answers, in order,
   // by method and URL prefix, each request header it declares checked as sent; a question none takes is unreachable, as
   // in a sealed World
   const answers = [...(journey.application ?? [])];
@@ -253,13 +228,39 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
     }
     return { status: a!.status, ...(a!.headers ? { headers: a!.headers } : {}), ...(a!.bodyError === undefined ? {} : { bodyError: a!.bodyError }), body: typeof a!.body === 'string' ? a!.body : a!.body === undefined ? '' : JSON.stringify(a!.body) };
   };
+  // the life's companions (`companions`): the packs its vendor calls, each over this walk's root, on the same World and
+  // clock; reached in process by the vendor's ctx.vendorFetch and by a step to a URL their host rules claim
+  const companions: Array<{ vendor: string; keys: string[]; fetch: InProcessTwin; databaseDoor?: string }> = [];
+  for (const vendor of journey.companions ?? []) {
+    const keys = routingKeys(vendor);
+    const companion = vendor === journey.vendor ? undefined : await vendorFetch(vendor);
+    if (!companion || !keys.length) { collect.push({ step: '(setup)', beat: 'ADAPTER', detail: vendor === journey.vendor ? `the life's own vendor ${vendor} is no companion of it` : !companion ? `no create<Name>Fetch adapter for the companion ${vendor}` : `the companion ${vendor} declares no host rules (pack facts) a call could reach it by` }); return {}; }
+    companions.push({ vendor, keys, fetch: companion(root, { application }), databaseDoor: companion.managedDatabaseDoor });
+  }
+  const eventFields = new Map<string, Set<string>>();
+  for (const vendor of [journey.vendor, ...companions.map((companion) => companion.vendor)]) {
+    const directory = directoryFor(vendor);
+    if (!fieldsByVendor.has(vendor)) fieldsByVendor.set(vendor, await vendorFieldsOf(directory));
+    for (const unit of unitsIn(directory)) {
+      const { manifest } = await import(join(unit, 'src', 'manifest.ts')) as { manifest: { service?: string; vendor: string } };
+      const service = manifest.service ?? manifest.vendor;
+      const fields = eventFields.get(service) ?? new Set<string>();
+      for (const field of specFields(vendor)) fields.add(field);
+      eventFields.set(service, fields);
+    }
+  }
+  const ownKeys = routingKeys(journey.vendor);
+  /** The companion that serves a step's URL: one whose host rules claim it where the life's own vendor's do not. */
+  const companionOf = (url: string) => (claims(ownKeys, url) ? undefined : companions.find((c) => claims(c.keys, url)));
+  // the life's scenario is the World's: written into the walk's World store, which the pack reads it from as a World's
+  // handlers/<vendor>.json is read
+  const scenarioPath = journey.scenario ? join(directoryFor(journey.vendor), 'journeys', journey.scenario) : undefined;
+  if (scenarioPath) getActiveWorldStore().write(scenarioPath, readFileSync(scenarioPath, 'utf8'));
+  const shape = await shapeOf(journey.vendor);
   const served = opts.served ? await opts.served(root, { ...(scenarioPath ? { scenarioPath } : {}), application }) : undefined;
   const twin = served ? served.fetch : factory(root, { ...(scenarioPath ? { scenarioPath } : {}), application });
-  const shape = await shapeOf(journey.vendor);
   const host = journey.host ?? 'twin.local';
   const origin = `${served?.scheme ?? 'http'}://${host}`;
-  // what an earlier walk captured, when this one continues it (the branch round-trip sends a life's write again)
-  const captured: Captured = { ...(opts.captured ?? {}) };
   // a capture's JSON value where it is not text (a number, a boolean, an object such as database-js's `session`)
   const typedValues: Record<string, unknown> = {};
   const remember = (name: string, v: unknown): void => {
@@ -280,24 +281,20 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
     // a body the client compresses or encodes as it sends it (posthog-js's gzip-js, its base64 form), captures filled first
     if (step.encode && typeof init.body === 'string') init.body = step.encode === 'gzip' ? new Uint8Array(gzipSync(Buffer.from(init.body))) : `data=${encodeURIComponent(Buffer.from(init.body).toString('base64'))}`;
     // a step may follow an absolute url an answer gave it (a hosted page's), as the application does
-    // a twin that throws answers 500, as a server would: the walk reports the step and goes on to the next
-    let res: Response;
-    try {
-      const request = new Request(/^https?:\/\//.test(path) ? path : `${origin}${path}`, init);
-      const companion = companionOf(path);
-      res = companion ? await companion.fetch(request) : served ? await served.fetch(request, { credentials: step.credentials ?? 'same-origin' }) : await twin(request);
-    }
-    catch (error) { res = new Response(`twin threw: ${error instanceof Error ? error.message : String(error)}`, { status: 500 }); }
-    const text = await res.text();
-    let body: unknown; try { body = parseExactJson(text); } catch { body = text; }
-    // an XML answer (S3's) is its root element's members (xml-view.ts), as a JSON answer is its object
-    if (typeof body === 'string' && /xml/.test(res.headers.get('content-type') ?? '')) body = xmlMembers(text) ?? text;
+    const companion = companionOf(path);
+    await ((companion?.fetch ?? twin) as ManagedDatabaseWalk).managedDatabaseUrl?.();
+    const request = new Request(/^https?:\/\//.test(path) ? path : `${origin}${path}`, init);
+    const answer = await readHttpResponse(async (request) => {
+      // a twin that throws answers 500, as a server would: the walk reports the step and goes on to the next
+      try { return companion ? await companion.fetch(request) : served ? await served.fetch(request, { credentials: step.credentials ?? 'same-origin' }) : await twin(request); }
+      catch (error) { if (served) throw error; return new Response(`twin threw: ${error instanceof Error ? error.message : String(error)}`, { status: 500 }); }
+    }, request);
+    const wireBody = decodeResponse(answer.text, answer.headers['content-type']);
+    let body = wireBody;
     // a hosted flow answers with a redirect: its target and query are what a step captures (redirect.code)
-    const location = res.status >= 300 && res.status < 400 ? res.headers.get('location') : null;
+    const location = answer.status >= 300 && answer.status < 400 ? answer.headers.location : undefined;
     if (location) body = { location, redirect: Object.fromEntries(new URL(location, origin).searchParams) };
-    // every cookie an answer sets, joined as one header: a sign-in sets several, and a map keeps only the last
-    const cookies = res.headers.getSetCookie();
-    return { status: res.status, body, text, headers: { ...Object.fromEntries(res.headers), ...(cookies.length ? { 'set-cookie': cookies.join(', ') } : {}) } };
+    return { status: answer.status, body, wireBody, request: { method: request.method, url: request.url, headers: Object.fromEntries(request.headers) }, text: answer.text, headers: answer.headers, ...(answer.bodyError ? { bodyError: answer.bodyError } : {}) };
   };
 
   // a wait moves the World clock; the walk puts it back where it found it, so a replay starts at the same instant
@@ -592,8 +589,7 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
       const ws = served!.socket!(at.path, opening);
       ws.onmessage = (e) => { const text = String(e.data); try { opened.frames.push(parseExactJson(text)); } catch { opened.frames.push(text); } };
       ws.onclose = (e) => { opened.closed ??= { code: e.code, reason: e.reason }; };
-      const up = await new Promise<boolean>((done) => { ws.onopen = () => done(true); ws.onerror = () => done(false); setTimeout(() => done(false), 5000); });
-      if (!up) { collect.push({ step: step.name, beat: 'STATUS', detail: `no socket at ${step.open}` }); return; }
+      if (!await openWebSocket(ws)) { collect.push({ step: step.name, beat: 'STATUS', detail: `no socket at ${step.open}` }); return; }
       sockets.set(step.socket, opened);
       wire.set(step.socket, ws);
     }
@@ -679,9 +675,10 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
   const sqlStep = async (step: SqlStep): Promise<void> => {
     const failuresBefore = collect.length;
     opts.hooks?.before?.(step as never, 'sql', step.sql);
-    let answered: { status: number; body: unknown } = { status: 0, body: undefined };
+    let answered: StepAnswer = { status: 0, body: undefined };
     try {
-      const url = await (twin as { managedDatabaseUrl?: () => Promise<string> }).managedDatabaseUrl?.();
+      const database = twin as ManagedDatabaseWalk;
+      const url = await database.managedDatabaseUrl?.();
       if (url === undefined) { collect.push({ step: step.name, beat: 'ADAPTER', detail: `${journey.vendor} binds no managed database in this walk, so a SQL step has none to run on` }); return; }
       let results;
       try {
@@ -694,9 +691,13 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
         if (step.refused !== code) collect.push({ step: step.name, beat: 'STATUS', detail: `Postgres refused it: ${code} ${(e as Error).message}${step.refused ? ` (wanted ${step.refused})` : ''}` });
         return;
       }
-      const last = results.at(-1) ?? { fields: [], rows: [], command: '' };
+      const last = results.at(-1) ?? { fields: [], fieldTypes: [], rows: [], command: '' };
       const body = { command: last.command, rows: last.rows.map((row) => Object.fromEntries(last.fields.map((field, i) => [field, row[i] ?? null]))) };
-      answered = { status: 0, body };
+      // Duplicate column names have the same last-column ownership as the visible row projection.
+      const fieldTypes = Object.fromEntries(last.fields.map((field, i) => [field, last.fieldTypes[i]]));
+      const timeFields = Object.fromEntries(Object.entries(fieldTypes).filter(([, type]) => type === 1114 || type === 1184)) as Record<string, 1114 | 1184>;
+      answered = { status: 0, body, ...(database.managedDatabaseClock === 'wall' ? { sql: { clock: database.managedDatabaseClock, timeFields } } : {}) };
+      for (const detail of timestampOrderFailures(body, timeFields, step.timeOrder ?? [])) collect.push({ step: step.name, beat: 'EXPECT', detail });
       if (step.refused !== undefined) { collect.push({ step: step.name, beat: 'STATUS', detail: `Postgres completed it (${last.command}), wanted ${step.refused}` }); return; }
       for (const [name, dot] of Object.entries(step.capture ?? {})) {
         const v = captureAt(body, dot);
@@ -711,7 +712,7 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
     }
   };
 
-  const pending = new Map<string, { method: string; path: string; body: unknown; headers: Record<string, string> | undefined; answer: Promise<Answer>; abort: AbortController; timer: ReturnType<typeof setTimeout> }>();
+  const pending = new Map<string, { method: string; path: string; body: unknown; headers: Record<string, string> | undefined; answer: Promise<Answer>; abort: AbortController }>();
   // the clock is world state, read and moved through the active World store (in memory for a walk)
   const clockFile = process.env.TWIN_WORLD_CLOCK_FILE;
   const store = getActiveWorldStore();
@@ -726,7 +727,9 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
   // what the World sends a receiver (an event's delivery) is read as the receiver reads it, for the twin's own fields
   const previousObserver = setDeliveryObserver((d) => {
     let body: unknown; try { body = parseExactJson(d.body); } catch { return; }
-    for (const key of leaked(body, specFields(journey.vendor), '', true)) collect.push({ step: stepNow, beat: 'LEAK', detail: `the event sent to ${d.url} carries the twin's own field ${key}` });
+    const fields = eventFields.get(d.service);
+    if (!fields) { collect.push({ step: stepNow, beat: 'LEAK', detail: `the event sent to ${d.url} names no selected owner for service ${d.service}` }); return; }
+    for (const key of leaked(body, fields, '', true)) collect.push({ step: stepNow, beat: 'LEAK', detail: `the event sent to ${d.url} carries the twin's own field ${key}` });
   });
   const previousTwins = setInProcessTwins(Object.fromEntries([
     ...companions.flatMap((c) => c.keys.map((key) => [key, c.fetch] as const)),
@@ -734,6 +737,7 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
   ]));
   try {
   for (const step of journey.steps) {
+    await settleEventDeliveries(root);
     stepNow = step.name;
     if (isMongo(step)) { await mongoStep(step); continue; }
     if (isMysql(step)) { await mysqlStep(step); continue; }
@@ -744,7 +748,7 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
     if (isSql(step)) { await sqlStep(step); continue; }
     if (isWait(step)) {
       if (!clockFile || startedAt === undefined) { collect.push({ step: step.name, beat: 'WAIT', detail: 'the walk has no World clock to move' }); continue; }
-      try { store.write(clockFile, new Date(Date.parse((store.read(clockFile) ?? '').trim()) + waitMs(step.wait)).toISOString()); } catch (e) { collect.push({ step: step.name, beat: 'WAIT', detail: (e as Error).message }); }
+      try { store.write(clockFile, new Date(Date.parse(worldNow()) + waitMs(step.wait)).toISOString()); } catch (e) { collect.push({ step: step.name, beat: 'WAIT', detail: (e as Error).message }); }
       continue;
     }
     let path: string;
@@ -769,8 +773,7 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
         // a form is signed as it is sent: its multipart type, whose parts OAuth 1.0a's base string never carries
         const sent = formed ? undefined : body === undefined ? undefined : typeof body === 'string' ? body : JSON.stringify(body);
         const contentType = Object.entries(headers ?? {}).find(([k]) => k.toLowerCase() === 'content-type')?.[1] ?? (formed ? formed.contentType : step.bodyBase64 === undefined ? defaultType(body) : undefined);
-        const instant = clockFile && store.exists(clockFile) ? Date.parse((store.read(clockFile) ?? '').trim()) : Number.NaN;
-        headers = { ...(headers ?? {}), authorization: oauth1Authorization(o, step.method, url, sent, contentType, Number.isNaN(instant) ? 0 : instant) };
+        headers = { ...(headers ?? {}), authorization: oauth1Authorization(o, step.method, url, sent, contentType, Date.parse(worldNow())) };
       }
       if (step.jwt) headers = { ...(headers ?? {}), authorization: jwtAuthorization(step.jwt, captured) };
       // an AWS SDK's request is signed as it sends it, at the World clock's instant
@@ -778,9 +781,8 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
         const url = /^https?:\/\//.test(path) ? path : `${origin}${path}`;
         // the bytes and the type sent: a form's multipart body and boundary, raw bytes, or the body's text
         const sent = formed ? formed.bytes : step.bodyBase64 !== undefined ? Uint8Array.from(Buffer.from(step.bodyBase64, 'base64')) : body === undefined ? undefined : typeof body === 'string' ? body : JSON.stringify(body);
-        const instant = clockFile && store.exists(clockFile) ? Date.parse((store.read(clockFile) ?? '').trim()) : Number.NaN;
         const base = { ...(formed ? { 'content-type': formed.contentType } : step.bodyBase64 === undefined ? { 'content-type': defaultType(body) } : {}), ...(journey.headers ?? {}), ...(headers ?? {}) };
-        headers = { ...(headers ?? {}), ...sigv4Headers(step.sigv4, captured, step.method, url, base, sent, Number.isNaN(instant) ? 0 : instant) };
+        headers = { ...(headers ?? {}), ...sigv4Headers(step.sigv4, captured, step.method, url, base, sent, Date.parse(worldNow())) };
       }
     } catch (e) { collect.push({ step: step.name, beat: 'CAPTURE', detail: (e as Error).message }); continue; }
     if (step.defer) {
@@ -788,14 +790,10 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
         collect.push({ step: step.name, beat: 'PENDING', detail: 'a deferred request needs a unique key and no response checks or state beats' }); continue;
       }
       const abort = new AbortController();
-      const timer = setTimeout(() => abort.abort(new Error('deferred request exceeded 10 s')), 10_000);
-      const answer = Promise.race([
-        send({ ...step, body, headers, ...(formed ? { formed } : {}) }, path, abort.signal),
-        new Promise<Answer>((_, reject) => abort.signal.addEventListener('abort', () => reject(abort.signal.reason), { once: true })),
-      ]).finally(() => clearTimeout(timer));
+      const answer = send({ ...step, body, headers, ...(formed ? { formed } : {}) }, path, abort.signal);
       // A rejected body read must not become an unhandled promise while socket steps run.
       void answer.catch(() => undefined);
-      pending.set(step.defer, { method: step.method, path, body, headers, answer, abort, timer });
+      pending.set(step.defer, { method: step.method, path, body, headers, answer, abort });
       continue;
     }
     const deferred = step.await ? pending.get(step.await) : undefined;
@@ -817,34 +815,56 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
     }
     const failuresBefore = collect.length;
     opts.hooks?.before?.(step, path, body);
-    let answered: { status: number; body: unknown; headers?: Record<string, string> } = { status: 0, body: undefined };
+    let answered: StepAnswer = { status: 0, body: undefined };
     // every step that ran reports, however it ended: a scorer counts the failing ones too
     try {
     let res: Answer;
     if (deferred) {
       pending.delete(step.await!);
       const got = await deferred.answer.then((a) => ({ a }), (error: unknown) => ({ error }));
-      // a body the vendor broke off, as the collect expects (`breaks`): its words are the answer
-      if ('error' in got) { if (step.breaks === undefined || !String(got.error).includes(step.breaks)) collect.push({ step: step.name, beat: 'PENDING', detail: String(got.error) }); continue; }
-      // over a listener, which ends a broken stream as a short body, the ordinary checks hold the part that arrived
+      if ('error' in got) { collect.push({ step: step.name, beat: 'PENDING', detail: String(got.error) }); continue; }
       res = got.a;
-    } else res = await send({ ...step, body, headers, ...(formed ? { formed } : {}) }, path);
-    answered = { status: res.status, body: res.body, headers: res.headers };
+    } else {
+      try { res = await send({ ...step, body, headers, ...(formed ? { formed } : {}) }, path); }
+      catch (error) { collect.push({ step: step.name, beat: 'CONNECTION', detail: String(error) }); continue; }
+    }
+    answered = { status: res.status, body: res.body, headers: res.headers, request: res.request, wireBody: res.wireBody };
+    if (step.breaks !== undefined) {
+      if (!res.bodyError || res.bodyError.cancelled) collect.push({ step: step.name, beat: 'PENDING', detail: 'expected a response body interrupted by the vendor; clean EOF or caller cancellation is not an interruption' });
+      else if (!served && !res.bodyError.message.includes(step.breaks)) collect.push({ step: step.name, beat: 'PENDING', detail: res.bodyError.message });
+    } else if (res.bodyError) collect.push({ step: step.name, beat: 'BODY', detail: res.bodyError.message });
     if (opts.verbose) console.log(`  ${String(res.status).padEnd(4)} ${step.method.padEnd(6)} ${path.slice(0, 70).padEnd(70)} ${step.name}`);
 
+    // A companion's answer belongs to its own schema, including when the status is unexpected.
+    const stepShape = serving === journey.vendor ? shape : await shapeOf(serving);
+    const companion = companionOf(res.request.url);
+    const databaseDoor = companion ? companion.databaseDoor : factory.managedDatabaseDoor;
+    if (databaseDoor && res.status >= 200 && res.status < 300) {
+      const selectedPath = new URL(stepShape ? stepShape.pathOf(res.request.method, res.request.url, res.request.headers) : res.request.url).pathname;
+      const declared = worldDoorMatcher([{ method: 'POST', path: databaseDoor }])(res.request.method, selectedPath);
+      if (declared) {
+        const connection = ((companion?.fetch ?? twin) as ManagedDatabaseWalk).managedDatabaseConnection;
+        if (!connection || !stepShape) collect.push({ step: step.name, beat: 'ADAPTER', detail: 'The selected native credential transport omitted its connection verification capability' });
+        else {
+          try {
+            const verified = await connection(res.request.method, selectedPath, res.status, res.wireBody);
+            if (!verified) throw new Error('The selected native credential transport supplied no authentication provenance');
+            answered.connection = verified;
+          } catch (error) {
+            collect.push({ step: step.name, beat: 'NATIVE-CONNECTION', detail: (error as Error).message });
+          }
+        }
+      }
+    }
+    const problems = responseProblems(stepShape, specFields(serving), {
+      method: res.request.method, path: res.request.url, requestHeaders: res.request.headers,
+      graphqlBody: graphqlBody ? body : undefined, status: res.status, mediaType: res.headers['content-type'], body: res.wireBody, text: res.text,
+    });
+    for (const detail of problems.shape) collect.push({ step: step.name, beat: 'SHAPE', detail });
+    for (const key of problems.leaks) collect.push({ step: step.name, beat: 'LEAK', detail: `answers the twin's own field ${key}` });
     const wanted = step.status;
     const ok = wanted === undefined ? res.status >= 200 && res.status < 300 : res.status === wanted;
     if (!ok) { collect.push({ step: step.name, beat: 'STATUS', detail: `${step.method} ${path} answered ${res.status}${wanted ? ` (wanted ${wanted})` : ' (wanted 2xx)'}: ${res.text.slice(0, 160)}` }); continue; }
-    // a step a companion served is checked against the companion's spec
-    const stepShape = serving === journey.vendor ? shape : await shapeOf(serving);
-    const answering = stepShape && res.status >= 200 && res.status < 300 && !stepShape.pinsOlder({ ...(journey.headers ?? {}), ...(headers ?? {}) }) ? stepShape.operationOf(step.method, path, { ...(journey.headers ?? {}), ...(headers ?? {}) }) : undefined;
-    if (answering) for (const detail of stepShape!.failures(answering.op, res.body, res.headers['content-type'], new Headers({ ...(journey.headers ?? {}), ...(headers ?? {}) }).get('accept') ?? undefined, res.status, { method: step.method, responseText: res.text })) collect.push({ step: step.name, beat: 'SHAPE', detail });
-    // what a pack keeps for itself is `_`-prefixed and never answered, under any name (Protocol 3, State)
-    const responseFields = new Set(specFields(serving));
-    if (body && typeof body === 'object' && typeof (body as { query?: unknown }).query === 'string' && graphqlBody) {
-      try { visit(parse((body as { query: string }).query), { Field: (field) => { responseFields.add(field.alias?.value ?? field.name.value); } }); } catch { /* malformed documents have no selected response keys */ }
-    }
-    if (res.status >= 200 && res.status < 300) for (const key of leaked(res.body, responseFields)) collect.push({ step: step.name, beat: 'LEAK', detail: `answers the twin's own field ${key}` });
     // a request that forbids tools (tool_choice none) is answered without a tool call, in any vendor's form
     if (res.status >= 200 && res.status < 300 && step.body && typeof step.body === 'object' && forbidsTools((step.body as { tool_choice?: unknown }).tool_choice)) for (const call of toolCallsIn(res.body)) collect.push({ step: step.name, beat: 'TOOLS', detail: `answered a tool call (${call}) to a request whose tool_choice is none` });
 
@@ -906,21 +926,37 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
   for (const key of pending.keys()) collect.push({ step: '(teardown)', beat: 'PENDING', detail: `deferred request ${key} was never collected` });
   return captured;
   } finally {
-    for (const request of pending.values()) { clearTimeout(request.timer); request.abort.abort(); }
-    for (const ws of wire.values()) ws.close();
-    const upgrade = (twin as { upgrade?: { close: (peer: Socket['peer']) => void } }).upgrade;
-    for (const socket of sockets.values()) if (!socket.closed) upgrade?.close(socket.peer);
-    await Promise.allSettled([...pending.values()].map((request) => request.answer));
-    setDeliveryObserver(previousObserver);
-    setInProcessTwins(previousTwins);
     try {
-      if (sqlSession) await sqlSession.then((session) => session.end(), () => {});
+      for (const request of pending.values()) request.abort.abort();
+      for (const ws of wire.values()) ws.close();
+      const upgrade = (twin as { upgrade?: { close: (peer: Socket['peer']) => void } }).upgrade;
+      for (const socket of sockets.values()) if (!socket.closed) upgrade?.close(socket.peer);
     } finally {
-      await served?.stop();
-      // a World's managed database ends with its walk (vendor-fetch.ts), so the next World declares it at the same port
-      await (twin as { stopDatabase?: () => Promise<void> }).stopDatabase?.();
+      try {
+        await Promise.allSettled([...pending.values()].map((request) => request.answer));
+        await settleEventDeliveries(root);
+      } finally {
+        setDeliveryObserver(previousObserver);
+        setInProcessTwins(previousTwins);
+        try {
+          if (sqlSession) await sqlSession.then((session) => session.end(), () => {});
+        } finally {
+          try {
+            await served?.stop();
+          } finally {
+            try {
+              // The selected twin's database ends with its walk.
+              const retired = await Promise.allSettled([twin, ...companions.map(companion => companion.fetch)]
+                .map(fetch => (fetch as ManagedDatabaseWalk).stopDatabase?.()));
+              const failures = retired.filter(result => result.status === 'rejected').map(result => result.reason);
+              if (failures.length) throw new AggregateError(failures, 'Walk native database retirement failed');
+            } finally {
+              if (clockFile && startedAt !== undefined) store.write(clockFile, startedAt);
+            }
+          }
+        }
+      }
     }
-    if (clockFile && startedAt !== undefined) store.write(clockFile, startedAt);
   }
 }
 

package/src/walk/journey-kit.ts

@@ -52,7 +52,7 @@ export type JourneyStep = {
   defer?: string;
   /** Collect a previously deferred request once, with all ordinary response checks. */
   await?: string;
-  /** A collect whose answer's body breaks off with these words (a stream the vendor ends in an error), not arriving whole. */
+  /** Require an interrupted body; in-process, its error also contains these words. Status and received bytes retain all checks. */
   breaks?: string;
   body?: unknown;
   /** A body of raw bytes (a gzip'd package, an image, an audio file), base64 in the journey and sent decoded: a
@@ -445,8 +445,8 @@ export const isWait = (step: Step): step is WaitStep => 'wait' in step;
  *  as written. Its answer is the last statement's `{ command, rows }`, each row its
  *  columns by name in Postgres's text format, which `expect` reads; a refusal fails the step with
  *  Postgres's SQLSTATE, unless the step names that SQLSTATE as its `refused` (the application's own catch of it), when
- *  its answer is `{ code, message }`. */
-export type SqlStep = { name: string; sql: string; values?: Array<string | null>; refused?: string; expect?: JourneyStep['expect']; capture?: JourneyStep['capture'] };
+ *  its answer is `{ code, message }`. `timeOrder` names native timestamp columns whose rows must be nondecreasing. */
+export type SqlStep = { name: string; sql: string; values?: Array<string | null>; refused?: string; timeOrder?: string[]; expect?: JourneyStep['expect']; capture?: JourneyStep['capture'] };
 export const isSql = (step: Step): step is SqlStep => 'sql' in step;
 
 /** A wait's length in milliseconds; an unreadable one throws. */
@@ -534,13 +534,15 @@ export function fill(template: string, captured: Captured, graphqlDocument = fal
   // `{encoded:name}` is a capture percent-encoded (encodeURIComponent), as a URL's path segment or query value or a form
   // field carries it: a URN a vendor's page prints encoded in its paths (LinkedIn's /rest/posts/urn%3Ali%3Ashare%3A…), a
   // sign-in's return path holding `&` that a browser posts back
-  const encoded = basic.replace(/\{encoded:([A-Za-z0-9_.]+)\}/g, (_, k: string) => {
+  const encoded = basic.replace(/\{(encoded|base64|md5):([A-Za-z0-9_.]+)\}/g, (_, transform: string, k: string) => {
     if (captured[k] === undefined) throw new Error(`journey: step refers to {${k}}, which no earlier step captured`);
+    if (transform === 'base64') return Buffer.from(captured[k]!, 'utf8').toString('base64');
+    if (transform === 'md5') return digest('md5', captured[k]!, 'base64');
     return encodeURIComponent(captured[k]!);
   });
   const selections = new Set<number>();
   // Only request query documents opt into GraphQL grammar (their selection sets, found in the text the captures fill,
-  // which `{basic:…}` and `{encoded:…}` have already filled); variables remain ordinary captured JSON data.
+  // which the credential and byte transforms have already filled); variables remain ordinary captured JSON data.
   if (graphqlDocument && !Object.hasOwn(captured, /^\{([A-Za-z_][A-Za-z0-9_.]*)\}$/.exec(encoded)?.[1] ?? '')) {
     try { visit(parse(encoded), { SelectionSet(node) { if (node.loc) selections.add(node.loc.start); } }); } catch { /* invalid documents still reach the vendor's own syntax error */ }
   }

package/src/walk/replay.ts

@@ -0,0 +1,103 @@
+/** A received step, with verified native engine provenance outside its unchanged response body. */
+export type ReplayStep = {
+  name: string; path: string; status: number; body: unknown;
+  sql?: { clock: 'wall'; timeFields: Record<string, 1114 | 1184> };
+  connection?: { kind: 'postgres'; field: string; authenticated: true };
+};
+
+/** PostgreSQL's ISO timestamp text, retaining its six fractional digits and timezone offset. */
+function timestamp(value: unknown, oid: 1114 | 1184): bigint | undefined {
+  if (value === 'infinity') return 10n ** 30n;
+  if (value === '-infinity') return -(10n ** 30n);
+  if (typeof value !== 'string') return undefined;
+  const m = /^(\d{4,6})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,6}))?(Z|([+-])(\d{2})(?::?(\d{2}))?(?::?(\d{2}))?)?( BC)?$/.exec(value);
+  if (!m || (oid === 1184) !== (m[8] !== undefined)) return undefined;
+  const [year, month, day, hour, minute, second] = m.slice(1, 7).map(Number);
+  if (!year || !month || month > 12 || !day || hour! > 23 || minute! > 59 || second! > 59) return undefined;
+  const astronomicalYear = m[13] ? 1 - year : year;
+  const leap = astronomicalYear % 4 === 0 && (astronomicalYear % 100 !== 0 || astronomicalYear % 400 === 0);
+  if (day > [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][month - 1]!) return undefined;
+  const zoneHour = Number(m[10] ?? 0), zoneMinute = Number(m[11] ?? 0), zoneSecond = Number(m[12] ?? 0);
+  if (zoneHour > 15 || zoneMinute > 59 || zoneSecond > 59) return undefined;
+  const adjustedYear = astronomicalYear - (month <= 2 ? 1 : 0);
+  const era = Math.floor(adjustedYear / 400), yearOfEra = adjustedYear - era * 400;
+  const dayOfYear = Math.floor((153 * (month + (month > 2 ? -3 : 9)) + 2) / 5) + day - 1;
+  const days = era * 146097 + yearOfEra * 365 + Math.floor(yearOfEra / 4) - Math.floor(yearOfEra / 100) + dayOfYear - 719468;
+  const offset = (zoneHour * 3600 + zoneMinute * 60 + zoneSecond) * (m[9] === '-' ? -1 : 1);
+  return BigInt(days) * 86_400_000_000n + BigInt(hour! * 3600 + minute! * 60 + second! - offset) * 1_000_000n + BigInt((m[7] ?? '').padEnd(6, '0'));
+}
+
+/** A journey's declared native timestamp columns are present, valid and nondecreasing down the returned rows. */
+export function timestampOrderFailures(body: unknown, timeFields: Record<string, 1114 | 1184>, columns: string[]): string[] {
+  if (!columns.length) return [];
+  const rows = body && typeof body === 'object' ? (body as { rows?: unknown }).rows : undefined;
+  if (!Array.isArray(rows)) return ['timestamp ordering requires SQL result rows'];
+  const failures: string[] = [];
+  for (const column of columns) {
+    const oid = timeFields[column];
+    if (oid !== 1114 && oid !== 1184) { failures.push(`${column} is not a native timestamp column`); continue; }
+    let previous: bigint | undefined;
+    for (const [i, row] of rows.entries()) {
+      const at = timestamp(row && typeof row === 'object' ? (row as Record<string, unknown>)[column] : undefined, oid);
+      if (at === undefined) failures.push(`rows.${i}.${column} is not a non-null native timestamp`);
+      else if (previous !== undefined && at < previous) failures.push(`rows.${i}.${column} precedes the previous row's timestamp`);
+      previous = at;
+    }
+  }
+  return failures;
+}
+
+/** Compare received traces by their selected engine declarations, retaining every undeclared byte. */
+export function compareReplays(first: ReadonlyArray<ReplayStep>, second: ReadonlyArray<ReplayStep>): { equal: boolean; diagnostics: string[] } {
+  const diagnostics: string[] = [];
+  if (!first.length) diagnostics.push('the journey answered no steps');
+  if (first.length !== second.length) diagnostics.push(`the replays answered ${first.length} and ${second.length} steps`);
+  const times = new Map<1114 | 1184, Array<{ first: bigint; second: bigint; where: string }>>();
+  for (let i = 0; i < Math.min(first.length, second.length); i++) {
+    const a = first[i]!, b = second[i]!;
+    const where = `step ${i + 1} (${a.name})`;
+    if (a.name !== b.name || a.path !== b.path || a.status !== b.status || JSON.stringify(a.sql) !== JSON.stringify(b.sql) || JSON.stringify(a.connection) !== JSON.stringify(b.connection)) {
+      diagnostics.push(`${where}: request, status or native engine provenance differs`);
+      continue;
+    }
+    const fields = a.sql?.clock === 'wall' ? a.sql.timeFields : {};
+    const same = (left: unknown, right: unknown, path: string[]): boolean => {
+      if (path.length === 1 && a.connection?.kind === 'postgres' && a.connection.authenticated === true && path[0] === a.connection.field)
+        return typeof left === 'string' && typeof right === 'string';
+      const oid = path.length === 3 && path[0] === 'rows' ? fields[path[2]!] : undefined;
+      if ((oid === 1114 || oid === 1184) && left !== null && right !== null) {
+        const x = timestamp(left, oid), y = timestamp(right, oid);
+        if (x === undefined || y === undefined) { diagnostics.push(`${where}: ${path.join('.')} is not a valid native timestamp`); return false; }
+        const values = times.get(oid) ?? [];
+        values.push({ first: x, second: y, where: `${where}: ${path.join('.')}` });
+        times.set(oid, values);
+        return true;
+      }
+      if (Array.isArray(left) || Array.isArray(right)) return Array.isArray(left) && Array.isArray(right) && left.length === right.length && left.every((value, n) => same(value, right[n], [...path, String(n)]));
+      if (left && right && typeof left === 'object' && typeof right === 'object') {
+        const keys = Object.keys(left);
+        return JSON.stringify(keys) === JSON.stringify(Object.keys(right)) && keys.every(key => same((left as Record<string, unknown>)[key], (right as Record<string, unknown>)[key], [...path, key]));
+      }
+      return JSON.stringify(left) === JSON.stringify(right);
+    };
+    if (!same(a.body, b.body, [])) diagnostics.push(`${where}: response bytes differ`);
+  }
+  // Equal timestamps form one group: ties may split or coincide, while a strict order cannot reverse.
+  for (const values of times.values()) {
+    values.sort((a, b) => a.first < b.first ? -1 : a.first > b.first ? 1 : 0);
+    let greatest: { value: bigint; where: string } | undefined;
+    for (let i = 0; i < values.length;) {
+      let end = i + 1;
+      while (end < values.length && values[end]!.first === values[i]!.first) end++;
+      let next = greatest;
+      for (let n = i; n < end; n++) {
+        const at = values[n]!;
+        if (greatest && at.second < greatest.value) diagnostics.push(`${at.where}: timestamp order reverses relative to ${greatest.where}`);
+        if (!next || at.second > next.value) next = { value: at.second, where: at.where };
+      }
+      greatest = next;
+      i = end;
+    }
+  }
+  return { equal: diagnostics.length === 0, diagnostics };
+}