@volter/world-core 3.0.155 → 3.0.156

package/src/mongodb/stream.ts

@@ -0,0 +1,213 @@
+// THE KERNEL'S MONGODB WIRE (`@volter/world-core/mongodb`): MongoDB's client/server message framing as a TwinStream,
+// so an unmodified MongoDB driver connects to a twin as to a server. The kernel holds the framing, which every
+// MongoDB-speaking vendor shares; the pack holds what a command means (it answers each command document from its own
+// engine over the tree) and the document codec (BSON, a library the pack installs). The messages served are what the
+// vendor's own driver sends, by citation (docs/contributing/architecture.md, "Other wires", the MongoDB wire): OP_MSG
+// with a body section and document-sequence sections, and the legacy OP_QUERY a driver's first hello may use, answered
+// with OP_REPLY. Any other message ends the connection.
+// source: archive:https://registry.npmjs.org/mongodb/-/mongodb-6.20.0.tgz#sha256=a12bd599cb4f39cdf222b32380ea3efd1f60c6d88615e0700632a303a9006b02!/package/src/cmap/commands.ts "moreToCome is an OP_MSG only concept"
+import type { TwinStream, TwinStreamSink } from '../twin-fetch.ts';
+
+/** A command or reply document as the pack's codec gives it: the pack decides how BSON's types appear in it. */
+export type MongoDocument = Record<string, unknown>;
+/** BSON, as the pack supplies it (the `bson` library, installed by the pack's package.json). The kernel frames
+ *  documents and never reads inside one but for a command's `$db`. */
+export type MongoCodec = { serialize(document: MongoDocument): Uint8Array; deserialize(bytes: Uint8Array): MongoDocument };
+
+/** What the sender's message said beside its command: whether it will take more than one reply to it (OP_MSG's exhaustAllowed). */
+export type MongoAsked = { exhaustAllowed: boolean };
+const TIMED = Symbol('mongodb timed reply');
+/**
+ * An answer the wire sends LATER, or goes on sending: the vendor's protocol does this in real time (an awaitable
+ * `hello` is held for the sender's maxAwaitTimeMS, and one sent with exhaustAllowed is answered again every such
+ * interval, each reply marked moreToCome). The pack says WHAT and WHEN as data and holds no clock: the same request
+ * gets the same answer every time. The wire does the holding and the repeating on the wall clock, and the delay is
+ * never read back into state (architecture, "The MongoDB wire").
+ */
+export type MongoTimedReply = { readonly [TIMED]: true; reply: MongoDocument; afterMs: number; everyMs?: number };
+/**
+ * `reply` after `afterMs` (0: at once); with `everyMs`, again each `everyMs` until the peer closes, where the sender
+ * allowed more than one reply. The two differ in the vendor's own answers: a hello naming the server's current
+ * topology is first answered after the wait, one naming another process at once, and both then repeat at the wait.
+ */
+export const timedReply = (reply: MongoDocument, afterMs: number, everyMs?: number): MongoTimedReply => ({ [TIMED]: true, reply, afterMs, ...(everyMs !== undefined ? { everyMs } : {}) });
+const isTimed = (answer: MongoDocument | MongoTimedReply): answer is MongoTimedReply => (answer as MongoTimedReply)[TIMED] === true;
+
+export type MongoStreamOptions<Session> = {
+  codec: MongoCodec;
+  /** A connection opened: what the pack keeps for it (its sessions, an open transaction); never the tree. */
+  open(peer: string): Session;
+  /** Answer ONE command document sent to `database`. A document sequence (an insert's `documents`, an update's
+   *  `updates`) arrives as that field holding the array. The answer is the reply document, `ok: 1` or the vendor's
+   *  error document; a command the pack does not serve is the pack's to answer as the vendor answers an unknown one. */
+  command(session: Session, database: string, command: MongoDocument, asked: MongoAsked): Promise<MongoDocument | MongoTimedReply>;
+  /** The connection went away: abort what it left open. */
+  close?(session: Session): void;
+  /** Told why a connection is ended by this stream and not by its peer: the pack threw answering a command, or a
+   *  message was one this wire does not serve. Ending the connection is all the wire can say to the client, so the
+   *  reason is said here; unset, it is written to the process's error output. A stream that ends connections in
+   *  silence leaves its operator with a client that hangs and nothing to read. */
+  ended?(reason: string, peer: string): void;
+  /** The largest message accepted; the vendor's server names 48,000,000 in its hello (`maxMessageSizeBytes`). */
+  maxMessageSizeBytes?: number;
+};
+
+export const MongoOp = { REPLY: 1, QUERY: 2004, MSG: 2013 } as const;
+/** OP_MSG's flag bits. */
+export const MongoMsgFlag = { CHECKSUM_PRESENT: 1 << 0, MORE_TO_COME: 1 << 1, EXHAUST_ALLOWED: 1 << 16 } as const;
+const HEADER = 16;
+const dec = new TextDecoder();
+const enc = new TextEncoder();
+
+const view = (bytes: Uint8Array): DataView => new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+function concat(parts: Uint8Array[]): Uint8Array {
+  const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0)); let at = 0;
+  for (const p of parts) { out.set(p, at); at += p.length; }
+  return out;
+}
+function header(length: number, requestId: number, responseTo: number, opcode: number): Uint8Array {
+  const out = new Uint8Array(HEADER), v = view(out);
+  v.setInt32(0, length, true); v.setInt32(4, requestId, true); v.setInt32(8, responseTo, true); v.setInt32(12, opcode, true);
+  return out;
+}
+
+/** An OP_MSG message carrying one document as its body section. */
+export function encodeMsg(codec: MongoCodec, document: MongoDocument, requestId: number, responseTo: number, flags = 0): Uint8Array {
+  const body = codec.serialize(document), bits = new Uint8Array(5);
+  view(bits).setUint32(0, flags, true);
+  return concat([header(HEADER + 4 + 1 + body.length, requestId, responseTo, MongoOp.MSG), bits, body]);
+}
+/** The command of an OP_MSG message: its body section, with each document sequence as the field it names. */
+export function decodeMsg(codec: MongoCodec, message: Uint8Array): { flags: number; document: MongoDocument } {
+  const v = view(message), flags = v.getUint32(HEADER, true), end = message.length - (flags & MongoMsgFlag.CHECKSUM_PRESENT ? 4 : 0);
+  let document: MongoDocument | undefined; const sequences: Record<string, MongoDocument[]> = {};
+  for (let at = HEADER + 4; at < end;) {
+    const kind = message[at]!; at += 1;
+    if (kind === 0) {
+      const size = v.getInt32(at, true);
+      if (size < 5 || at + size > end) throw new Error('OP_MSG body section runs past the message');
+      document = codec.deserialize(message.subarray(at, at + size)); at += size;
+    } else if (kind === 1) {
+      const size = v.getInt32(at, true), stop = at + size;
+      if (size < 4 || stop > end) throw new Error('OP_MSG document sequence runs past the message');
+      const nameEnd = message.indexOf(0, at + 4);
+      if (nameEnd < 0 || nameEnd >= stop) throw new Error('OP_MSG document sequence names no field');
+      const name = dec.decode(message.subarray(at + 4, nameEnd)), documents: MongoDocument[] = [];
+      for (let cursor = nameEnd + 1; cursor < stop;) {
+        const length = v.getInt32(cursor, true);
+        if (length < 5 || cursor + length > stop) throw new Error('OP_MSG document sequence holds a document that runs past it');
+        documents.push(codec.deserialize(message.subarray(cursor, cursor + length))); cursor += length;
+      }
+      sequences[name] = documents; at = stop;
+    } else throw new Error(`OP_MSG section kind ${kind} is unknown`);
+  }
+  if (!document) throw new Error('OP_MSG carries no body section');
+  return { flags, document: { ...document, ...sequences } };
+}
+
+/** MongoDB's wire, served: each command document answered by the pack, in arrival order, on one connection. */
+export function createMongoStream<Session>(options: MongoStreamOptions<Session>): TwinStream {
+  const limit = options.maxMessageSizeBytes ?? 48_000_000;
+  let nextResponse = 0;
+  return (sink: TwinStreamSink, peer: string) => {
+    const session = options.open(peer);
+    let buffer: Uint8Array = new Uint8Array(0);
+    let closed = false;
+    let chain: Promise<void> = Promise.resolve();
+    // A reply being held or repeated (a timed reply): its timer, and the messages that arrived meanwhile. A
+    // connection serves one command at a time, so nothing else is answered until a held reply is sent, and nothing
+    // at all while a repeating one runs (recording:hello-awaitable-exhaust-then-command.json: a ping sent into the
+    // stream has no reply). The timer is not work: `settled` does not wait for it, it does not keep the process
+    // alive (unref), and a close from either side clears it at once.
+    let held: ReturnType<typeof setTimeout> | undefined;
+    const parked: Uint8Array[] = [];
+    const release = (): void => { if (held !== undefined) { clearTimeout(held); held = undefined; } parked.length = 0; };
+    const hold = (ms: number, then: () => void): void => {
+      held = setTimeout(() => { held = undefined; if (!closed) then(); }, Math.max(0, ms));
+      (held as { unref?: () => void }).unref?.();
+    };
+    const end = (reason: string): void => {
+      if (closed) return;
+      closed = true; release();
+      (options.ended ?? ((why, from) => console.error(`mongodb wire: ended the connection from ${from}: ${why}`)))(reason, peer);
+      options.close?.(session); sink.end();
+    };
+
+    const answer = async (message: Uint8Array): Promise<void> => {
+      const v = view(message), requestId = v.getInt32(4, true), opcode = v.getInt32(12, true);
+      if (opcode === MongoOp.MSG) {
+        const { flags, document } = decodeMsg(options.codec, message);
+        const database = typeof document.$db === 'string' ? document.$db : 'admin';
+        const exhaustAllowed = (flags & MongoMsgFlag.EXHAUST_ALLOWED) !== 0;
+        const answered = await options.command(session, database, document, { exhaustAllowed });
+        // the sender waits for no reply to a message it marked moreToCome
+        if (flags & MongoMsgFlag.MORE_TO_COME) return;
+        if (!isTimed(answered)) { sink.write(encodeMsg(options.codec, answered, ++nextResponse, requestId)); return; }
+        if (answered.everyMs === undefined || !exhaustAllowed) {
+          // held once: one reply after the wait, with no flag; then what arrived meanwhile is answered in order
+          // source: recording:hello-awaitable-no-exhaust.json "received"
+          hold(answered.afterMs, () => {
+            sink.write(encodeMsg(options.codec, answered.reply, ++nextResponse, requestId));
+            for (const message of parked.splice(0)) take(message);
+          });
+          return;
+        }
+        // repeated: each reply marked moreToCome; the first answers the request, each later one answers this
+        // wire's own previous reply, until the peer closes
+        // source: recording:hello-awaitable-exhaust.json "received"
+        let answers = requestId;
+        const again = (): void => {
+          const id = ++nextResponse;
+          sink.write(encodeMsg(options.codec, answered.reply, id, answers, MongoMsgFlag.MORE_TO_COME));
+          answers = id;
+          hold(answered.everyMs!, again);
+        };
+        // source: recording:hello-stale-process-exhaust.json "received" (the first at once, the rest at the wait)
+        hold(answered.afterMs, again);
+        return;
+      }
+      if (opcode === MongoOp.QUERY) {
+        // the legacy form a driver's first hello may take: a query on `<database>.$cmd` whose document is the command
+        const nameEnd = message.indexOf(0, HEADER + 4);
+        if (nameEnd < 0) throw new Error('OP_QUERY names no collection');
+        const namespace = dec.decode(message.subarray(HEADER + 4, nameEnd));
+        if (!namespace.endsWith('.$cmd')) throw new Error(`OP_QUERY on ${namespace}: only a command on <database>.$cmd is served`);
+        const command = options.codec.deserialize(message.subarray(nameEnd + 1 + 8));
+        // the legacy form has no flags of OP_MSG's: nothing is held or repeated for it
+        const legacy = await options.command(session, namespace.slice(0, -'.$cmd'.length), command, { exhaustAllowed: false });
+        const reply = options.codec.serialize(isTimed(legacy) ? legacy.reply : legacy);
+        // OP_REPLY: responseFlags (AwaitCapable), cursorID 0, startingFrom 0, numberReturned 1, then the document
+        const fixed = new Uint8Array(20), f = view(fixed);
+        f.setInt32(0, 8, true); f.setInt32(16, 1, true);
+        sink.write(concat([header(HEADER + 20 + reply.length, ++nextResponse, requestId, MongoOp.REPLY), fixed, reply]));
+        return;
+      }
+      throw new Error(`opcode ${opcode} is not served`);
+    };
+
+    // One message, in arrival order: parked while a reply is held, answered otherwise.
+    const take = (message: Uint8Array): void => {
+      chain = chain.then(() => (closed ? undefined : held !== undefined ? void parked.push(message) : answer(message))).catch((error) => end(error instanceof Error ? error.message : String(error)));
+    };
+    return {
+      data(chunk: Uint8Array): void {
+        if (closed) return;
+        buffer = buffer.length ? concat([buffer, chunk]) : chunk;
+        while (buffer.length >= 4) {
+          const length = view(buffer).getInt32(0, true);
+          // where the citation stops: what the vendor's server does with a malformed or oversized message is not
+          // read; this ends the connection, which no driver's well-formed message reaches
+          if (length < HEADER || length > limit) { end(`a message of ${length} bytes (accepted: ${HEADER} to ${limit})`); return; }
+          if (buffer.length < length) break;
+          const message = buffer.subarray(0, length); buffer = buffer.subarray(length);
+          take(message.slice());
+        }
+      },
+      settled: () => chain,
+      close: () => { if (!closed) { closed = true; release(); options.close?.(session); } },
+    };
+  };
+}
+
+/** The bytes of a C string, for a client's OP_QUERY. */
+export const cstring = (text: string): Uint8Array => concat([enc.encode(text), new Uint8Array(1)]);

package/src/observe.ts

@@ -145,5 +145,5 @@ function foldResources(service: string, resources: ObservedResource[], opts: { r
     }
   }
   return report;
-  });
+  }, { operation: 'observation batch' });
 }

package/src/pack-fetch.ts

@@ -1,4 +1,5 @@
 import { parseExactJson } from './exact-json.ts';
+import { redisServiceTimes } from './redis/service-times.ts';
 import { readHostPort } from './host-port.ts';
 // A Protocol 3 pack's whole HTTP surface, served by the kernel from the pack's declarations (docs/contributing/
 // architecture.md, "The pack contract: the kernel's root"): the discovery door (`GET /twin`, the manifest's
@@ -431,6 +432,7 @@ export function createPackFetch(parts: PackParts, options: PackFetchOptions = {}
     return readable instanceof Response ? readable : readable ? atContentHost(readable) : undecodable(arrived);
   };
   const fetch = async (arrived: Request): Promise<Response> => {
+    redisServiceTimes.lap('1 request made by the stream');
     // the discovery door is the pack's own, whichever lane would take its path
     const asked = new URL(arrived.url).pathname.replace(/\/+$/, '') || '/';
     const discovery = arrived.method === 'GET' && m.discovery !== undefined && (asked === '/twin' || asked === '/twin/scenario');
@@ -460,6 +462,7 @@ export function createPackFetch(parts: PackParts, options: PackFetchOptions = {}
     const url = new URL(request.url);
     const path = url.pathname.replace(/\/+$/, '') || '/';
     const readOnly = (options.readOnly ?? false) || isReadOnlyRequest(request);
+    redisServiceTimes.lap('2 content and lane routes, decoding, routing');
     if (request.method === 'GET' && path === '/twin' && m.discovery) {
       const d = m.discovery;
       const discoveryDoors = [...(m.doors ?? []), ...(m.machines ? [{ id: 'machine-pool', method: 'POST', path: '/_twin/machine-pool', note: 'Enroll the World execution pool: { kind: light | docker | none | fake, images?: { image: { context, command?, runtime? } } }.' }] : [])];
@@ -562,12 +565,19 @@ export function createPackFetch(parts: PackParts, options: PackFetchOptions = {}
     if (!parts.around) return encoded(m, request, await decided(request));
     // The front reads a context before forwarding (possibly with new headers). Materialize a transport body before
     // those independent reads, rather than chaining tees of the incoming stream through the front and the scenario.
+    redisServiceTimes.lap('3 doors, screens and machine routes passed');
     const bytes = request.method === 'GET' || request.method === 'HEAD' ? undefined : await request.arrayBuffer();
     const buffered = new Request(request.url, { method: request.method, headers: request.headers, signal: request.signal, ...(bytes ? { body: bytes } : {}) });
     const matched = matchOperation(routes, buffered.method, url.pathname, url.searchParams, buffered.headers);
     const operation = matched?.operation ?? ({ id: 'unmatched', method: buffered.method, path: url.pathname, class: buffered.method === 'GET' ? 'retrieve' : 'action' } as DerivedOperation);
+    redisServiceTimes.lap('4 body buffered, operation matched');
     const ctx = await contextFor(m, { request: buffered.clone(), operation, params: matched?.params ?? pathParams(routes, url.pathname) }, scope);
-    return encoded(m, buffered, await parts.around(ctx, (rewritten) => decided(rewritten ?? buffered)));
+    redisServiceTimes.lap('5 the around context opened');
+    const answered = await parts.around(ctx, (rewritten) => decided(rewritten ?? buffered));
+    redisServiceTimes.lap('6d the handler, and the around after it');
+    const sent = await encoded(m, buffered, answered);
+    redisServiceTimes.lap('7 answer encoded');
+    return sent;
   };
   // the request's id is the correlation of every entry it appends (runtime contract D3), minted here, always: it is the
   // group a perform sends as one call (the real-system adapters, "Perform"). Under `auto` the group is performed once
@@ -743,8 +753,16 @@ export function createVendorFetch(manifest: VendorManifest, lanes: Record<string
   const ingest = Object.values(lanes).find((l) => l.fetch.ingest)?.fetch.ingest;
   const lane = laneRouter(manifest.vendor, manifest.lanes, lanes);
   return Object.assign(async (request: Request): Promise<Response> => {
-    for (const part of Object.values(lanes)) await (part.fetch as typeof part.fetch & { advanceClock?: (request: Request) => Promise<void> }).advanceClock?.(request);
     const path = new URL(request.url).pathname.replace(/\/+$/, '') || '/';
+    // the discovery door is answered before any lane's clock moves, as a pack's own is (createPackFetch): reading what
+    // the twin is moves no vendor clock
+    if (request.method === 'GET' && path === '/twin' && manifest.discovery) {
+      // the vendor's screens are its lanes' (a dashboard drawn by one lane is the vendor's)
+      const screens = lanesWorkspaces(lanes);
+      const laneDiscovery = await laneDoors(lanes, request, '/twin');
+      return Response.json({ ...statefulTwinManifest({ vendor: manifest.vendor, ...manifest.discovery }), ...(screens.length ? { screens } : {}), ...(Object.keys(laneDiscovery).length ? { lanes: laneDiscovery } : {}) });
+    }
+    for (const part of Object.values(lanes)) await (part.fetch as typeof part.fetch & { advanceClock?: (request: Request) => Promise<void> }).advanceClock?.(request);
     // the vendor's board is the frames of the lanes that lay one out, each drawn when any lane's screen takes its url
     if (request.method === 'GET' && path === '/twin/board') {
       const frames: BoardFrame[] = [];
@@ -758,12 +776,6 @@ export function createVendorFetch(manifest: VendorManifest, lanes: Record<string
       const draws = (url: string): boolean => Object.values(lanes).some((l) => l.fetch.draws?.(url) ?? false);
       return Response.json({ frames: frames.map((f) => ({ ...f, drawn: draws(f.url) })) });
     }
-    if (request.method === 'GET' && path === '/twin' && manifest.discovery) {
-      // the vendor's screens are its lanes' (a dashboard drawn by one lane is the vendor's)
-      const screens = lanesWorkspaces(lanes);
-      const laneDiscovery = await laneDoors(lanes, request, '/twin');
-      return Response.json({ ...statefulTwinManifest({ vendor: manifest.vendor, ...manifest.discovery }), ...(screens.length ? { screens } : {}), ...(Object.keys(laneDiscovery).length ? { lanes: laneDiscovery } : {}) });
-    }
     if (request.method === 'GET' && path === '/twin/scenario') {
       const laneScenarios = await laneDoors(lanes, request, '/twin/scenario');
       if (Object.keys(laneScenarios).length) return Response.json({ vendor: manifest.vendor, lanes: laneScenarios });

package/src/packRegistry.ts

@@ -15,6 +15,8 @@ import type { DerivedManifest, VendorManifest } from './derived-core.ts';
 import type { DerivedSurface } from './derived.ts';
 import type { TwinEmitter } from './emit.ts';
 import { registerAuthStrategy, registerStateSystem, type StateSystemAdapters } from './state-system.ts';
+import { PROTOCOL_VERSION, protocolStanding } from './protocol.ts';
+export { PROTOCOL_VERSION, PROTOCOL_MAJOR, protocolStanding } from './protocol.ts';
 
 /**
  * How a pack's clients ADDRESS it. The first three are HTTP API styles; `raw-tcp` is the
@@ -49,20 +51,6 @@ export type PackTransport = 'rest' | 'graphql' | 'web-api' | 'raw-tcp';
  *                       seam (fly's runtime, github's git plane, supabase's external stack).
  *   'proxy'           — the vendor itself is a forwarding plane (tunnel).
  */
-/** THE PLATFORM PROTOCOL VERSION (runtime contract R16): `major.minor`. The major names the
- *  descriptor shape, the factory signature, the doors and the adapter contract a package is
- *  written against, and the build form it is made in (the derived pack); a change that removes or
- *  alters a required shape bumps it, an additive change bumps minor. A package declares the major
- *  it targets (`protocol`), and the kernel serves its own major and no other: a package that
- *  declares another, or none, is refused. */
-export const PROTOCOL_VERSION = '3.0';
-export const PROTOCOL_MAJOR = Number(PROTOCOL_VERSION.split('.')[0]);
-/** Where a package's declared protocol stands against this kernel: `current`, or `refused`. */
-export function protocolStanding(declared: string | undefined): { major: number; standing: 'current' | 'refused' } {
-  const major = declared === undefined ? Number.NaN : Number(String(declared).split('.')[0]);
-  return major === PROTOCOL_MAJOR ? { major, standing: 'current' } : { major, standing: 'refused' };
-}
-
 export const PACK_ARCHETYPES = ['crud', 'generative', 'signed-protocol', 'engine-control', 'proxy'] as const;
 export type PackArchetype = (typeof PACK_ARCHETYPES)[number];
 
@@ -237,7 +225,7 @@ export type TwinPack = {
   /** Another pack of this vendor issues the API's credentials; architecture, descriptor World wiring. */
   credentialIssuer?: string;
   prismaAdapter?: { adapter: string; export: string; urlEnv: string };
-  managedService?: { kind: 'redis' | 'mongodb'; scheme: string; note: string };
+  managedService?: { kind: string; scheme: string; note: string; configurationArg?: '--configuration'; originalInputs?: { version: 1; manifestArg: '--original-inputs'; rootArg: '--original-root'; digestArg: '--original-inputs-sha256'; paths: string[] } };
   managedDatabase?: { kind: 'postgres'; arg: string; note: string };
   sidePorts?: Array<{ name: string; option: string; flag: string; path?: string; scheme?: string; note: string }>;
   stateSystem?: StateSystemAdapters;

package/src/protocol.ts

@@ -0,0 +1,13 @@
+/** THE PLATFORM PROTOCOL VERSION (runtime contract R16): `major.minor`. The major names the
+ *  descriptor shape, the factory signature, the doors and the adapter contract a package is
+ *  written against, and the build form it is made in (the derived pack); a change that removes or
+ *  alters a required shape bumps it, an additive change bumps minor. A package declares the major
+ *  it targets (`protocol`), and the kernel serves its own major and no other: a package that
+ *  declares another, or none, is refused. */
+export const PROTOCOL_VERSION = '3.0';
+export const PROTOCOL_MAJOR = Number(PROTOCOL_VERSION.split('.')[0]);
+/** Where a package's declared protocol stands against this kernel: `current`, or `refused`. */
+export function protocolStanding(declared: string | undefined): { major: number; standing: 'current' | 'refused' } {
+  const major = declared === undefined ? Number.NaN : Number(String(declared).split('.')[0]);
+  return major === PROTOCOL_MAJOR ? { major, standing: 'current' } : { major, standing: 'refused' };
+}

package/src/redis/engine.ts

@@ -46,7 +46,8 @@
 import { createHash } from 'node:crypto';
 import { projectResources } from '../actions.ts';
 import { treeChangesSince, treeStamp } from '../log.ts';
-import { applyTwinWrite, type TwinResource } from '../serve.ts';
+import { redisServiceTimes as times } from './service-times.ts';
+import { applyTwinWrites, type TwinResource, type TwinWriteInput } from '../serve.ts';
 import { getActiveWorldStore } from '../world-store.ts';
 import { LUA_NULL, luaScriptSha1, LuaError, LuaTable, runLua, type LuaHost, type LuaValue } from './lua.ts';
 import { inlineCommand } from './frames.ts';
@@ -110,7 +111,7 @@ type StoredZSet = Array<[string, string]>;
 export type StreamEntries = Array<[string, HashPairs]>;
 
 /** A point-in-time copy of a run's whole mutable state, for script rollback. */
-type Snapshot = { rows: Map<string, KeyRow>; touched: Map<string, string>; scripts: Map<string, string>; touchedScripts: Map<string, string> };
+type Snapshot = { rows: OverlayState<KeyRow>; touched: Map<string, string>; scripts: OverlayState<string>; touchedScripts: Map<string, string> };
 
 export type KeyRow = {
   name: string;
@@ -399,29 +400,71 @@ function patchImage(image: KeyspaceImage, changed: readonly TwinResource[], remo
   return true;
 }
 
+const GONE: unique symbol = Symbol('gone');
+type OverlayState<V> = { own: Map<string, V | typeof GONE>; moved: Set<string> };
+/**
+ * A run's map over the shared image: what the run changed is its own, the rest is read through. Copying the whole image
+ * per run (and again per script, for its rollback) cost every command the size of the keyspace (S2 R19: Twenty's queue
+ * twin, its queue scripts waiting behind each other's copies). Iteration is a `new Map(base)` changed in place's: the
+ * base's order with its changed values, then the keys added, a base key deleted and set again among them at its set.
+ */
+class Overlay<V> {
+  private own = new Map<string, V | typeof GONE>();
+  private moved = new Set<string>();
+  constructor(private readonly base: ReadonlyMap<string, V>) {}
+  get(key: string): V | undefined {
+    const mine = this.own.get(key);
+    return mine === GONE ? undefined : mine !== undefined || this.own.has(key) ? mine as V : this.base.get(key);
+  }
+  has(key: string): boolean { return this.own.has(key) ? this.own.get(key) !== GONE : this.base.has(key); }
+  set(key: string, value: V): void {
+    if (this.own.get(key) === GONE) { this.own.delete(key); this.moved.add(key); }
+    this.own.set(key, value);
+  }
+  delete(key: string): void {
+    this.own.delete(key);
+    this.moved.delete(key);
+    if (this.base.has(key)) this.own.set(key, GONE);
+  }
+  *entries(): IterableIterator<[string, V]> {
+    for (const [key, value] of this.base) {
+      if (this.moved.has(key)) continue;
+      if (!this.own.has(key)) { yield [key, value]; continue; }
+      const mine = this.own.get(key)!;
+      if (mine !== GONE) yield [key, mine];
+    }
+    for (const [key, mine] of this.own) if (mine !== GONE && (!this.base.has(key) || this.moved.has(key))) yield [key, mine];
+  }
+  *keys(): IterableIterator<string> { for (const [key] of this.entries()) yield key; }
+  *values(): IterableIterator<V> { for (const [, value] of this.entries()) yield value; }
+  /** What the run changed, copied: a rollback point the size of the changes, not of the keyspace. */
+  save(): OverlayState<V> { return { own: new Map(this.own), moved: new Set(this.moved) }; }
+  load(state: OverlayState<V>): void { this.own = new Map(state.own); this.moved = new Set(state.moved); }
+}
+
 /**
  * A synchronous, in-memory image of the keyspace for the duration of ONE request.
  *
  * Seeded from the keyspace's image (the tree folded by `projectResources`, memoized per World state — the kernel IS
  * the source of truth); every mutation records the key name in `touched`, and the flush writes exactly those keys
- * back. This run's maps are its own copies: nothing a run changes reaches the image or a later request except
+ * back. This run owns its overlays: nothing a run changes reaches the image or a later request except
  * through the tree.
  */
 export class KeySpace {
-  private readonly rows: Map<string, KeyRow>;
+  private readonly rows: Overlay<KeyRow>;
   private readonly touched = new Map<string, string>(); // key → the operation label to record
-  private readonly scripts: Map<string, string>; // sha1 → script source (a SEPARATE namespace)
+  private readonly scripts: Overlay<string>; // sha1 → script source (a SEPARATE namespace)
   private readonly touchedScripts = new Map<string, string>();
   /** name → the last write ordinal seen, INCLUDING for keys currently deleted or expired. */
-  private readonly revs: Map<string, number>;
-  /** The keyspace as the tree held it when this run began: shared, never changed in place. */
+  private readonly revs: Overlay<number>;
+  /** The shared base maps are patched by the next image(); a KeySpace must finish reading its base before it yields. */
   readonly image: KeyspaceImage;
   constructor(readonly ctx: RedisContext, readonly nowMs: number) {
-    // the image is shared and never changed in place; this run's maps are its own
+    // The base maps are patched by the next image(); finish reading this run's base before yielding.
     const image = this.image = ctx.dialect.storage?.image(ctx) ?? keyspaceImageOf(ctx);
-    this.rows = new Map(image.rows);
-    this.scripts = new Map(image.scripts);
-    this.revs = new Map(image.revs);
+    this.rows = new Overlay(image.rows);
+    this.scripts = new Overlay(image.scripts);
+    this.revs = new Overlay(image.revs);
   }
   /**
    * Is this row expired AS OF NOW? Checked on every read, not merely when the snapshot was built.
@@ -470,15 +513,13 @@ export class KeySpace {
   }
   /** A point-in-time copy, so a failed script can be rolled back to exactly where it started. */
   snapshot(): Snapshot {
-    return { rows: new Map(this.rows), touched: new Map(this.touched), scripts: new Map(this.scripts), touchedScripts: new Map(this.touchedScripts) };
+    return { rows: this.rows.save(), touched: new Map(this.touched), scripts: this.scripts.save(), touchedScripts: new Map(this.touchedScripts) };
   }
   restore(s: Snapshot): void {
-    this.rows.clear();
-    for (const [k, v] of s.rows) this.rows.set(k, v);
+    this.rows.load(s.rows);
     this.touched.clear();
     for (const [k, v] of s.touched) this.touched.set(k, v);
-    this.scripts.clear();
-    for (const [k, v] of s.scripts) this.scripts.set(k, v);
+    this.scripts.load(s.scripts);
     this.touchedScripts.clear();
     for (const [k, v] of s.touchedScripts) this.touchedScripts.set(k, v);
   }
@@ -524,39 +565,34 @@ export async function restoreKeyspace(image: readonly KeyImage[], ctx: RedisCont
 }
 
 async function flushKeySpace(space: KeySpace): Promise<void> {
+  // a run's writes land together, in this order, as they landed one at a time (serve.ts applyTwinWrites)
+  const writes: TwinWriteInput[] = [];
   for (const { name, operation, row } of space.pendingWrites()) {
-    await applyTwinWrite(
-      space.ctx.service,
-      {
-        operation,
-        subjectType: 'key',
-        subjectId: `${scopeOf(space.ctx)}key:${name}`,
-        // The kernel MERGES fields, so a delete must write EVERY field back to its "nothing here"
-        // value — leaving `v`/`pexpire_at` behind would let a later recreate inherit a dead value
-        // or a dead TTL.
-        fields: row === null
-          ? { name, kind: 'string', v: '', pexpire_at: null, last_id: '', gone: true, _rev: space.nextRev(name) }
-          : { name, kind: row.kind, v: row.v as never, pexpire_at: row.pexpireAt, last_id: row.lastId ?? '', gone: false, _rev: space.nextRev(name) },
-        occurredAt: space.ctx.occurredAt,
-        actor: { kind: 'agent' },
-      },
-      space.ctx.root,
-    );
+    writes.push({
+      operation,
+      subjectType: 'key',
+      subjectId: `${scopeOf(space.ctx)}key:${name}`,
+      // The kernel MERGES fields, so a delete must write EVERY field back to its "nothing here"
+      // value — leaving `v`/`pexpire_at` behind would let a later recreate inherit a dead value
+      // or a dead TTL.
+      fields: row === null
+        ? { name, kind: 'string', v: '', pexpire_at: null, last_id: '', gone: true, _rev: space.nextRev(name) }
+        : { name, kind: row.kind, v: row.v as never, pexpire_at: row.pexpireAt, last_id: row.lastId ?? '', gone: false, _rev: space.nextRev(name) },
+      occurredAt: space.ctx.occurredAt,
+      actor: { kind: 'agent' },
+    });
   }
   for (const { sha, operation, body } of space.pendingScriptWrites()) {
-    await applyTwinWrite(
-      space.ctx.service,
-      {
-        operation,
-        subjectType: 'script',
-        subjectId: `${scopeOf(space.ctx)}script:${sha}`,
-        fields: body === null ? { sha, body: '', gone: true } : { sha, body, gone: false },
-        occurredAt: space.ctx.occurredAt,
-        actor: { kind: 'agent' },
-      },
-      space.ctx.root,
-    );
+    writes.push({
+      operation,
+      subjectType: 'script',
+      subjectId: `${scopeOf(space.ctx)}script:${sha}`,
+      fields: body === null ? { sha, body: '', gone: true } : { sha, body, gone: false },
+      occurredAt: space.ctx.occurredAt,
+      actor: { kind: 'agent' },
+    });
   }
+  await applyTwinWrites(space.ctx.service, writes, space.ctx.root);
 }
 
 // ── typed value accessors (each asserts the key's type first) ─────────────────────────────────
@@ -1608,21 +1644,28 @@ export type RunItem = { result: RedisValue } | { error: string };
  * an unavailable command or a bad arity — are QUEUE-time and DO discard the whole batch, but the
  * caller (`upstash-twin.ts`) rejects those before this function is ever reached.
  */
-export async function execRedisRun(commands: string[][], ctx: RedisContext): Promise<{ items: RunItem[]; wrote: boolean; instant: number }> {
+export async function execRedisRun(commands: string[][], ctx: RedisContext): Promise<{ items: RunItem[]; wrote: boolean; instant: number; touched: string[] }> {
   const nowMs = Date.parse(ctx.occurredAt);
   if (!Number.isFinite(nowMs)) throw new RedisCommandError(`ERR twin: unparseable occurredAt '${ctx.occurredAt}'`);
-  const space = new KeySpace(ctx, nowMs);
+  // The phases a served command's time is counted in (service-times.ts): the image, the commands, their writes.
+  const space = times.timed('image', () => new KeySpace(ctx, nowMs));
   const items: RunItem[] = [];
-  for (const argv of commands) {
-    try {
-      items.push({ result: execOne(space, argv) });
-    } catch (e) {
-      if (e instanceof ReadOnlyError) throw e; // surfaces as HTTP 405 for the whole request
-      if (e instanceof RedisCommandError) { items.push({ error: e.message }); continue; }
-      throw e;
+  const began = times.now();
+  try {
+    for (const argv of commands) {
+      try {
+        items.push({ result: execOne(space, argv) });
+      } catch (e) {
+        if (e instanceof ReadOnlyError) throw e; // surfaces as HTTP 405 for the whole request
+        if (e instanceof RedisCommandError) { items.push({ error: e.message }); continue; }
+        throw e;
+      }
     }
-  }
+  } finally { times.add('execute', times.now() - began); }
   const wrote = space.dirty;
-  await (space.ctx.dialect.storage?.flush(space) ?? flushKeySpace(space));
-  return { items, wrote, instant: nowMs };
+  // The keys this run wrote, by name, read before the flush clears them: a stream wakes only the clients blocked on one.
+  const touched = space.touchedNames();
+  const flushing = times.now();
+  try { await (space.ctx.dialect.storage?.flush(space) ?? flushKeySpace(space)); } finally { times.add('flush', times.now() - flushing); }
+  return { items, wrote, instant: nowMs, touched };
 }