@volter/twin-standard 1.0.148 → 1.0.149

package/src/spec-ir-commands.ts

@@ -48,3 +48,84 @@ export function fromRedisCommands(files: Record<string, RedisCommandFile>, versi
   commands.sort((a, b) => a.id.localeCompare(b.id));
   return { format: 'command-table', version, commands };
 }
+
+// ── A command language whose commands are DOCUMENTS ─────────────────────────────────────────────────────────────
+// A command of MongoDB's wire protocol is not words with an arity: it is a document whose first key names the
+// command and whose other keys are named fields, some of them lists of statements (an update's `updates`) with
+// fields of their own, and every command also carries the fields the layers under it add (the session's, the
+// connection's). The vendor publishes no table of these that a pack may derive from (its reference and its driver
+// specifications are licensed NonCommercial); its own client is Apache-2.0 and builds every one of them. So the
+// table is the client's statement: the client's syntax tree is read by a program (as Tinybird's Python client's is,
+// spec-ir-client.ts) and what it read is the pack's vendored document, `spec/command-builders.json`; this reads that
+// data into the command table. The program is kept beside that document in the pack's `spec/`, as the table's
+// provenance, and is never a kernel script (Creating a pack, step 2).
+//
+// What the table says and does not: which commands a client can send and, for each, every field, its declared
+// type, and the condition the client sends it under. It says nothing of what a server does with them. A field's
+// condition is recorded, never evaluated: `always`; `given` (the condition tests only the option itself);
+// `wire-version` (it depends on what the server's hello says); `other`, kept as the client wrote it.
+
+export type FieldCondition = { form: 'always' | 'given' | 'wire-version' | 'other'; text?: string };
+/** A field of a command document: where the client sets it and under what condition. `by` names a helper that sets
+ *  it; `from` the builder it was read in, where that is not the command's own (a base class, a subclass). */
+export type CommandField = { name: string; type: string; optional?: boolean; sentWhen: FieldCondition; source: { file: string; line: number }; by?: string; from?: string };
+/** A command that is a document. `statements`: for a field that is a list of statements, the fields of one. */
+export type DocumentCommandDef = { id: string; fields: CommandField[]; statements?: Record<string, CommandField[]> };
+export type DocumentCommandTableSpec = { format: 'command-table'; language: 'document'; version: string; source: { package: string; commit: string; licence: string }; commands: DocumentCommandDef[]; envelope: CommandField[] };
+
+type BuilderField = CommandField & { typeFrom?: string; value?: string; also?: unknown[]; spreadUnder?: string };
+type Builder = { builder: string; kind: 'command' | 'statement'; file: string; line: number; command: string | null; class?: string; extends?: string; addsTo?: string; fields: BuilderField[] };
+export type DriverCommandBuilders = {
+  format: 'driver-command-builders';
+  driver: { package: string; version: string; commit: string; licence: string };
+  builders: Builder[];
+  envelope: Array<{ layer: string; file: string; line: number; fields: BuilderField[] }>;
+};
+
+const fieldOf = (f: BuilderField, from?: string): CommandField => ({
+  name: f.name, type: f.type, ...(f.optional ? { optional: true } : {}), sentWhen: f.sentWhen, source: f.source,
+  ...(f.by ? { by: f.by } : {}), ...(from ? { from } : {}),
+});
+
+/** The command table of a client's command builders: each command with its own fields, the fields its subclasses
+ *  add, the fields a base class adds to the commands that descend from it, and the fields of its statements. A builder this cannot place is
+ *  an error naming it; a table with a silent hole is never returned. */
+export function fromDriverCommandBuilders(doc: DriverCommandBuilders): DocumentCommandTableSpec {
+  const named = doc.builders.filter((b) => b.kind === 'command' && b.command !== null);
+  const classOf = new Map(named.filter((b) => b.class).map((b) => [b.class!, b.command!]));
+  const commands = new Map<string, DocumentCommandDef>();
+  const add = (id: string, fields: BuilderField[], from?: string): void => {
+    const held = commands.get(id) ?? { id, fields: [] };
+    for (const f of fields) if (!held.fields.some((h) => h.name === f.name)) held.fields.push(fieldOf(f, from));
+    commands.set(id, held);
+  };
+  for (const b of named) add(b.command!, b.fields);
+  // what a subclass adds to its parent's document, and what the base of every command adds to all of them
+  const everyCommand: Builder[] = [];
+  for (const b of doc.builders.filter((b) => b.kind === 'command' && b.command === null)) {
+    if (b.addsTo === undefined) throw new Error(`command-builders: ${b.builder} (${b.file}:${b.line}) names no command and adds to none`);
+    const parent = classOf.get(b.addsTo);
+    if (parent !== undefined) add(parent, b.fields, b.builder);
+    else if (b.class === b.addsTo) everyCommand.push(b);
+    else throw new Error(`command-builders: ${b.builder} (${b.file}:${b.line}) adds to ${b.addsTo}, which builds no named command`);
+  }
+  // the base's fields go to the commands whose class descends from it, and to no other (a getMore is not one)
+  const parentOf = new Map(doc.builders.filter((b) => b.class && b.extends).map((b) => [b.class!, b.extends!]));
+  const descends = (cls: string | undefined, base: string): boolean => { for (let at = cls, hops = 0; at !== undefined && hops < 16; at = parentOf.get(at), hops += 1) if (at === base) return true; return false; };
+  for (const b of everyCommand) for (const n of named) if (descends(n.extends, b.class!)) add(n.command!, b.fields, b.builder);
+  // a statement builder's fields belong to the command of its file that carries the statement list
+  for (const b of doc.builders.filter((b) => b.kind === 'statement')) {
+    const owner = named.find((n) => n.file === b.file && n.fields.some((f) => f.value === 'this.statements'));
+    const list = owner?.fields.find((f) => f.value === 'this.statements');
+    if (!owner || !list) throw new Error(`command-builders: ${b.builder} (${b.file}:${b.line}) builds a statement and no command of its file carries a statement list`);
+    (commands.get(owner.command!)!.statements ??= {})[list.name] = b.fields.map((f) => fieldOf(f));
+  }
+  const envelope: CommandField[] = [];
+  for (const layer of doc.envelope) for (const f of layer.fields) if (!envelope.some((h) => h.name === f.name)) envelope.push(fieldOf(f, layer.layer));
+  if (commands.size === 0) throw new Error('command-builders: no command was read');
+  return {
+    format: 'command-table', language: 'document', version: doc.driver.version,
+    source: { package: doc.driver.package, commit: doc.driver.commit, licence: doc.driver.licence },
+    commands: [...commands.values()].sort((a, b) => a.id.localeCompare(b.id)), envelope,
+  };
+}

package/src/types.ts

@@ -29,6 +29,8 @@ export interface ScoreReport {
 
 export interface CriterionResult {
   pass: boolean;
+  /** An interrupted check is not a verdict about unchanged source. */
+  cacheable?: boolean;
   /** why it failed, or what it measured when it passed */
   reason: string;
 }

package/src/walk/behavior-journey.ts

@@ -9,7 +9,8 @@ import { attributionName } from '../verdict.ts';
 //
 // The caller names the pack repository first (usePacks).
 
-import { mysql, redis } from '@volter/world-core';
+import { mongodb, mysql, redis } from '@volter/world-core';
+import { createRequire } from 'node:module';
 import { gzipSync } from 'node:zlib';
 import { parse, visit } from 'graphql';
 import { existsSync, readdirSync, readFileSync } from 'node:fs';
@@ -17,7 +18,7 @@ import { join } from 'node:path';
 import { databaseSession, type DatabaseSession, getActivePackAssets, getActiveWorldStore, LocalPackAssets, MemoryBlobStore, MemoryWorldStore, setActiveBlobStore, setActivePackAssets, setActiveWorldStore, setDeliveryObserver, setInProcessTwins, type InProcessTwin, type TwinStream, type TwinStreamConnection } from '@volter/world-core/runtime';
 import { createHostTable, type HostFacts } from '@volter/world-core/host-routing';
 import packFactsDocument from '@volter/world-core/generated/pack-facts.json' with { type: 'json' };
-import { type Captured, type GitStep, type Journey, type JourneyStep, type LineStep, type SocketStep, type SqlStep, type RespStep, type MysqlStep, isMysql, isResp, at, captureAt, echoFailures, expandFills, expectFailures, fill, fillJson, formBody, isGit, isLine, isSql, isSocket, isWait, jwtAuthorization, oauth1Authorization, presignedPath, sigv4Headers, waitMs } from './journey-kit.ts';
+import { type Captured, type GitStep, type Journey, type JourneyStep, type LineStep, type SocketStep, type SqlStep, type RespStep, type MysqlStep, type MongoStep, isMongo, isMysql, isResp, at, captureAt, echoFailures, expandFills, expectFailures, fill, fillJson, formBody, isGit, isLine, isSql, isSocket, isWait, jwtAuthorization, oauth1Authorization, presignedPath, sigv4Headers, waitMs } from './journey-kit.ts';
 import { lanesOf, lineShape, shapeChecker, type LineConnection, type ShapeChecker } from './shape.ts';
 import { walkPackDirectory } from './pack-directory.ts';
 import { xmlMembers } from './xml-view.ts';
@@ -446,6 +447,61 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
     finally { opts.hooks?.after?.(step as never, link, answered, collect.slice(before)); }
   };
 
+  // A MongoDB step speaks the pack's MongoDB wire as an application's driver does (@volter/world-core/mongodb's client), on
+  // the same byte-stream adapter the owned TCP listener serves. The document codec is BSON, the library the pack
+  // installs: the walk loads it from the pack's own dependencies, so a step is encoded by the library the pack decodes with.
+  const mongoLinks = new Map<string, { client: ReturnType<typeof mongodb.mongoClient>; database: string; close: () => void }>();
+  type Bson = { serialize(document: Record<string, unknown>): Uint8Array; deserialize(bytes: Uint8Array, options: Record<string, unknown>): Record<string, unknown>; EJSON: { deserialize(value: unknown, options: { relaxed: boolean }): Record<string, unknown>; serialize(value: unknown, options: { relaxed: boolean }): Record<string, unknown> } };
+  let bson: Bson | undefined;
+  const bareNumber = (value: unknown, at = 'mongo'): string | undefined => {
+    if (typeof value === 'number') return at;
+    if (Array.isArray(value)) { for (const [i, item] of value.entries()) { const found = bareNumber(item, `${at}[${i}]`); if (found) return found; } return undefined; }
+    if (value && typeof value === 'object') for (const [key, item] of Object.entries(value)) { const found = bareNumber(item, `${at}.${key}`); if (found) return found; }
+    return undefined;
+  };
+  const mongoStep = async (step: MongoStep): Promise<void> => {
+    const before = collect.length; let answered: { status: number; body: unknown } = { status: 0, body: undefined };
+    const link = step.link ?? 'mongo';
+    opts.hooks?.before?.(step as never, link, step.mongo);
+    try {
+      if (!bson) {
+        const pack = walkPackDirectory(packsDir(), journey.vendor, dependenciesDir());
+        if (!pack) throw new Error(`no pack directory for ${journey.vendor} to load its BSON library from`);
+        bson = createRequire(join(pack, 'package.json'))('bson') as Bson;
+      }
+      const codec: mongodb.MongoCodec = { serialize: (document) => bson!.serialize(document), deserialize: (bytes) => bson!.deserialize(bytes, { promoteValues: false, promoteLongs: false, bsonRegExp: true }) };
+      let held = mongoLinks.get(link);
+      if (!held) {
+        if (!served?.stream) streams ??= await vendorStream(packsDir(), journey.vendor, dependenciesDir());
+        const factory = served?.stream ?? streams?.(root, {}); if (!factory) throw new Error(`no create<Name>TwinStream adapter enrolled for ${journey.vendor}`);
+        let out: number[] = [];
+        const conn = factory({ write: (b) => { out.push(...b); }, end: () => {} }, '127.0.0.1');
+        const exchange: mongodb.MongoExchange = async (bytes) => { conn.data(bytes); await conn.settled(); const got = Uint8Array.from(out); out = []; return got; };
+        held = { client: mongodb.mongoClient(exchange, codec), database: 'test', close: () => conn.close() };
+        mongoLinks.set(link, held);
+      }
+      if (step.database) held.database = fill(step.database, captured);
+      const written = fillJson(step.mongo, captured);
+      const bare = bareNumber(written);
+      if (bare) throw new Error(`${bare} is a bare JSON number: write its kind ({ "$numberInt": "…" }, $numberLong or $numberDouble)`);
+      const reply = await held.client.command(held.database, bson.EJSON.deserialize(written, { relaxed: false }));
+      const body = bson.EJSON.serialize(reply, { relaxed: false });
+      const ok = (body.ok as Record<string, unknown> | undefined), failed = !ok || Number(Object.values(ok)[0]) !== 1;
+      const code = failed ? Number(Object.values((body.code as Record<string, unknown> | undefined) ?? { none: Number.NaN })[0]) : undefined;
+      answered = { status: code ?? 0, body };
+      if (opts.verbose) console.log(`  ${String(code ?? 'OK').padEnd(5)} ${link.padEnd(6)} ${String(Object.keys(step.mongo)[0] ?? '').padEnd(20)} ${step.name}`);
+      if (step.refused !== undefined ? code !== step.refused : failed) {
+        collect.push({ step: step.name, beat: 'STATUS', detail: failed ? `MongoDB refused it: ${code} ${String(body.codeName ?? '')} ${String(body.errmsg ?? '')}${step.refused ? ` (wanted ${step.refused})` : ''}` : `MongoDB completed it, wanted ${step.refused}` });
+        return;
+      }
+      for (const [name, dot] of Object.entries(step.capture ?? {})) { const value = captureAt(body, dot); if (value === undefined || value === null) collect.push({ step: step.name, beat: 'CAPTURE', detail: `nothing at "${dot}" in the answer to capture as {${name}}` }); else remember(name, value); }
+      try { for (const detail of expectFailures(step.expect ?? [], body, captured)) collect.push({ step: step.name, beat: 'EXPECT', detail }); }
+      catch (e) { collect.push({ step: step.name, beat: 'CAPTURE', detail: (e as Error).message }); }
+      if (step.close) { mongoLinks.get(link)?.close(); mongoLinks.delete(link); }
+    } catch (e) { collect.push({ step: step.name, beat: 'MONGO', detail: (e as Error).message }); }
+    finally { opts.hooks?.after?.(step as never, link, answered, collect.slice(before)); }
+  };
+
   // A git step is a git client's clone or push on the pack's smart-HTTP plane (packages/twin-standard/src/walk/git-client.ts); its answer is
   // checked as a request's is (status, captures, expectations).
   const gitStep = async (step: GitStep): Promise<void> => {
@@ -679,6 +735,7 @@ export async function walk(journey: Journey, root: string, collect: Failure[], o
   try {
   for (const step of journey.steps) {
     stepNow = step.name;
+    if (isMongo(step)) { await mongoStep(step); continue; }
     if (isMysql(step)) { await mysqlStep(step); continue; }
     if (isResp(step)) { await respStep(step); continue; }
     if (isLine(step)) { await speak(step); continue; }

package/src/walk/journey-kit.ts

@@ -327,7 +327,7 @@ export type Journey = {
   steps: Step[];
 };
 
-export type Step = JourneyStep | WaitStep | LineStep | GitStep | SocketStep | SqlStep | RespStep | MysqlStep;
+export type Step = JourneyStep | WaitStep | LineStep | GitStep | SocketStep | SqlStep | RespStep | MysqlStep | MongoStep;
 
 /** A step on a MySQL wire (the kernel's @volter/world-core/mysql, a pack's `create<Name>TwinStream`), spoken as an
  *  application's driver speaks it. `signIn` opens a named connection (`link`, `mysql` by default) with a user and
@@ -342,6 +342,19 @@ export type MysqlStep = {
 };
 export const isMysql = (step: Step): step is MysqlStep => 'mysql' in step || 'signIn' in step;
 
+/** A step on a pack's MongoDB wire, sent as an application's driver sends it (`@volter/world-core/mongodb`'s client)
+ *  on a named `link`, opened by the first step that names it. `mongo` is ONE command document, written in
+ *  canonical extended JSON so that what a document holds is written as it is: an ObjectId as `{ "$oid": … }`, a date
+ *  as `{ "$date": … }`, and a number with its kind (`{ "$numberInt": "1" }`, `$numberLong`, `$numberDouble`); a bare
+ *  JSON number is refused, because JSON does not say which of three kinds it is. `database` is the database it is
+ *  sent to (the link's last one, else `test`). Its answer is the reply document in the same form. A reply whose
+ *  `ok` is 0 fails the step unless `refused` names its `code`. */
+export type MongoStep = {
+  name: string; mongo: Record<string, unknown>; database?: string; link?: string; close?: boolean;
+  refused?: number; capture?: JourneyStep['capture']; expect?: JourneyStep['expect']; example?: string; exampleDiffers?: Record<string, string>;
+};
+export const isMongo = (step: Step): step is MongoStep => 'mongo' in step;
+
 /** A typed request/reply stream step. `resp` is one command or a pipeline of commands, on a named connection.
  * Captures and expectations see `{ replies, last, error }`; binary arguments use `{ base64 }`.
  * `read` drains pushes or a blocking command's answer without sending bytes. */