@volter/twin-planetscale 0.1.0

package/src/planetscale-wire.ts

@@ -0,0 +1,274 @@
+// THE psdb WIRE — protojson encoding of the `psdb.v1alpha1.Database` messages, and the row packing
+// that is the single most-gotten-wrong part of this protocol.
+//
+// GROUNDED in the ACTUALLY-INSTALLED `@planetscale/database@1.20.1` source (npm pack), which is the
+// authoritative decoder: `dist/index.js`'s `decodeRow`/`parse`/`execute` and `dist/cast.js`. Every
+// encoding rule below exists because that client's decoder requires it.
+//
+// ── HOW A ROW IS PACKED (dist/index.js `decodeRow`) ───────────────────────────────────────────
+//   { lengths: ['3', '-1', '2'], values: base64('abc' + '' + 'de') }
+// `values` is ONE base64 blob holding every non-null column's bytes CONCATENATED with no
+// separators; `lengths` is the per-column byte width as a DECIMAL STRING (protojson renders int64
+// as a string), and `-1` means SQL NULL. The client walks `lengths`, slicing `atob(values)` by
+// offset. Three consequences a naive encoder gets wrong, all silent:
+//   • a length is the BYTE count, not the character count — a multi-byte UTF-8 value packed with
+//     its character count shifts every subsequent column's slice, corrupting the whole row;
+//   • the client slices with `String.prototype.substring` over the `atob` output, i.e. over
+//     LATIN-1 CHARACTER units, then re-decodes with TextDecoder — so the bytes must be laid out as
+//     one char per byte before base64, which is what `packBytes` does;
+//   • `values` is OMITTED when the row's bytes are empty (an all-NULL row), and `decodeRow` handles
+//     that with `row.values ? atob(row.values) : ''` — emitting `""` would work, but omitting is
+//     what the vendor does and what the fixture in the client's own suite shows.
+//
+// ── PROTOJSON DEFAULTS ARE OMITTED, AND THE CLIENT DEPENDS ON IT ──────────────────────────────
+// protojson does not emit fields holding their zero value unless `EmitUnpopulated` is set, and
+// PlanetScale does not set it. So `rowsAffected: 0`, `insertId: 0`, an empty `fields`, an empty
+// `rows`, and `type: NULL_TYPE` (enum 0) are all ABSENT from the JSON. The client compensates
+// exactly there — `result?.rowsAffected ? parseInt(...) : 0`, `result?.insertId ?? '0'`,
+// `field.type ||= 'NULL'` — which is the proof that the absence is the vendor's real behaviour and
+// not an accident. A DDL statement's whole result is therefore the empty object `{}`.
+import type { Cell, Field, SessionVars, VitessType } from './planetscale-sql.ts';
+import { BINARY_CHARSET } from './planetscale-sql.ts';
+
+/** One packed row, as `ExecuteResponse.result.rows[]` carries it. */
+export type WireRow = { lengths: string[]; values?: string };
+
+/** `ExecuteResponse.result` — every key omitted when it holds its protojson zero value. */
+export type WireQueryResult = {
+  fields?: WireField[];
+  rows?: WireRow[];
+  rowsAffected?: string;
+  insertId?: string;
+};
+
+/** `query.Field`, protojson-shaped. Optional keys are omitted, never null. */
+export type WireField = {
+  name: string;
+  type?: string;
+  table?: string;
+  orgTable?: string;
+  database?: string;
+  orgName?: string;
+  columnLength?: number;
+  charset?: number;
+  flags?: number;
+  columnType?: string;
+};
+
+/** `vtrpc.RPCError` as it appears on `ExecuteResponse.error` (and in a Connect error body). */
+export type WireError = { code: string; message: string };
+
+/**
+ * Encode one cell to bytes.
+ *
+ * The charset decides the encoding, exactly as it decides the client's decoding: charset 63 is
+ * MySQL's binary collation, for which `cast` returns a `Uint8Array` built one byte per character
+ * (dist/cast.js `isBinary` → dist/text.js `uint8Array`), so a binary cell is laid out LATIN-1 —
+ * one char, one byte. Everything else is UTF-8, which is what `decodeUtf8` decodes.
+ */
+/** A binary cell holding a code point above 0xFF, which no write of the engine stores: its UTF-8 bytes, never a throw. */
+function wideBinaryCell(value: string): Uint8Array {
+  return new TextEncoder().encode(value);
+}
+
+export function encodeCell(value: string, charset: number | undefined): Uint8Array {
+  if (charset === BINARY_CHARSET) {
+    // A binary cell is expected to already be BYTE-SHAPED (one char per byte) — the engine's
+    // `storeValue` guarantees that on every write path. If a code point above 0xFF reaches here
+    // anyway (state written by an older build, or a caller folding resources in by hand), the
+    // answer is to UTF-8-encode it, NOT to throw.
+    //
+    // §9 round one, finding 3: this used to throw a plain `Error`. Because it is not a `SqlError`,
+    // it escaped `handlePlanetscaleTwinRequest` entirely, the server's catch-all turned it into an
+    // HTTP 500, and the client raised `UnknownError` — so one legal `INSERT INTO b (p) VALUES
+    // ('日')` made that table UNREADABLE FOREVER. Real MySQL just stores the UTF-8 bytes. A twin's
+    // read path must never be the thing that crashes; bytes are bytes.
+    let byteShaped = true;
+    for (let i = 0; i < value.length; i++) if (value.charCodeAt(i) > 0xff) { byteShaped = false; break; }
+    if (!byteShaped) return wideBinaryCell(value);
+    const out = new Uint8Array(value.length);
+    for (let i = 0; i < value.length; i++) out[i] = value.charCodeAt(i);
+    return out;
+  }
+  return new TextEncoder().encode(value);
+}
+
+/** Base64 of a byte sequence laid out one char per byte — the layout `atob` reproduces. */
+export function packBytes(bytes: Uint8Array): string {
+  let binary = '';
+  for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]!);
+  return btoa(binary);
+}
+
+/** Pack one result row: per-column byte lengths plus the concatenated payload. */
+export function packRow(fields: Field[], cells: Cell[]): WireRow {
+  const lengths: string[] = [];
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  for (let i = 0; i < fields.length; i++) {
+    const cell = cells[i] ?? null;
+    if (cell === null) { lengths.push('-1'); continue; }
+    // A JSON column reports charset 63 as MySQL's does, but its text is utf8mb4 (MySQL 8 manual,
+    // "The JSON Data Type": JSON strings use utf8mb4), so it packs as text, never as a byte string.
+    const bytes = encodeCell(cell, fields[i]?.type === 'JSON' ? undefined : fields[i]?.charset);
+    lengths.push(String(bytes.length));
+    chunks.push(bytes);
+    total += bytes.length;
+  }
+  if (total === 0) return { lengths };
+  const joined = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) { joined.set(chunk, offset); offset += chunk.length; }
+  return { lengths, values: packBytes(joined) };
+}
+
+/** The inverse — used by this pack's own tests and by the connector to read a real response. */
+export function unpackRow(row: WireRow): Array<string | null> {
+  const raw = row.values === undefined ? '' : atob(row.values);
+  let offset = 0;
+  return row.lengths.map((size) => {
+    const width = parseInt(size, 10);
+    if (width < 0) return null;
+    const slice = raw.substring(offset, offset + width);
+    offset += width;
+    return slice;
+  });
+}
+
+/** Field → protojson, dropping every key at its zero value (see the header). */
+export function encodeField(field: Field): WireField {
+  const out: WireField = { name: field.name };
+  if (field.type !== 'NULL') out.type = field.type;
+  if (field.table !== undefined) out.table = field.table;
+  if (field.orgTable !== undefined) out.orgTable = field.orgTable;
+  if (field.database !== undefined) out.database = field.database;
+  if (field.orgName !== undefined) out.orgName = field.orgName;
+  if (field.columnLength !== undefined && field.columnLength !== 0) out.columnLength = field.columnLength;
+  if (field.charset !== undefined && field.charset !== 0) out.charset = field.charset;
+  if (field.flags !== undefined && field.flags !== 0) out.flags = field.flags;
+  if (field.columnType !== undefined) out.columnType = field.columnType;
+  return out;
+}
+
+/** A whole result set → `ExecuteResponse.result`. `{}` for a DDL statement, by construction. */
+export function encodeQueryResult(input: {
+  fields: Field[];
+  rows: Cell[][];
+  rowsAffected: number;
+  insertId: number;
+}): WireQueryResult {
+  const out: WireQueryResult = {};
+  if (input.fields.length > 0) out.fields = input.fields.map(encodeField);
+  if (input.rows.length > 0) out.rows = input.rows.map((r) => packRow(input.fields, r));
+  if (input.rowsAffected !== 0) out.rowsAffected = String(input.rowsAffected);
+  if (input.insertId !== 0) out.insertId = String(input.insertId);
+  return out;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────────────────────
+// SESSION
+// ─────────────────────────────────────────────────────────────────────────────────────────────
+
+/**
+ * `psdb.v1alpha1.Session`, as far as this protocol can be observed.
+ *
+ * HONESTY NOTE: the client treats the session as COMPLETELY OPAQUE — `Connection.execute` stores
+ * whatever came back and echoes it on the next request, and reads no field of it (dist/index.js,
+ * `this.session = session`). So the only facts about its shape that this pack can assert are the
+ * ones its own protocol needs: an identity that survives a round trip, and the transaction state
+ * that makes `Connection.transaction()` work. The real Vitess `Session` proto carries many more
+ * fields (shard sessions, options, warnings, DDL strategy); this twin does NOT fabricate them, and
+ * `planetscale.session.vitess_session_fields` files that as a todo rather than inventing a shape.
+ */
+export type WireSession = {
+  signature: string;
+  vitessSession: {
+    autocommit: boolean;
+    targetString: string;
+    sessionUUID: string;
+    inTransaction: boolean;
+    /** vtgate.proto Session.user_defined_variables: "all the @variables defined for this session", each a BindVariable */
+    userDefinedVariables?: Record<string, { type?: string; value?: string }>;
+    /** vtgate.proto Session.system_variables: "all session variables set for this connection" (the twin keeps sql_mode) */
+    systemVariables?: Record<string, string>;
+    /** vtgate.proto Session.warnings: "non-fatal warnings from the previous query", each a QueryWarning (code, message) */
+    warnings?: Array<{ code?: number; message?: string }>;
+  };
+};
+
+/**
+ * The session variables a client's session carries, read back from what it echoes: user variables from
+ * `user_defined_variables` (a BindVariable each: its type, and its value as bytes, base64 in protojson; a NULL value is
+ * the zero type, NULL_TYPE, which protojson omits) and the SQL mode from `system_variables.sql_mode`. The session is the
+ * client's to echo, as Vitess keeps them; the twin stores none of it.
+ */
+export function sessionVarsOf(session: unknown): SessionVars {
+  const vs = (session as { vitessSession?: WireSession['vitessSession'] } | null | undefined)?.vitessSession;
+  const user: SessionVars['user'] = {};
+  for (const [name, bv] of Object.entries(vs?.userDefinedVariables ?? {})) {
+    if (bv === null || typeof bv !== 'object') continue;
+    const type = typeof bv.type === 'string' && bv.type !== 'NULL_TYPE' ? (bv.type as VitessType) : undefined;
+    const bytes = atob(bv.value ?? '');
+    // a binary string keeps its bytes (one character per byte, as the engine holds binary cells); text is UTF-8
+    user[name.toLowerCase()] = type === undefined ? { value: null, type: 'NULL' } : { value: type === 'VARBINARY' ? bytes : new TextDecoder().decode(Uint8Array.from(bytes, (c) => c.charCodeAt(0))), type };
+  }
+  const sqlMode = vs?.systemVariables?.sql_mode;
+  const collationConnection = vs?.systemVariables?.collation_connection;
+  const timeZone = vs?.systemVariables?.time_zone;
+  const warnings = Array.isArray(vs?.warnings) ? vs!.warnings.map((w) => ({ level: 'Warning', code: Number(w?.code ?? 0), message: String(w?.message ?? '') })) : [];
+  return { user, ...(typeof sqlMode === 'string' ? { sqlMode } : {}), ...(typeof collationConnection === 'string' ? { collationConnection } : {}), ...(typeof timeZone === 'string' ? { timeZone } : {}), ...(warnings.length ? { warnings } : {}) };
+}
+
+function encodeVars(vars: SessionVars | undefined): Pick<WireSession['vitessSession'], 'userDefinedVariables' | 'systemVariables' | 'warnings'> {
+  if (vars === undefined) return {};
+  const names = Object.keys(vars.user);
+  return {
+    ...(names.length ? { userDefinedVariables: Object.fromEntries(names.sort().map((n) => {
+      const v = vars.user[n]!;
+      const bytes = v.value === null ? undefined : v.type === 'VARBINARY' ? Uint8Array.from(v.value, (c) => c.charCodeAt(0)) : new TextEncoder().encode(v.value);
+      return [n, bytes === undefined ? {} : { type: v.type, value: packBytes(bytes) }];
+    })) } : {}),
+    ...(vars.sqlMode !== undefined || vars.collationConnection !== undefined || vars.timeZone !== undefined ? { systemVariables: {
+      ...(vars.sqlMode !== undefined ? { sql_mode: vars.sqlMode } : {}), ...(vars.collationConnection !== undefined ? { collation_connection: vars.collationConnection } : {}),
+      ...(vars.timeZone !== undefined ? { time_zone: vars.timeZone } : {}),
+    } } : {}),
+    ...(vars.warnings?.length ? { warnings: vars.warnings.map((w) => ({ code: w.code, message: w.message })) } : {}),
+  };
+}
+
+export type WireUser = { username: string; psid: string };
+
+/** `CreateSessionResponse`. `branch` and `user` come straight from ps-http-sim's own response. */
+export type WireCreateSessionResponse = { branch: string; user: WireUser; session: WireSession };
+
+export type WireExecuteResponse = {
+  session?: WireSession;
+  result?: WireQueryResult;
+  error?: WireError;
+  /** seconds, as a float — the client multiplies by 1000 for `ExecutedQuery.time`. */
+  timing?: number;
+};
+
+export function makeSession(id: string, opts: { branch: string; inTransaction: boolean; vars?: SessionVars }): WireSession {
+  return {
+    // A stable, derivable signature — NOT entropy: the served response must be a pure function of
+    // (request, stored state), so the same session id always renders the same signature.
+    signature: packBytes(new TextEncoder().encode(`twin:${opts.branch}:${id}`)),
+    vitessSession: {
+      autocommit: !opts.inTransaction,
+      targetString: `${opts.branch}@primary`,
+      sessionUUID: id,
+      inTransaction: opts.inTransaction,
+      ...encodeVars(opts.vars),
+    },
+  };
+}
+
+/** Read the session id a client echoed back. Returns null for a missing/unrecognisable session. */
+export function sessionIdOf(session: unknown): string | null {
+  if (session === null || typeof session !== 'object') return null;
+  const vs = (session as { vitessSession?: unknown }).vitessSession;
+  if (vs === null || typeof vs !== 'object') return null;
+  const id = (vs as { sessionUUID?: unknown }).sessionUUID;
+  return typeof id === 'string' && id !== '' ? id : null;
+}

package/src/semantics/psdb.ts

@@ -0,0 +1,57 @@
+// The psdb RPCs' handlers, by operationId (docs/contributing/architecture.md, "The semantics layer"). Each delegates
+// to the pack's request handler (planetscale-twin.ts), which keeps psdb's order and both of its envelopes: Connect's
+// method and verb checks, then Basic auth (401 in `{session, error}`), then the body (400 `{code, message}`), then the
+// RPC, whose query errors are 200 `{session, error}` and whose read-only refusal is per statement (manifest.ts says why
+// the kernel's `crossCutting` is not in front). CloseSession asks the declared session machine (manifest.ts) before it
+// closes one. Prepare answers Connect's `unimplemented`, as ps-http-sim does. StreamExecute has no handler.
+//
+// These are dispatch handlers, not `bindSemantics` ones: a semantics context parses the body before the handler runs,
+// and a malformed body would throw there instead of answering psdb's 400 after its auth check.
+import type { DerivedCall, DerivedHandler } from '@volter/world-core';
+import { handlePlanetscaleTwinRequest, type PlanetscaleRequest, type PlanetscaleResponse } from '../planetscale-twin.ts';
+
+/** What a psdb request is answered against: the server's options and the instant it happens at. */
+export type PsdbOptions = {
+  root?: string;
+  readOnly?: boolean;
+  username?: string;
+  password?: string;
+  database?: string;
+  branch?: string;
+};
+
+/** A fetch Request as the psdb request handler reads it. */
+export async function psdbRequest(request: Request, options: PsdbOptions, occurredAt: string): Promise<PlanetscaleRequest> {
+  const url = new URL(request.url);
+  const body = request.method !== 'GET' && request.method !== 'HEAD' ? await request.text() : '';
+  const headers: Record<string, string> = {};
+  request.headers.forEach((value, key) => { headers[key.toLowerCase()] = value; });
+  return {
+    method: request.method,
+    path: url.pathname + (url.search || ''),
+    body,
+    headers,
+    readOnly: options.readOnly ?? false,
+    occurredAt,
+    ...(options.root !== undefined ? { root: options.root } : {}),
+    ...(options.username !== undefined ? { username: options.username } : {}),
+    ...(options.password !== undefined ? { password: options.password } : {}),
+    ...(options.database !== undefined ? { database: options.database } : {}),
+    ...(options.branch !== undefined ? { branch: options.branch } : {}),
+  };
+}
+
+/** The handler's answer as a fetch Response: only its own headers, none added. */
+export const psdbResponse = ({ status, body, headers }: PlanetscaleResponse): Response =>
+  new Response(JSON.stringify(body), { status, headers: { ...(headers ?? {}) } });
+
+/** A fault inside the twin, in Connect's transport shape; never an exception out of the fetch. */
+export const psdbInternal = (error: unknown): Response =>
+  new Response(JSON.stringify({ code: 'internal', message: String(error) }), { status: 500, headers: { 'content-type': 'application/json' } });
+
+export function psdbHandlers(options: PsdbOptions, now: () => string): Record<string, DerivedHandler> {
+  const answer = async (request: Request): Promise<Response> =>
+    psdbResponse(await handlePlanetscaleTwinRequest(await psdbRequest(request, options, now())));
+  const serve = ({ request }: DerivedCall): Promise<Response> => answer(request).catch(psdbInternal);
+  return { CreateSession: serve, Execute: serve, CloseSession: serve, Prepare: serve };
+}