cursedbelt-core 1.0.0

package/src/core/wire/marshalWorker.ts

@@ -0,0 +1,361 @@
+/**
+ * The marshalling Worker (05 §B1, second half) — `validate` + `transform` for a
+ * large inbound payload, off the thread that has to answer requests.
+ *
+ * `marshalling.ts` has always had the SIZE BOUNDARY and the injection seam; what
+ * it never had was anything to hand a large payload to, so a >512KB webhook body
+ * got a 503 with no documentation anywhere that a limit existed. This module is
+ * the missing half: a real, long-lived worker with a request/response protocol,
+ * on both runtimes cursedbelt ships to.
+ *
+ * ## Why the marshaller names its own worker entry
+ *
+ * A marshaller is mostly FUNCTIONS (`validate`, `transform`, a Zod schema), and
+ * functions do not survive structured clone — a worker cannot be handed one, it
+ * has to `import` it. The obvious design (one shared worker that resolves any
+ * marshaller by id from a registry module) needs the host to declare, register
+ * and keep in sync a second module listing every marshaller, and it gives that
+ * one worker the authority to run all of them.
+ *
+ * Instead each marshaller declares `workerEntry`: the module that constructs
+ * exactly that marshaller and calls {@link serveMarshalWorker}. The worker can
+ * therefore only ever run the one marshaller it was built for, the host wires
+ * nothing, and — the part that matters for the gate — "can this marshaller
+ * offload?" stops being a boolean somebody remembered to set and becomes a
+ * question about an artifact that either exists or does not. V3 checks it
+ * statically (`marshaller-worker-offload-missing`), so the 503 path below is
+ * unreachable in any app that passed verification.
+ *
+ * ## The two runtimes
+ *
+ * Server (Bun/Node) uses `node:worker_threads`; the browser uses a module
+ * `Worker`. The import of `node:worker_threads` is deliberately DYNAMIC and
+ * inside the server branch — a static import would put a Node builtin in the
+ * module graph of every client bundle that touches wire code.
+ */
+import { AppError } from '../../shared/errors';
+import {
+  type CCWireMarshaller,
+  MARSHALLER_SCHEMA_VIOLATION,
+  MARSHALLER_VALIDATION_FAILED,
+  type MarshalWorker,
+} from './marshalling';
+
+export const MARSHALLER_WORKER_FAILED = 'MARSHALLER_WORKER_FAILED';
+export const MARSHALLER_WORKER_TIMEOUT = 'MARSHALLER_WORKER_TIMEOUT';
+
+/** How long one payload may occupy the worker before the host gives up. */
+export const MARSHAL_WORKER_DEFAULT_TIMEOUT_MS = 30_000;
+
+/* -------------------------------------------------------------- protocol */
+
+export interface MarshalWorkerRequest {
+  /** Correlates the reply — several payloads are in flight at once. */
+  id: number;
+  payload: unknown;
+}
+
+export type MarshalWorkerResponse =
+  | { id: number; ok: true; value: unknown }
+  | { id: number; ok: false; code: string; message: string };
+
+/* ----------------------------------------------- the worker side (inside) */
+
+/** The two host runtimes look identical from inside a worker if we reduce them
+ *  to "give me messages / let me post one". */
+interface WorkerChannel {
+  onMessage(handler: (data: unknown) => void): void;
+  post(data: unknown): void;
+}
+
+async function resolveWorkerChannel(): Promise<WorkerChannel> {
+  // Browser/Web Worker: `self` has postMessage and no `window`.
+  const maybeSelf = globalThis as unknown as {
+    postMessage?: (data: unknown) => void;
+    addEventListener?: (type: string, listener: (event: { data: unknown }) => void) => void;
+    window?: unknown;
+  };
+  if (
+    maybeSelf.window === undefined &&
+    typeof maybeSelf.postMessage === 'function' &&
+    typeof maybeSelf.addEventListener === 'function'
+  ) {
+    const post = maybeSelf.postMessage.bind(globalThis);
+    const listen = maybeSelf.addEventListener.bind(globalThis);
+    return {
+      onMessage: (handler) => listen('message', (event) => handler(event.data)),
+      post,
+    };
+  }
+
+  const { parentPort } = await import('node:worker_threads');
+  if (!parentPort)
+    throw new AppError('serveMarshalWorker was called outside a worker thread.', {
+      code: MARSHALLER_WORKER_FAILED,
+      status: 500,
+    });
+  return {
+    onMessage: (handler) => parentPort.on('message', handler),
+    post: (data) => parentPort.postMessage(data),
+  };
+}
+
+/**
+ * Run `marshaller` inside a worker, answering one payload per message.
+ *
+ * A worker entry is three lines:
+ *
+ * ```ts
+ * import { serveMarshalWorker } from 'cursedbelt-core/wire';
+ * import { sheetRowsMarshaller } from './sheetRows';
+ * await serveMarshalWorker(sheetRowsMarshaller);
+ * ```
+ *
+ * **The schema check is deliberately NOT run here.** `marshalInbound` re-checks
+ * every worker result against the token's canonical schema on the host side,
+ * because a worker is a thread boundary, not a trust boundary — a result that
+ * validated itself is exactly the kind of assurance that stops being true the
+ * day someone edits the worker entry. Doing it in both places would also mean
+ * two implementations of one law, and the worker's copy is the one no test
+ * would cover.
+ */
+export async function serveMarshalWorker<Inbound, Out>(
+  marshaller: Pick<CCWireMarshaller<Inbound, Out>, 'id' | 'validate' | 'transform'>,
+): Promise<void> {
+  const channel = await resolveWorkerChannel();
+
+  channel.onMessage((data) => {
+    const request = data as MarshalWorkerRequest;
+    if (typeof request?.id !== 'number') return; // not ours; ignore rather than crash the worker
+    try {
+      if (!marshaller.validate(request.payload as Inbound)) {
+        channel.post({
+          id: request.id,
+          ok: false,
+          code: MARSHALLER_VALIDATION_FAILED,
+          message: `Inbound payload rejected by marshaller '${marshaller.id}'.`,
+        } satisfies MarshalWorkerResponse);
+        return;
+      }
+      const value = marshaller.transform(request.payload as Inbound);
+      channel.post({ id: request.id, ok: true, value } satisfies MarshalWorkerResponse);
+    } catch (error) {
+      // A throwing transform must not take the worker down with it: the next
+      // payload is a different payload, and a dead worker turns one bad webhook
+      // into an outage.
+      channel.post({
+        id: request.id,
+        ok: false,
+        code: MARSHALLER_WORKER_FAILED,
+        message: error instanceof Error ? error.message : String(error),
+      } satisfies MarshalWorkerResponse);
+    }
+  });
+}
+
+/* ------------------------------------------------- the host side (outside) */
+
+/** The subset of both runtimes' Worker the client below needs. */
+interface HostWorker {
+  post(data: unknown): void;
+  onMessage(handler: (data: unknown) => void): void;
+  onError(handler: (error: unknown) => void): void;
+  terminate(): void;
+}
+
+export interface MarshalWorkerClientOptions {
+  /** Module specifier / URL of the entry that calls {@link serveMarshalWorker}. */
+  entry: string | URL;
+  /** Per-payload ceiling. Defaults to {@link MARSHAL_WORKER_DEFAULT_TIMEOUT_MS}. */
+  timeoutMs?: number;
+  /** Spawn override — the seam tests use to run without a real thread. */
+  spawn?: (entry: string | URL) => HostWorker;
+}
+
+export interface MarshalWorkerClient extends MarshalWorker {
+  /** Stop the thread. Pending payloads reject; further calls are refused. */
+  dispose(): Promise<void>;
+}
+
+function spawnHostWorker(entry: string | URL): HostWorker {
+  const WebWorker = (globalThis as { Worker?: new (url: string | URL, options?: { type: string }) => unknown })
+    .Worker;
+  // A browser (or Bun's web-Worker shim) — `type: 'module'` so the entry may use
+  // static imports, which every entry will.
+  if (typeof WebWorker === 'function') {
+    const worker = new WebWorker(entry, { type: 'module' }) as {
+      postMessage(data: unknown): void;
+      addEventListener(type: string, listener: (event: { data?: unknown; message?: string }) => void): void;
+      terminate(): void;
+    };
+    return {
+      post: (data) => worker.postMessage(data),
+      onMessage: (handler) => worker.addEventListener('message', (event) => handler(event.data)),
+      onError: (handler) => worker.addEventListener('error', (event) => handler(event.message ?? event)),
+      terminate: () => worker.terminate(),
+    };
+  }
+
+  // Node/Bun worker_threads. Constructed lazily and synchronously-wrapped: the
+  // client below queues sends until the thread exists, so callers never see the
+  // construction race.
+  let pending: unknown[] = [];
+  let realPost: ((data: unknown) => void) | null = null;
+  let terminated = false;
+  const messageHandlers: ((data: unknown) => void)[] = [];
+  const errorHandlers: ((error: unknown) => void)[] = [];
+  let terminateReal: (() => void) | null = null;
+
+  void (async () => {
+    try {
+      const { Worker } = await import('node:worker_threads');
+      if (terminated) return;
+      const worker = new Worker(entry as string | URL);
+      worker.on('message', (data: unknown) => {
+        for (const handler of messageHandlers) handler(data);
+      });
+      worker.on('error', (error: unknown) => {
+        for (const handler of errorHandlers) handler(error);
+      });
+      // ANY exit is a failure for this client: the worker is long-lived by
+      // design, so it should outlive every payload. Gating this on `code !== 0`
+      // left a cleanly-exited thread invisible — payloads in flight waited out
+      // their full timeout instead of failing at once, and every later `run()`
+      // posted into a dead thread and did the same.
+      worker.on('exit', (code: number) => {
+        const how = code === 0 ? 'exited cleanly while still in use' : `exited with code ${code}`;
+        for (const handler of errorHandlers) handler(new Error(`marshal worker ${how}`));
+      });
+      // `unref` would let the process exit with payloads in flight; the client's
+      // own timeout is the liveness backstop instead.
+      realPost = (data) => worker.postMessage(data);
+      terminateReal = () => void worker.terminate();
+      for (const queued of pending) realPost(queued);
+      pending = [];
+    } catch (error) {
+      for (const handler of errorHandlers) handler(error);
+    }
+  })();
+
+  return {
+    post: (data) => (realPost ? realPost(data) : pending.push(data)),
+    onMessage: (handler) => messageHandlers.push(handler),
+    onError: (handler) => errorHandlers.push(handler),
+    terminate: () => {
+      terminated = true;
+      terminateReal?.();
+    },
+  };
+}
+
+/**
+ * A long-lived worker for ONE marshaller, multiplexing concurrent payloads.
+ *
+ * Long-lived rather than one-worker-per-payload deliberately: thread startup
+ * plus module load is tens of milliseconds, which on a burst of imports would
+ * cost more than the parse it is avoiding, and a spawn-per-payload design lets
+ * an import storm fork unbounded threads.
+ */
+export function createMarshalWorkerClient(options: MarshalWorkerClientOptions): MarshalWorkerClient {
+  const timeoutMs = options.timeoutMs ?? MARSHAL_WORKER_DEFAULT_TIMEOUT_MS;
+  const worker = (options.spawn ?? spawnHostWorker)(options.entry);
+
+  interface Pending {
+    resolve(value: unknown): void;
+    reject(error: unknown): void;
+    timer: ReturnType<typeof setTimeout>;
+  }
+  const pending = new Map<number, Pending>();
+  let nextId = 1;
+  let disposed = false;
+
+  const settle = (id: number, apply: (p: Pending) => void): void => {
+    const entry = pending.get(id);
+    if (!entry) return; // already timed out — the late reply is discarded
+    pending.delete(id);
+    clearTimeout(entry.timer);
+    apply(entry);
+  };
+
+  worker.onMessage((data) => {
+    const response = data as MarshalWorkerResponse;
+    if (typeof response?.id !== 'number') return;
+    settle(response.id, (p) => {
+      if (response.ok) {
+        p.resolve(response.value);
+        return;
+      }
+      p.reject(
+        new AppError(response.message, {
+          // A validation failure is the CALLER's fault (400) and must keep its
+          // own code; anything else the worker reports is ours (500).
+          code: response.code,
+          status: response.code === MARSHALLER_VALIDATION_FAILED ? 400 : 500,
+          context: { worker: String(options.entry) },
+        }),
+      );
+    });
+  });
+
+  // A worker that dies takes every payload in flight with it — failing them all
+  // now beats each one waiting out its own timeout.
+  worker.onError((error) => {
+    const message = error instanceof Error ? error.message : String(error);
+    for (const id of [...pending.keys()])
+      settle(id, (p) =>
+        p.reject(
+          new AppError(`Marshalling worker failed: ${message}`, {
+            code: MARSHALLER_WORKER_FAILED,
+            status: 500,
+            context: { worker: String(options.entry) },
+          }),
+        ),
+      );
+  });
+
+  return {
+    run<Inbound, Out>(_marshallerId: string, payload: Inbound): Promise<Out> {
+      if (disposed)
+        return Promise.reject(
+          new AppError('Marshalling worker has been disposed.', {
+            code: MARSHALLER_WORKER_FAILED,
+            status: 500,
+            context: { worker: String(options.entry) },
+          }),
+        );
+      const id = nextId++;
+      return new Promise<Out>((resolve, reject) => {
+        const timer = setTimeout(() => {
+          pending.delete(id);
+          reject(
+            new AppError(`Marshalling worker did not answer within ${timeoutMs}ms.`, {
+              code: MARSHALLER_WORKER_TIMEOUT,
+              status: 504,
+              context: { worker: String(options.entry), timeoutMs },
+            }),
+          );
+        }, timeoutMs);
+        pending.set(id, { resolve: resolve as (value: unknown) => void, reject, timer });
+        worker.post({ id, payload } satisfies MarshalWorkerRequest);
+      });
+    },
+
+    async dispose(): Promise<void> {
+      disposed = true;
+      for (const id of [...pending.keys()])
+        settle(id, (p) =>
+          p.reject(
+            new AppError('Marshalling worker was disposed while a payload was in flight.', {
+              code: MARSHALLER_WORKER_FAILED,
+              status: 503,
+              context: { worker: String(options.entry) },
+            }),
+          ),
+        );
+      worker.terminate();
+    },
+  };
+}
+
+/** Re-exported so a caller catching worker failures needs one import. */
+export { MARSHALLER_SCHEMA_VIOLATION, MARSHALLER_VALIDATION_FAILED };

package/src/core/wire/marshalling.spec.ts

@@ -0,0 +1,139 @@
+/**
+ * Port marshalling + the 512KB worker gate (02 §7.1).
+ */
+import { describe, expect, it } from 'bun:test';
+import { z } from 'zod';
+import {
+  type CCWireMarshaller,
+  createMarshallerRegistry,
+  MARSHALLER_WORKER_OFFLOAD_BYTES,
+  marshalInbound,
+  payloadBytes,
+} from './marshalling';
+
+const schema = z.object({ rows: z.array(z.object({ id: z.string() })) });
+
+const marshaller = (over: Partial<CCWireMarshaller<unknown, z.infer<typeof schema>>> = {}) =>
+  ({
+    id: 'csv-rows',
+    maxPayloadBytes: 10 * 1024 * 1024,
+    validate: () => true,
+    transform: (inbound: unknown) => ({ rows: (inbound as { rows: { id: string }[] }).rows }),
+    targetZodSchema: schema,
+    ...over,
+  }) as CCWireMarshaller<unknown, z.infer<typeof schema>>;
+
+describe('payloadBytes', () => {
+  it('measures UTF-8 bytes, not characters', () => {
+    // A CJK document is ~3 bytes/char; sizing by character count sails past the
+    // offload threshold on the main thread.
+    expect(payloadBytes('あ')).toBe(3);
+    expect(payloadBytes('a')).toBe(1);
+  });
+
+  it('treats a non-serializable payload as over-threshold rather than crashing', () => {
+    const cyclic: Record<string, unknown> = {};
+    cyclic.self = cyclic;
+    expect(payloadBytes(cyclic)).toBe(Number.POSITIVE_INFINITY);
+  });
+});
+
+describe('marshalInbound', () => {
+  it('validates, transforms, and schema-checks a normal payload on the main thread', async () => {
+    const result = await marshalInbound(marshaller(), { rows: [{ id: '1' }] });
+    expect(result).toEqual({ rows: [{ id: '1' }] });
+  });
+
+  it('rejects a payload over the marshaller’s own limit before scheduling any work', async () => {
+    let transformed = false;
+    await expect(
+      marshalInbound(
+        marshaller({
+          maxPayloadBytes: 4,
+          transform: () => {
+            transformed = true;
+            return { rows: [] };
+          },
+        }),
+        { rows: [{ id: 'long-enough-to-exceed' }] },
+      ),
+    ).rejects.toMatchObject({ code: 'MARSHALLER_PAYLOAD_TOO_LARGE' });
+    expect(transformed).toBe(false);
+  });
+
+  it('refuses a validate() rejection with a typed error', async () => {
+    await expect(
+      marshalInbound(marshaller({ validate: () => false }), { rows: [] }),
+    ).rejects.toMatchObject({ code: 'MARSHALLER_VALIDATION_FAILED' });
+  });
+
+  it('refuses transform output that violates the token’s canonical schema', async () => {
+    await expect(
+      marshalInbound(marshaller({ transform: () => ({ rows: 'not an array' }) as never }), { rows: [] }),
+    ).rejects.toMatchObject({ code: 'MARSHALLER_SCHEMA_VIOLATION' });
+  });
+
+  it('offloads past the threshold instead of blocking the calling thread', async () => {
+    let ranOnMainThread = false;
+    let offloaded = false;
+
+    const result = await marshalInbound(
+      marshaller({
+        transform: () => {
+          ranOnMainThread = true;
+          return { rows: [] };
+        },
+      }),
+      { rows: [{ id: 'x' }] },
+      {
+        offloadThresholdBytes: 4,
+        worker: {
+          async run() {
+            offloaded = true;
+            return { rows: [{ id: 'from-worker' }] } as never;
+          },
+        },
+      },
+    );
+
+    expect(offloaded).toBe(true);
+    expect(ranOnMainThread).toBe(false);
+    expect(result).toEqual({ rows: [{ id: 'from-worker' }] });
+  });
+
+  it('still schema-checks the worker’s result — a thread boundary is not a trust boundary', async () => {
+    await expect(
+      marshalInbound(marshaller(), { rows: [] }, {
+        offloadThresholdBytes: 1,
+        worker: { async run() { return { rows: 'garbage' } as never; } },
+      }),
+    ).rejects.toMatchObject({ code: 'MARSHALLER_SCHEMA_VIOLATION' });
+  });
+
+  it('refuses an oversized payload when no worker is configured', async () => {
+    await expect(
+      marshalInbound(marshaller(), { rows: [{ id: 'x' }] }, { offloadThresholdBytes: 1 }),
+    ).rejects.toMatchObject({ code: 'MARSHALLER_PAYLOAD_TOO_LARGE' });
+  });
+
+  it('pins the 512KB threshold', () => {
+    expect(MARSHALLER_WORKER_OFFLOAD_BYTES).toBe(512 * 1024);
+  });
+});
+
+describe('createMarshallerRegistry', () => {
+  it('refuses a marshaller with no declared payload ceiling', () => {
+    const registry = createMarshallerRegistry();
+    expect(() => registry.register('knowledge:records', marshaller({ maxPayloadBytes: 0 }))).toThrow(
+      /maxPayloadBytes/,
+    );
+  });
+
+  it('registers and resolves by token id', () => {
+    const registry = createMarshallerRegistry();
+    const m = marshaller();
+    registry.register('knowledge:records', m);
+    expect(registry.forToken('knowledge:records')).toBe(m);
+    expect(registry.forToken('media:videos')).toBeUndefined();
+  });
+});