cursedbelt-core 1.0.0

package/src/core/wire/ingress.ts

@@ -0,0 +1,182 @@
+/**
+ * The composable ingress seam (05 §B1, owner decision E16/D8) — the ONE thing
+ * between `marshalInbound` and a route that accepts external data.
+ *
+ * ## Why this exists rather than a webhook endpoint
+ *
+ * D8 asked which ingress should call `marshalInbound` first: an inbound webhook,
+ * a CSV/spreadsheet import, or neither. The owner delegated the choice with a
+ * binding condition — *"ensure the architecture is ready to add it later, so we
+ * don't have to refactor and can just build on top of it."*
+ *
+ * The answer is neither, yet, and the condition is the deliverable. Shipping a
+ * webhook endpoint that no product asked for would create a new orphan seam of
+ * exactly the kind Law #1 exists to prevent, and it would guess wrong about the
+ * part that actually differs between a webhook and an import (authentication,
+ * routing, what the marshalled value is handed to). What does NOT differ — and
+ * what every ingress would otherwise re-derive, differently — is this:
+ *
+ *   1. reading the body as bytes without buffering a stream into RAM twice;
+ *   2. finding the marshaller for the token being fed;
+ *   3. owning a worker per `workerEntry` — a live OS thread, so exactly one
+ *      thing must own disposing of it — and resolving it lazily and cached;
+ *   4. handing those failures up as errors that already KNOW their HTTP status,
+ *      so no route re-derives the mapping. The marshalling tier's codes are
+ *      unchanged and deliberately distinct: 413 over the declared ceiling, 400
+ *      for a payload `validate` refuses (the caller's shape is wrong), 502 when
+ *      the TRANSFORM OUTPUT violates the token's canonical schema (an upstream
+ *      provider changed shape under us — not our error budget), 503 for a
+ *      payload that needs offload from a marshaller with no worker.
+ *
+ * Step 3 is the one that would go wrong. `createMarshalWorkerClient` spawns a
+ * thread per call; a route handler that called it per request would leak one
+ * OS thread per webhook, and the symptom would be a server that gets slower for
+ * a week and then dies. Nothing about that is discoverable from the marshalling
+ * API, which is precisely why it belongs here once instead of in each ingress.
+ *
+ * ## Adding an ingress later
+ *
+ * It is a route, not a refactor:
+ *
+ * ```ts
+ * const ingress = createMarshalledIngress({ registry });
+ * app.post('/api/hooks/:token', session, async (c) => {
+ *   const value = await ingress.ingestRequest(c.req.param('token'), c.req.raw);
+ *   await service.acceptExternal(scopeOf(c), value);
+ *   return c.json({ success: true }, 202);
+ * });
+ * ```
+ *
+ * A CSV import is the same three lines with a different marshaller and a
+ * different service call. Neither needs to know that a worker exists.
+ */
+import { AppError } from '../../shared/errors';
+import { createMarshalWorkerClient, type MarshalWorkerClient } from './marshalWorker';
+import {
+  type CCWireMarshaller,
+  marshalInbound,
+  type MarshallerRegistry,
+  type MarshalWorker,
+} from './marshalling';
+
+/** No marshaller is registered for the token an ingress was asked to feed. */
+export const INGRESS_TOKEN_UNKNOWN = 'INGRESS_TOKEN_UNKNOWN';
+
+export interface MarshalledIngressOptions {
+  /** The registry the app's wire tokens were registered on. */
+  readonly registry: MarshallerRegistry;
+  /**
+   * Per-payload worker ceiling. A marshalling worker that hangs would otherwise
+   * hold the request open for as long as the client is willing to wait.
+   */
+  readonly workerTimeoutMs?: number;
+  /**
+   * Spawn override — the seam a test uses to run the offload path without a real
+   * OS thread. Production never sets it.
+   */
+  readonly spawn?: (entry: string | URL) => unknown;
+}
+
+export interface MarshalledIngress {
+  /**
+   * Marshal an already-decoded payload for `tokenId`.
+   *
+   * @throws {AppError} `INGRESS_TOKEN_UNKNOWN` (404) when nothing is registered;
+   *   `MARSHALLER_PAYLOAD_TOO_LARGE` (413/503), `MARSHALLER_VALIDATION_FAILED`
+   *   (400) or `MARSHALLER_SCHEMA_VIOLATION` (502) from the marshaller itself.
+   */
+  ingest<Out = unknown>(tokenId: string, payload: unknown): Promise<Out>;
+  /**
+   * Read a `Request` body and marshal it for `tokenId`. JSON bodies are parsed;
+   * anything else is handed to the marshaller as bytes, because a marshaller for
+   * a CSV or a binary feed wants the bytes and re-encoding them would be lossy.
+   */
+  ingestRequest<Out = unknown>(tokenId: string, request: Request): Promise<Out>;
+  /**
+   * Stop every worker this ingress spawned. Idempotent. A host calls it on
+   * shutdown; forgetting to is a thread leak, which is why the ingress owns the
+   * threads rather than each route.
+   */
+  dispose(): Promise<void>;
+}
+
+/**
+ * Build the ingress. One per app — it caches a worker per `workerEntry`, and two
+ * ingresses over the same registry would each spawn their own set.
+ */
+export const createMarshalledIngress = (options: MarshalledIngressOptions): MarshalledIngress => {
+  /** entry → client. The cache IS the point: see the header, step 3. */
+  const workers = new Map<string, MarshalWorkerClient>();
+  let disposed = false;
+
+  const resolveWorker = (marshaller: CCWireMarshaller<never, never>): MarshalWorker | undefined => {
+    const entry = marshaller.workerEntry;
+    if (entry === undefined) return undefined;
+    if (disposed)
+      throw new AppError('This ingress has been disposed and cannot spawn a marshalling worker.', {
+        code: 'INGRESS_DISPOSED',
+        status: 503,
+        isOperational: false,
+        context: { marshaller: marshaller.id },
+      });
+    const existing = workers.get(entry);
+    if (existing) return existing;
+    const client = createMarshalWorkerClient({
+      entry,
+      ...(options.workerTimeoutMs === undefined ? {} : { timeoutMs: options.workerTimeoutMs }),
+      ...(options.spawn === undefined ? {} : { spawn: options.spawn as never }),
+    });
+    workers.set(entry, client);
+    return client;
+  };
+
+  const marshallerFor = (tokenId: string): CCWireMarshaller => {
+    const marshaller = options.registry.forToken(tokenId);
+    if (!marshaller)
+      // 404, not 400: from outside, a token with no marshaller and a token that
+      // does not exist are the same fact, and distinguishing them would make
+      // this endpoint an enumeration oracle for the app's wire surface.
+      throw new AppError(`No marshaller is registered for wire token '${tokenId}'.`, {
+        code: INGRESS_TOKEN_UNKNOWN,
+        status: 404,
+        context: { tokenId },
+      });
+    return marshaller;
+  };
+
+  return {
+    async ingest<Out>(tokenId: string, payload: unknown): Promise<Out> {
+      return (await marshalInbound(marshallerFor(tokenId) as CCWireMarshaller<unknown, Out>, payload, {
+        resolveWorker,
+      })) as Out;
+    },
+
+    async ingestRequest<Out>(tokenId: string, request: Request): Promise<Out> {
+      const contentType = request.headers.get('content-type') ?? '';
+      // JSON is parsed here so the marshaller sees a value rather than a string
+      // it would have to parse itself (and parse DIFFERENTLY per marshaller).
+      // A malformed body is the CALLER's error — 400 — and must not read as a
+      // validation failure of a payload we never managed to decode.
+      const payload = contentType.includes('application/json')
+        ? await request.json().catch(() => {
+            throw new AppError('Request body is not valid JSON.', {
+              code: 'INGRESS_BODY_MALFORMED',
+              status: 400,
+              context: { tokenId, contentType },
+            });
+          })
+        : new Uint8Array(await request.arrayBuffer());
+      return await this.ingest<Out>(tokenId, payload);
+    },
+
+    async dispose(): Promise<void> {
+      disposed = true;
+      const clients = [...workers.values()];
+      workers.clear();
+      // `allSettled`: one thread that refuses to die must not prevent the rest
+      // from being reaped, and a shutdown path that throws leaves the others
+      // running forever.
+      await Promise.allSettled(clients.map((client) => client.dispose()));
+    },
+  };
+};

package/src/core/wire/marshalWorker.spec.ts

@@ -0,0 +1,196 @@
+/**
+ * The marshalling Worker (05 §B1) — proven over a REAL thread.
+ *
+ * The seam has had an injectable `MarshalWorker` since it was written, and the
+ * old suite exercised that seam with a fake. What it could never show is the
+ * thing the 503 was actually about: that a payload really does cross a thread
+ * boundary and come back correct. So the first block here spawns the genuine
+ * `node:worker_threads` worker and drives it end to end; the fake-spawn block
+ * below covers the lifecycle edges (timeout, crash, dispose) that a real thread
+ * cannot be made to hit deterministically.
+ */
+import { describe, expect, it } from 'bun:test';
+import { z } from 'zod';
+import {
+  createMarshalWorkerClient,
+  MARSHALLER_WORKER_FAILED,
+  MARSHALLER_WORKER_TIMEOUT,
+} from './marshalWorker';
+import { type CCWireMarshaller, marshalInbound, requiresWorkerOffload } from './marshalling';
+
+const schema = z.object({ rows: z.array(z.object({ id: z.string() })) });
+const WORKER_ENTRY = new URL('./__fixtures__/rowsMarshalWorker.ts', import.meta.url);
+
+const marshaller = (over: Partial<CCWireMarshaller<unknown, z.infer<typeof schema>>> = {}) =>
+  ({
+    id: 'rows',
+    tokenId: 'knowledge:records',
+    maxPayloadBytes: 10 * 1024 * 1024,
+    workerEntry: WORKER_ENTRY.pathname,
+    validate: () => true,
+    transform: (inbound: unknown) => ({ rows: (inbound as { rows: { id: string }[] }).rows }),
+    targetZodSchema: schema,
+    ...over,
+  }) as CCWireMarshaller<unknown, z.infer<typeof schema>>;
+
+describe('a real marshalling worker', () => {
+  it('runs validate + transform off-thread and returns the result', async () => {
+    const client = createMarshalWorkerClient({ entry: WORKER_ENTRY, timeoutMs: 20_000 });
+    try {
+      const result = await client.run('rows', { rows: [{ id: 'a' }] });
+      // The `worker:` prefix can only have been added by the fixture module,
+      // which this process never imported — so the work genuinely happened there.
+      expect(result).toEqual({ rows: [{ id: 'worker:a' }] });
+    } finally {
+      await client.dispose();
+    }
+  }, 30_000);
+
+  it('multiplexes concurrent payloads onto one thread, each getting its own reply', async () => {
+    const client = createMarshalWorkerClient({ entry: WORKER_ENTRY, timeoutMs: 20_000 });
+    try {
+      const results = await Promise.all(
+        ['a', 'b', 'c', 'd'].map((id) => client.run<{ rows: { id: string }[] }, z.infer<typeof schema>>('rows', { rows: [{ id }] })),
+      );
+      // Correlation by request id is the whole reason this is not a queue of
+      // one: replies may arrive in any order and must not be swapped.
+      expect(results.map((r) => r.rows[0]?.id)).toEqual(['worker:a', 'worker:b', 'worker:c', 'worker:d']);
+    } finally {
+      await client.dispose();
+    }
+  }, 30_000);
+
+  it('reports a validate() rejection as the caller’s fault, not the server’s', async () => {
+    const client = createMarshalWorkerClient({ entry: WORKER_ENTRY, timeoutMs: 20_000 });
+    try {
+      await expect(client.run('rows', { notRows: true })).rejects.toMatchObject({
+        code: 'MARSHALLER_VALIDATION_FAILED',
+        status: 400,
+      });
+    } finally {
+      await client.dispose();
+    }
+  }, 30_000);
+
+  it('survives a throwing transform — one bad payload is not an outage', async () => {
+    const client = createMarshalWorkerClient({ entry: WORKER_ENTRY, timeoutMs: 20_000 });
+    try {
+      await expect(client.run('rows', { rows: [{ id: 'boom' }] })).rejects.toMatchObject({
+        code: MARSHALLER_WORKER_FAILED,
+      });
+      // The thread must still be there. A worker that dies on a malformed body
+      // turns one hostile webhook into a denial of service.
+      const after = await client.run('rows', { rows: [{ id: 'after' }] });
+      expect(after).toEqual({ rows: [{ id: 'worker:after' }] });
+    } finally {
+      await client.dispose();
+    }
+  }, 30_000);
+
+  it('is the DEFAULT path: marshalInbound resolves the marshaller’s own workerEntry', async () => {
+    const client = createMarshalWorkerClient({ entry: WORKER_ENTRY, timeoutMs: 20_000 });
+    try {
+      let ranOnMainThread = false;
+      const result = await marshalInbound(
+        marshaller({
+          transform: () => {
+            ranOnMainThread = true;
+            return { rows: [] };
+          },
+        }),
+        { rows: [{ id: 'x' }] },
+        // No `worker:` — the point of the commit is that an ingress route wires
+        // nothing per-marshaller; it supplies one resolver and offload happens.
+        { offloadThresholdBytes: 4, resolveWorker: () => client },
+      );
+      expect(result).toEqual({ rows: [{ id: 'worker:x' }] });
+      expect(ranOnMainThread).toBe(false);
+    } finally {
+      await client.dispose();
+    }
+  }, 30_000);
+});
+
+describe('marshal worker lifecycle', () => {
+  /** A worker that never answers — the only honest way to test a timeout. */
+  const silentSpawn = () => ({
+    post: () => {},
+    onMessage: () => {},
+    onError: () => {},
+    terminate: () => {},
+  });
+
+  it('times out a payload rather than holding the request open forever', async () => {
+    const client = createMarshalWorkerClient({ entry: 'silent', timeoutMs: 20, spawn: silentSpawn });
+    await expect(client.run('rows', { rows: [] })).rejects.toMatchObject({
+      code: MARSHALLER_WORKER_TIMEOUT,
+      status: 504,
+    });
+    await client.dispose();
+  });
+
+  it('fails every payload in flight when the thread dies, instead of leaving each to time out', async () => {
+    let fail: ((error: unknown) => void) | undefined;
+    const client = createMarshalWorkerClient({
+      entry: 'crashy',
+      timeoutMs: 60_000, // far longer than the test — only the crash can settle these
+      spawn: () => ({
+        post: () => {},
+        onMessage: () => {},
+        onError: (handler) => {
+          fail = handler;
+        },
+        terminate: () => {},
+      }),
+    });
+    const inFlight = [client.run('rows', {}), client.run('rows', {})];
+    fail?.(new Error('thread died'));
+    for (const promise of inFlight)
+      await expect(promise).rejects.toMatchObject({ code: MARSHALLER_WORKER_FAILED });
+  });
+
+  it('refuses work after dispose rather than queueing onto a dead thread', async () => {
+    let terminated = false;
+    const client = createMarshalWorkerClient({
+      entry: 'disposable',
+      spawn: () => ({ ...silentSpawn(), terminate: () => { terminated = true; } }),
+    });
+    await client.dispose();
+    expect(terminated).toBe(true);
+    await expect(client.run('rows', {})).rejects.toMatchObject({ code: MARSHALLER_WORKER_FAILED });
+  });
+});
+
+describe('requiresWorkerOffload — a derived fact, not a declared one', () => {
+  it('is false for a marshaller that can never exceed the main-thread limit', () => {
+    // 256KB ceiling: no payload it accepts can reach the 512KB boundary, so
+    // demanding a worker of it would be noise.
+    expect(requiresWorkerOffload({ maxPayloadBytes: 256 * 1024 })).toBe(false);
+  });
+
+  it('is true for a marshaller whose ceiling is above it', () => {
+    expect(requiresWorkerOffload({ maxPayloadBytes: 2 * 1024 * 1024 })).toBe(true);
+  });
+
+  it('names the missing artifact when an offload-needing marshaller has no entry', async () => {
+    // The 503 that D1 was about. It is still reachable in code — V3 is what makes
+    // it unreachable in a verified app — and when it fires it must say what to
+    // build, not merely that something is missing.
+    const error = await marshalInbound(
+      marshaller({ workerEntry: undefined }),
+      { rows: [{ id: 'x' }] },
+      { offloadThresholdBytes: 1 },
+      // A two-armed `then`, not `.catch`: `.catch` leaves the RESOLVED type in the
+      // union, so `error.code` was `undefined` on the path where the call did not
+      // reject at all — the exact case this test exists to rule out.
+    ).then(
+      () => {
+        throw new Error('expected MARSHALLER_PAYLOAD_TOO_LARGE, but the call resolved');
+      },
+      (e: unknown) => e as { code: string; context?: Record<string, unknown> },
+    );
+
+    expect(error.code).toBe('MARSHALLER_PAYLOAD_TOO_LARGE');
+    expect(String(error.context?.hint)).toContain('workerEntry');
+  });
+});