@stigmer/server 3.14.0-dev.20260910084630

package/src/transport/demux.ts

@@ -0,0 +1,190 @@
+/**
+ * Cleartext protocol demux for the unified port — the TS equivalent of Go's
+ * h2c wrapper (backend/libs/go/grpc/server.go, StartHTTP).
+ *
+ * Why this exists: Go's h2c handler transparently serves HTTP/2
+ * prior-knowledge AND HTTP/1.1 on one cleartext socket. Node's cleartext
+ * `http2.createServer` does not fall back to HTTP/1.1 (its `allowHTTP1`
+ * option is TLS/ALPN-only), and browsers only speak HTTP/1.1 on cleartext.
+ * The port's clients therefore split by protocol: native gRPC (runner,
+ * conformance harness) opens HTTP/2 with prior knowledge; browser gRPC-Web,
+ * Connect-over-1.1, and the REST lanes speak HTTP/1.1.
+ *
+ * Mechanism: every HTTP/2 prior-knowledge connection MUST begin with the
+ * 24-byte client connection preface (RFC 9113 §3.4). We peek the first bytes
+ * of each socket, decide the moment the bytes diverge from (or complete) the
+ * preface, and hand the socket to the matching protocol server. HTTP/1.1
+ * requests diverge at byte 0 ("GET", "POST", "OPTI"…), so the common browser
+ * path decides on the first chunk.
+ *
+ * The two handoffs are deliberately different (spike SP-A finding,
+ * 2026-08-23): Node's HTTP/1.1 parser reads the socket as a JS stream, so
+ * `unshift` replays the peeked bytes for it. Node's HTTP/2 core instead
+ * consumes the socket's NATIVE handle — bytes unshifted into the JS stream
+ * buffer never reach it, and the session fails with "bad client magic". The
+ * h2 lane therefore hands over a small replay Duplex (peeked bytes first,
+ * then the live socket), which http2 accepts through its JSStreamSocket
+ * path. Only HTTP/2 connections pay the wrapper; they are few and
+ * long-lived (the runner and harness hold channels open), so the per-byte
+ * JS-stream hop amortizes and HTTP/1.1 traffic stays on the raw socket.
+ *
+ * The CLI's readiness gate is a bare TCP probe (connect + close, no data —
+ * client-apps/cli/src/local/daemon/components.ts serverGate). Sockets that
+ * close before a decision are simply released; they must never log errors
+ * or crash the process.
+ *
+ * Proven by spike SP-A (D2 spike register): the co-located test exercises
+ * all four verified client transports against this demux.
+ */
+import type { Server as Http1Server } from "node:http";
+import type { Http2Server } from "node:http2";
+import { createServer, type Server, type Socket } from "node:net";
+import { Duplex } from "node:stream";
+
+/**
+ * The HTTP/2 client connection preface (RFC 9113 §3.4). Byte-exact; a
+ * cleartext client either opens with these 24 bytes or it is not speaking
+ * HTTP/2 prior-knowledge.
+ */
+export const HTTP2_CONNECTION_PREFACE = Buffer.from(
+  "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n",
+  "latin1",
+);
+
+export interface DemuxTargets {
+  /** Serves browser gRPC-Web, Connect over 1.1, and the REST lanes. */
+  http1: Http1Server;
+  /** Serves native gRPC and the HTTP/2 flavors of Connect/gRPC-Web. */
+  http2: Http2Server;
+}
+
+/**
+ * Creates the TCP front server that peeks each connection's first bytes and
+ * routes the socket to the HTTP/1.1 or HTTP/2 server. The returned server is
+ * the one that binds the unified port; the protocol servers never listen
+ * themselves.
+ */
+export function createProtocolDemuxServer(targets: DemuxTargets): Server {
+  return createServer({ noDelay: true }, (socket) => {
+    routeByPreface(socket, targets);
+  });
+}
+
+function routeByPreface(socket: Socket, targets: DemuxTargets): void {
+  const peeked: Buffer[] = [];
+  let peekedLength = 0;
+
+  const cleanup = (): void => {
+    socket.removeListener("data", onData);
+    socket.removeListener("error", onPeekError);
+    socket.removeListener("close", cleanup);
+  };
+
+  // A socket that errors or closes mid-peek (TCP probes, port scanners,
+  // clients giving up) is released without routing — silently, because the
+  // readiness gate produces one of these on every daemon boot.
+  const onPeekError = (): void => {
+    cleanup();
+    socket.destroy();
+  };
+
+  const onData = (chunk: Buffer): void => {
+    peeked.push(chunk);
+    peekedLength += chunk.length;
+
+    const seen = Buffer.concat(
+      peeked,
+      Math.min(peekedLength, HTTP2_CONNECTION_PREFACE.length),
+    );
+    const comparable = seen.subarray(
+      0,
+      Math.min(seen.length, HTTP2_CONNECTION_PREFACE.length),
+    );
+
+    if (
+      !HTTP2_CONNECTION_PREFACE.subarray(0, comparable.length).equals(
+        comparable,
+      )
+    ) {
+      handoffHttp1();
+      return;
+    }
+    if (peekedLength >= HTTP2_CONNECTION_PREFACE.length) {
+      handoffHttp2();
+      return;
+    }
+    // A strict prefix of the preface: keep peeking until it completes or
+    // diverges. (No real client fragments 24 bytes, but TCP permits it.)
+  };
+
+  function handoffHttp1(): void {
+    cleanup();
+    socket.pause();
+    // The HTTP/1.1 parser reads the JS stream, so unshift replays the
+    // peeked bytes and the server parses the connection from byte 0.
+    socket.unshift(Buffer.concat(peeked, peekedLength));
+    targets.http1.emit("connection", socket);
+    socket.resume();
+  }
+
+  function handoffHttp2(): void {
+    cleanup();
+    socket.pause();
+    const replay = createReplayDuplex(
+      socket,
+      Buffer.concat(peeked, peekedLength),
+    );
+    targets.http2.emit("connection", replay);
+  }
+
+  socket.on("data", onData);
+  socket.on("error", onPeekError);
+  socket.on("close", cleanup);
+}
+
+/**
+ * A Duplex that delivers `head` before the socket's live bytes and forwards
+ * writes/shutdown to the socket. Required for the h2 handoff only: http2
+ * consumes the native socket handle directly, so JS-level `unshift` cannot
+ * replay peeked bytes for it (verified in spike SP-A). Backpressure maps
+ * both ways: a full Duplex buffer pauses the socket; http2 reads resume it.
+ */
+function createReplayDuplex(socket: Socket, head: Buffer): Duplex {
+  const replay = new Duplex({
+    read(): void {
+      socket.resume();
+    },
+    write(
+      chunk: Buffer,
+      encoding: BufferEncoding,
+      callback: (error?: Error | null) => void,
+    ): void {
+      socket.write(chunk, encoding, callback);
+    },
+    final(callback: (error?: Error | null) => void): void {
+      socket.end(callback);
+    },
+    destroy(
+      error: Error | null,
+      callback: (error?: Error | null) => void,
+    ): void {
+      socket.destroy();
+      callback(error);
+    },
+  });
+
+  socket.on("data", (chunk) => {
+    if (!replay.push(chunk)) {
+      socket.pause();
+    }
+  });
+  socket.on("end", () => {
+    replay.push(null);
+  });
+  socket.on("error", (error) => {
+    replay.destroy(error);
+  });
+
+  replay.push(head);
+  return replay;
+}

package/src/transport/health.ts

@@ -0,0 +1,127 @@
+/**
+ * gRPC health service (grpc.health.v1) — the readiness gate.
+ *
+ * Ports the Go server's health posture (backend/libs/go/grpc/server.go):
+ * the overall status ("" service) starts NOT_SERVING at construction and
+ * flips SERVING only after the composition root completes — "ready" means
+ * "wired and answering RPCs", never merely "port bound" (server.go:739-742).
+ * Shutdown flips NOT_SERVING FIRST, before the drain (Stop, :247-265).
+ * The CLI's serverGate TCP probe keys off the port bind, which the boot
+ * sequence therefore performs only after this state is SERVING.
+ *
+ * Semantics match grpc-go's health server: Check on an unknown service is
+ * NOT_FOUND; Watch answers immediately with the current status
+ * (SERVICE_UNKNOWN if unset), stays open, and streams changes.
+ */
+import { Code, ConnectError } from "@connectrpc/connect";
+import type { ConnectRouter } from "@connectrpc/connect";
+import {
+  Health,
+  HealthCheckResponse_ServingStatus as ServingStatus,
+} from "@stigmer/protos/grpc/health/v1/health_pb";
+
+type StatusListener = (status: ServingStatus) => void;
+
+/** The "" service name = the server's overall health (gRPC convention). */
+const OVERALL = "";
+
+export class HealthState {
+  private readonly statuses = new Map<string, ServingStatus>();
+  private readonly listeners = new Map<string, Set<StatusListener>>();
+
+  constructor() {
+    this.statuses.set(OVERALL, ServingStatus.NOT_SERVING);
+  }
+
+  setOverall(status: ServingStatus): void {
+    this.statuses.set(OVERALL, status);
+    for (const listener of this.listeners.get(OVERALL) ?? []) {
+      listener(status);
+    }
+  }
+
+  status(service: string): ServingStatus | undefined {
+    return this.statuses.get(service);
+  }
+
+  subscribe(service: string, listener: StatusListener): () => void {
+    const set = this.listeners.get(service) ?? new Set<StatusListener>();
+    set.add(listener);
+    this.listeners.set(service, set);
+    return () => {
+      set.delete(listener);
+    };
+  }
+}
+
+/** Registers the health service backed by the given state. */
+export function registerHealthService(
+  router: ConnectRouter,
+  state: HealthState,
+): void {
+  router.service(Health, {
+    check: (request) => {
+      const status = state.status(request.service);
+      if (status === undefined) {
+        throw new ConnectError(
+          `unknown service ${request.service}`,
+          Code.NotFound,
+        );
+      }
+      return { status };
+    },
+
+    list: () => {
+      const statuses: Record<string, { status: ServingStatus }> = {};
+      const overall = state.status(OVERALL);
+      if (overall !== undefined) {
+        statuses[OVERALL] = { status: overall };
+      }
+      return { statuses };
+    },
+
+    watch: async function* (request, context) {
+      // Immediate current status (SERVICE_UNKNOWN for unset — grpc-go keeps
+      // the stream OPEN in that case rather than failing, so a client can
+      // watch a service that registers later).
+      let latest =
+        state.status(request.service) ?? ServingStatus.SERVICE_UNKNOWN;
+      yield { status: latest };
+
+      // Stream every change until the client disconnects. The queue-plus-
+      // notify shape keeps updates ordered even if several arrive while the
+      // consumer is mid-yield.
+      const pending: ServingStatus[] = [];
+      let notify: (() => void) | undefined;
+      const unsubscribe = state.subscribe(request.service, (status) => {
+        pending.push(status);
+        notify?.();
+      });
+      const abort = new Promise<void>((resolve) => {
+        context.signal.addEventListener("abort", () => resolve(), {
+          once: true,
+        });
+      });
+
+      try {
+        while (!context.signal.aborted) {
+          if (pending.length === 0) {
+            await Promise.race([
+              abort,
+              new Promise<void>((resolve) => (notify = resolve)),
+            ]);
+            notify = undefined;
+            continue;
+          }
+          const next = pending.shift();
+          if (next !== undefined && next !== latest) {
+            latest = next;
+            yield { status: next };
+          }
+        }
+      } finally {
+        unsubscribe();
+      }
+    },
+  });
+}

package/src/transport/lanes.ts

@@ -0,0 +1,17 @@
+/**
+ * Shared request/response types for the unified port's lanes. The demux
+ * routes each connection to the HTTP/1.1 or HTTP/2 server, and BOTH run the
+ * same lane router — so every lane handler must accept both protocol
+ * stacks' shapes (the members used — headers, method, url, setHeader,
+ * statusCode, end — behave identically on both).
+ */
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { Http2ServerRequest, Http2ServerResponse } from "node:http2";
+
+export type LaneRequest = IncomingMessage | Http2ServerRequest;
+export type LaneResponse = ServerResponse | Http2ServerResponse;
+
+export type LaneHandler = (
+  request: LaneRequest,
+  response: LaneResponse,
+) => void;

package/src/transport/registry/__tests__/lanes.test.ts

@@ -0,0 +1,131 @@
+/**
+ * Registry-lane tests, mirroring the CW-10 conformance suite's assertions
+ * (test/conformance/src/suites/registry-proxy.conformance.test.ts — the
+ * self-declared gate for these lanes) against the FULLY composed server,
+ * demux and lane router included. When #4 lands the `local` target, the
+ * real suite runs these same assertions over the wire; until then this
+ * file keeps the contract enforced in this package's own gate.
+ *
+ * The byte-pin of the bundled data files against Go's embeds lives with
+ * the data now — src/domain/workflow/registry/__tests__/bundled.test.ts
+ * (the registry moved home to the domain, workflow-family DD-A).
+ */
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+
+import { loadConfig } from "../../../boot/config.js";
+import { composeServer, type ComposedServer } from "../../../boot/compose.js";
+import { createLogger } from "../../../boot/logger.js";
+
+const silentLogger = createLogger({
+  level: "error",
+  pretty: false,
+  write: () => {},
+});
+
+let server: ComposedServer;
+let baseUrl: string;
+
+let testDir: string;
+
+beforeAll(async () => {
+  // Refresh pinned off, as the conformance harness pins it — the served
+  // model registry IS the embedded snapshot, deterministic offline.
+  // Every filesystem-touching stage is pinned into a throwaway dir: this
+  // test previously composed against the DEFAULT paths, which meant
+  // opening the developer's real ~/.stigmer/stigmer.db — and with the
+  // skill domain's boot-time staging wipe (#8) it would now also clear
+  // ~/.stigmer/storage/skills-staging. Tests never touch the home dir.
+  testDir = mkdtempSync(path.join(tmpdir(), "registry-lanes-test-"));
+  const config = loadConfig({
+    STIGMER_MODEL_REGISTRY_REFRESH: "off",
+    // No engine behind composed tests: 127.0.0.1:1 is deterministically
+    // closed, so boots fail the non-fatal connect fast and can never touch
+    // a live local Temporal (the conformance CRUD harness does the same).
+    TEMPORAL_HOST_PORT: "127.0.0.1:1",
+    DB_PATH: path.join(testDir, "stigmer.db"),
+    ARTIFACT_LOCAL_BASE_PATH: path.join(testDir, "artifacts"),
+    STORAGE_PATH: path.join(testDir, "storage"),
+  });
+  server = await composeServer({
+    config,
+    logger: silentLogger,
+    portOverride: 0,
+    host: "127.0.0.1",
+  });
+  const port = await server.start();
+  baseUrl = `http://127.0.0.1:${port}`;
+});
+
+afterAll(async () => {
+  await server.shutdown();
+  rmSync(testDir, { recursive: true, force: true });
+});
+
+const routes = [
+  { path: "/v1/proxy/task-kind-registry", topLevelKey: "descriptors" },
+  { path: "/v1/proxy/model-registry", topLevelKey: "models" },
+] as const;
+
+describe("registry proxy lanes (CW-10 contract)", () => {
+  for (const { path, topLevelKey } of routes) {
+    describe(`GET ${path}`, () => {
+      it("answers cacheable JSON with allow-all CORS", async () => {
+        const response = await fetch(baseUrl + path);
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get("content-type")).toBe("application/json");
+        expect(response.headers.get("cache-control")).toContain("max-age");
+        expect(response.headers.get("access-control-allow-origin")).toBe("*");
+
+        const document = (await response.json()) as Record<string, unknown>;
+        expect(document[topLevelKey]).toBeDefined();
+      });
+
+      it("answers the OPTIONS preflight with 204 and the GET allow-list", async () => {
+        const response = await fetch(baseUrl + path, {
+          method: "OPTIONS",
+          headers: {
+            Origin: "http://localhost:3000",
+            "Access-Control-Request-Method": "GET",
+          },
+        });
+
+        expect(response.status).toBe(204);
+        expect(response.headers.get("access-control-allow-origin")).toBe("*");
+        expect(response.headers.get("access-control-allow-methods")).toBe(
+          "GET, OPTIONS",
+        );
+        expect(response.headers.get("access-control-allow-headers")).toBe(
+          "Authorization, Content-Type",
+        );
+      });
+
+      it("refuses non-GET methods with 405", async () => {
+        const response = await fetch(baseUrl + path, { method: "POST" });
+        expect(response.status).toBe(405);
+      });
+    });
+  }
+
+  it("model registry serves the embedded snapshot deterministically (no network behind it)", async () => {
+    const first = await fetch(baseUrl + "/v1/proxy/model-registry");
+    const second = await fetch(baseUrl + "/v1/proxy/model-registry");
+
+    const firstBody = await first.text();
+    const secondBody = await second.text();
+    expect(secondBody).toBe(firstBody);
+
+    const document = JSON.parse(firstBody) as { models?: unknown[] };
+    expect(Array.isArray(document.models)).toBe(true);
+    expect((document.models as unknown[]).length).toBeGreaterThan(0);
+  });
+
+  it("unknown /v1/proxy paths fall through to the router's 404", async () => {
+    const response = await fetch(baseUrl + "/v1/proxy/does-not-exist");
+    expect(response.status).toBe(404);
+  });
+});

package/src/transport/registry/lanes.ts

@@ -0,0 +1,65 @@
+/**
+ * Registry proxy lanes — the unified port's plain-HTTP JSON endpoints,
+ * ported from pkg/domain/workflow/registry/{task_kind_registry,
+ * model_registry}.go and wrapped in the registryCORS contract
+ * (pkg/server/registry_cors.go, oss#571). The CW-10 conformance suite
+ * (test/conformance/src/suites/registry-proxy.conformance.test.ts) is the
+ * executable spec these lanes are built against.
+ *
+ * The documents come from the workflow domain's registry module — the
+ * domain owns the registry, the transport serves FROM it (Go's ownership,
+ * restored by the workflow-family port's DD-A; the model-registry store's
+ * refresh lifecycle is the composition root's concern now, not a lane
+ * concern).
+ *
+ * Behavior per lane (Go task_kind_registry.go:33-51):
+ *   OPTIONS → 204 + the fixed registryCORS allow-lists,
+ *   GET     → 200, application/json, Cache-Control: public, max-age=3600,
+ *             Access-Control-Allow-Origin: *,
+ *   other   → 405 (with the allow-all header — the CORS wrap is
+ *             unconditional in Go).
+ */
+import type { ModelCatalogProvider } from "../../domain/workflow/registry/model-catalog-provider.js";
+import { applyRegistryCorsHeaders, handleRegistryPreflight } from "../cors.js";
+import type { LaneHandler } from "../lanes.js";
+
+/** Registries are static per release: cacheable for an hour (Go handlers). */
+const REGISTRY_CACHE_CONTROL = "public, max-age=3600";
+
+export interface RegistryLanes {
+  taskKindRegistryLane: LaneHandler;
+  modelRegistryLane: LaneHandler;
+}
+
+export interface RegistryLanesOptions {
+  /** The task-kind registry document (static per release). */
+  taskKindRegistryDocument: string;
+  /** The composed model-catalog provider the lane serves from (DD-008). */
+  modelRegistryStore: ModelCatalogProvider;
+}
+
+export function createRegistryLanes(
+  options: RegistryLanesOptions,
+): RegistryLanes {
+  return {
+    taskKindRegistryLane: registryLane(() => options.taskKindRegistryDocument),
+    modelRegistryLane: registryLane(() => options.modelRegistryStore.document()),
+  };
+}
+
+function registryLane(document: () => string): LaneHandler {
+  return (request, response) => {
+    if (handleRegistryPreflight(request, response)) {
+      return;
+    }
+    applyRegistryCorsHeaders(response);
+    if (request.method !== "GET") {
+      response.statusCode = 405;
+      response.end();
+      return;
+    }
+    response.setHeader("Content-Type", "application/json");
+    response.setHeader("Cache-Control", REGISTRY_CACHE_CONTROL);
+    response.end(document());
+  };
+}