@catheadowl/ukp 0.1.1 → 0.2.1

package/src/server.ts

@@ -1,8 +1,10 @@
 import { dirname, join } from "node:path";
-import { readFileSync, writeFileSync } from "node:fs";
+import { readFileSync, writeFileSync, fstatSync, constants as fsConstants } from "node:fs";
+import { createServer as netCreateServer, connect as netConnect, type Server as NetServer } from "node:net";
 import { ENDPOINT_NAME, loadManifest, type LoadedManifest } from "./config/manifest.ts";
 import { localPathOf, readRegistry } from "./registry.ts";
 import { FILE_NATIVE_CAPABILITIES, resolveFileNativeCapability } from "./config/file-native.ts";
+import { EXTERNAL_PROVIDER, EXTERNAL_TOOL_CAPABILITIES } from "./config/external-tool.ts";
 import {
   PROPOSE_SLUG,
   ProposeBusyError,
@@ -83,11 +85,24 @@ export interface ServeConfig {
    * identity under the Service folder (`.ukp/tls/`) — in door mode under the
    * registry's directory, since a door has no single Service folder;
    * `certificates` serves operator-provided PEM files (Let's Encrypt IP
-   * certs, mkcert, private CA). `opensslCommand` is a test injection point
-   * for the self-signing. */
+   * certs, mkcert, private CA). `sanEntries` (the `--tls-san` flag, already
+   * normalized `IP:x`/`DNS:y`) merges extra coverage into the self-signed
+   * SAN — a NAT/EIP public address no NIC carries; a persisted certificate
+   * lacking a requested entry is re-signed over the same key (pin unchanged,
+   * clients re-anchor). `opensslCommand` is a test injection point for the
+   * self-signing. */
   tls?:
-    | { mode: "self-signed"; opensslCommand?: readonly string[] }
+    | { mode: "self-signed"; opensslCommand?: readonly string[]; sanEntries?: readonly string[] }
     | { mode: "certificates"; certPath: string; keyPath: string };
+  /** W9 / ADR-REM-006: self-reap after this many seconds without requests.
+   * The orphan backstop for on-demand-woken doors — a no-TTY remote command
+   * can escape SIGHUP on an abrupt disconnect, so the door bounds its own
+   * lifetime instead of trusting the session to reap it. */
+  maxIdleSeconds?: number;
+  /** W10 / ADR-REM-006: serve on the systemd socket-activation listener
+   * (LISTEN_FDS fd 3) instead of binding a port — the port belongs to the
+   * socket unit; the process only exists while in use. Linux-only. */
+  systemdSocket?: boolean;
 }
 
 export interface DiscoveryDocument {
@@ -136,11 +151,57 @@ export interface ServeInfo {
   port: number;
   authRequired: boolean;
   /** TLS identity summary (W5' / D-079) when serving over HTTPS; the pin is
-   * what remote clients TOFU-pin at registration. */
+   * what remote clients TOFU-pin at registration. `source` "re-signed" marks
+   * the one start that grew SAN coverage over the existing key (--tls-san
+   * added an entry the persisted certificate lacked) — pin unchanged. */
   tls?: {
     pin: string;
     san: string;
-    source: "generated" | "persisted" | "operator";
+    source: "generated" | "re-signed" | "persisted" | "operator";
+  };
+  /** Present when --max-idle armed the self-reap timer (W9). */
+  maxIdleSeconds?: number;
+}
+
+/** W9 / ADR-REM-006 (--max-idle): wrap a serve handler so the listener stops
+ * itself after `maxIdleSeconds` without requests. Requests re-arm the timer
+ * on entry and again once the response settles; the exit mirrors the SIGINT
+ * path (stderr line + server.stop(true)), after which the process exits with
+ * its already-set code. The timer is unref'd so a SIGINT/SIGTERM stop is
+ * immediate — the listener, not the timer, is what holds the serve process
+ * open. `stopped` fences the settle-rearm of a request that outlived the
+ * window (no second fire after the first stop). The bind indirection exists
+ * because Bun.serve hands back the server only after `fetch` is fixed. */
+function armIdleExit(
+  handler: (request: Request) => Response | Promise<Response>,
+  maxIdleSeconds: number,
+): { fetch: (request: Request) => Response | Promise<Response>; bind: (stop: () => void) => void } {
+  const control = { stop: (): void => {} };
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let stopped = false;
+  const rearm = () => {
+    if (stopped) return;
+    if (timer !== undefined) clearTimeout(timer);
+    timer = setTimeout(() => {
+      stopped = true;
+      console.error(`ukp serve: idle exit (--max-idle ${maxIdleSeconds}s without requests)`);
+      control.stop();
+    }, maxIdleSeconds * 1000);
+    // Best-effort unref: present in Bun and Node; a missing no-op keeps this
+    // portable without changing the firing behavior while serving.
+    (timer as { unref?: () => void }).unref?.();
+  };
+  rearm();
+  return {
+    fetch: (request: Request) => {
+      rearm();
+      const response = handler(request);
+      Promise.resolve(response).then(rearm, () => {});
+      return response;
+    },
+    bind(stop: () => void) {
+      control.stop = stop;
+    },
   };
 }
 
@@ -171,13 +232,19 @@ function readInstanceUid(serviceFolder: string): string {
 }
 
 /** Capability projection shared by endpoint and door documents: derived
- * file-native defaults (read/nav) are always present; declared capabilities
- * override or extend them (a declared nav replaces the derived entry). */
+ * file-native defaults (read/nav) are always present, then the external-tool
+ * base tier (rg, ADR-RG-003 — served on /v1/rg for every endpoint; a missing
+ * binary degrades to per-endpoint availability data per the W6 wire
+ * contract, never an undeclared capability); declared capabilities override
+ * or extend both (a declared nav or rg replaces the default entry). */
 function projectCapabilities(loaded: LoadedManifest): DiscoveryDocument["capabilities"] {
   const capabilities: DiscoveryDocument["capabilities"] = {};
   for (const [name, spec] of Object.entries(FILE_NATIVE_CAPABILITIES)) {
     if (spec.derived) capabilities[name] = { provider: "file", derived: true };
   }
+  for (const name of Object.keys(EXTERNAL_TOOL_CAPABILITIES)) {
+    capabilities[name] = { provider: EXTERNAL_PROVIDER, derived: true };
+  }
   for (const name of Object.keys(loaded.manifest.capabilities).sort()) {
     const declared = loaded.manifest.capabilities[name]!;
     capabilities[name] = { provider: declared.provider ?? "file" };
@@ -449,45 +516,202 @@ function parseRgParams(
 export interface StartedServe {
   server: ReturnType<typeof Bun.serve>;
   info: ServeInfo;
+  /** Composite stop: in socket-activation mode this also closes the fd3
+   * acceptor (Bun.serve's own stop alone would leave the process alive
+   * holding the systemd-passed listener). Safe to call from the SIGINT/
+   * SIGTERM/SIGHUP handlers and the idle-exit path alike. */
+  stopAll: () => void;
+}
+
+/** W10 / ADR-REM-006: parse the systemd LISTEN_FDS convention — exactly one
+ * listening fd expected (fd 3), PID-guarded, and the variables are CONSUMED
+ * (unset) so any child process does not double-count inherited activation
+ * fds (the podman fd-counting pitfall, knowledge §5.2). */
+const S_ISSOCK = fsConstants.S_IFSOCK;
+
+export function parseListenFds(env: NodeJS.ProcessEnv): { fd: number } | { error: string } {
+  if ((env.LISTEN_FDS ?? "") === "" || (env.LISTEN_PID ?? "") === "") {
+    return {
+      error:
+        "--systemd-socket requires systemd socket activation: LISTEN_FDS/LISTEN_PID are not set (run under a .socket unit or systemd-socket-activate)",
+    };
+  }
+  if (Number(env.LISTEN_PID) !== process.pid) {
+    return { error: `LISTEN_PID ${env.LISTEN_PID} does not match this process (${process.pid}) — inherited activation variables belong to another process` };
+  }
+  const count = Number(env.LISTEN_FDS);
+  if (!Number.isSafeInteger(count) || count < 1) {
+    return { error: `LISTEN_FDS=${String(env.LISTEN_FDS)} is not a positive integer` };
+  }
+  if (count !== 1) {
+    return { error: `expected exactly one listening fd (LISTEN_FDS=1), got ${count} — run one socket unit per serve instance` };
+  }
+  delete env.LISTEN_FDS;
+  delete env.LISTEN_PID;
+  return { fd: 3 };
+}
+
+/** W10 / ADR-REM-006: the in-process bridge. Bun.serve cannot adopt a raw
+ * listening fd (no public API; ali probe 2026-09-19), but node:net can
+ * (`createServer().listen({fd: 3})` verified under bun). Each connection the
+ * systemd-held listener accepts is piped to the real Bun.serve listener on
+ * an ephemeral loopback port — the fake-ssh proxy pattern, in-process. */
+export function createSocketBridge(acceptor: NetServer, targetPort: number): { destroyAll: () => void } {
+  // Live connections are tracked so stopAll can force-close the pairs — a
+  // peer that receives FIN but never closes (half-open keep-alive shape)
+  // would otherwise hold the process open after the idle exit forever
+  // (review P2: probe-confirmed liveness hang), letting stale serve
+  // processes accumulate beside systemd's fresh spawns.
+  const live = new Set<{ socket: import("node:net").Socket; upstream: import("node:net").Socket }>();
+  acceptor.on("connection", (socket) => {
+    const upstream = netConnect({ host: "127.0.0.1", port: targetPort });
+    const pair = { socket, upstream };
+    live.add(pair);
+    socket.pipe(upstream);
+    upstream.pipe(socket);
+    const drop = () => {
+      live.delete(pair);
+      socket.destroy();
+      upstream.destroy();
+    };
+    socket.on("error", drop);
+    upstream.on("error", drop);
+    socket.on("close", () => {
+      live.delete(pair);
+      upstream.destroy();
+    });
+    upstream.on("close", () => {
+      live.delete(pair);
+      socket.destroy();
+    });
+  });
+  return {
+    destroyAll() {
+      for (const pair of live) {
+        pair.socket.destroy();
+        pair.upstream.destroy();
+      }
+      live.clear();
+    },
+  };
+}
+
+/** One serve listener, both modes (W10): plain binding by default; under
+ * `systemdSocket` the systemd-passed fd 3 is adopted via node:net and
+ * bridged to the real Bun.serve listener on an ephemeral loopback port. */
+function startServeListener(
+  handler: (request: Request) => Response | Promise<Response>,
+  options: {
+    host: string;
+    port?: number;
+    tlsMaterial?: { cert: string; key: string };
+    maxIdleSeconds?: number;
+    systemdSocket?: boolean;
+  },
+): { server: ReturnType<typeof Bun.serve>; stopAll: () => void; publicUrl: string; port: number } {
+  const idle = options.maxIdleSeconds !== undefined ? armIdleExit(handler, options.maxIdleSeconds) : undefined;
+  const fetch = idle?.fetch ?? handler;
+  const tls = options.tlsMaterial !== undefined ? { tls: { cert: options.tlsMaterial.cert, key: options.tlsMaterial.key } } : {};
+  const scheme = options.tlsMaterial !== undefined ? "https" : "http";
+
+  if (options.systemdSocket !== true) {
+    const server = Bun.serve({ hostname: options.host, port: options.port ?? 8570, ...tls, fetch });
+    idle?.bind(() => server.stop(true));
+    const port = server.port ?? (options.port ?? 8570);
+    return { server, stopAll: () => server.stop(true), publicUrl: `${scheme}://${options.host}:${port}`, port };
+  }
+
+  if (process.platform !== "linux") {
+    throw new ServeSetupError("--systemd-socket is Linux-only (systemd socket activation)");
+  }
+  const parsed = parseListenFds(process.env);
+  if ("error" in parsed) throw new ServeSetupError(parsed.error);
+  // Validate the fd synchronously: systemd passes a bound LISTENING socket;
+  // anything else (EBADF, wrong type) must fail setup, not crash later.
+  let isSocket = false;
+  try {
+    const stats = fstatSync(parsed.fd);
+    isSocket = (stats.mode & S_ISSOCK) === S_ISSOCK;
+  } catch {
+    isSocket = false;
+  }
+  if (!isSocket) {
+    throw new ServeSetupError(`fd ${parsed.fd} is not an open socket — socket activation must pass a bound listening socket`);
+  }
+  const server = Bun.serve({ hostname: "127.0.0.1", port: 0, ...tls, fetch });
+  const acceptor = netCreateServer();
+  const bridge = createSocketBridge(acceptor, server.port ?? 0);
+  acceptor.on("error", (error) => {
+    // Async setup failure channel (e.g. fd 3 is a CONNECTED socket — passes
+    // the S_IFSOCK pre-check but cannot listen): no hard exit from library
+    // code — report, stop both listeners, and let the loop drain with a
+    // failing exit code.
+    console.error(`ukp serve: fd${parsed.fd} acceptor failed: ${error instanceof Error ? error.message : String(error)}`);
+    process.exitCode = 1;
+    acceptor.close();
+    bridge.destroyAll();
+    server.stop(true);
+  });
+  acceptor.listen({ fd: parsed.fd });
+  const stopAll = () => {
+    acceptor.close();
+    bridge.destroyAll();
+    server.stop(true);
+  };
+  idle?.bind(stopAll);
+  return {
+    server,
+    stopAll,
+    publicUrl: `systemd:fd${parsed.fd} (bridge ${scheme}://127.0.0.1:${server.port})`,
+    port: server.port ?? 0,
+  };
 }
 
 /** TLS material resolution (W5' / D-079), shared by both serve modes: a
- * self-signed identity is generated/persisted under `identityDir` (the
- * Service folder's `.ukp/tls/` in endpoint mode, the registry directory in
- * door mode — a door has no single Service folder), explicit certificates
+ * self-signed identity is generated/re-signed/persisted under `identityDir`
+ * (the Service folder's `.ukp/tls/` in endpoint mode, the registry directory
+ * in door mode — a door has no single Service folder), explicit certificates
  * are read from the operator's paths. Bun.serve takes PEM contents
- * (path-string handling is platform-dependent). */
+ * (path-string handling is platform-dependent). Every failure — unreadable
+ * operator files, an unusable persisted identity, openssl unable to sign —
+ * surfaces as one ServeSetupError naming the certificate path. */
 function resolveTlsMaterial(
   config: ServeConfig,
   identityDir: string,
-): { cert: string; key: string; pin: string; san: string; source: "generated" | "persisted" | "operator" } | undefined {
+): { cert: string; key: string; pin: string; san: string; source: "generated" | "re-signed" | "persisted" | "operator" } | undefined {
   if (config.tls === undefined) return undefined;
   const tls = config.tls;
-  const paths = tls.mode === "self-signed"
-    ? (() => {
-        const identity = ensureSelfSignedTlsFiles(
-          identityDir,
-          tls.mode === "self-signed" && tls.opensslCommand !== undefined ? { opensslCommand: tls.opensslCommand } : {},
-        );
-        return {
-          certPath: identity.certPath,
-          keyPath: identity.keyPath,
-          source: identity.created ? ("generated" as const) : ("persisted" as const),
-        };
-      })()
-    : { certPath: tls.certPath, keyPath: tls.keyPath, source: "operator" as const };
+  let certPath: string | undefined;
   try {
-    const cert = readFileSync(paths.certPath, "utf8");
+    let keyPath: string;
+    let source: "generated" | "re-signed" | "persisted" | "operator";
+    if (tls.mode === "self-signed") {
+      const identity = ensureSelfSignedTlsFiles(
+        identityDir,
+        {
+          ...(tls.opensslCommand !== undefined ? { opensslCommand: tls.opensslCommand } : {}),
+          ...(tls.sanEntries !== undefined ? { extraSanEntries: tls.sanEntries } : {}),
+        },
+      );
+      certPath = identity.certPath;
+      keyPath = identity.keyPath;
+      source = identity.source;
+    } else {
+      certPath = tls.certPath;
+      keyPath = tls.keyPath;
+      source = "operator";
+    }
+    const cert = readFileSync(certPath, "utf8");
     return {
       cert,
-      key: readFileSync(paths.keyPath, "utf8"),
+      key: readFileSync(keyPath, "utf8"),
       pin: spkiPinOf(cert),
       san: certSanOf(cert),
-      source: paths.source,
+      source,
     };
   } catch (error) {
     const reason = error instanceof Error ? error.message : String(error);
-    throw new ServeSetupError(`TLS material is unusable (${paths.certPath}): ${reason}`);
+    throw new ServeSetupError(`TLS material is unusable (${certPath ?? join(identityDir, "cert.pem")}): ${reason}`);
   }
 }
 
@@ -650,12 +874,14 @@ function startDoorServer(config: ServeConfig, host: string, tlsMaterial: ReturnT
     return doorNotFound(undefined, config.registryPath);
   };
 
-  const server = Bun.serve({
-    hostname: host,
-    port: config.port ?? 8570,
-    ...(tlsMaterial !== undefined ? { tls: { cert: tlsMaterial.cert, key: tlsMaterial.key } } : {}),
-    fetch: handler,
+  const listener = startServeListener(handler, {
+    host,
+    ...(config.port !== undefined ? { port: config.port } : {}),
+    ...(tlsMaterial !== undefined ? { tlsMaterial } : {}),
+    ...(config.maxIdleSeconds !== undefined ? { maxIdleSeconds: config.maxIdleSeconds } : {}),
+    ...(config.systemdSocket === true ? { systemdSocket: true } : {}),
   });
+  const { server } = listener;
 
   const info: ServeInfo = {
     mode: "door",
@@ -663,15 +889,16 @@ function startDoorServer(config: ServeConfig, host: string, tlsMaterial: ReturnT
       endpoints: listDoorEndpoints(config.registryPath).map((endpoint) => endpoint.name),
       write: listWriteEndpoints(config.registryPath),
     },
-    url: `${tlsMaterial !== undefined ? "https" : "http"}://${host}:${server.port ?? (config.port ?? 8570)}`,
+    url: listener.publicUrl,
     host,
-    port: server.port ?? (config.port ?? 8570),
+    port: listener.port,
     authRequired: (config.tokens?.length ?? 0) > 0,
     ...(tlsMaterial !== undefined
       ? { tls: { pin: tlsMaterial.pin, san: tlsMaterial.san, source: tlsMaterial.source } }
       : {}),
+    ...(config.maxIdleSeconds !== undefined ? { maxIdleSeconds: config.maxIdleSeconds } : {}),
   };
-  return { server, info };
+  return { server, info, stopAll: listener.stopAll };
 }
 
 export function startUkpServer(config: ServeConfig): StartedServe {
@@ -781,27 +1008,30 @@ export function startUkpServer(config: ServeConfig): StartedServe {
     );
   };
 
-  const server = Bun.serve({
-    hostname: host,
-    port: config.port ?? 8570,
-    ...(tlsMaterial !== undefined ? { tls: { cert: tlsMaterial.cert, key: tlsMaterial.key } } : {}),
-    fetch: handler,
+  const listener = startServeListener(handler, {
+    host,
+    ...(config.port !== undefined ? { port: config.port } : {}),
+    ...(tlsMaterial !== undefined ? { tlsMaterial } : {}),
+    ...(config.maxIdleSeconds !== undefined ? { maxIdleSeconds: config.maxIdleSeconds } : {}),
+    ...(config.systemdSocket === true ? { systemdSocket: true } : {}),
   });
+  const { server } = listener;
 
   const info: ServeInfo = {
     mode: "endpoint",
     endpoint: binding.name,
     folder: serviceFolder,
     instanceUid: readInstanceUid(serviceFolder),
-    url: `${tlsMaterial !== undefined ? "https" : "http"}://${host}:${server.port ?? (config.port ?? 8570)}`,
+    url: listener.publicUrl,
     host,
-    port: server.port ?? (config.port ?? 8570),
+    port: listener.port,
     authRequired: (config.tokens?.length ?? 0) > 0,
     ...(tlsMaterial !== undefined
       ? { tls: { pin: tlsMaterial.pin, san: tlsMaterial.san, source: tlsMaterial.source } }
       : {}),
+    ...(config.maxIdleSeconds !== undefined ? { maxIdleSeconds: config.maxIdleSeconds } : {}),
   };
-  return { server, info };
+  return { server, info, stopAll: listener.stopAll };
 }
 
 async function handleSearch(

package/bun.lock

@@ -1,35 +0,0 @@
-{
-  "lockfileVersion": 1,
-  "configVersion": 1,
-  "workspaces": {
-    "": {
-      "name": "@catheadowl/ukp",
-      "dependencies": {
-        "commander": "15.0.0",
-        "smol-toml": "1.7.1",
-        "zod": "4.4.3",
-      },
-      "devDependencies": {
-        "@types/bun": "1.3.5",
-        "typescript": "5.9.3",
-      },
-    },
-  },
-  "packages": {
-    "@types/bun": ["@types/bun@1.3.5", "https://registry.npmmirror.com/@types/bun/-/bun-1.3.5.tgz", { "dependencies": { "bun-types": "1.3.5" } }, "sha512-RnygCqNrd3srIPEWBd5LFeUYG7plCoH2Yw9WaZGyNmdTEei+gWaHqydbaIRkIkcbXwhBT94q78QljxN0Sk838w=="],
-
-    "@types/node": ["@types/node@26.1.1", "https://registry.npmmirror.com/@types/node/-/node-26.1.1.tgz", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="],
-
-    "bun-types": ["bun-types@1.3.5", "https://registry.npmmirror.com/bun-types/-/bun-types-1.3.5.tgz", { "dependencies": { "@types/node": "*" } }, "sha512-inmAYe2PFLs0SUbFOWSVD24sg1jFlMPxOjOSSCYqUgn4Hsc3rDc7dFvfVYjFPNHtov6kgUeulV4SxbuIV/stPw=="],
-
-    "commander": ["commander@15.0.0", "https://registry.npmmirror.com/commander/-/commander-15.0.0.tgz", {}, "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg=="],
-
-    "smol-toml": ["smol-toml@1.7.1", "https://registry.npmmirror.com/smol-toml/-/smol-toml-1.7.1.tgz", {}, "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ=="],
-
-    "typescript": ["typescript@5.9.3", "https://registry.npmmirror.com/typescript/-/typescript-5.9.3.tgz", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
-
-    "undici-types": ["undici-types@8.3.0", "https://registry.npmmirror.com/undici-types/-/undici-types-8.3.0.tgz", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="],
-
-    "zod": ["zod@4.4.3", "https://registry.npmmirror.com/zod/-/zod-4.4.3.tgz", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="],
-  }
-}

package/tsconfig.json

@@ -1,12 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "ESNext",
-    "moduleResolution": "Bundler",
-    "strict": true,
-    "noEmit": true,
-    "allowImportingTsExtensions": true,
-    "types": ["bun"]
-  },
-  "include": ["src", "test"]
-}