cartethyia 2.6.6 → 2.7.0

package/src/console/remote-support/contracts.ts

@@ -21,7 +21,9 @@ export type RemoteSupportTool = (typeof REMOTE_SUPPORT_TOOLS)[number];
 export type RemoteSupportBlocker =
   | "not_installed"
   | "unsupported_platform"
-  | "launch_failed";
+  | "launch_failed"
+  /** The publisher launched but has not published an address yet. */
+  | "not_ready";
 
 export interface RemoteSupportStatus {
   /** Which publisher is up, or `null` when neither is. */

package/src/console/remote-support/service.ts

@@ -186,9 +186,95 @@ async function waitForUrl(dir: string, tool: RemoteSupportTool): Promise<string
  * live hostname comes from the daemon's own status. The JSON shape has drifted
  * across releases (`DNSName` at the root in some, nested under `Self` in
  * others), so both locations are probed and the first hit wins.
+ *
+ * `status` is probed as well as `serve status`: the latter answers `{}` until
+ * the node has actually joined the tailnet, which is exactly when the operator
+ * is staring at a "started" publisher with no link. `status` carries
+ * `Self.DNSName` and `BackendState` throughout, so it names the node while
+ * serve is still coming up.
  */
 function tailscaleUrlFromDaemon(): string | null {
+  const name = tailscaleDnsName();
+  return name === null ? null : `https://${name}`;
+}
+
+/**
+ * Why a publisher that started has not published an address.
+ *
+ * For Tailscale the answer is almost always that the node is not on the
+ * tailnet yet — still starting, or never logged in — and both are the
+ * operator's move, not the gateway's. Naming which one turns a dead end into
+ * something to do.
+ */
+function notReadyReason(tool: RemoteSupportTool): string {
+  if (tool !== "tailscale") return `${tool} started but has not published an address yet.`;
+  const state = spawnSync("tailscale", ["status", "--json"], { encoding: "utf8", windowsHide: true });
+  let backend: string | null = null;
+  let authUrl: string | null = null;
+  try {
+    const bag = JSON.parse(state.stdout) as Record<string, unknown>;
+    backend = typeof bag["BackendState"] === "string" ? (bag["BackendState"] as string) : null;
+    authUrl = typeof bag["AuthURL"] === "string" && (bag["AuthURL"] as string).length > 0 ? (bag["AuthURL"] as string) : null;
+  } catch {
+    // An unreadable status still leaves the generic message below.
+  }
+  if (authUrl !== null) return `Tailscale needs to be logged in. Open ${authUrl} once, then start the tunnel again.`;
+  if (backend !== null && backend !== "Running") {
+    return `Tailscale is not connected yet (state: ${backend}). Wait for it to finish starting, then start the tunnel again.`;
+  }
+  return "Tailscale has no tailnet address yet. Run `tailscale up` once, then start the tunnel again.";
+}
+
+/** This node's MagicDNS name, or `null` when the daemon has none yet. */
+function tailscaleDnsName(): string | null {
+  for (const args of [
+    ["serve", "status", "--json"],
+    ["status", "--json"],
+  ] as const) {
+    const name = dnsNameFromStatus(args);
+    if (name !== null) return name;
+  }
+  return null;
+}
+
+/**
+ * Whether `tailscale serve` is actually publishing anything.
+ *
+ * `serve status --json` is `{}` when nothing is served and gains `Web`
+ * (handlers) or `TCP` entries when something is. The node being online says
+ * nothing about this: stopping the tunnel leaves the node up and empties the
+ * status, which is why an online node alone was read as "still active".
+ */
+export function tailscaleServing(): boolean {
   const result = spawnSync("tailscale", ["serve", "status", "--json"], { encoding: "utf8", windowsHide: true });
+  if (result.status !== 0) return false;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(result.stdout);
+  } catch {
+    return false;
+  }
+  return serveStatusHasHandler(parsed);
+}
+
+/**
+ * Whether a parsed `serve status` body publishes anything.
+ *
+ * Split from the spawn so the shape rules are testable without a daemon: `{}`
+ * is the empty answer, and any entry under `Web` or `TCP` is a live one.
+ */
+export function serveStatusHasHandler(parsed: unknown): boolean {
+  const bag: Record<string, unknown> =
+    typeof parsed === "object" && parsed !== null ? (parsed as Record<string, unknown>) : {};
+  for (const key of ["Web", "TCP"]) {
+    const entries = bag[key];
+    if (typeof entries === "object" && entries !== null && Object.keys(entries).length > 0) return true;
+  }
+  return false;
+}
+
+function dnsNameFromStatus(args: readonly string[]): string | null {
+  const result = spawnSync("tailscale", [...args], { encoding: "utf8", windowsHide: true });
   if (result.status !== 0) return null;
   let parsed: unknown;
   try {
@@ -198,12 +284,24 @@ function tailscaleUrlFromDaemon(): string | null {
   }
   const bag: Record<string, unknown> =
     typeof parsed === "object" && parsed !== null ? (parsed as Record<string, unknown>) : {};
+  // Only `status` can prove the node is up — `serve status` carries no node
+  // state. A stopped node still reports its last DNS name, and publishing that
+  // as a live link is worse than publishing nothing: the operator opens an
+  // address that cannot resolve.
+  if (args[0] === "status") {
+    const self = selfOf(bag);
+    const online = self === null ? null : self["Online"];
+    if (online === false) return null;
+  }
   const direct = typeof bag["DNSName"] === "string" ? (bag["DNSName"] as string) : null;
-  const self =
-    typeof bag["Self"] === "object" && bag["Self"] !== null ? (bag["Self"] as Record<string, unknown>) : null;
+  const self = selfOf(bag);
   const nested = self && typeof self["DNSName"] === "string" ? (self["DNSName"] as string) : null;
-  const hostname = (direct ?? nested)?.trim().replace(/\/+$/, "");
-  return hostname && hostname.length > 0 ? `https://${hostname}` : null;
+  const hostname = (direct ?? nested)?.trim().replace(/\/+$/, "").replace(/\.$/, "");
+  return hostname && hostname.length > 0 ? hostname : null;
+}
+
+function selfOf(bag: Record<string, unknown>): Record<string, unknown> | null {
+  return typeof bag["Self"] === "object" && bag["Self"] !== null ? (bag["Self"] as Record<string, unknown>) : null;
 }
 
 /**
@@ -281,7 +379,11 @@ export function createRemoteSupportService(config: RemoteSupportServiceConfig):
       // Tailscale runs as a system service — its serve daemon owns its own
       // lifecycle, so a pid file is not reliable. Ask the daemon.
       if (tool === "tailscale") {
-        const url = tailscaleUrlFromDaemon();
+        // Both halves are required. A node that is up but serving nothing is
+        // not publishing an address, and reporting it as active left the
+        // toggle on and a stale link in place after the operator stopped it —
+        // `serve status` goes back to `{}` but the node stays `Online`.
+        const url = tailscaleServing() ? tailscaleUrlFromDaemon() : null;
         if (url) return { tool: "tailscale", url };
         continue;
       }
@@ -348,7 +450,14 @@ export function createRemoteSupportService(config: RemoteSupportServiceConfig):
           installed: await detectInstalled(),
         };
       }
-      return await readStatus();
+      const status = await readStatus();
+      // A publisher that started but published no address is the state that
+      // used to look like a silent success: the UI showed the tool as active
+      // with no link and no reason. Say what is missing instead.
+      if (status.active !== null && status.url === null) {
+        return { ...status, blocker: "not_ready", message: notReadyReason(status.active) };
+      }
+      return status;
     },
 
     async stop(tool) {

package/src/main.ts

@@ -1,169 +1,167 @@
-// Must run before any module that registers a route: it wires Elysia's TypeBox
-// bridge statically, which the source-run path needs under NODE_ENV=production.
-import "./boot/elysia-typebox-bridge";
-import { createGatewayApp, createGatewayShell } from "./boot/app";
-import { bootstrap } from "./boot/lifecycle";
-import type { CartethyiaBoot } from "./boot/lifecycle";
-import {
-  resolveDrainToken,
-  resolveIdleTimeout,
-  resolveMaxBodyBytes,
-  resolvePort,
-  resolveProcessTopology,
-} from "./config/runtime";
-import { Manifest } from "elysia";
-import { log } from "./observability/logger";
-
-const port = resolvePort();
-const entryMemory = process.memoryUsage();
-log.info(
-  `[startup] phase=entrypoint-ready elapsed_ms=${Math.round(process.uptime() * 1_000)} ` +
-    `rss_mb=${(entryMemory.rss / 1024 / 1024).toFixed(1)} ` +
-    `heap_used_mb=${(entryMemory.heapUsed / 1024 / 1024).toFixed(1)}`,
-);
-
-declare global {
-  // eslint-disable-next-line no-var -- globalThis augmentation requires `var`
-  var __cartethyiaBoot: CartethyiaBoot | undefined;
-  // eslint-disable-next-line no-var
-  var __cartethyiaSignalsRegistered: boolean | undefined;
-}
-
-const boot = Manifest.isCapturing()
-  ? undefined
-  : (globalThis.__cartethyiaBoot ??= await bootstrap());
-
-const app = boot
-  ? createGatewayApp({
-      mode: "production",
-      db: boot.deps.db,
-      proxyPreparer: boot.deps.proxyPreparer,
-      resolveProviderAdapter: boot.deps.resolveProviderAdapter,
-      byokUpstreamHosts: boot.deps.byokUpstreamHosts,
-      networkBindingFactory: boot.deps.networkBindingFactory,
-      ipAbuseProtection: boot.deps.ipAbuseProtection,
-      trustedProxyBoundary: boot.deps.trustedProxyBoundary,
-      poolSelector: boot.deps.poolSelector,
-      snapshotService: boot.deps.snapshotService,
-      readiness: boot.deps.readiness,
-      telemetryBuffer: boot.deps.telemetryBuffer,
-      resolveOAuthRefresher: boot.deps.resolveOAuthRefresher,
-      oauthRefreshService: boot.deps.oauthRefreshService,
-      modelStrikes: boot.deps.modelStrikes,
-      maxBodyBytes: resolveMaxBodyBytes(),
-      scheduledTasks: boot.deps.scheduledTasks,
-      // The sampler is constructed in the dependency graph and already
-      // sampling on its own interval; without passing it here the app cannot
-      // bind its in-flight source, so `cartethyia_process_in_flight` stays
-      // pinned at the constructor's `() => 0` placeholder forever while the
-      // registry it was meant to read reports real flights.
-      runtimeMetricsSampler: boot.deps.runtimeMetricsSampler,
-      shutdownCoordinator: boot.shutdownCoordinator,
-      // Signal-free graceful stop for platforms where a catchable signal cannot
-      // be delivered (Windows). Off unless `CARTETHYIA_DRAIN_TOKEN` is set.
-      ...(resolveDrainToken() !== undefined
-        ? { drainToken: resolveDrainToken() as string, triggerDrain: () => shutdown("SIGTERM") }
-        : {}),
-      // The console mounts in every boot: without REDIS_URL it runs on the
-      // in-process memory backend like the rest of the gateway.
-      consoleApi: {
-        db: boot.deps.db,
-        accessResolver: () => undefined,
-        routeSnapshotService: boot.deps.snapshotService,
-        poolSelector: boot.deps.poolSelector,
-        telemetryBuffer: boot.deps.telemetryBuffer,
-        providerRegistry: boot.deps.providerRegistry,
-        bundledModelCatalog: boot.deps.bundledModelCatalog,
-        networkBindingFactory: boot.deps.networkBindingFactory,
-        redis: boot.deps.redis,
-        oauthRefreshService: boot.deps.oauthRefreshService,
-        admissionService: boot.deps.admissionService,
-        modelStrikes: boot.deps.modelStrikes,
-        readRoutingAccountInflight: boot.deps.readRoutingAccountInflight,
-        // The same three writers the shutdown coordinator drains, so a manual
-        // flush from the console does exactly what a graceful stop would do
-        // for the queues, minus the stop.
-        flushBuffers: {
-          flush: async () => {
-            await boot.deps.telemetryBuffer.flush();
-            await boot.deps.usageFlush.flush();
-            await boot.deps.healthEventFlush.flush();
-          },
-        },
-      },
-    })
-  : createGatewayShell();
-export { app };
-
-function shutdown(signal: "SIGINT" | "SIGTERM", reason: "SIGINT" | "SIGTERM" | "update" = signal): void {
-  if (!boot) return;
-  log.info(`[shutdown] ${signal} received, draining...`);
-  // Derived from the coordinator's own budget, not a hand-kept literal: a
-  // force-exit shorter than the drain would hard-kill a process that was about
-  // to finish gracefully, truncating the in-flight responses the drain exists
-  // to protect.
-  const forceExit = setTimeout(() => {
-    log.error("[shutdown] forced exit after drain budget elapsed");
-    process.exit(1);
-  }, boot.shutdownCoordinator.totalShutdownBudgetMs());
-  forceExit.unref();
-  boot.shutdownCoordinator
-    .begin(reason)
-    .then(() => {
-      log.info("[shutdown] complete");
-      process.exit(0);
-    })
-    .catch((error: unknown) => {
-      log.error("[shutdown] failed", error as Error);
-      process.exit(1);
-    });
-}
-
-if (boot) {
-  boot.server = app.listen(
-    {
-      port,
-      // Cap idle sockets so a Slowloris/keep-alive-abuse client cannot
-      // pin one of the server's `maxRequestBodySize`-budgeted sockets
-      // indefinitely. 60 s aligns with the DB pool idle timeout so all
-      // long-idle resources reap together.
-      idleTimeout: resolveIdleTimeout(),
-      // SO_REUSEPORT load-balancing, so several processes can share the port
-      // without a reverse proxy. Tied to the declared topology rather than
-      // left always-on: with one process declared, enabling it lets an
-      // unrelated process on the same port silently take half the traffic —
-      // which is worse than a bind error, because nothing reports it.
-      reusePort: resolveProcessTopology().multiProcess,
-      // 8 MiB cap on request bodies keeps a rogue client from allocating
-      // gigabytes of memory in the ingress path before validation rejects.
-      maxRequestBodySize: resolveMaxBodyBytes(),
-    },
-    () => {
-      log.info(`Cartethyia listening on :${port} (Bun ${Bun.version})`);
-      const memory = process.memoryUsage();
-      log.info(
-        `[startup] phase=listener-ready rss_mb=${(memory.rss / 1024 / 1024).toFixed(1)} ` +
-          `heap_used_mb=${(memory.heapUsed / 1024 / 1024).toFixed(1)}`,
-      );
-    },
-  );
-  // Started here rather than inside the dependency builder: the first tick of
-  // the lease sweep and the health sweep should not run before the listener
-  // exists to serve traffic.
-  boot.deps.scheduledTasks.start();
-  if (!globalThis.__cartethyiaSignalsRegistered) {
-    globalThis.__cartethyiaSignalsRegistered = true;
-    process.on("SIGINT", () => shutdown("SIGINT"));
-    process.on("SIGTERM", () => shutdown("SIGTERM"));
-    // An in-place update signals SIGUSR2 before swapping the image so the old
-    // process drains with the `update` reason and its callers are told the
-    // replacement is seconds away instead of seeing a generic shutdown. Not
-    // available on Windows, where the listener is simply never registered —
-    // production runs in Linux containers, which is where an update happens.
-    try {
-      process.on("SIGUSR2", () => shutdown("SIGTERM", "update"));
-    } catch {
-      // Platform without SIGUSR2: update drains fall back to the SIGTERM notice.
-    }
-  }
-}
+// Must run before any module that registers a route: it wires Elysia's TypeBox
+// bridge statically, which the source-run path needs under NODE_ENV=production.
+import "./boot/elysia-typebox-bridge";
+import { createGatewayApp, createGatewayShell } from "./boot/app";
+import { bootstrap } from "./boot/lifecycle";
+import type { CartethyiaBoot } from "./boot/lifecycle";
+import {
+  resolveDrainToken,
+  resolveIdleTimeout,
+  resolveMaxBodyBytes,
+  resolvePort,
+  resolveProcessTopology,
+} from "./config/runtime";
+import { Manifest } from "elysia";
+import { log } from "./observability/logger";
+
+const port = resolvePort();
+const entryMemory = process.memoryUsage();
+log.info(
+  `[startup] phase=entrypoint-ready elapsed_ms=${Math.round(process.uptime() * 1_000)} ` +
+    `rss_mb=${(entryMemory.rss / 1024 / 1024).toFixed(1)} ` +
+    `heap_used_mb=${(entryMemory.heapUsed / 1024 / 1024).toFixed(1)}`,
+);
+
+declare global {
+  // eslint-disable-next-line no-var -- globalThis augmentation requires `var`
+  var __cartethyiaBoot: CartethyiaBoot | undefined;
+  // eslint-disable-next-line no-var
+  var __cartethyiaSignalsRegistered: boolean | undefined;
+}
+
+const boot = Manifest.isCapturing()
+  ? undefined
+  : (globalThis.__cartethyiaBoot ??= await bootstrap());
+
+const app = boot
+  ? createGatewayApp({
+      mode: "production",
+      db: boot.deps.db,
+      proxyPreparer: boot.deps.proxyPreparer,
+      resolveProviderAdapter: boot.deps.resolveProviderAdapter,
+      byokUpstreamHosts: boot.deps.byokUpstreamHosts,
+      networkBindingFactory: boot.deps.networkBindingFactory,
+      ipAbuseProtection: boot.deps.ipAbuseProtection,
+      trustedProxyBoundary: boot.deps.trustedProxyBoundary,
+      poolSelector: boot.deps.poolSelector,
+      snapshotService: boot.deps.snapshotService,
+      readiness: boot.deps.readiness,
+      telemetryBuffer: boot.deps.telemetryBuffer,
+      resolveOAuthRefresher: boot.deps.resolveOAuthRefresher,
+      oauthRefreshService: boot.deps.oauthRefreshService,
+      maxBodyBytes: resolveMaxBodyBytes(),
+      scheduledTasks: boot.deps.scheduledTasks,
+      // The sampler is constructed in the dependency graph and already
+      // sampling on its own interval; without passing it here the app cannot
+      // bind its in-flight source, so `cartethyia_process_in_flight` stays
+      // pinned at the constructor's `() => 0` placeholder forever while the
+      // registry it was meant to read reports real flights.
+      runtimeMetricsSampler: boot.deps.runtimeMetricsSampler,
+      shutdownCoordinator: boot.shutdownCoordinator,
+      // Signal-free graceful stop for platforms where a catchable signal cannot
+      // be delivered (Windows). Off unless `CARTETHYIA_DRAIN_TOKEN` is set.
+      ...(resolveDrainToken() !== undefined
+        ? { drainToken: resolveDrainToken() as string, triggerDrain: () => shutdown("SIGTERM") }
+        : {}),
+      // The console mounts in every boot: without REDIS_URL it runs on the
+      // in-process memory backend like the rest of the gateway.
+      consoleApi: {
+        db: boot.deps.db,
+        accessResolver: () => undefined,
+        routeSnapshotService: boot.deps.snapshotService,
+        poolSelector: boot.deps.poolSelector,
+        telemetryBuffer: boot.deps.telemetryBuffer,
+        providerRegistry: boot.deps.providerRegistry,
+        bundledModelCatalog: boot.deps.bundledModelCatalog,
+        networkBindingFactory: boot.deps.networkBindingFactory,
+        redis: boot.deps.redis,
+        oauthRefreshService: boot.deps.oauthRefreshService,
+        admissionService: boot.deps.admissionService,
+        readRoutingAccountInflight: boot.deps.readRoutingAccountInflight,
+        // The same three writers the shutdown coordinator drains, so a manual
+        // flush from the console does exactly what a graceful stop would do
+        // for the queues, minus the stop.
+        flushBuffers: {
+          flush: async () => {
+            await boot.deps.telemetryBuffer.flush();
+            await boot.deps.usageFlush.flush();
+            await boot.deps.healthEventFlush.flush();
+          },
+        },
+      },
+    })
+  : createGatewayShell();
+export { app };
+
+function shutdown(signal: "SIGINT" | "SIGTERM", reason: "SIGINT" | "SIGTERM" | "update" = signal): void {
+  if (!boot) return;
+  log.info(`[shutdown] ${signal} received, draining...`);
+  // Derived from the coordinator's own budget, not a hand-kept literal: a
+  // force-exit shorter than the drain would hard-kill a process that was about
+  // to finish gracefully, truncating the in-flight responses the drain exists
+  // to protect.
+  const forceExit = setTimeout(() => {
+    log.error("[shutdown] forced exit after drain budget elapsed");
+    process.exit(1);
+  }, boot.shutdownCoordinator.totalShutdownBudgetMs());
+  forceExit.unref();
+  boot.shutdownCoordinator
+    .begin(reason)
+    .then(() => {
+      log.info("[shutdown] complete");
+      process.exit(0);
+    })
+    .catch((error: unknown) => {
+      log.error("[shutdown] failed", error as Error);
+      process.exit(1);
+    });
+}
+
+if (boot) {
+  boot.server = app.listen(
+    {
+      port,
+      // Cap idle sockets so a Slowloris/keep-alive-abuse client cannot
+      // pin one of the server's `maxRequestBodySize`-budgeted sockets
+      // indefinitely. 60 s aligns with the DB pool idle timeout so all
+      // long-idle resources reap together.
+      idleTimeout: resolveIdleTimeout(),
+      // SO_REUSEPORT load-balancing, so several processes can share the port
+      // without a reverse proxy. Tied to the declared topology rather than
+      // left always-on: with one process declared, enabling it lets an
+      // unrelated process on the same port silently take half the traffic —
+      // which is worse than a bind error, because nothing reports it.
+      reusePort: resolveProcessTopology().multiProcess,
+      // 8 MiB cap on request bodies keeps a rogue client from allocating
+      // gigabytes of memory in the ingress path before validation rejects.
+      maxRequestBodySize: resolveMaxBodyBytes(),
+    },
+    () => {
+      log.info(`Cartethyia listening on :${port} (Bun ${Bun.version})`);
+      const memory = process.memoryUsage();
+      log.info(
+        `[startup] phase=listener-ready rss_mb=${(memory.rss / 1024 / 1024).toFixed(1)} ` +
+          `heap_used_mb=${(memory.heapUsed / 1024 / 1024).toFixed(1)}`,
+      );
+    },
+  );
+  // Started here rather than inside the dependency builder: the first tick of
+  // the lease sweep and the health sweep should not run before the listener
+  // exists to serve traffic.
+  boot.deps.scheduledTasks.start();
+  if (!globalThis.__cartethyiaSignalsRegistered) {
+    globalThis.__cartethyiaSignalsRegistered = true;
+    process.on("SIGINT", () => shutdown("SIGINT"));
+    process.on("SIGTERM", () => shutdown("SIGTERM"));
+    // An in-place update signals SIGUSR2 before swapping the image so the old
+    // process drains with the `update` reason and its callers are told the
+    // replacement is seconds away instead of seeing a generic shutdown. Not
+    // available on Windows, where the listener is simply never registered —
+    // production runs in Linux containers, which is where an update happens.
+    try {
+      process.on("SIGUSR2", () => shutdown("SIGTERM", "update"));
+    } catch {
+      // Platform without SIGUSR2: update drains fall back to the SIGTERM notice.
+    }
+  }
+}