cartethyia 2.4.5 → 2.5.0

package/src/console/updates/repository-updates.ts

@@ -1,34 +1,39 @@
-/**
- * Public release discovery for the console's update indicator.
- *
- * The browser never talks to GitHub directly: a dashboard-side fetch would put
- * every console user's GitHub rate limit behind one shared, unauthenticated
- * origin, and a CORS or rate-limit failure would surface as a console error
- * nobody can act on. The request is made here, server-side, and any failure
- * degrades to "no update known" rather than an error the operator must read.
- */
-const REPOSITORY = "risunCode/Cartethyia";
-
-/** Timeout for one GitHub read. A hung read must not hold a console query open. */
-const REQUEST_TIMEOUT_MS = 10_000;
-
-export interface LatestRelease {
-  readonly tag_name: string;
-  readonly html_url: string;
-}
-
-/** Fetches the latest public release. Returns `null` when none is known. */
-export async function fetchLatestRelease(): Promise<LatestRelease | null> {
-  try {
-    const response = await fetch(`https://api.github.com/repos/${REPOSITORY}/releases/latest`, {
-      headers: { accept: "application/vnd.github+json", "user-agent": "Cartethyia-updater" },
-      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
-    });
-    if (!response.ok) return null;
-    const value = (await response.json()) as { readonly tag_name?: unknown; readonly html_url?: unknown };
-    if (typeof value.tag_name !== "string" || typeof value.html_url !== "string") return null;
-    return { tag_name: value.tag_name, html_url: value.html_url };
-  } catch {
-    return null;
-  }
-}
+/**
+ * Public release discovery for the console's update indicator.
+ *
+ * The browser never talks to GitHub directly: a dashboard-side fetch would put
+ * every console user's GitHub rate limit behind one shared, unauthenticated
+ * origin, and a CORS or rate-limit failure would surface as a console error
+ * nobody can act on. The request is made here, server-side, and any failure
+ * degrades to "no update known" rather than an error the operator must read.
+ *
+ * Releases are read from the ALTERNATIVE repository (`risunCodex`), not the
+ * primary: the primary repository is hidden from public view, so its releases
+ * endpoint answers 404 and the console would report "no update known" forever —
+ * a silent failure that reads exactly like "you are up to date".
+ */
+const REPOSITORY = "risunCodex/Cartethyia";
+
+/** Timeout for one GitHub read. A hung read must not hold a console query open. */
+const REQUEST_TIMEOUT_MS = 10_000;
+
+export interface LatestRelease {
+  readonly tag_name: string;
+  readonly html_url: string;
+}
+
+/** Fetches the latest public release. Returns `null` when none is known. */
+export async function fetchLatestRelease(): Promise<LatestRelease | null> {
+  try {
+    const response = await fetch(`https://api.github.com/repos/${REPOSITORY}/releases/latest`, {
+      headers: { accept: "application/vnd.github+json", "user-agent": "Cartethyia-updater" },
+      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+    });
+    if (!response.ok) return null;
+    const value = (await response.json()) as { readonly tag_name?: unknown; readonly html_url?: unknown };
+    if (typeof value.tag_name !== "string" || typeof value.html_url !== "string") return null;
+    return { tag_name: value.tag_name, html_url: value.html_url };
+  } catch {
+    return null;
+  }
+}

package/src/console/updates/routes.ts

@@ -1,6 +1,73 @@
 import { Elysia } from "elysia";
-import { consoleErrorHandler } from "../shared/errors";
-import { fetchLatestRelease } from "./repository-updates";
+import { consoleErrorHandler, requireTenantScope } from "../shared/errors";
+import { consoleSseResponse, createConsoleSseStream } from "../observability/sse";
+import type { ObservabilityStore } from "../observability/contracts";
+import type { ConsoleAccessResolver } from "../auth/access";
+import { fetchLatestRelease, type LatestRelease } from "./repository-updates";
+import { isNewerRelease } from "./version-compare";
+
+const MEMORY_SAMPLE_INTERVAL_MS = 10_000;
+const UPDATE_SAMPLE_INTERVAL_MS = 60_000;
+const HIGH_MEMORY_PERCENT = 85;
+const HIGH_MEMORY_CLEAR_PERCENT = 80;
+const MEMORY_GROWTH_WINDOW_MS = 60_000;
+const MEMORY_GROWTH_RATIO = 1.2;
+const MAX_MEMORY_SAMPLES = 8;
+const RELEASES_URL = "https://github.com/risunCodex/Cartethyia/releases";
+
+export interface NotificationEvent {
+  readonly id: string;
+  readonly kind: "memory-high" | "memory-growth" | "update-available";
+  readonly title: string;
+  readonly message: string;
+  readonly createdAt: string;
+  readonly level: "warning" | "info";
+  readonly href?: string;
+}
+
+export function memoryNotifications(
+  samples: readonly { readonly at: number; readonly rss: number; readonly percent: number }[],
+  previous: { high: boolean; growth: boolean },
+): { notifications: NotificationEvent[]; high: boolean; growth: boolean } {
+  const latest = samples.at(-1);
+  if (latest === undefined) return { notifications: [], ...previous };
+  const notifications: NotificationEvent[] = [];
+  const high = previous.high
+    ? latest.percent >= HIGH_MEMORY_CLEAR_PERCENT
+    : latest.percent >= HIGH_MEMORY_PERCENT;
+  if (high && !previous.high) {
+    notifications.push({
+      id: "memory-high",
+      kind: "memory-high",
+      title: "High gateway memory",
+      message: `Gateway RSS is using ${latest.percent.toFixed(1)}% of system memory.`,
+      createdAt: new Date(latest.at).toISOString(),
+      level: "warning",
+    });
+  }
+
+  const oldest = samples.find((sample) => latest.at - sample.at >= MEMORY_GROWTH_WINDOW_MS);
+  const growth = oldest !== undefined &&
+    latest.rss >= oldest.rss * MEMORY_GROWTH_RATIO &&
+    samples.filter((sample) => sample.at >= oldest.at).every((sample, index, window) =>
+      index === 0 || sample.rss >= (window[index - 1]?.rss ?? sample.rss),
+    );
+  if (growth && !previous.growth && !high) {
+    notifications.push({
+      id: "memory-growth",
+      kind: "memory-growth",
+      title: "Sustained memory growth",
+      message: `Gateway RSS grew ${Math.round((latest.rss / oldest.rss - 1) * 100)}% over the last minute.`,
+      createdAt: new Date(latest.at).toISOString(),
+      level: "warning",
+    });
+  }
+  return {
+    notifications,
+    high: high && latest.percent >= HIGH_MEMORY_CLEAR_PERCENT,
+    growth,
+  };
+}
 
 /**
  * Read-only update discovery for the console sidebar.
@@ -10,8 +77,99 @@ import { fetchLatestRelease } from "./repository-updates";
  * answers `null` rather than an error: an absent update badge is the honest
  * result when the upstream cannot be reached.
  */
-export function createUpdateRoutes(): Elysia {
+export function createUpdateRoutes(config: {
+  readonly accessResolver: ConsoleAccessResolver;
+  readonly observabilityStore: ObservabilityStore;
+  readonly fetchRelease?: () => Promise<LatestRelease | null>;
+}): Elysia {
+  const fetchRelease = config.fetchRelease ?? fetchLatestRelease;
   return new Elysia()
     .error(consoleErrorHandler("Update check failed"))
-    .get("/updates/release", async () => await fetchLatestRelease()) as unknown as Elysia;
+    .get("/updates/release", async () => await fetchRelease())
+    .get("/notifications/stream", ({ request }) => {
+      const tenantId = requireTenantScope(
+        config.accessResolver(request),
+        "dashboard:read",
+      ).tenantId;
+      return consoleSseResponse(
+        createConsoleSseStream(request.signal, (sender) => {
+          let closed = false;
+          let memoryTimer: ReturnType<typeof setTimeout> | undefined;
+          let updateTimer: ReturnType<typeof setTimeout> | undefined;
+          let latestTag: string | undefined;
+          let memoryState = { high: false, growth: false };
+          let samples: Array<{ at: number; rss: number; percent: number }> = [];
+
+          const publishMemory = async (): Promise<void> => {
+            if (closed) return;
+            try {
+              const at = Date.now();
+              // Fresh, not `health`: that read is cached for 10 s and this
+              // samples on that exact cadence, so the cached bytes would make
+              // consecutive samples identical and a growth trend unreadable.
+              const live = await config.observabilityStore.liveMemory();
+              samples = [
+                ...samples,
+                { at, rss: live.memory_bytes, percent: live.memory_percent },
+              ].filter((sample) => at - sample.at <= MEMORY_GROWTH_WINDOW_MS * 2)
+                .slice(-MAX_MEMORY_SAMPLES);
+              const computed = memoryNotifications(samples, memoryState);
+              memoryState = { high: computed.high, growth: computed.growth };
+              for (const notification of computed.notifications) sender.send("notification", notification);
+            } catch {
+              // A transient check must not terminate the feed or hide later events.
+            }
+          };
+
+          const publishUpdate = async (): Promise<void> => {
+            if (closed) return;
+            try {
+              const release = await fetchRelease();
+              if (release === null || release.tag_name === latestTag) return;
+              const health = (await config.observabilityStore.health(tenantId)).value;
+              if (!isNewerRelease(health.version, release.tag_name)) return;
+              latestTag = release.tag_name;
+              sender.send("notification", {
+                id: `update-${release.tag_name}`,
+                kind: "update-available",
+                title: "Cartethyia update available",
+                message: `Version ${release.tag_name} is available; this gateway runs ${health.version}.`,
+                createdAt: new Date().toISOString(),
+                level: "info",
+                href: RELEASES_URL,
+              } satisfies NotificationEvent);
+            } catch {
+              // Release discovery is best-effort and cannot interrupt memory sampling.
+            }
+          };
+
+          const scheduleMemory = (): void => {
+            memoryTimer = setTimeout(() => {
+              void publishMemory().finally(() => {
+                if (!closed) scheduleMemory();
+              });
+            }, MEMORY_SAMPLE_INTERVAL_MS);
+          };
+          const scheduleUpdate = (): void => {
+            updateTimer = setTimeout(() => {
+              void publishUpdate().finally(() => {
+                if (!closed) scheduleUpdate();
+              });
+            }, UPDATE_SAMPLE_INTERVAL_MS);
+          };
+
+          void publishMemory().finally(() => {
+            if (!closed) scheduleMemory();
+          });
+          void publishUpdate().finally(() => {
+            if (!closed) scheduleUpdate();
+          });
+          return () => {
+            closed = true;
+            clearTimeout(memoryTimer);
+            clearTimeout(updateTimer);
+          };
+        }),
+      );
+    }) as unknown as Elysia;
 }

package/src/main.ts

@@ -1,3 +1,6 @@
+// Must run before any module that registers a route: it wires Elysia's TypeBox
+// bridge statically, which the source-run path needs under NODE_ENV=production.
+import "./boot/elysia-typebox-bridge";
 import { createGatewayApp, createGatewayShell } from "./boot/app";
 import { bootstrap } from "./boot/lifecycle";
 import type { CartethyiaBoot } from "./boot/lifecycle";

package/src/network/http2-bidi.ts

@@ -0,0 +1,85 @@
+// Transport-neutral bidirectional HTTP/2 egress contract.
+//
+// Cursor's Run is a Connect-RPC bidi stream: one long-lived writable request
+// whose response body is an unbounded stream of protobuf frames. Neither
+// `fetch` nor the WebSocket session contract can express it — `fetch` needs the
+// whole body up front, and a WebSocket is a different protocol — so the egress
+// boundary gets its own contract.
+//
+// This module is transport-neutral on purpose: it declares *what* a validated
+// bidi stream is, and `network/pool/resolver.ts` decides *how* one is dialed
+// (direct TLS to a pinned, SSRF-validated address, or through the selected
+// tenant pool's CONNECT/SOCKS tunnel). A provider adapter consumes the contract
+// and never opens a socket of its own.
+
+/** Response metadata of one opened bidi stream. */
+export interface Http2BidiResponse {
+  /** HTTP/2 `:status`; `0` before the response headers arrive. */
+  readonly status: number;
+  readonly headers: Readonly<Record<string, string>>;
+}
+
+/** Terminal reason a bidi stream ended. */
+export type Http2BidiEndReason =
+  /** The server closed the stream cleanly. */
+  | { readonly kind: "remote-clean-close" }
+  /** The server failed it: an HTTP status, a gRPC trailer, or a transport error. */
+  | { readonly kind: "remote-error"; readonly error: Error }
+  /** The client closed or aborted it. */
+  | { readonly kind: "local-close" };
+
+/**
+ * One validated bidirectional HTTP/2 stream.
+ *
+ * Bytes in are an async iterable (so a provider pumps frames naturally and
+ * backpressure comes from the consumer); bytes out are `write`, which resolves
+ * once the chunk has drained — a bidi Run can carry megabytes, and an
+ * unawaited write is how a stream overruns its session memory and dies with
+ * `ENHANCE_YOUR_CALM`.
+ */
+export interface ValidatedHttp2BidiStream {
+  readonly response: Http2BidiResponse;
+  /** Response trailers, populated when the server sends them (gRPC status). */
+  readonly trailers: Readonly<Record<string, string>>;
+  /** Response body bytes, in arrival order. */
+  readonly body: AsyncIterable<Uint8Array>;
+  /** Writes one request chunk, resolving when it has drained. */
+  write(chunk: Uint8Array): Promise<void>;
+  /** Half-closes the request side; the response may still be streaming. */
+  end(): void;
+  /** Tears the stream down immediately. Idempotent. */
+  abort(reason?: unknown): void;
+  /** Registers a terminal listener; the returned function unregisters it. */
+  onTerminal(listener: (reason: Http2BidiEndReason) => void): () => void;
+}
+
+/** Options for opening one validated bidi stream. */
+export interface Http2BidiOpenOptions {
+  readonly signal?: AbortSignal;
+  /** Bytes read idle before the stream is declared dead. */
+  readonly readIdleMs?: number;
+  /** Upper bound on buffered inbound bytes before the stream is refused. */
+  readonly maxBufferedBytes?: number;
+  /**
+   * Host allowlist applied on every new connection, in addition to the SSRF
+   * policy. A long-lived stream carries the account credential for its whole
+   * life, so "is this host one the caller may reach at all" is a per-provider
+   * decision the binding enforces rather than trusts the adapter to apply.
+   */
+  readonly allowHostname?: (hostname: string) => boolean;
+}
+
+/**
+ * The dialer contract a provider adapter is handed.
+ *
+ * Declared here rather than in the provider layer so the dependency runs one
+ * way: `provider-contracts` imports this module, never the reverse. It used to
+ * be declared there and re-exported from here, which made the two import each
+ * other — type-only, so it never deadlocked, but it meant this
+ * transport-neutral contract had to know about providers.
+ */
+export type ValidatedOutboundHttp2Bidi = (
+  url: URL,
+  headers: Readonly<Record<string, string>>,
+  options?: Http2BidiOpenOptions,
+) => Promise<ValidatedHttp2BidiStream>;

package/src/network/outbound-fetch.ts

@@ -34,6 +34,57 @@ function agentForTarget(
   return agent;
 }
 
+/**
+ * Keep-alive options for the direct (non-pool) egress agents.
+ *
+ * Matched to `network/pool/agent.ts`, which builds pool agents from
+ * `resolveProxyMaxSockets()` / `resolveProxyMaxFreeSockets()` and
+ * `PROXY_KEEP_ALIVE_TIMEOUT_MS`. Direct egress has no per-pool config to
+ * read, so it uses the documented defaults those resolvers return.
+ */
+const DIRECT_AGENT_OPTIONS = {
+  keepAlive: true,
+  maxSockets: 100,
+  maxFreeSockets: 20,
+  timeout: 60_000,
+} as const;
+
+/**
+ * An agent whose socket key includes the SSRF-validated address.
+ *
+ * The direct (non-pool) path pins egress by passing a `lookup` that returns the
+ * one validated address, but `Agent#getName` — the keep-alive socket key —
+ * hashes only host/port/localAddress/family. Two requests to the same
+ * `host:port` that validated to *different* addresses therefore share a pooled
+ * socket, and the second one silently egresses to the first one's address:
+ * the pin is bypassed without any error surfacing.
+ *
+ * Including the address in the key makes each validated address its own socket
+ * bucket, so a re-resolved address always dials fresh while the *same* address
+ * still reuses its socket (keep-alive is preserved, not defeated).
+ *
+ * This is a real gap, not a theoretical one: with no agent at all the request
+ * falls back to the global agent, which has the same key shape *and* is shared
+ * with unrelated traffic.
+ */
+class PinnedAddressAgent extends HttpAgent {
+  override getName(options: Record<string, unknown> = {}): string {
+    const base = super.getName(options as never);
+    const pinned = (options["lookup"] as { pinnedAddress?: string } | undefined)?.pinnedAddress;
+    return `${base}:${pinned ?? "-"}`;
+  }
+}
+class PinnedAddressHttpsAgent extends HttpsAgent {
+  override getName(options: Record<string, unknown> = {}): string {
+    const base = super.getName(options as never);
+    const pinned = (options["lookup"] as { pinnedAddress?: string } | undefined)?.pinnedAddress;
+    return `${base}:${pinned ?? "-"}`;
+  }
+}
+
+const DIRECT_HTTP_AGENT = new PinnedAddressAgent(DIRECT_AGENT_OPTIONS);
+const DIRECT_HTTPS_AGENT = new PinnedAddressHttpsAgent(DIRECT_AGENT_OPTIONS);
+
 /**
  * Builds the application-relay request used by hosted relay front doors.
  * These endpoints intentionally do not receive CONNECT: they accept the
@@ -99,7 +150,8 @@ function decompressBody(
   return { stream: body.pipe(decoder), encoding: undefined };
 }
 
-async function pinnedFetch(  url: URL,
+async function pinnedFetch(
+  url: URL,
   init: RequestInit,
   resolvedAddress: string | undefined,
   agent?: HttpAgent | HttpsAgent,
@@ -126,27 +178,38 @@ async function pinnedFetch(  url: URL,
     outboundHeaders["content-length"] =
       typeof body === "string" ? String(Buffer.byteLength(body, "utf8")) : String(body.byteLength);
   }
-  const lookup = ((
-    _hostname: string,
-    options: { all?: boolean; family?: number } | number,
-    callback: (
-      error: Error | null,
-      address?: string | Array<{ address: string; family: number }>,
-      family?: number,
-    ) => void,
-  ): void => {
-    if (resolvedAddress === undefined) {
-      callback(new Error("no validated address available"));
-      return;
-    }
-    const opts = typeof options === "number" ? { family: options } : options;
-    const family = resolvedAddress.includes(":") ? 6 : 4;
-    if (opts?.all) {
-      callback(null, [{ address: resolvedAddress, family }] as never);
-      return;
-    }
-    callback(null, resolvedAddress, family);
-  }) as unknown as NonNullable<RequestOptions["lookup"]>;
+  const lookup = Object.assign(
+    (
+      _hostname: string,
+      options: { all?: boolean; family?: number } | number,
+      callback: (
+        error: Error | null,
+        address?: string | Array<{ address: string; family: number }>,
+        family?: number,
+      ) => void,
+    ): void => {
+      if (resolvedAddress === undefined) {
+        callback(new Error("no validated address available"));
+        return;
+      }
+      const opts = typeof options === "number" ? { family: options } : options;
+      const family = resolvedAddress.includes(":") ? 6 : 4;
+      if (opts?.all) {
+        callback(null, [{ address: resolvedAddress, family }] as never);
+        return;
+      }
+      callback(null, resolvedAddress, family);
+    },
+    // Read by `PinnedAddressAgent#getName` so the keep-alive socket key
+    // carries the validated address. See the class comment for why.
+    { pinnedAddress: resolvedAddress },
+  ) as unknown as NonNullable<RequestOptions["lookup"]>;
+  // Direct egress gets a dedicated agent as well as the pinning `lookup`.
+  // Passing only `lookup` left the request on the global agent, whose socket
+  // key ignores `lookup` — so a re-resolved address could reuse a socket
+  // dialed against an earlier one and egress past the SSRF check. The direct
+  // agent keys its sockets on the pinned address, so the pin holds.
+  const directAgent = url.protocol === "https:" ? DIRECT_HTTPS_AGENT : DIRECT_HTTP_AGENT;
   return new Promise<Response>((resolve, reject) => {
     const nodeRequest = request(
       {
@@ -157,7 +220,7 @@ async function pinnedFetch(  url: URL,
         method: init.method ?? "GET",
         headers: outboundHeaders,
         signal: init.signal ?? undefined,
-        ...(agent ? { agent } : { lookup }),
+        ...(agent ? { agent } : { agent: directAgent, lookup }),
         ...(url.protocol === "https:" ? { servername: url.hostname } : {}),
       },
       (response) => {
@@ -230,7 +293,7 @@ export function createValidatedFetch(options: ValidatedFetchOptions = {}): Valid
         // The egress dials the pool/relay, which resolves the target itself.
         // Counting these lets ops confirm how much traffic depends on
         // proxy-resolved DNS rather than the local system resolver.
-        metrics.cartethyia_proxy_dial_dns_fallback_total.inc();
+        metrics.cartethyia_proxy_dial_proxy_resolved_total.inc();
       }
       try {
         if (!poolDialsTarget) {