cartethyia 2.5.3 → 2.5.6

package/src/console/routing/pools/store.ts

@@ -166,8 +166,8 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
       ...(row.lastSpeedtestAt ? { lastSpeedtestAt: row.lastSpeedtestAt.toISOString() } : {}),
       ...(() => {
         // A verdict describes the address recorded beside it. Once the pool is
-        // re-pointed, that address is gone and the verdict is a claim about a
-        // proxy that no longer exists — report "never probed" instead of
+        // re-pointed, that address is gone and the verdict is a claim about an
+        // address the pool no longer uses — report "never probed" instead of
         // showing a colour the operator cannot trust.
         const probe = storedProviderProbe(row, endpoint ?? "");
         return probe === undefined ? {} : { providerProbe: probe };
@@ -350,10 +350,10 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
   /**
    * Banks a probe verdict on the pool row.
    *
-   * The verdict used to live in the operator's browser, which made it
-   * decorative: another machine saw nothing, the gateway saw nothing, and
-   * admission kept choosing the pool the badge called blocked. Storing it here
-   * is what makes the badge a fact about the pool rather than about one tab.
+   * The verdict is stored here rather than per-viewer so every reader agrees:
+   * another machine, the gateway, and the operator alike. Otherwise admission
+   * keeps choosing the pool the badge calls blocked. Storing it here is what
+   * makes the badge a fact about the pool rather than about one tab.
    *
    * `poolId` is the pool row; `endpoint` is what the verdict was measured
    * against, so a later edit invalidates it on read. A probe against an unsaved
@@ -458,8 +458,8 @@ export class DrizzleNetworkPoolStore implements NetworkPoolStore {
       .set({
         endpointConfig,
         kind: effectiveKind === "https" ? "http" : effectiveKind,
-        // Fields default to `undefined` when omitted from the patch; that
-        // used to overwrite the current row with `undefined` and clobber
+        // Fields default to `undefined` when omitted from the patch, which
+        // would overwrite the current row with `undefined` and clobber
         // existing values. Fall back to the current record so a partial
         // PATCH truly leaves un-mentioned fields alone.
         maxInflight: patch.maxInflight ?? current.maxInflight,

package/src/console/settings/contracts.ts

@@ -7,6 +7,10 @@ import { t } from "elysia";
 import type { AccessDecision } from "../../security/access-control";
 import type { RedisBackend } from "../../persistence/redis";
 import type { ConsoleSettingsPreferences } from "../../persistence/schema";
+import { RTK_LEVELS } from "../../transport/request/rtk/rtk-constants";
+import { PONYTAIL_LEVELS } from "../../transport/request/rtk/ponytail-prompt";
+
+export { RTK_LEVELS, PONYTAIL_LEVELS };
 
 export interface RuntimeSettingsResponse {
   readonly redisBackendActual: RedisBackend;
@@ -17,6 +21,8 @@ export interface RuntimeSettingsResponse {
   readonly telemetryPayloads: TelemetryPayloadMode;
   /** Capture depth for `full` mode; `minimum` when unset. */
   readonly telemetryPayloadDepth: TelemetryPayloadDepth;
+  /** How long a captured frame is kept, in minutes; `null` means deployment default. */
+  readonly telemetryPayloadRetentionMinutes: number | null;
   readonly privacyMode: PrivacyMode;
   /** Compacts bulky tool-result text before dispatch (RTK prune). Default false. */
   readonly rtkPruneEnabled: boolean;
@@ -26,6 +32,19 @@ export interface RuntimeSettingsResponse {
   readonly ponyTailEnabled: boolean;
   /** PonyTail directive intensity; `full` by default. */
   readonly ponyTailLevel: PonyTailLevel;
+  /**
+   * Ignores an account-wide cooldown when building route candidates. Off by
+   * default: dialing a cooled-down account reproduces the refusal that parked
+   * it, and can exhaust the candidate list. Overrides nothing else — the
+   * cooldown is still recorded and still recovered by the sweep.
+   */
+  readonly forceDisableCooldown: boolean;
+  /** Per-tenant upstream request timeout in seconds; null means deployment default. */
+  readonly upstreamTimeoutSeconds: number | null;
+  /** Per-tenant telemetry compaction age in days; null means deployment default. */
+  readonly telemetryWindowMinutes: number | null;
+  /** Per-tenant inline attachment ceiling in bytes; null means deployment default. */
+  readonly imageClampMaxBytes: number | null;
   readonly updatedAt: string;
 }
 
@@ -50,6 +69,17 @@ export interface RuntimeSettingsConfig {
   readonly store: RuntimeSettingsStore;
   readonly accessResolver: ConsoleAccessResolver;
   readonly auditSink?: AuditSink;
+  /**
+   * Structurally satisfied by `InMemoryRouteSnapshotService`.
+   *
+   * Two preferences here are read on the request path, not per request: the
+   * route snapshot reads `forceDisableCooldown` when it builds candidates, and
+   * the admission path reads `tenantConcurrencyLimit`. The snapshot is cached
+   * until something invalidates it, so without this a saved setting stays
+   * invisible to `/v1/*` until some unrelated console write happens to
+   * invalidate — the toggle looks broken while the row is already correct.
+   */
+  readonly snapshotInvalidator?: { invalidate(): unknown };
 }
 
 
@@ -62,13 +92,25 @@ export type TelemetryPayloadMode = (typeof TELEMETRY_PAYLOAD_MODES)[number];
 export const TELEMETRY_PAYLOAD_DEPTHS = ["minimum", "moderate", "maximum"] as const;
 /** Capture depth for `full` mode accepted by the runtime-settings PATCH. */
 export type TelemetryPayloadDepth = (typeof TELEMETRY_PAYLOAD_DEPTHS)[number];
+/**
+ * How long a captured payload frame is kept, in whole minutes, or `null` for
+ * the deployment default.
+ *
+ * Only the offered values are accepted, and a value off the list is *rejected*
+ * rather than clamped: clamping would answer a request for thirty days with
+ * seven and report success, and an operator who asked for the longer window
+ * would believe they got it.
+ */
+export const TELEMETRY_PAYLOAD_RETENTION_MINUTES = [15, 30, 60, 360, 720, 1440, 10080] as const;
+/** Payload retention in minutes accepted by the runtime-settings PATCH. */
+export type TelemetryPayloadRetentionMinutes =
+  | (typeof TELEMETRY_PAYLOAD_RETENTION_MINUTES)[number]
+  | null;
 export const PRIVACY_MODES = ["masked", "full"] as const;
 /** Provider/model label privacy accepted by the runtime-settings PATCH. */
 export type PrivacyMode = (typeof PRIVACY_MODES)[number];
-export const PONYTAIL_LEVELS = ["lite", "full", "ultra"] as const;
 /** PonyTail directive intensity accepted by the runtime-settings PATCH. */
 export type PonyTailLevel = (typeof PONYTAIL_LEVELS)[number];
-export const RTK_LEVELS = ["lite", "full", "ultra"] as const;
 /** RTK prune strength accepted by the runtime-settings PATCH. */
 export type RtkLevel = (typeof RTK_LEVELS)[number];
 
@@ -107,6 +149,19 @@ export function createRuntimeSettingsOperations(deps: RuntimeSettingsConfig) {
         if (patch.telemetryPayloadDepth !== undefined && !TELEMETRY_PAYLOAD_DEPTHS.includes(patch.telemetryPayloadDepth)) {
           throw new ConsoleDomainError("invalid_request", 400, "Invalid telemetryPayloadDepth");
         }
+        if (patch.telemetryPayloadRetentionMinutes !== undefined) {
+          const minutes = patch.telemetryPayloadRetentionMinutes;
+          if (
+            minutes !== null &&
+            !(TELEMETRY_PAYLOAD_RETENTION_MINUTES as readonly number[]).includes(minutes)
+          ) {
+            throw new ConsoleDomainError(
+              "invalid_request",
+              400,
+              "Invalid telemetryPayloadRetentionMinutes",
+            );
+          }
+        }
         if (patch.privacyMode !== undefined && !PRIVACY_MODES.includes(patch.privacyMode)) {
           throw new ConsoleDomainError("invalid_request", 400, "Invalid privacyMode");
         }
@@ -127,6 +182,14 @@ export function createRuntimeSettingsOperations(deps: RuntimeSettingsConfig) {
           target: a.tenantId,
           detail: patch as Record<string, unknown>,
         });
+        // Saved preferences are routing changes: `forceDisableCooldown` decides
+        // candidate eligibility and `tenantConcurrencyLimit` decides admission,
+        // and both are read from the cached route snapshot. Invalidate after the
+        // write commits so the next `/v1/*` request uses them instead of the
+        // revision that was built before the save. A failure here must not fail
+        // the save — the row is already correct, and the snapshot rebuilds on
+        // the next read anyway.
+        await Promise.resolve(deps.snapshotInvalidator?.invalidate()).catch(() => undefined);
         return updated;
       },
   };
@@ -143,6 +206,12 @@ export const runtimeUpdateBody = t.Object({
   responsesReasoningSummary: t.Optional(literalUnion(RESPONSES_REASONING_SUMMARIES)),
   telemetryPayloads: t.Optional(literalUnion(TELEMETRY_PAYLOAD_MODES)),
   telemetryPayloadDepth: t.Optional(literalUnion(TELEMETRY_PAYLOAD_DEPTHS)),
+  telemetryPayloadRetentionMinutes: t.Optional(
+    t.Union([
+      t.Null(),
+      t.Union(TELEMETRY_PAYLOAD_RETENTION_MINUTES.map((minutes) => t.Literal(minutes))),
+    ]),
+  ),
   privacyMode: t.Optional(literalUnion(PRIVACY_MODES)),
   tenantConcurrencyLimit: t.Optional(t.Union([t.Null(), t.Number()])),
   thinkingNormalizationEnabled: t.Optional(t.Boolean()),
@@ -150,4 +219,28 @@ export const runtimeUpdateBody = t.Object({
   rtkPruneLevel: t.Optional(literalUnion(RTK_LEVELS)),
   ponyTailEnabled: t.Optional(t.Boolean()),
   ponyTailLevel: t.Optional(literalUnion(PONYTAIL_LEVELS)),
+  /*
+   * Operational overrides. Every field a caller may patch must appear here:
+   * this schema is what the route parses the body against, and a key it does
+   * not name is dropped before the store sees it. These four were added to the
+   * request type and the store but never to this object, so a save returned
+   * 200 and changed nothing — the mutation's toast fired while the stored
+   * value stayed put.
+   *
+   * `t.Null()` is what "Deployment default" sends, and it must survive:
+   * clearing an override is a real edit, not an absent one.
+   *
+   * The bounds mirror the store's own normalizers, so an out-of-range value is
+   * rejected here rather than silently clamped on the way in.
+   */
+  forceDisableCooldown: t.Optional(t.Boolean()),
+  upstreamTimeoutSeconds: t.Optional(
+    t.Union([t.Null(), t.Number({ minimum: 5, maximum: 600 })]),
+  ),
+  telemetryWindowMinutes: t.Optional(
+    t.Union([t.Null(), t.Number({ minimum: 1, maximum: 1440 })]),
+  ),
+  imageClampMaxBytes: t.Optional(
+    t.Union([t.Null(), t.Number({ minimum: 64 * 1024, maximum: 100 * 1024 * 1024 })]),
+  ),
 });

package/src/console/settings/store.ts

@@ -11,6 +11,7 @@ import {
   RTK_LEVELS,
   TELEMETRY_PAYLOAD_DEPTHS,
   TELEMETRY_PAYLOAD_MODES,
+  TELEMETRY_PAYLOAD_RETENTION_MINUTES,
   type PonyTailLevel,
   type ResponsesReasoningSummary,
   type RtkLevel,
@@ -68,6 +69,62 @@ function normalizeTelemetryPayloadDepth(value: unknown): TelemetryPayloadDepth {
   return "minimum";
 }
 
+/**
+ * Bounds a per-tenant numeric override, or `null` for "use the default".
+ *
+ * Clamped rather than rejected: these are operational knobs an operator tunes
+ * live, and a PATCH that throws on an out-of-range value leaves them stuck at
+ * whatever was stored. Clamping keeps the value inside the range the request
+ * path can actually honour — the same range the deployment env var accepts.
+ *
+ * `null` (and every non-finite value) means "no override", which is distinct
+ * from "override with zero".
+ */
+function normalizeBoundedNumber(value: unknown, min: number, max: number): number | null {
+  if (value === null || value === undefined) return null;
+  const parsed = typeof value === "number" ? value : Number(value);
+  if (!Number.isFinite(parsed)) return null;
+  // A value at or below zero is not a setting, it is a typo: clamping it to
+  // the floor would silently give the request a 5-second budget when the
+  // operator meant "no override". Non-positive therefore reads as absent.
+  if (parsed <= 0) return null;
+  return Math.min(max, Math.max(min, Math.floor(parsed)));
+}
+
+/** Upstream timeout in seconds: 5 s floor, 600 s ceiling, matching the env var. */
+function normalizeUpstreamTimeoutSeconds(value: unknown): number | null {
+  return normalizeBoundedNumber(value, 5, 600);
+}
+
+/** Source window in minutes: 1 floor, 1440 (one day) ceiling, matching the env var. */
+function normalizeTelemetryWindowMinutes(value: unknown): number | null {
+  return normalizeBoundedNumber(value, 1, 1440);
+}
+
+/** Inline attachment ceiling in bytes: 64 KB floor, 100 MB ceiling. */
+function normalizeImageClampMaxBytes(value: unknown): number | null {
+  return normalizeBoundedNumber(value, 64 * 1024, 100 * 1024 * 1024);
+}
+
+/**
+ * How long a captured payload frame is kept, in whole minutes.
+ *
+ * Only one of the offered values survives; anything else reads as "deployment
+ * default" rather than being clamped to the nearest one. A stored value from
+ * before this list existed would otherwise be silently rewritten to a window
+ * the operator never chose, and dropping it to `null` is the same outcome the
+ * operator gets by never having set it.
+ */
+function normalizePayloadRetentionMinutes(value: unknown): number | null {
+  if (value === null || value === undefined) return null;
+  const parsed = typeof value === "number" ? value : Number(value);
+  if (!Number.isFinite(parsed)) return null;
+  const minutes = Math.floor(parsed);
+  return (TELEMETRY_PAYLOAD_RETENTION_MINUTES as readonly number[]).includes(minutes)
+    ? minutes
+    : null;
+}
+
 function mapRuntimeSettingsRow(
   row: typeof consoleSettings.$inferSelect | undefined,
   redisBackend: RedisBackend,
@@ -83,6 +140,9 @@ function mapRuntimeSettingsRow(
       : "detailed",
     telemetryPayloads: normalizeTelemetryPayloadMode(prefs.telemetryPayloads),
     telemetryPayloadDepth: normalizeTelemetryPayloadDepth(prefs.telemetryPayloadDepth),
+    telemetryPayloadRetentionMinutes: normalizePayloadRetentionMinutes(
+      prefs.telemetryPayloadRetentionMinutes,
+    ),
     privacyMode: prefs.privacyMode === "full" ? "full" : "masked",
     rtkPruneEnabled: prefs.rtkPruneEnabled === true,
     rtkPruneLevel: normalizeRtkLevel(prefs.rtkPruneLevel),
@@ -90,11 +150,15 @@ function mapRuntimeSettingsRow(
     // RTK above works. `ponyTailLevel` selects the intensity and is kept while
     // the feature is off so a re-enable restores it, but a stored level is NOT
     // consent: the dashboard lets an operator choose a strength with the toggle
-    // still off, which writes a level and no flag. Reading that as enabled made
-    // the console report the directive as on while the dispatch gate — which
-    // requires the flag — never injected it.
+    // still off, which writes a level and no flag. Reading that as enabled
+    // would report the directive as on while the dispatch gate — which requires
+    // the flag — never injects it.
     ponyTailEnabled: prefs.ponyTailEnabled === true,
     ponyTailLevel: normalizePonyTailLevel(prefs.ponyTailLevel),
+    forceDisableCooldown: prefs.forceDisableCooldown === true,
+    upstreamTimeoutSeconds: normalizeUpstreamTimeoutSeconds(prefs.upstreamTimeoutSeconds),
+    telemetryWindowMinutes: normalizeTelemetryWindowMinutes(prefs.telemetryWindowMinutes),
+    imageClampMaxBytes: normalizeImageClampMaxBytes(prefs.imageClampMaxBytes),
     updatedAt,
   };
 }
@@ -128,11 +192,25 @@ export class DrizzleRuntimeSettingsStore implements RuntimeSettingsStore {
       patchPrefs.responsesReasoningSummary = patch.responsesReasoningSummary;
     if (patch.telemetryPayloads !== undefined) patchPrefs.telemetryPayloads = patch.telemetryPayloads;
     if (patch.telemetryPayloadDepth !== undefined) patchPrefs.telemetryPayloadDepth = patch.telemetryPayloadDepth;
+    if (patch.telemetryPayloadRetentionMinutes !== undefined)
+      patchPrefs.telemetryPayloadRetentionMinutes = normalizePayloadRetentionMinutes(
+        patch.telemetryPayloadRetentionMinutes,
+      );
     if (patch.privacyMode !== undefined) patchPrefs.privacyMode = patch.privacyMode;
     if (patch.rtkPruneEnabled !== undefined) patchPrefs.rtkPruneEnabled = patch.rtkPruneEnabled;
     if (patch.rtkPruneLevel !== undefined) patchPrefs.rtkPruneLevel = patch.rtkPruneLevel;
     if (patch.ponyTailEnabled !== undefined) patchPrefs.ponyTailEnabled = patch.ponyTailEnabled;
     if (patch.ponyTailLevel !== undefined) patchPrefs.ponyTailLevel = patch.ponyTailLevel;
+    if (patch.forceDisableCooldown !== undefined)
+      patchPrefs.forceDisableCooldown = patch.forceDisableCooldown === true;
+    if (patch.upstreamTimeoutSeconds !== undefined)
+      patchPrefs.upstreamTimeoutSeconds = normalizeUpstreamTimeoutSeconds(patch.upstreamTimeoutSeconds);
+    if (patch.telemetryWindowMinutes !== undefined)
+      patchPrefs.telemetryWindowMinutes = normalizeTelemetryWindowMinutes(
+        patch.telemetryWindowMinutes,
+      );
+    if (patch.imageClampMaxBytes !== undefined)
+      patchPrefs.imageClampMaxBytes = normalizeImageClampMaxBytes(patch.imageClampMaxBytes);
 
     const rows = await this.db
       .insert(consoleSettings)

package/src/console/shared/cursor.ts

@@ -13,8 +13,8 @@ const decodeCache = new Map<string, unknown>();
  * Maximum nesting depth a decoded cursor may have.
  *
  * `JSON.parse` accepts arbitrary nesting (it is iterative in this runtime), but
- * every traversal of the result is not: `deepFreeze` used to recurse once per
- * level, and `structuredClone` recurses again. A payload past the stack limit
+ * every traversal of the result is not: `structuredClone` recurses once per
+ * level. A payload past the stack limit
  * therefore threw `RangeError: Maximum call stack size exceeded` out of a function
  * that documents returning `undefined` for a malformed value — failing OPEN into a
  * 500 where the contract is "fall back to the first page", and leaving a partially
@@ -116,8 +116,8 @@ function copyDecoded<T>(value: unknown): T {
       // `structuredClone` recurses, so a value past its own budget throws. The
       // depth limit makes this unreachable for anything `decodeCursor` accepts, but
       // the documented contract is `undefined` for a value that cannot be decoded,
-      // and a cache hit on an entry written before the limit existed could still
-      // reach here.
+      // and the documented contract is `undefined` for a value that cannot be
+      // decoded.
       return undefined as T;
     }
   }
@@ -133,10 +133,10 @@ export interface DatedCursor {
 /**
  * Decodes a `(createdAt, id)` keyset cursor, rejecting anything malformed.
  *
- * Both the audit log and the telemetry event listing paginate on the same pair
- * and both had written the same validation locally — a shape check plus a
- * parseable timestamp. It lives here so the two listings cannot drift: a cursor
- * the audit store accepts must be one the stats store accepts.
+ * Both the audit log and the telemetry event listing paginate on the same pair,
+ * so the shape check plus parseable-timestamp validation lives here and the two
+ * listings cannot drift: a cursor the audit store accepts must be one the stats
+ * store accepts.
  *
  * Deliberately not generic over `T extends DatedCursor`: a caller could then
  * instantiate it with a wider type and read fields that were never validated.

package/src/console/shared/errors.ts

@@ -141,9 +141,7 @@ export function requireGlobalAdmin(access: AccessDecision | undefined): AccessDe
 /**
  * Builds the console's catch-all error handler for an Elysia route group.
  *
- * Every console route group used to wrap each handler in its own try/catch
- * calling `errorResponse` with a group-specific fallback message — 95 copies,
- * 22 of them in one file. Elysia's global `error(handler)` hook receives any
+ * Elysia's global `error(handler)` hook receives any
  * throw from the group (including a `.use()`d child), so one hook per group
  * produces the identical envelope with the identical fallback text.
  *

package/src/console/updates/routes.ts

@@ -1,4 +1,5 @@
 import { Elysia } from "elysia";
+import { spawn } from "node:child_process";
 import { consoleErrorHandler, requireTenantScope } from "../shared/errors";
 import { consoleSseResponse, createConsoleSseStream } from "../observability/sse";
 import type { ObservabilityStore } from "../observability/contracts";
@@ -86,6 +87,21 @@ export function createUpdateRoutes(config: {
   return new Elysia()
     .error(consoleErrorHandler("Update check failed"))
     .get("/updates/release", async () => await fetchRelease())
+    .post("/updates/apply", ({ request }) => {
+      requireTenantScope(config.accessResolver(request), "dashboard:write");
+      const isWin = process.platform === "win32";
+      const shell = isWin ? "cmd.exe" : "/bin/sh";
+      const install = "npm install -g cartethyia@latest --prefer-online";
+      const restart = isWin
+        ? "cartethyia start"
+        : "systemctl restart cartethyia 2>/dev/null || cartethyia start";
+      const script = `${install} && ${restart}`;
+      const args = isWin ? ["/c", script] : ["-c", script];
+      const child = spawn(shell, args, { detached: true, stdio: "ignore" });
+      child.unref();
+      setTimeout(() => process.exit(0), 1500);
+      return { started: true };
+    })
     .get("/notifications/stream", ({ request }) => {
       const tenantId = requireTenantScope(
         config.accessResolver(request),

package/src/main.ts

@@ -68,6 +68,16 @@ const app = boot
         admissionService: boot.deps.admissionService,
         modelStrikes: boot.deps.modelStrikes,
         readRoutingAccountInflight: boot.deps.readRoutingAccountInflight,
+        // The same three writers the shutdown coordinator drains, so a manual
+        // flush from the console does exactly what a graceful stop would do
+        // for the queues, minus the stop.
+        flushBuffers: {
+          flush: async () => {
+            await boot.deps.telemetryBuffer.flush();
+            await boot.deps.usageFlush.flush();
+            await boot.deps.healthEventFlush.flush();
+          },
+        },
       },
     })
   : createGatewayShell();

package/src/network/http2-bidi.ts

@@ -73,10 +73,8 @@ export interface Http2BidiOpenOptions {
  * The dialer contract a provider adapter is handed.
  *
  * Declared here rather than in the provider layer so the dependency runs one
- * way: `provider-contracts` imports this module, never the reverse. It used to
- * be declared there and re-exported from here, which made the two import each
- * other — type-only, so it never deadlocked, but it meant this
- * transport-neutral contract had to know about providers.
+ * way: `provider-contracts` imports this module, never the reverse, and this
+ * transport-neutral contract stays free of any knowledge of providers.
  */
 export type ValidatedOutboundHttp2Bidi = (
   url: URL,