cartethyia 2.5.3 → 2.5.6

package/src/console/console-router-guards.ts

@@ -14,7 +14,7 @@ import { parseCookieValue, SESSION_COOKIE_NAME, isCsrfValid } from "../security/
 /**
  * Stateless double-submit CSRF guard for unsafe `/console/api/*` mutations.
  * Compares the `x-csrf-token` header against the readable `csrf_token`
- * cookie — zero database I/O, so dashboard writes no longer pay a token
+ * cookie — zero database I/O, so dashboard writes pay no token
  * SELECT + UPDATE on top of the session lookup, and there is no token
  * store whose outage could masquerade as an auth failure.
  */

package/src/console/console-router.ts

@@ -48,6 +48,13 @@ export interface ConsoleApiCompositionDeps {
   readonly routeSnapshotService: { invalidate(): Promise<number> };
   readonly poolSelector: NetworkPoolSelector;
   readonly telemetryBuffer: TelemetryBatchBuffer;
+  /**
+   * Queued writers the console can drain on demand (telemetry, usage, health
+   * events). Optional for the same reason the other reduced composition
+   * surface is: absent means the flush control is not offered rather than the
+   * console failing to build.
+   */
+  readonly flushBuffers?: { flush(): Promise<void> };
   /**
    * The request state store owns the live in-flight gauge, so the console reads
    * its snapshot/subscription from here rather than a free-standing module.
@@ -189,6 +196,7 @@ export function createConsoleRouter(deps: ConsoleApiCompositionDeps): Elysia {
     admissionService: deps.admissionService,
     ...(deps.modelStrikes ? { modelStrikes: deps.modelStrikes } : {}),
     readRoutingAccountInflight: deps.readRoutingAccountInflight,
+    ...(deps.flushBuffers ? { flushBuffers: deps.flushBuffers } : {}),
     credentialService,
     // The backup surface re-authenticates the operator, so it needs the current
     // user's hash. Read from the session on the request that asks for it, never

package/src/console/data-management/contracts.ts

@@ -0,0 +1,100 @@
+/**
+ * Disk housekeeping: what the telemetry and payload directories hold, and the
+ * reclaim actions an operator can run from the console.
+ *
+ * Everything here is files on disk, except one deliberate row-level action.
+ * Reclaiming a file is safe because the data it held is already past the point
+ * where the gateway would read it: payload frames expire after the tenant's
+ * chosen retention, and telemetry files past the retention window are outside
+ * every query's scan bound. Deleting rows is a different decision with a different
+ * blast radius, and the only row data the console offers purging is the health
+ * & error log — purely an operator's diagnostic trail with no request-path
+ * reader, unlike everything else in this panel.
+ *
+ * Sizes are bytes as `number`. A directory is scanned on each read rather than
+ * cached, because the point of the panel is to show what is true now — a
+ * stale size would be worse than none.
+ */
+
+/** One measured category of on-disk data. */
+export interface DiskUsageBucket {
+  /** Stable id, so the dashboard can key rows without matching on copy. */
+  readonly id: "telemetry_events" | "telemetry_index" | "payload_frames";
+  readonly label: string;
+  readonly bytes: number;
+  readonly files: number;
+  /** Why this data exists, shown next to the size. */
+  readonly note: string;
+  /** Bytes reclaimable right now, when that is knowable without a dry run. */
+  readonly reclaimableBytes: number | null;
+  readonly reclaimableFiles: number | null;
+}
+
+export interface DiskUsageResponse {
+  readonly buckets: readonly DiskUsageBucket[];
+  readonly totalBytes: number;
+  /** Retention in days; telemetry older than this is outside every scan. */
+  readonly retentionDays: number;
+  /** Payload frames live this long by default before they are reclaimable. */
+  readonly payloadRetentionMs: number;
+  /**
+   * The longest retention a tenant may have chosen. A frame is only counted as
+   * reclaimable once it is older than *this*, because a tenant who raised the
+   * window has frames the default would call dead while their rows still
+   * address them.
+   */
+  readonly payloadRetentionMaxMs: number;
+  /**
+   * Stored health & error rows, for the purge row's label. Absent when the
+   * composition does not wire the health-event store (a reduced console),
+   * which is also when the purge control is not offered.
+   */
+  readonly healthEventCount?: number;
+}
+
+export type ReclaimAction = "payload_frames" | "telemetry_retention";
+
+export interface ReclaimResponse {
+  readonly action: ReclaimAction;
+  readonly deletedFiles: number;
+  readonly reclaimedBytes: number;
+}
+
+/** Row-level deletion targets. The health & error log is the only one. */
+export type PurgeTarget = "health_events";
+
+export interface PurgeResponse {
+  readonly target: PurgeTarget;
+  /** Rows actually deleted — the number the operator's decision produced. */
+  readonly deleted: number;
+}
+
+export interface FlushResponse {
+  readonly flushed: boolean;
+}
+
+/**
+ * A reclaim the caller is not allowed to make.
+ *
+ * Kept as a named class so the route maps it to 403 rather than letting it
+ * surface as a 500 — an operator asking for something out of policy is a
+ * normal request, not a failure.
+ */
+export class ReclaimNotAllowedError extends Error {
+  readonly status = 403;
+
+  constructor(message: string) {
+    super(message);
+    this.name = "ReclaimNotAllowedError";
+  }
+}
+
+/** A purge whose backing store is not wired in this composition. */
+export class PurgeNotWiredError extends Error {
+  readonly status = 501;
+
+  constructor() {
+    super("Health & error log purging is not offered in this deployment's console composition.");
+    this.name = "PurgeNotWiredError";
+  }
+}

package/src/console/data-management/operations.ts

@@ -0,0 +1,234 @@
+import { readdir, stat, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { payloadDirectory, prunePayloadFrames } from "../../observability/payload-store";
+import type {
+  DiskUsageBucket,
+  DiskUsageResponse,
+  PurgeTarget,
+  ReclaimAction,
+  ReclaimResponse,
+} from "./contracts";
+import { PurgeNotWiredError, ReclaimNotAllowedError } from "./contracts";
+
+/**
+ * Reads and reclaims the on-disk telemetry and payload directories, and
+ * purges the health & error log — the console's one row-level deletion.
+ *
+ * Reclaiming files is safe because the data is already past the point the
+ * gateway would read it: payload frames expire minutes after they are written,
+ * and telemetry past the retention window sits outside every query's scan
+ * bound. The health & error log purge is the deliberate exception: the rows are
+ * an operator's diagnostic trail that no request-path code reads, so clearing
+ * them costs nothing but the history itself.
+ *
+ * The telemetry directory is injected rather than read from its module
+ * constant so the scanner can be pointed at a fixture. The payload directory
+ * is not: reclaim goes through `prunePayloadFrames`, which resolves the
+ * directory itself, and a second path would let the two drift so the scanner
+ * measured one place while the prune emptied another. The health-event store
+ * hook is injected for the same reason: this module should know its numbers,
+ * not how storage reaches them.
+ */
+
+export interface DataManagementConfig {
+  readonly telemetryDir: string;
+  readonly retentionDays: number;
+  readonly payloadRetentionMs: number;
+  /**
+   * The longest payload retention a tenant may have chosen, and therefore the
+   * only cutoff a frame file can be judged dead against without reading it.
+   * Defaults to `payloadRetentionMs` for callers that predate per-tenant
+   * retention, which is then the true ceiling in that deployment.
+   */
+  readonly payloadRetentionMaxMs?: number;
+  /**
+   * The health & error log store, backing the console's only row-level purge.
+   * Absent (a reduced console composition) means the purge is not offered
+   * rather than failing to boot.
+   */
+  readonly healthEvents?:
+    | {
+        count(): Promise<number>;
+        purgeAll(): Promise<number>;
+      }
+    | undefined;
+}
+
+interface Scan {
+  readonly bytes: number;
+  readonly files: number;
+}
+
+async function scan(
+  directory: string,
+  match: (name: string) => boolean,
+  modifiedBefore?: number,
+): Promise<Scan> {
+  const names = await readdir(directory).catch(() => [] as string[]);
+  let bytes = 0;
+  let files = 0;
+  for (const name of names) {
+    if (!match(name)) continue;
+    const info = await stat(join(directory, name)).catch(() => undefined);
+    if (!info?.isFile()) continue;
+    if (modifiedBefore !== undefined && info.mtimeMs >= modifiedBefore) continue;
+    bytes += info.size;
+    files += 1;
+  }
+  return { bytes, files };
+}
+
+const isIndex = (name: string): boolean => name.endsWith(".index.json");
+const isHourly = (name: string): boolean => name.endsWith(".events.jsonl");
+const isFrame = (name: string): boolean =>
+  name.endsWith(".jsonl");
+
+/** The UTC day a telemetry file name starts with, compared to a cutoff. */
+function dayBefore(name: string, cutoff: number): boolean {
+  const at = Date.parse(`${name.slice(0, 10)}T00:00:00Z`);
+  return Number.isFinite(at) && at < cutoff;
+}
+
+export function createDataManagementOperations(config: DataManagementConfig) {
+  const payloadDir = (): string => payloadDirectory();
+  /**
+   * The only cutoff a frame file can be judged dead against without parsing it.
+   *
+   * A file's mtime says when it was last written, not what its frames' tenants
+   * chose — so a file written an hour ago can hold a frame with a week to live.
+   * The ceiling is therefore the bound, and the default retention is only used
+   * when no ceiling was configured.
+   */
+  const payloadMax = (): number => config.payloadRetentionMaxMs ?? config.payloadRetentionMs;
+
+  async function usage(): Promise<DiskUsageResponse> {
+    const retentionCutoff = Date.now() - config.retentionDays * 24 * 60 * 60_000;
+    const payloadCutoff = Date.now() - payloadMax();
+
+    const [hourly, compacted, staleCompacted, frames, staleFrames, healthEventCount] = await Promise.all([
+      scan(config.telemetryDir, isHourly),
+      scan(config.telemetryDir, isIndex),
+      scan(config.telemetryDir, (name) => isIndex(name) && dayBefore(name, retentionCutoff)),
+      scan(payloadDir(), isFrame),
+      scan(payloadDir(), isFrame, payloadCutoff),
+      config.healthEvents?.count(),
+    ]);
+
+    const buckets: DiskUsageBucket[] = [
+      {
+        id: "telemetry_events",
+        label: "Request event files",
+        bytes: hourly.bytes,
+        files: hourly.files,
+        note: "Feeds the request list and detail drawer.",
+        reclaimableBytes: null,
+        reclaimableFiles: null,
+      },
+      {
+        id: "telemetry_index",
+        label: "Day indexes",
+        bytes: compacted.bytes,
+        files: compacted.files,
+        note: "Feeds usage totals, breakdowns and the chart.",
+        reclaimableBytes: staleCompacted.bytes,
+        reclaimableFiles: staleCompacted.files,
+      },
+      {
+        id: "payload_frames",
+        label: "Payload frames",
+        bytes: frames.bytes,
+        files: frames.files,
+        note: `Captured bodies, expiring ${Math.round(
+          config.payloadRetentionMs / 60_000,
+        )} min after write by default.`,
+        reclaimableBytes: staleFrames.bytes,
+        reclaimableFiles: staleFrames.files,
+      },
+    ];
+
+    return {
+      buckets,
+      totalBytes: buckets.reduce((sum, bucket) => sum + bucket.bytes, 0),
+      retentionDays: config.retentionDays,
+      payloadRetentionMs: config.payloadRetentionMs,
+      payloadRetentionMaxMs: payloadMax(),
+      ...(healthEventCount === undefined ? {} : { healthEventCount }),
+    };
+  }
+
+  async function reclaim(action: ReclaimAction): Promise<ReclaimResponse> {
+    if (action === "payload_frames") {
+      // Delegated rather than reimplemented: frame expiry lives inside each
+      // file, so a file's mtime alone cannot say whether it still holds a live
+      // frame. Measuring around the call is how the bytes are reported without
+      // duplicating that rule.
+      const before = await scan(payloadDir(), isFrame);
+      // `before` is the expiry bound: a frame still live at this instant is
+      // kept. The second argument covers files that cannot be parsed — one
+      // written after that bound might still hold a live frame in the part
+      // that failed to parse, so it is left for a later pass.
+      const now = new Date();
+      const deletedFiles = await prunePayloadFrames(
+        now,
+        new Date(now.getTime() - payloadMax()),
+      );
+      const after = await scan(payloadDir(), isFrame);
+      return {
+        action,
+        deletedFiles,
+        reclaimedBytes: Math.max(0, before.bytes - after.bytes),
+      };
+    }
+    if (action === "telemetry_retention") {
+      // Only day indexes past retention, and only together with their events.
+      // Deleting one without the other is the failure this avoids: an index
+      // with no events reports totals for requests that are gone.
+      const cutoff = Date.now() - config.retentionDays * 24 * 60 * 60_000;
+      return unlinkWhere(action, config.telemetryDir, (name) =>
+        (isIndex(name) || isHourly(name)) && dayBefore(name, cutoff),
+      );
+    }
+    throw new ReclaimNotAllowedError(`Unknown reclaim action: ${action}`);
+  }
+
+  /** The one row-level deletion the console offers. */
+  async function purge(target: PurgeTarget): Promise<number> {
+    switch (target) {
+      case "health_events": {
+        const store = config.healthEvents;
+        if (store === undefined) throw new PurgeNotWiredError();
+        return await store.purgeAll();
+      }
+    }
+  }
+
+  return { usage, reclaim, purge };
+}
+
+async function unlinkWhere(
+  action: ReclaimAction,
+  directory: string,
+  match: (name: string) => boolean,
+): Promise<ReclaimResponse> {
+  const names = await readdir(directory).catch(() => [] as string[]);
+  let deletedFiles = 0;
+  let reclaimedBytes = 0;
+  for (const name of names) {
+    if (!match(name)) continue;
+    const path = join(directory, name);
+    // Measured before the unlink, since the size is gone afterwards.
+    const size = await stat(path)
+      .then((value) => value.size)
+      .catch(() => 0);
+    const removed = await unlink(path).then(
+      () => true,
+      () => false,
+    );
+    if (!removed) continue;
+    deletedFiles += 1;
+    reclaimedBytes += size;
+  }
+  // The response names the action, so a caller that fired two reclaims can
+  // tell which result is which rather than both reading the same thing.
+  return { action, deletedFiles, reclaimedBytes };
+}

package/src/console/data-management/routes.ts

@@ -0,0 +1,103 @@
+import { Elysia } from "elysia";
+import { t } from "elysia";
+import { errorResponse, requireAnyScope } from "../shared/errors";
+import type { AccessDecision } from "../../security/access-control";
+import { createDataManagementOperations, type DataManagementConfig } from "./operations";
+import type { PurgeTarget, ReclaimAction, ReclaimResponse } from "./contracts";
+
+const WRITE_SCOPES = ["dashboard:write"] as const;
+
+const reclaimBody = t.Object({
+  action: t.Union([
+    t.Literal("payload_frames"),
+    t.Literal("telemetry_retention"),
+  ]),
+});
+
+const purgeBody = t.Object({
+  target: t.Literal("health_events"),
+});
+
+/**
+ * Console routes for disk housekeeping.
+ *
+ * Reclaim and flush both need `dashboard:write`. Reading is deliberately the
+ * same scope as the rest of the console rather than something narrower: the
+ * sizes are not sensitive, but splitting the two would mean a caller who can
+ * already write could not see what they are about to affect.
+ */
+export function createDataManagementRoutes(
+  config: DataManagementConfig & {
+    readonly accessResolver: (request: Request) => AccessDecision | undefined;
+    readonly auditSink?: {
+      record(entry: {
+        readonly access: AccessDecision;
+        readonly action: string;
+        readonly target: string | null;
+        readonly detail?: Record<string, unknown>;
+      }): Promise<void> | void;
+    } | undefined;
+    readonly flusher?: { flush(): Promise<void> } | undefined;
+  },
+): Elysia {
+  const operations = createDataManagementOperations(config);
+
+  return new Elysia({ prefix: "/data-management" })
+    .get("/usage", async ({ request, set }) => {
+      try {
+        const access = requireAnyScope(config.accessResolver(request), WRITE_SCOPES);
+        void access;
+        return await operations.usage();
+      } catch (error) {
+        return errorResponse(error, set, "Could not read disk usage");
+      }
+    })
+    .post("/reclaim", { body: reclaimBody }, async ({ request, body, set }) => {
+      try {
+        const access = requireAnyScope(config.accessResolver(request), WRITE_SCOPES);
+        const action = body.action as ReclaimAction;
+        const result: ReclaimResponse = await operations.reclaim(action);
+        await config.auditSink?.record({
+          access,
+          action: `data_management.reclaim.${action}`,
+          target: access.tenantId ?? null,
+          detail: { deletedFiles: result.deletedFiles, reclaimedBytes: result.reclaimedBytes },
+        });
+        set.status = 200;
+        return result;
+      } catch (error) {
+        return errorResponse(error, set, "Reclaim failed");
+      }
+    })
+    .post("/flush", async ({ request, set }) => {
+      try {
+        const access = requireAnyScope(config.accessResolver(request), WRITE_SCOPES);
+        await config.flusher?.flush();
+        await config.auditSink?.record({
+          access,
+          action: "data_management.flush",
+          target: access.tenantId ?? null,
+        });
+        set.status = 200;
+        return { flushed: true };
+      } catch (error) {
+        return errorResponse(error, set, "Flush failed");
+      }
+    })
+    .post("/purge", { body: purgeBody }, async ({ request, body, set }) => {
+      try {
+        const access = requireAnyScope(config.accessResolver(request), WRITE_SCOPES);
+        const deleted: number = await operations.purge(body.target as PurgeTarget);
+        await config.auditSink?.record({
+          access,
+          action: `data_management.purge.${body.target}`,
+          target: access.tenantId ?? null,
+          detail: { deleted },
+        });
+        set.status = 200;
+        return { target: body.target, deleted } satisfies { target: PurgeTarget; deleted: number };
+      } catch (error) {
+        return errorResponse(error, set, "Purge failed");
+      }
+    }) as unknown as Elysia;
+}

package/src/console/domain-registration.ts

@@ -23,6 +23,9 @@ import { createNetworkPoolRoutes } from "./routing/pools/routes";
 import { DrizzleModelRoutingStore } from "./routing/model/store";
 import { DrizzleNetworkPoolStore } from "./routing/pools/store";
 import { createRuntimeSettingsRoutes } from "./settings/routes";
+import { createDataManagementRoutes } from "./data-management/routes";
+import { healthEventBuffer } from "../persistence/health-event-flush";
+import { countAllHealthEvents, deleteAllHealthEvents } from "../providers/operations/account-health-service";
 import { DrizzleRuntimeSettingsStore } from "./settings/store";
 import { createObservabilityRoutes } from "./observability/routes";
 import { DrizzleObservabilityStore } from "./observability/store";
@@ -33,9 +36,15 @@ import { createModelAbuseRoutes } from "./admin/model-abuse/routes";
 import { DrizzleAuditReadStore } from "./audit/store";
 import { createPerformanceRoutes } from "./observability/performance";
 import { createUpdateRoutes } from "./updates/routes";
+import { createRemoteSupportRoutes } from "./remote-support/routes";
+import { createRemoteSupportService } from "./remote-support/service";
+import { resolvePort } from "../config/runtime";
 import { createBackupRoutes } from "./backup/routes";
 import { BackupService } from "./backup/service";
 import type { ConsoleCredentialService } from "./auth/credentials";
+import { resolveTelemetryRetentionDays } from "../config/telemetry";
+import { MAX_PAYLOAD_RETENTION_MS, payloadRetentionMs } from "../observability/payload-capture";
+import { TELEMETRY_EVENT_DIR } from "../observability/event-log";
 
 import { InMemoryOAuthFlowStore, OAuthFlowStore } from "../providers/authentication/oauth-flow-store";
 import { OAuthCallbackListener } from "./providers/oauth/callback-listener";
@@ -80,6 +89,14 @@ export interface ConsoleDomainContext {
   readonly routeSnapshotService: { invalidate(): Promise<number> };
   readonly poolSelector: NetworkPoolSelector;
   readonly telemetryBuffer: TelemetryBatchBuffer;
+  /**
+   * Queued writers the console can drain on demand. Optional because a flush
+   * is only meaningful when the process owns the buffers; a caller that does
+   * not pass them simply loses the button rather than failing to boot.
+   */
+  readonly flushBuffers?: {
+    flush(): Promise<void>;
+  };
   /** Owns the live in-flight gauge the console surfaces. */
   readonly stateStore?: LiveInFlightSource;
   readonly admissionService: Pick<ApiKeyAdmissionService, "purgeKey">;
@@ -227,6 +244,10 @@ export function registerConsoleDomains(
     accessResolver: ctx.accessResolver,
     observabilityStore,
   }));
+  console.use(createRemoteSupportRoutes({
+    accessResolver: ctx.accessResolver,
+    service: createRemoteSupportService({ port: resolvePort() }),
+  }));
   console.use(createAuditRoutes({ store: auditReadStore, accessResolver: ctx.accessResolver }));
   if (ctx.modelStrikes) {
     console.use(createModelAbuseRoutes({
@@ -267,7 +288,30 @@ export function registerConsoleDomains(
     // same SSRF-validated fetch as every other egress.
     relayFetch: createValidatedFetch({ policy: resolveSsrfPolicy() }),
   }));
-  console.use(createRuntimeSettingsRoutes({ store: runtimeSettingsStore, accessResolver: ctx.accessResolver, auditSink: ctx.auditRecorder }));
+  console.use(createRuntimeSettingsRoutes({ store: runtimeSettingsStore, accessResolver: ctx.accessResolver, auditSink: ctx.auditRecorder, snapshotInvalidator: ctx.routeSnapshotService }));
+  console.use(
+    createDataManagementRoutes({
+      telemetryDir: TELEMETRY_EVENT_DIR,
+      retentionDays: resolveTelemetryRetentionDays(),
+      payloadRetentionMs: payloadRetentionMs(),
+      payloadRetentionMaxMs: MAX_PAYLOAD_RETENTION_MS,
+      accessResolver: ctx.accessResolver,
+      auditSink: ctx.auditRecorder,
+      flusher: ctx.flushBuffers,
+      // The console's one row-level purge: the health & error log is a
+      // diagnostic trail no request-path code reads. Dropping the buffered
+      // writes before the DELETE is what makes it sound — otherwise a batch
+      // in flight would land behind the purge and resurrect history that was
+      // just counted as deleted.
+      healthEvents: {
+        count: () => countAllHealthEvents(ctx.db),
+        purgeAll: async () => {
+          await healthEventBuffer()?.drain();
+          return await deleteAllHealthEvents(ctx.db);
+        },
+      },
+    }),
+  );
   console.use(
     createBackupRoutes({
       accessResolver: ctx.accessResolver,