cartethyia 2.7.4 → 2.7.6

package/src/console/quota/daily-growth-job.ts

@@ -0,0 +1,308 @@
+/**
+ * Background execution for the manual daily-growth batch (`POST /quota/daily-growth`).
+ *
+ * The route used to run the whole batch synchronously — chunked 3-parallel,
+ * up to two upstream HTTP calls per account, no global timeout — so a large
+ * roster held the console request open and blocked the event loop. This module
+ * moves the orchestration to an in-process background job: the route enqueues
+ * and returns `{ jobId }` immediately, and a status route reads the job's
+ * progress.
+ *
+ * The per-account runner (report then claim, egress rotation, per-leg timeout)
+ * is injected by the route — the job owns only the orchestration: the
+ * bounded-concurrency worker loop, the single-result aggregation, and the
+ * deduplication guards. This keeps the module free of the route's imports and
+ * avoids a cycle between the two quota modules.
+ *
+ * One job per (tenant, provider) key is allowed at a time. The key is the
+ * double-claim guard between two manual triggers: enqueuing again while a job
+ * for the same tenant+provider is still queued/running returns the existing
+ * job id instead of stamping the same accounts twice. In parallel with the
+ * daily sweep, the ledger is the guard — the runner stamps
+ * `markCheckinAttempted` after a successful claim, and this runner skips
+ * accounts the sweep (or an earlier run) already settled today, so the same
+ * account is never claimed twice in one window.
+ *
+ * The registry is intentionally process-local: the manual trigger is an
+ * operator action on a single gateway, so a cross-process queue would buy
+ * nothing here.
+ */
+import { randomUUID } from "node:crypto";
+import type { CartethyiaDatabase } from "../../persistence/db-handle";
+import type { RedisClient } from "../../persistence/redis";
+import { checkinAttemptedToday } from "../../workers/daily-checkin";
+import type { ValidatedFetch } from "../../network/outbound-fetch";
+
+/** Lifecycle states of a daily-growth job. */
+export type DailyGrowthJobState = "queued" | "running" | "done" | "failed";
+
+/** One account the job failed to settle, with the reason. */
+export interface DailyGrowthJobError {
+  readonly accountId: string;
+  readonly error: string;
+}
+
+/** Aggregate outcome of one job. */
+export interface DailyGrowthJobResult {
+  /** Accounts the job actually attempted (excludes skipped/already-settled). */
+  readonly total: number;
+  readonly succeeded: number;
+  readonly failed: number;
+  readonly errors: readonly DailyGrowthJobError[];
+}
+
+/** Public view of a job, as the status route returns it. */
+export interface DailyGrowthJobView {
+  readonly jobId: string;
+  readonly state: DailyGrowthJobState;
+  readonly providerId: string;
+  readonly createdAt: string;
+  readonly finishedAt: string | null;
+  readonly total: number;
+  readonly succeeded: number;
+  readonly failed: number;
+  readonly errors: readonly DailyGrowthJobError[];
+}
+
+/** One account the job will run the daily-growth pass over. */
+export interface DailyGrowthJobAccount {
+  readonly id: string;
+  readonly tenantId: string | null;
+  readonly label: string;
+}
+
+/** Outcome the route's per-account runner reports. */
+export interface DailyGrowthAccountOutcome {
+  readonly ok: boolean;
+  /** Present when `ok` is false: the reason the account was not settled. */
+  readonly error?: string;
+}
+
+/** Everything the background runner needs to execute one job. */
+export interface DailyGrowthJobDeps {
+  readonly db: CartethyiaDatabase;
+  readonly redis: RedisClient | undefined;
+  /** Runs one account's daily-growth pass (report then claim). */
+  readonly runAccount: (account: DailyGrowthJobAccount) => Promise<DailyGrowthAccountOutcome>;
+  /** Invalidated once when the job finishes, so the page re-reads the ledger. */
+  readonly snapshotInvalidator?: { invalidate(): Promise<number> };
+}
+
+/** Bound on accounts processed at once. Mirrors the old synchronous chunk size. */
+const DAILY_GROWTH_CONCURRENCY = 3;
+/**
+ * Per-account leg budget. Each individual upstream call already carries its own
+ * 15s abort timeout; this bounds the whole leg (credential resolve + report +
+ * check-in) so a single stuck account cannot hold a job worker open forever.
+ */
+export const DAILY_GROWTH_LEG_TIMEOUT_MS = 60_000;
+
+/**
+ * Wraps a fetcher so every request also carries `budget`, even when the callee
+ * sets its own per-request timeout. The buddy/qoder billing helpers pass an
+ * `AbortSignal.timeout(...)` of their own, which would override a plain
+ * `signalFetch`-style injection; combining signals intersects both bounds.
+ */
+export function withLegBudget(
+  fetcher: ValidatedFetch | typeof fetch,
+  budget: AbortSignal,
+): typeof fetch {
+  const wrapped = (
+    input: Parameters<typeof fetch>[0],
+    init?: Parameters<typeof fetch>[1],
+  ): Promise<Response> =>
+    fetcher(input, {
+      ...init,
+      signal: init?.signal ? AbortSignal.any([budget, init.signal]) : budget,
+    });
+  return Object.assign(wrapped, {
+    preconnect: (fetcher as typeof fetch).preconnect,
+  });
+}
+
+/** Registry cap; oldest finished jobs are evicted so the map cannot grow unbounded. */
+const DAILY_GROWTH_JOB_MAX = 64;
+
+interface DailyGrowthJobRecord {
+  readonly id: string;
+  readonly key: string;
+  readonly tenantId: string;
+  readonly providerId: string;
+  readonly createdAt: number;
+  state: DailyGrowthJobState;
+  finishedAt: number | null;
+  readonly accounts: readonly DailyGrowthJobAccount[];
+  result: DailyGrowthJobResult;
+}
+
+const jobs = new Map<string, DailyGrowthJobRecord>();
+/** tenant+provider key → job id; the double-trigger guard. */
+const jobsByKey = new Map<string, string>();
+/**
+ * Accounts currently being worked by any job. Global accounts (`tenant_id IS
+ * NULL`) are visible to every tenant, so two tenants can enqueue jobs over the
+ * same global account; this set keeps them from stamping it twice in the same
+ * window. Tenant-owned accounts cannot collide this way (they are unique to
+ * their tenant), but the set is uniform and cheap.
+ */
+const inFlightAccounts = new Set<string>();
+
+function jobView(job: DailyGrowthJobRecord): DailyGrowthJobView {
+  return {
+    jobId: job.id,
+    state: job.state,
+    providerId: job.providerId,
+    createdAt: new Date(job.createdAt).toISOString(),
+    finishedAt: job.finishedAt === null ? null : new Date(job.finishedAt).toISOString(),
+    total: job.result.total,
+    succeeded: job.result.succeeded,
+    failed: job.result.failed,
+    errors: job.result.errors,
+  };
+}
+
+/** Evicts the oldest finished jobs once the registry exceeds its cap. */
+function evictFinishedJobs(): void {
+  if (jobs.size <= DAILY_GROWTH_JOB_MAX) return;
+  const finished: DailyGrowthJobRecord[] = [];
+  for (const job of jobs.values()) {
+    if (job.state === "done" || job.state === "failed") finished.push(job);
+  }
+  finished.sort((left, right) => left.createdAt - right.createdAt);
+  let excess = jobs.size - DAILY_GROWTH_JOB_MAX;
+  for (const job of finished) {
+    if (excess <= 0) break;
+    jobs.delete(job.id);
+    if (jobsByKey.get(job.key) === job.id) jobsByKey.delete(job.key);
+    excess -= 1;
+  }
+}
+
+/**
+ * Runs one job to completion. Never rejects: every failure is collected into
+ * the job's result, so the status route always has an answer.
+ */
+async function runJob(record: DailyGrowthJobRecord, deps: DailyGrowthJobDeps): Promise<void> {
+  record.state = "running";
+  const succeeded: string[] = [];
+  const errors: DailyGrowthJobError[] = [];
+  const attempted: string[] = [];
+  let cursor = 0;
+  const accounts = record.accounts;
+
+  const runOne = async (account: DailyGrowthJobAccount): Promise<void> => {
+    // Double-claim guard with the daily sweep (and any other job): the ledger
+    // is shared, so an account the sweep already settled today is skipped
+    // rather than claimed again — the same `markCheckinAttempted` semantics
+    // the button uses.
+    if (inFlightAccounts.has(account.id)) return;
+    try {
+      if (await checkinAttemptedToday({ db: deps.db, redis: deps.redis, accountId: account.id })) {
+        return;
+      }
+    } catch {
+      // An unreadable ledger reads as "not attempted" — the safe direction for
+      // a manual trigger, which asserts "try now".
+    }
+    inFlightAccounts.add(account.id);
+    attempted.push(account.id);
+    try {
+      const outcome = await deps.runAccount(account);
+      if (outcome.ok) {
+        succeeded.push(account.id);
+      } else {
+        errors.push({ accountId: account.id, error: outcome.error ?? "Daily growth failed" });
+      }
+    } finally {
+      inFlightAccounts.delete(account.id);
+    }
+  };
+
+  await Promise.all(
+    Array.from({ length: Math.min(DAILY_GROWTH_CONCURRENCY, accounts.length) }, async () => {
+      for (;;) {
+        const index = cursor;
+        cursor += 1;
+        if (index >= accounts.length) return;
+        await runOne(accounts[index]!);
+      }
+    }),
+  );
+
+  record.result = {
+    total: attempted.length,
+    succeeded: succeeded.length,
+    failed: errors.length,
+    errors,
+  };
+  record.finishedAt = Date.now();
+  record.state = "done";
+  if (deps.snapshotInvalidator) {
+    try {
+      await deps.snapshotInvalidator.invalidate();
+    } catch {
+      // Best-effort: a failed snapshot invalidate must not fail the job.
+    }
+  }
+}
+
+/**
+ * Enqueues a daily-growth job and returns its id immediately. When a job for
+ * the same (tenant, provider) is already queued or running, the existing job
+ * id is returned instead — the operator pressed the button twice, and running
+ * the same accounts twice would double-claim.
+ */
+export function enqueueDailyGrowthJob(args: {
+  readonly deps: DailyGrowthJobDeps;
+  readonly tenantId: string;
+  readonly providerId: string;
+  readonly accounts: readonly DailyGrowthJobAccount[];
+}): string {
+  const key = `${args.tenantId}:${args.providerId}`;
+  const existing = jobsByKey.get(key);
+  if (existing !== undefined) {
+    const job = jobs.get(existing);
+    if (job !== undefined && (job.state === "queued" || job.state === "running")) {
+      return job.id;
+    }
+    jobsByKey.delete(key);
+  }
+
+  const id = randomUUID();
+  const record: DailyGrowthJobRecord = {
+    id,
+    key,
+    tenantId: args.tenantId,
+    providerId: args.providerId,
+    createdAt: Date.now(),
+    state: "queued",
+    finishedAt: null,
+    accounts: args.accounts,
+    result: { total: 0, succeeded: 0, failed: 0, errors: [] },
+  };
+  jobs.set(id, record);
+  jobsByKey.set(key, id);
+  evictFinishedJobs();
+  void runJob(record, args.deps).catch(() => {
+    record.state = "failed";
+    record.finishedAt = Date.now();
+  });
+  return id;
+}
+
+/**
+ * Reads one job's status. Scoped to the requesting tenant: a job enqueued by
+ * another tenant is not visible, so one tenant cannot probe another's work.
+ */
+export function getDailyGrowthJob(jobId: string, tenantId: string): DailyGrowthJobView | null {
+  const job = jobs.get(jobId);
+  if (job === undefined || job.tenantId !== tenantId) return null;
+  return jobView(job);
+}
+
+/** Test seam: clears the registry so suites do not leak state across runs. */
+export function clearDailyGrowthJobsForTests(): void {
+  jobs.clear();
+  jobsByKey.clear();
+  inFlightAccounts.clear();
+}

package/src/console/quota/native-identity.ts

@@ -20,7 +20,7 @@ import { ConsoleDomainError, requireGlobalAdmin, requireTenantScope } from "../s
 import type { ConsoleAccessResolver } from "../auth/access";
 
 /** True when `token` looks like a plausible machine token. */
-export function isValidNativeIdentityToken(token: string): boolean {
+function isValidNativeIdentityToken(token: string): boolean {
   return typeof token === "string" && token.length >= 8 && token.length <= 512;
 }
 
@@ -120,7 +120,7 @@ export function createNativeIdentityOps(
  * (`tenant_id IS NULL`) need platform admin; tenant-owned accounts need the
  * tenant scope for the owning tenant.
  */
-export interface NativeIdentityScope {
+interface NativeIdentityScope {
   readonly tenantId: string | null;
 }
 

package/src/console/quota/quota-view.ts

@@ -11,7 +11,7 @@ import {
 } from "../../providers/quota/refresh";
 
 /** Shape returned for quota data on dashboard account responses. */
-export interface QuotaRefreshView {
+interface QuotaRefreshView {
   readonly source: string;
   readonly status: "ready" | "error" | "refreshing";
   readonly plan: string | null;

package/src/console/remote-support/routes.ts

@@ -14,7 +14,7 @@ import { consoleErrorHandler, requireTenantScope } from "../shared/errors";
 import { REMOTE_SUPPORT_TOOLS } from "./contracts";
 import type { RemoteSupportService } from "./service";
 
-export interface RemoteSupportRoutesConfig {
+interface RemoteSupportRoutesConfig {
   readonly accessResolver: ConsoleAccessResolver;
   readonly service: RemoteSupportService;
 }

package/src/console/remote-support/service.ts

@@ -33,7 +33,7 @@ const URL_POLL_MS = 500;
 /** Cached install detection so we don't spawn `where`/`which` on every poll. */
 const installedCache = new Map<RemoteSupportTool, boolean>();
 
-export interface RemoteSupportServiceConfig {
+interface RemoteSupportServiceConfig {
   /** Defaults to the gateway's configured port. */
   readonly port: number;
   /** Overridable for tests. Defaults to a subdir of the user config dir. */
@@ -245,7 +245,7 @@ function tailscaleDnsName(): string | null {
  * nothing about this: stopping the tunnel leaves the node up and empties the
  * status, which is why an online node alone was read as "still active".
  */
-export function tailscaleServing(): boolean {
+function tailscaleServing(): boolean {
   const result = spawnSync("tailscale", ["serve", "status", "--json"], { encoding: "utf8", windowsHide: true });
   if (result.status !== 0) return false;
   let parsed: unknown;

package/src/console/routing/model/contracts.ts

@@ -102,7 +102,7 @@ export interface ModelRoutingStore {
 
 /** Structurally satisfied by `InMemoryRouteSnapshotService` as-is — no import
  * of routing internals into the console domain layer. */
-export interface ModelRoutingSnapshotInvalidator {
+interface ModelRoutingSnapshotInvalidator {
   invalidate(): Promise<number>;
 }
 

package/src/console/routing/pools/contracts.ts

@@ -223,7 +223,7 @@ export type PoolStatus = (typeof POOL_STATUSES)[number];
  * stay aligned with the `pool_routing_strategy` enum in `schema.ts`.
  */
 export const POOL_ROUTING_STRATEGIES = ["least_loaded", "round_robin", "random"] as const;
-export type PoolRoutingStrategyValue = (typeof POOL_ROUTING_STRATEGIES)[number];
+type PoolRoutingStrategyValue = (typeof POOL_ROUTING_STRATEGIES)[number];
 
 export interface PoolStrategySetting {
   strategy: PoolRoutingStrategyValue;
@@ -456,7 +456,7 @@ export const MAX_BATCH_PROBE_TARGETS = 100;
  */
 export const PROVIDER_PROBE_BATCH_CONCURRENCY = 4;
 
-export interface NetworkPoolAgentReleaser {
+interface NetworkPoolAgentReleaser {
   releasePool(poolId: string, tenantId: string): Promise<void>;
 }
 export interface NetworkPoolConfig {

package/src/console/routing/pools/import-parse.ts

@@ -18,7 +18,7 @@
 import { POOL_KINDS, POOL_STATUSES } from "./contracts";
 
 /** One row that parsed, ready to become a pool. */
-export interface ParsedProxyRow {
+interface ParsedProxyRow {
   readonly kind: (typeof POOL_KINDS)[number];
   readonly endpoint: string;
   readonly credential: string | null;
@@ -26,12 +26,12 @@ export interface ParsedProxyRow {
 }
 
 /** One row that did not parse, with the reason. */
-export interface SkippedProxyRow {
+interface SkippedProxyRow {
   readonly line: number;
   readonly reason: string;
 }
 
-export interface ParsedProxyFile {
+interface ParsedProxyFile {
   readonly rows: readonly ParsedProxyRow[];
   readonly skipped: readonly SkippedProxyRow[];
 }

package/src/console/routing/pools/relay-deploy.ts

@@ -36,7 +36,7 @@ export const RELAY_TARGETS: readonly RelayTarget[] = ["cloudflare", "vercel", "d
  * The same handler body runs on Workers and Vercel Edge (`export default {
  * fetch }`); the Deno entry wraps it in `Deno.serve`.
  */
-export const RELAY_WORKER_SOURCE = `// Cartethyia hosted relay.
+const RELAY_WORKER_SOURCE = `// Cartethyia hosted relay.
 //
 // Forwards a request to the origin named by the x-relay-target header, at the
 // path named by x-relay-path. Deployed to Cloudflare Workers / Vercel Edge.
@@ -83,7 +83,7 @@ export default handler;
 `;
 
 /** A Deno Deploy entry wraps the shared handler in `Deno.serve`. */
-export const RELAY_DENO_SOURCE = `${RELAY_WORKER_SOURCE}
+const RELAY_DENO_SOURCE = `${RELAY_WORKER_SOURCE}
 Deno.serve((request) => handler.fetch(request));
 `;
 
@@ -111,7 +111,7 @@ export interface RelayDeployResult {
 }
 
 /** Outbound fetch seam (the validated fetch), so tests can stub deploys. */
-export type RelayFetch = (url: string, init: RequestInit) => Promise<Response>;
+type RelayFetch = (url: string, init: RequestInit) => Promise<Response>;
 
 function generateName(prefix: string): string {
   return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;

package/src/console/settings/contracts.ts

@@ -53,6 +53,12 @@ export interface RuntimeSettingsResponse {
   readonly telemetryWindowMinutes: number | null;
   /** Per-tenant inline attachment ceiling in bytes; null means deployment default. */
   readonly imageClampMaxBytes: number | null;
+  /**
+   * Ordered vision-model ids used to describe images when the selected model
+   * is not image-capable. Empty means the legacy `[image]` placeholder is
+   * used instead.
+   */
+  readonly visionHandoffModels: readonly string[];
   readonly updatedAt: string;
   /*
    * What each `null` actually resolves to, in the same units the row stores.
@@ -67,7 +73,7 @@ export interface RuntimeSettingsResponse {
 }
 
 /** The deployment-level values a `null` runtime setting falls back to. */
-export interface RuntimeSettingsDefaults {
+interface RuntimeSettingsDefaults {
   /** Seconds; from `CARTETHYIA_UPSTREAM_TIMEOUT_MS`. */
   readonly upstreamTimeoutSeconds: number;
   /** Minutes; what the payload store expires frames after. */
@@ -122,7 +128,7 @@ export type TelemetryPayloadMode = (typeof TELEMETRY_PAYLOAD_MODES)[number];
 export const TELEMETRY_PAYLOAD_DEPTHS = ["minimum", "moderate", "maximum"] as const;
 /** Capture depth for `full` mode accepted by the runtime-settings PATCH. */
 export type TelemetryPayloadDepth = (typeof TELEMETRY_PAYLOAD_DEPTHS)[number];
-export const PRIVACY_MODES = ["masked", "full"] as const;
+const PRIVACY_MODES = ["masked", "full"] as const;
 /** Provider/model label privacy accepted by the runtime-settings PATCH. */
 export type PrivacyMode = (typeof PRIVACY_MODES)[number];
 /** PonyTail directive intensity accepted by the runtime-settings PATCH. */
@@ -130,6 +136,9 @@ export type PonyTailLevel = (typeof PONYTAIL_LEVELS)[number];
 /** RTK prune strength accepted by the runtime-settings PATCH. */
 export type RtkLevel = (typeof RTK_LEVELS)[number];
 
+/** Upper bound on the number of ordered vision-handoff models. */
+export const VISION_HANDOFF_MAX_MODELS = 8;
+
 export function createRuntimeSettingsOperations(deps: RuntimeSettingsConfig) {
   const operations = {
     async get(access: AccessDecision | undefined): Promise<RuntimeSettingsResponse> {
@@ -178,6 +187,23 @@ export function createRuntimeSettingsOperations(deps: RuntimeSettingsConfig) {
         if (patch.rtkPruneLevel !== undefined && !RTK_LEVELS.includes(patch.rtkPruneLevel)) {
           throw new ConsoleDomainError("invalid_request", 400, "Invalid rtkPruneLevel");
         }
+        if (patch.visionHandoffModels !== undefined) {
+          if (!Array.isArray(patch.visionHandoffModels)) {
+            throw new ConsoleDomainError("invalid_request", 400, "visionHandoffModels must be an array of model ids");
+          }
+          if (patch.visionHandoffModels.length > VISION_HANDOFF_MAX_MODELS) {
+            throw new ConsoleDomainError(
+              "invalid_request",
+              400,
+              `visionHandoffModels may list at most ${VISION_HANDOFF_MAX_MODELS} models`,
+            );
+          }
+          for (const model of patch.visionHandoffModels) {
+            if (typeof model !== "string" || model.trim().length === 0) {
+              throw new ConsoleDomainError("invalid_request", 400, "visionHandoffModels entries must be non-empty model ids");
+            }
+          }
+        }
         const updated = await deps.store.update(a.tenantId, patch);
         await deps.auditSink?.record({
           access: a,
@@ -245,4 +271,5 @@ export const runtimeUpdateBody = t.Object({
   imageClampMaxBytes: t.Optional(
     t.Union([t.Null(), t.Number({ minimum: 64 * 1024, maximum: 100 * 1024 * 1024 })]),
   ),
+  visionHandoffModels: t.Optional(t.Array(t.String())),
 });

package/src/console/settings/store.ts

@@ -15,6 +15,7 @@ import {
   RTK_LEVELS,
   TELEMETRY_PAYLOAD_DEPTHS,
   TELEMETRY_PAYLOAD_MODES,
+  VISION_HANDOFF_MAX_MODELS,
   type PonyTailLevel,
   type ResponsesReasoningSummary,
   type RtkLevel,
@@ -115,6 +116,22 @@ function normalizeConcurrencyTolerance(value: unknown): number | null {
   return normalizeBoundedNumber(value, 0, 1024);
 }
 
+/** Ordered vision-handoff models: non-empty strings, trimmed, de-duplicated. */
+function normalizeVisionHandoffModels(value: unknown): readonly string[] {
+  if (!Array.isArray(value)) return [];
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const entry of value) {
+    if (typeof entry !== "string") continue;
+    const model = entry.trim();
+    if (model.length === 0 || seen.has(model)) continue;
+    seen.add(model);
+    out.push(model);
+    if (out.length >= VISION_HANDOFF_MAX_MODELS) break;
+  }
+  return out;
+}
+
 function mapRuntimeSettingsRow(
   row: typeof consoleSettings.$inferSelect | undefined,
   redisBackend: RedisBackend,
@@ -148,6 +165,7 @@ function mapRuntimeSettingsRow(
     upstreamTimeoutSeconds: normalizeUpstreamTimeoutSeconds(prefs.upstreamTimeoutSeconds),
     telemetryWindowMinutes: normalizeTelemetryWindowMinutes(prefs.telemetryWindowMinutes),
     imageClampMaxBytes: normalizeImageClampMaxBytes(prefs.imageClampMaxBytes),
+    visionHandoffModels: normalizeVisionHandoffModels(prefs.visionHandoffModels),
     updatedAt,
     /*
      * Resolved at request time, not at module load: all four come from env the
@@ -211,6 +229,8 @@ export class DrizzleRuntimeSettingsStore implements RuntimeSettingsStore {
       );
     if (patch.imageClampMaxBytes !== undefined)
       patchPrefs.imageClampMaxBytes = normalizeImageClampMaxBytes(patch.imageClampMaxBytes);
+    if (patch.visionHandoffModels !== undefined)
+      patchPrefs.visionHandoffModels = normalizeVisionHandoffModels(patch.visionHandoffModels);
 
     const rows = await this.db
       .insert(consoleSettings)