cartethyia 2.7.0 → 2.7.2

package/src/boot/dependencies.ts

@@ -48,6 +48,7 @@ import {
   ValidatedNetworkBindingFactory,
 } from "../network/pool/resolver";
 import { NetworkPoolSelector } from "../network/pool/selector";
+import { restorePoolProbeVerdicts, restoreProviderCooldownMarkers } from "../network/pool-cooldown-store";
 import { ScheduledTaskRegistry } from "../workers/scheduler";
 import { quotaRefreshSweep } from "../workers/quota-refresh-sweep";
 import { markProviderApiKeyCredential } from "../console/quota/account-quota-shared";
@@ -55,15 +56,11 @@ import { dailyCheckinEgressForPass } from "../workers/daily-checkin";
 import {
   buddyAdoptTask,
   buddyActivityReportTask,
-  buddyLotteryTask,
-  buddyRedeemTask,
   buddyStreakTask,
-  buddyTravelTask,
-  buddyTrialTask,
   syncBuddyUsage,
 } from "../workers/buddy-growth-tasks";
 import { createAccountSecretResolver } from "../providers/operations/provider-credential-service";
-import { quotaCacheSize } from "../console/quota/cache";
+import { quotaCacheSize } from "../providers/quota/cache";
 import { modelCatalogCacheSize } from "../providers/operations/model-catalog-cache";
 import { preferencesReaderFor } from "../persistence/tenant-preferences";
 import { sweepExpiredCooldowns } from "../providers/operations/account-health-service";
@@ -260,39 +257,6 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
     redis ? new RedisAdmissionController(redis) : undefined,
     redis,
   );
-  const readRoutingAccountInflight = async (
-    providerId: string,
-    tenantId: string | null,
-  ): Promise<readonly { accountId: string; inflight: number }[]> => {
-    if (providerId === "" || tenantId === null) return [];
-    const accounts = await new DrizzleProviderCatalogStore(db, {
-      telemetryBuffer,
-      bundledModelCatalog: bundledCatalog,
-      providerRegistry: registry,
-      outboundFetchFor: async () => ({ fetch: networkBindingFactory.fetch(undefined, tenantId) }),
-      snapshotInvalidator: snapshotService,
-    }).listAccounts(tenantId, providerId);
-    const owned = new Set(accounts.map((account) => account.id));
-    // Scoped read: only the provider's own candidates for this tenant are
-    // queried, so the console read costs one exact MGET over the accounts it
-    // is about to report on — not a fleet-wide `KEYS` scan whose cost is the
-    // size of every bucket every process ever wrote.
-    const routeSnapshot = await snapshotService.getSnapshot();
-    const ownedCandidates = routeSnapshot.candidates.filter(
-      (candidate) =>
-        candidate.provider_id === providerId &&
-        candidate.provider_account_id !== undefined &&
-        owned.has(candidate.provider_account_id),
-    );
-    const snapshot = await routingEngine.accountInflightSnapshot(ownedCandidates);
-    const totals = new Map<string, number>();
-    for (const [bucket, count] of snapshot) {
-      const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
-      if (!owned.has(accountId)) continue;
-      totals.set(accountId, (totals.get(accountId) ?? 0) + count);
-    }
-    return [...totals].map(([accountId, inflight]) => ({ accountId, inflight }));
-  };
   // Peer processes' settings mutations arrive here. Subscribed once for the
   // process: the revision counter is process-global, and the handler only
   // moves it forward.
@@ -379,6 +343,10 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
   scheduledTasks.register({
     name: "account-health-sweep",
     intervalMs: 30_000,
+    // Offsets the pass from lease-sweep and runtime-metrics, which share the
+    // same cadence: under the registry's single-flight gate, aligned ticks
+    // make the 30s tasks skip each other when one runs long.
+    initialDelayMs: 5_000,
     // Recovers cooldowns deployment-wide and invalidates the shared snapshot.
     shared: true,
     run: async () => {
@@ -420,6 +388,22 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
   });
   const telemetryBuffer = new TelemetryBatchBuffer(db);
   const poolSelector = new NetworkPoolSelector(redis);
+  // Replay the provider cooldown markers the previous process recorded
+  // (`network_pools.provider_cooldowns`). Without this a restart made dispatch
+  // forget every egress the upstream was still refusing: the flags are
+  // process-local otherwise, so the pool re-entered rotation and took 429s the
+  // first request after boot re-discovered. Expired entries are dropped by the
+  // restore itself. Best-effort: a failed read leaves selection matching what
+  // a fresh deployment would do.
+  try {
+    const restored = await restoreProviderCooldownMarkers(poolSelector, db);
+    const replayed = await restorePoolProbeVerdicts(poolSelector, db);
+    if (restored > 0 || replayed > 0) {
+      log.info("[pool-selector] restored provider cooldown state", { mirrored: restored, probeVerdictsApplied: replayed });
+    }
+  } catch (error) {
+    log.warn("[pool-selector] provider cooldown restore failed", { error: String(error) });
+  }
   const poolAgentResolver = new PoolAgentResolver(
     new DrizzleNetworkPoolLoader(db),
     { ssrfPolicy, db },
@@ -429,6 +413,49 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
     undefined,
     poolAgentResolver,
   );
+  // The account-inflight reader the console queries. It lives after the deps
+  // it closes over (telemetry buffer, binding factory), so the closure never
+  // touches a TDZ binding; it also reuses one catalog store per tenant instead
+  // of constructing a fresh store object on every console read.
+  let lastInflightTenant = "";
+  let lastInflightStore: DrizzleProviderCatalogStore | undefined;
+  const readRoutingAccountInflight = async (
+    providerId: string,
+    tenantId: string | null,
+  ): Promise<readonly { accountId: string; inflight: number }[]> => {
+    if (providerId === "" || tenantId === null) return [];
+    if (tenantId !== lastInflightTenant || lastInflightStore === undefined) {
+      lastInflightStore = new DrizzleProviderCatalogStore(db, {
+        telemetryBuffer,
+        bundledModelCatalog: bundledCatalog,
+        providerRegistry: registry,
+        outboundFetchFor: async () => ({ fetch: networkBindingFactory.fetch(undefined, tenantId) }),
+        snapshotInvalidator: snapshotService,
+      });
+      lastInflightTenant = tenantId;
+    }
+    const accounts = await lastInflightStore.listAccounts(tenantId, providerId);
+    const owned = new Set(accounts.map((account) => account.id));
+    // Scoped read: only the provider's own candidates for this tenant are
+    // queried, so the console read costs one exact MGET over the accounts it
+    // is about to report on — not a fleet-wide `KEYS` scan whose cost is the
+    // size of every bucket every process ever wrote.
+    const routeSnapshot = await snapshotService.getSnapshot();
+    const ownedCandidates = routeSnapshot.candidates.filter(
+      (candidate) =>
+        candidate.provider_id === providerId &&
+        candidate.provider_account_id !== undefined &&
+        owned.has(candidate.provider_account_id),
+    );
+    const snapshot = await routingEngine.accountInflightSnapshot(ownedCandidates);
+    const totals = new Map<string, number>();
+    for (const [bucket, count] of snapshot) {
+      const accountId = bucket.slice(bucket.lastIndexOf(":") + 1);
+      if (!owned.has(accountId)) continue;
+      totals.set(accountId, (totals.get(accountId) ?? 0) + count);
+    }
+    return [...totals].map(([accountId, inflight]) => ({ accountId, inflight }));
+  };
   // Runtime gauges (memory + bounded-collection sizes) sampled on the shared
   // maintenance cadence. The sampler reads live sizes through closures so the
   // observability layer never holds direct references to routing/pool state.
@@ -606,36 +633,12 @@ export async function buildProductionDeps(): Promise<ProductionDeps> {
     initialDelayMs: 5 * 60_000,
     run: () => buddyStreakTask(buddyTaskDeps),
   });
-  scheduledTasks.register({
-    name: "buddy-redeem",
-    intervalMs: 24 * 60 * 60_000,
-    initialDelayMs: 8 * 60_000,
-    run: () => buddyRedeemTask(buddyTaskDeps),
-  });
-  scheduledTasks.register({
-    name: "buddy-lottery",
-    intervalMs: 24 * 60 * 60_000,
-    initialDelayMs: 11 * 60_000,
-    run: () => buddyLotteryTask(buddyTaskDeps),
-  });
-  scheduledTasks.register({
-    name: "buddy-travel",
-    intervalMs: 24 * 60 * 60_000,
-    initialDelayMs: 14 * 60_000,
-    run: () => buddyTravelTask(buddyTaskDeps),
-  });
   scheduledTasks.register({
     name: "buddy-adopt",
     intervalMs: 24 * 60 * 60_000,
     initialDelayMs: 17 * 60_000,
     run: () => buddyAdoptTask(buddyTaskDeps),
   });
-  scheduledTasks.register({
-    name: "buddy-trial",
-    intervalMs: 24 * 60 * 60_000,
-    initialDelayMs: 20 * 60_000,
-    run: () => buddyTrialTask(buddyTaskDeps),
-  });
   // Growth-activity report: the `/v2/report` batch that lights the login streak
   // and scores the daily growth event. Once a day with its own ledger — it used
   // to ride the quota sweep with no marker of its own, so its eight POSTs ran

package/src/config/cooldowns.ts

@@ -168,6 +168,21 @@ export function resolveAccountDisabledRecoveryMs(): number {
   return _disabledRecovery;
 }
 
+let _poolProviderRateLimitCooldown: number | undefined;
+/**
+ * Fallback cooldown for a provider-scoped egress 429 when the upstream named no
+ * window (default 1h). See `CARTETHYIA_POOL_PROVIDER_RATE_LIMIT_COOLDOWN_MS`.
+ * An upstream `retryAfterMs` always wins over this.
+ */
+export function resolvePoolProviderRateLimitCooldownMs(): number {
+  if (_poolProviderRateLimitCooldown === undefined)
+    _poolProviderRateLimitCooldown = readInt(
+    "CARTETHYIA_POOL_PROVIDER_RATE_LIMIT_COOLDOWN_MS",
+    CONFIG_SPEC.CARTETHYIA_POOL_PROVIDER_RATE_LIMIT_COOLDOWN_MS,
+  );
+  return _poolProviderRateLimitCooldown;
+}
+
 /**
  * Resets all cached cooldown values. Call this in tests that override
  * cooldown env-vars between test cases.
@@ -180,6 +195,7 @@ export function resetCooldownCacheForTests(): void {
   _unclassifiedCooldown = undefined;
   _escalationMaxCooldown = undefined;
   _poolCooldown = undefined;
+  _poolProviderRateLimitCooldown = undefined;
   _authRejectedCooldown = undefined;
   _policyBlockCooldown = undefined;
   _disabledRecovery = undefined;

package/src/config/providers.ts

@@ -4,7 +4,7 @@
  * public origin.
  */
 
-import { CONFIG_SPEC, readInt, readRequired, readText } from "./spec";
+import { CONFIG_SPEC, readInt, readOptionalText, readRequired, readText } from "./spec";
 
 /** Reads and normalizes the public OAuth origin. */
 export function requirePublicOrigin(): string {
@@ -48,6 +48,18 @@ export function resolveKimiQuotaBaseUrl(): string {
   );
 }
 
+/**
+ * Operator override for the directory installation identities are persisted in.
+ * Read here so the only literal `process.env` for this value stays in config;
+ * the providers that stamp an install id consume the resolver instead.
+ */
+export function resolveInstallIdDir(): string | undefined {
+  return readOptionalText(
+    "CARTETHYIA_INSTALL_ID_DIR",
+    CONFIG_SPEC.CARTETHYIA_INSTALL_ID_DIR,
+  );
+}
+
 /**
  * Operator override for the OpenCode free-tier ids upstream refuses, or
  * `undefined` when unset — meaning the bundled, measured list stands.

package/src/config/resources.ts

@@ -1,70 +1,70 @@
-/**
- * Canonical resource policy: how much memory this process may use, and how
- * much room it must keep free.
- *
- * The memory figure used to be decided in unrelated places — a host-wide
- * `os.totalmem()` in the headroom guard, a fallback constant in
- * `runtime-metrics.ts` — which meant a container limited to 512 MiB shed on
- * the same threshold as a 4 GiB one and only reacted once the *host* was
- * nearly full. On a 1-core / 2 GiB VPS that is the difference between
- * shedding early and being OOM-killed, and the OOM kills the dashboard too —
- * the one surface that would have explained what happened.
- *
- * How many requests may run at once is NOT decided here. That is admission's
- * job: the tenant ceiling and each key's own limit, both set in Settings. A
- * second, process-wide ceiling derived from memory sat below the number an
- * operator could configure and refused requests that every configured limit
- * still allowed, which is why it is gone.
- */
-import { CONFIG_SPEC, readOptionalInt } from "./spec";
-
-/** Used when no limit is discoverable at all (native dev, macOS, Windows). */
-export const FALLBACK_MEMORY_LIMIT_BYTES = 512 * 1024 * 1024;
-
-
-/**
- * How many copies of a request body are resident at once.
- *
- * Measured, not assumed: the socket read buffer, the parsed object graph, and
- * the upstream re-serialization each hold a body-sized allocation, and garbage
- * collection lags rather than keeping them to one. Budgeting one copy halves
- * the real peak and lets the process admit twice the concurrency it can hold.
- */
-export const REQUEST_BODY_TRANSIENT_COPIES = 2;
-
-/** Share of the memory limit all in-flight request bodies together may hold. */
-export const REQUEST_BODY_BUDGET_FRACTION = 0.25;
-
-/** Share of the memory limit kept free for allocation spikes and GC slack. */
-export const MEMORY_HEADROOM_FRACTION = 0.2;
-
-
-/**
- * Bytes kept free below the memory limit.
- *
- * A guard that only reacts once RSS equals the limit has already lost: the
- * allocation that breaches it happens between two samples, and the spike that
- * matters — an upstream response landing all at once — arrives in far less
- * than one sample interval.
- */
-export function deriveMemoryHeadroomBytes(memoryLimitBytes: number): number {
-  return Math.ceil(memoryLimitBytes * MEMORY_HEADROOM_FRACTION);
-}
-
-
-/**
- * Slack added on top of every key's concurrency limit.
- *
- * Operator configuration, not something a key carries about itself: it is
- * deliberate hidden allowance so a run that lands a few requests over the
- * number an operator chose is not refused at that number. The effective
- * ceiling is `limit + tolerance`, and only requests past it are refused.
- */
-export function resolveConcurrencyTolerance(): number {
-  const value = readOptionalInt(
-    "CARTETHYIA_CONCURRENCY_TOLERANCE",
-    CONFIG_SPEC.CARTETHYIA_CONCURRENCY_TOLERANCE,
-  );
-  return value === undefined ? 0 : Math.max(0, value);
-}
-
+/**
+ * Canonical resource policy: how much memory this process may use, and how
+ * much room it must keep free.
+ *
+ * The memory figure used to be decided in unrelated places — a host-wide
+ * `os.totalmem()` in the headroom guard, a fallback constant in
+ * `runtime-metrics.ts` — which meant a container limited to 512 MiB shed on
+ * the same threshold as a 4 GiB one and only reacted once the *host* was
+ * nearly full. On a 1-core / 2 GiB VPS that is the difference between
+ * shedding early and being OOM-killed, and the OOM kills the dashboard too —
+ * the one surface that would have explained what happened.
+ *
+ * How many requests may run at once is NOT decided here. That is admission's
+ * job: the tenant ceiling and each key's own limit, both set in Settings. A
+ * second, process-wide ceiling derived from memory sat below the number an
+ * operator could configure and refused requests that every configured limit
+ * still allowed, which is why it is gone.
+ */
+import { CONFIG_SPEC, readOptionalInt } from "./spec";
+
+/** Used when no limit is discoverable at all (native dev, macOS, Windows). */
+export const FALLBACK_MEMORY_LIMIT_BYTES = 512 * 1024 * 1024;
+
+
+/**
+ * How many copies of a request body are resident at once.
+ *
+ * Measured, not assumed: the socket read buffer, the parsed object graph, and
+ * the upstream re-serialization each hold a body-sized allocation, and garbage
+ * collection lags rather than keeping them to one. Budgeting one copy halves
+ * the real peak and lets the process admit twice the concurrency it can hold.
+ */
+export const REQUEST_BODY_TRANSIENT_COPIES = 2;
+
+/** Share of the memory limit all in-flight request bodies together may hold. */
+export const REQUEST_BODY_BUDGET_FRACTION = 0.25;
+
+/** Share of the memory limit kept free for allocation spikes and GC slack. */
+export const MEMORY_HEADROOM_FRACTION = 0.2;
+
+
+/**
+ * Bytes kept free below the memory limit.
+ *
+ * A guard that only reacts once RSS equals the limit has already lost: the
+ * allocation that breaches it happens between two samples, and the spike that
+ * matters — an upstream response landing all at once — arrives in far less
+ * than one sample interval.
+ */
+export function deriveMemoryHeadroomBytes(memoryLimitBytes: number): number {
+  return Math.ceil(memoryLimitBytes * MEMORY_HEADROOM_FRACTION);
+}
+
+
+/**
+ * Slack added on top of every key's concurrency limit.
+ *
+ * Operator configuration, not something a key carries about itself: it is
+ * deliberate hidden allowance so a run that lands a few requests over the
+ * number an operator chose is not refused at that number. The effective
+ * ceiling is `limit + tolerance`, and only requests past it are refused.
+ */
+export function resolveConcurrencyTolerance(): number {
+  const value = readOptionalInt(
+    "CARTETHYIA_CONCURRENCY_TOLERANCE",
+    CONFIG_SPEC.CARTETHYIA_CONCURRENCY_TOLERANCE,
+  );
+  return value === undefined ? 0 : Math.max(0, value);
+}
+

package/src/config/security.ts

@@ -1,37 +1,37 @@
-/**
- * Security resolvers: the application encryption key and the abuse-protection
- * ceiling.
- */
-
-import { CONFIG_SPEC, readInt, readRequired } from "./spec";
-
-const ENCRYPTION_KEY_LENGTH = 32;
-
-/**
- * Per-IP request ceiling for the unauthenticated abuse-protection layer.
- * Defaults to 240 requests per window so an unset env does not mean unlimited;
- * operators tune it via `IP_RATE_MAX_PER_WINDOW`.
- */
-export function resolveIpRateLimit(): number {
-  return readInt("IP_RATE_MAX_PER_WINDOW", CONFIG_SPEC.IP_RATE_MAX_PER_WINDOW);
-}
-
-/** Reads the application encryption key without caching it. */
-export function requireEncryptionKeyEnv(): string {
-  return readRequired("CARTETHYIA_ENCRYPTION_KEY", CONFIG_SPEC.CARTETHYIA_ENCRYPTION_KEY);
-}
-
-/** Decodes and validates a base64 or hexadecimal 256-bit encryption key. */
-export function decodeEncryptionKey(raw: string): Buffer {
-  const candidate = raw.trim();
-  const buf = /^[0-9a-fA-F]{64}$/.test(candidate)
-    ? Buffer.from(candidate, "hex")
-    : Buffer.from(candidate, "base64");
-  if (buf.length !== ENCRYPTION_KEY_LENGTH) {
-    throw new Error(
-      `CARTETHYIA_ENCRYPTION_KEY must decode to exactly ${ENCRYPTION_KEY_LENGTH} bytes (got ${buf.length}); ` +
-        "provide a base64 or hex encoded 256-bit key",
-    );
-  }
-  return buf;
-}
+/**
+ * Security resolvers: the application encryption key and the abuse-protection
+ * ceiling.
+ */
+
+import { CONFIG_SPEC, readInt, readRequired } from "./spec";
+
+const ENCRYPTION_KEY_LENGTH = 32;
+
+/**
+ * Per-IP request ceiling for the unauthenticated abuse-protection layer.
+ * Defaults to 240 requests per window so an unset env does not mean unlimited;
+ * operators tune it via `IP_RATE_MAX_PER_WINDOW`.
+ */
+export function resolveIpRateLimit(): number {
+  return readInt("IP_RATE_MAX_PER_WINDOW", CONFIG_SPEC.IP_RATE_MAX_PER_WINDOW);
+}
+
+/** Reads the application encryption key without caching it. */
+export function requireEncryptionKeyEnv(): string {
+  return readRequired("CARTETHYIA_ENCRYPTION_KEY", CONFIG_SPEC.CARTETHYIA_ENCRYPTION_KEY);
+}
+
+/** Decodes and validates a base64 or hexadecimal 256-bit encryption key. */
+export function decodeEncryptionKey(raw: string): Buffer {
+  const candidate = raw.trim();
+  const buf = /^[0-9a-fA-F]{64}$/.test(candidate)
+    ? Buffer.from(candidate, "hex")
+    : Buffer.from(candidate, "base64");
+  if (buf.length !== ENCRYPTION_KEY_LENGTH) {
+    throw new Error(
+      `CARTETHYIA_ENCRYPTION_KEY must decode to exactly ${ENCRYPTION_KEY_LENGTH} bytes (got ${buf.length}); ` +
+        "provide a base64 or hex encoded 256-bit key",
+    );
+  }
+  return buf;
+}