cartethyia 2.7.3 → 2.7.5

package/src/console/quota/account-quota-tenant.ts

@@ -4,17 +4,22 @@ import type { Elysia } from "elysia";
 import { and, eq, inArray } from "drizzle-orm";
 import { providerAccounts, providers } from "../../persistence/schema";
 import { providerDisplayName } from "../../providers/provider-metadata";
-import { invalidateCredentialCache } from "../../providers/operations/provider-credential-service";
+import {
+  invalidateCredentialCache,
+  invalidateCredentialCaches,
+} from "../../providers/operations/provider-credential-service";
 import { requireTenantScope } from "../shared/errors";
 import { BUDDY_PROVIDER_IDS } from "../../providers/provider-metadata";
-import { readBuddyUsageSnapshots } from "../../providers/integrations/buddy/buddy-usage-store";
 import { GatewayError } from "../../protocol/gateway-error";
 import {
   GLOBAL_QUOTA_LENS,
   getCachedQuotaEntries,
   getCachedQuotaEntry,
   invalidateQuotaCache,
+  invalidateQuotaCaches,
+  invalidateQuotaCacheForAccount,
 } from "../../providers/quota/cache";
+import { readQuotaSnapshot, readQuotaSnapshots } from "../../providers/quota/snapshot";
 import {
   QUOTA_REFRESH_TIMEOUT_MS,
   refreshAccountQuota,
@@ -47,13 +52,16 @@ import {
   DrizzleQoderNativeIdentityStore,
   type QoderNativeIdentityStore,
 } from "../../providers/integrations/qoder/qoder-native-identity-store";
-import { buddyAccountUid } from "../../providers/integrations/buddy/buddy-oauth-shared";
+import { claimZcodeGifts, type ZcodeGiftResult } from "../../providers/integrations/zcode-start-plan/gifts";
+import { ZCODE_START_PLAN_PROVIDER_ID } from "../../providers/integrations/zcode-start-plan/provider";
 import {
   supportsAccountReset,
   consumeAccountResetCredit,
+  getCachedResetCredits,
   listAccountResetCredits,
 } from "../../providers/operations/account-reset-service";
 import {
+  enqueueBackgroundQuotaRefresh,
   pendingQuotaRefreshes,
   toQuotaAccountHealth,
   toQuotaView,
@@ -63,6 +71,14 @@ import {
   type AccountQuotaRoutesDeps,
   type AccountStatus,
 } from "./account-quota-shared";
+import {
+  DAILY_GROWTH_LEG_TIMEOUT_MS,
+  enqueueDailyGrowthJob,
+  getDailyGrowthJob,
+  withLegBudget,
+  type DailyGrowthJobAccount,
+  type DailyGrowthJobDeps,
+} from "./daily-growth-job";
 
 /** Daily check-in only exists on the buddy billing facade; everything else 400s with intent. */
 function supportsAccountCheckin(providerId: string): boolean {
@@ -132,10 +148,6 @@ async function runAccountCheckin(
   });
 }
 
-/** Bound on accounts one manual daily-growth batch may cover. */
-/** Concurrent accounts inside one manual daily-growth batch. */
-const DAILY_GROWTH_CONCURRENCY = 3;
-
 /**
  * One account's daily growth pass, report first then claim.
  *
@@ -174,6 +186,12 @@ async function runDailyGrowthForAccount(
   // fails closed instead of reporting under another account's identity. Qoder
   // has no activity-report leg — its daily grant is purely the check-in.
   const qoderAccount = isQoderCheckinProvider(String(providerId));
+  // dynamic-import: B-02 lazy graph — buddy uid decoder, only for buddy report legs.
+  // The buddy-oauth-shared graph is lazy-loaded: the uid decoder is only
+  // needed for buddy accounts' report leg (B-02).
+  const { buddyAccountUid } = await import(
+    "../../providers/integrations/buddy/buddy-oauth-shared"
+  );
   const userId = buddyAccountUid(credential);
   let report: { ok: boolean; message: string };
   if (qoderAccount) {
@@ -352,6 +370,13 @@ export function registerAccountQuotaTenantRoutes(
         getCachedQuotaEntries(access.tenantId, ownedIds, redis),
         getCachedQuotaEntries(GLOBAL_QUOTA_LENS, globalIds, redis),
       ]);
+      // A page open right after a restart finds the in-process cache empty.
+      // Fall back to the persisted snapshots (which survive a restart) so the
+      // page still renders the last known balance; the sweep refreshes it.
+      const snapshots = await readQuotaSnapshots(
+        db,
+        rows.filter((row) => !globalCache.has(row.id) && !ownedCache.has(row.id)).map((row) => row.id),
+      );
 
       const pending = pendingQuotaRefreshes();
       const providerIds = [...new Set(rows.map((row) => row.providerId))];
@@ -369,6 +394,15 @@ export function registerAccountQuotaTenantRoutes(
           icon: providerId,
         }));
       let refreshing = 0;
+      // Cold-cache backfill: the overview renders whatever is cached, and the
+      // periodic sweep owns freshness — but with many accounts (e.g. a large
+      // Qoder roster) the sweep's per-pass cap cannot keep every cache warm,
+      // so a page open would show accounts with no badge for minutes. Enqueue
+      // a bounded background refresh for accounts whose cache entry is missing
+      // (not merely stale), so the badge fills in without waiting for the next
+      // sweep pass and without fanning out to the whole roster at once.
+      const coldCacheCap = 20;
+      let coldCacheEnqueued = 0;
       // The sweep's day markers answer "attempted today" without an upstream
       // request; only check-in-capable accounts can carry one, so only they
       // are read.
@@ -385,6 +419,12 @@ export function registerAccountQuotaTenantRoutes(
       // the upstream: the page renders every account at once, and re-hitting the
       // billing endpoint per account would make opening it scale with the
       // account count. One batch read for every buddy row.
+      // dynamic-import: B-02 lazy graph — buddy usage snapshots, only when the page renders buddy accounts.
+      // The buddy-usage-store graph is lazy-loaded: it is only needed when the
+      // page actually renders buddy accounts (B-02).
+      const { readBuddyUsageSnapshots } = await import(
+        "../../providers/integrations/buddy/buddy-usage-store"
+      );
       const usageByAccount = await readBuddyUsageSnapshots(
         db,
         rows.filter((row) => isBuddyProvider(row.providerId)).map((row) => row.id),
@@ -392,6 +432,9 @@ export function registerAccountQuotaTenantRoutes(
       const accounts = rows.map((row) => {
         const entry =
           row.tenantId === null ? globalCache.get(row.id) : ownedCache.get(row.id);
+        const snapshot = entry ?? snapshots.get(row.id);
+        const entryQuota = snapshot ? snapshot.quota : null;
+        const entryFetchedAt = snapshot ? snapshot.fetchedAt : null;
         // No background refresh is enqueued here: enqueuing one for every
         // account whose cached value had aged out means a single page open hits
         // every provider's quota endpoint at once — and because the page also
@@ -404,19 +447,34 @@ export function registerAccountQuotaTenantRoutes(
         const canRefresh = providerRegistry.hasQuotaCollector(row.providerId);
         const isRefreshing = canRefresh && (pending.has(row.id) || entry === undefined);
         if (isRefreshing) refreshing += 1;
+        // Backfill a missing cache entry on page open (bounded, see above).
+        if (
+          canRefresh &&
+          entry === undefined &&
+          !pending.has(row.id) &&
+          coldCacheEnqueued < coldCacheCap
+        ) {
+          coldCacheEnqueued += 1;
+          enqueueBackgroundQuotaRefresh(refreshDeps, {
+            accountId: row.id,
+            providerId: row.providerId,
+            tenantId: row.tenantId,
+            credentialKind: row.credentialKind,
+          });
+        }
         return {
           id: row.id,
           provider: row.providerId,
           name: row.label || row.id,
           credentialHint: row.credentialKind,
           active: row.status === "active",
-          quota: entry
-            ? toQuotaView(entry.quota, row, {
-                fetchedAt: entry.fetchedAt,
-              lastAttemptAt: entry.lastAttemptAt,
-              refreshError: entry.refreshError,
-              ...(isRefreshing ? { status: "refreshing" as const } : {}),
-            })
+          quota: entryQuota
+            ? toQuotaView(entryQuota, row, {
+                fetchedAt: entryFetchedAt,
+                lastAttemptAt: entry?.lastAttemptAt ?? null,
+                refreshError: entry?.refreshError ?? null,
+                ...(isRefreshing ? { status: "refreshing" as const } : {}),
+              })
             : null,
           pending: isRefreshing,
           health: toQuotaAccountHealth(row),
@@ -478,13 +536,33 @@ export function registerAccountQuotaTenantRoutes(
           health: toQuotaAccountHealth(account),
         };
       }
+      // Cache is cold (e.g. right after a restart): render the last persisted
+      // snapshot instead of hitting upstream on every page open. The sweep
+      // refreshes it; a page open does not need to pay the upstream cost.
+      const snapshot = await readQuotaSnapshot(db, accountId);
+      if (snapshot) {
+        return {
+          accountId,
+          quota: toQuotaView(snapshot.quota, account, {
+            fetchedAt: snapshot.fetchedAt,
+            lastAttemptAt: snapshot.fetchedAt,
+          }),
+          health: toQuotaAccountHealth(account),
+        };
+      }
 
       // Cold read: this is the one path that legitimately waits on upstream, and
       // it shares the in-flight fetch with any queued background fill so a cold
       // page open never issues two requests for the same account.
       const outcome = await refreshAccountQuota(
         refreshDeps,
-        { accountId, providerId: account.providerId, tenantId: account.tenantId, credentialKind: account.credentialKind },
+        {
+          accountId,
+          providerId: account.providerId,
+          tenantId: account.tenantId,
+          credentialKind: account.credentialKind,
+          authState: account.authState,
+        },
         signalFetch(timeoutSignal(request.signal, QUOTA_REFRESH_TIMEOUT_MS)),
       );
       const fresh = await getCachedQuotaEntry(access.tenantId, accountId, redis);
@@ -525,7 +603,7 @@ export function registerAccountQuotaTenantRoutes(
         throw new GatewayError(
           "proxy_pool_unavailable",
           503,
-          "Account test cannot select a proxy because proxy routing is not configured.",
+          "Account test cannot select a proxy because proxy routing is not configured (request {requestId}). No bridge to cross — configure a pool first 🏗️",
           { reason: "proxy_routing_unconfigured" },
           "network",
         );
@@ -546,7 +624,16 @@ export function registerAccountQuotaTenantRoutes(
       try {
         outcome = await refreshAccountQuota(
           refreshDeps,
-          { accountId, providerId: account.providerId, tenantId: account.tenantId, credentialKind: account.credentialKind },
+          {
+            accountId,
+            providerId: account.providerId,
+            tenantId: account.tenantId,
+            credentialKind: account.credentialKind,
+            // Kiro's usage surface is profile-scoped: without the account's auth
+            // state the collector cannot resolve a `profileArn`, and the GET is
+            // refused with `400 Invalid profileArn.`.
+            authState: account.authState,
+          },
           egress.fetch,
         );
       } finally {
@@ -591,6 +678,7 @@ export function registerAccountQuotaTenantRoutes(
             providerId: providerAccounts.providerId,
             tenantId: providerAccounts.tenantId,
             credentialKind: providerAccounts.credentialKind,
+            authState: providerAccounts.authState,
           })
           .from(providerAccounts)
           .where(
@@ -751,17 +839,60 @@ export function registerAccountQuotaTenantRoutes(
       });
       return { success: true };
     })
+    .post(
+      "/accounts/batch-delete",
+      {
+        body: t.Object({
+          accountIds: t.Array(t.String()),
+        }),
+      },
+      async ({ request, body }) => {
+        const access = requireTenantScope(accessResolver(request), "dashboard:write");
+        const accountIds = [...new Set(body.accountIds)];
+        if (accountIds.length === 0) {
+          return { ok: true, deleted: [], missing: [] };
+        }
+        // One DELETE statement for the whole batch, scoped to the tenant: a
+        // global/shared account (`tenant_id IS NULL`) is never matchable, so
+        // it lands in `missing` rather than being deleted by a tenant.
+        const deleted = await db
+          .delete(providerAccounts)
+          .where(
+            and(
+              inArray(providerAccounts.id, accountIds),
+              eq(providerAccounts.tenantId, access.tenantId),
+            ),
+          )
+          .returning({ id: providerAccounts.id });
+        const deletedIds = deleted.map((row) => row.id);
+        const deletedSet = new Set(deletedIds);
+        const missing = accountIds.filter((id) => !deletedSet.has(id));
+        // One credential-cache pass, one quota-cache pass, one snapshot
+        // invalidation, one audit row — never one per deleted account.
+        invalidateCredentialCaches(deletedIds);
+        await invalidateQuotaCaches(access.tenantId, deletedIds, redis);
+        await snapshotInvalidator?.invalidate();
+        await auditRecorder?.record({
+          access,
+          action: "provider_account.batch_deleted",
+          target: access.tenantId,
+          detail: { deleted: deletedIds, missing },
+        });
+        return { ok: true, deleted: deletedIds, missing };
+      },
+    )
     /**
-     * Runs the daily growth pass over a provider's accounts in one request:
-     * the growth-activity report first, then the check-in claim — the order the
+     * Enqueues a background daily-growth job over a provider's accounts: the
+     * growth-activity report first, then the check-in claim — the order the
      * sweep uses — with egress rotated per account the same way the worker
      * does, so a batch leaves from more than one address.
      *
-     * A manual trigger is the operator asserting "try now", so it bypasses the
-     * sweep's once-per-day ledger; the upstream stays idempotent either way.
-     * Per-account outcomes are reported individually: a batch is not
-     * all-or-nothing, and one account's closed check-in must not hide the
-     * others' streaks.
+     * Running the whole batch in the request held it open for every account's
+     * upstream calls; the route now returns `{ jobId }` immediately and the
+     * work drains through a bounded background worker (see the job status
+     * route below). A manual trigger is the operator asserting "try now", so
+     * it bypasses the sweep's once-per-day ledger; the upstream stays
+     * idempotent either way.
      */
     .post(
       "/quota/daily-growth",
@@ -807,59 +938,66 @@ export function registerAccountQuotaTenantRoutes(
           };
         }
         // No size cap, for the same reason as the quota refresh: the work is
-        // already drained through a bounded worker pool, so refusing the request
-        // protected nothing and just forced the operator to batch by hand.
-        // One egress rotator for the whole request, exactly as the sweep builds
-        // it, so consecutive accounts leave from consecutive pools.
+        // already drained through a bounded worker pool behind the job, so
+        // refusing the request protected nothing and just forced the operator
+        // to batch by hand.
+        //
+        // One egress rotator for the whole job, exactly as the sweep builds it,
+        // so consecutive accounts leave from consecutive pools. The job carries
+        // the per-account run (report then claim) through the injected runner.
         const rotator =
           networkBindingFactory !== undefined
             ? dailyCheckinEgressForPass({ db, networkBindingFactory })
             : undefined;
+        // One leg budget for the whole job: every upstream fetch the per-account
+        // run issues carries it on top of the helpers' own per-request timeouts.
+        const legSignal = AbortSignal.timeout(DAILY_GROWTH_LEG_TIMEOUT_MS);
+        const rotatorWithBudget =
+          rotator === undefined
+            ? undefined
+            : async (accountId: string, tenantId?: string | null) => {
+                const base = await rotator(accountId, tenantId);
+                return withLegBudget(base, legSignal) as unknown as ValidatedFetch;
+              };
 
-        const results: {
-          readonly accountId: string;
-          readonly label: string;
-          readonly report: { readonly ok: boolean; readonly message: string };
-          readonly checkin: { readonly ok: boolean; readonly message: string };
-        }[] = [];
-
-        for (let offset = 0; offset < accounts.length; offset += DAILY_GROWTH_CONCURRENCY) {
-          const batch = accounts.slice(offset, offset + DAILY_GROWTH_CONCURRENCY);
-          const settled = await Promise.all(
-            batch.map(async (account) => {
-              const outcome = await runDailyGrowthForAccount(
-                providerId as ProviderId,
-                account,
-                refreshDeps,
-                rotator,
-              );
-              return { account, outcome };
-            }),
+        const runAccount = async (account: DailyGrowthJobAccount) => {
+          const outcome = await runDailyGrowthForAccount(
+            providerId as ProviderId,
+            account,
+            refreshDeps,
+            rotatorWithBudget,
           );
-          for (const { account, outcome } of settled) {
-            results.push({
-              accountId: account.id,
-              label: account.label || account.id,
-              report: outcome.report,
-              checkin: outcome.checkin,
-            });
+          if (!outcome.checkin.ok) {
+            return { ok: false, error: outcome.checkin.message };
           }
-        }
+          return { ok: true };
+        };
 
-        // The ledger is shared with the sweep, so invalidate once at the end
-        // rather than per account — the page re-reads "attempted today" from it.
-        await snapshotInvalidator?.invalidate();
-        const reportsOk = results.filter((entry) => entry.report.ok).length;
-        const claimed = results.filter((entry) => entry.checkin.ok).length;
-        return {
-          ok: true,
-          status: 200,
-          providerId,
-          data: results,
-          message: `${results.length} account${results.length === 1 ? "" : "s"} · report ${reportsOk}/${results.length} · check-in ${claimed}/${results.length}`,
+        const jobDeps: DailyGrowthJobDeps = {
+          db,
+          redis,
+          runAccount,
+          ...(snapshotInvalidator ? { snapshotInvalidator } : {}),
         };
+        const jobId = enqueueDailyGrowthJob({
+          deps: jobDeps,
+          tenantId: access.tenantId,
+          providerId,
+          accounts,
+        });
+        set.status = 202;
+        return { jobId, status: 202 };
       },
     )
+    .get("/quota/daily-growth/status/:jobId", async ({ request, params, set }) => {
+      const access = requireTenantScope(accessResolver(request), "dashboard:read");
+      const job = getDailyGrowthJob(params.jobId, access.tenantId);
+      if (job === null) {
+        set.status = 404;
+        return { error: `Job ${params.jobId} not found`, code: "not_found" };
+      }
+      return job;
+    })
     .post("/accounts/:id/checkin", async ({ request, params, set }) => {
       const access = requireTenantScope(accessResolver(request), "dashboard:write");
       const rows = await db
@@ -929,6 +1067,75 @@ export function registerAccountQuotaTenantRoutes(
         message: checkinStateMessage(result),
       };
     })
+    // Gift grants are an explicit action, not a side effect of a quota read.
+    .post("/accounts/:id/claim-gifts", async ({ request, params, set }) => {
+      const access = requireTenantScope(accessResolver(request), "dashboard:write");
+      const rows = await db
+        .select()
+        .from(providerAccounts)
+        .where(
+          and(eq(providerAccounts.id, params.id), eq(providerAccounts.tenantId, access.tenantId)),
+        )
+        .limit(1);
+      const account = rows[0];
+      if (!account) {
+        set.status = 404;
+        return { error: `Account ${params.id} not found`, code: "not_found" };
+      }
+      if (account.providerId !== ZCODE_START_PLAN_PROVIDER_ID) {
+        set.status = 400;
+        return {
+          error: `Gift plans are only claimable for ${ZCODE_START_PLAN_PROVIDER_ID}`,
+          code: "gifts_unsupported",
+        };
+      }
+      let result: ZcodeGiftResult;
+      try {
+        const credential = await refreshDeps.resolveCredential(account.providerId as ProviderId, account.id);
+        const fetcher = networkBindingFactory
+          ? await dailyCheckinEgressForPass({ db, networkBindingFactory })(account.id, access.tenantId)
+          : globalThis.fetch;
+        result = await claimZcodeGifts({ accountId: account.id, credential, fetcher });
+      } catch {
+        set.status = 502;
+        return {
+          ok: false, status: 502,
+          data: { id: account.id, state: "error", claimed: [], detail: "Gift claim request failed" },
+          message: "Gift claim request failed", code: "gift_claim_failed",
+        };
+      }
+      if (result.claimed.length > 0) {
+        // Cold readers must not resurrect the balance from before the grant.
+        await db.update(providerAccounts).set({ quotaSnapshot: null }).where(
+          and(eq(providerAccounts.id, account.id), eq(providerAccounts.tenantId, access.tenantId)),
+        );
+        await invalidateQuotaCacheForAccount(account.id, redis);
+        await snapshotInvalidator?.invalidate();
+      }
+      if (result.state === "error") {
+        set.status = 502;
+        return {
+          ok: false,
+          status: 502,
+          data: { id: account.id, ...result },
+          message: result.detail ?? "Gift claim failed",
+          code: "gift_claim_failed",
+        };
+      }
+      return {
+        ok: true,
+        status: 200,
+        data: { id: account.id, ...result },
+        message:
+          result.state === "nothing"
+            ? "No gift plan to claim"
+            : result.state === "claimed"
+              ? `Claimed ${result.claimed.join(", ")}`
+              : result.state === "already_claimed"
+                ? "Every gift plan was claimed before"
+                : (result.detail ?? "Claim it in the ZCode app"),
+      };
+    })
     .post("/accounts/:id/activity-report", async ({ request, params, set }) => {
       const access = requireTenantScope(accessResolver(request), "dashboard:write");
       const rows = await db
@@ -977,6 +1184,12 @@ export function registerAccountQuotaTenantRoutes(
         // The growth event scores by uid, and only a JWT carries one. An
         // opaque API key must fail closed here — reporting it under another
         // account's uid would poison someone else's streak.
+        // dynamic-import: B-02 lazy graph — buddy uid decoder on this buddy-only surface.
+        // The buddy-oauth-shared graph is lazy-loaded: the uid decoder is only
+        // needed on this buddy-only surface (B-02).
+        const { buddyAccountUid } = await import(
+          "../../providers/integrations/buddy/buddy-oauth-shared"
+        );
         const userId = buddyAccountUid(credential);
         if (!userId) {
           set.status = 400;
@@ -1043,7 +1256,12 @@ export function registerAccountQuotaTenantRoutes(
           account.providerId as ProviderId,
           account.id,
         );
-        const list = await listAccountResetCredits(account.providerId, credential);
+        // Cached per account for a short TTL: every dialog open was hitting the
+        // upstream live. The cache is cleared on a successful redemption, so a
+        // consumed credit is not shown again for the rest of the TTL.
+        const list = await getCachedResetCredits(account.providerId, account.id, () =>
+          listAccountResetCredits(account.providerId, credential),
+        );
         return list ?? { availableCount: 0, credits: [] };
       } catch (err) {
         return { availableCount: 0, credits: [] };

package/src/console/quota/account-test-egress.ts

@@ -138,7 +138,7 @@ export async function selectAccountTestEgress(
     throw new GatewayError(
       "proxy_pool_unavailable",
       503,
-      "No active proxy pool is available because coordination is unavailable.",
+      "No active proxy pool is available because coordination is unavailable (request {requestId}). The bouncer stepped out — come back shortly 🚪",
       { reason: "coordination_unavailable" },
       "network",
     );
@@ -159,10 +159,10 @@ export async function selectAccountTestEgress(
     const status = failure.reason === "at_capacity" ? 429 : 503;
     const message =
       failure.reason === "at_capacity"
-        ? "Every proxy pool for this provider is at its concurrency limit. Retry shortly, or add a pool or raise its capacity."
+        ? "Every proxy pool for this provider is at its concurrency limit (request {requestId}). Retry shortly, or add a pool or raise its capacity. The pools are full right now — chill 🌊"
         : failure.reason === "cooldown"
-          ? "Every proxy pool for this provider is cooling down after upstream failures. Retry once the cooldown resets."
-          : "No active proxy pool is configured for this tenant and provider. Add one, or route this provider through direct egress.";
+          ? "Every proxy pool for this provider is cooling down after upstream failures (request {requestId}). Retry once the cooldown resets. The proxies are catching their breath — zzz 💤"
+          : "No active proxy pool is configured for this tenant and provider (request {requestId}). Add one, or route this provider through direct egress. No bridge to cross — configure a pool first 🏗️";
     throw new GatewayError(
       code,
       status,