agent-nuvira 3.1.1 → 3.2.0

package/dist/web-dashboard/server.js

@@ -25,11 +25,14 @@ import { getAutoRouter } from '../learning/auto-router.js';
 import { readRecallHits } from '../context/session-recall.js';
 import { getRouterPromotion } from '../learning/router-promotion.js';
 import { AUTH_CLEAR_THRESHOLD } from '../learning/key-hygiene.js';
-import { ACTION_LOG_FILENAME, aggregateActionTelemetry, readActionTelemetryFile } from '../learning/model-registry.js';
+import { ACTION_LOG_FILENAME, aggregateActionTelemetry, readActionTelemetryFile, getModelRegistry, } from '../learning/model-registry.js';
+import { classifyModelEntitlement } from '../inference/model-entitlement.js';
 import { AdminSessions, countAdminRoleUsers, isAdminConfigured, listAdminUsers, removeAdminUser, roleForUser, verifyAdmin, writeAdminUser, MIN_ADMIN_PASSWORD_LENGTH, } from './src/admin-auth.js';
 import { TaskRunner } from './task-runner.js';
 import { WhatsAppPairingManager } from './whatsapp-pairing.js';
 import { ChatConsole, newChatSessionId } from './chat-console.js';
+import { ChatRetryBroker } from './chat-retry.js';
+import { markFailoverAttempts } from '../learning/resilient-call.js';
 import { buildProjectContext, formatProjectText } from './project-context.js';
 import { isVaultRef } from '../enterprise/vault.js';
 import { ROLES, roleCan } from '../enterprise/rbac.js';
@@ -322,6 +325,50 @@ export function broadcastTyping(event) {
         catch { /* client disconnected */ }
     }
 }
+/**
+ * Push a deferred-retry update to the open dashboard.
+ *
+ * The retry runs minutes or hours after the turn that queued it, long after the
+ * per-session chat stream has been torn down (the client unsubscribes when the
+ * POST resolves), so it rides the PERSISTENT `/api/sse` channel the app opens
+ * once at load. The session id is in the payload because that channel is shared
+ * by every session and by every other panel.
+ */
+export function broadcastChatRetry(event) {
+    const payload = `event: chat-retry\ndata: ${JSON.stringify({ ...event, serverTime: Date.now() })}\n\n`;
+    for (const client of sseClients) {
+        try {
+            client.res.write(payload);
+        }
+        catch { /* client disconnected */ }
+    }
+}
+/**
+ * The deferred-retry drain (one timer per process; idempotent).
+ *
+ * Armed with the server and cleared on shutdown. A retry is meant to outlive the
+ * user's attention — that is the whole point of the offer — so unlike the quota
+ * watcher this is NOT tied to an open SSE client: closing the tab must not
+ * cancel the work the agent promised to finish.
+ */
+let chatRetryTimer = null;
+function armChatRetryDrain() {
+    if (chatRetryTimer)
+        return;
+    chatRetryTimer = setInterval(() => {
+        void chatRetryBroker.drain().catch(() => undefined);
+    }, CHAT_RETRY_DRAIN_INTERVAL_MS);
+    // Never hold the event loop open for a background convenience.
+    chatRetryTimer.unref?.();
+}
+function disarmChatRetryDrain() {
+    if (!chatRetryTimer)
+        return;
+    clearInterval(chatRetryTimer);
+    chatRetryTimer = null;
+}
+/** How often the dashboard checks for a retry whose wait is over (ms). */
+const CHAT_RETRY_DRAIN_INTERVAL_MS = 15_000;
 // ─── ZIP Generation (no external dependencies) ─────────────────────────────
 /** CRC-32 lookup table (precomputed). */
 const CRC_TABLE = new Uint32Array(256);
@@ -806,7 +853,9 @@ export function readTracesData() {
     const traces = [...file.traces]
         .sort((a, b) => b.startedAt - a.startedAt)
         .map((t) => {
-        const { steps, ...rest } = t;
+        // Session 3 — the full stable layer is detail-only (the index would grow
+        // by ~16KB per trace otherwise); it is shown in the replay view.
+        const { steps, systemPrompt: _systemPrompt, ...rest } = t;
         const stepCount = steps.length;
         const failedSteps = steps.filter((s) => !s.success).length;
         const totalTokens = steps.reduce((sum, s) => sum + s.inputTokens + s.outputTokens, 0);
@@ -958,12 +1007,81 @@ async function readModelsHealth() {
         checkVLLMProvider(),
         checkBedrockProvider(),
     ]);
-    const providers = results.filter(Boolean);
-    const totalModels = providers.reduce((sum, p) => sum + p.models.length, 0);
-    const available = providers.reduce((sum, p) => sum + p.models.filter((m) => m.status === 'available').length, 0);
-    const limited = providers.reduce((sum, p) => sum + p.models.filter((m) => m.status === 'limited').length, 0);
-    const unavailable = providers.reduce((sum, p) => sum + p.models.filter((m) => m.status === 'unavailable').length, 0);
-    return { providers, lastChecked: Date.now(), totalModels, available, limited, unavailable };
+    const probed = results.filter(Boolean);
+    const totalModels = probed.reduce((sum, p) => sum + p.models.length, 0);
+    const available = probed.reduce((sum, p) => sum + p.models.filter((m) => m.status === 'available').length, 0);
+    const limited = probed.reduce((sum, p) => sum + p.models.filter((m) => m.status === 'limited').length, 0);
+    const unavailable = probed.reduce((sum, p) => sum + p.models.filter((m) => m.status === 'unavailable').length, 0);
+    // ── Reconcile the LISTING with the ROUTER (Models-page audit) ─────────────
+    // Everything above answers "does the provider LIST this model?" — and for
+    // OpenRouter that is 400+ ids stamped with the PROVIDER's rate-limit status,
+    // so hundreds of models read "Available" that the router can never pick (the
+    // registry holds 443 OpenRouter entries with 0 verified). Observed live: the
+    // page said 509 available while the agent routed 87% of a day's calls to one
+    // model. Two truths about the same models, on the same screen, with nothing
+    // connecting them — which is what made the page untrustworthy.
+    //
+    // The fix is to ask the REGISTRY, through the SAME predicate routing uses
+    // (`isUsable`). Duplicating the staleness/park rules here would recreate the
+    // exact divergence this is closing.
+    const registry = getModelRegistry();
+    const now = Date.now();
+    let routable = 0;
+    let registryTotal = 0;
+    let registryVerified = 0;
+    for (const provider of registry.getTrackedProviders()) {
+        for (const entry of registry.getAllModelsForProvider(provider)) {
+            registryTotal += 1;
+            if (entry.status === 'verified')
+                registryVerified += 1;
+        }
+    }
+    const providers = probed.map((p) => {
+        let providerRoutable = 0;
+        const models = p.models.map((m) => {
+            const entry = registry.getEntry(p.provider, m.id);
+            const usable = entry ? registry.isUsable(p.provider, m.id, now) : false;
+            if (usable) {
+                providerRoutable += 1;
+                routable += 1;
+            }
+            return {
+                ...m,
+                /** The registry's own verdict for this exact pair. */
+                registryStatus: entry?.status,
+                /** True only when the ROUTER would use it right now. */
+                routable: usable,
+                // WHY each unusable verdict is unusable — the raw learned reason, so the
+                // cell can distinguish "this id does not exist on the endpoint" from
+                // "your key cannot use it". Without it the page could only say
+                // "unavailable", which is the ambiguity that made "requires purchase"
+                // tempting and wrong.
+                registryError: entry?.lastError,
+                /** A retired provider × model pair (the id is not served there). */
+                registryDead: entry?.deadPair === true,
+                // Cost/entitlement, LABELLED never filtered — a user who just bought
+                // credits must still see the models they paid for.
+                entitlement: classifyModelEntitlement(p.provider, m.id),
+                // Parked/quota state lived only in the registry payload, so the per-model
+                // health cells could never render it — the type and the chips existed but
+                // nothing populated them on this path.
+                parked: entry ? entry.quotaParkedUntil > now : false,
+                resetsInMs: entry?.resetsInMs ?? 0,
+            };
+        });
+        return { ...p, models, routable: providerRoutable };
+    });
+    return {
+        providers,
+        lastChecked: now,
+        totalModels,
+        available,
+        limited,
+        unavailable,
+        routable,
+        registryTotal,
+        registryVerified,
+    };
 }
 /** Check local Ollama provider — no rate limits to parse */
 async function checkLocalProvider() {
@@ -2269,6 +2387,21 @@ let whatsappPairing = new WhatsAppPairingManager();
 // P4 — chat sessions persist through ~/.nuvira/memory/chat-sessions.json so the
 // sidebar can resume any past conversation after a dashboard restart.
 let chatConsole = new ChatConsole({ persistPath: join(MEMORY_DIR, 'chat-sessions.json') });
+/**
+ * The dashboard's half of the deferred retry loop (one per process).
+ *
+ * Reads `chatConsole` lazily through its closures, so the test hook that swaps
+ * the console keeps working: a broker captured at module load would freeze the
+ * original console and retry into a session nobody is looking at.
+ */
+const chatRetryBroker = new ChatRetryBroker({
+    // `recordTurn: false` — the broker writes the retry outcome itself, once,
+    // with the header that explains an answer arriving unprompted.
+    answer: (sessionId, text) => chatConsole.answer(sessionId, text, { recordTurn: false }),
+    isBusy: (sessionId) => chatConsole.isBusy(sessionId),
+    notify: (event) => broadcastChatRetry(event),
+    persistTurn: (sessionId, user, assistant) => chatConsole.appendTurns(sessionId, user, assistant),
+});
 // Phase 4 (AGENTIC_CAPABILITY_ASSESSMENT Addendum v4) — the chat console's
 // loop turns record their engine + per-tool telemetry into the DAG store (a
 // local hook, never a cross-module import: chat-console must not know about
@@ -2334,6 +2467,14 @@ export function setChatConsoleForTest(console) {
     };
     chatConsole = console;
 }
+/**
+ * Test hook: run one deferred-retry drain immediately, instead of waiting for
+ * the 15s tick. Mirrors `setChatConsoleForTest` — the seam a real-HTTP test
+ * needs to observe a retry that is otherwise hours of wall-clock away.
+ */
+export async function runChatRetryDrainForTest() {
+    await chatRetryBroker.drain();
+}
 /**
  * The LIVE server handle, kept module-level so the Shutdown route can close
  * the listeners (primary + IPv6 twin) before exiting — a dashboard stopped
@@ -2358,6 +2499,9 @@ let dashboardShutdownAction = () => {
             }
             catch { /* best-effort */ }
         }
+        // The queue itself is PERSISTED and deliberately left alone: a restart
+        // resumes it (the normal case for a quota wait). Only the timer stops.
+        disarmChatRetryDrain();
         process.exit(0);
     }, 150);
 };
@@ -5218,11 +5362,30 @@ function handleRequest(req, res) {
             // stream closes): for an ABORTED fetch the response never completes, so
             // writableEnded stays false and the turn is cancelled; after a normal
             // completion writableEnded is true and this is a no-op.
+            // A pending retry offer is answered HERE, before any turn runs: the reply
+            // belongs to the offer, not to the composer's next request. A message that
+            // is neither yes nor no falls through to normal handling.
+            const retryReply = chatRetryBroker.consumeAnswer(sessionId, message);
+            if (retryReply) {
+                writeJson(res, 200, {
+                    ok: true,
+                    sessionId,
+                    content: retryReply,
+                    followups: [],
+                    provider: null,
+                    model: null,
+                    generationFailed: false,
+                });
+                return;
+            }
             const onResClose = () => {
                 if (!res.writableEnded)
                     chatConsole.abort(sessionId);
             };
             res.on('close', onResClose);
+            // Mark the failover log BEFORE the turn so a failure can report which
+            // models were actually tried (same contract as the gateway).
+            const attemptMark = markFailoverAttempts();
             const result = await chatConsole.answer(sessionId, message, { provider, model, projectContext, projectPath, attachments });
             // The response is written below — remove the disconnect listener so a
             // post-completion close can never touch the console again.
@@ -5234,14 +5397,39 @@ function handleRequest(req, res) {
                 });
                 return;
             }
+            // A failed generation gets the SAME treatment as the gateway: report what
+            // was tried, offer to keep checking, and queue the ask so the offer is
+            // enforced (see ChatRetryBroker). Without this the console was a dead end
+            // — the reader had to wait and re-send the message themselves.
+            let content = result.content ?? '';
+            // Did the offer we are about to send get a mechanism behind it? The client
+            // uses this to stop offering "↻ Retry" for a turn the server is already
+            // retrying on its own — a manual re-send on top of the queued one would
+            // run the same ask twice.
+            let retryQueued = false;
+            if (result.generationFailed === true) {
+                const report = chatRetryBroker.onFailure(sessionId, message, attemptMark);
+                if (report) {
+                    content = content.trim() ? `${content}\n\n${report}` : report;
+                    retryQueued = true;
+                    // The console stored whatever the ENGINE returned (often nothing), so
+                    // the report the reader is about to see would vanish on reload. Record
+                    // the composed bubble — the same contract as the gateway's `record()`.
+                    chatConsole.amendLastAssistantTurn(sessionId, content);
+                }
+                else if (!content.trim()) {
+                    content = "🤖 I couldn't get an answer from the model just now — please try again in a moment.";
+                }
+            }
             writeJson(res, 200, {
                 ok: true,
                 sessionId,
-                content: result.content ?? '',
+                content,
                 followups: result.followups ?? [],
                 provider: result.provider ?? null,
                 model: result.model ?? null,
                 generationFailed: result.generationFailed === true,
+                retryQueued,
             });
         })();
         return;
@@ -5540,6 +5728,9 @@ function loadApiKeysFromConfig() {
     }
 }
 export function createDashboardServer(opts) {
+    // Deferred retries for this dashboard's own chat sessions. Armed here (not
+    // per SSE client): a promised retry must run whether or not the tab is open.
+    armChatRetryDrain();
     // Bind values are resolved at CALL time: explicit override → env var →
     // import-time default. (PORT/HOST above are module constants, so the CLI's
     // `dashboard --port X` can NOT rely on setting BUFF_DASHBOARD_PORT after