@antseed/node 0.2.156 → 0.2.158

package/dist/seller-request-handler.d.ts

@@ -14,12 +14,22 @@ import type { SerializedHttpRequest } from './types/http.js';
 import { VerificationMux } from './verification/verification-mux.js';
 import type { UnitBillingContext, UnitBillingModelV1 } from './types/billing.js';
 import type { ResourceOwnershipStore } from './resources/resource-ownership-store.js';
+import type { ServicePerformanceTracker } from './health/service-performance.js';
 type ProviderTokenPricing = import('./interfaces/seller-provider.js').ProviderTokenPricingUsdPerMillion;
 /**
  * Routing protocols whose requests do not name a service (`oneServicePerSeller`) let a seller
  * offer at most one service of that protocol; the request path then addresses it.
  */
 export declare function assertSingleRoutingService(providers: readonly Provider[]): void;
+/**
+ * Upstreams sharing a provider name must announce a service they both list
+ * with the same terms: price, API protocols, unit billing models and
+ * capabilities. Buyers see one entry per provider name and bill by it, while
+ * the seller serves through whichever of those upstreams is healthy. A native
+ * video service has one upstream per name: a job stays in the account that
+ * accepted it, so its creates and retrieves must reach the same upstream.
+ */
+export declare function assertSameNameUpstreamsAgree(providers: readonly Provider[]): void;
 export interface SellerRequestHandlerDeps {
     identity: Identity;
     providers: Provider[];
@@ -34,6 +44,8 @@ export interface SellerRequestHandlerDeps {
     reserveEstimateOverdraftUsdc?: bigint;
     /** Persistent buyer ownership and pending charges for stateful video jobs. Video follow-ups fail closed without it. */
     resourceOwnershipStore?: ResourceOwnershipStore | null;
+    /** Records how each service served buyers, for the seller to announce. */
+    servicePerformance?: ServicePerformanceTracker | null;
     emit: (event: string, ...args: unknown[]) => boolean;
 }
 /**
@@ -46,6 +58,8 @@ export interface SellerRequestHandlerDeps {
 export declare class SellerRequestHandler {
     private readonly _deps;
     private readonly _providerLoadCounts;
+    /** Per channel: the highest spend a catch-up NeedAuth was already sent for, so parked requests ask once. */
+    private readonly _catchUpPrompted;
     private readonly _attestRateWindows;
     private _metadataRefreshTimer;
     constructor(deps: SellerRequestHandlerDeps);

package/dist/seller-request-handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"seller-request-handler.d.ts","sourceRoot":"","sources":["../src/seller-request-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EACV,QAAQ,EAET,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAuB,KAAK,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AACjF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AACjF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACtF,OAAO,KAAK,EAAoB,qBAAqB,EAAE,MAAM,wCAAwC,CAAC;AAUtG,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,KAAK,EACV,qBAAqB,EAEtB,MAAM,iBAAiB,CAAC;AAgBzB,OAAO,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AAIrE,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAA8C,MAAM,oBAAoB,CAAC;AAG7H,OAAO,KAAK,EAAyB,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAU7G,KAAK,oBAAoB,GAAG,OAAO,iCAAiC,EAAE,iCAAiC,CAAC;AASxG;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,IAAI,CAQ/E;AA0CD,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,oBAAoB,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAClD,sBAAsB,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD,qBAAqB,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;IACrD,cAAc,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAC5C,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IACtC,SAAS,EAAE,aAAa,GAAG,IAAI,CAAC;IAChC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC,uHAAuH;IACvH,sBAAsB,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC;CACtD;AAgBD;;;;;;GAMG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAA6B;IACjE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAuD;IAC1F,OAAO,CAAC,qBAAqB,CAA8C;gBAE/D,IAAI,EAAE,wBAAwB;IAI1C,OAAO,CAAC,YAAY;IAoBpB;;;OAGG;IACH,gBAAgB,CACd,IAAI,EAAE,cAAc,EACpB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,eAAe,GAC/B;QAAE,GAAG,EAAE,QAAQ,CAAA;KAAE;IA0vBpB,oDAAoD;IACpD,OAAO,CAAC,oBAAoB;IA4B5B;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IA8C7B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAuC1B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAa5B,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,cAAc;IAiBtB;;;;;OAKG;YACW,qBAAqB;IAiBnC,OAAO,CAAC,oBAAoB;IA4C5B,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,GAAG,SAAS;IAuBnE,sBAAsB,CACpB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,qBAAqB,GAC7B,oBAAoB;IAWvB,+BAA+B,CAC7B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,kBAAkB,GAC1B,kBAAkB,GAAG,SAAS;IAMjC,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAY7D,yBAAyB,IAAI,IAAI;IAUjC,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,wBAAwB;IAehC,OAAO,CAAC,yBAAyB;IASjC,OAAO,CAAC,4BAA4B;IAYpC,OAAO,CAAC,+BAA+B;IAevC,OAAO,CAAC,4BAA4B;IAWpC;;;;;OAKG;YACW,gBAAgB;IAmB9B,OAAO,CAAC,wBAAwB;IAmBhC,OAAO,CAAC,gCAAgC;IA0BxC,OAAO,CAAC,uBAAuB;IAiB/B,OAAO,CAAC,qBAAqB;YAcf,eAAe;IAW7B,OAAO,CAAC,uBAAuB;IAe/B,OAAO,CAAC,8BAA8B;IAetC,OAAO,CAAC,wBAAwB;IAmBhC,OAAO,CAAC,2BAA2B;CA8BpC"}
+{"version":3,"file":"seller-request-handler.d.ts","sourceRoot":"","sources":["../src/seller-request-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EACV,QAAQ,EAET,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAuB,KAAK,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AACjF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AACjF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACtF,OAAO,KAAK,EAAoB,qBAAqB,EAAE,MAAM,wCAAwC,CAAC;AAUtG,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,KAAK,EACV,qBAAqB,EAEtB,MAAM,iBAAiB,CAAC;AAgBzB,OAAO,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AAKrE,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAA8C,MAAM,oBAAoB,CAAC;AAG7H,OAAO,KAAK,EAAyB,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAU7G,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAIjF,KAAK,oBAAoB,GAAG,OAAO,iCAAiC,EAAE,iCAAiC,CAAC;AASxG;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,IAAI,CAQ/E;AAED;;;;;;;GAOG;AACH,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,IAAI,CA4BjF;AAmDD,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,oBAAoB,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAClD,sBAAsB,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD,qBAAqB,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;IACrD,cAAc,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAC5C,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IACtC,SAAS,EAAE,aAAa,GAAG,IAAI,CAAC;IAChC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC,uHAAuH;IACvH,sBAAsB,CAAC,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD,0EAA0E;IAC1E,kBAAkB,CAAC,EAAE,yBAAyB,GAAG,IAAI,CAAC;IACtD,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC;CACtD;AAoBD;;;;;;GAMG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAA6B;IACjE,4GAA4G;IAC5G,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA6B;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAuD;IAC1F,OAAO,CAAC,qBAAqB,CAA8C;gBAE/D,IAAI,EAAE,wBAAwB;IAI1C,OAAO,CAAC,YAAY;IAoBpB;;;OAGG;IACH,gBAAgB,CACd,IAAI,EAAE,cAAc,EACpB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,eAAe,GAC/B;QAAE,GAAG,EAAE,QAAQ,CAAA;KAAE;IA02BpB,oDAAoD;IACpD,OAAO,CAAC,oBAAoB;IA4B5B;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IA8C7B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAuC1B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAa5B,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,cAAc;IAiBtB;;;;;OAKG;YACW,qBAAqB;IAiBnC,OAAO,CAAC,oBAAoB;IA4C5B,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,GAAG,SAAS;IA8BnE,sBAAsB,CACpB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,qBAAqB,GAC7B,oBAAoB;IAWvB,+BAA+B,CAC7B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,kBAAkB,GAC1B,kBAAkB,GAAG,SAAS;IAMjC,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAY7D,yBAAyB,IAAI,IAAI;IAUjC,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,wBAAwB;IAehC,OAAO,CAAC,yBAAyB;IASjC,OAAO,CAAC,4BAA4B;IAYpC,OAAO,CAAC,+BAA+B;IAevC,OAAO,CAAC,4BAA4B;IAWpC;;;;;OAKG;YACW,gBAAgB;IAmB9B,OAAO,CAAC,wBAAwB;IAmBhC,OAAO,CAAC,gCAAgC;IA0BxC,OAAO,CAAC,uBAAuB;IAiB/B,OAAO,CAAC,qBAAqB;YAcf,eAAe;IAW7B,OAAO,CAAC,uBAAuB;IAe/B,OAAO,CAAC,8BAA8B;IAetC,OAAO,CAAC,wBAAwB;IAmBhC,OAAO,CAAC,2BAA2B;CA8BpC"}

package/dist/seller-request-handler.js

@@ -13,11 +13,15 @@ import { debugLog, debugWarn } from './utils/debug.js';
 import { CONNECTION_CAPABILITY_RESPONSE_AUTH_V1, DEFAULT_FIRST_SIGN_CAP, DEFAULT_TOP_UP_SETTLED_THRESHOLD_BPS, computeOneOffChannelPlan, PAYMENT_CODE_CHANNEL_EXHAUSTED, PAYMENT_CODE_ONE_OFF_CHANNEL_REQUIRED, } from './types/protocol.js';
 import { createResponseAuthPayload, createStreamingResponseHash } from './verification/response-auth.js';
 import { VIDEO_DOWNLOAD_STREAM_HEADER, VIDEO_DOWNLOAD_STREAM_VERSION } from '@antseed/protocol/http';
+import { isServiceHealthFailing, preferHealthyUpstream, serviceHealthOf } from '@antseed/protocol/peer-metadata';
 import { hasJsonContentType, tryParseJsonObject } from './utils/json-codec.js';
 import { captureUnitBillingContext, computeFinalUnitBilling, estimateUnitRequestCost, isFreeUnitBillingModel } from './billing/unit.js';
 import { nativeVideoAcceptance, nativeVideoDelivered, nativeVideoRoute, requestService } from '@antseed/api-adapter';
+import { isNativeVideoProtocol } from '@antseed/protocol/service-api';
 import { detectRequestServiceApiProtocol, extractRequestBodyFields, selectTargetProtocolForRequest, } from '@antseed/api-adapter';
 import { parseResponseUsage } from './utils/response-usage.js';
+import { buildSelfHealthErrorResponse, isUpstreamFailureStatus } from './health/self-health-error.js';
+import { StreamOutcome } from './health/stream-outcome.js';
 import { routingProtocolForPath, routingProtocolOf, routingRequestOf, routingWireProtocol, ROUTING_PROTOCOLS } from '@antseed/protocol/routing';
 /** A seller's services of one routing protocol. */
 function routingServices(providers, protocol) {
@@ -39,6 +43,50 @@ export function assertSingleRoutingService(providers) {
         }
     }
 }
+/**
+ * Upstreams sharing a provider name must announce a service they both list
+ * with the same terms: price, API protocols, unit billing models and
+ * capabilities. Buyers see one entry per provider name and bill by it, while
+ * the seller serves through whichever of those upstreams is healthy. A native
+ * video service has one upstream per name: a job stays in the account that
+ * accepted it, so its creates and retrieves must reach the same upstream.
+ */
+export function assertSameNameUpstreamsAgree(providers) {
+    const terms = new Map();
+    for (const provider of providers) {
+        for (const service of provider.services) {
+            const video = provider.serviceApiProtocols?.[service]?.some(isNativeVideoProtocol) ?? false;
+            const announced = canonicalJson({
+                pricing: provider.pricing.services?.[service] ?? provider.pricing.defaults,
+                protocols: [...(provider.serviceApiProtocols?.[service] ?? [])].sort(),
+                unitBilling: provider.serviceUnitBillingModels?.[service] ?? null,
+                capabilities: provider.serviceCapabilities?.[service] ?? null,
+            });
+            const key = `${provider.name.toLowerCase()}\u0000${service}`;
+            const first = terms.get(key);
+            if (!first) {
+                terms.set(key, { provider, terms: announced });
+            }
+            else if (video) {
+                throw new Error(`Providers sharing the name "${provider.name}" both offer the video service "${service}". `
+                    + 'A video job stays with the upstream that accepted it, so offer it from one of them only.');
+            }
+            else if (first.terms !== announced) {
+                throw new Error(`Providers sharing the name "${provider.name}" offer "${service}" with different pricing, API protocols, billing or capabilities. `
+                    + 'Buyers see one entry per provider name, so offer it from one of them only, or on the same terms.');
+            }
+        }
+    }
+}
+/** JSON with object keys sorted, so equal values compare equal. */
+function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalJson).join(',')}]`;
+    if (value && typeof value === 'object') {
+        return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(',')}}`;
+    }
+    return JSON.stringify(value) ?? 'null';
+}
 const FREE_TOKEN_PRICING = { inputUsdPerMillion: 0, outputUsdPerMillion: 0 };
 function isZeroTokenPricing(pricing) {
     return pricing.inputUsdPerMillion === 0
@@ -80,6 +128,10 @@ function takeHeader(headers, name) {
 const METADATA_REFRESH_DEBOUNCE_MS = 200;
 /** Time to wait for a catch-up SpendingAuth before returning 402. */
 const DEFAULT_CATCH_UP_WAIT_MS = 5_000;
+/** How long a request waits for a closing channel's close() to land before it is answered. */
+const CLOSING_CHANNEL_WAIT_MS = 15_000;
+/** How long a request at the reserve ceiling waits for a running top-up before a retryable answer. */
+const TOP_UP_WAIT_MS = 15_000;
 /** Per-buyer rate limit for the free attestation route. */
 const ATTEST_RATE_WINDOW_MS = 60_000;
 const ATTEST_RATE_MAX_PER_WINDOW = 10;
@@ -94,6 +146,8 @@ const ATTEST_RATE_MAX_TRACKED_PEERS = 1024;
 export class SellerRequestHandler {
     _deps;
     _providerLoadCounts = new Map();
+    /** Per channel: the highest spend a catch-up NeedAuth was already sent for, so parked requests ask once. */
+    _catchUpPrompted = new Map();
     _attestRateWindows = new Map();
     _metadataRefreshTimer = null;
     constructor(deps) {
@@ -470,9 +524,8 @@ export class SellerRequestHandler {
                     await spm.waitForPendingAuths(buyerPeerId);
                     // Re-read after the await — the session may have been evicted (timeout
                     // checker, disconnect) while the on-chain top-up was confirming.
-                    const session = spm.getChannelByPeer(buyerPeerId);
-                    if (!session) {
-                        debugWarn(`[SellerHandler] Session evicted during waitForPendingAuths for ${buyerPeerId.slice(0, 12)}... — returning 402`);
+                    const sendSessionExpired = (during) => {
+                        debugWarn(`[SellerHandler] Session evicted during ${during} for ${buyerPeerId.slice(0, 12)}... — returning 402`);
                         mux.sendProxyResponse({
                             requestId: request.requestId,
                             statusCode: 402,
@@ -482,19 +535,20 @@ export class SellerRequestHandler {
                                 message: 'Session expired, please renegotiate',
                             })),
                         });
+                    };
+                    const session = spm.getChannelByPeer(buyerPeerId);
+                    if (!session) {
+                        sendSessionExpired('waitForPendingAuths');
                         return;
                     }
                     let accepted = spm.getAcceptedCumulative(session.sessionId);
-                    const spent = spm.getCumulativeSpend(session.sessionId);
+                    let spent = spm.getCumulativeSpend(session.sessionId);
                     // Serving headroom only includes funds locked on-chain. Pending
                     // top-ups are not counted until topUp() succeeds, otherwise a large
                     // request could push spend above the current reserve before the extra
                     // funds are actually locked.
-                    const reserveMax = spm.getEffectiveReserveMax(session.sessionId);
-                    const isBlocked = spm.isChannelBlocked(session.sessionId);
-                    // If spend has caught up and there is no headroom left in the reserve,
-                    // stop serving before accepting any additional request cost.
-                    const isAtExactSpendLimit = spent > 0n && spent === accepted && reserveMax > 0n && accepted >= reserveMax;
+                    let reserveMax = spm.getEffectiveReserveMax(session.sessionId);
+                    let isBlocked = spm.isChannelBlocked(session.sessionId);
                     if (spent > 0n && spent > accepted) {
                         // Race cover: the buyer's SpendingAuth for the *previous* response's
                         // NeedAuth may still be on the wire when this request arrives. The
@@ -505,21 +559,96 @@ export class SellerRequestHandler {
                         // operation this wait is a no-op; under pipelined requests (a new
                         // request dispatched before the prior NeedAuth → SpendingAuth has
                         // completed) it hides the round-trip latency from the buyer.
-                        const caughtUp = await spm.awaitAcceptedAtLeast(session.sessionId, spent, DEFAULT_CATCH_UP_WAIT_MS);
-                        accepted = spm.getAcceptedCumulative(session.sessionId);
-                        if (caughtUp && spent <= accepted) {
+                        // Ask for the catch-up now instead of only after the wait: after a
+                        // restart (or a lost NeedAuth) nothing else would prompt the buyer,
+                        // and every parked request would end in a 402. Once per spend level,
+                        // and only for spend the reserve can cover (beyond it the channel is closed below).
+                        const signable = reserveMax <= 0n || spent <= reserveMax;
+                        if (signable && spent > (this._catchUpPrompted.get(session.sessionId) ?? 0n)) {
+                            this._catchUpPrompted.delete(session.sessionId);
+                            this._catchUpPrompted.set(session.sessionId, spent);
+                            if (this._catchUpPrompted.size > 1_024)
+                                this._catchUpPrompted.delete(this._catchUpPrompted.keys().next().value);
+                            // A channel-level prompt: no requestId, since it is not about the
+                            // request that is waiting (its cost is not known yet).
+                            this._sendNeedAuthBestEffort(paymentMux, {
+                                channelId: session.sessionId,
+                                requiredCumulativeAmount: spent.toString(),
+                                currentAcceptedCumulative: accepted.toString(),
+                                deposit: session.authMax ?? '0',
+                            }, buyerPeerId, 'catch-up-before-wait');
+                        }
+                        // Other requests can finish while this one is parked: re-read the
+                        // spend after each catch-up and keep waiting, within one budget,
+                        // so an auth for the earlier spend never releases newer unpaid work.
+                        const deadline = Date.now() + DEFAULT_CATCH_UP_WAIT_MS;
+                        let caughtUp = true;
+                        while (caughtUp && spent > accepted) {
+                            caughtUp = await spm.awaitAcceptedAtLeast(session.sessionId, spent, Math.max(0, deadline - Date.now()));
+                            accepted = spm.getAcceptedCumulative(session.sessionId);
+                            spent = spm.getCumulativeSpend(session.sessionId);
+                        }
+                        // A channel closed or evicted while parked has its counters reset
+                        // to zero, which would read as caught up: renegotiate instead.
+                        if (spm.getChannelByPeer(buyerPeerId)?.sessionId !== session.sessionId) {
+                            sendSessionExpired('the catch-up wait');
+                            return;
+                        }
+                        // A top-up can land while parked: judge exhaustion on the current
+                        // ceiling, or a channel that just grew is closed as exhausted.
+                        reserveMax = spm.getEffectiveReserveMax(session.sessionId);
+                        isBlocked = spm.isChannelBlocked(session.sessionId);
+                        if (spent <= accepted) {
                             debugLog(`[SellerHandler] Caught up before 402 for ${buyerPeerId.slice(0, 12)}... (spent=${spent} accepted=${accepted})`);
                         }
                     }
-                    const remainingLockedReserve = reserveMax > spent ? reserveMax - spent : 0n;
+                    // A stored ceiling can lag the chain (a top-up that confirmed while
+                    // the seller was down): check it before treating the channel as full.
                     const reserveEstimateOverdraft = this._deps.reserveEstimateOverdraftUsdc;
+                    const estimateExceedsReserve = () => reserveEstimateOverdraft != null
+                        && reserveMax > 0n
+                        && estimatedRequestCost > 0n
+                        && estimatedRequestCost > (reserveMax > spent ? reserveMax - spent : 0n) + reserveEstimateOverdraft;
+                    const atCeiling = () => reserveMax > 0n && (accepted >= reserveMax || spent > reserveMax || estimateExceedsReserve());
+                    if (atCeiling()) {
+                        // A top-up mining in the background may lift the ceiling: wait for
+                        // it (bounded) rather than close a channel that is being extended.
+                        let topUpLanded = true;
+                        if (spm.isTopUpRunning(session.sessionId)) {
+                            topUpLanded = await spm.waitForTopUp(session.sessionId, TOP_UP_WAIT_MS);
+                        }
+                        reserveMax = await spm.refreshReserveMaxFromChain(session.sessionId);
+                        // The channel may have closed during the read: never admit work
+                        // on a channel that is gone, and judge the others on fresh counters.
+                        if (spm.getChannelByPeer(buyerPeerId)?.sessionId !== session.sessionId) {
+                            sendSessionExpired('the reserve ceiling check');
+                            return;
+                        }
+                        accepted = spm.getAcceptedCumulative(session.sessionId);
+                        spent = spm.getCumulativeSpend(session.sessionId);
+                        isBlocked = spm.isChannelBlocked(session.sessionId);
+                        if (!topUpLanded && atCeiling()) {
+                            // Still mining: a short retry, never an exhausted (closing) answer.
+                            mux.sendProxyResponse({
+                                requestId: request.requestId,
+                                statusCode: 503,
+                                headers: { 'content-type': 'application/json', 'retry-after': '2' },
+                                body: new TextEncoder().encode(JSON.stringify({
+                                    error: 'top_up_pending',
+                                    message: 'The payment channel is being extended on-chain; retry shortly',
+                                })),
+                            });
+                            return;
+                        }
+                    }
+                    // If spend has caught up and there is no headroom left in the reserve,
+                    // stop serving before accepting any additional request cost.
+                    const isAtExactSpendLimit = spent > 0n && spent === accepted && reserveMax > 0n && accepted >= reserveMax;
+                    const remainingLockedReserve = reserveMax > spent ? reserveMax - spent : 0n;
                     const effectiveEstimateLimit = reserveEstimateOverdraft != null
                         ? remainingLockedReserve + reserveEstimateOverdraft
                         : null;
-                    const estimatedCostExceedsLockedReserve = effectiveEstimateLimit != null
-                        && reserveMax > 0n
-                        && estimatedRequestCost > 0n
-                        && estimatedRequestCost > effectiveEstimateLimit;
+                    const estimatedCostExceedsLockedReserve = estimateExceedsReserve();
                     if (isBlocked || (spent > 0n && (spent > accepted || isAtExactSpendLimit)) || estimatedCostExceedsLockedReserve) {
                         const baseRequirements = spm.getPaymentRequirements(request.requestId, buyerPeerId, requestPricing);
                         // Tell the buyer exactly how much delivered spend remains unsigned.
@@ -625,6 +754,8 @@ export class SellerRequestHandler {
             let downloadHash;
             let heldDoneChunkData = null;
             let responseStartedAt = startTime;
+            // First generated token and in-stream failures of a streamed answer, for the performance this seller announces.
+            const streamOutcome = new StreamOutcome();
             let responseForAuth = null;
             let streamAuthStatusCode = 0;
             let streamAuthHeaders = null;
@@ -636,6 +767,21 @@ export class SellerRequestHandler {
             // between serving the request and claiming its cost.
             const isBillable = !isServedFree && !oneOffChannelId && (spm?.hasSession(buyerPeerId) ?? false);
             if (isBillable && spm.hasClosingChannel(buyerPeerId)) {
+                // A close takes seconds on-chain (a disconnect, an exhausted channel).
+                // Let it finish, then ask for a new channel: the buyer opens one and
+                // retries this request, instead of the client getting an error.
+                if (await spm.waitForClosingChannel(buyerPeerId, CLOSING_CHANNEL_WAIT_MS) && !spm.hasSession(buyerPeerId)) {
+                    mux.sendProxyResponse({
+                        requestId: request.requestId,
+                        statusCode: 402,
+                        headers: { 'content-type': 'application/json' },
+                        body: new TextEncoder().encode(JSON.stringify({
+                            error: 'payment_required',
+                            message: 'Session expired, please renegotiate',
+                        })),
+                    });
+                    return;
+                }
                 mux.sendProxyResponse({
                     requestId: request.requestId,
                     statusCode: 503,
@@ -668,6 +814,8 @@ export class SellerRequestHandler {
                         onResponseChunk: (chunk) => {
                             if (!streamedResponseStarted)
                                 return;
+                            if (!downloadHash)
+                                streamOutcome.observe(chunk.data);
                             if (downloadHash) {
                                 downloadHash.update(chunk.data);
                                 if (!chunk.done)
@@ -690,13 +838,19 @@ export class SellerRequestHandler {
                         response = this._acceptVideoCreate(videoRoute, request, response, buyerPeerId, requestedModel, requestBilling, unitBillingModel, oneOffChannelId);
                     }
                     statusCode = response.statusCode;
+                    // The upstream's answer for its account and model health (an out-of-credit error is seen here first).
+                    // Polling or downloading a video already paid for says nothing about the account's credit.
+                    const upstreamBody = response.body ?? new Uint8Array(0);
+                    this._deps.emit('provider:response', { provider, service: requestedModel, statusCode, body: statusCode >= 400 ? new TextDecoder().decode(upstreamBody.slice(0, 2_000)) : '', billable: videoRoute?.action !== 'retrieve' });
+                    const health = serviceHealthOf(provider.serviceHealth, requestedModel);
+                    if (!streamedResponseStarted && isServiceHealthFailing(health) && isUpstreamFailureStatus(statusCode)) {
+                        response = buildSelfHealthErrorResponse(request.requestId, requestedModel, health, response);
+                        statusCode = response.statusCode;
+                    }
                     responseBody = response.body ?? new Uint8Array(0);
                     responseForAuth = response;
                     if (downloadHash)
                         responseForAuth = { ...response, streamedBody: downloadHash.finish() };
-                    // The upstream's answer for its account health (an out-of-credit error is seen here first).
-                    // Polling or downloading a video already paid for says nothing about the account's credit.
-                    this._deps.emit('provider:response', { provider, statusCode, body: statusCode >= 400 ? new TextDecoder().decode(responseBody.slice(0, 2_000)) : '', billable: videoRoute?.action !== 'retrieve' });
                     if (statusCode >= 400) {
                         const errBody = new TextDecoder().decode(responseBody).slice(0, 200);
                         debugWarn(`[SellerHandler] Provider error response: status=${statusCode} provider="${provider.name}" model="${requestedModel}" buyer=${buyerPeerId.slice(0, 12)}... (${Date.now() - startTime}ms) body=${errBody}`);
@@ -735,6 +889,7 @@ export class SellerRequestHandler {
                 catch (err) {
                     const message = err instanceof Error ? err.message : "Internal error";
                     failureMessage = message;
+                    this._deps.emit('provider:failure', { provider, service: requestedModel });
                     debugWarn(`[SellerHandler] Provider exception: provider="${provider.name}" model="${requestedModel}" buyer=${buyerPeerId.slice(0, 12)}... (${Date.now() - startTime}ms) ${message}`);
                     if (videoRoute?.action === 'create')
                         this._closeOneOffBestEffort(oneOffChannelId, 'video create failed');
@@ -767,13 +922,17 @@ export class SellerRequestHandler {
                         }
                     }
                     else {
-                        statusCode = 500;
-                        responseForAuth = {
-                            requestId: request.requestId,
-                            statusCode: 500,
-                            headers: { "content-type": "text/plain" },
-                            body: responseBody,
-                        };
+                        const health = serviceHealthOf(provider.serviceHealth, requestedModel);
+                        responseForAuth = isServiceHealthFailing(health)
+                            ? buildSelfHealthErrorResponse(request.requestId, requestedModel, health, { statusCode: 500, headers: {}, body: responseBody })
+                            : {
+                                requestId: request.requestId,
+                                statusCode: 500,
+                                headers: { "content-type": "text/plain" },
+                                body: responseBody,
+                            };
+                        statusCode = responseForAuth.statusCode;
+                        responseBody = responseForAuth.body;
                         mux.sendProxyResponse(responseForAuth);
                     }
                 }
@@ -784,7 +943,17 @@ export class SellerRequestHandler {
                     unitCostUsdc = finalBilling.costUsdc;
                 }
                 // Record metering
-                const latencyMs = Date.now() - startTime;
+                const finishedAt = Date.now();
+                const latencyMs = finishedAt - startTime;
+                // Video jobs are polled and downloaded on their own schedule: their timing says nothing about the model.
+                if (!videoRoute && requestedModel !== 'unknown') {
+                    const { firstTokenAt } = streamOutcome;
+                    this._deps.servicePerformance?.record(provider.name, requestedModel, {
+                        ok: failureMessage === null && !streamOutcome.failed && !isUpstreamFailureStatus(statusCode),
+                        ...(firstTokenAt !== null ? { firstTokenMs: firstTokenAt - startTime, streamMs: finishedAt - firstTokenAt } : {}),
+                        outputTokens: responseUsage.outputTokens,
+                    }, finishedAt);
+                }
                 if (this._deps.sessionTracker) {
                     await this._deps.sessionTracker.recordMetering({
                         buyerPeerId,
@@ -1090,12 +1259,20 @@ export class SellerRequestHandler {
         const videoRoute = nativeVideoRoute(request);
         const providers = this._deps.providers;
         const matchesService = (provider) => provider.services.includes(requestedService) && (!videoRoute || Boolean(provider.serviceApiProtocols?.[requestedService]?.includes(videoRoute.protocol)));
+        // Several upstreams can share a provider name (with the same terms, see
+        // assertSameNameUpstreamsAgree); buyers merge their announcements with the
+        // same preference (mergeProviderAnnouncements). A native video job lives
+        // in the account of the upstream that accepted it, so video requests keep
+        // to the first matching upstream: a retrieve reaches the job's upstream
+        // even after it starts failing.
+        const pick = (candidates) => (videoRoute ? candidates[0] : preferHealthyUpstream(candidates, requestedService));
+        const serving = providers.filter(matchesService);
         let provider;
         if (requestedProvider) {
-            provider = providers.find((candidate) => candidate.name.toLowerCase() === requestedProvider && matchesService(candidate));
+            provider = pick(serving.filter((candidate) => candidate.name.toLowerCase() === requestedProvider));
         }
         if (!provider && !(videoRoute && requestedProvider)) {
-            provider = providers.find((candidate) => matchesService(candidate));
+            provider = pick(serving);
         }
         return provider;
     }