@antseed/cli 0.1.196 → 0.1.198

package/dist/proxy/buyer-proxy.js

@@ -8,14 +8,15 @@ import { buildRoutingServices, findPinnedRoutingService } from './routing-servic
 import { readFileSync, watchFile, unwatchFile } from 'node:fs';
 import { readFile, writeFile, rename, mkdir, readdir, stat, unlink } from 'node:fs/promises';
 import { join } from 'node:path';
-import { ANTSEED_BUYER_FAULT_ERROR_CODE, ANTSEED_FAULT_ATTRIBUTION_HEADER, ANTSEED_ATTEST_PATH, adaptPeerFaultErrorResponse, computeTrustScore, decodeSweepRequest, faultAttributionOf, faultCodeOf, isModelRouteCoolingDown, isModelRouteEligible, modelRouteReputationScore, modelRouteTotalPrice, normalizedModelReputationScore, peerSupportsCooperativeClose, isRoutingSelection, DEFAULT_BUYER_IDENTITY, rankModelRoutes, SellerHealthBook, orderAdaptiveRoutes, adaptiveFirstTokenTimeoutMs, sanitizePeerDisplayName, } from '@antseed/node';
+import { ANTSEED_BUYER_FAULT_ERROR_CODE, ANTSEED_FAULT_ATTRIBUTION_HEADER, ANTSEED_ATTEST_PATH, adaptPeerFaultErrorResponse, computeTrustScore, decodeSweepRequest, faultAttributionOf, faultCodeOf, isModelRouteCoolingDown, isModelRouteEligible, modelRouteReputationScore, modelRouteTotalPrice, normalizedModelReputationScore, peerSupportsCooperativeClose, isRoutingSelection, DEFAULT_BUYER_IDENTITY, rankModelRoutes, SellerHealthBook, orderAdaptiveRoutes, adaptiveFirstTokenTimeoutMs, sanitizePeerDisplayName, VIDEO_JOB_RETIRED_HEADER, } from '@antseed/node';
+import { encodeServedSelection, ROUTE_SELECTION_HEADER } from './route-selection.js';
 import { canonicalModelKey } from '@antseed/node/model-identity';
 import { createStreamingAdapter, detectRequestServiceApiProtocol, extractRequestBodyFields, transformRequest, transformResponse, } from './service-api-adapter.js';
-import { DEBUG, log, extractRequestedService, summarizeRequestShape, summarizeErrorResponse, requestWantsStreaming, parsePeerPinnedService, rewritePeerPinnedServiceInBody, substituteRoutedModelAlias, overrideRoutedModelInBody, ROUTED_MODEL_ALIAS, SYSTEM_PROXY_SOURCE_HEADER, SYSTEM_ROUTED_MODEL_HEADER, BUYER_IDENTITY_HEADER, normalizePeerId, takeRoutingOptions, } from './request-utils.js';
+import { DEBUG, log, extractRequestedService, summarizeRequestShape, summarizeErrorResponse, requestWantsStreaming, parsePeerPinnedService, rewritePeerPinnedServiceInBody, substituteRoutedModelAlias, overrideRoutedModelInBody, ROUTED_MODEL_ALIAS, SYSTEM_PROXY_SOURCE_HEADER, SYSTEM_ROUTED_MODEL_HEADER, BUYER_IDENTITY_HEADER, MAX_VIDEO_COST_HEADER, normalizePeerId, takeRoutingOptions, } from './request-utils.js';
 import { BuyerIdentityLoader } from '../buyer-identities/loader.js';
 import { buildNetworkModels, effectiveModelReputationScore, parseModelTypeFilter, } from './network-models.js';
 import { findMissingRequiredParameters, findUnannouncedRequestParameters, findAdvertisedServiceOffer, findAdvertisedServiceProtocols, getExplicitProviderOverride, getExplicitPeerIdOverride, parseRequiredParametersHeader, REQUIRED_PARAMETERS_HEADER, resolvePeerRoutePlan, selectCandidatePeersForRouting, } from './routing.js';
-import { computeResponseTelemetry, attachAntseedTelemetryHeaders, attachStreamingAntseedHeaders, } from './telemetry.js';
+import { computeResponseTelemetry, attachAntseedTelemetryHeaders, ROUTER_PEER_ID_HEADER, ROUTER_RANK_HEADER, ROUTER_SERVICE_HEADER, withoutRouterHeaders, attachStreamingAntseedHeaders, } from './telemetry.js';
 import { DEFAULT_BUYER_PEER_REFRESH_INTERVAL_MS } from '../config/defaults.js';
 import { extractConversationIdentity, extractFirstUserSnippet, isCompletionRequestPath, isTitleGenerationRequest, parseRequestBodyObject, } from './conversation-identity.js';
 import { ConversationStore } from './conversation-store.js';
@@ -310,7 +311,8 @@ function dispatchFailure(statusCode, responseHeaders, body) {
 function writeDispatchFailure(res, result) {
     if (res.headersSent || res.writableEnded)
         return;
-    res.writeHead(result.statusCode, result.responseHeaders);
+    // A refusal can carry the seller's own headers: never let it name a router.
+    res.writeHead(result.statusCode, withoutRouterHeaders(result.responseHeaders));
     res.end(result.responseBody);
 }
 function isBuyerFault(result) {
@@ -434,6 +436,10 @@ export function sanitizePeerBuyerFaultMarker(response) {
         ? { ...response, body: Buffer.from(JSON.stringify(parsed)) }
         : response;
 }
+/** A non-negative integer amount of USDC base units, or null when absent or malformed. */
+function parseUsdcBaseUnits(value) {
+    return value && /^\d{1,15}$/.test(value) ? BigInt(value) : null;
+}
 function buyerFaultStatusCode(faultCode) {
     switch (faultCode) {
         case 'invalid-request': return 400;
@@ -810,6 +816,12 @@ export class BuyerProxy {
     _routingPreferences;
     _stateWriteChain = Promise.resolve();
     _resourceRoutes = new ResourceRoutes();
+    /** Retrieves of one video job run one at a time, so a job retired as failed cannot be delivered by a retrieve already under way. */
+    _videoJobQueues = new Map();
+    /** Jobs whose retirement is being saved: left out of the saved routes, still routable in memory until the save succeeds. */
+    /** The saved video routes existed but could not be read at start (see `start`). */
+    _resourceRoutesUnread = false;
+    _retiringVideoJobs = new Set();
     _cachedPeers = [];
     _cacheLastUpdatedAtMs = 0;
     _cacheMutationEpoch = 0;
@@ -1311,7 +1323,9 @@ export class BuyerProxy {
             now,
             affinityKey: input.affinityKey ?? null,
             ...this._adaptiveRouting,
-            ...(input.deterministic ? { spreadTolerance: 0, explorationRate: 0 } : {}),
+            // The preview keeps the real sharing pool (so it can show who shares the
+            // traffic) but always draws its first seller and never probes.
+            ...(input.deterministic ? { random: () => 0, explorationRate: 0 } : {}),
         });
         // Without routing preferences the buyer ranks by reputation, as before;
         // health only moves sellers that fail often behind the others.
@@ -1340,6 +1354,7 @@ export class BuyerProxy {
         // Conversation affinity is a soft preference: it only reorders sellers the
         // policy already allows, and yields to an explicit per-model route.
         const preferredPeerId = input.preferredPeerId;
+        let conversationFirst = false;
         if (preferredPeerId && !findModelRoute(effectivePolicy, requestedService)) {
             const preferredIndex = candidates.findIndex((candidate) => (candidate.peer.peerId.toLowerCase() === preferredPeerId
                 && !isCoolingDown(this._peerHealth.get(candidate.peer.peerId), now)
@@ -1349,7 +1364,17 @@ export class BuyerProxy {
                 if (preferred)
                     candidates.unshift(preferred);
             }
-        }
+            conversationFirst = preferredIndex >= 0;
+        }
+        // How the eligible sellers were put in order. Only the buyer's adaptive
+        // order, unsorted by the policy and with no route chain, spreads traffic.
+        const policySort = effectivePolicy?.sort && effectivePolicy.sort !== 'balanced' ? effectivePolicy.sort : null;
+        const selectionOrdering = policySort ?? (routingPreferences ? 'expected-cost' : 'trust');
+        const spreads = selectionOrdering === 'expected-cost' && !chainPosition?.size;
+        const pool = spreads ? adaptive.pool : [];
+        const poolWeight = (candidate) => 1 / Math.max(healthOf(candidate).score, 1e-9) ** 2;
+        const poolTotal = pool.reduce((sum, candidate) => sum + poolWeight(candidate), 0);
+        const selections = new Map();
         candidates.forEach((candidate, index) => {
             const verdict = verdicts.get(candidate.peerId);
             if (!verdict)
@@ -1364,8 +1389,8 @@ export class BuyerProxy {
                 verdict.reasons.push('cooling down (tried last)');
             else if (health.unhealthy)
                 verdict.reasons.push(`failing lately: ${health.stats?.lastFailureReason ?? 'failed requests'} (tried after healthy sellers)`);
-            if (adaptive.pool.length > 1 && adaptive.pool.includes(candidate))
-                verdict.reasons.push(`shares traffic with ${adaptive.pool.length - 1} other seller(s)`);
+            if (pool.length > 1 && pool.includes(candidate))
+                verdict.reasons.push(`shares traffic with ${pool.length - 1} other seller(s)`);
             if (modelRouteTotalPrice(candidate) === 0)
                 verdict.reasons.push('free');
             verdict.reasons.push(`success ~${Math.round(health.successProbability * 100)}%${health.stats ? '' : ' (no data yet)'}`);
@@ -1375,8 +1400,49 @@ export class BuyerProxy {
             const throughput = this._peerThroughput(candidate.peerId, candidate.serviceId);
             if (throughput !== null)
                 verdict.reasons.push(`~${Math.round(throughput)} tok/s`);
+            const healthKey = this._healthServiceKey(candidate.serviceId);
+            const stats = health.stats;
+            const inPool = pool.includes(candidate);
+            const maxConcurrency = typeof candidate.peer.maxConcurrency === 'number' && candidate.peer.maxConcurrency > 0 ? candidate.peer.maxConcurrency : null;
+            const selection = {
+                expectedCostUsd: health.score,
+                expectedPriceUsd: health.expectedPriceUsd,
+                expectedSeconds: health.expectedSeconds,
+                successRate: health.successProbability,
+                observations: stats ? stats.successes + stats.failures : 0,
+                record: !stats ? 'none' : this._sellerHealth.hasRecord(candidate.peerId, healthKey) ? 'model' : 'other-models',
+                firstTokenMs: latency,
+                tokensPerSecond: throughput,
+                lastSuccessAt: stats && stats.lastSuccessAt > 0 ? stats.lastSuccessAt : null,
+                lastFailureAt: stats && stats.lastFailureAt > 0 ? stats.lastFailureAt : null,
+                lastFailureReason: stats?.lastFailureReason ?? null,
+                failing: health.unhealthy,
+                coolingDown: isCooling(candidate),
+                stale: health.stale,
+                ordering: selectionOrdering,
+                sharesTraffic: inPool && pool.length > 1,
+                trafficShare: inPool && pool.length > 1 && poolTotal > 0 ? poolWeight(candidate) / poolTotal : null,
+                poolSize: pool.length,
+                channel: modelRouteTotalPrice(candidate) === 0 ? 'free' : openChannels ? (openChannels.has(candidate.peerId.toLowerCase()) ? 'open' : 'new') : null,
+                inFlight: this._inFlightBySeller.get(candidate.peerId.toLowerCase()) ?? 0,
+                maxConcurrency,
+                routeChainPosition: position !== undefined ? position + 1 : null,
+            };
+            verdict.selection = selection;
+            selections.set(candidate, selection);
         });
-        return { candidates, verdicts: [...verdicts.values()], effectivePolicy, explored: adaptive.explored };
+        // How the first seller came to be first, for the request's record.
+        const first = candidates[0];
+        let firstPick = 'order';
+        if (first) {
+            if (first === adaptive.explored)
+                firstPick = 'exploration';
+            else if (conversationFirst || (input.affinityKey && pool.length > 1 && pool.includes(first) && first === adaptive.ordered[0]))
+                firstPick = 'conversation';
+            else if (first === adaptive.ordered[0] && pool.includes(first))
+                firstPick = pool.length > 1 ? 'spread' : 'best';
+        }
+        return { candidates, verdicts: [...verdicts.values()], effectivePolicy, explored: adaptive.explored, selections, firstPick };
     }
     /**
      * Without routing preferences: best effective reputation first, and only
@@ -1434,6 +1500,7 @@ export class BuyerProxy {
             inputUsdPerMillion: verdict.offer.inputUsdPerMillion ?? null,
             outputUsdPerMillion: verdict.offer.outputUsdPerMillion ?? null,
             trustScore: normalizedModelReputationScore(verdict.peer),
+            selection: modelAllowed && verdict.eligible ? verdict.selection ?? null : null,
         }))
             .sort((a, b) => (a.rank ?? Number.POSITIVE_INFINITY) - (b.rank ?? Number.POSITIVE_INFINITY)
             || a.peerId.localeCompare(b.peerId));
@@ -1648,7 +1715,13 @@ export class BuyerProxy {
             const state = JSON.parse(await readFile(this._stateFile, 'utf8'));
             this._resourceRoutes.hydrate(state.resourceRoutes);
         }
-        catch { }
+        catch (error) {
+            // Routes that could not be read may come back on a later start: until then no job is said to be retired.
+            if (error.code !== 'ENOENT') {
+                this._resourceRoutesUnread = true;
+                console.warn(`[buyer-proxy] Could not read saved video routes: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
         // Adopt persisted session overrides (peer pin, default routed model) so
         // they survive daemon restart. A --peer CLI flag beats the persisted pin
         // at startup; runtime `connection set` writes still take over via the
@@ -1909,8 +1982,50 @@ export class BuyerProxy {
         }).catch(() => { });
         return this._stateWriteChain;
     }
+    async _serializeVideoJob(job, run) {
+        const previous = this._videoJobQueues.get(job) ?? Promise.resolve();
+        const current = previous.then(run, run);
+        const tail = current.catch(() => { });
+        this._videoJobQueues.set(job, tail);
+        try {
+            await current;
+        }
+        finally {
+            if (this._videoJobQueues.get(job) === tail)
+                this._videoJobQueues.delete(job);
+        }
+    }
+    /**
+     * Retires a video job that failed or that its seller no longer has: later
+     * retrieves get `video_route_not_found` and never reach the seller, so it can
+     * never be delivered or charged, and a gateway may release the price it held
+     * for it. False (route kept) when the retirement could not be saved, so a
+     * restart cannot bring the job back.
+     */
+    async _retireVideoJob(protocol, resourceId) {
+        if (!this._resourceRoutes.resolve(protocol, resourceId))
+            return true;
+        // The route stays usable until the retirement is saved: a retrieve arriving meanwhile waits its turn
+        // (retrieves of one job run one at a time) instead of seeing the job as gone before that is durable.
+        const key = `${protocol}\n${resourceId}`;
+        this._retiringVideoJobs.add(key);
+        try {
+            await this._persistResourceRoutes();
+            this._resourceRoutes.forget(protocol, resourceId);
+            return true;
+        }
+        catch (error) {
+            console.error('[proxy] Retired video route was not saved; the job is kept:', error);
+            return false;
+        }
+        finally {
+            this._retiringVideoJobs.delete(key);
+        }
+    }
     async _persistResourceRoutes() {
-        const write = this._stateWriteChain.then(() => mergeJsonStateFile(this._stateDir, this._stateFile, { resourceRoutes: this._resourceRoutes.snapshot() }));
+        const write = this._stateWriteChain.then(() => mergeJsonStateFile(this._stateDir, this._stateFile, {
+            resourceRoutes: this._resourceRoutes.snapshot().filter((route) => !this._retiringVideoJobs.has(`${route.protocol}\n${route.resourceId}`)),
+        }));
         this._stateWriteChain = write.catch(() => { });
         return write;
     }
@@ -3247,7 +3362,10 @@ export class BuyerProxy {
             // Re-classify with the body: some APIs (Venice) carry the job ID in it.
             const prepared = prepareVideoRequest(nativeVideoRoute(serializedReq) ?? nativeVideo, serializedReq.headers, this._resourceRoutes);
             if ('error' in prepared) {
-                res.writeHead(prepared.error.statusCode, { 'content-type': 'application/json' });
+                // No route for the job: this buyer can never deliver or charge it. Unless its saved routes could not be
+                // read at start, when the job may still be there on the next one.
+                const retired = prepared.error.body.error.code === 'video_route_not_found' && !this._resourceRoutesUnread ? { [VIDEO_JOB_RETIRED_HEADER]: '1' } : {};
+                res.writeHead(prepared.error.statusCode, { 'content-type': 'application/json', ...retired });
                 res.end(JSON.stringify(prepared.error.body));
                 return;
             }
@@ -3386,6 +3504,11 @@ export class BuyerProxy {
         const conversationId = this._trackRouterConversation(context);
         if (conversationId && selection.service)
             this._conversations.setLastRouter(conversationId, `${selection.service.peerId}@${selection.service.serviceId}`);
+        // Merged into whatever answer this request gets, so a gateway can tell routed requests (and failed rankings) apart.
+        if (selection.service) {
+            res.setHeader(ROUTER_PEER_ID_HEADER, selection.service.peerId.toLowerCase());
+            res.setHeader(ROUTER_SERVICE_HEADER, selection.service.serviceId);
+        }
         let recommendations;
         try {
             const client = this._modelRoutingClient;
@@ -3403,7 +3526,7 @@ export class BuyerProxy {
                 signal: AbortSignal.any([signal, AbortSignal.timeout(120_000)]),
                 onRoutingRequest: requestId => {
                     if (spendAttributionTag)
-                        this._spendAttribution.track(requestId, spendAttributionTag);
+                        this._spendAttribution.track(requestId, spendAttributionTag, 'routing');
                     if (conversationId)
                         this._trackRequestConversation(requestId, conversationId, undefined, 'routing');
                 },
@@ -3426,10 +3549,11 @@ export class BuyerProxy {
             return;
         }
         let lastFailure = null;
-        for (const recommendation of recommendations) {
+        for (const [index, recommendation] of recommendations.entries()) {
             // Never switch sellers once the user has disconnected or a response has started.
             if (signal.aborted || res.headersSent || res.writableEnded)
                 return;
+            res.setHeader(ROUTER_RANK_HEADER, String(index + 1));
             // Each attempt has its own billing ID, but all attempts count as one chat turn.
             const attempt = requestForRecommendation(request, recommendation, randomUUID());
             if (spendAttributionTag)
@@ -3703,7 +3827,7 @@ export class BuyerProxy {
                 ({ candidatePeers: modelPeers, routePlanByPeerId: modelPlans } = selectModelPeers(discoveredPeers));
             }
             const router = this._node.router;
-            const { candidates, effectivePolicy, explored } = this._planModelRoutes({
+            const { candidates, effectivePolicy, explored, selections, firstPick } = this._planModelRoutes({
                 modelPeers,
                 modelPlans,
                 request: serializedReq,
@@ -3798,6 +3922,18 @@ export class BuyerProxy {
                         continue;
                     }
                 }
+                // Why this seller serves the request, for the gateway's request log. A
+                // seller that fails is replaced by the next one's before any header goes out.
+                const selection = selections.get(selected);
+                if (selection && !res.headersSent) {
+                    res.setHeader(ROUTE_SELECTION_HEADER, encodeServedSelection({
+                        ...selection,
+                        rank: index + 1,
+                        of: candidates.length,
+                        attempt: index,
+                        pick: index === 0 ? firstPick : failedSellers.size > 0 || lastRetry !== null ? 'failover' : 'order',
+                    }));
+                }
                 for (let peerAttempt = 0; peerAttempt < MODEL_RATE_LIMIT_MAX_ATTEMPTS_PER_PEER; peerAttempt += 1) {
                     log(`Auto-selected peer ${selected.peer.peerId.slice(0, 12)}... for model="${requestedService}" `
                         + `service="${selected.serviceId}" reputation=${selected.reputation} `
@@ -4139,7 +4275,7 @@ export class BuyerProxy {
                     + `did not announce for this service: ${unannounced.join(', ')} — the upstream may ignore or reject them`);
             }
         }
-        const { 'x-antseed-pin-peer': _pinPeer, 'x-antseed-prefer-peer': _preferPeer, [BUYER_IDENTITY_HEADER]: buyerIdentity, [REQUIRED_PARAMETERS_HEADER]: _requiredParameters, 'x-vpr-session-id': _vprSession, 
+        const { 'x-antseed-pin-peer': _pinPeer, 'x-antseed-prefer-peer': _preferPeer, [BUYER_IDENTITY_HEADER]: buyerIdentity, [MAX_VIDEO_COST_HEADER]: maxVideoCostHeader, [REQUIRED_PARAMETERS_HEADER]: _requiredParameters, 'x-vpr-session-id': _vprSession, 
         // Legacy desktop builds (pre AntStation → VPR rename) still send this.
         'x-antstation-session-id': _antstationSession, ...headersForPeer } = serializedReq.headers;
         let requestForPeer = {
@@ -4213,9 +4349,18 @@ export class BuyerProxy {
         const wantsStreaming = clientWantsStreaming && !videoRoute;
         if (videoRoute?.action === 'retrieve') {
             // The seller hands a job only to the buyer that created it: retrieve with the same wallet.
-            await downloadVideo(requestForPeer, res, (request, callbacks, signal) => this._node.sendRequestStream(selectedPeer, request, callbacks, {
-                signal, pinned: true, ...(buyerIdentity ? { buyerIdentity } : {}),
-            }), requestSignal);
+            const { protocol, resourceId } = videoRoute;
+            await this._serializeVideoJob(`${protocol}\n${resourceId ?? ''}`, async () => {
+                // Checked again in turn: a retrieve queued behind one that retired the job must not reach the seller.
+                if (!resourceId || !this._resourceRoutes.resolve(protocol, resourceId)) {
+                    res.writeHead(404, { 'content-type': 'application/json', [VIDEO_JOB_RETIRED_HEADER]: '1' });
+                    res.end(JSON.stringify({ error: { code: 'video_route_not_found', message: 'Unknown video job' } }));
+                    return;
+                }
+                await downloadVideo(requestForPeer, res, (request, callbacks, signal) => this._node.sendRequestStream(selectedPeer, request, callbacks, {
+                    signal, pinned: true, ...(buyerIdentity ? { buyerIdentity } : {}),
+                }), requestSignal, () => this._retireVideoJob(protocol, resourceId));
+            });
             return { done: true };
         }
         const peerResponseProtocol = selectedRoutePlan.selection?.targetProtocol ?? requestProtocol;
@@ -4370,10 +4515,12 @@ export class BuyerProxy {
                 return { done: true, emptyAnswer: answeredEmpty };
             }
             else {
+                const maxVideoCostUsdc = videoRoute?.action === 'create' ? parseUsdcBaseUnits(maxVideoCostHeader) : null;
                 const upstreamResponse = await this._node.sendRequest(selectedPeer, requestForPeer, {
                     signal: attemptSignal,
                     pinned,
                     ...(buyerIdentity ? { buyerIdentity } : {}),
+                    ...(maxVideoCostUsdc !== null ? { maxVideoCostUsdc } : {}),
                 });
                 if (upstreamResponse.statusCode >= 400 && !adaptResponse) {
                     log(`Upstream raw error detail: ${summarizeErrorResponse(upstreamResponse)}`);