@antseed/cli 0.1.197 → 0.1.198

package/dist/proxy/buyer-proxy.js

@@ -9,6 +9,7 @@ import { readFileSync, watchFile, unwatchFile } from 'node:fs';
 import { readFile, writeFile, rename, mkdir, readdir, stat, unlink } from 'node:fs/promises';
 import { join } from 'node:path';
 import { ANTSEED_BUYER_FAULT_ERROR_CODE, ANTSEED_FAULT_ATTRIBUTION_HEADER, ANTSEED_ATTEST_PATH, adaptPeerFaultErrorResponse, computeTrustScore, decodeSweepRequest, faultAttributionOf, faultCodeOf, isModelRouteCoolingDown, isModelRouteEligible, modelRouteReputationScore, modelRouteTotalPrice, normalizedModelReputationScore, peerSupportsCooperativeClose, isRoutingSelection, DEFAULT_BUYER_IDENTITY, rankModelRoutes, SellerHealthBook, orderAdaptiveRoutes, adaptiveFirstTokenTimeoutMs, sanitizePeerDisplayName, VIDEO_JOB_RETIRED_HEADER, } from '@antseed/node';
+import { encodeServedSelection, ROUTE_SELECTION_HEADER } from './route-selection.js';
 import { canonicalModelKey } from '@antseed/node/model-identity';
 import { createStreamingAdapter, detectRequestServiceApiProtocol, extractRequestBodyFields, transformRequest, transformResponse, } from './service-api-adapter.js';
 import { DEBUG, log, extractRequestedService, summarizeRequestShape, summarizeErrorResponse, requestWantsStreaming, parsePeerPinnedService, rewritePeerPinnedServiceInBody, substituteRoutedModelAlias, overrideRoutedModelInBody, ROUTED_MODEL_ALIAS, SYSTEM_PROXY_SOURCE_HEADER, SYSTEM_ROUTED_MODEL_HEADER, BUYER_IDENTITY_HEADER, MAX_VIDEO_COST_HEADER, normalizePeerId, takeRoutingOptions, } from './request-utils.js';
@@ -1322,7 +1323,9 @@ export class BuyerProxy {
             now,
             affinityKey: input.affinityKey ?? null,
             ...this._adaptiveRouting,
-            ...(input.deterministic ? { spreadTolerance: 0, explorationRate: 0 } : {}),
+            // The preview keeps the real sharing pool (so it can show who shares the
+            // traffic) but always draws its first seller and never probes.
+            ...(input.deterministic ? { random: () => 0, explorationRate: 0 } : {}),
         });
         // Without routing preferences the buyer ranks by reputation, as before;
         // health only moves sellers that fail often behind the others.
@@ -1351,6 +1354,7 @@ export class BuyerProxy {
         // Conversation affinity is a soft preference: it only reorders sellers the
         // policy already allows, and yields to an explicit per-model route.
         const preferredPeerId = input.preferredPeerId;
+        let conversationFirst = false;
         if (preferredPeerId && !findModelRoute(effectivePolicy, requestedService)) {
             const preferredIndex = candidates.findIndex((candidate) => (candidate.peer.peerId.toLowerCase() === preferredPeerId
                 && !isCoolingDown(this._peerHealth.get(candidate.peer.peerId), now)
@@ -1360,7 +1364,17 @@ export class BuyerProxy {
                 if (preferred)
                     candidates.unshift(preferred);
             }
-        }
+            conversationFirst = preferredIndex >= 0;
+        }
+        // How the eligible sellers were put in order. Only the buyer's adaptive
+        // order, unsorted by the policy and with no route chain, spreads traffic.
+        const policySort = effectivePolicy?.sort && effectivePolicy.sort !== 'balanced' ? effectivePolicy.sort : null;
+        const selectionOrdering = policySort ?? (routingPreferences ? 'expected-cost' : 'trust');
+        const spreads = selectionOrdering === 'expected-cost' && !chainPosition?.size;
+        const pool = spreads ? adaptive.pool : [];
+        const poolWeight = (candidate) => 1 / Math.max(healthOf(candidate).score, 1e-9) ** 2;
+        const poolTotal = pool.reduce((sum, candidate) => sum + poolWeight(candidate), 0);
+        const selections = new Map();
         candidates.forEach((candidate, index) => {
             const verdict = verdicts.get(candidate.peerId);
             if (!verdict)
@@ -1375,8 +1389,8 @@ export class BuyerProxy {
                 verdict.reasons.push('cooling down (tried last)');
             else if (health.unhealthy)
                 verdict.reasons.push(`failing lately: ${health.stats?.lastFailureReason ?? 'failed requests'} (tried after healthy sellers)`);
-            if (adaptive.pool.length > 1 && adaptive.pool.includes(candidate))
-                verdict.reasons.push(`shares traffic with ${adaptive.pool.length - 1} other seller(s)`);
+            if (pool.length > 1 && pool.includes(candidate))
+                verdict.reasons.push(`shares traffic with ${pool.length - 1} other seller(s)`);
             if (modelRouteTotalPrice(candidate) === 0)
                 verdict.reasons.push('free');
             verdict.reasons.push(`success ~${Math.round(health.successProbability * 100)}%${health.stats ? '' : ' (no data yet)'}`);
@@ -1386,8 +1400,49 @@ export class BuyerProxy {
             const throughput = this._peerThroughput(candidate.peerId, candidate.serviceId);
             if (throughput !== null)
                 verdict.reasons.push(`~${Math.round(throughput)} tok/s`);
+            const healthKey = this._healthServiceKey(candidate.serviceId);
+            const stats = health.stats;
+            const inPool = pool.includes(candidate);
+            const maxConcurrency = typeof candidate.peer.maxConcurrency === 'number' && candidate.peer.maxConcurrency > 0 ? candidate.peer.maxConcurrency : null;
+            const selection = {
+                expectedCostUsd: health.score,
+                expectedPriceUsd: health.expectedPriceUsd,
+                expectedSeconds: health.expectedSeconds,
+                successRate: health.successProbability,
+                observations: stats ? stats.successes + stats.failures : 0,
+                record: !stats ? 'none' : this._sellerHealth.hasRecord(candidate.peerId, healthKey) ? 'model' : 'other-models',
+                firstTokenMs: latency,
+                tokensPerSecond: throughput,
+                lastSuccessAt: stats && stats.lastSuccessAt > 0 ? stats.lastSuccessAt : null,
+                lastFailureAt: stats && stats.lastFailureAt > 0 ? stats.lastFailureAt : null,
+                lastFailureReason: stats?.lastFailureReason ?? null,
+                failing: health.unhealthy,
+                coolingDown: isCooling(candidate),
+                stale: health.stale,
+                ordering: selectionOrdering,
+                sharesTraffic: inPool && pool.length > 1,
+                trafficShare: inPool && pool.length > 1 && poolTotal > 0 ? poolWeight(candidate) / poolTotal : null,
+                poolSize: pool.length,
+                channel: modelRouteTotalPrice(candidate) === 0 ? 'free' : openChannels ? (openChannels.has(candidate.peerId.toLowerCase()) ? 'open' : 'new') : null,
+                inFlight: this._inFlightBySeller.get(candidate.peerId.toLowerCase()) ?? 0,
+                maxConcurrency,
+                routeChainPosition: position !== undefined ? position + 1 : null,
+            };
+            verdict.selection = selection;
+            selections.set(candidate, selection);
         });
-        return { candidates, verdicts: [...verdicts.values()], effectivePolicy, explored: adaptive.explored };
+        // How the first seller came to be first, for the request's record.
+        const first = candidates[0];
+        let firstPick = 'order';
+        if (first) {
+            if (first === adaptive.explored)
+                firstPick = 'exploration';
+            else if (conversationFirst || (input.affinityKey && pool.length > 1 && pool.includes(first) && first === adaptive.ordered[0]))
+                firstPick = 'conversation';
+            else if (first === adaptive.ordered[0] && pool.includes(first))
+                firstPick = pool.length > 1 ? 'spread' : 'best';
+        }
+        return { candidates, verdicts: [...verdicts.values()], effectivePolicy, explored: adaptive.explored, selections, firstPick };
     }
     /**
      * Without routing preferences: best effective reputation first, and only
@@ -1445,6 +1500,7 @@ export class BuyerProxy {
             inputUsdPerMillion: verdict.offer.inputUsdPerMillion ?? null,
             outputUsdPerMillion: verdict.offer.outputUsdPerMillion ?? null,
             trustScore: normalizedModelReputationScore(verdict.peer),
+            selection: modelAllowed && verdict.eligible ? verdict.selection ?? null : null,
         }))
             .sort((a, b) => (a.rank ?? Number.POSITIVE_INFINITY) - (b.rank ?? Number.POSITIVE_INFINITY)
             || a.peerId.localeCompare(b.peerId));
@@ -3771,7 +3827,7 @@ export class BuyerProxy {
                 ({ candidatePeers: modelPeers, routePlanByPeerId: modelPlans } = selectModelPeers(discoveredPeers));
             }
             const router = this._node.router;
-            const { candidates, effectivePolicy, explored } = this._planModelRoutes({
+            const { candidates, effectivePolicy, explored, selections, firstPick } = this._planModelRoutes({
                 modelPeers,
                 modelPlans,
                 request: serializedReq,
@@ -3866,6 +3922,18 @@ export class BuyerProxy {
                         continue;
                     }
                 }
+                // Why this seller serves the request, for the gateway's request log. A
+                // seller that fails is replaced by the next one's before any header goes out.
+                const selection = selections.get(selected);
+                if (selection && !res.headersSent) {
+                    res.setHeader(ROUTE_SELECTION_HEADER, encodeServedSelection({
+                        ...selection,
+                        rank: index + 1,
+                        of: candidates.length,
+                        attempt: index,
+                        pick: index === 0 ? firstPick : failedSellers.size > 0 || lastRetry !== null ? 'failover' : 'order',
+                    }));
+                }
                 for (let peerAttempt = 0; peerAttempt < MODEL_RATE_LIMIT_MAX_ATTEMPTS_PER_PEER; peerAttempt += 1) {
                     log(`Auto-selected peer ${selected.peer.peerId.slice(0, 12)}... for model="${requestedService}" `
                         + `service="${selected.serviceId}" reputation=${selected.reputation} `