@antseed/cli 0.1.143 → 0.1.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/cli/commands/config/index.d.ts.map +1 -1
- package/dist/cli/commands/config/index.js +9 -0
- package/dist/cli/commands/config/index.js.map +1 -1
- package/dist/cli/commands/seller/capability-prompts.d.ts +17 -0
- package/dist/cli/commands/seller/capability-prompts.d.ts.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.js +123 -0
- package/dist/cli/commands/seller/capability-prompts.js.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.test.d.ts +2 -0
- package/dist/cli/commands/seller/capability-prompts.test.d.ts.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.test.js +99 -0
- package/dist/cli/commands/seller/capability-prompts.test.js.map +1 -0
- package/dist/cli/commands/seller/setup.d.ts.map +1 -1
- package/dist/cli/commands/seller/setup.js +12 -0
- package/dist/cli/commands/seller/setup.js.map +1 -1
- package/dist/cli/commands/seller/start.d.ts +3 -2
- package/dist/cli/commands/seller/start.d.ts.map +1 -1
- package/dist/cli/commands/seller/start.js +39 -3
- package/dist/cli/commands/seller/start.js.map +1 -1
- package/dist/cli/commands/seller/start.test.js +46 -1
- package/dist/cli/commands/seller/start.test.js.map +1 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +6 -0
- package/dist/config/loader.js.map +1 -1
- package/dist/config/loader.test.js +33 -0
- package/dist/config/loader.test.js.map +1 -1
- package/dist/config/service-metadata.d.ts +5 -0
- package/dist/config/service-metadata.d.ts.map +1 -0
- package/dist/config/service-metadata.js +47 -0
- package/dist/config/service-metadata.js.map +1 -0
- package/dist/config/service-metadata.test.d.ts +2 -0
- package/dist/config/service-metadata.test.d.ts.map +1 -0
- package/dist/config/service-metadata.test.js +18 -0
- package/dist/config/service-metadata.test.js.map +1 -0
- package/dist/config/types.d.ts +5 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js +2 -0
- package/dist/config/validation.js.map +1 -1
- package/dist/proxy/buyer-proxy.d.ts +76 -9
- package/dist/proxy/buyer-proxy.d.ts.map +1 -1
- package/dist/proxy/buyer-proxy.js +508 -68
- package/dist/proxy/buyer-proxy.js.map +1 -1
- package/dist/proxy/buyer-proxy.test.js +416 -13
- package/dist/proxy/buyer-proxy.test.js.map +1 -1
- package/dist/proxy/conversation-store.d.ts +6 -1
- package/dist/proxy/conversation-store.d.ts.map +1 -1
- package/dist/proxy/conversation-store.js +9 -2
- package/dist/proxy/conversation-store.js.map +1 -1
- package/dist/proxy/conversation-store.test.js +24 -0
- package/dist/proxy/conversation-store.test.js.map +1 -1
- package/dist/proxy/peer-attribution.d.ts +100 -0
- package/dist/proxy/peer-attribution.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.js +175 -0
- package/dist/proxy/peer-attribution.js.map +1 -0
- package/dist/proxy/peer-attribution.test.d.ts +2 -0
- package/dist/proxy/peer-attribution.test.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.test.js +140 -0
- package/dist/proxy/peer-attribution.test.js.map +1 -0
- package/dist/proxy/peer-health.d.ts +73 -0
- package/dist/proxy/peer-health.d.ts.map +1 -0
- package/dist/proxy/peer-health.js +182 -0
- package/dist/proxy/peer-health.js.map +1 -0
- package/dist/proxy/peer-health.test.d.ts +2 -0
- package/dist/proxy/peer-health.test.d.ts.map +1 -0
- package/dist/proxy/peer-health.test.js +197 -0
- package/dist/proxy/peer-health.test.js.map +1 -0
- package/dist/proxy/routing-parameters.test.d.ts +2 -0
- package/dist/proxy/routing-parameters.test.d.ts.map +1 -0
- package/dist/proxy/routing-parameters.test.js +97 -0
- package/dist/proxy/routing-parameters.test.js.map +1 -0
- package/dist/proxy/routing.d.ts +8 -0
- package/dist/proxy/routing.d.ts.map +1 -1
- package/dist/proxy/routing.js +36 -1
- package/dist/proxy/routing.js.map +1 -1
- package/dist/proxy/service-api-adapter.d.ts +1 -1
- package/dist/proxy/service-api-adapter.d.ts.map +1 -1
- package/dist/proxy/service-api-adapter.js +1 -1
- package/dist/proxy/service-api-adapter.js.map +1 -1
- package/package.json +6 -6
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
import { createServer } from 'node:http';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
3
|
import { watchFile, unwatchFile } from 'node:fs';
|
|
4
|
-
import { readFile, writeFile, rename, mkdir } from 'node:fs/promises';
|
|
4
|
+
import { readFile, writeFile, rename, mkdir, readdir, stat, unlink } from 'node:fs/promises';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
-
import { ANTSEED_ATTEST_PATH, computeOnChainReputationScore, decodeSweepRequest, } from '@antseed/node';
|
|
6
|
+
import { ANTSEED_BUYER_FAULT_ERROR_CODE, ANTSEED_FAULT_ATTRIBUTION_HEADER, ANTSEED_ATTEST_PATH, computeOnChainReputationScore, decodeSweepRequest, faultAttributionOf, faultCodeOf, peerSupportsCooperativeClose, } from '@antseed/node';
|
|
7
7
|
import { createStreamingAdapter, detectRequestServiceApiProtocol, transformRequest, transformResponse, } from './service-api-adapter.js';
|
|
8
|
-
import { DEBUG, log, extractRequestedService, summarizeRequestShape, summarizeErrorResponse, requestWantsStreaming, parsePeerPinnedService, rewritePeerPinnedServiceInBody, substituteRoutedModelAlias, overrideRoutedModelInBody, ROUTED_MODEL_ALIAS, SYSTEM_PROXY_SOURCE_HEADER, SYSTEM_ROUTED_MODEL_HEADER, } from './request-utils.js';
|
|
9
|
-
import { getExplicitProviderOverride, getExplicitPeerIdOverride, resolvePeerRoutePlan, selectCandidatePeersForRouting, } from './routing.js';
|
|
8
|
+
import { DEBUG, log, extractRequestedService, summarizeRequestShape, summarizeErrorResponse, requestWantsStreaming, parsePeerPinnedService, rewritePeerPinnedServiceInBody, substituteRoutedModelAlias, overrideRoutedModelInBody, ROUTED_MODEL_ALIAS, SYSTEM_PROXY_SOURCE_HEADER, SYSTEM_ROUTED_MODEL_HEADER, normalizePeerId, } from './request-utils.js';
|
|
9
|
+
import { findUnannouncedRequestParameters, getExplicitProviderOverride, getExplicitPeerIdOverride, resolvePeerRoutePlan, selectCandidatePeersForRouting, } from './routing.js';
|
|
10
10
|
import { computeResponseTelemetry, attachAntseedTelemetryHeaders, attachStreamingAntseedHeaders, } from './telemetry.js';
|
|
11
11
|
import { DEFAULT_BUYER_PEER_REFRESH_INTERVAL_MS } from '../config/defaults.js';
|
|
12
12
|
import { extractConversationIdentity, extractFirstUserSnippet, isCompletionRequestPath, isTitleGenerationRequest, parseRequestBodyObject, } from './conversation-identity.js';
|
|
13
13
|
import { ConversationStore } from './conversation-store.js';
|
|
14
|
+
import { recordPeerFailureEntry, clearPeerHealthEntry, isCoolingDown, parsePersistedPeerHealth, prunePeerHealth, serializePeerHealth, reasonEscalates, } from './peer-health.js';
|
|
15
|
+
import { PeerAttributionTracker, HEARTBEAT_MS } from './peer-attribution.js';
|
|
14
16
|
import { estimateAnthropicPromptTokens, isCountTokensPath } from './count-tokens.js';
|
|
15
17
|
import { getCachedVerdict, runVerifier, verifierSupportFingerprint } from '../plugins/verifier.js';
|
|
16
18
|
// Re-export for backward compatibility (used by tests and other consumers)
|
|
@@ -50,7 +52,6 @@ export function isModelNotFoundResponse(response) {
|
|
|
50
52
|
* liveness (`lastReachedAt`) even if the DHT record is older.
|
|
51
53
|
*/
|
|
52
54
|
const CARRY_FORWARD_TTL_MS = 2 * 60 * 60_000;
|
|
53
|
-
const PEER_FAILURE_WINDOW_MS = 5 * 60_000;
|
|
54
55
|
/**
|
|
55
56
|
* Requests kept in the spend-attribution map. Entries outlive their request on
|
|
56
57
|
* purpose (a seller-initiated auth can land after the response), so this is
|
|
@@ -61,11 +62,42 @@ const MAX_TRACKED_REQUEST_CONVERSATIONS = 512;
|
|
|
61
62
|
const MODEL_NOT_FOUND_REFRESH_THROTTLE_MS = 30_000;
|
|
62
63
|
/** Verification is expensive; bound how many verdicts we retain (TTL = peer-cache TTL). */
|
|
63
64
|
const VERIFY_CACHE_MAX_ENTRIES = 1024;
|
|
65
|
+
/**
|
|
66
|
+
* Statuses that prove the peer is alive and serving. Any response short of a
|
|
67
|
+
* server error counts: a peer that answers 400 or 404 is reachable, and
|
|
68
|
+
* treating only 2xx as proof would leave a stale cooldown on a healthy peer
|
|
69
|
+
* that happens to reject every request.
|
|
70
|
+
*/
|
|
71
|
+
function isProofOfLife(statusCode) {
|
|
72
|
+
return statusCode < 500 && statusCode !== 408;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Map a seller's response status onto a health reason, or null when the status
|
|
76
|
+
* says nothing about the peer's liveness.
|
|
77
|
+
*/
|
|
78
|
+
function failureReasonForStatus(statusCode) {
|
|
79
|
+
if (statusCode === 408)
|
|
80
|
+
return 'seller-timeout';
|
|
81
|
+
// Rate limiting is capacity pressure, not death — recorded, never escalated.
|
|
82
|
+
if (statusCode === 429)
|
|
83
|
+
return 'seller-busy';
|
|
84
|
+
if (statusCode >= 500 && statusCode <= 599)
|
|
85
|
+
return 'seller-5xx';
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
function responseFaultAttribution(response) {
|
|
89
|
+
const attribution = response.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER]?.toLowerCase();
|
|
90
|
+
return attribution === 'buyer' || attribution === 'peer' || attribution === 'unknown'
|
|
91
|
+
? attribution
|
|
92
|
+
: 'peer';
|
|
93
|
+
}
|
|
64
94
|
function adaptOpenAICompatibleErrorResponse(response, requestProtocol) {
|
|
65
95
|
if (response.statusCode !== 402) {
|
|
66
96
|
return response;
|
|
67
97
|
}
|
|
68
|
-
if (requestProtocol !== 'openai-responses'
|
|
98
|
+
if (requestProtocol !== 'openai-responses'
|
|
99
|
+
&& requestProtocol !== 'openai-chat-completions'
|
|
100
|
+
&& requestProtocol !== 'openai-images') {
|
|
69
101
|
return response;
|
|
70
102
|
}
|
|
71
103
|
let parsed = null;
|
|
@@ -97,6 +129,76 @@ function adaptOpenAICompatibleErrorResponse(response, requestProtocol) {
|
|
|
97
129
|
body: Buffer.from(JSON.stringify(wrappedError)),
|
|
98
130
|
};
|
|
99
131
|
}
|
|
132
|
+
function adaptBuyerFaultErrorResponse(response, requestProtocol) {
|
|
133
|
+
if (response.statusCode < 400
|
|
134
|
+
|| response.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER]?.toLowerCase() !== 'buyer') {
|
|
135
|
+
return sanitizePeerBuyerFaultMarker(response);
|
|
136
|
+
}
|
|
137
|
+
let parsed = {};
|
|
138
|
+
try {
|
|
139
|
+
parsed = JSON.parse(Buffer.from(response.body).toString('utf-8'));
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// Buyer-generated failures should be JSON, but keep a useful fallback if
|
|
143
|
+
// a future path emits plain text.
|
|
144
|
+
}
|
|
145
|
+
const nestedError = parsed.error && typeof parsed.error === 'object' && !Array.isArray(parsed.error)
|
|
146
|
+
? parsed.error
|
|
147
|
+
: null;
|
|
148
|
+
const reason = [nestedError?.code, parsed.code, parsed.reason, nestedError?.type, parsed.error]
|
|
149
|
+
.find((value) => typeof value === 'string' && value.trim().length > 0);
|
|
150
|
+
const message = [nestedError?.message, parsed.message, parsed.error]
|
|
151
|
+
.find((value) => typeof value === 'string' && value.trim().length > 0)
|
|
152
|
+
?? 'The request failed on the buyer.';
|
|
153
|
+
const body = requestProtocol === 'anthropic-messages'
|
|
154
|
+
? {
|
|
155
|
+
type: 'error',
|
|
156
|
+
error: {
|
|
157
|
+
type: ANTSEED_BUYER_FAULT_ERROR_CODE,
|
|
158
|
+
message: reason ? `${message} (${reason})` : message,
|
|
159
|
+
},
|
|
160
|
+
}
|
|
161
|
+
: {
|
|
162
|
+
error: {
|
|
163
|
+
type: 'api_error',
|
|
164
|
+
code: ANTSEED_BUYER_FAULT_ERROR_CODE,
|
|
165
|
+
message,
|
|
166
|
+
...(reason ? { param: reason } : {}),
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
return {
|
|
170
|
+
...response,
|
|
171
|
+
headers: { ...response.headers, 'content-type': 'application/json' },
|
|
172
|
+
body: Buffer.from(JSON.stringify(body)),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
function sanitizePeerBuyerFaultMarker(response) {
|
|
176
|
+
if (response.statusCode < 400)
|
|
177
|
+
return response;
|
|
178
|
+
let parsed;
|
|
179
|
+
try {
|
|
180
|
+
parsed = JSON.parse(Buffer.from(response.body).toString('utf-8'));
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
return response;
|
|
184
|
+
}
|
|
185
|
+
let changed = false;
|
|
186
|
+
const scrub = (record) => {
|
|
187
|
+
for (const key of ['code', 'type', 'errorCode']) {
|
|
188
|
+
if (record[key] === ANTSEED_BUYER_FAULT_ERROR_CODE) {
|
|
189
|
+
record[key] = 'upstream_error';
|
|
190
|
+
changed = true;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
scrub(parsed);
|
|
195
|
+
if (parsed.error && typeof parsed.error === 'object' && !Array.isArray(parsed.error)) {
|
|
196
|
+
scrub(parsed.error);
|
|
197
|
+
}
|
|
198
|
+
return changed
|
|
199
|
+
? { ...response, body: Buffer.from(JSON.stringify(parsed)) }
|
|
200
|
+
: response;
|
|
201
|
+
}
|
|
100
202
|
/**
|
|
101
203
|
* Inject the buyer-known peerId into a 402 payment_required JSON body.
|
|
102
204
|
* The seller doesn't include its own peerId (and shouldn't — self-reported
|
|
@@ -309,6 +411,82 @@ export function makeVerifierReach(node, peer, chosenId, signal) {
|
|
|
309
411
|
return { statusCode: resp.statusCode, headers: resp.headers, body: resp.body };
|
|
310
412
|
};
|
|
311
413
|
}
|
|
414
|
+
const STATE_TMP_PATTERN = /^\.buyer\.state\..+\.json\.tmp$/;
|
|
415
|
+
const STATE_TMP_SWEEP_MIN_AGE_MS = 60_000;
|
|
416
|
+
const STATE_RENAME_RETRY_DELAYS_MS = [25, 50, 100, 200];
|
|
417
|
+
/**
|
|
418
|
+
* Rename with a short bounded retry. On Windows, renaming over a file a
|
|
419
|
+
* reader briefly holds open fails with EPERM/EACCES/EBUSY; those clear within
|
|
420
|
+
* milliseconds, so retrying recovers the write instead of dropping it.
|
|
421
|
+
*/
|
|
422
|
+
export async function renameWithRetry(from, to) {
|
|
423
|
+
for (let attempt = 0;; attempt++) {
|
|
424
|
+
try {
|
|
425
|
+
await rename(from, to);
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
catch (err) {
|
|
429
|
+
const code = err.code;
|
|
430
|
+
const retryable = code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
|
|
431
|
+
if (!retryable || attempt >= STATE_RENAME_RETRY_DELAYS_MS.length)
|
|
432
|
+
throw err;
|
|
433
|
+
await new Promise((resolve) => setTimeout(resolve, STATE_RENAME_RETRY_DELAYS_MS[attempt]));
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Delete leftover `.buyer.state.<uuid>.json.tmp` files from state writes whose
|
|
439
|
+
* rename failed in an earlier run. The age floor protects a temp file another
|
|
440
|
+
* process (e.g. `antseed buyer connection set`) is writing right now.
|
|
441
|
+
*/
|
|
442
|
+
export async function sweepStaleStateTmpFiles(dir, minAgeMs = STATE_TMP_SWEEP_MIN_AGE_MS) {
|
|
443
|
+
let names;
|
|
444
|
+
try {
|
|
445
|
+
names = await readdir(dir);
|
|
446
|
+
}
|
|
447
|
+
catch {
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
const now = Date.now();
|
|
451
|
+
for (const name of names) {
|
|
452
|
+
if (!STATE_TMP_PATTERN.test(name))
|
|
453
|
+
continue;
|
|
454
|
+
const filePath = join(dir, name);
|
|
455
|
+
try {
|
|
456
|
+
const info = await stat(filePath);
|
|
457
|
+
if (now - info.mtimeMs >= minAgeMs)
|
|
458
|
+
await unlink(filePath);
|
|
459
|
+
}
|
|
460
|
+
catch {
|
|
461
|
+
// already gone or unreadable; nothing to clean
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Atomic read-merge-write of a JSON state file via a sibling temp file. The
|
|
467
|
+
* temp file never survives: a failed rename unlinks it before rethrowing.
|
|
468
|
+
*/
|
|
469
|
+
export async function mergeJsonStateFile(stateDir, stateFile, patch) {
|
|
470
|
+
await mkdir(stateDir, { recursive: true });
|
|
471
|
+
let existing = {};
|
|
472
|
+
try {
|
|
473
|
+
const raw = await readFile(stateFile, 'utf-8');
|
|
474
|
+
existing = JSON.parse(raw);
|
|
475
|
+
}
|
|
476
|
+
catch {
|
|
477
|
+
// file doesn't exist yet
|
|
478
|
+
}
|
|
479
|
+
const data = { ...existing, ...patch };
|
|
480
|
+
const tmp = join(stateDir, `.buyer.state.${randomUUID()}.json.tmp`);
|
|
481
|
+
await writeFile(tmp, JSON.stringify(data, null, 2));
|
|
482
|
+
try {
|
|
483
|
+
await renameWithRetry(tmp, stateFile);
|
|
484
|
+
}
|
|
485
|
+
catch (err) {
|
|
486
|
+
await unlink(tmp).catch(() => { });
|
|
487
|
+
throw err;
|
|
488
|
+
}
|
|
489
|
+
}
|
|
312
490
|
export class BuyerProxy {
|
|
313
491
|
_server;
|
|
314
492
|
_node;
|
|
@@ -348,7 +526,16 @@ export class BuyerProxy {
|
|
|
348
526
|
_consecutiveEmptyDiscoveries = 0;
|
|
349
527
|
_lastModelNotFoundRefreshAtMs = 0;
|
|
350
528
|
_bgRefreshHandle = null;
|
|
351
|
-
|
|
529
|
+
/**
|
|
530
|
+
* Per-peer failure streaks and cooldowns. Advisory only: a cooling-down peer
|
|
531
|
+
* is still dispatched to when a request names it, so routing can never
|
|
532
|
+
* deadlock and pinned conversations keep working.
|
|
533
|
+
*/
|
|
534
|
+
_peerHealth = new Map();
|
|
535
|
+
/** Decides whether a failure is the peer's fault at all. */
|
|
536
|
+
_attribution = new PeerAttributionTracker();
|
|
537
|
+
_heartbeatHandle = null;
|
|
538
|
+
_now;
|
|
352
539
|
/** Latest relayer receipt per sweep authNonce, for CLI progress polling. */
|
|
353
540
|
_sweepReceipts = new Map();
|
|
354
541
|
/**
|
|
@@ -370,6 +557,7 @@ export class BuyerProxy {
|
|
|
370
557
|
this._stateFile = join(config.dataDir, 'buyer.state.json');
|
|
371
558
|
this._conversations = new ConversationStore(config.dataDir);
|
|
372
559
|
this._pinnedPeer = config.pinnedPeerId?.toLowerCase() ?? null;
|
|
560
|
+
this._now = config.now ?? (() => Date.now());
|
|
373
561
|
this._server = createServer((req, res) => {
|
|
374
562
|
this._handleRequest(req, res).catch((err) => {
|
|
375
563
|
log('Unhandled error:', err);
|
|
@@ -437,6 +625,10 @@ export class BuyerProxy {
|
|
|
437
625
|
}
|
|
438
626
|
async start() {
|
|
439
627
|
this._startedAtMs = Date.now();
|
|
628
|
+
// Clean up temp files orphaned by state writes whose rename failed in a
|
|
629
|
+
// previous run — each carries a full discovered-peers snapshot, so left
|
|
630
|
+
// alone they accumulate into real disk usage.
|
|
631
|
+
await sweepStaleStateTmpFiles(this._stateDir);
|
|
440
632
|
// Hydrate the in-memory peer cache from the persisted state file BEFORE
|
|
441
633
|
// the server starts accepting requests. This lets the first request after
|
|
442
634
|
// startup route from the warm cache without blocking on DHT discovery.
|
|
@@ -455,6 +647,7 @@ export class BuyerProxy {
|
|
|
455
647
|
});
|
|
456
648
|
});
|
|
457
649
|
this._startBackgroundRefresh();
|
|
650
|
+
this._startSuspendHeartbeat();
|
|
458
651
|
// Trigger initial discovery immediately so the desktop can show services
|
|
459
652
|
// without waiting for the first request or 5-minute interval. The sweep
|
|
460
653
|
// emits each accepted metadata document as it arrives, so buyer.state.json
|
|
@@ -467,6 +660,11 @@ export class BuyerProxy {
|
|
|
467
660
|
try {
|
|
468
661
|
const raw = await readFile(this._stateFile, 'utf-8');
|
|
469
662
|
const parsed = JSON.parse(raw);
|
|
663
|
+
// Cooldowns survive a restart — a peer that died ten seconds before we
|
|
664
|
+
// exited is still dead — but the parser clamps anything expired or
|
|
665
|
+
// impossibly distant, so a restart can never extend one. Nothing new can
|
|
666
|
+
// escalate until a success re-establishes that the buyer is healthy.
|
|
667
|
+
this._peerHealth = parsePersistedPeerHealth(parsed, this._now());
|
|
470
668
|
const peers = parsePersistedPeers(parsed);
|
|
471
669
|
if (peers.length === 0) {
|
|
472
670
|
return;
|
|
@@ -499,6 +697,10 @@ export class BuyerProxy {
|
|
|
499
697
|
clearInterval(this._bgRefreshHandle);
|
|
500
698
|
this._bgRefreshHandle = null;
|
|
501
699
|
}
|
|
700
|
+
if (this._heartbeatHandle) {
|
|
701
|
+
clearInterval(this._heartbeatHandle);
|
|
702
|
+
this._heartbeatHandle = null;
|
|
703
|
+
}
|
|
502
704
|
await this._writeStateFile('stopped');
|
|
503
705
|
await this._conversations.flush();
|
|
504
706
|
return new Promise((resolve) => {
|
|
@@ -554,22 +756,13 @@ export class BuyerProxy {
|
|
|
554
756
|
_mergeStateFile(patch) {
|
|
555
757
|
this._stateWriteChain = this._stateWriteChain.then(async () => {
|
|
556
758
|
try {
|
|
557
|
-
await
|
|
558
|
-
let existing = {};
|
|
559
|
-
try {
|
|
560
|
-
const raw = await readFile(this._stateFile, 'utf-8');
|
|
561
|
-
existing = JSON.parse(raw);
|
|
562
|
-
}
|
|
563
|
-
catch {
|
|
564
|
-
// file doesn't exist yet
|
|
565
|
-
}
|
|
566
|
-
const data = { ...existing, ...patch };
|
|
567
|
-
const tmp = join(this._stateDir, `.buyer.state.${randomUUID()}.json.tmp`);
|
|
568
|
-
await writeFile(tmp, JSON.stringify(data, null, 2));
|
|
569
|
-
await rename(tmp, this._stateFile);
|
|
759
|
+
await mergeJsonStateFile(this._stateDir, this._stateFile, patch);
|
|
570
760
|
}
|
|
571
|
-
catch {
|
|
572
|
-
//
|
|
761
|
+
catch (err) {
|
|
762
|
+
// Non-fatal for the proxy, but the write itself is lost — session
|
|
763
|
+
// pin, default route, and peer-cache updates in this patch were not
|
|
764
|
+
// persisted. Always audible: state writes are minutes apart.
|
|
765
|
+
console.error('[proxy] buyer.state.json write failed:', err instanceof Error ? err.message : String(err));
|
|
573
766
|
}
|
|
574
767
|
}).catch(() => { });
|
|
575
768
|
return this._stateWriteChain;
|
|
@@ -707,36 +900,119 @@ export class BuyerProxy {
|
|
|
707
900
|
});
|
|
708
901
|
}
|
|
709
902
|
/**
|
|
710
|
-
*
|
|
711
|
-
*
|
|
712
|
-
*
|
|
903
|
+
* Record a failed request against a peer.
|
|
904
|
+
*
|
|
905
|
+
* Recording is unconditional — the streak and reason are useful diagnostics
|
|
906
|
+
* either way — but only failures the attribution gates accept as the peer's
|
|
907
|
+
* own move the cooldown. Discovery metadata is never evicted: a cooling-down
|
|
908
|
+
* peer stays routable, it just stops being *chosen*.
|
|
713
909
|
*/
|
|
714
|
-
_recordPeerFailure(peerId, reason) {
|
|
715
|
-
const now =
|
|
716
|
-
const
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
910
|
+
_recordPeerFailure(peerId, reason, fault = 'unknown') {
|
|
911
|
+
const now = this._now();
|
|
912
|
+
const { verdict, rollbackPeerIds } = this._attribution.classify({
|
|
913
|
+
peerId,
|
|
914
|
+
reasonEscalates: reasonEscalates(reason),
|
|
915
|
+
fault,
|
|
916
|
+
now,
|
|
917
|
+
});
|
|
918
|
+
const previous = this._peerHealth.get(peerId);
|
|
919
|
+
const entry = recordPeerFailureEntry(previous, reason, now, verdict.escalate);
|
|
920
|
+
this._peerHealth.set(peerId, entry);
|
|
921
|
+
if (rollbackPeerIds.length > 0) {
|
|
922
|
+
this._rollbackPeerHealth(rollbackPeerIds, 'buyer-side outage detected');
|
|
923
|
+
}
|
|
924
|
+
if (verdict.escalate && isCoolingDown(entry, now)) {
|
|
925
|
+
const seconds = Math.round((entry.cooldownUntil - now) / 1000);
|
|
926
|
+
log(`Peer ${peerId.slice(0, 12)}... cooling down for ${seconds}s after `
|
|
927
|
+
+ `${entry.failureStreak} failures (reason=${reason}).`);
|
|
928
|
+
}
|
|
929
|
+
else {
|
|
930
|
+
const why = verdict.escalate ? 'below cooldown threshold' : verdict.suppressedBy;
|
|
931
|
+
log(`Peer ${peerId.slice(0, 12)}... failure recorded (reason=${reason}); `
|
|
932
|
+
+ `not cooling down: ${why}.`);
|
|
933
|
+
}
|
|
934
|
+
void this._persistPeerHealthToState();
|
|
935
|
+
}
|
|
936
|
+
/**
|
|
937
|
+
* Fold a seller's HTTP response into that peer's health.
|
|
938
|
+
*
|
|
939
|
+
* Control-plane paths are exempt for the same reason `isRouterSuccess`
|
|
940
|
+
* exempts them: a failing `/v1/models` says nothing about the peer's ability
|
|
941
|
+
* to serve inference.
|
|
942
|
+
*/
|
|
943
|
+
_recordPeerResponseHealth(peerId, statusCode, path) {
|
|
944
|
+
if (isControlPlaneServicesPath(path)) {
|
|
945
|
+
if (isProofOfLife(statusCode))
|
|
946
|
+
this._rememberSuccessfulPeer(peerId);
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
949
|
+
const reason = failureReasonForStatus(statusCode);
|
|
950
|
+
if (reason && statusCode >= 500) {
|
|
951
|
+
const now = this._now();
|
|
952
|
+
if (isCoolingDown(this._peerHealth.get(peerId), now)) {
|
|
953
|
+
this._rememberSuccessfulPeer(peerId);
|
|
954
|
+
}
|
|
955
|
+
this._recordPeerFailure(peerId, reason, 'peer');
|
|
956
|
+
return;
|
|
957
|
+
}
|
|
958
|
+
if (isProofOfLife(statusCode)) {
|
|
959
|
+
// A 402, a 400, even a 429 — the peer answered, so it is alive and any
|
|
960
|
+
// cooldown is stale. Throttling still gets stamped as the last reason so
|
|
961
|
+
// "alive but refusing work" stays visible in diagnostics.
|
|
962
|
+
this._rememberSuccessfulPeer(peerId);
|
|
963
|
+
if (reason) {
|
|
964
|
+
const entry = this._peerHealth.get(peerId);
|
|
965
|
+
if (entry) {
|
|
966
|
+
this._peerHealth.set(peerId, { ...entry, lastReason: reason, lastFailureAt: this._now() });
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
if (reason)
|
|
972
|
+
this._recordPeerFailure(peerId, reason, 'peer');
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* Undo cooldowns that turned out to be our fault.
|
|
976
|
+
*
|
|
977
|
+
* When the attribution gates conclude the buyer itself was down — a suspend,
|
|
978
|
+
* a dropped network — the failures recorded during that window blamed the
|
|
979
|
+
* wrong party, so the streaks they created are wound back to zero.
|
|
980
|
+
*/
|
|
981
|
+
_rollbackPeerHealth(peerIds, why) {
|
|
982
|
+
let changed = false;
|
|
983
|
+
for (const peerId of peerIds) {
|
|
984
|
+
const entry = this._peerHealth.get(peerId);
|
|
985
|
+
if (!entry || (entry.failureStreak === 0 && entry.cooldownUntil === 0))
|
|
986
|
+
continue;
|
|
987
|
+
this._peerHealth.set(peerId, {
|
|
988
|
+
...entry,
|
|
989
|
+
failureStreak: 0,
|
|
990
|
+
windowStartedAt: 0,
|
|
991
|
+
episodeStartedAt: 0,
|
|
992
|
+
cooldownUntil: 0,
|
|
993
|
+
});
|
|
994
|
+
changed = true;
|
|
995
|
+
}
|
|
996
|
+
if (changed) {
|
|
997
|
+
log(`Cleared peer cooldowns for ${peerIds.length} peer(s): ${why}.`);
|
|
998
|
+
void this._persistPeerHealthToState();
|
|
999
|
+
}
|
|
729
1000
|
}
|
|
730
1001
|
/**
|
|
731
1002
|
* A peer told us it does not serve the requested model. Our cached
|
|
732
1003
|
* metadata for it is stale (the seller may have just unadvertised the
|
|
733
|
-
* model after failing its own health checks), so
|
|
734
|
-
*
|
|
735
|
-
*
|
|
1004
|
+
* model after failing its own health checks), so refresh discovery
|
|
1005
|
+
* metadata in the background — throttled, since one broken model can
|
|
1006
|
+
* produce a burst of these.
|
|
1007
|
+
*
|
|
1008
|
+
* Deliberately does NOT touch peer health: the response itself is proof of
|
|
1009
|
+
* life (`_recordPeerResponseHealth` treats any sub-500 answer as such), and
|
|
1010
|
+
* a peer that is healthy for its other models must not cool down over one
|
|
1011
|
+
* stale catalog entry. The router still learns via `onResult(success:false)`
|
|
1012
|
+
* so scoring reflects the miss.
|
|
736
1013
|
*/
|
|
737
1014
|
_onModelNotFound(peerId, requestedService) {
|
|
738
|
-
this.
|
|
739
|
-
const now = Date.now();
|
|
1015
|
+
const now = this._now();
|
|
740
1016
|
if (now - this._lastModelNotFoundRefreshAtMs < MODEL_NOT_FOUND_REFRESH_THROTTLE_MS) {
|
|
741
1017
|
return;
|
|
742
1018
|
}
|
|
@@ -748,17 +1024,56 @@ export class BuyerProxy {
|
|
|
748
1024
|
/**
|
|
749
1025
|
* Stamp `lastReachedAt` on a peer after a successful request so the
|
|
750
1026
|
* carry-forward heuristic can trust local transport liveness even when the
|
|
751
|
-
* DHT record grows stale. Persisted so the signal survives restarts.
|
|
752
|
-
*
|
|
1027
|
+
* DHT record grows stale. Persisted so the signal survives restarts.
|
|
1028
|
+
*
|
|
1029
|
+
* A response is also proof that the buyer's own network, DHT, chain RPC and
|
|
1030
|
+
* wallet are working, which is what lets other peers' failures be attributed
|
|
1031
|
+
* to them rather than to us.
|
|
753
1032
|
*/
|
|
754
1033
|
_rememberSuccessfulPeer(peerId) {
|
|
755
|
-
this.
|
|
1034
|
+
const now = this._now();
|
|
1035
|
+
this._attribution.recordSuccess(peerId, now);
|
|
1036
|
+
const previous = this._peerHealth.get(peerId);
|
|
1037
|
+
if (previous && (previous.failureStreak > 0 || previous.cooldownUntil > 0)) {
|
|
1038
|
+
log(`Peer ${peerId.slice(0, 12)}... recovered; cooldown cleared.`);
|
|
1039
|
+
}
|
|
1040
|
+
this._peerHealth.set(peerId, clearPeerHealthEntry(previous, now));
|
|
1041
|
+
void this._persistPeerHealthToState();
|
|
756
1042
|
const cached = this._cachedPeers.find((p) => p.peerId === peerId);
|
|
757
1043
|
if (cached) {
|
|
758
|
-
cached.lastReachedAt =
|
|
1044
|
+
cached.lastReachedAt = now;
|
|
759
1045
|
this._persistPeersToState();
|
|
760
1046
|
}
|
|
761
1047
|
}
|
|
1048
|
+
/**
|
|
1049
|
+
* Watch for the wall clock jumping forward, which means the machine slept.
|
|
1050
|
+
* On wake every pending timeout and keepalive fires at once, so without this
|
|
1051
|
+
* a single closed lid would cool down every peer the buyer knows.
|
|
1052
|
+
*/
|
|
1053
|
+
_startSuspendHeartbeat() {
|
|
1054
|
+
if (this._heartbeatHandle)
|
|
1055
|
+
return;
|
|
1056
|
+
this._attribution.onHeartbeat(this._now());
|
|
1057
|
+
this._heartbeatHandle = setInterval(() => {
|
|
1058
|
+
const result = this._attribution.onHeartbeat(this._now());
|
|
1059
|
+
if (result && result.rollbackPeerIds.length > 0) {
|
|
1060
|
+
this._rollbackPeerHealth(result.rollbackPeerIds, 'machine resumed from sleep');
|
|
1061
|
+
}
|
|
1062
|
+
else if (result) {
|
|
1063
|
+
log('Detected a wall-clock jump; suspending peer cooldowns briefly.');
|
|
1064
|
+
}
|
|
1065
|
+
}, HEARTBEAT_MS);
|
|
1066
|
+
this._heartbeatHandle.unref?.();
|
|
1067
|
+
}
|
|
1068
|
+
/** Persist health separately from `discoveredPeers`, which is rebuilt wholesale. */
|
|
1069
|
+
async _persistPeerHealthToState() {
|
|
1070
|
+
const now = this._now();
|
|
1071
|
+
this._peerHealth = prunePeerHealth(this._peerHealth, now);
|
|
1072
|
+
await this._mergeStateFile({
|
|
1073
|
+
peerHealth: serializePeerHealth(this._peerHealth),
|
|
1074
|
+
peerHealthUpdatedAt: now,
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
762
1077
|
async _discoverPeersFromNetwork() {
|
|
763
1078
|
log('Discovering peers via DHT...');
|
|
764
1079
|
const peers = await this._node.discoverPeers();
|
|
@@ -894,6 +1209,69 @@ export class BuyerProxy {
|
|
|
894
1209
|
res.end(JSON.stringify({ ok: true, peers: payload }));
|
|
895
1210
|
return;
|
|
896
1211
|
}
|
|
1212
|
+
if (path === '/_antseed/peer-health' && method === 'GET') {
|
|
1213
|
+
const now = this._now();
|
|
1214
|
+
const attribution = this._attribution.snapshot(now);
|
|
1215
|
+
// `buyerHealthy` and `suppressedUntil` are what make "why is this peer
|
|
1216
|
+
// (not) cooling down" answerable from outside the process.
|
|
1217
|
+
const peers = [...this._peerHealth.entries()].map(([peerId, entry]) => ({
|
|
1218
|
+
peerId,
|
|
1219
|
+
failureStreak: entry.failureStreak,
|
|
1220
|
+
lastFailureAt: entry.lastFailureAt,
|
|
1221
|
+
lastReason: entry.lastReason,
|
|
1222
|
+
cooldownUntil: entry.cooldownUntil,
|
|
1223
|
+
coolingDown: isCoolingDown(entry, now),
|
|
1224
|
+
cooldownMsRemaining: isCoolingDown(entry, now) ? entry.cooldownUntil - now : 0,
|
|
1225
|
+
lastSuccessAt: entry.lastSuccessAt,
|
|
1226
|
+
}));
|
|
1227
|
+
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1228
|
+
res.end(JSON.stringify({
|
|
1229
|
+
ok: true,
|
|
1230
|
+
now,
|
|
1231
|
+
buyerHealthy: this._attribution.isBuyerHealthy(now),
|
|
1232
|
+
lastAnySuccessAt: attribution.lastAnySuccessAt,
|
|
1233
|
+
suppressionActive: attribution.suppressedUntil > 0,
|
|
1234
|
+
suppressedUntil: attribution.suppressedUntil,
|
|
1235
|
+
lastSuppressedBy: attribution.lastSuppressedBy,
|
|
1236
|
+
peers,
|
|
1237
|
+
}));
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1240
|
+
if (path === '/_antseed/peer-health/clear' && method === 'POST') {
|
|
1241
|
+
const chunks = [];
|
|
1242
|
+
let totalSize = 0;
|
|
1243
|
+
for await (const chunk of req) {
|
|
1244
|
+
totalSize += chunk.length;
|
|
1245
|
+
if (totalSize > 8192) {
|
|
1246
|
+
res.writeHead(413, { 'content-type': 'application/json' });
|
|
1247
|
+
res.end(JSON.stringify({ ok: false, error: 'Request body too large' }));
|
|
1248
|
+
return;
|
|
1249
|
+
}
|
|
1250
|
+
chunks.push(chunk);
|
|
1251
|
+
}
|
|
1252
|
+
let peerId;
|
|
1253
|
+
try {
|
|
1254
|
+
const body = JSON.parse(Buffer.concat(chunks).toString());
|
|
1255
|
+
peerId = typeof body.peerId === 'string' ? body.peerId.trim().toLowerCase() : '';
|
|
1256
|
+
}
|
|
1257
|
+
catch {
|
|
1258
|
+
res.writeHead(400, { 'content-type': 'application/json' });
|
|
1259
|
+
res.end(JSON.stringify({ ok: false, error: 'Invalid JSON body' }));
|
|
1260
|
+
return;
|
|
1261
|
+
}
|
|
1262
|
+
const normalized = normalizePeerId(peerId) ?? peerId;
|
|
1263
|
+
if (!/^[0-9a-f]{40}$/.test(normalized)) {
|
|
1264
|
+
res.writeHead(400, { 'content-type': 'application/json' });
|
|
1265
|
+
res.end(JSON.stringify({ ok: false, error: 'peerId must be a 40-character hex peer id' }));
|
|
1266
|
+
return;
|
|
1267
|
+
}
|
|
1268
|
+
// Deliberately does not stamp a success: the user is asking us to give
|
|
1269
|
+
// the peer another chance, not asserting that it answered.
|
|
1270
|
+
this._rollbackPeerHealth([normalized], 'cleared by request');
|
|
1271
|
+
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1272
|
+
res.end(JSON.stringify({ ok: true, peerId: normalized }));
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
897
1275
|
if (path === '/_antseed/route' && method === 'GET') {
|
|
898
1276
|
res.writeHead(200, { 'content-type': 'application/json' });
|
|
899
1277
|
res.end(JSON.stringify({ ok: true, model: this._defaultRoutedModel }));
|
|
@@ -984,8 +1362,11 @@ export class BuyerProxy {
|
|
|
984
1362
|
res.end(JSON.stringify({ ok: false, error: 'pinnedModel must be "<peerId>@<service>" (or empty to clear)' }));
|
|
985
1363
|
return;
|
|
986
1364
|
}
|
|
987
|
-
|
|
988
|
-
|
|
1365
|
+
// 'user' marks a seller the user chose for this specific chat — the
|
|
1366
|
+
// desktop's re-point sweep skips those; everything else stays 'auto'.
|
|
1367
|
+
const peerSource = parsed.peerSource === 'user' ? 'user' : 'auto';
|
|
1368
|
+
conversation = this._conversations.setPinnedModel(id, pin.length > 0 ? pin : null, peerSource);
|
|
1369
|
+
log(`Conversation ${id.slice(0, 40)} pin: ${pin || 'cleared'}${pin ? ` (${peerSource})` : ''}`);
|
|
989
1370
|
}
|
|
990
1371
|
if ('label' in parsed) {
|
|
991
1372
|
const label = typeof parsed.label === 'string' ? parsed.label : null;
|
|
@@ -1048,8 +1429,17 @@ export class BuyerProxy {
|
|
|
1048
1429
|
const channels = all
|
|
1049
1430
|
? this._node.getAllBuyerChannels()
|
|
1050
1431
|
: this._node.getActiveBuyerChannels();
|
|
1432
|
+
const peers = await this._getPeers();
|
|
1433
|
+
const peersById = new Map(peers.map((peer) => [peer.peerId, peer]));
|
|
1434
|
+
const channelsWithCapabilities = channels.map((channel) => {
|
|
1435
|
+
const peer = peersById.get(channel.peerId);
|
|
1436
|
+
return {
|
|
1437
|
+
...channel,
|
|
1438
|
+
cooperativeCloseSupported: peer ? peerSupportsCooperativeClose(peer) : false,
|
|
1439
|
+
};
|
|
1440
|
+
});
|
|
1051
1441
|
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1052
|
-
res.end(JSON.stringify({ ok: true, channels }));
|
|
1442
|
+
res.end(JSON.stringify({ ok: true, channels: channelsWithCapabilities }));
|
|
1053
1443
|
return;
|
|
1054
1444
|
}
|
|
1055
1445
|
if (path.startsWith('/_antseed/buyer-usage') && method === 'GET') {
|
|
@@ -1168,6 +1558,8 @@ export class BuyerProxy {
|
|
|
1168
1558
|
const isKnownApiPath = normalizedPath.startsWith('/v1/messages') ||
|
|
1169
1559
|
normalizedPath.startsWith('/v1/chat/completions') ||
|
|
1170
1560
|
normalizedPath.startsWith('/v1/responses') ||
|
|
1561
|
+
normalizedPath.startsWith('/v1/images/generations') ||
|
|
1562
|
+
normalizedPath.startsWith('/v1/images/edits') ||
|
|
1171
1563
|
normalizedPath.startsWith('/v1/models');
|
|
1172
1564
|
if (!isKnownApiPath) {
|
|
1173
1565
|
res.writeHead(404, { 'content-type': 'application/json' });
|
|
@@ -1589,6 +1981,15 @@ export class BuyerProxy {
|
|
|
1589
1981
|
if (!selectedRoutePlan) {
|
|
1590
1982
|
return { done: false, statusCode: 502, responseBody: Buffer.from('No compatible provider route'), responseHeaders: { 'content-type': 'text/plain' }, errorMessage: null };
|
|
1591
1983
|
}
|
|
1984
|
+
// Soft supportedParameters check — only for direct routes, since a
|
|
1985
|
+
// protocol transform rebuilds the body for the target protocol anyway.
|
|
1986
|
+
if (!selectedRoutePlan.selection?.requiresTransform) {
|
|
1987
|
+
const unannounced = findUnannouncedRequestParameters(selectedPeer, selectedRoutePlan.provider, requestedService, requestProtocol, serializedReq);
|
|
1988
|
+
if (unannounced.length > 0) {
|
|
1989
|
+
log(`Warning: request to "${requestedService}" carries parameters peer ${selectedPeer.peerId.slice(0, 12)} `
|
|
1990
|
+
+ `did not announce for this service: ${unannounced.join(', ')} — the upstream may ignore or reject them`);
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1592
1993
|
const { 'x-antseed-pin-peer': _pinPeer, 'x-antseed-prefer-peer': _preferPeer, ...headersForPeer } = serializedReq.headers;
|
|
1593
1994
|
let requestForPeer = {
|
|
1594
1995
|
...serializedReq,
|
|
@@ -1690,8 +2091,10 @@ export class BuyerProxy {
|
|
|
1690
2091
|
}
|
|
1691
2092
|
},
|
|
1692
2093
|
}, { signal: requestSignal });
|
|
1693
|
-
let responseForClient = response;
|
|
1694
|
-
if (!streamed
|
|
2094
|
+
let responseForClient = adaptBuyerFaultErrorResponse(response, requestProtocol);
|
|
2095
|
+
if (!streamed
|
|
2096
|
+
&& adaptResponse
|
|
2097
|
+
&& responseForClient.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER]?.toLowerCase() !== 'buyer') {
|
|
1695
2098
|
responseForClient = adaptResponse(response);
|
|
1696
2099
|
}
|
|
1697
2100
|
responseForClient = adaptOpenAICompatibleErrorResponse(responseForClient, requestProtocol);
|
|
@@ -1706,13 +2109,14 @@ export class BuyerProxy {
|
|
|
1706
2109
|
log(`${prefix}: ${summarizeErrorResponse(responseForClient)}`);
|
|
1707
2110
|
}
|
|
1708
2111
|
const telemetry = computeResponseTelemetry(requestForPeer, responseForClient.headers, responseForClient.body, selectedPeer);
|
|
2112
|
+
const responseFault = responseFaultAttribution(responseForClient);
|
|
1709
2113
|
const modelNotFound = !streamed
|
|
1710
2114
|
&& !isControlPlaneServicesPath(requestForPeer.path)
|
|
1711
2115
|
&& isModelNotFoundResponse(responseForClient);
|
|
1712
2116
|
if (modelNotFound) {
|
|
1713
2117
|
this._onModelNotFound(selectedPeer.peerId, requestedService);
|
|
1714
2118
|
}
|
|
1715
|
-
if (router) {
|
|
2119
|
+
if (router && responseFault !== 'buyer') {
|
|
1716
2120
|
router.onResult(selectedPeer, {
|
|
1717
2121
|
success: !modelNotFound
|
|
1718
2122
|
&& isRouterSuccess(responseForClient.statusCode, requestForPeer.path, retryableStatusCodes),
|
|
@@ -1720,11 +2124,14 @@ export class BuyerProxy {
|
|
|
1720
2124
|
tokens: telemetry.usage.totalTokens,
|
|
1721
2125
|
});
|
|
1722
2126
|
}
|
|
2127
|
+
if (responseFault === 'buyer') {
|
|
2128
|
+
this._recordPeerFailure(selectedPeer.peerId, 'buyer-local', 'buyer');
|
|
2129
|
+
}
|
|
2130
|
+
else {
|
|
2131
|
+
this._recordPeerResponseHealth(selectedPeer.peerId, responseForClient.statusCode, requestForPeer.path);
|
|
2132
|
+
}
|
|
1723
2133
|
if (streamed) {
|
|
1724
2134
|
// Headers already sent to client, can't retry
|
|
1725
|
-
if (responseForClient.statusCode >= 200 && responseForClient.statusCode < 400) {
|
|
1726
|
-
this._rememberSuccessfulPeer(selectedPeer.peerId);
|
|
1727
|
-
}
|
|
1728
2135
|
if (!res.writableEnded) {
|
|
1729
2136
|
res.end();
|
|
1730
2137
|
}
|
|
@@ -1741,9 +2148,6 @@ export class BuyerProxy {
|
|
|
1741
2148
|
errorMessage: null,
|
|
1742
2149
|
};
|
|
1743
2150
|
}
|
|
1744
|
-
if (responseForClient.statusCode >= 200 && responseForClient.statusCode < 400) {
|
|
1745
|
-
this._rememberSuccessfulPeer(selectedPeer.peerId);
|
|
1746
|
-
}
|
|
1747
2151
|
res.writeHead(responseForClient.statusCode, responseHeaders);
|
|
1748
2152
|
res.end(Buffer.from(responseForClient.body));
|
|
1749
2153
|
return { done: true };
|
|
@@ -1753,8 +2157,9 @@ export class BuyerProxy {
|
|
|
1753
2157
|
if (upstreamResponse.statusCode >= 400 && !adaptResponse) {
|
|
1754
2158
|
log(`Upstream raw error detail: ${summarizeErrorResponse(upstreamResponse)}`);
|
|
1755
2159
|
}
|
|
1756
|
-
let response = upstreamResponse;
|
|
1757
|
-
if (adaptResponse
|
|
2160
|
+
let response = adaptBuyerFaultErrorResponse(upstreamResponse, requestProtocol);
|
|
2161
|
+
if (adaptResponse
|
|
2162
|
+
&& response.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER]?.toLowerCase() !== 'buyer') {
|
|
1758
2163
|
response = adaptResponse(response);
|
|
1759
2164
|
}
|
|
1760
2165
|
response = adaptOpenAICompatibleErrorResponse(response, requestProtocol);
|
|
@@ -1771,13 +2176,14 @@ export class BuyerProxy {
|
|
|
1771
2176
|
}
|
|
1772
2177
|
const telemetry = computeResponseTelemetry(requestForPeer, response.headers, response.body, selectedPeer);
|
|
1773
2178
|
const responseHeaders = attachAntseedTelemetryHeaders(response.headers, selectedPeer, telemetry, requestForPeer.requestId, latencyMs);
|
|
2179
|
+
const responseFault = responseFaultAttribution(response);
|
|
1774
2180
|
const modelNotFound = !isControlPlaneServicesPath(requestForPeer.path)
|
|
1775
2181
|
&& isModelNotFoundResponse(response);
|
|
1776
2182
|
if (modelNotFound) {
|
|
1777
2183
|
this._onModelNotFound(selectedPeer.peerId, requestedService);
|
|
1778
2184
|
}
|
|
1779
2185
|
// Report result to router for learning
|
|
1780
|
-
if (router) {
|
|
2186
|
+
if (router && responseFault !== 'buyer') {
|
|
1781
2187
|
router.onResult(selectedPeer, {
|
|
1782
2188
|
success: !modelNotFound
|
|
1783
2189
|
&& isRouterSuccess(response.statusCode, requestForPeer.path, retryableStatusCodes),
|
|
@@ -1785,13 +2191,16 @@ export class BuyerProxy {
|
|
|
1785
2191
|
tokens: telemetry.usage.totalTokens,
|
|
1786
2192
|
});
|
|
1787
2193
|
}
|
|
2194
|
+
if (responseFault === 'buyer') {
|
|
2195
|
+
this._recordPeerFailure(selectedPeer.peerId, 'buyer-local', 'buyer');
|
|
2196
|
+
}
|
|
2197
|
+
else {
|
|
2198
|
+
this._recordPeerResponseHealth(selectedPeer.peerId, response.statusCode, requestForPeer.path);
|
|
2199
|
+
}
|
|
1788
2200
|
// Check if retryable
|
|
1789
2201
|
if (retryableStatusCodes.has(response.statusCode)) {
|
|
1790
2202
|
return { done: false, statusCode: response.statusCode, responseBody: Buffer.from(response.body), responseHeaders, errorMessage: null };
|
|
1791
2203
|
}
|
|
1792
|
-
if (response.statusCode >= 200 && response.statusCode < 400) {
|
|
1793
|
-
this._rememberSuccessfulPeer(selectedPeer.peerId);
|
|
1794
|
-
}
|
|
1795
2204
|
// Forward response headers and body to the HTTP client
|
|
1796
2205
|
res.writeHead(response.statusCode, responseHeaders);
|
|
1797
2206
|
res.end(Buffer.from(response.body));
|
|
@@ -1836,7 +2245,14 @@ export class BuyerProxy {
|
|
|
1836
2245
|
}
|
|
1837
2246
|
return { done: true };
|
|
1838
2247
|
}
|
|
1839
|
-
this
|
|
2248
|
+
// Whose fault was this? Errors raised by our own wallet, deposits,
|
|
2249
|
+
// transport or state machine say nothing about the peer, and telling the
|
|
2250
|
+
// user to blame the seller for their empty deposit sends them chasing the
|
|
2251
|
+
// wrong fix. Anything untagged stays 'unknown', where the attribution
|
|
2252
|
+
// gates decide.
|
|
2253
|
+
const fault = faultAttributionOf(err);
|
|
2254
|
+
const faultCode = faultCodeOf(err);
|
|
2255
|
+
this._recordPeerFailure(selectedPeer.peerId, fault === 'buyer' ? 'buyer-local' : 'request-failed', fault);
|
|
1840
2256
|
if (res.headersSent) {
|
|
1841
2257
|
// Headers already sent (streaming), can't retry
|
|
1842
2258
|
if (!res.writableEnded) {
|
|
@@ -1844,6 +2260,30 @@ export class BuyerProxy {
|
|
|
1844
2260
|
}
|
|
1845
2261
|
return { done: true };
|
|
1846
2262
|
}
|
|
2263
|
+
if (fault === 'buyer') {
|
|
2264
|
+
const buyerResponse = adaptBuyerFaultErrorResponse({
|
|
2265
|
+
requestId: requestForPeer.requestId,
|
|
2266
|
+
statusCode: 503,
|
|
2267
|
+
headers: {
|
|
2268
|
+
'content-type': 'application/json',
|
|
2269
|
+
[ANTSEED_FAULT_ATTRIBUTION_HEADER]: 'buyer',
|
|
2270
|
+
},
|
|
2271
|
+
body: Buffer.from(JSON.stringify({
|
|
2272
|
+
error: {
|
|
2273
|
+
type: 'buyer_request_failed',
|
|
2274
|
+
code: faultCode ?? 'buyer_request_failed',
|
|
2275
|
+
message,
|
|
2276
|
+
},
|
|
2277
|
+
})),
|
|
2278
|
+
}, requestProtocol);
|
|
2279
|
+
return {
|
|
2280
|
+
done: false,
|
|
2281
|
+
statusCode: buyerResponse.statusCode,
|
|
2282
|
+
responseBody: Buffer.from(buyerResponse.body),
|
|
2283
|
+
responseHeaders: buyerResponse.headers,
|
|
2284
|
+
errorMessage: message,
|
|
2285
|
+
};
|
|
2286
|
+
}
|
|
1847
2287
|
return { done: false, statusCode: 502, responseBody: Buffer.from(`P2P request failed: ${message}`), responseHeaders: { 'content-type': 'text/plain' }, errorMessage: message };
|
|
1848
2288
|
}
|
|
1849
2289
|
}
|