@antseed/cli 0.1.144 → 0.1.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/proxy/buyer-proxy.d.ts +59 -9
- package/dist/proxy/buyer-proxy.d.ts.map +1 -1
- package/dist/proxy/buyer-proxy.js +391 -47
- package/dist/proxy/buyer-proxy.js.map +1 -1
- package/dist/proxy/buyer-proxy.test.js +339 -11
- package/dist/proxy/buyer-proxy.test.js.map +1 -1
- package/dist/proxy/peer-attribution.d.ts +100 -0
- package/dist/proxy/peer-attribution.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.js +175 -0
- package/dist/proxy/peer-attribution.js.map +1 -0
- package/dist/proxy/peer-attribution.test.d.ts +2 -0
- package/dist/proxy/peer-attribution.test.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.test.js +140 -0
- package/dist/proxy/peer-attribution.test.js.map +1 -0
- package/dist/proxy/peer-health.d.ts +73 -0
- package/dist/proxy/peer-health.d.ts.map +1 -0
- package/dist/proxy/peer-health.js +182 -0
- package/dist/proxy/peer-health.js.map +1 -0
- package/dist/proxy/peer-health.test.d.ts +2 -0
- package/dist/proxy/peer-health.test.d.ts.map +1 -0
- package/dist/proxy/peer-health.test.js +197 -0
- package/dist/proxy/peer-health.test.js.map +1 -0
- package/package.json +4 -4
|
@@ -3,14 +3,16 @@ import { randomUUID } from 'node:crypto';
|
|
|
3
3
|
import { watchFile, unwatchFile } from 'node:fs';
|
|
4
4
|
import { readFile, writeFile, rename, mkdir, readdir, stat, unlink } from 'node:fs/promises';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
-
import { ANTSEED_ATTEST_PATH, computeOnChainReputationScore, decodeSweepRequest, peerSupportsCooperativeClose, } from '@antseed/node';
|
|
6
|
+
import { ANTSEED_BUYER_FAULT_ERROR_CODE, ANTSEED_FAULT_ATTRIBUTION_HEADER, ANTSEED_ATTEST_PATH, computeOnChainReputationScore, decodeSweepRequest, faultAttributionOf, faultCodeOf, peerSupportsCooperativeClose, } from '@antseed/node';
|
|
7
7
|
import { createStreamingAdapter, detectRequestServiceApiProtocol, transformRequest, transformResponse, } from './service-api-adapter.js';
|
|
8
|
-
import { DEBUG, log, extractRequestedService, summarizeRequestShape, summarizeErrorResponse, requestWantsStreaming, parsePeerPinnedService, rewritePeerPinnedServiceInBody, substituteRoutedModelAlias, overrideRoutedModelInBody, ROUTED_MODEL_ALIAS, SYSTEM_PROXY_SOURCE_HEADER, SYSTEM_ROUTED_MODEL_HEADER, } from './request-utils.js';
|
|
8
|
+
import { DEBUG, log, extractRequestedService, summarizeRequestShape, summarizeErrorResponse, requestWantsStreaming, parsePeerPinnedService, rewritePeerPinnedServiceInBody, substituteRoutedModelAlias, overrideRoutedModelInBody, ROUTED_MODEL_ALIAS, SYSTEM_PROXY_SOURCE_HEADER, SYSTEM_ROUTED_MODEL_HEADER, normalizePeerId, } from './request-utils.js';
|
|
9
9
|
import { findUnannouncedRequestParameters, getExplicitProviderOverride, getExplicitPeerIdOverride, resolvePeerRoutePlan, selectCandidatePeersForRouting, } from './routing.js';
|
|
10
10
|
import { computeResponseTelemetry, attachAntseedTelemetryHeaders, attachStreamingAntseedHeaders, } from './telemetry.js';
|
|
11
11
|
import { DEFAULT_BUYER_PEER_REFRESH_INTERVAL_MS } from '../config/defaults.js';
|
|
12
12
|
import { extractConversationIdentity, extractFirstUserSnippet, isCompletionRequestPath, isTitleGenerationRequest, parseRequestBodyObject, } from './conversation-identity.js';
|
|
13
13
|
import { ConversationStore } from './conversation-store.js';
|
|
14
|
+
import { recordPeerFailureEntry, clearPeerHealthEntry, isCoolingDown, parsePersistedPeerHealth, prunePeerHealth, serializePeerHealth, reasonEscalates, } from './peer-health.js';
|
|
15
|
+
import { PeerAttributionTracker, HEARTBEAT_MS } from './peer-attribution.js';
|
|
14
16
|
import { estimateAnthropicPromptTokens, isCountTokensPath } from './count-tokens.js';
|
|
15
17
|
import { getCachedVerdict, runVerifier, verifierSupportFingerprint } from '../plugins/verifier.js';
|
|
16
18
|
// Re-export for backward compatibility (used by tests and other consumers)
|
|
@@ -50,7 +52,6 @@ export function isModelNotFoundResponse(response) {
|
|
|
50
52
|
* liveness (`lastReachedAt`) even if the DHT record is older.
|
|
51
53
|
*/
|
|
52
54
|
const CARRY_FORWARD_TTL_MS = 2 * 60 * 60_000;
|
|
53
|
-
const PEER_FAILURE_WINDOW_MS = 5 * 60_000;
|
|
54
55
|
/**
|
|
55
56
|
* Requests kept in the spend-attribution map. Entries outlive their request on
|
|
56
57
|
* purpose (a seller-initiated auth can land after the response), so this is
|
|
@@ -61,6 +62,35 @@ const MAX_TRACKED_REQUEST_CONVERSATIONS = 512;
|
|
|
61
62
|
const MODEL_NOT_FOUND_REFRESH_THROTTLE_MS = 30_000;
|
|
62
63
|
/** Verification is expensive; bound how many verdicts we retain (TTL = peer-cache TTL). */
|
|
63
64
|
const VERIFY_CACHE_MAX_ENTRIES = 1024;
|
|
65
|
+
/**
|
|
66
|
+
* Statuses that prove the peer is alive and serving. Any response short of a
|
|
67
|
+
* server error counts: a peer that answers 400 or 404 is reachable, and
|
|
68
|
+
* treating only 2xx as proof would leave a stale cooldown on a healthy peer
|
|
69
|
+
* that happens to reject every request.
|
|
70
|
+
*/
|
|
71
|
+
function isProofOfLife(statusCode) {
|
|
72
|
+
return statusCode < 500 && statusCode !== 408;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Map a seller's response status onto a health reason, or null when the status
|
|
76
|
+
* says nothing about the peer's liveness.
|
|
77
|
+
*/
|
|
78
|
+
function failureReasonForStatus(statusCode) {
|
|
79
|
+
if (statusCode === 408)
|
|
80
|
+
return 'seller-timeout';
|
|
81
|
+
// Rate limiting is capacity pressure, not death — recorded, never escalated.
|
|
82
|
+
if (statusCode === 429)
|
|
83
|
+
return 'seller-busy';
|
|
84
|
+
if (statusCode >= 500 && statusCode <= 599)
|
|
85
|
+
return 'seller-5xx';
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
function responseFaultAttribution(response) {
|
|
89
|
+
const attribution = response.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER]?.toLowerCase();
|
|
90
|
+
return attribution === 'buyer' || attribution === 'peer' || attribution === 'unknown'
|
|
91
|
+
? attribution
|
|
92
|
+
: 'peer';
|
|
93
|
+
}
|
|
64
94
|
function adaptOpenAICompatibleErrorResponse(response, requestProtocol) {
|
|
65
95
|
if (response.statusCode !== 402) {
|
|
66
96
|
return response;
|
|
@@ -99,6 +129,76 @@ function adaptOpenAICompatibleErrorResponse(response, requestProtocol) {
|
|
|
99
129
|
body: Buffer.from(JSON.stringify(wrappedError)),
|
|
100
130
|
};
|
|
101
131
|
}
|
|
132
|
+
function adaptBuyerFaultErrorResponse(response, requestProtocol) {
|
|
133
|
+
if (response.statusCode < 400
|
|
134
|
+
|| response.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER]?.toLowerCase() !== 'buyer') {
|
|
135
|
+
return sanitizePeerBuyerFaultMarker(response);
|
|
136
|
+
}
|
|
137
|
+
let parsed = {};
|
|
138
|
+
try {
|
|
139
|
+
parsed = JSON.parse(Buffer.from(response.body).toString('utf-8'));
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// Buyer-generated failures should be JSON, but keep a useful fallback if
|
|
143
|
+
// a future path emits plain text.
|
|
144
|
+
}
|
|
145
|
+
const nestedError = parsed.error && typeof parsed.error === 'object' && !Array.isArray(parsed.error)
|
|
146
|
+
? parsed.error
|
|
147
|
+
: null;
|
|
148
|
+
const reason = [nestedError?.code, parsed.code, parsed.reason, nestedError?.type, parsed.error]
|
|
149
|
+
.find((value) => typeof value === 'string' && value.trim().length > 0);
|
|
150
|
+
const message = [nestedError?.message, parsed.message, parsed.error]
|
|
151
|
+
.find((value) => typeof value === 'string' && value.trim().length > 0)
|
|
152
|
+
?? 'The request failed on the buyer.';
|
|
153
|
+
const body = requestProtocol === 'anthropic-messages'
|
|
154
|
+
? {
|
|
155
|
+
type: 'error',
|
|
156
|
+
error: {
|
|
157
|
+
type: ANTSEED_BUYER_FAULT_ERROR_CODE,
|
|
158
|
+
message: reason ? `${message} (${reason})` : message,
|
|
159
|
+
},
|
|
160
|
+
}
|
|
161
|
+
: {
|
|
162
|
+
error: {
|
|
163
|
+
type: 'api_error',
|
|
164
|
+
code: ANTSEED_BUYER_FAULT_ERROR_CODE,
|
|
165
|
+
message,
|
|
166
|
+
...(reason ? { param: reason } : {}),
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
return {
|
|
170
|
+
...response,
|
|
171
|
+
headers: { ...response.headers, 'content-type': 'application/json' },
|
|
172
|
+
body: Buffer.from(JSON.stringify(body)),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
function sanitizePeerBuyerFaultMarker(response) {
|
|
176
|
+
if (response.statusCode < 400)
|
|
177
|
+
return response;
|
|
178
|
+
let parsed;
|
|
179
|
+
try {
|
|
180
|
+
parsed = JSON.parse(Buffer.from(response.body).toString('utf-8'));
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
return response;
|
|
184
|
+
}
|
|
185
|
+
let changed = false;
|
|
186
|
+
const scrub = (record) => {
|
|
187
|
+
for (const key of ['code', 'type', 'errorCode']) {
|
|
188
|
+
if (record[key] === ANTSEED_BUYER_FAULT_ERROR_CODE) {
|
|
189
|
+
record[key] = 'upstream_error';
|
|
190
|
+
changed = true;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
scrub(parsed);
|
|
195
|
+
if (parsed.error && typeof parsed.error === 'object' && !Array.isArray(parsed.error)) {
|
|
196
|
+
scrub(parsed.error);
|
|
197
|
+
}
|
|
198
|
+
return changed
|
|
199
|
+
? { ...response, body: Buffer.from(JSON.stringify(parsed)) }
|
|
200
|
+
: response;
|
|
201
|
+
}
|
|
102
202
|
/**
|
|
103
203
|
* Inject the buyer-known peerId into a 402 payment_required JSON body.
|
|
104
204
|
* The seller doesn't include its own peerId (and shouldn't — self-reported
|
|
@@ -426,7 +526,16 @@ export class BuyerProxy {
|
|
|
426
526
|
_consecutiveEmptyDiscoveries = 0;
|
|
427
527
|
_lastModelNotFoundRefreshAtMs = 0;
|
|
428
528
|
_bgRefreshHandle = null;
|
|
429
|
-
|
|
529
|
+
/**
|
|
530
|
+
* Per-peer failure streaks and cooldowns. Advisory only: a cooling-down peer
|
|
531
|
+
* is still dispatched to when a request names it, so routing can never
|
|
532
|
+
* deadlock and pinned conversations keep working.
|
|
533
|
+
*/
|
|
534
|
+
_peerHealth = new Map();
|
|
535
|
+
/** Decides whether a failure is the peer's fault at all. */
|
|
536
|
+
_attribution = new PeerAttributionTracker();
|
|
537
|
+
_heartbeatHandle = null;
|
|
538
|
+
_now;
|
|
430
539
|
/** Latest relayer receipt per sweep authNonce, for CLI progress polling. */
|
|
431
540
|
_sweepReceipts = new Map();
|
|
432
541
|
/**
|
|
@@ -448,6 +557,7 @@ export class BuyerProxy {
|
|
|
448
557
|
this._stateFile = join(config.dataDir, 'buyer.state.json');
|
|
449
558
|
this._conversations = new ConversationStore(config.dataDir);
|
|
450
559
|
this._pinnedPeer = config.pinnedPeerId?.toLowerCase() ?? null;
|
|
560
|
+
this._now = config.now ?? (() => Date.now());
|
|
451
561
|
this._server = createServer((req, res) => {
|
|
452
562
|
this._handleRequest(req, res).catch((err) => {
|
|
453
563
|
log('Unhandled error:', err);
|
|
@@ -537,6 +647,7 @@ export class BuyerProxy {
|
|
|
537
647
|
});
|
|
538
648
|
});
|
|
539
649
|
this._startBackgroundRefresh();
|
|
650
|
+
this._startSuspendHeartbeat();
|
|
540
651
|
// Trigger initial discovery immediately so the desktop can show services
|
|
541
652
|
// without waiting for the first request or 5-minute interval. The sweep
|
|
542
653
|
// emits each accepted metadata document as it arrives, so buyer.state.json
|
|
@@ -549,6 +660,11 @@ export class BuyerProxy {
|
|
|
549
660
|
try {
|
|
550
661
|
const raw = await readFile(this._stateFile, 'utf-8');
|
|
551
662
|
const parsed = JSON.parse(raw);
|
|
663
|
+
// Cooldowns survive a restart — a peer that died ten seconds before we
|
|
664
|
+
// exited is still dead — but the parser clamps anything expired or
|
|
665
|
+
// impossibly distant, so a restart can never extend one. Nothing new can
|
|
666
|
+
// escalate until a success re-establishes that the buyer is healthy.
|
|
667
|
+
this._peerHealth = parsePersistedPeerHealth(parsed, this._now());
|
|
552
668
|
const peers = parsePersistedPeers(parsed);
|
|
553
669
|
if (peers.length === 0) {
|
|
554
670
|
return;
|
|
@@ -581,6 +697,10 @@ export class BuyerProxy {
|
|
|
581
697
|
clearInterval(this._bgRefreshHandle);
|
|
582
698
|
this._bgRefreshHandle = null;
|
|
583
699
|
}
|
|
700
|
+
if (this._heartbeatHandle) {
|
|
701
|
+
clearInterval(this._heartbeatHandle);
|
|
702
|
+
this._heartbeatHandle = null;
|
|
703
|
+
}
|
|
584
704
|
await this._writeStateFile('stopped');
|
|
585
705
|
await this._conversations.flush();
|
|
586
706
|
return new Promise((resolve) => {
|
|
@@ -780,36 +900,119 @@ export class BuyerProxy {
|
|
|
780
900
|
});
|
|
781
901
|
}
|
|
782
902
|
/**
|
|
783
|
-
*
|
|
784
|
-
*
|
|
785
|
-
*
|
|
903
|
+
* Record a failed request against a peer.
|
|
904
|
+
*
|
|
905
|
+
* Recording is unconditional — the streak and reason are useful diagnostics
|
|
906
|
+
* either way — but only failures the attribution gates accept as the peer's
|
|
907
|
+
* own move the cooldown. Discovery metadata is never evicted: a cooling-down
|
|
908
|
+
* peer stays routable, it just stops being *chosen*.
|
|
786
909
|
*/
|
|
787
|
-
_recordPeerFailure(peerId, reason) {
|
|
788
|
-
const now =
|
|
789
|
-
const
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
910
|
+
_recordPeerFailure(peerId, reason, fault = 'unknown') {
|
|
911
|
+
const now = this._now();
|
|
912
|
+
const { verdict, rollbackPeerIds } = this._attribution.classify({
|
|
913
|
+
peerId,
|
|
914
|
+
reasonEscalates: reasonEscalates(reason),
|
|
915
|
+
fault,
|
|
916
|
+
now,
|
|
917
|
+
});
|
|
918
|
+
const previous = this._peerHealth.get(peerId);
|
|
919
|
+
const entry = recordPeerFailureEntry(previous, reason, now, verdict.escalate);
|
|
920
|
+
this._peerHealth.set(peerId, entry);
|
|
921
|
+
if (rollbackPeerIds.length > 0) {
|
|
922
|
+
this._rollbackPeerHealth(rollbackPeerIds, 'buyer-side outage detected');
|
|
923
|
+
}
|
|
924
|
+
if (verdict.escalate && isCoolingDown(entry, now)) {
|
|
925
|
+
const seconds = Math.round((entry.cooldownUntil - now) / 1000);
|
|
926
|
+
log(`Peer ${peerId.slice(0, 12)}... cooling down for ${seconds}s after `
|
|
927
|
+
+ `${entry.failureStreak} failures (reason=${reason}).`);
|
|
928
|
+
}
|
|
929
|
+
else {
|
|
930
|
+
const why = verdict.escalate ? 'below cooldown threshold' : verdict.suppressedBy;
|
|
931
|
+
log(`Peer ${peerId.slice(0, 12)}... failure recorded (reason=${reason}); `
|
|
932
|
+
+ `not cooling down: ${why}.`);
|
|
933
|
+
}
|
|
934
|
+
void this._persistPeerHealthToState();
|
|
935
|
+
}
|
|
936
|
+
/**
|
|
937
|
+
* Fold a seller's HTTP response into that peer's health.
|
|
938
|
+
*
|
|
939
|
+
* Control-plane paths are exempt for the same reason `isRouterSuccess`
|
|
940
|
+
* exempts them: a failing `/v1/models` says nothing about the peer's ability
|
|
941
|
+
* to serve inference.
|
|
942
|
+
*/
|
|
943
|
+
_recordPeerResponseHealth(peerId, statusCode, path) {
|
|
944
|
+
if (isControlPlaneServicesPath(path)) {
|
|
945
|
+
if (isProofOfLife(statusCode))
|
|
946
|
+
this._rememberSuccessfulPeer(peerId);
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
949
|
+
const reason = failureReasonForStatus(statusCode);
|
|
950
|
+
if (reason && statusCode >= 500) {
|
|
951
|
+
const now = this._now();
|
|
952
|
+
if (isCoolingDown(this._peerHealth.get(peerId), now)) {
|
|
953
|
+
this._rememberSuccessfulPeer(peerId);
|
|
954
|
+
}
|
|
955
|
+
this._recordPeerFailure(peerId, reason, 'peer');
|
|
956
|
+
return;
|
|
957
|
+
}
|
|
958
|
+
if (isProofOfLife(statusCode)) {
|
|
959
|
+
// A 402, a 400, even a 429 — the peer answered, so it is alive and any
|
|
960
|
+
// cooldown is stale. Throttling still gets stamped as the last reason so
|
|
961
|
+
// "alive but refusing work" stays visible in diagnostics.
|
|
962
|
+
this._rememberSuccessfulPeer(peerId);
|
|
963
|
+
if (reason) {
|
|
964
|
+
const entry = this._peerHealth.get(peerId);
|
|
965
|
+
if (entry) {
|
|
966
|
+
this._peerHealth.set(peerId, { ...entry, lastReason: reason, lastFailureAt: this._now() });
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
if (reason)
|
|
972
|
+
this._recordPeerFailure(peerId, reason, 'peer');
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* Undo cooldowns that turned out to be our fault.
|
|
976
|
+
*
|
|
977
|
+
* When the attribution gates conclude the buyer itself was down — a suspend,
|
|
978
|
+
* a dropped network — the failures recorded during that window blamed the
|
|
979
|
+
* wrong party, so the streaks they created are wound back to zero.
|
|
980
|
+
*/
|
|
981
|
+
_rollbackPeerHealth(peerIds, why) {
|
|
982
|
+
let changed = false;
|
|
983
|
+
for (const peerId of peerIds) {
|
|
984
|
+
const entry = this._peerHealth.get(peerId);
|
|
985
|
+
if (!entry || (entry.failureStreak === 0 && entry.cooldownUntil === 0))
|
|
986
|
+
continue;
|
|
987
|
+
this._peerHealth.set(peerId, {
|
|
988
|
+
...entry,
|
|
989
|
+
failureStreak: 0,
|
|
990
|
+
windowStartedAt: 0,
|
|
991
|
+
episodeStartedAt: 0,
|
|
992
|
+
cooldownUntil: 0,
|
|
993
|
+
});
|
|
994
|
+
changed = true;
|
|
995
|
+
}
|
|
996
|
+
if (changed) {
|
|
997
|
+
log(`Cleared peer cooldowns for ${peerIds.length} peer(s): ${why}.`);
|
|
998
|
+
void this._persistPeerHealthToState();
|
|
999
|
+
}
|
|
802
1000
|
}
|
|
803
1001
|
/**
|
|
804
1002
|
* A peer told us it does not serve the requested model. Our cached
|
|
805
1003
|
* metadata for it is stale (the seller may have just unadvertised the
|
|
806
|
-
* model after failing its own health checks), so
|
|
807
|
-
*
|
|
808
|
-
*
|
|
1004
|
+
* model after failing its own health checks), so refresh discovery
|
|
1005
|
+
* metadata in the background — throttled, since one broken model can
|
|
1006
|
+
* produce a burst of these.
|
|
1007
|
+
*
|
|
1008
|
+
* Deliberately does NOT touch peer health: the response itself is proof of
|
|
1009
|
+
* life (`_recordPeerResponseHealth` treats any sub-500 answer as such), and
|
|
1010
|
+
* a peer that is healthy for its other models must not cool down over one
|
|
1011
|
+
* stale catalog entry. The router still learns via `onResult(success:false)`
|
|
1012
|
+
* so scoring reflects the miss.
|
|
809
1013
|
*/
|
|
810
1014
|
_onModelNotFound(peerId, requestedService) {
|
|
811
|
-
this.
|
|
812
|
-
const now = Date.now();
|
|
1015
|
+
const now = this._now();
|
|
813
1016
|
if (now - this._lastModelNotFoundRefreshAtMs < MODEL_NOT_FOUND_REFRESH_THROTTLE_MS) {
|
|
814
1017
|
return;
|
|
815
1018
|
}
|
|
@@ -821,17 +1024,56 @@ export class BuyerProxy {
|
|
|
821
1024
|
/**
|
|
822
1025
|
* Stamp `lastReachedAt` on a peer after a successful request so the
|
|
823
1026
|
* carry-forward heuristic can trust local transport liveness even when the
|
|
824
|
-
* DHT record grows stale. Persisted so the signal survives restarts.
|
|
825
|
-
*
|
|
1027
|
+
* DHT record grows stale. Persisted so the signal survives restarts.
|
|
1028
|
+
*
|
|
1029
|
+
* A response is also proof that the buyer's own network, DHT, chain RPC and
|
|
1030
|
+
* wallet are working, which is what lets other peers' failures be attributed
|
|
1031
|
+
* to them rather than to us.
|
|
826
1032
|
*/
|
|
827
1033
|
_rememberSuccessfulPeer(peerId) {
|
|
828
|
-
this.
|
|
1034
|
+
const now = this._now();
|
|
1035
|
+
this._attribution.recordSuccess(peerId, now);
|
|
1036
|
+
const previous = this._peerHealth.get(peerId);
|
|
1037
|
+
if (previous && (previous.failureStreak > 0 || previous.cooldownUntil > 0)) {
|
|
1038
|
+
log(`Peer ${peerId.slice(0, 12)}... recovered; cooldown cleared.`);
|
|
1039
|
+
}
|
|
1040
|
+
this._peerHealth.set(peerId, clearPeerHealthEntry(previous, now));
|
|
1041
|
+
void this._persistPeerHealthToState();
|
|
829
1042
|
const cached = this._cachedPeers.find((p) => p.peerId === peerId);
|
|
830
1043
|
if (cached) {
|
|
831
|
-
cached.lastReachedAt =
|
|
1044
|
+
cached.lastReachedAt = now;
|
|
832
1045
|
this._persistPeersToState();
|
|
833
1046
|
}
|
|
834
1047
|
}
|
|
1048
|
+
/**
|
|
1049
|
+
* Watch for the wall clock jumping forward, which means the machine slept.
|
|
1050
|
+
* On wake every pending timeout and keepalive fires at once, so without this
|
|
1051
|
+
* a single closed lid would cool down every peer the buyer knows.
|
|
1052
|
+
*/
|
|
1053
|
+
_startSuspendHeartbeat() {
|
|
1054
|
+
if (this._heartbeatHandle)
|
|
1055
|
+
return;
|
|
1056
|
+
this._attribution.onHeartbeat(this._now());
|
|
1057
|
+
this._heartbeatHandle = setInterval(() => {
|
|
1058
|
+
const result = this._attribution.onHeartbeat(this._now());
|
|
1059
|
+
if (result && result.rollbackPeerIds.length > 0) {
|
|
1060
|
+
this._rollbackPeerHealth(result.rollbackPeerIds, 'machine resumed from sleep');
|
|
1061
|
+
}
|
|
1062
|
+
else if (result) {
|
|
1063
|
+
log('Detected a wall-clock jump; suspending peer cooldowns briefly.');
|
|
1064
|
+
}
|
|
1065
|
+
}, HEARTBEAT_MS);
|
|
1066
|
+
this._heartbeatHandle.unref?.();
|
|
1067
|
+
}
|
|
1068
|
+
/** Persist health separately from `discoveredPeers`, which is rebuilt wholesale. */
|
|
1069
|
+
async _persistPeerHealthToState() {
|
|
1070
|
+
const now = this._now();
|
|
1071
|
+
this._peerHealth = prunePeerHealth(this._peerHealth, now);
|
|
1072
|
+
await this._mergeStateFile({
|
|
1073
|
+
peerHealth: serializePeerHealth(this._peerHealth),
|
|
1074
|
+
peerHealthUpdatedAt: now,
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
835
1077
|
async _discoverPeersFromNetwork() {
|
|
836
1078
|
log('Discovering peers via DHT...');
|
|
837
1079
|
const peers = await this._node.discoverPeers();
|
|
@@ -967,6 +1209,69 @@ export class BuyerProxy {
|
|
|
967
1209
|
res.end(JSON.stringify({ ok: true, peers: payload }));
|
|
968
1210
|
return;
|
|
969
1211
|
}
|
|
1212
|
+
if (path === '/_antseed/peer-health' && method === 'GET') {
|
|
1213
|
+
const now = this._now();
|
|
1214
|
+
const attribution = this._attribution.snapshot(now);
|
|
1215
|
+
// `buyerHealthy` and `suppressedUntil` are what make "why is this peer
|
|
1216
|
+
// (not) cooling down" answerable from outside the process.
|
|
1217
|
+
const peers = [...this._peerHealth.entries()].map(([peerId, entry]) => ({
|
|
1218
|
+
peerId,
|
|
1219
|
+
failureStreak: entry.failureStreak,
|
|
1220
|
+
lastFailureAt: entry.lastFailureAt,
|
|
1221
|
+
lastReason: entry.lastReason,
|
|
1222
|
+
cooldownUntil: entry.cooldownUntil,
|
|
1223
|
+
coolingDown: isCoolingDown(entry, now),
|
|
1224
|
+
cooldownMsRemaining: isCoolingDown(entry, now) ? entry.cooldownUntil - now : 0,
|
|
1225
|
+
lastSuccessAt: entry.lastSuccessAt,
|
|
1226
|
+
}));
|
|
1227
|
+
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1228
|
+
res.end(JSON.stringify({
|
|
1229
|
+
ok: true,
|
|
1230
|
+
now,
|
|
1231
|
+
buyerHealthy: this._attribution.isBuyerHealthy(now),
|
|
1232
|
+
lastAnySuccessAt: attribution.lastAnySuccessAt,
|
|
1233
|
+
suppressionActive: attribution.suppressedUntil > 0,
|
|
1234
|
+
suppressedUntil: attribution.suppressedUntil,
|
|
1235
|
+
lastSuppressedBy: attribution.lastSuppressedBy,
|
|
1236
|
+
peers,
|
|
1237
|
+
}));
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1240
|
+
if (path === '/_antseed/peer-health/clear' && method === 'POST') {
|
|
1241
|
+
const chunks = [];
|
|
1242
|
+
let totalSize = 0;
|
|
1243
|
+
for await (const chunk of req) {
|
|
1244
|
+
totalSize += chunk.length;
|
|
1245
|
+
if (totalSize > 8192) {
|
|
1246
|
+
res.writeHead(413, { 'content-type': 'application/json' });
|
|
1247
|
+
res.end(JSON.stringify({ ok: false, error: 'Request body too large' }));
|
|
1248
|
+
return;
|
|
1249
|
+
}
|
|
1250
|
+
chunks.push(chunk);
|
|
1251
|
+
}
|
|
1252
|
+
let peerId;
|
|
1253
|
+
try {
|
|
1254
|
+
const body = JSON.parse(Buffer.concat(chunks).toString());
|
|
1255
|
+
peerId = typeof body.peerId === 'string' ? body.peerId.trim().toLowerCase() : '';
|
|
1256
|
+
}
|
|
1257
|
+
catch {
|
|
1258
|
+
res.writeHead(400, { 'content-type': 'application/json' });
|
|
1259
|
+
res.end(JSON.stringify({ ok: false, error: 'Invalid JSON body' }));
|
|
1260
|
+
return;
|
|
1261
|
+
}
|
|
1262
|
+
const normalized = normalizePeerId(peerId) ?? peerId;
|
|
1263
|
+
if (!/^[0-9a-f]{40}$/.test(normalized)) {
|
|
1264
|
+
res.writeHead(400, { 'content-type': 'application/json' });
|
|
1265
|
+
res.end(JSON.stringify({ ok: false, error: 'peerId must be a 40-character hex peer id' }));
|
|
1266
|
+
return;
|
|
1267
|
+
}
|
|
1268
|
+
// Deliberately does not stamp a success: the user is asking us to give
|
|
1269
|
+
// the peer another chance, not asserting that it answered.
|
|
1270
|
+
this._rollbackPeerHealth([normalized], 'cleared by request');
|
|
1271
|
+
res.writeHead(200, { 'content-type': 'application/json' });
|
|
1272
|
+
res.end(JSON.stringify({ ok: true, peerId: normalized }));
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
970
1275
|
if (path === '/_antseed/route' && method === 'GET') {
|
|
971
1276
|
res.writeHead(200, { 'content-type': 'application/json' });
|
|
972
1277
|
res.end(JSON.stringify({ ok: true, model: this._defaultRoutedModel }));
|
|
@@ -1786,8 +2091,10 @@ export class BuyerProxy {
|
|
|
1786
2091
|
}
|
|
1787
2092
|
},
|
|
1788
2093
|
}, { signal: requestSignal });
|
|
1789
|
-
let responseForClient = response;
|
|
1790
|
-
if (!streamed
|
|
2094
|
+
let responseForClient = adaptBuyerFaultErrorResponse(response, requestProtocol);
|
|
2095
|
+
if (!streamed
|
|
2096
|
+
&& adaptResponse
|
|
2097
|
+
&& responseForClient.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER]?.toLowerCase() !== 'buyer') {
|
|
1791
2098
|
responseForClient = adaptResponse(response);
|
|
1792
2099
|
}
|
|
1793
2100
|
responseForClient = adaptOpenAICompatibleErrorResponse(responseForClient, requestProtocol);
|
|
@@ -1802,13 +2109,14 @@ export class BuyerProxy {
|
|
|
1802
2109
|
log(`${prefix}: ${summarizeErrorResponse(responseForClient)}`);
|
|
1803
2110
|
}
|
|
1804
2111
|
const telemetry = computeResponseTelemetry(requestForPeer, responseForClient.headers, responseForClient.body, selectedPeer);
|
|
2112
|
+
const responseFault = responseFaultAttribution(responseForClient);
|
|
1805
2113
|
const modelNotFound = !streamed
|
|
1806
2114
|
&& !isControlPlaneServicesPath(requestForPeer.path)
|
|
1807
2115
|
&& isModelNotFoundResponse(responseForClient);
|
|
1808
2116
|
if (modelNotFound) {
|
|
1809
2117
|
this._onModelNotFound(selectedPeer.peerId, requestedService);
|
|
1810
2118
|
}
|
|
1811
|
-
if (router) {
|
|
2119
|
+
if (router && responseFault !== 'buyer') {
|
|
1812
2120
|
router.onResult(selectedPeer, {
|
|
1813
2121
|
success: !modelNotFound
|
|
1814
2122
|
&& isRouterSuccess(responseForClient.statusCode, requestForPeer.path, retryableStatusCodes),
|
|
@@ -1816,11 +2124,14 @@ export class BuyerProxy {
|
|
|
1816
2124
|
tokens: telemetry.usage.totalTokens,
|
|
1817
2125
|
});
|
|
1818
2126
|
}
|
|
2127
|
+
if (responseFault === 'buyer') {
|
|
2128
|
+
this._recordPeerFailure(selectedPeer.peerId, 'buyer-local', 'buyer');
|
|
2129
|
+
}
|
|
2130
|
+
else {
|
|
2131
|
+
this._recordPeerResponseHealth(selectedPeer.peerId, responseForClient.statusCode, requestForPeer.path);
|
|
2132
|
+
}
|
|
1819
2133
|
if (streamed) {
|
|
1820
2134
|
// Headers already sent to client, can't retry
|
|
1821
|
-
if (responseForClient.statusCode >= 200 && responseForClient.statusCode < 400) {
|
|
1822
|
-
this._rememberSuccessfulPeer(selectedPeer.peerId);
|
|
1823
|
-
}
|
|
1824
2135
|
if (!res.writableEnded) {
|
|
1825
2136
|
res.end();
|
|
1826
2137
|
}
|
|
@@ -1837,9 +2148,6 @@ export class BuyerProxy {
|
|
|
1837
2148
|
errorMessage: null,
|
|
1838
2149
|
};
|
|
1839
2150
|
}
|
|
1840
|
-
if (responseForClient.statusCode >= 200 && responseForClient.statusCode < 400) {
|
|
1841
|
-
this._rememberSuccessfulPeer(selectedPeer.peerId);
|
|
1842
|
-
}
|
|
1843
2151
|
res.writeHead(responseForClient.statusCode, responseHeaders);
|
|
1844
2152
|
res.end(Buffer.from(responseForClient.body));
|
|
1845
2153
|
return { done: true };
|
|
@@ -1849,8 +2157,9 @@ export class BuyerProxy {
|
|
|
1849
2157
|
if (upstreamResponse.statusCode >= 400 && !adaptResponse) {
|
|
1850
2158
|
log(`Upstream raw error detail: ${summarizeErrorResponse(upstreamResponse)}`);
|
|
1851
2159
|
}
|
|
1852
|
-
let response = upstreamResponse;
|
|
1853
|
-
if (adaptResponse
|
|
2160
|
+
let response = adaptBuyerFaultErrorResponse(upstreamResponse, requestProtocol);
|
|
2161
|
+
if (adaptResponse
|
|
2162
|
+
&& response.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER]?.toLowerCase() !== 'buyer') {
|
|
1854
2163
|
response = adaptResponse(response);
|
|
1855
2164
|
}
|
|
1856
2165
|
response = adaptOpenAICompatibleErrorResponse(response, requestProtocol);
|
|
@@ -1867,13 +2176,14 @@ export class BuyerProxy {
|
|
|
1867
2176
|
}
|
|
1868
2177
|
const telemetry = computeResponseTelemetry(requestForPeer, response.headers, response.body, selectedPeer);
|
|
1869
2178
|
const responseHeaders = attachAntseedTelemetryHeaders(response.headers, selectedPeer, telemetry, requestForPeer.requestId, latencyMs);
|
|
2179
|
+
const responseFault = responseFaultAttribution(response);
|
|
1870
2180
|
const modelNotFound = !isControlPlaneServicesPath(requestForPeer.path)
|
|
1871
2181
|
&& isModelNotFoundResponse(response);
|
|
1872
2182
|
if (modelNotFound) {
|
|
1873
2183
|
this._onModelNotFound(selectedPeer.peerId, requestedService);
|
|
1874
2184
|
}
|
|
1875
2185
|
// Report result to router for learning
|
|
1876
|
-
if (router) {
|
|
2186
|
+
if (router && responseFault !== 'buyer') {
|
|
1877
2187
|
router.onResult(selectedPeer, {
|
|
1878
2188
|
success: !modelNotFound
|
|
1879
2189
|
&& isRouterSuccess(response.statusCode, requestForPeer.path, retryableStatusCodes),
|
|
@@ -1881,13 +2191,16 @@ export class BuyerProxy {
|
|
|
1881
2191
|
tokens: telemetry.usage.totalTokens,
|
|
1882
2192
|
});
|
|
1883
2193
|
}
|
|
2194
|
+
if (responseFault === 'buyer') {
|
|
2195
|
+
this._recordPeerFailure(selectedPeer.peerId, 'buyer-local', 'buyer');
|
|
2196
|
+
}
|
|
2197
|
+
else {
|
|
2198
|
+
this._recordPeerResponseHealth(selectedPeer.peerId, response.statusCode, requestForPeer.path);
|
|
2199
|
+
}
|
|
1884
2200
|
// Check if retryable
|
|
1885
2201
|
if (retryableStatusCodes.has(response.statusCode)) {
|
|
1886
2202
|
return { done: false, statusCode: response.statusCode, responseBody: Buffer.from(response.body), responseHeaders, errorMessage: null };
|
|
1887
2203
|
}
|
|
1888
|
-
if (response.statusCode >= 200 && response.statusCode < 400) {
|
|
1889
|
-
this._rememberSuccessfulPeer(selectedPeer.peerId);
|
|
1890
|
-
}
|
|
1891
2204
|
// Forward response headers and body to the HTTP client
|
|
1892
2205
|
res.writeHead(response.statusCode, responseHeaders);
|
|
1893
2206
|
res.end(Buffer.from(response.body));
|
|
@@ -1932,7 +2245,14 @@ export class BuyerProxy {
|
|
|
1932
2245
|
}
|
|
1933
2246
|
return { done: true };
|
|
1934
2247
|
}
|
|
1935
|
-
this
|
|
2248
|
+
// Whose fault was this? Errors raised by our own wallet, deposits,
|
|
2249
|
+
// transport or state machine say nothing about the peer, and telling the
|
|
2250
|
+
// user to blame the seller for their empty deposit sends them chasing the
|
|
2251
|
+
// wrong fix. Anything untagged stays 'unknown', where the attribution
|
|
2252
|
+
// gates decide.
|
|
2253
|
+
const fault = faultAttributionOf(err);
|
|
2254
|
+
const faultCode = faultCodeOf(err);
|
|
2255
|
+
this._recordPeerFailure(selectedPeer.peerId, fault === 'buyer' ? 'buyer-local' : 'request-failed', fault);
|
|
1936
2256
|
if (res.headersSent) {
|
|
1937
2257
|
// Headers already sent (streaming), can't retry
|
|
1938
2258
|
if (!res.writableEnded) {
|
|
@@ -1940,6 +2260,30 @@ export class BuyerProxy {
|
|
|
1940
2260
|
}
|
|
1941
2261
|
return { done: true };
|
|
1942
2262
|
}
|
|
2263
|
+
if (fault === 'buyer') {
|
|
2264
|
+
const buyerResponse = adaptBuyerFaultErrorResponse({
|
|
2265
|
+
requestId: requestForPeer.requestId,
|
|
2266
|
+
statusCode: 503,
|
|
2267
|
+
headers: {
|
|
2268
|
+
'content-type': 'application/json',
|
|
2269
|
+
[ANTSEED_FAULT_ATTRIBUTION_HEADER]: 'buyer',
|
|
2270
|
+
},
|
|
2271
|
+
body: Buffer.from(JSON.stringify({
|
|
2272
|
+
error: {
|
|
2273
|
+
type: 'buyer_request_failed',
|
|
2274
|
+
code: faultCode ?? 'buyer_request_failed',
|
|
2275
|
+
message,
|
|
2276
|
+
},
|
|
2277
|
+
})),
|
|
2278
|
+
}, requestProtocol);
|
|
2279
|
+
return {
|
|
2280
|
+
done: false,
|
|
2281
|
+
statusCode: buyerResponse.statusCode,
|
|
2282
|
+
responseBody: Buffer.from(buyerResponse.body),
|
|
2283
|
+
responseHeaders: buyerResponse.headers,
|
|
2284
|
+
errorMessage: message,
|
|
2285
|
+
};
|
|
2286
|
+
}
|
|
1943
2287
|
return { done: false, statusCode: 502, responseBody: Buffer.from(`P2P request failed: ${message}`), responseHeaders: { 'content-type': 'text/plain' }, errorMessage: message };
|
|
1944
2288
|
}
|
|
1945
2289
|
}
|