@antseed/cli 0.1.143 → 0.1.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/cli/commands/config/index.d.ts.map +1 -1
- package/dist/cli/commands/config/index.js +9 -0
- package/dist/cli/commands/config/index.js.map +1 -1
- package/dist/cli/commands/seller/capability-prompts.d.ts +17 -0
- package/dist/cli/commands/seller/capability-prompts.d.ts.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.js +123 -0
- package/dist/cli/commands/seller/capability-prompts.js.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.test.d.ts +2 -0
- package/dist/cli/commands/seller/capability-prompts.test.d.ts.map +1 -0
- package/dist/cli/commands/seller/capability-prompts.test.js +99 -0
- package/dist/cli/commands/seller/capability-prompts.test.js.map +1 -0
- package/dist/cli/commands/seller/setup.d.ts.map +1 -1
- package/dist/cli/commands/seller/setup.js +12 -0
- package/dist/cli/commands/seller/setup.js.map +1 -1
- package/dist/cli/commands/seller/start.d.ts +3 -2
- package/dist/cli/commands/seller/start.d.ts.map +1 -1
- package/dist/cli/commands/seller/start.js +39 -3
- package/dist/cli/commands/seller/start.js.map +1 -1
- package/dist/cli/commands/seller/start.test.js +46 -1
- package/dist/cli/commands/seller/start.test.js.map +1 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +6 -0
- package/dist/config/loader.js.map +1 -1
- package/dist/config/loader.test.js +33 -0
- package/dist/config/loader.test.js.map +1 -1
- package/dist/config/service-metadata.d.ts +5 -0
- package/dist/config/service-metadata.d.ts.map +1 -0
- package/dist/config/service-metadata.js +47 -0
- package/dist/config/service-metadata.js.map +1 -0
- package/dist/config/service-metadata.test.d.ts +2 -0
- package/dist/config/service-metadata.test.d.ts.map +1 -0
- package/dist/config/service-metadata.test.js +18 -0
- package/dist/config/service-metadata.test.js.map +1 -0
- package/dist/config/types.d.ts +5 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js +2 -0
- package/dist/config/validation.js.map +1 -1
- package/dist/proxy/buyer-proxy.d.ts +76 -9
- package/dist/proxy/buyer-proxy.d.ts.map +1 -1
- package/dist/proxy/buyer-proxy.js +508 -68
- package/dist/proxy/buyer-proxy.js.map +1 -1
- package/dist/proxy/buyer-proxy.test.js +416 -13
- package/dist/proxy/buyer-proxy.test.js.map +1 -1
- package/dist/proxy/conversation-store.d.ts +6 -1
- package/dist/proxy/conversation-store.d.ts.map +1 -1
- package/dist/proxy/conversation-store.js +9 -2
- package/dist/proxy/conversation-store.js.map +1 -1
- package/dist/proxy/conversation-store.test.js +24 -0
- package/dist/proxy/conversation-store.test.js.map +1 -1
- package/dist/proxy/peer-attribution.d.ts +100 -0
- package/dist/proxy/peer-attribution.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.js +175 -0
- package/dist/proxy/peer-attribution.js.map +1 -0
- package/dist/proxy/peer-attribution.test.d.ts +2 -0
- package/dist/proxy/peer-attribution.test.d.ts.map +1 -0
- package/dist/proxy/peer-attribution.test.js +140 -0
- package/dist/proxy/peer-attribution.test.js.map +1 -0
- package/dist/proxy/peer-health.d.ts +73 -0
- package/dist/proxy/peer-health.d.ts.map +1 -0
- package/dist/proxy/peer-health.js +182 -0
- package/dist/proxy/peer-health.js.map +1 -0
- package/dist/proxy/peer-health.test.d.ts +2 -0
- package/dist/proxy/peer-health.test.d.ts.map +1 -0
- package/dist/proxy/peer-health.test.js +197 -0
- package/dist/proxy/peer-health.test.js.map +1 -0
- package/dist/proxy/routing-parameters.test.d.ts +2 -0
- package/dist/proxy/routing-parameters.test.d.ts.map +1 -0
- package/dist/proxy/routing-parameters.test.js +97 -0
- package/dist/proxy/routing-parameters.test.js.map +1 -0
- package/dist/proxy/routing.d.ts +8 -0
- package/dist/proxy/routing.d.ts.map +1 -1
- package/dist/proxy/routing.js +36 -1
- package/dist/proxy/routing.js.map +1 -1
- package/dist/proxy/service-api-adapter.d.ts +1 -1
- package/dist/proxy/service-api-adapter.d.ts.map +1 -1
- package/dist/proxy/service-api-adapter.js +1 -1
- package/dist/proxy/service-api-adapter.js.map +1 -1
- package/package.json +6 -6
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import assert from 'node:assert/strict';
|
|
2
|
-
import {
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { mkdir, mkdtemp, readdir, readFile, rm, utimes, writeFile } from 'node:fs/promises';
|
|
3
4
|
import { tmpdir } from 'node:os';
|
|
4
5
|
import { join } from 'node:path';
|
|
5
6
|
import { Readable } from 'node:stream';
|
|
6
7
|
import test from 'node:test';
|
|
7
|
-
import { CONNECTION_CAPABILITY_RELAYS_SWEEPS_V1 } from '@antseed/node';
|
|
8
|
+
import { ANTSEED_BUYER_FAULT_ERROR_CODE, ANTSEED_FAULT_ATTRIBUTION_HEADER, CONNECTION_CAPABILITY_COOPERATIVE_CLOSE_V1, CONNECTION_CAPABILITY_RELAYS_SWEEPS_V1, buyerFault, } from '@antseed/node';
|
|
8
9
|
import { DEFAULT_BUYER_PEER_REFRESH_INTERVAL_MS } from '../config/defaults.js';
|
|
9
|
-
import { BuyerProxy, isModelNotFoundResponse, makeVerifierReach, parsePeerPinnedService, parsePersistedPeers, rewritePeerPinnedServiceInBody, selectCandidatePeersForRouting, substituteRoutedModelAlias, } from './buyer-proxy.js';
|
|
10
|
+
import { BuyerProxy, isModelNotFoundResponse, makeVerifierReach, mergeJsonStateFile, parsePeerPinnedService, parsePersistedPeers, rewritePeerPinnedServiceInBody, selectCandidatePeersForRouting, substituteRoutedModelAlias, sweepStaleStateTmpFiles, } from './buyer-proxy.js';
|
|
10
11
|
import { overrideRoutedModelInBody, SYSTEM_ROUTED_MODEL_HEADER } from './request-utils.js';
|
|
11
12
|
function makePeer(seed, providers) {
|
|
12
13
|
const repeated = (seed.repeat(40) + 'a'.repeat(40)).slice(0, 40);
|
|
@@ -57,18 +58,48 @@ function makeProxyResponse() {
|
|
|
57
58
|
},
|
|
58
59
|
};
|
|
59
60
|
}
|
|
60
|
-
function makeBuyerProxyWithPeers(initialPeers, refreshedPeers = initialPeers, router = null) {
|
|
61
|
+
function makeBuyerProxyWithPeers(initialPeers, refreshedPeers = initialPeers, router = null, now) {
|
|
61
62
|
const proxy = new BuyerProxy({
|
|
62
63
|
port: 0,
|
|
63
64
|
dataDir: '/tmp/antseed-test',
|
|
64
65
|
node: {
|
|
65
66
|
router,
|
|
66
67
|
},
|
|
68
|
+
...(now ? { now } : {}),
|
|
67
69
|
});
|
|
68
70
|
proxy._getPeers = async (options) => options?.forceRefresh ? refreshedPeers : initialPeers;
|
|
69
71
|
proxy._cacheLastUpdatedAtMs = Date.now();
|
|
70
72
|
return proxy;
|
|
71
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* A hand-cranked clock for peer-health tests.
|
|
76
|
+
*
|
|
77
|
+
* Health bookkeeping coalesces failures landing within a second of each other,
|
|
78
|
+
* so back-to-back in-process requests would otherwise register as one episode.
|
|
79
|
+
* Real failures are spaced by connect/request timeouts; these tests advance the
|
|
80
|
+
* clock explicitly to model that without sleeping.
|
|
81
|
+
*/
|
|
82
|
+
function makeTestClock(start = 1_700_000_000_000) {
|
|
83
|
+
let current = start;
|
|
84
|
+
return {
|
|
85
|
+
now: () => current,
|
|
86
|
+
advance(ms) { current += ms; },
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
/** Router stub that permits every peer and ignores result telemetry. */
|
|
90
|
+
function permissiveRouter() {
|
|
91
|
+
return { allowsPeerForPolicy: () => true, onResult: () => { } };
|
|
92
|
+
}
|
|
93
|
+
/** Drive `times` failed requests at a pinned peer, spaced past the coalesce window. */
|
|
94
|
+
async function failRepeatedly(proxy, peer, clock, times = 3) {
|
|
95
|
+
for (let i = 0; i < times; i += 1) {
|
|
96
|
+
await invokeProxy(proxy, makeProxyRequest({ headers: { 'x-antseed-pin-peer': peer.peerId } }));
|
|
97
|
+
clock.advance(5_000);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function healthOf(proxy, peer) {
|
|
101
|
+
return proxy._peerHealth.get(peer.peerId);
|
|
102
|
+
}
|
|
72
103
|
async function invokeProxy(proxy, req) {
|
|
73
104
|
const res = makeProxyResponse();
|
|
74
105
|
await proxy._handleRequest(req, res);
|
|
@@ -212,6 +243,25 @@ test('peers control endpoint exposes relay capability metadata', async () => {
|
|
|
212
243
|
assert.equal(res.statusCode, 200);
|
|
213
244
|
assert.deepEqual(body.peers[0]?.capabilities, [CONNECTION_CAPABILITY_RELAYS_SWEEPS_V1]);
|
|
214
245
|
});
|
|
246
|
+
test('channels endpoint exposes cooperative-close support from peer capabilities', async () => {
|
|
247
|
+
const supportedPeer = makePeer('a', ['openai']);
|
|
248
|
+
supportedPeer.capabilities = [CONNECTION_CAPABILITY_COOPERATIVE_CLOSE_V1];
|
|
249
|
+
const unsupportedPeer = makePeer('b', ['openai']);
|
|
250
|
+
const proxy = makeBuyerProxyWithPeers([supportedPeer, unsupportedPeer]);
|
|
251
|
+
proxy._node.getAllBuyerChannels = () => [
|
|
252
|
+
{ sessionId: 'supported', peerId: supportedPeer.peerId },
|
|
253
|
+
{ sessionId: 'unsupported', peerId: unsupportedPeer.peerId },
|
|
254
|
+
{ sessionId: 'unknown', peerId: 'c'.repeat(40) },
|
|
255
|
+
];
|
|
256
|
+
const res = await invokeProxy(proxy, makeProxyRequest({ method: 'GET', path: '/_antseed/channels?all=1' }));
|
|
257
|
+
const body = JSON.parse(res.body);
|
|
258
|
+
assert.equal(res.statusCode, 200);
|
|
259
|
+
assert.deepEqual(body.channels.map((channel) => [channel.sessionId, channel.cooperativeCloseSupported]), [
|
|
260
|
+
['supported', true],
|
|
261
|
+
['unsupported', false],
|
|
262
|
+
['unknown', false],
|
|
263
|
+
]);
|
|
264
|
+
});
|
|
215
265
|
test('selectCandidatePeersForRouting excludes peers when requested service is not in provider metadata', () => {
|
|
216
266
|
const openAiPeer = makePeer('a', ['openai']);
|
|
217
267
|
openAiPeer.providerServiceApiProtocols = {
|
|
@@ -355,7 +405,7 @@ test('pinned proxy request enforces buyer routing policy', async () => {
|
|
|
355
405
|
assert.match(res.body, /outside your buyer routing policy/);
|
|
356
406
|
assert.match(res.body, /pricing\/reputation limits/);
|
|
357
407
|
});
|
|
358
|
-
test('
|
|
408
|
+
test('a buyer-attributed failure returns 503 and never blames the peer', async () => {
|
|
359
409
|
const peer = makePeer('a', ['openai']);
|
|
360
410
|
const routerResults = [];
|
|
361
411
|
const router = {
|
|
@@ -367,21 +417,81 @@ test('local buyer payment failures only update diagnostic failure state', async
|
|
|
367
417
|
const proxy = makeBuyerProxyWithPeers([peer], [peer], router);
|
|
368
418
|
proxy._cachedPeers = [peer];
|
|
369
419
|
proxy._node.sendRequest = async () => {
|
|
370
|
-
throw
|
|
420
|
+
throw buyerFault('Insufficient buyer deposits for reserve top-up: available=0 required=1000', 'buyer-deposits-insufficient');
|
|
371
421
|
};
|
|
372
422
|
const res = await invokeProxy(proxy, makeProxyRequest({
|
|
373
423
|
headers: {
|
|
374
424
|
'x-antseed-pin-peer': peer.peerId,
|
|
375
425
|
},
|
|
376
426
|
}));
|
|
377
|
-
|
|
378
|
-
|
|
427
|
+
// 503, not 502: our empty deposit is not the seller's fault, and the user
|
|
428
|
+
// should be pointed at the deposit rather than at a different peer.
|
|
429
|
+
assert.equal(res.statusCode, 503);
|
|
430
|
+
assert.equal(JSON.parse(res.body).error.code, ANTSEED_BUYER_FAULT_ERROR_CODE);
|
|
431
|
+
assert.equal(JSON.parse(res.body).error.param, 'buyer-deposits-insufficient');
|
|
432
|
+
assert.equal(res.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER], 'buyer');
|
|
379
433
|
assert.equal(routerResults.length, 0);
|
|
380
|
-
|
|
381
|
-
assert.equal(
|
|
434
|
+
const health = proxy._peerHealth.get(peer.peerId);
|
|
435
|
+
assert.equal(health?.lastReason, 'buyer-local');
|
|
436
|
+
assert.equal(health?.failureStreak, 0, 'a buyer fault must never build a peer streak');
|
|
437
|
+
assert.equal(health?.cooldownUntil, 0);
|
|
382
438
|
assert.equal(proxy._cachedPeers[0]?.peerId, peer.peerId);
|
|
383
439
|
});
|
|
384
|
-
test('
|
|
440
|
+
test('a buyer-authored 503 does not affect router metrics or peer health', async () => {
|
|
441
|
+
const peer = makePeer('a', ['openai']);
|
|
442
|
+
const routerResults = [];
|
|
443
|
+
const router = {
|
|
444
|
+
allowsPeerForPolicy: () => true,
|
|
445
|
+
onResult: (_peer, result) => {
|
|
446
|
+
routerResults.push(result);
|
|
447
|
+
},
|
|
448
|
+
};
|
|
449
|
+
const proxy = makeBuyerProxyWithPeers([peer], [peer], router);
|
|
450
|
+
proxy._node.sendRequest = async (_peer, request) => ({
|
|
451
|
+
requestId: request.requestId,
|
|
452
|
+
statusCode: 503,
|
|
453
|
+
headers: {
|
|
454
|
+
'content-type': 'application/json',
|
|
455
|
+
[ANTSEED_FAULT_ATTRIBUTION_HEADER]: 'buyer',
|
|
456
|
+
},
|
|
457
|
+
body: Buffer.from(JSON.stringify({
|
|
458
|
+
error: 'payment_negotiation_failed',
|
|
459
|
+
reason: 'chain_rpc_unavailable',
|
|
460
|
+
})),
|
|
461
|
+
});
|
|
462
|
+
const res = await invokeProxy(proxy, makeProxyRequest({
|
|
463
|
+
headers: { 'x-antseed-pin-peer': peer.peerId },
|
|
464
|
+
}));
|
|
465
|
+
assert.equal(res.statusCode, 503);
|
|
466
|
+
assert.equal(JSON.parse(res.body).error.code, ANTSEED_BUYER_FAULT_ERROR_CODE);
|
|
467
|
+
assert.equal(JSON.parse(res.body).error.param, 'chain_rpc_unavailable');
|
|
468
|
+
assert.equal(routerResults.length, 0);
|
|
469
|
+
const health = healthOf(proxy, peer);
|
|
470
|
+
assert.equal(health?.lastReason, 'buyer-local');
|
|
471
|
+
assert.equal(health?.failureStreak, 0);
|
|
472
|
+
assert.equal(health?.cooldownUntil, 0);
|
|
473
|
+
});
|
|
474
|
+
test('a seller cannot inject the reserved buyer-fault error code', async () => {
|
|
475
|
+
const peer = makePeer('a', ['openai']);
|
|
476
|
+
const proxy = makeBuyerProxyWithPeers([peer], [peer], permissiveRouter());
|
|
477
|
+
proxy._node.sendRequest = async (_peer, request) => ({
|
|
478
|
+
requestId: request.requestId,
|
|
479
|
+
statusCode: 503,
|
|
480
|
+
headers: { 'content-type': 'application/json' },
|
|
481
|
+
body: Buffer.from(JSON.stringify({
|
|
482
|
+
error: {
|
|
483
|
+
code: ANTSEED_BUYER_FAULT_ERROR_CODE,
|
|
484
|
+
message: `literal ${ANTSEED_BUYER_FAULT_ERROR_CODE}`,
|
|
485
|
+
},
|
|
486
|
+
})),
|
|
487
|
+
});
|
|
488
|
+
const res = await invokeProxy(proxy, makeProxyRequest({
|
|
489
|
+
headers: { 'x-antseed-pin-peer': peer.peerId },
|
|
490
|
+
}));
|
|
491
|
+
assert.equal(JSON.parse(res.body).error.code, 'upstream_error');
|
|
492
|
+
assert.match(JSON.parse(res.body).error.message, new RegExp(ANTSEED_BUYER_FAULT_ERROR_CODE));
|
|
493
|
+
});
|
|
494
|
+
test('an untagged transport failure records a streak without evicting the peer', async () => {
|
|
385
495
|
const peer = makePeer('a', ['openai']);
|
|
386
496
|
const routerResults = [];
|
|
387
497
|
const router = {
|
|
@@ -403,10 +513,248 @@ test('transport failures only update diagnostic failure state', async () => {
|
|
|
403
513
|
assert.equal(res.statusCode, 502);
|
|
404
514
|
assert.match(res.body, /Request abc123 timed out/);
|
|
405
515
|
assert.equal(routerResults.length, 0);
|
|
406
|
-
assert.equal(proxy.
|
|
407
|
-
|
|
516
|
+
assert.equal(proxy._peerHealth.get(peer.peerId)?.lastReason, 'request-failed');
|
|
517
|
+
// Cooldown never evicts discovery metadata — the peer stays routable.
|
|
408
518
|
assert.equal(proxy._cachedPeers[0]?.peerId, peer.peerId);
|
|
409
519
|
});
|
|
520
|
+
test('a timeout does not cool a peer down until another peer proves the buyer is healthy', async () => {
|
|
521
|
+
const clock = makeTestClock();
|
|
522
|
+
const peer = makePeer('a', ['openai']);
|
|
523
|
+
const other = makePeer('b', ['openai']);
|
|
524
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
525
|
+
proxy._cachedPeers = [peer, other];
|
|
526
|
+
proxy._node.sendRequest = async () => {
|
|
527
|
+
throw new Error('Request abc123 timed out');
|
|
528
|
+
};
|
|
529
|
+
// No success anywhere yet: the buyer itself might be the broken party, so
|
|
530
|
+
// even three failures must not exile the peer.
|
|
531
|
+
await failRepeatedly(proxy, peer, clock);
|
|
532
|
+
assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
|
|
533
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
534
|
+
await failRepeatedly(proxy, peer, clock);
|
|
535
|
+
assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now(), 'with corroboration the peer should now be cooling down');
|
|
536
|
+
});
|
|
537
|
+
test('a cooling-down peer is still dispatched to when a request names it', async () => {
|
|
538
|
+
const clock = makeTestClock();
|
|
539
|
+
const peer = makePeer('a', ['openai']);
|
|
540
|
+
const other = makePeer('b', ['openai']);
|
|
541
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
542
|
+
proxy._cachedPeers = [peer, other];
|
|
543
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
544
|
+
proxy._node.sendRequest = async () => {
|
|
545
|
+
throw new Error('Request abc123 timed out');
|
|
546
|
+
};
|
|
547
|
+
await failRepeatedly(proxy, peer, clock);
|
|
548
|
+
assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
|
|
549
|
+
let dispatched = false;
|
|
550
|
+
proxy._node.sendRequest = async (_peer, request) => {
|
|
551
|
+
dispatched = true;
|
|
552
|
+
return {
|
|
553
|
+
requestId: request.requestId,
|
|
554
|
+
statusCode: 200,
|
|
555
|
+
headers: { 'content-type': 'application/json' },
|
|
556
|
+
body: Buffer.from('{}'),
|
|
557
|
+
};
|
|
558
|
+
};
|
|
559
|
+
const res = await invokeProxy(proxy, makeProxyRequest({ headers: { 'x-antseed-pin-peer': peer.peerId } }));
|
|
560
|
+
assert.equal(dispatched, true, 'cooldown is advisory; a named peer must still be tried');
|
|
561
|
+
assert.equal(res.statusCode, 200);
|
|
562
|
+
// And the response clears the cooldown, because the peer plainly answered.
|
|
563
|
+
assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
|
|
564
|
+
});
|
|
565
|
+
test('a 429 records capacity pressure without ever cooling the peer down', async () => {
|
|
566
|
+
const clock = makeTestClock();
|
|
567
|
+
const peer = makePeer('a', ['openai']);
|
|
568
|
+
const other = makePeer('b', ['openai']);
|
|
569
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
570
|
+
proxy._cachedPeers = [peer, other];
|
|
571
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
572
|
+
proxy._node.sendRequest = async (_peer, request) => ({
|
|
573
|
+
requestId: request.requestId,
|
|
574
|
+
statusCode: 429,
|
|
575
|
+
headers: { 'content-type': 'text/plain' },
|
|
576
|
+
body: Buffer.from('slow down'),
|
|
577
|
+
});
|
|
578
|
+
await failRepeatedly(proxy, peer, clock, 5);
|
|
579
|
+
const health = healthOf(proxy, peer);
|
|
580
|
+
assert.equal(health?.lastReason, 'seller-busy');
|
|
581
|
+
assert.equal(health?.failureStreak, 0);
|
|
582
|
+
assert.equal(health?.cooldownUntil, 0);
|
|
583
|
+
});
|
|
584
|
+
test('a seller 503 escalates once the buyer is corroborated as healthy', async () => {
|
|
585
|
+
const clock = makeTestClock();
|
|
586
|
+
const peer = makePeer('a', ['openai']);
|
|
587
|
+
const other = makePeer('b', ['openai']);
|
|
588
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
589
|
+
proxy._cachedPeers = [peer, other];
|
|
590
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
591
|
+
proxy._node.sendRequest = async (_peer, request) => ({
|
|
592
|
+
requestId: request.requestId,
|
|
593
|
+
statusCode: 503,
|
|
594
|
+
headers: { 'content-type': 'text/plain' },
|
|
595
|
+
body: Buffer.from('seller down'),
|
|
596
|
+
});
|
|
597
|
+
await failRepeatedly(proxy, peer, clock);
|
|
598
|
+
const health = healthOf(proxy, peer);
|
|
599
|
+
assert.equal(health?.lastReason, 'seller-5xx');
|
|
600
|
+
assert.ok(health?.cooldownUntil > clock.now());
|
|
601
|
+
});
|
|
602
|
+
test('non-standard seller 5xx responses also build a cooldown streak', async () => {
|
|
603
|
+
const clock = makeTestClock();
|
|
604
|
+
const peer = makePeer('a', ['openai']);
|
|
605
|
+
const other = makePeer('b', ['openai']);
|
|
606
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
607
|
+
proxy._cachedPeers = [peer, other];
|
|
608
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
609
|
+
proxy._node.sendRequest = async (_peer, request) => ({
|
|
610
|
+
requestId: request.requestId,
|
|
611
|
+
statusCode: 522,
|
|
612
|
+
headers: { 'content-type': 'text/plain' },
|
|
613
|
+
body: Buffer.from('connection timed out'),
|
|
614
|
+
});
|
|
615
|
+
await failRepeatedly(proxy, peer, clock);
|
|
616
|
+
const health = healthOf(proxy, peer);
|
|
617
|
+
assert.equal(health?.lastReason, 'seller-5xx');
|
|
618
|
+
assert.ok(health?.cooldownUntil > clock.now());
|
|
619
|
+
});
|
|
620
|
+
test('a clean 4xx counts as proof of life and clears a cooldown', async () => {
|
|
621
|
+
const clock = makeTestClock();
|
|
622
|
+
const peer = makePeer('a', ['openai']);
|
|
623
|
+
const other = makePeer('b', ['openai']);
|
|
624
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
625
|
+
proxy._cachedPeers = [peer, other];
|
|
626
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
627
|
+
proxy._node.sendRequest = async () => {
|
|
628
|
+
throw new Error('Request abc123 timed out');
|
|
629
|
+
};
|
|
630
|
+
await failRepeatedly(proxy, peer, clock);
|
|
631
|
+
assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
|
|
632
|
+
proxy._node.sendRequest = async (_peer, request) => ({
|
|
633
|
+
requestId: request.requestId,
|
|
634
|
+
statusCode: 400,
|
|
635
|
+
headers: { 'content-type': 'text/plain' },
|
|
636
|
+
body: Buffer.from('bad request'),
|
|
637
|
+
});
|
|
638
|
+
await invokeProxy(proxy, makeProxyRequest({ headers: { 'x-antseed-pin-peer': peer.peerId } }));
|
|
639
|
+
const health = healthOf(proxy, peer);
|
|
640
|
+
assert.equal(health?.cooldownUntil, 0);
|
|
641
|
+
assert.equal(health?.failureStreak, 0);
|
|
642
|
+
});
|
|
643
|
+
test('a non-standard seller 5xx proves reachability and restarts the failure streak', async () => {
|
|
644
|
+
const clock = makeTestClock();
|
|
645
|
+
const peer = makePeer('a', ['openai']);
|
|
646
|
+
const other = makePeer('b', ['openai']);
|
|
647
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
648
|
+
proxy._cachedPeers = [peer, other];
|
|
649
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
650
|
+
proxy._node.sendRequest = async () => { throw new Error('Request timed out'); };
|
|
651
|
+
await failRepeatedly(proxy, peer, clock);
|
|
652
|
+
assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
|
|
653
|
+
proxy._node.sendRequest = async (_peer, request) => ({
|
|
654
|
+
requestId: request.requestId,
|
|
655
|
+
statusCode: 507,
|
|
656
|
+
headers: { 'content-type': 'text/plain' },
|
|
657
|
+
body: Buffer.from('insufficient storage'),
|
|
658
|
+
});
|
|
659
|
+
await invokeProxy(proxy, makeProxyRequest({ headers: { 'x-antseed-pin-peer': peer.peerId } }));
|
|
660
|
+
assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
|
|
661
|
+
assert.equal(healthOf(proxy, peer)?.failureStreak, 1);
|
|
662
|
+
});
|
|
663
|
+
test('a successful control-plane response clears a stale cooldown', async () => {
|
|
664
|
+
const clock = makeTestClock();
|
|
665
|
+
const peer = makePeer('a', ['openai']);
|
|
666
|
+
const other = makePeer('b', ['openai']);
|
|
667
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
668
|
+
proxy._cachedPeers = [peer, other];
|
|
669
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
670
|
+
proxy._node.sendRequest = async () => { throw new Error('Request timed out'); };
|
|
671
|
+
await failRepeatedly(proxy, peer, clock);
|
|
672
|
+
assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
|
|
673
|
+
proxy._node.sendRequest = async (_peer, request) => ({
|
|
674
|
+
requestId: request.requestId,
|
|
675
|
+
statusCode: 200,
|
|
676
|
+
headers: { 'content-type': 'application/json' },
|
|
677
|
+
body: Buffer.from('{"data":[]}'),
|
|
678
|
+
});
|
|
679
|
+
await invokeProxy(proxy, makeProxyRequest({
|
|
680
|
+
method: 'GET',
|
|
681
|
+
path: '/v1/models',
|
|
682
|
+
headers: { 'x-antseed-pin-peer': peer.peerId },
|
|
683
|
+
}));
|
|
684
|
+
assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
|
|
685
|
+
assert.equal(healthOf(proxy, peer)?.failureStreak, 0);
|
|
686
|
+
});
|
|
687
|
+
test('a buyer-side outage rolls back the cooldowns it caused', async () => {
|
|
688
|
+
const clock = makeTestClock();
|
|
689
|
+
const peers = ['a', 'b', 'c', 'd'].map((seed) => makePeer(seed, ['openai']));
|
|
690
|
+
const proxy = makeBuyerProxyWithPeers(peers, peers, permissiveRouter(), clock.now);
|
|
691
|
+
proxy._cachedPeers = peers;
|
|
692
|
+
proxy._rememberSuccessfulPeer(peers[3].peerId);
|
|
693
|
+
proxy._node.sendRequest = async () => {
|
|
694
|
+
throw new Error('Connection to peer failed');
|
|
695
|
+
};
|
|
696
|
+
// Every peer failing at once is the signature of a dropped network, not of
|
|
697
|
+
// three sellers dying simultaneously.
|
|
698
|
+
for (const peer of peers.slice(0, 3)) {
|
|
699
|
+
await failRepeatedly(proxy, peer, clock);
|
|
700
|
+
}
|
|
701
|
+
for (const peer of peers.slice(0, 3)) {
|
|
702
|
+
const health = healthOf(proxy, peer);
|
|
703
|
+
assert.equal(health?.cooldownUntil, 0, `${peer.peerId.slice(0, 4)} should not be cooling down`);
|
|
704
|
+
assert.equal(health?.failureStreak, 0, `${peer.peerId.slice(0, 4)} streak should be rolled back`);
|
|
705
|
+
}
|
|
706
|
+
});
|
|
707
|
+
test('GET /_antseed/peer-health reports cooldowns and buyer health', async () => {
|
|
708
|
+
const clock = makeTestClock();
|
|
709
|
+
const peer = makePeer('a', ['openai']);
|
|
710
|
+
const other = makePeer('b', ['openai']);
|
|
711
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
712
|
+
proxy._cachedPeers = [peer, other];
|
|
713
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
714
|
+
proxy._node.sendRequest = async () => {
|
|
715
|
+
throw new Error('Request abc123 timed out');
|
|
716
|
+
};
|
|
717
|
+
await failRepeatedly(proxy, peer, clock);
|
|
718
|
+
const res = await invokeProxy(proxy, makeProxyRequest({ method: 'GET', path: '/_antseed/peer-health' }));
|
|
719
|
+
assert.equal(res.statusCode, 200);
|
|
720
|
+
const body = JSON.parse(res.body);
|
|
721
|
+
assert.equal(body.ok, true);
|
|
722
|
+
assert.equal(body.buyerHealthy, true);
|
|
723
|
+
const entry = body.peers.find((p) => p.peerId === peer.peerId);
|
|
724
|
+
assert.equal(entry.coolingDown, true);
|
|
725
|
+
assert.ok(entry.cooldownMsRemaining > 0);
|
|
726
|
+
assert.equal(entry.lastReason, 'request-failed');
|
|
727
|
+
});
|
|
728
|
+
test('POST /_antseed/peer-health/clear gives a peer another chance', async () => {
|
|
729
|
+
const clock = makeTestClock();
|
|
730
|
+
const peer = makePeer('a', ['openai']);
|
|
731
|
+
const other = makePeer('b', ['openai']);
|
|
732
|
+
const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
|
|
733
|
+
proxy._cachedPeers = [peer, other];
|
|
734
|
+
proxy._rememberSuccessfulPeer(other.peerId);
|
|
735
|
+
proxy._node.sendRequest = async () => {
|
|
736
|
+
throw new Error('Request abc123 timed out');
|
|
737
|
+
};
|
|
738
|
+
await failRepeatedly(proxy, peer, clock);
|
|
739
|
+
assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
|
|
740
|
+
const res = await invokeProxy(proxy, makeProxyRequest({
|
|
741
|
+
method: 'POST',
|
|
742
|
+
path: '/_antseed/peer-health/clear',
|
|
743
|
+
body: { peerId: peer.peerId },
|
|
744
|
+
}));
|
|
745
|
+
assert.equal(res.statusCode, 200);
|
|
746
|
+
assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
|
|
747
|
+
});
|
|
748
|
+
test('POST /_antseed/peer-health/clear rejects a malformed peer id', async () => {
|
|
749
|
+
const peer = makePeer('a', ['openai']);
|
|
750
|
+
const proxy = makeBuyerProxyWithPeers([peer], [peer], { allowsPeerForPolicy: () => true });
|
|
751
|
+
const res = await invokeProxy(proxy, makeProxyRequest({
|
|
752
|
+
method: 'POST',
|
|
753
|
+
path: '/_antseed/peer-health/clear',
|
|
754
|
+
body: { peerId: 'nope' },
|
|
755
|
+
}));
|
|
756
|
+
assert.equal(res.statusCode, 400);
|
|
757
|
+
});
|
|
410
758
|
test('/v1/models retryable response reports router success', async () => {
|
|
411
759
|
const peer = makePeer('a', ['openai']);
|
|
412
760
|
const routerResults = [];
|
|
@@ -1315,4 +1663,59 @@ test('isModelNotFoundResponse: detects seller and upstream model_not_found rejec
|
|
|
1315
1663
|
error: { code: 'model_not_found' },
|
|
1316
1664
|
})), false);
|
|
1317
1665
|
});
|
|
1666
|
+
test('mergeJsonStateFile: merges into existing state and leaves no temp files', async () => {
|
|
1667
|
+
const dir = await mkdtemp(join(tmpdir(), 'antseed-buyer-state-'));
|
|
1668
|
+
try {
|
|
1669
|
+
const stateFile = join(dir, 'buyer.state.json');
|
|
1670
|
+
await mergeJsonStateFile(dir, stateFile, { a: 1, pinnedPeerId: 'old' });
|
|
1671
|
+
await mergeJsonStateFile(dir, stateFile, { pinnedPeerId: 'new', b: 2 });
|
|
1672
|
+
const state = JSON.parse(await readFile(stateFile, 'utf-8'));
|
|
1673
|
+
assert.deepEqual(state, { a: 1, pinnedPeerId: 'new', b: 2 });
|
|
1674
|
+
const tmpLeftovers = (await readdir(dir)).filter((name) => name.endsWith('.json.tmp'));
|
|
1675
|
+
assert.deepEqual(tmpLeftovers, []);
|
|
1676
|
+
}
|
|
1677
|
+
finally {
|
|
1678
|
+
await rm(dir, { recursive: true, force: true });
|
|
1679
|
+
}
|
|
1680
|
+
});
|
|
1681
|
+
test('mergeJsonStateFile: unlinks the temp file when the rename fails', async () => {
|
|
1682
|
+
const dir = await mkdtemp(join(tmpdir(), 'antseed-buyer-state-'));
|
|
1683
|
+
try {
|
|
1684
|
+
// A directory at the state-file path makes rename() fail on every platform
|
|
1685
|
+
// with a non-retryable code, exercising the cleanup path.
|
|
1686
|
+
const stateFile = join(dir, 'buyer.state.json');
|
|
1687
|
+
await mkdir(stateFile);
|
|
1688
|
+
await assert.rejects(mergeJsonStateFile(dir, stateFile, { a: 1 }));
|
|
1689
|
+
const tmpLeftovers = (await readdir(dir)).filter((name) => name.endsWith('.json.tmp'));
|
|
1690
|
+
assert.deepEqual(tmpLeftovers, []);
|
|
1691
|
+
}
|
|
1692
|
+
finally {
|
|
1693
|
+
await rm(dir, { recursive: true, force: true });
|
|
1694
|
+
}
|
|
1695
|
+
});
|
|
1696
|
+
test('sweepStaleStateTmpFiles: removes aged temp files, keeps fresh and unrelated ones', async () => {
|
|
1697
|
+
const dir = await mkdtemp(join(tmpdir(), 'antseed-buyer-state-'));
|
|
1698
|
+
try {
|
|
1699
|
+
const stale = join(dir, '.buyer.state.11111111-1111-1111-1111-111111111111.json.tmp');
|
|
1700
|
+
const fresh = join(dir, '.buyer.state.22222222-2222-2222-2222-222222222222.json.tmp');
|
|
1701
|
+
const unrelated = join(dir, 'buyer.state.json');
|
|
1702
|
+
await writeFile(stale, '{}');
|
|
1703
|
+
await writeFile(fresh, '{}');
|
|
1704
|
+
await writeFile(unrelated, '{}');
|
|
1705
|
+
const past = new Date(Date.now() - 5 * 60_000);
|
|
1706
|
+
await utimes(stale, past, past);
|
|
1707
|
+
await sweepStaleStateTmpFiles(dir);
|
|
1708
|
+
const remaining = (await readdir(dir)).sort();
|
|
1709
|
+
assert.deepEqual(remaining, [
|
|
1710
|
+
'.buyer.state.22222222-2222-2222-2222-222222222222.json.tmp',
|
|
1711
|
+
'buyer.state.json',
|
|
1712
|
+
]);
|
|
1713
|
+
}
|
|
1714
|
+
finally {
|
|
1715
|
+
await rm(dir, { recursive: true, force: true });
|
|
1716
|
+
}
|
|
1717
|
+
});
|
|
1718
|
+
test('sweepStaleStateTmpFiles: tolerates a missing directory', async () => {
|
|
1719
|
+
await sweepStaleStateTmpFiles(join(tmpdir(), 'antseed-does-not-exist', randomUUID()));
|
|
1720
|
+
});
|
|
1318
1721
|
//# sourceMappingURL=buyer-proxy.test.js.map
|