@antseed/cli 0.1.144 → 0.1.146

package/dist/proxy/buyer-proxy.test.js

@@ -5,7 +5,7 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { Readable } from 'node:stream';
 import test from 'node:test';
-import { CONNECTION_CAPABILITY_COOPERATIVE_CLOSE_V1, CONNECTION_CAPABILITY_RELAYS_SWEEPS_V1, } from '@antseed/node';
+import { ANTSEED_BUYER_FAULT_ERROR_CODE, ANTSEED_FAULT_ATTRIBUTION_HEADER, CONNECTION_CAPABILITY_COOPERATIVE_CLOSE_V1, CONNECTION_CAPABILITY_RELAYS_SWEEPS_V1, buyerFault, } from '@antseed/node';
 import { DEFAULT_BUYER_PEER_REFRESH_INTERVAL_MS } from '../config/defaults.js';
 import { BuyerProxy, isModelNotFoundResponse, makeVerifierReach, mergeJsonStateFile, parsePeerPinnedService, parsePersistedPeers, rewritePeerPinnedServiceInBody, selectCandidatePeersForRouting, substituteRoutedModelAlias, sweepStaleStateTmpFiles, } from './buyer-proxy.js';
 import { overrideRoutedModelInBody, SYSTEM_ROUTED_MODEL_HEADER } from './request-utils.js';
@@ -58,18 +58,48 @@ function makeProxyResponse() {
         },
     };
 }
-function makeBuyerProxyWithPeers(initialPeers, refreshedPeers = initialPeers, router = null) {
+function makeBuyerProxyWithPeers(initialPeers, refreshedPeers = initialPeers, router = null, now) {
     const proxy = new BuyerProxy({
         port: 0,
         dataDir: '/tmp/antseed-test',
         node: {
             router,
         },
+        ...(now ? { now } : {}),
     });
     proxy._getPeers = async (options) => options?.forceRefresh ? refreshedPeers : initialPeers;
     proxy._cacheLastUpdatedAtMs = Date.now();
     return proxy;
 }
+/**
+ * A hand-cranked clock for peer-health tests.
+ *
+ * Health bookkeeping coalesces failures landing within a second of each other,
+ * so back-to-back in-process requests would otherwise register as one episode.
+ * Real failures are spaced by connect/request timeouts; these tests advance the
+ * clock explicitly to model that without sleeping.
+ */
+function makeTestClock(start = 1_700_000_000_000) {
+    let current = start;
+    return {
+        now: () => current,
+        advance(ms) { current += ms; },
+    };
+}
+/** Router stub that permits every peer and ignores result telemetry. */
+function permissiveRouter() {
+    return { allowsPeerForPolicy: () => true, onResult: () => { } };
+}
+/** Drive `times` failed requests at a pinned peer, spaced past the coalesce window. */
+async function failRepeatedly(proxy, peer, clock, times = 3) {
+    for (let i = 0; i < times; i += 1) {
+        await invokeProxy(proxy, makeProxyRequest({ headers: { 'x-antseed-pin-peer': peer.peerId } }));
+        clock.advance(5_000);
+    }
+}
+function healthOf(proxy, peer) {
+    return proxy._peerHealth.get(peer.peerId);
+}
 async function invokeProxy(proxy, req) {
     const res = makeProxyResponse();
     await proxy._handleRequest(req, res);
@@ -375,7 +405,7 @@ test('pinned proxy request enforces buyer routing policy', async () => {
     assert.match(res.body, /outside your buyer routing policy/);
     assert.match(res.body, /pricing\/reputation limits/);
 });
-test('local buyer payment failures only update diagnostic failure state', async () => {
+test('a buyer-attributed failure returns 503 and never blames the peer', async () => {
     const peer = makePeer('a', ['openai']);
     const routerResults = [];
     const router = {
@@ -387,21 +417,81 @@ test('local buyer payment failures only update diagnostic failure state', async
     const proxy = makeBuyerProxyWithPeers([peer], [peer], router);
     proxy._cachedPeers = [peer];
     proxy._node.sendRequest = async () => {
-        throw new Error('Insufficient buyer deposits for reserve top-up: available=0 required=1000');
+        throw buyerFault('Insufficient buyer deposits for reserve top-up: available=0 required=1000', 'buyer-deposits-insufficient');
     };
     const res = await invokeProxy(proxy, makeProxyRequest({
         headers: {
             'x-antseed-pin-peer': peer.peerId,
         },
     }));
-    assert.equal(res.statusCode, 502);
-    assert.match(res.body, /Insufficient buyer deposits/);
+    // 503, not 502: our empty deposit is not the seller's fault, and the user
+    // should be pointed at the deposit rather than at a different peer.
+    assert.equal(res.statusCode, 503);
+    assert.equal(JSON.parse(res.body).error.code, ANTSEED_BUYER_FAULT_ERROR_CODE);
+    assert.equal(JSON.parse(res.body).error.param, 'buyer-deposits-insufficient');
+    assert.equal(res.headers[ANTSEED_FAULT_ATTRIBUTION_HEADER], 'buyer');
     assert.equal(routerResults.length, 0);
-    assert.equal(proxy._peerFailures.get(peer.peerId)?.count, 1);
-    assert.equal(proxy._peerFailures.get(peer.peerId)?.lastReason, 'request-failed');
+    const health = proxy._peerHealth.get(peer.peerId);
+    assert.equal(health?.lastReason, 'buyer-local');
+    assert.equal(health?.failureStreak, 0, 'a buyer fault must never build a peer streak');
+    assert.equal(health?.cooldownUntil, 0);
     assert.equal(proxy._cachedPeers[0]?.peerId, peer.peerId);
 });
-test('transport failures only update diagnostic failure state', async () => {
+test('a buyer-authored 503 does not affect router metrics or peer health', async () => {
+    const peer = makePeer('a', ['openai']);
+    const routerResults = [];
+    const router = {
+        allowsPeerForPolicy: () => true,
+        onResult: (_peer, result) => {
+            routerResults.push(result);
+        },
+    };
+    const proxy = makeBuyerProxyWithPeers([peer], [peer], router);
+    proxy._node.sendRequest = async (_peer, request) => ({
+        requestId: request.requestId,
+        statusCode: 503,
+        headers: {
+            'content-type': 'application/json',
+            [ANTSEED_FAULT_ATTRIBUTION_HEADER]: 'buyer',
+        },
+        body: Buffer.from(JSON.stringify({
+            error: 'payment_negotiation_failed',
+            reason: 'chain_rpc_unavailable',
+        })),
+    });
+    const res = await invokeProxy(proxy, makeProxyRequest({
+        headers: { 'x-antseed-pin-peer': peer.peerId },
+    }));
+    assert.equal(res.statusCode, 503);
+    assert.equal(JSON.parse(res.body).error.code, ANTSEED_BUYER_FAULT_ERROR_CODE);
+    assert.equal(JSON.parse(res.body).error.param, 'chain_rpc_unavailable');
+    assert.equal(routerResults.length, 0);
+    const health = healthOf(proxy, peer);
+    assert.equal(health?.lastReason, 'buyer-local');
+    assert.equal(health?.failureStreak, 0);
+    assert.equal(health?.cooldownUntil, 0);
+});
+test('a seller cannot inject the reserved buyer-fault error code', async () => {
+    const peer = makePeer('a', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer], [peer], permissiveRouter());
+    proxy._node.sendRequest = async (_peer, request) => ({
+        requestId: request.requestId,
+        statusCode: 503,
+        headers: { 'content-type': 'application/json' },
+        body: Buffer.from(JSON.stringify({
+            error: {
+                code: ANTSEED_BUYER_FAULT_ERROR_CODE,
+                message: `literal ${ANTSEED_BUYER_FAULT_ERROR_CODE}`,
+            },
+        })),
+    });
+    const res = await invokeProxy(proxy, makeProxyRequest({
+        headers: { 'x-antseed-pin-peer': peer.peerId },
+    }));
+    assert.equal(JSON.parse(res.body).error.code, 'upstream_error');
+    assert.match(JSON.parse(res.body).error.message, new RegExp(ANTSEED_BUYER_FAULT_ERROR_CODE));
+});
+test('an untagged transport failure records a streak without evicting the peer', async () => {
     const peer = makePeer('a', ['openai']);
     const routerResults = [];
     const router = {
@@ -423,10 +513,248 @@ test('transport failures only update diagnostic failure state', async () => {
     assert.equal(res.statusCode, 502);
     assert.match(res.body, /Request abc123 timed out/);
     assert.equal(routerResults.length, 0);
-    assert.equal(proxy._peerFailures.get(peer.peerId)?.count, 1);
-    assert.equal(proxy._peerFailures.get(peer.peerId)?.lastReason, 'request-failed');
+    assert.equal(proxy._peerHealth.get(peer.peerId)?.lastReason, 'request-failed');
+    // Cooldown never evicts discovery metadata — the peer stays routable.
     assert.equal(proxy._cachedPeers[0]?.peerId, peer.peerId);
 });
+test('a timeout does not cool a peer down until another peer proves the buyer is healthy', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._node.sendRequest = async () => {
+        throw new Error('Request abc123 timed out');
+    };
+    // No success anywhere yet: the buyer itself might be the broken party, so
+    // even three failures must not exile the peer.
+    await failRepeatedly(proxy, peer, clock);
+    assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
+    proxy._rememberSuccessfulPeer(other.peerId);
+    await failRepeatedly(proxy, peer, clock);
+    assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now(), 'with corroboration the peer should now be cooling down');
+});
+test('a cooling-down peer is still dispatched to when a request names it', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async () => {
+        throw new Error('Request abc123 timed out');
+    };
+    await failRepeatedly(proxy, peer, clock);
+    assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
+    let dispatched = false;
+    proxy._node.sendRequest = async (_peer, request) => {
+        dispatched = true;
+        return {
+            requestId: request.requestId,
+            statusCode: 200,
+            headers: { 'content-type': 'application/json' },
+            body: Buffer.from('{}'),
+        };
+    };
+    const res = await invokeProxy(proxy, makeProxyRequest({ headers: { 'x-antseed-pin-peer': peer.peerId } }));
+    assert.equal(dispatched, true, 'cooldown is advisory; a named peer must still be tried');
+    assert.equal(res.statusCode, 200);
+    // And the response clears the cooldown, because the peer plainly answered.
+    assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
+});
+test('a 429 records capacity pressure without ever cooling the peer down', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async (_peer, request) => ({
+        requestId: request.requestId,
+        statusCode: 429,
+        headers: { 'content-type': 'text/plain' },
+        body: Buffer.from('slow down'),
+    });
+    await failRepeatedly(proxy, peer, clock, 5);
+    const health = healthOf(proxy, peer);
+    assert.equal(health?.lastReason, 'seller-busy');
+    assert.equal(health?.failureStreak, 0);
+    assert.equal(health?.cooldownUntil, 0);
+});
+test('a seller 503 escalates once the buyer is corroborated as healthy', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async (_peer, request) => ({
+        requestId: request.requestId,
+        statusCode: 503,
+        headers: { 'content-type': 'text/plain' },
+        body: Buffer.from('seller down'),
+    });
+    await failRepeatedly(proxy, peer, clock);
+    const health = healthOf(proxy, peer);
+    assert.equal(health?.lastReason, 'seller-5xx');
+    assert.ok(health?.cooldownUntil > clock.now());
+});
+test('non-standard seller 5xx responses also build a cooldown streak', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async (_peer, request) => ({
+        requestId: request.requestId,
+        statusCode: 522,
+        headers: { 'content-type': 'text/plain' },
+        body: Buffer.from('connection timed out'),
+    });
+    await failRepeatedly(proxy, peer, clock);
+    const health = healthOf(proxy, peer);
+    assert.equal(health?.lastReason, 'seller-5xx');
+    assert.ok(health?.cooldownUntil > clock.now());
+});
+test('a clean 4xx counts as proof of life and clears a cooldown', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async () => {
+        throw new Error('Request abc123 timed out');
+    };
+    await failRepeatedly(proxy, peer, clock);
+    assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
+    proxy._node.sendRequest = async (_peer, request) => ({
+        requestId: request.requestId,
+        statusCode: 400,
+        headers: { 'content-type': 'text/plain' },
+        body: Buffer.from('bad request'),
+    });
+    await invokeProxy(proxy, makeProxyRequest({ headers: { 'x-antseed-pin-peer': peer.peerId } }));
+    const health = healthOf(proxy, peer);
+    assert.equal(health?.cooldownUntil, 0);
+    assert.equal(health?.failureStreak, 0);
+});
+test('a non-standard seller 5xx proves reachability and restarts the failure streak', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async () => { throw new Error('Request timed out'); };
+    await failRepeatedly(proxy, peer, clock);
+    assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
+    proxy._node.sendRequest = async (_peer, request) => ({
+        requestId: request.requestId,
+        statusCode: 507,
+        headers: { 'content-type': 'text/plain' },
+        body: Buffer.from('insufficient storage'),
+    });
+    await invokeProxy(proxy, makeProxyRequest({ headers: { 'x-antseed-pin-peer': peer.peerId } }));
+    assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
+    assert.equal(healthOf(proxy, peer)?.failureStreak, 1);
+});
+test('a successful control-plane response clears a stale cooldown', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async () => { throw new Error('Request timed out'); };
+    await failRepeatedly(proxy, peer, clock);
+    assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
+    proxy._node.sendRequest = async (_peer, request) => ({
+        requestId: request.requestId,
+        statusCode: 200,
+        headers: { 'content-type': 'application/json' },
+        body: Buffer.from('{"data":[]}'),
+    });
+    await invokeProxy(proxy, makeProxyRequest({
+        method: 'GET',
+        path: '/v1/models',
+        headers: { 'x-antseed-pin-peer': peer.peerId },
+    }));
+    assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
+    assert.equal(healthOf(proxy, peer)?.failureStreak, 0);
+});
+test('a buyer-side outage rolls back the cooldowns it caused', async () => {
+    const clock = makeTestClock();
+    const peers = ['a', 'b', 'c', 'd'].map((seed) => makePeer(seed, ['openai']));
+    const proxy = makeBuyerProxyWithPeers(peers, peers, permissiveRouter(), clock.now);
+    proxy._cachedPeers = peers;
+    proxy._rememberSuccessfulPeer(peers[3].peerId);
+    proxy._node.sendRequest = async () => {
+        throw new Error('Connection to peer failed');
+    };
+    // Every peer failing at once is the signature of a dropped network, not of
+    // three sellers dying simultaneously.
+    for (const peer of peers.slice(0, 3)) {
+        await failRepeatedly(proxy, peer, clock);
+    }
+    for (const peer of peers.slice(0, 3)) {
+        const health = healthOf(proxy, peer);
+        assert.equal(health?.cooldownUntil, 0, `${peer.peerId.slice(0, 4)} should not be cooling down`);
+        assert.equal(health?.failureStreak, 0, `${peer.peerId.slice(0, 4)} streak should be rolled back`);
+    }
+});
+test('GET /_antseed/peer-health reports cooldowns and buyer health', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async () => {
+        throw new Error('Request abc123 timed out');
+    };
+    await failRepeatedly(proxy, peer, clock);
+    const res = await invokeProxy(proxy, makeProxyRequest({ method: 'GET', path: '/_antseed/peer-health' }));
+    assert.equal(res.statusCode, 200);
+    const body = JSON.parse(res.body);
+    assert.equal(body.ok, true);
+    assert.equal(body.buyerHealthy, true);
+    const entry = body.peers.find((p) => p.peerId === peer.peerId);
+    assert.equal(entry.coolingDown, true);
+    assert.ok(entry.cooldownMsRemaining > 0);
+    assert.equal(entry.lastReason, 'request-failed');
+});
+test('POST /_antseed/peer-health/clear gives a peer another chance', async () => {
+    const clock = makeTestClock();
+    const peer = makePeer('a', ['openai']);
+    const other = makePeer('b', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer, other], [peer, other], permissiveRouter(), clock.now);
+    proxy._cachedPeers = [peer, other];
+    proxy._rememberSuccessfulPeer(other.peerId);
+    proxy._node.sendRequest = async () => {
+        throw new Error('Request abc123 timed out');
+    };
+    await failRepeatedly(proxy, peer, clock);
+    assert.ok(healthOf(proxy, peer)?.cooldownUntil > clock.now());
+    const res = await invokeProxy(proxy, makeProxyRequest({
+        method: 'POST',
+        path: '/_antseed/peer-health/clear',
+        body: { peerId: peer.peerId },
+    }));
+    assert.equal(res.statusCode, 200);
+    assert.equal(healthOf(proxy, peer)?.cooldownUntil, 0);
+});
+test('POST /_antseed/peer-health/clear rejects a malformed peer id', async () => {
+    const peer = makePeer('a', ['openai']);
+    const proxy = makeBuyerProxyWithPeers([peer], [peer], { allowsPeerForPolicy: () => true });
+    const res = await invokeProxy(proxy, makeProxyRequest({
+        method: 'POST',
+        path: '/_antseed/peer-health/clear',
+        body: { peerId: 'nope' },
+    }));
+    assert.equal(res.statusCode, 400);
+});
 test('/v1/models retryable response reports router success', async () => {
     const peer = makePeer('a', ['openai']);
     const routerResults = [];