mixdog 0.9.53 → 0.9.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/agent-model-liveness-test.mjs +68 -0
- package/scripts/anthropic-transport-policy-test.mjs +260 -0
- package/scripts/gemini-provider-test.mjs +396 -1
- package/scripts/grok-oauth-refresh-race-test.mjs +76 -1
- package/scripts/openai-oauth-ws-1006-retry-test.mjs +81 -1
- package/scripts/process-lifecycle-test.mjs +13 -2
- package/scripts/provider-admission-scheduler-test.mjs +4 -5
- package/scripts/provider-contract-test.mjs +91 -33
- package/scripts/provider-toolcall-test.mjs +97 -17
- package/src/repl.mjs +11 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +62 -25
- package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +8 -2
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +50 -23
- package/src/runtime/agent/orchestrator/providers/gemini-cache.mjs +15 -4
- package/src/runtime/agent/orchestrator/providers/gemini-stream.mjs +136 -27
- package/src/runtime/agent/orchestrator/providers/gemini.mjs +104 -20
- package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +96 -75
- package/src/runtime/agent/orchestrator/providers/lib/anthropic-request-utils.mjs +40 -1
- package/src/runtime/agent/orchestrator/providers/openai-compat-stream.mjs +5 -0
- package/src/runtime/agent/orchestrator/providers/openai-compat-wire.mjs +35 -4
- package/src/runtime/agent/orchestrator/providers/openai-compat-xai.mjs +3 -1
- package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +49 -9
- package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +14 -2
- package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs +9 -4
- package/src/runtime/agent/orchestrator/providers/retry-classifier.mjs +53 -13
- package/src/runtime/agent/orchestrator/session/manager/ask-session.mjs +53 -7
- package/src/runtime/agent/orchestrator/session/manager/turn-interruption.mjs +23 -1
- package/src/runtime/agent/orchestrator/stall-policy.mjs +6 -13
- package/src/runtime/shared/memory-snapshot.mjs +45 -36
- package/src/runtime/shared/process-lifecycle.mjs +113 -36
- package/src/session-runtime/session-turn-api.mjs +1 -0
- package/src/tui/dist/index.mjs +31 -0
- package/src/tui/engine/turn.mjs +31 -0
|
@@ -427,7 +427,7 @@ test('Gemini function-response media is nested, referenced, and MIME-safe', () =
|
|
|
427
427
|
});
|
|
428
428
|
|
|
429
429
|
test('Gemini global cache accepts a fresh default-five-minute entry', () => {
|
|
430
|
-
assert.
|
|
430
|
+
assert.equal(GEMINI_GLOBAL_CACHE_MIN_LIVE_MS, 75_000);
|
|
431
431
|
geminiGlobalCaches.clear();
|
|
432
432
|
const now = Date.now();
|
|
433
433
|
_setGeminiGlobalCache('k', {
|
|
@@ -437,6 +437,16 @@ test('Gemini global cache accepts a fresh default-five-minute entry', () => {
|
|
|
437
437
|
assert.equal(_getGeminiGlobalCache('k', now)?.cacheName, 'cachedContents/fresh');
|
|
438
438
|
});
|
|
439
439
|
|
|
440
|
+
test('Gemini global cache rejects entries inside first-byte headroom', () => {
|
|
441
|
+
geminiGlobalCaches.clear();
|
|
442
|
+
const now = Date.now();
|
|
443
|
+
_setGeminiGlobalCache('near-expiry', {
|
|
444
|
+
cacheName: 'cachedContents/near-expiry',
|
|
445
|
+
cacheExpiresAt: now + 70_000,
|
|
446
|
+
});
|
|
447
|
+
assert.equal(_getGeminiGlobalCache('near-expiry', now), null);
|
|
448
|
+
});
|
|
449
|
+
|
|
440
450
|
test('Gemini cache identity includes toolConfig', () => {
|
|
441
451
|
const base = {
|
|
442
452
|
model: 'gemini-2.5-flash',
|
|
@@ -518,6 +528,47 @@ test('Gemini cachedContents creation carries toolConfig with the tool schema', a
|
|
|
518
528
|
}
|
|
519
529
|
});
|
|
520
530
|
|
|
531
|
+
test('Gemini cachedContents creation retries transient failures within policy', async () => {
|
|
532
|
+
geminiGlobalCaches.clear();
|
|
533
|
+
let calls = 0;
|
|
534
|
+
const provider = new GeminiProvider(hermeticConfig({ fetchFn: async () => {
|
|
535
|
+
calls += 1;
|
|
536
|
+
if (calls === 1) {
|
|
537
|
+
return {
|
|
538
|
+
ok: false,
|
|
539
|
+
status: 503,
|
|
540
|
+
headers: new Headers({ 'Retry-After': '0' }),
|
|
541
|
+
async text() { return '{"error":{"message":"temporarily unavailable"}}'; },
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
return {
|
|
545
|
+
ok: true,
|
|
546
|
+
async json() {
|
|
547
|
+
return {
|
|
548
|
+
name: 'cachedContents/retried',
|
|
549
|
+
usageMetadata: { totalTokenCount: 3000 },
|
|
550
|
+
};
|
|
551
|
+
},
|
|
552
|
+
};
|
|
553
|
+
} }));
|
|
554
|
+
try {
|
|
555
|
+
const name = await provider._ensureGeminiCache({
|
|
556
|
+
apiKey: 'test-key',
|
|
557
|
+
model: 'gemini-2.5-flash',
|
|
558
|
+
systemInstruction: 'x'.repeat(9000),
|
|
559
|
+
contents: [
|
|
560
|
+
{ role: 'user', parts: [{ text: 'prefix' }] },
|
|
561
|
+
{ role: 'user', parts: [{ text: 'latest' }] },
|
|
562
|
+
],
|
|
563
|
+
opts: { providerState: {}, iteration: 1 },
|
|
564
|
+
});
|
|
565
|
+
assert.equal(name, 'cachedContents/retried');
|
|
566
|
+
assert.equal(calls, 2);
|
|
567
|
+
} finally {
|
|
568
|
+
geminiGlobalCaches.clear();
|
|
569
|
+
}
|
|
570
|
+
});
|
|
571
|
+
|
|
521
572
|
test('Gemini constructor and send are network-hermetic through injected transports', async () => {
|
|
522
573
|
let preconnectCalls = 0;
|
|
523
574
|
let fetchCalls = 0;
|
|
@@ -849,6 +900,133 @@ test('Gemini REST EOF finalizes buffered leaked tools before classifying truncat
|
|
|
849
900
|
assert.equal(classifyError(error), 'permanent');
|
|
850
901
|
});
|
|
851
902
|
|
|
903
|
+
test('Gemini REST malformed SSE is retryable corruption even before a later STOP', async () => {
|
|
904
|
+
const body = new ReadableStream({
|
|
905
|
+
start(controller) {
|
|
906
|
+
controller.enqueue(new TextEncoder().encode([
|
|
907
|
+
'data: {not-json}',
|
|
908
|
+
'data: {"candidates":[{"finishReason":"STOP"}]}',
|
|
909
|
+
'',
|
|
910
|
+
].join('\n')));
|
|
911
|
+
controller.close();
|
|
912
|
+
},
|
|
913
|
+
});
|
|
914
|
+
const error = await consumeGeminiRestStreamResponse(
|
|
915
|
+
{ body },
|
|
916
|
+
{ label: 'REST corrupt SSE' },
|
|
917
|
+
).then(() => null, (caught) => caught);
|
|
918
|
+
assert.equal(error.streamCorruption, true);
|
|
919
|
+
assert.equal(error.code, 'TRUNCATED_STREAM');
|
|
920
|
+
assert.equal(classifyError(error), 'transient');
|
|
921
|
+
});
|
|
922
|
+
|
|
923
|
+
test('Gemini REST already-aborted signal does not lock the response body', async () => {
|
|
924
|
+
const controller = new AbortController();
|
|
925
|
+
controller.abort(new Error('already closed'));
|
|
926
|
+
const body = new ReadableStream({ start() {} });
|
|
927
|
+
await assert.rejects(
|
|
928
|
+
consumeGeminiRestStreamResponse(
|
|
929
|
+
{ body },
|
|
930
|
+
{ label: 'REST pre-abort', signal: controller.signal },
|
|
931
|
+
),
|
|
932
|
+
/already closed/,
|
|
933
|
+
);
|
|
934
|
+
assert.equal(body.locked, false);
|
|
935
|
+
});
|
|
936
|
+
|
|
937
|
+
test('Gemini SDK first-byte timeout cancels generation before rejecting', async () => {
|
|
938
|
+
let cancelled = false;
|
|
939
|
+
let returned = false;
|
|
940
|
+
const streamResult = {
|
|
941
|
+
stream: {
|
|
942
|
+
[Symbol.asyncIterator]() {
|
|
943
|
+
return {
|
|
944
|
+
next() { return new Promise(() => {}); },
|
|
945
|
+
async return() {
|
|
946
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
947
|
+
returned = true;
|
|
948
|
+
return { done: true };
|
|
949
|
+
},
|
|
950
|
+
};
|
|
951
|
+
},
|
|
952
|
+
},
|
|
953
|
+
};
|
|
954
|
+
const error = await consumeGeminiSdkStream(streamResult, {
|
|
955
|
+
label: 'SDK timeout cancellation',
|
|
956
|
+
firstByteTimeoutMs: 5,
|
|
957
|
+
cancelGeneration: () => { cancelled = true; },
|
|
958
|
+
}).then(() => null, (caught) => caught);
|
|
959
|
+
assert.equal(error.code, 'EGEMINITIMEOUT');
|
|
960
|
+
assert.equal(cancelled, true);
|
|
961
|
+
assert.equal(returned, true);
|
|
962
|
+
});
|
|
963
|
+
|
|
964
|
+
test('Gemini SDK timeout is bounded when iterator return never settles', async () => {
|
|
965
|
+
let cancelled = false;
|
|
966
|
+
const startedAt = Date.now();
|
|
967
|
+
const streamResult = {
|
|
968
|
+
stream: {
|
|
969
|
+
[Symbol.asyncIterator]() {
|
|
970
|
+
return {
|
|
971
|
+
next() { return new Promise(() => {}); },
|
|
972
|
+
return() { return new Promise(() => {}); },
|
|
973
|
+
};
|
|
974
|
+
},
|
|
975
|
+
},
|
|
976
|
+
response: new Promise(() => {}),
|
|
977
|
+
};
|
|
978
|
+
const error = await consumeGeminiSdkStream(streamResult, {
|
|
979
|
+
label: 'SDK stuck cancellation',
|
|
980
|
+
firstByteTimeoutMs: 5,
|
|
981
|
+
cancellationGraceMs: 20,
|
|
982
|
+
cancelGeneration: () => { cancelled = true; },
|
|
983
|
+
}).then(() => null, (caught) => caught);
|
|
984
|
+
assert.equal(error.code, 'EGEMINITIMEOUT');
|
|
985
|
+
assert.equal(cancelled, true);
|
|
986
|
+
assert.ok(Date.now() - startedAt < 200, 'stuck iterator.return must not deadlock timeout rejection');
|
|
987
|
+
});
|
|
988
|
+
|
|
989
|
+
test('Gemini SDK parser errors are retryable stream corruption', async () => {
|
|
990
|
+
const parseFailure = Object.assign(
|
|
991
|
+
new Error('[GoogleGenerativeAI Error]: Error parsing JSON response: Unexpected end of JSON input'),
|
|
992
|
+
{ name: 'GoogleGenerativeAIError' },
|
|
993
|
+
);
|
|
994
|
+
const streamResult = {
|
|
995
|
+
stream: {
|
|
996
|
+
[Symbol.asyncIterator]() {
|
|
997
|
+
return {
|
|
998
|
+
async next() { throw parseFailure; },
|
|
999
|
+
async return() { return { done: true }; },
|
|
1000
|
+
};
|
|
1001
|
+
},
|
|
1002
|
+
},
|
|
1003
|
+
response: Promise.reject(parseFailure),
|
|
1004
|
+
};
|
|
1005
|
+
const error = await consumeGeminiSdkStream(streamResult, {
|
|
1006
|
+
label: 'SDK corrupt SSE',
|
|
1007
|
+
}).then(() => null, (caught) => caught);
|
|
1008
|
+
assert.equal(error.streamCorruption, true);
|
|
1009
|
+
assert.equal(error.code, 'TRUNCATED_STREAM');
|
|
1010
|
+
assert.equal(classifyError(error), 'transient');
|
|
1011
|
+
});
|
|
1012
|
+
|
|
1013
|
+
test('Gemini native function call is retry-safe until provider dispatch', async () => {
|
|
1014
|
+
const streamResult = {
|
|
1015
|
+
stream: {
|
|
1016
|
+
async *[Symbol.asyncIterator]() {
|
|
1017
|
+
yield { candidates: [{ content: { parts: [{ functionCall: { name: 'read', args: {} } }] } }] };
|
|
1018
|
+
throw Object.assign(new Error('socket reset'), { code: 'ECONNRESET' });
|
|
1019
|
+
},
|
|
1020
|
+
},
|
|
1021
|
+
};
|
|
1022
|
+
const error = await consumeGeminiSdkStream(streamResult, {
|
|
1023
|
+
label: 'SDK pre-dispatch function call',
|
|
1024
|
+
}).then(() => null, (caught) => caught);
|
|
1025
|
+
assert.equal(error.emittedToolCall, undefined);
|
|
1026
|
+
assert.equal(error.unsafeToRetry, undefined);
|
|
1027
|
+
assert.equal(classifyError(error), 'transient');
|
|
1028
|
+
});
|
|
1029
|
+
|
|
852
1030
|
test('Gemini SDK failure finalizes buffered leaked tools before retry-safety stamps', async () => {
|
|
853
1031
|
const streamResult = {
|
|
854
1032
|
stream: {
|
|
@@ -1051,3 +1229,220 @@ test('Gemini auth reload retries a status-coded 401 exactly once', async () => {
|
|
|
1051
1229
|
assert.equal(reloads, 1);
|
|
1052
1230
|
assert.equal(opts.providerState.gemini, undefined);
|
|
1053
1231
|
});
|
|
1232
|
+
|
|
1233
|
+
test('Gemini evicted cachedContent is invalidated and retried uncached', async () => {
|
|
1234
|
+
geminiGlobalCaches.clear();
|
|
1235
|
+
_setGeminiGlobalCache('evicted-key', {
|
|
1236
|
+
cacheName: 'cachedContents/evicted',
|
|
1237
|
+
cacheExpiresAt: Date.now() + 5 * 60_000,
|
|
1238
|
+
});
|
|
1239
|
+
let cacheChecks = 0;
|
|
1240
|
+
let restCalls = 0;
|
|
1241
|
+
let sdkCalls = 0;
|
|
1242
|
+
const provider = new GeminiProvider(hermeticConfig({
|
|
1243
|
+
fetchFn: async () => {
|
|
1244
|
+
restCalls += 1;
|
|
1245
|
+
return {
|
|
1246
|
+
ok: false,
|
|
1247
|
+
status: 404,
|
|
1248
|
+
headers: new Headers(),
|
|
1249
|
+
async text() {
|
|
1250
|
+
return JSON.stringify({ error: { message: 'Cached content not found', status: 'NOT_FOUND' } });
|
|
1251
|
+
},
|
|
1252
|
+
};
|
|
1253
|
+
},
|
|
1254
|
+
genAI: {
|
|
1255
|
+
getGenerativeModel() {
|
|
1256
|
+
return {
|
|
1257
|
+
async generateContentStream() {
|
|
1258
|
+
sdkCalls += 1;
|
|
1259
|
+
return sdkStream([{
|
|
1260
|
+
candidates: [{ content: { role: 'model', parts: [{ text: 'uncached ok' }] }, finishReason: 'STOP' }],
|
|
1261
|
+
}]);
|
|
1262
|
+
},
|
|
1263
|
+
};
|
|
1264
|
+
},
|
|
1265
|
+
},
|
|
1266
|
+
}));
|
|
1267
|
+
provider._ensureGeminiCache = async ({ skipExplicitCache }) => {
|
|
1268
|
+
cacheChecks += 1;
|
|
1269
|
+
return skipExplicitCache ? null : 'cachedContents/evicted';
|
|
1270
|
+
};
|
|
1271
|
+
const opts = {
|
|
1272
|
+
providerState: { gemini: { cacheName: 'cachedContents/evicted', cachePrefixContentCount: 0 } },
|
|
1273
|
+
};
|
|
1274
|
+
const result = await provider.send(
|
|
1275
|
+
[{ role: 'user', content: 'hello' }],
|
|
1276
|
+
'gemini-2.5-flash',
|
|
1277
|
+
[],
|
|
1278
|
+
opts,
|
|
1279
|
+
);
|
|
1280
|
+
assert.equal(result.content, 'uncached ok');
|
|
1281
|
+
assert.equal(restCalls, 1);
|
|
1282
|
+
assert.equal(sdkCalls, 1);
|
|
1283
|
+
assert.equal(cacheChecks, 2);
|
|
1284
|
+
assert.equal(opts.providerState.gemini, undefined);
|
|
1285
|
+
assert.equal(geminiGlobalCaches.size, 0);
|
|
1286
|
+
});
|
|
1287
|
+
|
|
1288
|
+
test('Gemini REST rate-limit response preserves Retry-After for request retry', async () => {
|
|
1289
|
+
let calls = 0;
|
|
1290
|
+
let firstCallAt = 0;
|
|
1291
|
+
let secondCallAt = 0;
|
|
1292
|
+
const provider = new GeminiProvider(hermeticConfig({
|
|
1293
|
+
fetchFn: async () => {
|
|
1294
|
+
calls += 1;
|
|
1295
|
+
if (calls === 1) {
|
|
1296
|
+
firstCallAt = Date.now();
|
|
1297
|
+
return {
|
|
1298
|
+
ok: false,
|
|
1299
|
+
status: 429,
|
|
1300
|
+
headers: new Headers({ 'Retry-After': '0.05' }),
|
|
1301
|
+
async text() {
|
|
1302
|
+
return JSON.stringify({
|
|
1303
|
+
error: {
|
|
1304
|
+
status: 'RESOURCE_EXHAUSTED',
|
|
1305
|
+
message: 'resource exhausted; retry after the capacity window',
|
|
1306
|
+
details: [{
|
|
1307
|
+
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
|
|
1308
|
+
retryDelay: '0s',
|
|
1309
|
+
}],
|
|
1310
|
+
},
|
|
1311
|
+
});
|
|
1312
|
+
},
|
|
1313
|
+
};
|
|
1314
|
+
}
|
|
1315
|
+
secondCallAt = Date.now();
|
|
1316
|
+
return {
|
|
1317
|
+
ok: true,
|
|
1318
|
+
body: new ReadableStream({
|
|
1319
|
+
start(controller) {
|
|
1320
|
+
controller.enqueue(new TextEncoder().encode(
|
|
1321
|
+
'data: {"candidates":[{"content":{"role":"model","parts":[{"text":"retried"}]},"finishReason":"STOP"}]}\n',
|
|
1322
|
+
));
|
|
1323
|
+
controller.close();
|
|
1324
|
+
},
|
|
1325
|
+
}),
|
|
1326
|
+
};
|
|
1327
|
+
},
|
|
1328
|
+
}));
|
|
1329
|
+
provider._ensureGeminiCache = async () => 'cachedContents/live';
|
|
1330
|
+
const result = await provider.send(
|
|
1331
|
+
[{ role: 'user', content: 'hello' }],
|
|
1332
|
+
'gemini-2.5-flash',
|
|
1333
|
+
[],
|
|
1334
|
+
{ providerState: { gemini: { cachePrefixContentCount: 0 } } },
|
|
1335
|
+
);
|
|
1336
|
+
assert.equal(result.content, 'retried');
|
|
1337
|
+
assert.equal(calls, 2);
|
|
1338
|
+
assert.ok(secondCallAt - firstCallAt >= 40, 'Retry-After must set the retry delay');
|
|
1339
|
+
});
|
|
1340
|
+
|
|
1341
|
+
test('Gemini structured RetryInfo outranks resource-exhausted permanence', async () => {
|
|
1342
|
+
let calls = 0;
|
|
1343
|
+
let firstCallAt = 0;
|
|
1344
|
+
let secondCallAt = 0;
|
|
1345
|
+
const provider = new GeminiProvider(hermeticConfig({
|
|
1346
|
+
fetchFn: async () => {
|
|
1347
|
+
calls += 1;
|
|
1348
|
+
if (calls === 1) {
|
|
1349
|
+
firstCallAt = Date.now();
|
|
1350
|
+
return {
|
|
1351
|
+
ok: false,
|
|
1352
|
+
status: 429,
|
|
1353
|
+
headers: new Headers(),
|
|
1354
|
+
async text() {
|
|
1355
|
+
return JSON.stringify({
|
|
1356
|
+
error: {
|
|
1357
|
+
status: 'RESOURCE_EXHAUSTED',
|
|
1358
|
+
message: 'resource exhausted',
|
|
1359
|
+
details: [{
|
|
1360
|
+
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
|
|
1361
|
+
retryDelay: '0.05s',
|
|
1362
|
+
}],
|
|
1363
|
+
},
|
|
1364
|
+
});
|
|
1365
|
+
},
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
secondCallAt = Date.now();
|
|
1369
|
+
return {
|
|
1370
|
+
ok: true,
|
|
1371
|
+
body: new ReadableStream({
|
|
1372
|
+
start(controller) {
|
|
1373
|
+
controller.enqueue(new TextEncoder().encode(
|
|
1374
|
+
'data: {"candidates":[{"content":{"role":"model","parts":[{"text":"retry-info"}]},"finishReason":"STOP"}]}\n',
|
|
1375
|
+
));
|
|
1376
|
+
controller.close();
|
|
1377
|
+
},
|
|
1378
|
+
}),
|
|
1379
|
+
};
|
|
1380
|
+
},
|
|
1381
|
+
}));
|
|
1382
|
+
provider._ensureGeminiCache = async () => 'cachedContents/live';
|
|
1383
|
+
const result = await provider.send(
|
|
1384
|
+
[{ role: 'user', content: 'hello' }],
|
|
1385
|
+
'gemini-2.5-flash',
|
|
1386
|
+
[],
|
|
1387
|
+
{ providerState: { gemini: { cachePrefixContentCount: 0 } } },
|
|
1388
|
+
);
|
|
1389
|
+
assert.equal(result.content, 'retry-info');
|
|
1390
|
+
assert.equal(calls, 2);
|
|
1391
|
+
assert.ok(secondCallAt - firstCallAt >= 40, 'RetryInfo.retryDelay must set the retry delay');
|
|
1392
|
+
});
|
|
1393
|
+
|
|
1394
|
+
test('Gemini RESOURCE_EXHAUSTED without Retry-After remains permanent', async () => {
|
|
1395
|
+
let calls = 0;
|
|
1396
|
+
const provider = new GeminiProvider(hermeticConfig({
|
|
1397
|
+
fetchFn: async () => {
|
|
1398
|
+
calls += 1;
|
|
1399
|
+
return {
|
|
1400
|
+
ok: false,
|
|
1401
|
+
status: 429,
|
|
1402
|
+
headers: new Headers(),
|
|
1403
|
+
async text() {
|
|
1404
|
+
return JSON.stringify({
|
|
1405
|
+
error: {
|
|
1406
|
+
status: 'RESOURCE_EXHAUSTED',
|
|
1407
|
+
message: 'capacity quota exhausted',
|
|
1408
|
+
details: [{ '@type': 'type.googleapis.com/google.rpc.QuotaFailure' }],
|
|
1409
|
+
},
|
|
1410
|
+
});
|
|
1411
|
+
},
|
|
1412
|
+
};
|
|
1413
|
+
},
|
|
1414
|
+
}));
|
|
1415
|
+
provider._ensureGeminiCache = async () => 'cachedContents/live';
|
|
1416
|
+
const error = await provider.send(
|
|
1417
|
+
[{ role: 'user', content: 'hello' }],
|
|
1418
|
+
'gemini-2.5-flash',
|
|
1419
|
+
[],
|
|
1420
|
+
{ providerState: { gemini: { cachePrefixContentCount: 0 } } },
|
|
1421
|
+
).then(() => null, (caught) => caught);
|
|
1422
|
+
assert.equal(calls, 1);
|
|
1423
|
+
assert.equal(error.code, 'RESOURCE_EXHAUSTED');
|
|
1424
|
+
assert.equal(error.geminiStatus, 'RESOURCE_EXHAUSTED');
|
|
1425
|
+
assert.equal(error.details[0]['@type'], 'type.googleapis.com/google.rpc.QuotaFailure');
|
|
1426
|
+
assert.equal(classifyError(error), 'permanent');
|
|
1427
|
+
});
|
|
1428
|
+
|
|
1429
|
+
test('Gemini availability probe rejects an unresolved generation within its bound', async () => {
|
|
1430
|
+
let receivedSignal = null;
|
|
1431
|
+
const startedAt = Date.now();
|
|
1432
|
+
const provider = new GeminiProvider(hermeticConfig({
|
|
1433
|
+
genAI: {
|
|
1434
|
+
getGenerativeModel() {
|
|
1435
|
+
return {
|
|
1436
|
+
async generateContent(_input, options) {
|
|
1437
|
+
receivedSignal = options?.signal;
|
|
1438
|
+
return new Promise(() => {});
|
|
1439
|
+
},
|
|
1440
|
+
};
|
|
1441
|
+
},
|
|
1442
|
+
},
|
|
1443
|
+
}));
|
|
1444
|
+
assert.equal(await provider.isAvailable(), false);
|
|
1445
|
+
assert.equal(receivedSignal instanceof AbortSignal, true);
|
|
1446
|
+
assert.equal(receivedSignal.aborted, true);
|
|
1447
|
+
assert.ok(Date.now() - startedAt < 1_500, 'availability probe must be bounded');
|
|
1448
|
+
});
|
|
@@ -26,12 +26,13 @@ after(() => {
|
|
|
26
26
|
rmSync(root, { recursive: true, force: true });
|
|
27
27
|
});
|
|
28
28
|
|
|
29
|
-
function writeTokens(accessToken, refreshToken, expiresAt) {
|
|
29
|
+
function writeTokens(accessToken, refreshToken, expiresAt, extra = {}) {
|
|
30
30
|
writeJsonAtomicSync(tokenPath, {
|
|
31
31
|
access_token: accessToken,
|
|
32
32
|
refresh_token: refreshToken,
|
|
33
33
|
expires_at: expiresAt,
|
|
34
34
|
token_endpoint: 'https://auth.x.ai/oauth/token',
|
|
35
|
+
...extra,
|
|
35
36
|
}, { lock: true, fsyncDir: true, mode: 0o600, secret: true });
|
|
36
37
|
}
|
|
37
38
|
|
|
@@ -196,3 +197,77 @@ writeFileSync(process.env.RESULT_PATH, JSON.stringify({
|
|
|
196
197
|
assert.deepEqual(JSON.parse(readFileSync(resultPath, 'utf8')), { adopted: true, calls: 1 });
|
|
197
198
|
assert.ok(statSync(tokenPath).size > 0);
|
|
198
199
|
});
|
|
200
|
+
|
|
201
|
+
test('refresh retries transient responses three times, preserves principal fields, and stops on invalid_client', async () => {
|
|
202
|
+
const resultPath = join(root, 'retry-policy-result');
|
|
203
|
+
writeTokens('fixture-retry-access', 'fixture-retry-refresh', Date.now() + 1_000, {
|
|
204
|
+
principal_type: 'user',
|
|
205
|
+
principal_id: 'principal-123',
|
|
206
|
+
});
|
|
207
|
+
const childSource = `
|
|
208
|
+
const { writeFileSync } = await import('node:fs');
|
|
209
|
+
const { GrokOAuthProvider } = await import(${JSON.stringify(moduleUrl)});
|
|
210
|
+
const provider = new GrokOAuthProvider({preconnect:false});
|
|
211
|
+
const bodies = [];
|
|
212
|
+
let retryCalls = 0;
|
|
213
|
+
globalThis.fetch = async (_url, init) => {
|
|
214
|
+
retryCalls += 1;
|
|
215
|
+
bodies.push(Object.fromEntries(init.body.entries()));
|
|
216
|
+
if (retryCalls === 1) {
|
|
217
|
+
return {ok:false,status:400,async text(){return JSON.stringify({error:'temporarily_unavailable'});}};
|
|
218
|
+
}
|
|
219
|
+
if (retryCalls === 2) {
|
|
220
|
+
return {ok:false,status:401,async text(){return '{}';}};
|
|
221
|
+
}
|
|
222
|
+
return {ok:true,status:200,async text(){return JSON.stringify({
|
|
223
|
+
access_token:'fixture-retry-new-access',
|
|
224
|
+
refresh_token:'fixture-retry-new-refresh',
|
|
225
|
+
expires_in:600
|
|
226
|
+
});}};
|
|
227
|
+
};
|
|
228
|
+
const refreshed = await provider.ensureAuth({forceRefresh:true});
|
|
229
|
+
let terminalCalls = 0;
|
|
230
|
+
globalThis.fetch = async () => {
|
|
231
|
+
terminalCalls += 1;
|
|
232
|
+
return {ok:false,status:400,async text(){return JSON.stringify({error:'invalid_client'});}};
|
|
233
|
+
};
|
|
234
|
+
let terminalCode = null;
|
|
235
|
+
try {
|
|
236
|
+
await provider.ensureAuth({forceRefresh:true});
|
|
237
|
+
} catch (error) {
|
|
238
|
+
terminalCode = error.oauthError;
|
|
239
|
+
}
|
|
240
|
+
writeFileSync(process.env.RESULT_PATH, JSON.stringify({
|
|
241
|
+
retryCalls,
|
|
242
|
+
bodies,
|
|
243
|
+
principalType: refreshed.principal_type,
|
|
244
|
+
principalId: refreshed.principal_id,
|
|
245
|
+
terminalCalls,
|
|
246
|
+
terminalCode
|
|
247
|
+
}));
|
|
248
|
+
`;
|
|
249
|
+
const child = spawn(process.execPath, ['--input-type=module', '--eval', childSource], {
|
|
250
|
+
env: {
|
|
251
|
+
...process.env,
|
|
252
|
+
MIXDOG_DATA_DIR: root,
|
|
253
|
+
GROK_OAUTH_CREDENTIALS_PATH: tokenPath,
|
|
254
|
+
RESULT_PATH: resultPath,
|
|
255
|
+
},
|
|
256
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
257
|
+
});
|
|
258
|
+
await waitForExit(child, 'refresh retry policy child');
|
|
259
|
+
const result = JSON.parse(readFileSync(resultPath, 'utf8'));
|
|
260
|
+
assert.equal(result.retryCalls, 3);
|
|
261
|
+
assert.equal(result.terminalCalls, 1);
|
|
262
|
+
assert.equal(result.terminalCode, 'invalid_client');
|
|
263
|
+
assert.equal(result.principalType, 'user');
|
|
264
|
+
assert.equal(result.principalId, 'principal-123');
|
|
265
|
+
for (const body of result.bodies) {
|
|
266
|
+
assert.equal(body.principal_type, 'user');
|
|
267
|
+
assert.equal(body.principal_id, 'principal-123');
|
|
268
|
+
assert.equal(body.refresh_token, 'fixture-retry-refresh');
|
|
269
|
+
}
|
|
270
|
+
const persisted = JSON.parse(readFileSync(tokenPath, 'utf8'));
|
|
271
|
+
assert.equal(persisted.principal_type, 'user');
|
|
272
|
+
assert.equal(persisted.principal_id, 'principal-123');
|
|
273
|
+
});
|
|
@@ -4,7 +4,10 @@ import assert from 'node:assert/strict';
|
|
|
4
4
|
import { EventEmitter } from 'node:events';
|
|
5
5
|
import { OpenAIOAuthProvider } from '../src/runtime/agent/orchestrator/providers/openai-oauth.mjs';
|
|
6
6
|
import { _acquireWithRetry, sendViaWebSocket } from '../src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs';
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
sendViaHttpSse,
|
|
9
|
+
_shouldUseOpenAIHttpFallback,
|
|
10
|
+
} from '../src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs';
|
|
8
11
|
import { applyAskTerminalUsageTotals } from '../src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs';
|
|
9
12
|
import {
|
|
10
13
|
_clearWebSocketPoolForTest,
|
|
@@ -155,6 +158,83 @@ test('acquire timeout reconnects successfully with progress, not a terminal WS e
|
|
|
155
158
|
}
|
|
156
159
|
});
|
|
157
160
|
|
|
161
|
+
test('OAuth WS-handshake HTTP 429 is terminal and cannot enter the HTTP fallback gate', async () => {
|
|
162
|
+
const savedEnv = Object.fromEntries([
|
|
163
|
+
'MIXDOG_OAI_TRANSPORT',
|
|
164
|
+
'MIXDOG_OPENAI_HTTP_FALLBACK',
|
|
165
|
+
'MIXDOG_OPENAI_OAUTH_HTTP_FALLBACK',
|
|
166
|
+
'MIXDOG_OPENAI_OAUTH_WS_WARMUP',
|
|
167
|
+
'MIXDOG_AGENT_TRACE_DISABLE',
|
|
168
|
+
].map((name) => [name, process.env[name]]));
|
|
169
|
+
Object.assign(process.env, {
|
|
170
|
+
MIXDOG_OAI_TRANSPORT: 'auto',
|
|
171
|
+
MIXDOG_OPENAI_HTTP_FALLBACK: '1',
|
|
172
|
+
MIXDOG_OPENAI_OAUTH_HTTP_FALLBACK: '1',
|
|
173
|
+
MIXDOG_OPENAI_OAUTH_WS_WARMUP: '0',
|
|
174
|
+
MIXDOG_AGENT_TRACE_DISABLE: '1',
|
|
175
|
+
});
|
|
176
|
+
let acquires = 0;
|
|
177
|
+
let sleeps = 0;
|
|
178
|
+
let httpCalls = 0;
|
|
179
|
+
const provider = new OpenAIOAuthProvider({});
|
|
180
|
+
provider.ensureAuth = async () => ({ access_token: 'test-token' });
|
|
181
|
+
try {
|
|
182
|
+
await assert.rejects(
|
|
183
|
+
provider.send([], 'gpt-5.5', [], {
|
|
184
|
+
sessionId: 'openai-oauth-ws-429-test',
|
|
185
|
+
_prebuiltBody: { model: 'gpt-5.5', input: [] },
|
|
186
|
+
_sendViaWebSocketFn: async (args) => {
|
|
187
|
+
try {
|
|
188
|
+
return await sendViaWebSocket({
|
|
189
|
+
...args,
|
|
190
|
+
_acquireWithRetryFn: async () => {
|
|
191
|
+
acquires += 1;
|
|
192
|
+
throw Object.assign(new Error('handshake rate limited'), { httpStatus: 429 });
|
|
193
|
+
},
|
|
194
|
+
_sleepFn: async () => { sleeps += 1; },
|
|
195
|
+
});
|
|
196
|
+
} catch (err) {
|
|
197
|
+
// Force the provider's real exhausted-WS gate; the
|
|
198
|
+
// OAuth predicate itself must still reject this 429.
|
|
199
|
+
err.wsRetriesExhausted = true;
|
|
200
|
+
throw err;
|
|
201
|
+
}
|
|
202
|
+
},
|
|
203
|
+
_sendViaHttpSseFn: async () => {
|
|
204
|
+
httpCalls += 1;
|
|
205
|
+
return { content: 'must not fallback', toolCalls: [], usage: {} };
|
|
206
|
+
},
|
|
207
|
+
}),
|
|
208
|
+
(err) => err?.httpStatus === 429,
|
|
209
|
+
);
|
|
210
|
+
assert.equal(_shouldUseOpenAIHttpFallback({ httpStatus: 429, wsRetriesExhausted: true }), false);
|
|
211
|
+
assert.equal(acquires, 1, 'OAuth 429 must not consume the five-retry WS budget');
|
|
212
|
+
assert.equal(sleeps, 0, 'OAuth 429 must not schedule WS reconnect backoff');
|
|
213
|
+
assert.equal(httpCalls, 0, 'OAuth 429 must not enter HTTP/SSE fallback');
|
|
214
|
+
} finally {
|
|
215
|
+
for (const [name, value] of Object.entries(savedEnv)) {
|
|
216
|
+
if (value == null) delete process.env[name];
|
|
217
|
+
else process.env[name] = value;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test('non-OAuth WS handshakes retain their existing 429 retry budget', async () => {
|
|
223
|
+
let acquires = 0;
|
|
224
|
+
await assert.rejects(
|
|
225
|
+
sendViaWebSocket(wsArgs({
|
|
226
|
+
traceProvider: 'xai',
|
|
227
|
+
_acquireWithRetryFn: async () => {
|
|
228
|
+
acquires += 1;
|
|
229
|
+
throw Object.assign(new Error('handshake rate limited'), { httpStatus: 429 });
|
|
230
|
+
},
|
|
231
|
+
_sleepFn: async () => {},
|
|
232
|
+
})),
|
|
233
|
+
(err) => err?.httpStatus === 429 && err?.wsRetriesExhausted === true,
|
|
234
|
+
);
|
|
235
|
+
assert.equal(acquires, 6, 'non-OAuth 429 preserves the five-retry WS budget');
|
|
236
|
+
});
|
|
237
|
+
|
|
158
238
|
for (const failure of ['callback error', 'callback timeout']) {
|
|
159
239
|
test(`send ${failure} drops the socket and retries on a fresh one`, async () => {
|
|
160
240
|
const acquires = [];
|
|
@@ -104,7 +104,7 @@ test('vanished evidence uses only old-process fields and both ledger files stay
|
|
|
104
104
|
}
|
|
105
105
|
});
|
|
106
106
|
|
|
107
|
-
test('PID reuse is detected
|
|
107
|
+
test('PID reuse is detected once across a finish/rebegin while uncertain live identity is preserved', async () => {
|
|
108
108
|
const root = tempRoot();
|
|
109
109
|
let child;
|
|
110
110
|
try {
|
|
@@ -126,7 +126,12 @@ test('PID reuse is detected by process identity while uncertain live identity is
|
|
|
126
126
|
const name = readdirSync(paths.markerDir).find((entry) => entry.startsWith(`${child.pid}-`));
|
|
127
127
|
if (name) {
|
|
128
128
|
childMarker = join(paths.markerDir, name);
|
|
129
|
-
|
|
129
|
+
try {
|
|
130
|
+
currentIdentity = JSON.parse(readFileSync(childMarker, 'utf8')).processIdentity;
|
|
131
|
+
} catch {
|
|
132
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
130
135
|
if (currentIdentity?.kind === 'linux-start-ticks' || currentIdentity?.method === 'powershell') break;
|
|
131
136
|
}
|
|
132
137
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
@@ -191,6 +196,12 @@ test('PID reuse is detected by process identity while uncertain live identity is
|
|
|
191
196
|
processIdentity: currentIdentity,
|
|
192
197
|
}));
|
|
193
198
|
beginProcessLifecycle({ directory: root, configureReports: false });
|
|
199
|
+
assert.equal(finishProcessLifecycle('clean-shutdown', 0), true);
|
|
200
|
+
beginProcessLifecycle({ directory: root, configureReports: false });
|
|
201
|
+
const reapDeadline = Date.now() + 5000;
|
|
202
|
+
while (existsSync(reused) && Date.now() < reapDeadline) {
|
|
203
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
204
|
+
}
|
|
194
205
|
finishProcessLifecycle('clean-shutdown', 0);
|
|
195
206
|
assert.equal(entries(paths.ledger).filter((row) => row.reason === 'prior-process-vanished').length, 2);
|
|
196
207
|
assert.equal(existsSync(reused), false);
|
|
@@ -340,14 +340,14 @@ test('xAI legacy cache-lane knobs cannot create a secondary queue or timeout', a
|
|
|
340
340
|
assert.deepEqual((await Promise.all([first, second])).map((item) => item.value), [0, 1]);
|
|
341
341
|
});
|
|
342
342
|
|
|
343
|
-
test('Anthropic
|
|
343
|
+
test('Anthropic OAuth subscription 429 fails fast while API-key 429 retries request-locally', async () => {
|
|
344
344
|
const oauth = Object.create(AnthropicOAuthProvider.prototype);
|
|
345
345
|
oauth.credentials = { accessToken: 'fixture', expiresAt: Date.now() + 60_000 };
|
|
346
346
|
oauth.config = {};
|
|
347
347
|
oauth.fastModeBetaHeaderLatched = false;
|
|
348
348
|
oauth.ensureAuth = async () => oauth.credentials;
|
|
349
349
|
let oauthAttempts = 0;
|
|
350
|
-
|
|
350
|
+
await assert.rejects(oauth.send([], 'claude-sonnet-4-5', [], {
|
|
351
351
|
_doRequestFn: async () => {
|
|
352
352
|
oauthAttempts += 1;
|
|
353
353
|
return {
|
|
@@ -364,9 +364,8 @@ test('Anthropic providers retry request-local pre-output 429 responses', async (
|
|
|
364
364
|
_parseSSEFn: async () => ({
|
|
365
365
|
content: 'oauth-ok', model: 'claude', toolCalls: [], usage: {},
|
|
366
366
|
}),
|
|
367
|
-
});
|
|
368
|
-
assert.equal(
|
|
369
|
-
assert.equal(oauthAttempts, 2);
|
|
367
|
+
}), (error) => error?.status === 429 && error?.retryAfterMs === 0);
|
|
368
|
+
assert.equal(oauthAttempts, 1);
|
|
370
369
|
|
|
371
370
|
const direct = Object.create(AnthropicProvider.prototype);
|
|
372
371
|
direct.name = 'anthropic';
|