audit-tools 0.32.68 → 0.33.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/audit/cli/advanceAuditCommand.d.ts.map +1 -1
- package/dist/audit/cli/advanceAuditCommand.js +8 -3
- package/dist/audit/cli/advanceAuditCommand.js.map +1 -1
- package/dist/audit/cli/args.d.ts +9 -27
- package/dist/audit/cli/args.d.ts.map +1 -1
- package/dist/audit/cli/args.js +119 -50
- package/dist/audit/cli/args.js.map +1 -1
- package/dist/audit/cli/auditStep.d.ts +15 -1
- package/dist/audit/cli/auditStep.d.ts.map +1 -1
- package/dist/audit/cli/auditStep.js +2 -0
- package/dist/audit/cli/auditStep.js.map +1 -1
- package/dist/audit/cli/dispatch/quotaPool.d.ts +2 -1
- package/dist/audit/cli/dispatch/quotaPool.d.ts.map +1 -1
- package/dist/audit/cli/dispatch/quotaPool.js +43 -77
- package/dist/audit/cli/dispatch/quotaPool.js.map +1 -1
- package/dist/audit/cli/dispatch.d.ts.map +1 -1
- package/dist/audit/cli/dispatch.js +14 -6
- package/dist/audit/cli/dispatch.js.map +1 -1
- package/dist/audit/cli/hybridDispatch.d.ts +3 -1
- package/dist/audit/cli/hybridDispatch.d.ts.map +1 -1
- package/dist/audit/cli/hybridDispatch.js +6 -0
- package/dist/audit/cli/hybridDispatch.js.map +1 -1
- package/dist/audit/cli/nextStepCommand.d.ts +0 -9
- package/dist/audit/cli/nextStepCommand.d.ts.map +1 -1
- package/dist/audit/cli/nextStepCommand.js +173 -92
- package/dist/audit/cli/nextStepCommand.js.map +1 -1
- package/dist/audit/cli/nextStepHelpers.d.ts +11 -2
- package/dist/audit/cli/nextStepHelpers.d.ts.map +1 -1
- package/dist/audit/cli/nextStepHelpers.js +56 -8
- package/dist/audit/cli/nextStepHelpers.js.map +1 -1
- package/dist/audit/cli/prepareDispatchCommand.d.ts.map +1 -1
- package/dist/audit/cli/prepareDispatchCommand.js +15 -9
- package/dist/audit/cli/prepareDispatchCommand.js.map +1 -1
- package/dist/audit/cli/prompts.d.ts +12 -32
- package/dist/audit/cli/prompts.d.ts.map +1 -1
- package/dist/audit/cli/prompts.js +19 -33
- package/dist/audit/cli/prompts.js.map +1 -1
- package/dist/audit/cli/providerConfirmationStep.d.ts +13 -5
- package/dist/audit/cli/providerConfirmationStep.d.ts.map +1 -1
- package/dist/audit/cli/providerConfirmationStep.js +47 -8
- package/dist/audit/cli/providerConfirmationStep.js.map +1 -1
- package/dist/audit/cli/quotaCommand.d.ts.map +1 -1
- package/dist/audit/cli/quotaCommand.js +16 -10
- package/dist/audit/cli/quotaCommand.js.map +1 -1
- package/dist/audit/cli/rollingAuditDispatch.d.ts.map +1 -1
- package/dist/audit/cli/rollingAuditDispatch.js +14 -1
- package/dist/audit/cli/rollingAuditDispatch.js.map +1 -1
- package/dist/audit/cli/semanticReviewStep.d.ts +11 -1
- package/dist/audit/cli/semanticReviewStep.d.ts.map +1 -1
- package/dist/audit/cli/semanticReviewStep.js +11 -15
- package/dist/audit/cli/semanticReviewStep.js.map +1 -1
- package/dist/audit/cli/validateCommand.d.ts.map +1 -1
- package/dist/audit/cli/validateCommand.js +3 -3
- package/dist/audit/cli/validateCommand.js.map +1 -1
- package/dist/audit/cli.js +3 -3
- package/dist/audit/cli.js.map +1 -1
- package/dist/audit/contracts/wrapperResponse.d.ts +6 -6
- package/dist/audit/orchestrator/advance.d.ts.map +1 -1
- package/dist/audit/orchestrator/advance.js +39 -7
- package/dist/audit/orchestrator/advance.js.map +1 -1
- package/dist/audit/orchestrator/advanceTypes.d.ts +52 -1
- package/dist/audit/orchestrator/advanceTypes.d.ts.map +1 -1
- package/dist/audit/orchestrator/executorRunners.d.ts.map +1 -1
- package/dist/audit/orchestrator/executorRunners.js +9 -1
- package/dist/audit/orchestrator/executorRunners.js.map +1 -1
- package/dist/audit/orchestrator/intakeExecutors.d.ts +27 -3
- package/dist/audit/orchestrator/intakeExecutors.d.ts.map +1 -1
- package/dist/audit/orchestrator/intakeExecutors.js +119 -19
- package/dist/audit/orchestrator/intakeExecutors.js.map +1 -1
- package/dist/audit/orchestrator/nextStep.d.ts +7 -0
- package/dist/audit/orchestrator/nextStep.d.ts.map +1 -1
- package/dist/audit/orchestrator/nextStep.js +3 -0
- package/dist/audit/orchestrator/nextStep.js.map +1 -1
- package/dist/audit/orchestrator/providerConfirmation.d.ts.map +1 -1
- package/dist/audit/orchestrator/providerConfirmation.js +0 -2
- package/dist/audit/orchestrator/providerConfirmation.js.map +1 -1
- package/dist/audit/orchestrator/rollingDispatch.d.ts.map +1 -1
- package/dist/audit/orchestrator/rollingDispatch.js +2 -0
- package/dist/audit/orchestrator/rollingDispatch.js.map +1 -1
- package/dist/audit/orchestrator/state.d.ts +17 -0
- package/dist/audit/orchestrator/state.d.ts.map +1 -1
- package/dist/audit/orchestrator/state.js +13 -1
- package/dist/audit/orchestrator/state.js.map +1 -1
- package/dist/audit/providers/claudeWorkerProvider.d.ts +10 -0
- package/dist/audit/providers/claudeWorkerProvider.d.ts.map +1 -0
- package/dist/audit/providers/claudeWorkerProvider.js +15 -0
- package/dist/audit/providers/claudeWorkerProvider.js.map +1 -0
- package/dist/audit/providers/index.d.ts.map +1 -1
- package/dist/audit/providers/index.js +2 -0
- package/dist/audit/providers/index.js.map +1 -1
- package/dist/audit/quota/index.d.ts +133 -128
- package/dist/audit/quota/index.d.ts.map +1 -1
- package/dist/audit/supervisor/sessionConfig.d.ts +3 -11
- package/dist/audit/supervisor/sessionConfig.d.ts.map +1 -1
- package/dist/audit/supervisor/sessionConfig.js +3 -25
- package/dist/audit/supervisor/sessionConfig.js.map +1 -1
- package/dist/audit/types/analyzerCapability.d.ts +6 -6
- package/dist/audit/types/externalAnalyzer.d.ts +10 -10
- package/dist/remediate/index.d.ts.map +1 -1
- package/dist/remediate/index.js +2 -2
- package/dist/remediate/index.js.map +1 -1
- package/dist/remediate/phases/plan.d.ts.map +1 -1
- package/dist/remediate/phases/plan.js +5 -4
- package/dist/remediate/phases/plan.js.map +1 -1
- package/dist/remediate/providers/claudeWorkerProvider.d.ts +11 -0
- package/dist/remediate/providers/claudeWorkerProvider.d.ts.map +1 -0
- package/dist/remediate/providers/claudeWorkerProvider.js +16 -0
- package/dist/remediate/providers/claudeWorkerProvider.js.map +1 -0
- package/dist/remediate/providers/index.d.ts.map +1 -1
- package/dist/remediate/providers/index.js +2 -0
- package/dist/remediate/providers/index.js.map +1 -1
- package/dist/remediate/steps/contractPipeline.d.ts.map +1 -1
- package/dist/remediate/steps/contractPipeline.js +9 -3
- package/dist/remediate/steps/contractPipeline.js.map +1 -1
- package/dist/remediate/steps/dispatch/acceptNode.d.ts +1 -1
- package/dist/remediate/steps/dispatch/acceptNode.d.ts.map +1 -1
- package/dist/remediate/steps/dispatch/acceptNode.js.map +1 -1
- package/dist/remediate/steps/dispatch/marshal.d.ts.map +1 -1
- package/dist/remediate/steps/dispatch/marshal.js +5 -4
- package/dist/remediate/steps/dispatch/marshal.js.map +1 -1
- package/dist/remediate/steps/dispatch/waveScheduling.d.ts +4 -2
- package/dist/remediate/steps/dispatch/waveScheduling.d.ts.map +1 -1
- package/dist/remediate/steps/dispatch/waveScheduling.js +33 -85
- package/dist/remediate/steps/dispatch/waveScheduling.js.map +1 -1
- package/dist/remediate/steps/nextStep.d.ts +21 -10
- package/dist/remediate/steps/nextStep.d.ts.map +1 -1
- package/dist/remediate/steps/nextStep.js +45 -25
- package/dist/remediate/steps/nextStep.js.map +1 -1
- package/dist/remediate/steps/sessionConfigLoad.d.ts +46 -0
- package/dist/remediate/steps/sessionConfigLoad.d.ts.map +1 -0
- package/dist/remediate/steps/sessionConfigLoad.js +39 -0
- package/dist/remediate/steps/sessionConfigLoad.js.map +1 -0
- package/dist/shared/config/resolveSessionConfig.d.ts +75 -0
- package/dist/shared/config/resolveSessionConfig.d.ts.map +1 -0
- package/dist/shared/config/resolveSessionConfig.js +125 -0
- package/dist/shared/config/resolveSessionConfig.js.map +1 -0
- package/dist/shared/dispatch/admissionLoop.d.ts +12 -12
- package/dist/shared/dispatch/coordinator.d.ts.map +1 -1
- package/dist/shared/dispatch/coordinator.js +52 -6
- package/dist/shared/dispatch/coordinator.js.map +1 -1
- package/dist/shared/dispatch/hybridDispatch.d.ts +6 -0
- package/dist/shared/dispatch/hybridDispatch.d.ts.map +1 -1
- package/dist/shared/dispatch/hybridDispatch.js +6 -0
- package/dist/shared/dispatch/hybridDispatch.js.map +1 -1
- package/dist/shared/dispatch/providerLaunchFinalize.d.ts +2 -1
- package/dist/shared/dispatch/providerLaunchFinalize.d.ts.map +1 -1
- package/dist/shared/dispatch/providerLaunchFinalize.js +123 -68
- package/dist/shared/dispatch/providerLaunchFinalize.js.map +1 -1
- package/dist/shared/dispatch/rollingDispatch.d.ts +69 -2
- package/dist/shared/dispatch/rollingDispatch.d.ts.map +1 -1
- package/dist/shared/dispatch/rollingDispatch.js +177 -7
- package/dist/shared/dispatch/rollingDispatch.js.map +1 -1
- package/dist/shared/dispatch/unifiedRolling.d.ts +23 -0
- package/dist/shared/dispatch/unifiedRolling.d.ts.map +1 -1
- package/dist/shared/dispatch/unifiedRolling.js +2 -0
- package/dist/shared/dispatch/unifiedRolling.js.map +1 -1
- package/dist/shared/friction/stepBoundaryCapture.d.ts +52 -1
- package/dist/shared/friction/stepBoundaryCapture.d.ts.map +1 -1
- package/dist/shared/friction/stepBoundaryCapture.js +87 -0
- package/dist/shared/friction/stepBoundaryCapture.js.map +1 -1
- package/dist/shared/index.d.ts +20 -10
- package/dist/shared/index.d.ts.map +1 -1
- package/dist/shared/index.js +20 -6
- package/dist/shared/index.js.map +1 -1
- package/dist/shared/io/stateDir.d.ts +27 -0
- package/dist/shared/io/stateDir.d.ts.map +1 -0
- package/dist/shared/io/stateDir.js +40 -0
- package/dist/shared/io/stateDir.js.map +1 -0
- package/dist/shared/providers/auditorSources.d.ts +169 -0
- package/dist/shared/providers/auditorSources.d.ts.map +1 -0
- package/dist/shared/providers/auditorSources.js +453 -0
- package/dist/shared/providers/auditorSources.js.map +1 -0
- package/dist/shared/providers/claudeWorkerProvider.d.ts +71 -0
- package/dist/shared/providers/claudeWorkerProvider.d.ts.map +1 -0
- package/dist/shared/providers/claudeWorkerProvider.js +133 -0
- package/dist/shared/providers/claudeWorkerProvider.js.map +1 -0
- package/dist/shared/providers/providerConfirmation.d.ts +3 -20
- package/dist/shared/providers/providerConfirmation.d.ts.map +1 -1
- package/dist/shared/providers/providerConfirmation.js +27 -40
- package/dist/shared/providers/providerConfirmation.js.map +1 -1
- package/dist/shared/providers/providerFactory.d.ts +13 -1
- package/dist/shared/providers/providerFactory.d.ts.map +1 -1
- package/dist/shared/providers/providerFactory.js +12 -0
- package/dist/shared/providers/providerFactory.js.map +1 -1
- package/dist/shared/providers/proxyCatalog.d.ts +95 -0
- package/dist/shared/providers/proxyCatalog.d.ts.map +1 -0
- package/dist/shared/providers/proxyCatalog.js +426 -0
- package/dist/shared/providers/proxyCatalog.js.map +1 -0
- package/dist/shared/providers/sharedProviderConfirmation.d.ts +264 -75
- package/dist/shared/providers/sharedProviderConfirmation.d.ts.map +1 -1
- package/dist/shared/providers/sharedProviderConfirmation.js +534 -113
- package/dist/shared/providers/sharedProviderConfirmation.js.map +1 -1
- package/dist/shared/quota/apiPool.d.ts +34 -11
- package/dist/shared/quota/apiPool.d.ts.map +1 -1
- package/dist/shared/quota/apiPool.js +52 -23
- package/dist/shared/quota/apiPool.js.map +1 -1
- package/dist/shared/quota/capacity.d.ts +79 -50
- package/dist/shared/quota/capacity.d.ts.map +1 -1
- package/dist/shared/quota/capacity.js +40 -5
- package/dist/shared/quota/capacity.js.map +1 -1
- package/dist/shared/quota/errorParsing.d.ts +39 -0
- package/dist/shared/quota/errorParsing.d.ts.map +1 -1
- package/dist/shared/quota/errorParsing.js +76 -0
- package/dist/shared/quota/errorParsing.js.map +1 -1
- package/dist/shared/quota/hostPool.d.ts +107 -0
- package/dist/shared/quota/hostPool.d.ts.map +1 -0
- package/dist/shared/quota/hostPool.js +103 -0
- package/dist/shared/quota/hostPool.js.map +1 -0
- package/dist/shared/quota/quotaSource.d.ts +2 -2
- package/dist/shared/types/auditorDescriptor.d.ts +120 -0
- package/dist/shared/types/auditorDescriptor.d.ts.map +1 -0
- package/dist/shared/types/auditorDescriptor.js +28 -0
- package/dist/shared/types/auditorDescriptor.js.map +1 -0
- package/dist/shared/types/providerConfirmation.d.ts +64 -6
- package/dist/shared/types/providerConfirmation.d.ts.map +1 -1
- package/dist/shared/types/providerConfirmation.js +1 -1
- package/dist/shared/types/rollingDispatch.d.ts +45 -1
- package/dist/shared/types/rollingDispatch.d.ts.map +1 -1
- package/dist/shared/types/sessionConfig.d.ts +149 -67
- package/dist/shared/types/sessionConfig.d.ts.map +1 -1
- package/dist/shared/types/sessionConfig.js +87 -2
- package/dist/shared/types/sessionConfig.js.map +1 -1
- package/dist/shared/validation/sessionConfig.d.ts +26 -14
- package/dist/shared/validation/sessionConfig.d.ts.map +1 -1
- package/dist/shared/validation/sessionConfig.js +95 -69
- package/dist/shared/validation/sessionConfig.js.map +1 -1
- package/docs/audit-pkg/contracts.md +4 -4
- package/docs/audit-pkg/operator-guide.md +43 -24
- package/package.json +1 -1
- package/skills/audit-code/SKILL.md +17 -14
- package/skills/audit-code/audit-code.prompt.md +17 -16
- package/wrapper/audit-code-wrapper-install-hosts.mjs +6 -10
- package/wrapper/remediate-code-wrapper-install-hosts.mjs +4 -8
- package/dist/shared/quota/repairProxyRegistry.d.ts +0 -84
- package/dist/shared/quota/repairProxyRegistry.d.ts.map +0 -1
- package/dist/shared/quota/repairProxyRegistry.js +0 -144
- package/dist/shared/quota/repairProxyRegistry.js.map +0 -1
|
@@ -1,6 +1,103 @@
|
|
|
1
|
-
import { detectRateLimitFromChannel, detectCreditExhaustionFromChannel, detectQuotaSuspicious } from "../quota/errorParsing.js";
|
|
1
|
+
import { detectRateLimitFromChannel, detectCreditExhaustionFromChannel, detectModelUnavailableFromChannel, detectRequestTooLargeFromChannel, detectQuotaSuspicious, } from "../quota/errorParsing.js";
|
|
2
2
|
import { appendTokenUsageLine } from "../io/tokenUsageLedger.js";
|
|
3
3
|
import { readOptionalJsonFile, readOptionalTextFile } from "../io/json.js";
|
|
4
|
+
/**
|
|
5
|
+
* Classify a worker's failure text by scanning stderr and stdout in order for known
|
|
6
|
+
* quota/API-error patterns. Reused identically by both not-accepted (exit ≠ 0) and
|
|
7
|
+
* accepted-but-no-result-file branches so they cannot drift.
|
|
8
|
+
*
|
|
9
|
+
* **Scan order (load-bearing — F4 requires stderr request-too-large BEFORE rate-limit
|
|
10
|
+
* so a combined "413 ... retry rate limit" cannot cooldown-poison a healthy pool):**
|
|
11
|
+
* 1. stderr: credit-exhaustion (permanent, no reset timer)
|
|
12
|
+
* 2. stderr: request-too-large (per-packet sizing fault, no cooldown)
|
|
13
|
+
* 3. stderr: model-unavailable (404, permanent pool exclusion)
|
|
14
|
+
* 4. stderr: rate-limit (transient 429)
|
|
15
|
+
* 5. stdout: credit-exhaustion
|
|
16
|
+
* 6. stdout: request-too-large
|
|
17
|
+
* 7. stdout: model-unavailable
|
|
18
|
+
* 8. stdout: rate-limit
|
|
19
|
+
* 9. TIER 2 `detectQuotaSuspicious` over both channels combined (fallback pre-filter)
|
|
20
|
+
*
|
|
21
|
+
* Returns the corresponding RollingDispatchResult fragment or null if no match.
|
|
22
|
+
*/
|
|
23
|
+
function classifyFailureChannels(packet, stderrText, stdoutText) {
|
|
24
|
+
// Stderr TIER 1: credit exhaustion (permanent, checked first — must never fall through to rate-limit)
|
|
25
|
+
const stderrCreditCheck = detectCreditExhaustionFromChannel("error", stderrText);
|
|
26
|
+
if (stderrCreditCheck.isCreditExhausted) {
|
|
27
|
+
return {
|
|
28
|
+
outcome: "credit_exhausted",
|
|
29
|
+
creditExhaustion: { channel: "error", text: stderrText, rawMatch: stderrCreditCheck.rawMatch },
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
// Stderr TIER 1: request-too-large (F4 — must be checked BEFORE rate-limit to prevent cooldown-poisoning)
|
|
33
|
+
const stderrTooLargeCheck = detectRequestTooLargeFromChannel("error", stderrText);
|
|
34
|
+
if (stderrTooLargeCheck.isRequestTooLarge) {
|
|
35
|
+
return {
|
|
36
|
+
outcome: "packet_too_large",
|
|
37
|
+
packetTooLarge: { channel: "error", text: stderrText, rawMatch: stderrTooLargeCheck.rawMatch },
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
// Stderr TIER 1: model-unavailable (permanent pool exclusion, checked before rate-limit)
|
|
41
|
+
const stderrModelUnavailableCheck = detectModelUnavailableFromChannel("error", stderrText);
|
|
42
|
+
if (stderrModelUnavailableCheck.isModelUnavailable) {
|
|
43
|
+
return {
|
|
44
|
+
outcome: "model_unavailable",
|
|
45
|
+
modelUnavailable: { channel: "error", text: stderrText, rawMatch: stderrModelUnavailableCheck.rawMatch },
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
// Stderr TIER 1: rate-limit (transient, applies cooldown)
|
|
49
|
+
const stderrLimitCheck = detectRateLimitFromChannel("error", stderrText);
|
|
50
|
+
if (stderrLimitCheck.isRateLimited) {
|
|
51
|
+
return {
|
|
52
|
+
outcome: "rate_limited",
|
|
53
|
+
rateLimit: { channel: "error", text: stderrText },
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
// Stdout TIER 1: credit exhaustion (some providers report to stdout instead)
|
|
57
|
+
const stdoutCreditCheck = detectCreditExhaustionFromChannel("status", stdoutText);
|
|
58
|
+
if (stdoutCreditCheck.isCreditExhausted) {
|
|
59
|
+
return {
|
|
60
|
+
outcome: "credit_exhausted",
|
|
61
|
+
creditExhaustion: { channel: "status", text: stdoutText, rawMatch: stdoutCreditCheck.rawMatch },
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
// Stdout TIER 1: request-too-large
|
|
65
|
+
const stdoutTooLargeCheck = detectRequestTooLargeFromChannel("status", stdoutText);
|
|
66
|
+
if (stdoutTooLargeCheck.isRequestTooLarge) {
|
|
67
|
+
return {
|
|
68
|
+
outcome: "packet_too_large",
|
|
69
|
+
packetTooLarge: { channel: "status", text: stdoutText, rawMatch: stdoutTooLargeCheck.rawMatch },
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
// Stdout TIER 1: model-unavailable
|
|
73
|
+
const stdoutModelUnavailableCheck = detectModelUnavailableFromChannel("status", stdoutText);
|
|
74
|
+
if (stdoutModelUnavailableCheck.isModelUnavailable) {
|
|
75
|
+
return {
|
|
76
|
+
outcome: "model_unavailable",
|
|
77
|
+
modelUnavailable: { channel: "status", text: stdoutText, rawMatch: stdoutModelUnavailableCheck.rawMatch },
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
// Stdout TIER 1: rate-limit
|
|
81
|
+
const stdoutLimitCheck = detectRateLimitFromChannel("status", stdoutText);
|
|
82
|
+
if (stdoutLimitCheck.isRateLimited) {
|
|
83
|
+
return {
|
|
84
|
+
outcome: "rate_limited",
|
|
85
|
+
rateLimit: { channel: "status", text: stdoutText },
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
// TIER 2 (Slice A2b): broad pre-filter on combined channels (fallback for unclassified quota-suspicious text)
|
|
89
|
+
const combinedText = [stderrText, stdoutText].filter((t) => t.length > 0).join("\n");
|
|
90
|
+
if (combinedText.length > 0 && detectQuotaSuspicious(combinedText)) {
|
|
91
|
+
return {
|
|
92
|
+
outcome: "quota_unclassified",
|
|
93
|
+
quotaUnclassified: {
|
|
94
|
+
channel: stderrText.length > 0 ? "error" : "status",
|
|
95
|
+
text: combinedText,
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
4
101
|
/**
|
|
5
102
|
* The shared launch-result finalize tail run by BOTH per-packet dispatchers
|
|
6
103
|
* (audit `makeAuditProviderPacketDispatcher`, remediate `makeProviderNodeDispatcher`)
|
|
@@ -11,7 +108,8 @@ import { readOptionalJsonFile, readOptionalTextFile } from "../io/json.js";
|
|
|
11
108
|
* read-only-vs-git-mutating / AuditResult-vs-ImplementWorkerResult divergences.
|
|
12
109
|
*
|
|
13
110
|
* In order:
|
|
14
|
-
* 1. not-accepted →
|
|
111
|
+
* 1. not-accepted → classify the failure text via the shared `classifyFailureChannels`
|
|
112
|
+
* helper (mirrors the accepted-but-no-result-file branch so both cannot drift);
|
|
15
113
|
* 2. channel-isolated credit-exhaustion (TIER 1) then session-limit (TIER 1)
|
|
16
114
|
* detection on stderr (CE-003) → credit_exhausted / non-consuming
|
|
17
115
|
* rate_limited re-queue (the result file is never scanned for these strings);
|
|
@@ -29,40 +127,31 @@ import { readOptionalJsonFile, readOptionalTextFile } from "../io/json.js";
|
|
|
29
127
|
*/
|
|
30
128
|
export async function finalizeProviderLaunchResult(launch, ctx) {
|
|
31
129
|
const { packet } = ctx;
|
|
130
|
+
// Read both channels for classification (reused by both not-accepted and accepted-but-no-result branches)
|
|
131
|
+
//
|
|
132
|
+
// DELIBERATE semantic change (2026-07-17, gap-fix lap): an ACCEPTED launch whose
|
|
133
|
+
// result file landed + parses is a SUCCESS even if a channel carries limit text —
|
|
134
|
+
// an agentic worker that retried through transient 429s and still delivered must
|
|
135
|
+
// not have its completed work discarded and re-run (the old order scanned stderr
|
|
136
|
+
// before the result read and re-queued such packets). Channel classification now
|
|
137
|
+
// applies only when the launch was rejected or no result landed; result CONTENT
|
|
138
|
+
// is still never scanned (CE-003) and is adjudicated by the merge downstream.
|
|
139
|
+
const stderrText = (await readOptionalTextFile(ctx.stderrPath)) ?? "";
|
|
140
|
+
const stdoutText = (await readOptionalTextFile(ctx.stdoutPath)) ?? "";
|
|
141
|
+
// The not-accepted branch NOW runs the channel classifier BEFORE returning error.
|
|
142
|
+
// This fixes the dogfood gap: a nonzero-exit worker's 429/404/413 text is now scanned.
|
|
32
143
|
if (!launch.accepted) {
|
|
144
|
+
const classification = classifyFailureChannels(packet, stderrText, stdoutText);
|
|
145
|
+
if (classification) {
|
|
146
|
+
return { packet, ...classification };
|
|
147
|
+
}
|
|
148
|
+
// No classification matched — return the original error
|
|
33
149
|
return {
|
|
34
150
|
packet,
|
|
35
151
|
outcome: "error",
|
|
36
152
|
error: new Error(launch.error ?? `provider ${ctx.providerName} rejected ${ctx.entityLabel}`),
|
|
37
153
|
};
|
|
38
154
|
}
|
|
39
|
-
// Channel-isolated session-limit detection (CE-003): only the error/status channel
|
|
40
|
-
// (stderr) is inspected — the result file is never scanned for limit strings, so a
|
|
41
|
-
// healthy result quoting a limit never triggers a re-queue.
|
|
42
|
-
const stderrText = (await readOptionalTextFile(ctx.stderrPath)) ?? "";
|
|
43
|
-
// Credit exhaustion (out of prepaid usage credits) is checked FIRST and is
|
|
44
|
-
// architecturally distinct from a rate limit: it carries no reset time, so a
|
|
45
|
-
// positive match here must never fall through to the resettable rate_limited
|
|
46
|
-
// path below. Same channel isolation (CE-003) applies — only stderr, never the
|
|
47
|
-
// consumed result file.
|
|
48
|
-
const creditCheck = detectCreditExhaustionFromChannel("error", stderrText);
|
|
49
|
-
if (creditCheck.isCreditExhausted) {
|
|
50
|
-
return {
|
|
51
|
-
packet,
|
|
52
|
-
outcome: "credit_exhausted",
|
|
53
|
-
creditExhaustion: { channel: "error", text: stderrText, rawMatch: creditCheck.rawMatch },
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
const limitCheck = detectRateLimitFromChannel("error", stderrText);
|
|
57
|
-
if (limitCheck.isRateLimited) {
|
|
58
|
-
// Non-consuming re-queue: the rolling engine drops the provider and puts this
|
|
59
|
-
// packet back into the pending pool so it retries once the cooldown passes.
|
|
60
|
-
return {
|
|
61
|
-
packet,
|
|
62
|
-
outcome: "rate_limited",
|
|
63
|
-
rateLimit: { channel: "error", text: stderrText },
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
155
|
// Reactive cost verification: relay the endpoint-reported cost (when the provider
|
|
67
156
|
// surfaced one) so `handleResult` can demote a declared-free pool that started
|
|
68
157
|
// charging. Absent for providers that report no cost.
|
|
@@ -71,45 +160,11 @@ export async function finalizeProviderLaunchResult(launch, ctx) {
|
|
|
71
160
|
// Contents are adjudicated by the deterministic merge downstream.
|
|
72
161
|
const result = await readOptionalJsonFile(ctx.resultPath);
|
|
73
162
|
if (!result) {
|
|
74
|
-
//
|
|
75
|
-
//
|
|
76
|
-
const
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
return {
|
|
80
|
-
packet,
|
|
81
|
-
outcome: "credit_exhausted",
|
|
82
|
-
creditExhaustion: { channel: "status", text: stdoutText, rawMatch: stdoutCreditCheck.rawMatch },
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
|
-
const stdoutLimitCheck = detectRateLimitFromChannel("status", stdoutText);
|
|
86
|
-
if (stdoutLimitCheck.isRateLimited) {
|
|
87
|
-
return {
|
|
88
|
-
packet,
|
|
89
|
-
outcome: "rate_limited",
|
|
90
|
-
rateLimit: { channel: "status", text: stdoutText },
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
// TIER 2 (Slice A2b): neither channel matched a PRECISE credit/rate-limit
|
|
94
|
-
// pattern. Before falling through to a silent, unclassified raw `error` —
|
|
95
|
-
// the exact "worker AND dispatcher both die raw" failure mode
|
|
96
|
-
// credit-exhaustion detection fixed, but only for text it recognizes — run
|
|
97
|
-
// the deliberately broad `detectQuotaSuspicious` pre-filter across BOTH
|
|
98
|
-
// channels combined. A positive match here degrades CONSERVATIVELY
|
|
99
|
-
// (re-queue with a reversible cooldown, the pool is never permanently
|
|
100
|
-
// excluded — see the `quota_unclassified` branch in rollingDispatch.ts) and
|
|
101
|
-
// carries the verbatim text so the operator can classify it and improve the
|
|
102
|
-
// pattern set, instead of a silent death with no signal at all.
|
|
103
|
-
const combinedText = [stderrText, stdoutText].filter((t) => t.length > 0).join("\n");
|
|
104
|
-
if (combinedText.length > 0 && detectQuotaSuspicious(combinedText)) {
|
|
105
|
-
return {
|
|
106
|
-
packet,
|
|
107
|
-
outcome: "quota_unclassified",
|
|
108
|
-
quotaUnclassified: {
|
|
109
|
-
channel: stderrText.length > 0 ? "error" : "status",
|
|
110
|
-
text: combinedText,
|
|
111
|
-
},
|
|
112
|
-
};
|
|
163
|
+
// No result file landed — reuse the same channel classifier so both branches cannot drift.
|
|
164
|
+
// Channel isolation (CE-003) is preserved: result channel is never scanned.
|
|
165
|
+
const classification = classifyFailureChannels(packet, stderrText, stdoutText);
|
|
166
|
+
if (classification) {
|
|
167
|
+
return { packet, ...classification };
|
|
113
168
|
}
|
|
114
169
|
return {
|
|
115
170
|
packet,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providerLaunchFinalize.js","sourceRoot":"","sources":["../../../src/shared/dispatch/providerLaunchFinalize.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"providerLaunchFinalize.js","sourceRoot":"","sources":["../../../src/shared/dispatch/providerLaunchFinalize.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,0BAA0B,EAC1B,iCAAiC,EACjC,iCAAiC,EACjC,gCAAgC,EAChC,qBAAqB,GACtB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AA4B3E;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,uBAAuB,CAC9B,MAAsC,EACtC,UAAkB,EAClB,UAAkB;IAElB,sGAAsG;IACtG,MAAM,iBAAiB,GAAG,iCAAiC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACjF,IAAI,iBAAiB,CAAC,iBAAiB,EAAE,CAAC;QACxC,OAAO;YACL,OAAO,EAAE,kBAAkB;YAC3B,gBAAgB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,EAAE;SAC/F,CAAC;IACJ,CAAC;IAED,0GAA0G;IAC1G,MAAM,mBAAmB,GAAG,gCAAgC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAClF,IAAI,mBAAmB,CAAC,iBAAiB,EAAE,CAAC;QAC1C,OAAO;YACL,OAAO,EAAE,kBAAkB;YAC3B,cAAc,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,EAAE;SAC/F,CAAC;IACJ,CAAC;IAED,yFAAyF;IACzF,MAAM,2BAA2B,GAAG,iCAAiC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAC3F,IAAI,2BAA2B,CAAC,kBAAkB,EAAE,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,mBAAmB;YAC5B,gBAAgB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,2BAA2B,CAAC,QAAQ,EAAE;SACzG,CAAC;IACJ,CAAC;IAED,0DAA0D;IAC1D,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACzE,IAAI,gBAAgB,CAAC,aAAa,EAAE,CAAC;QACnC,OAAO;YACL,OAAO,EAAE,cAAc;YACvB,SAAS,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE;SAClD,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,MAAM,iBAAiB,GAAG,iCAAiC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAClF,IAAI,iBAAiB,CAAC,iBAAiB,EAAE,CAAC;QACxC,OAAO;YACL,OAAO,EAAE,kBAAkB;YAC3B,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,EAAE;SAChG,CAAC;IACJ,CAAC;IAED,mCAAmC;IACnC,MAAM,mBAAmB,GAAG,gCAAgC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACnF,IAAI,mBAAmB,CAAC,iBAAiB,EAAE,CAAC;QAC1C,OAAO;YACL,OAAO,EAAE,kBAAkB;YAC3B,cAAc,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,EAAE;SAChG,CAAC;IACJ,CAAC;IAED,mCAAmC;IACnC,MAAM,2BAA2B,GAAG,iCAAiC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC5F,IAAI,2BAA2B,CAAC,kBAAkB,EAAE,CAAC;QACnD,OAAO;YACL,OAAO,EAAE,mBAAmB;YAC5B,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,2BAA2B,CAAC,QAAQ,EAAE;SAC1G,CAAC;IACJ,CAAC;IAED,4BAA4B;IAC5B,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC1E,IAAI,gBAAgB,CAAC,aAAa,EAAE,CAAC;QACnC,OAAO;YACL,OAAO,EAAE,cAAc;YACvB,SAAS,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE;SACnD,CAAC;IACJ,CAAC;IAED,8GAA8G;IAC9G,MAAM,YAAY,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,qBAAqB,CAAC,YAAY,CAAC,EAAE,CAAC;QACnE,OAAO;YACL,OAAO,EAAE,oBAAoB;YAC7B,iBAAiB,EAAE;gBACjB,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;gBACnD,IAAI,EAAE,YAAY;aACnB;SACF,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,MAAgC,EAChC,GAA2C;IAE3C,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC;IAEvB,0GAA0G;IAC1G,EAAE;IACF,iFAAiF;IACjF,kFAAkF;IAClF,iFAAiF;IACjF,iFAAiF;IACjF,iFAAiF;IACjF,gFAAgF;IAChF,8EAA8E;IAC9E,MAAM,UAAU,GAAG,CAAC,MAAM,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IACtE,MAAM,UAAU,GAAG,CAAC,MAAM,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAEtE,kFAAkF;IAClF,uFAAuF;IACvF,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,cAAc,GAAG,uBAAuB,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/E,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,CAAC;QACvC,CAAC;QACD,wDAAwD;QACxD,OAAO;YACL,MAAM;YACN,OAAO,EAAE,OAAO;YAChB,KAAK,EAAE,IAAI,KAAK,CACd,MAAM,CAAC,KAAK,IAAI,YAAY,GAAG,CAAC,YAAY,aAAa,GAAG,CAAC,WAAW,EAAE,CAC3E;SACF,CAAC;IACJ,CAAC;IAED,kFAAkF;IAClF,+EAA+E;IAC/E,sDAAsD;IACtD,MAAM,YAAY,GAChB,MAAM,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpF,kFAAkF;IAClF,kEAAkE;IAClE,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAU,GAAG,CAAC,UAAU,CAAC,CAAC;IACnE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,2FAA2F;QAC3F,4EAA4E;QAC5E,MAAM,cAAc,GAAG,uBAAuB,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/E,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,CAAC;QACvC,CAAC;QAED,OAAO;YACL,MAAM;YACN,OAAO,EAAE,OAAO;YAChB,KAAK,EAAE,IAAI,KAAK,CACd,cAAc,GAAG,CAAC,WAAW,uBAAuB,GAAG,CAAC,UAAU,EAAE,CACrE;SACF,CAAC;IACJ,CAAC;IAED,kFAAkF;IAClF,gFAAgF;IAChF,0EAA0E;IAC1E,gFAAgF;IAChF,qEAAqE;IACrE,MAAM,oBAAoB,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE;QACtD,SAAS,EAAE,GAAG,CAAC,QAAQ;QACvB,OAAO,EAAE,GAAG,CAAC,MAAM;QACnB,YAAY,EAAE,MAAM,CAAC,aAAa,EAAE,WAAW,IAAI,IAAI;QACvD,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,YAAY,IAAI,IAAI;QACzD,iBAAiB,EAAE,MAAM,CAAC,aAAa,EAAE,eAAe,IAAI,IAAI;QAChE,qBAAqB,EAAE,MAAM,CAAC,aAAa,EAAE,mBAAmB,IAAI,IAAI;QACxE,iBAAiB,EAAE,MAAM,CAAC,eAAe,IAAI,IAAI;KAClD,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,YAAY,EAAE,CAAC;AACzD,CAAC"}
|
|
@@ -81,7 +81,7 @@ export interface RollingDispatchPacket<TPacket> {
|
|
|
81
81
|
/** Outcome of dispatching a single packet. */
|
|
82
82
|
export interface RollingDispatchResult<TPacket> {
|
|
83
83
|
packet: RollingDispatchPacket<TPacket>;
|
|
84
|
-
outcome: "success" | "rate_limited" | "timeout" | "error" | "credit_exhausted" | "quota_unclassified";
|
|
84
|
+
outcome: "success" | "rate_limited" | "timeout" | "error" | "credit_exhausted" | "model_unavailable" | "packet_too_large" | "quota_unclassified";
|
|
85
85
|
/** Actual tokens consumed, if the provider reports it. */
|
|
86
86
|
actualTokens?: number;
|
|
87
87
|
/**
|
|
@@ -129,6 +129,28 @@ export interface RollingDispatchResult<TPacket> {
|
|
|
129
129
|
text: string;
|
|
130
130
|
rawMatch: string | null;
|
|
131
131
|
};
|
|
132
|
+
/**
|
|
133
|
+
* The worker ERROR/STATUS channel evidence that classified a `model_unavailable`
|
|
134
|
+
* outcome (HTTP 404, model not found — permanent pool exclusion, the model is not
|
|
135
|
+
* served by this provider). Carried so the consumer's `onModelUnavailable` hook can
|
|
136
|
+
* surface it as reviewable friction. Absent on non-model_unavailable outcomes.
|
|
137
|
+
*/
|
|
138
|
+
modelUnavailable?: {
|
|
139
|
+
channel: WorkerOutputChannel;
|
|
140
|
+
text: string;
|
|
141
|
+
rawMatch: string | null;
|
|
142
|
+
};
|
|
143
|
+
/**
|
|
144
|
+
* The worker ERROR/STATUS channel evidence that classified a `packet_too_large`
|
|
145
|
+
* outcome (HTTP 413, request/payload too large — a per-packet sizing fault for
|
|
146
|
+
* this particular pool). Carried so the consumer's `onPacketTooLarge` hook can
|
|
147
|
+
* surface it as reviewable friction. Absent on non-packet_too_large outcomes.
|
|
148
|
+
*/
|
|
149
|
+
packetTooLarge?: {
|
|
150
|
+
channel: WorkerOutputChannel;
|
|
151
|
+
text: string;
|
|
152
|
+
rawMatch: string | null;
|
|
153
|
+
};
|
|
132
154
|
/**
|
|
133
155
|
* The worker ERROR/STATUS channel VERBATIM evidence that classified a
|
|
134
156
|
* `quota_unclassified` outcome (Slice A2b, TIER 2): the broad
|
|
@@ -224,6 +246,24 @@ export interface RollingDispatchState<TPacket> {
|
|
|
224
246
|
* documented follow-up (docs/backlog.md) — bounded efficiency only, not a safety gap.
|
|
225
247
|
*/
|
|
226
248
|
quotaUnclassifiedPoolIds: Set<string>;
|
|
249
|
+
/**
|
|
250
|
+
* Pool ids permanently excluded after a `model_unavailable` result (HTTP 404, the
|
|
251
|
+
* model is not served by this provider). Monotonic within a run — a pool is never
|
|
252
|
+
* re-added — mirrors `exhaustedPoolIds` behavior for credit exhaustion but carries
|
|
253
|
+
* availability semantics instead of billing semantics. Unlike exhausted-pool
|
|
254
|
+
* re-routing (a timed backoff), a 404 has no reset condition: once a model is
|
|
255
|
+
* unavailable, the pool is gone for the run.
|
|
256
|
+
*/
|
|
257
|
+
modelUnavailablePoolIds: Set<string>;
|
|
258
|
+
/**
|
|
259
|
+
* Packet id → set of pool ids that 413'd this packet. When a `packet_too_large`
|
|
260
|
+
* result lands, a pool is never permanently excluded from the whole run (unlike
|
|
261
|
+
* credit_exhausted or model_unavailable); instead, ONLY THIS PACKET is skipped for
|
|
262
|
+
* THIS POOL on the next selection pass. Other packets freely bind to the pool, and
|
|
263
|
+
* this packet retries on any other pool. Discriminator is (packetId, poolId) — each
|
|
264
|
+
* distinct pair is a separate sizing fault signal.
|
|
265
|
+
*/
|
|
266
|
+
oversizedPacketPools: Map<string, Set<string>>;
|
|
227
267
|
}
|
|
228
268
|
/** Consumer-provided configuration for the rolling dispatcher. */
|
|
229
269
|
export interface RollingDispatchConfig<TPacket> {
|
|
@@ -266,6 +306,33 @@ export interface RollingDispatchConfig<TPacket> {
|
|
|
266
306
|
poolId: string;
|
|
267
307
|
rawMatch: string | null;
|
|
268
308
|
}) => void;
|
|
309
|
+
/**
|
|
310
|
+
* Model unavailable (HTTP 404, model not found — permanent pool exclusion):
|
|
311
|
+
* invoked once per pool the FIRST time a `model_unavailable` result lands for it —
|
|
312
|
+
* the model is not served by this provider and the pool is permanently excluded from
|
|
313
|
+
* this run's admissible set (monotonic `modelUnavailablePoolIds` — never a timed
|
|
314
|
+
* cooldown). The consumer wires it to friction emission. Best-effort — a throwing
|
|
315
|
+
* hook never aborts dispatch. Omit to leave the exclusion silent (no friction
|
|
316
|
+
* emitted, exclusion still happens).
|
|
317
|
+
*/
|
|
318
|
+
onModelUnavailable?: (info: {
|
|
319
|
+
poolId: string;
|
|
320
|
+
rawMatch: string | null;
|
|
321
|
+
}) => void;
|
|
322
|
+
/**
|
|
323
|
+
* Packet too large (HTTP 413, payload/request too large — per-packet sizing fault):
|
|
324
|
+
* invoked every time a `packet_too_large` result lands — the pool is NOT permanently
|
|
325
|
+
* excluded (unlike model_unavailable or credit_exhausted); only THIS PACKET is
|
|
326
|
+
* skipped for THIS POOL on re-selection. Each (packet,pool) pair is distinct signal,
|
|
327
|
+
* so the hook fires per distinct pair, not per pool. The consumer wires it to
|
|
328
|
+
* friction emission. Best-effort — a throwing hook never aborts dispatch. Omit to
|
|
329
|
+
* leave the re-queue silent (no friction, re-queue still happens).
|
|
330
|
+
*/
|
|
331
|
+
onPacketTooLarge?: (info: {
|
|
332
|
+
poolId: string;
|
|
333
|
+
packetId: string;
|
|
334
|
+
rawMatch: string | null;
|
|
335
|
+
}) => void;
|
|
269
336
|
/**
|
|
270
337
|
* Quota-unclassified harvest (Slice A2b, TIER 2): invoked once per pool the
|
|
271
338
|
* FIRST time a `quota_unclassified` result lands for it — a worker death whose
|
|
@@ -417,7 +484,7 @@ export declare function scorePacketComplexity<TPacket>(packet: RollingDispatchPa
|
|
|
417
484
|
*
|
|
418
485
|
* Returns null if no eligible pool currently has quota headroom.
|
|
419
486
|
*/
|
|
420
|
-
export declare function selectProvider<TPacket>(packet: RollingDispatchPacket<TPacket>, confirmedPools: CapacityPool[], inFlightTracker: InFlightTokenTracker, quotaStateEntries: Record<string, QuotaStateEntry>, sessionConfig: SessionConfig, exhaustedPoolIds?: ReadonlySet<string>, pausedPoolResetAt?: ReadonlyMap<string, number>, costDemotedPoolIds?: ReadonlySet<string>, now?: number): ProviderSlot | null;
|
|
487
|
+
export declare function selectProvider<TPacket>(packet: RollingDispatchPacket<TPacket>, confirmedPools: CapacityPool[], inFlightTracker: InFlightTokenTracker, quotaStateEntries: Record<string, QuotaStateEntry>, sessionConfig: SessionConfig, exhaustedPoolIds?: ReadonlySet<string>, pausedPoolResetAt?: ReadonlyMap<string, number>, costDemotedPoolIds?: ReadonlySet<string>, oversizedPacketPools?: ReadonlyMap<string, ReadonlySet<string>>, now?: number): ProviderSlot | null;
|
|
421
488
|
/**
|
|
422
489
|
* Create a rolling dispatcher for the given configuration.
|
|
423
490
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rollingDispatch.d.ts","sourceRoot":"","sources":["../../../src/shared/dispatch/rollingDispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,KAAK,EAAE,YAAY,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"rollingDispatch.d.ts","sourceRoot":"","sources":["../../../src/shared/dispatch/rollingDispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,KAAK,EAAE,YAAY,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AAEpF,OAAO,KAAK,EAAE,eAAe,EAA4B,MAAM,mBAAmB,CAAC;AAcnF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAEpE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AASvE,oEAAoE;AACpE,MAAM,WAAW,qBAAqB,CAAC,OAAO;IAC5C,8DAA8D;IAC9D,EAAE,EAAE,MAAM,CAAC;IACX,gEAAgE;IAChE,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,8CAA8C;AAC9C,MAAM,WAAW,qBAAqB,CAAC,OAAO;IAC5C,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACvC,OAAO,EACH,SAAS,GACT,cAAc,GACd,SAAS,GACT,OAAO,GACP,kBAAkB,GAClB,mBAAmB,GACnB,kBAAkB,GAClB,oBAAoB,CAAC;IACzB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE;QACV,OAAO,EAAE,mBAAmB,CAAC;QAC7B,IAAI,EAAE,MAAM,CAAC;QACb;;;;;;;;;WASG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE;QACjB,OAAO,EAAE,mBAAmB,CAAC;QAC7B,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;KACzB,CAAC;IACF;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE;QACjB,OAAO,EAAE,mBAAmB,CAAC;QAC7B,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;KACzB,CAAC;IACF;;;;;OAKG;IACH,cAAc,CAAC,EAAE;QACf,OAAO,EAAE,mBAAmB,CAAC;QAC7B,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;KACzB,CAAC;IACF;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE;QAClB,OAAO,EAAE,mBAAmB,CAAC;QAC7B,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;CACH;AAED,mEAAmE;AACnE,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wDAAwD;AACxD,MAAM,WAAW,aAAa,CAAC,OAAO;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACvC,YAAY,EAAE,YAAY,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IACjD;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+CAA+C;AAC/C,MAAM,WAAW,oBAAoB,CAAC,OAAO;IAC3C,YAAY,EAAE,qBAAqB,CAAC,OAAO,CAAC,EAAE,CAAC;IAC/C,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9C,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1B;;;;;OAKG;IACH,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B;;;;;;;;OAQG;IACH,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC;;;;OAIG;IACH,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB;;;;;;;;;;;OAWG;IACH,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC;;;;;;;;;;;;;;;;OAgBG;IACH,wBAAwB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtC;;;;;;;OAOG;IACH,uBAAuB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC;;;;;;;OAOG;IACH,oBAAoB,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;CAChD;AAED,kEAAkE;AAClE,MAAM,WAAW,qBAAqB,CAAC,OAAO;IAC5C,qDAAqD;IACrD,cAAc,EAAE,YAAY,EAAE,CAAC;IAC/B,aAAa,EAAE,aAAa,CAAC;IAC7B;;;OAGG;IACH,cAAc,EAAE,CACd,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,EACtC,IAAI,EAAE,YAAY,KACf,OAAO,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,uFAAuF;IACvF,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC;IAC5D;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,KAAK,IAAI,CAAC;IACX;;;;;;;;;OASG;IACH,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,KAAK,IAAI,CAAC;IAChF;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,KAAK,IAAI,CAAC;IACjF;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,CAAC,IAAI,EAAE;QACxB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;KACzB,KAAK,IAAI,CAAC;IACX;;;;;;;;;;OAUG;IACH,mBAAmB,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACvE;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC;;;;;;;;;;OAUG;IACH,iBAAiB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;IAClD;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,CAChB,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,EACtC,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,KACnC,IAAI,CAAC;IACV;;;;;;;;;;;OAWG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IAC/C;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,CACzB,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,EACtC,MAAM,EAAE,MAAM,KACX,MAAM,CAAC;CACb;AAED,6EAA6E;AAC7E,MAAM,WAAW,sBAAsB;IACrC;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB,CAAC,OAAO;IACxC;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC;IACzD;;;;OAIG;IACH,GAAG,IAAI,OAAO,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACjD,sDAAsD;IACtD,QAAQ,IAAI,QAAQ,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;IACpD;;;;;OAKG;IACH,WAAW,IAAI,yBAAyB,GAAG,IAAI,CAAC;CACjD;AAMD;;;GAGG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAE1D,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAKtD,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAKrD,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;CAG1C;AAMD,6EAA6E;AAC7E,wBAAgB,qBAAqB,CAAC,OAAO,EAC3C,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,GACrC,MAAM,CAER;AAoDD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,cAAc,CAAC,OAAO,EACpC,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,EACtC,cAAc,EAAE,YAAY,EAAE,EAC9B,eAAe,EAAE,oBAAoB,EACrC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,EAClD,aAAa,EAAE,aAAa,EAC5B,gBAAgB,GAAE,WAAW,CAAC,MAAM,CAAa,EACjD,iBAAiB,GAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAa,EAC1D,kBAAkB,GAAE,WAAW,CAAC,MAAM,CAAa,EACnD,oBAAoB,GAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAa,EAC1E,GAAG,GAAE,MAAmB,GACvB,YAAY,GAAG,IAAI,CA8IrB;AAMD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAC7C,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,EACtC,OAAO,GAAE,sBAA2B,GACnC,iBAAiB,CAAC,OAAO,CAAC,CAg3B5B"}
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
* carries the verbatim matched text so an operator can classify the death and
|
|
56
56
|
* improve `errorParsing.ts`'s pattern set (mirrors `onCreditExhausted`).
|
|
57
57
|
*/
|
|
58
|
+
import { AGENTIC_WORKER_HARNESS_OVERHEAD_TOKENS } from "../quota/capacity.js";
|
|
58
59
|
import { scheduleWave, } from "../quota/scheduler.js";
|
|
59
60
|
import { recordWaveOutcome, readQuotaState, emptyQuotaState, quotaSnapshotWindowPctMap, foldSlopeObservationFromPctMaps, } from "../quota/state.js";
|
|
60
61
|
import { buildEmptyPoolTerminal, buildQuotaPausedTerminal } from "../quota/capacity.js";
|
|
@@ -166,7 +167,7 @@ function isPoolQuotaDegraded(schedule, budgetTokens) {
|
|
|
166
167
|
*
|
|
167
168
|
* Returns null if no eligible pool currently has quota headroom.
|
|
168
169
|
*/
|
|
169
|
-
export function selectProvider(packet, confirmedPools, inFlightTracker, quotaStateEntries, sessionConfig, exhaustedPoolIds = new Set(), pausedPoolResetAt = new Map(), costDemotedPoolIds = new Set(), now = Date.now()) {
|
|
170
|
+
export function selectProvider(packet, confirmedPools, inFlightTracker, quotaStateEntries, sessionConfig, exhaustedPoolIds = new Set(), pausedPoolResetAt = new Map(), costDemotedPoolIds = new Set(), oversizedPacketPools = new Map(), now = Date.now()) {
|
|
170
171
|
const complexity = scorePacketComplexity(packet);
|
|
171
172
|
const highComplexity = complexity >= 0.5;
|
|
172
173
|
// Pause-honor (piece D): a pool paused until a stated host session-limit reset
|
|
@@ -177,6 +178,22 @@ export function selectProvider(packet, confirmedPools, inFlightTracker, quotaSta
|
|
|
177
178
|
const resetAt = pausedPoolResetAt.get(poolId);
|
|
178
179
|
return resetAt != null && now < resetAt;
|
|
179
180
|
};
|
|
181
|
+
// Packet-too-large skip (F4): this packet has been rejected as too large by some
|
|
182
|
+
// pools (HTTP 413). Skip those pools for THIS PACKET only — other packets freely
|
|
183
|
+
// bind to them. Allows packet to retry on a different pool.
|
|
184
|
+
const isTooLargeForThisPacket = (poolId) => {
|
|
185
|
+
const tooLargePools = oversizedPacketPools.get(packet.id);
|
|
186
|
+
return tooLargePools != null && tooLargePools.has(poolId);
|
|
187
|
+
};
|
|
188
|
+
// Context-fit skip (U2): this packet does not fit within a pool's declared context cap
|
|
189
|
+
// (plus harness overhead). Skip pools that cannot fit the packet. A pool with unknown
|
|
190
|
+
// context cap (null) always fits. This is the selection-time guard; partition-time fit
|
|
191
|
+
// checking in hybridDispatch ensures every in-process assignment is guaranteed to fit.
|
|
192
|
+
const doesNotFitContext = (pool) => {
|
|
193
|
+
if (pool.contextCapTokens == null)
|
|
194
|
+
return false; // unknown cap → always fits
|
|
195
|
+
return packet.estimatedTokens + AGENTIC_WORKER_HARNESS_OVERHEAD_TOKENS > pool.contextCapTokens;
|
|
196
|
+
};
|
|
180
197
|
// Ask scheduleWave whether each pool can accept one more slot, accounting for
|
|
181
198
|
// in-flight tokens as additional estimated slot cost. The schedule already
|
|
182
199
|
// folds in the real-time quota snapshot, the resolved provider limits, and any
|
|
@@ -227,7 +244,10 @@ export function selectProvider(packet, confirmedPools, inFlightTracker, quotaSta
|
|
|
227
244
|
});
|
|
228
245
|
};
|
|
229
246
|
const evaluated = confirmedPools
|
|
230
|
-
.filter((p) => !exhaustedPoolIds.has(p.id) &&
|
|
247
|
+
.filter((p) => !exhaustedPoolIds.has(p.id) &&
|
|
248
|
+
!isPausedNow(p.id) &&
|
|
249
|
+
!isTooLargeForThisPacket(p.id) &&
|
|
250
|
+
!doesNotFitContext(p))
|
|
231
251
|
.map((pool) => ({ pool, schedule: scheduleForPool(pool) }));
|
|
232
252
|
// Single ordering pass (INV-QD-14 proactive spill), in priority order — quota
|
|
233
253
|
// management is one track, always-on, so this ordering always applies:
|
|
@@ -286,7 +306,7 @@ export function selectProvider(packet, confirmedPools, inFlightTracker, quotaSta
|
|
|
286
306
|
* separate concurrency cap unless `options.maxConcurrentPerPool` is set.
|
|
287
307
|
*/
|
|
288
308
|
export function createRollingDispatcher(config, options = {}) {
|
|
289
|
-
const { confirmedPools, sessionConfig, dispatchPacket, onResult, onCostDrift, onCreditExhausted, onQuotaUnclassified, costDemotedPoolIds: injectedCostDemotedPoolIds, isPacketEscalated, recordRateLimit, reservationLedger, resolvePoolBudget, resolveOutputReservation: resolveOutputReservationFn, } = config;
|
|
309
|
+
const { confirmedPools, sessionConfig, dispatchPacket, onResult, onCostDrift, onCreditExhausted, onModelUnavailable, onPacketTooLarge, onQuotaUnclassified, costDemotedPoolIds: injectedCostDemotedPoolIds, isPacketEscalated, recordRateLimit, reservationLedger, resolvePoolBudget, resolveOutputReservation: resolveOutputReservationFn, } = config;
|
|
290
310
|
const state = {
|
|
291
311
|
pendingQueue: [],
|
|
292
312
|
inFlight: new Map(),
|
|
@@ -298,6 +318,8 @@ export function createRollingDispatcher(config, options = {}) {
|
|
|
298
318
|
// + levels; a standalone dispatcher owns a fresh per-instance set.
|
|
299
319
|
costDemotedPoolIds: injectedCostDemotedPoolIds ?? new Set(),
|
|
300
320
|
quotaUnclassifiedPoolIds: new Set(),
|
|
321
|
+
modelUnavailablePoolIds: new Set(),
|
|
322
|
+
oversizedPacketPools: new Map(),
|
|
301
323
|
};
|
|
302
324
|
const inFlightTracker = new InFlightTokenTracker();
|
|
303
325
|
const allResults = [];
|
|
@@ -469,6 +491,9 @@ export function createRollingDispatcher(config, options = {}) {
|
|
|
469
491
|
// cooldown is a TIMED reset; credit exhaustion has none. The permanent-for-
|
|
470
492
|
// the-run exclusion lives entirely in the in-memory exhaustedPoolIds set
|
|
471
493
|
// below, never in the persisted quota-state cooldown.
|
|
494
|
+
// model_unavailable and packet_too_large record as 'error' — no cooldown
|
|
495
|
+
// semantics. A 404 is permanent (model not served), a 413 is a per-packet
|
|
496
|
+
// sizing fault; neither carries reset timing.
|
|
472
497
|
// quota_unclassified records as 'rate_limited' — NOT 'error' — precisely so
|
|
473
498
|
// recordWaveOutcomeUnsafe DOES apply its exponential-backoff cooldown_until
|
|
474
499
|
// (state.ts:412-430). That reversible, timed cooldown IS the "conservative
|
|
@@ -479,8 +504,10 @@ export function createRollingDispatcher(config, options = {}) {
|
|
|
479
504
|
: result.outcome === "rate_limited" ? "rate_limited"
|
|
480
505
|
: result.outcome === "error" ? "error"
|
|
481
506
|
: result.outcome === "credit_exhausted" ? "error"
|
|
482
|
-
: result.outcome === "
|
|
483
|
-
: "
|
|
507
|
+
: result.outcome === "model_unavailable" ? "error"
|
|
508
|
+
: result.outcome === "packet_too_large" ? "error"
|
|
509
|
+
: result.outcome === "quota_unclassified" ? "rate_limited"
|
|
510
|
+
: "timeout";
|
|
484
511
|
try {
|
|
485
512
|
await recordWaveOutcome(providerModelKey, { outcome: quotaOutcome });
|
|
486
513
|
}
|
|
@@ -615,6 +642,113 @@ export function createRollingDispatcher(config, options = {}) {
|
|
|
615
642
|
}
|
|
616
643
|
return;
|
|
617
644
|
}
|
|
645
|
+
// Model unavailable (HTTP 404, model not found): the model is not served by
|
|
646
|
+
// this provider, so the pool is permanently excluded from this run's admissible
|
|
647
|
+
// set (monotonic modelUnavailablePoolIds). Unlike rate_limited/credit_exhausted
|
|
648
|
+
// (which carry reset/operator-action semantics), a 404 is permanent: the model
|
|
649
|
+
// simply doesn't exist at this provider. The pool is added to exhaustedPoolIds
|
|
650
|
+
// and the packet is re-queued so a different pool absorbs it. If this was the
|
|
651
|
+
// last admissible pool, the run degrades to the SAME empty_pool / quota_paused
|
|
652
|
+
// stranding path (INV-QD-07).
|
|
653
|
+
if (result.outcome === "model_unavailable") {
|
|
654
|
+
// Fire the hook only the FIRST time a given pool hits model-unavailable
|
|
655
|
+
// (mirrors the `firstForPool` gate on `onCreditExhausted`).
|
|
656
|
+
const firstForPool = !state.modelUnavailablePoolIds.has(providerSlot.poolId);
|
|
657
|
+
state.modelUnavailablePoolIds.add(providerSlot.poolId);
|
|
658
|
+
// Add to exhaustedPoolIds so selectProvider skips it (permanent exclusion for the run)
|
|
659
|
+
state.exhaustedPoolIds.add(providerSlot.poolId);
|
|
660
|
+
if (firstForPool) {
|
|
661
|
+
try {
|
|
662
|
+
onModelUnavailable?.({
|
|
663
|
+
poolId: providerSlot.poolId,
|
|
664
|
+
rawMatch: result.modelUnavailable?.rawMatch ?? null,
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
// Best-effort: a throwing friction hook must never abort dispatch.
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
if (!state.completedIds.has(packet.id) && !state.pendingQueue.some((q) => q.id === packet.id)) {
|
|
672
|
+
state.pendingQueue.unshift(packet);
|
|
673
|
+
}
|
|
674
|
+
try {
|
|
675
|
+
process.stderr.write(JSON.stringify({
|
|
676
|
+
ts: new Date().toISOString(),
|
|
677
|
+
kind: "rolling_dispatch_requeue_model_unavailable",
|
|
678
|
+
packet_id: packet.id,
|
|
679
|
+
exhausted_pool_id: providerSlot.poolId,
|
|
680
|
+
}) + "\n");
|
|
681
|
+
}
|
|
682
|
+
catch {
|
|
683
|
+
// Observability must never abort a run.
|
|
684
|
+
}
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
687
|
+
// Packet too large (HTTP 413, payload/request too large): a per-packet sizing
|
|
688
|
+
// fault for this particular (packet, pool) combination. UNLIKE model_unavailable
|
|
689
|
+
// or credit_exhausted, the pool is NOT permanently excluded — only THIS PACKET
|
|
690
|
+
// is skipped for THIS POOL on the next selection pass. The packet is re-queued
|
|
691
|
+
// to retry on a different pool. The pool remains available for other packets
|
|
692
|
+
// whose payload might fit.
|
|
693
|
+
if (result.outcome === "packet_too_large") {
|
|
694
|
+
// Track this (packet, pool) pair so selectProvider skips it on re-selection
|
|
695
|
+
if (!state.oversizedPacketPools.has(packet.id)) {
|
|
696
|
+
state.oversizedPacketPools.set(packet.id, new Set());
|
|
697
|
+
}
|
|
698
|
+
state.oversizedPacketPools.get(packet.id).add(providerSlot.poolId);
|
|
699
|
+
// Fire the hook every time this outcome lands (unlike model_unavailable which
|
|
700
|
+
// fires once per pool — each (packet,pool) pair is distinct signal).
|
|
701
|
+
try {
|
|
702
|
+
onPacketTooLarge?.({
|
|
703
|
+
poolId: providerSlot.poolId,
|
|
704
|
+
packetId: packet.id,
|
|
705
|
+
rawMatch: result.packetTooLarge?.rawMatch ?? null,
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
catch {
|
|
709
|
+
// Best-effort: a throwing friction hook must never abort dispatch.
|
|
710
|
+
}
|
|
711
|
+
// Livelock guard: a 413 skip is PERMANENT for the (packet, pool) pair (a
|
|
712
|
+
// packet never shrinks), so a packet whose skip-set now covers EVERY
|
|
713
|
+
// non-excluded pool can never dispatch again — re-queueing it would spin
|
|
714
|
+
// the dispatch loop forever (noPoolCanAcceptNow is pool-level and never
|
|
715
|
+
// fires for a per-packet condition). Strand it LOUD instead. A merely
|
|
716
|
+
// PAUSED pool still counts as potentially-available only if the packet
|
|
717
|
+
// has not 413'd it — pauses reset, 413s don't.
|
|
718
|
+
const skipSet = state.oversizedPacketPools.get(packet.id);
|
|
719
|
+
const anyPoolCouldEverTake = confirmedPools.some((p) => !state.exhaustedPoolIds.has(p.id) && !skipSet.has(p.id));
|
|
720
|
+
if (!anyPoolCouldEverTake) {
|
|
721
|
+
if (!state.completedIds.has(packet.id))
|
|
722
|
+
state.strandedIds.add(packet.id);
|
|
723
|
+
try {
|
|
724
|
+
process.stderr.write(JSON.stringify({
|
|
725
|
+
ts: new Date().toISOString(),
|
|
726
|
+
kind: "rolling_dispatch_stranded_packet_too_large_all_pools",
|
|
727
|
+
packet_id: packet.id,
|
|
728
|
+
skipped_pool_ids: [...skipSet].sort(),
|
|
729
|
+
}) + "\n");
|
|
730
|
+
}
|
|
731
|
+
catch {
|
|
732
|
+
// Observability must never abort a run.
|
|
733
|
+
}
|
|
734
|
+
return;
|
|
735
|
+
}
|
|
736
|
+
if (!state.completedIds.has(packet.id) && !state.pendingQueue.some((q) => q.id === packet.id)) {
|
|
737
|
+
state.pendingQueue.unshift(packet);
|
|
738
|
+
}
|
|
739
|
+
try {
|
|
740
|
+
process.stderr.write(JSON.stringify({
|
|
741
|
+
ts: new Date().toISOString(),
|
|
742
|
+
kind: "rolling_dispatch_requeue_packet_too_large",
|
|
743
|
+
packet_id: packet.id,
|
|
744
|
+
pool_id: providerSlot.poolId,
|
|
745
|
+
}) + "\n");
|
|
746
|
+
}
|
|
747
|
+
catch {
|
|
748
|
+
// Observability must never abort a run.
|
|
749
|
+
}
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
618
752
|
// Quota-unclassified (Slice A2b, TIER 2, backlog HIGH 2026-07-11): a worker
|
|
619
753
|
// death whose text was quota-SUSPICIOUS (the broad `detectQuotaSuspicious`
|
|
620
754
|
// pre-filter matched) but classified as NEITHER precise class above. This is
|
|
@@ -781,8 +915,9 @@ export function createRollingDispatcher(config, options = {}) {
|
|
|
781
915
|
let forcedUsed = false;
|
|
782
916
|
for (const packet of getDispatchablePackets()) {
|
|
783
917
|
// selectProvider skips pools in exhaustedPoolIds, so a re-queued packet
|
|
784
|
-
// re-routes to a surviving pool (INV-QD-07).
|
|
785
|
-
|
|
918
|
+
// re-routes to a surviving pool (INV-QD-07). It also skips pools in
|
|
919
|
+
// oversizedPacketPools for THIS PACKET if it's been rejected as too large.
|
|
920
|
+
const slot = selectProvider(packet, confirmedPools, inFlightTracker, quotaStateCache.entries, sessionConfig, state.exhaustedPoolIds, state.pausedPoolResetAt, state.costDemotedPoolIds, state.oversizedPacketPools, Date.now());
|
|
786
921
|
if (slot === null)
|
|
787
922
|
continue;
|
|
788
923
|
// Per-pool in-flight COUNT cap. The pool's own endpoint-declared cap
|
|
@@ -853,6 +988,41 @@ export function createRollingDispatcher(config, options = {}) {
|
|
|
853
988
|
strandPending();
|
|
854
989
|
break;
|
|
855
990
|
}
|
|
991
|
+
// Per-packet never-dispatchable strand (U2/F1, distinct from the
|
|
992
|
+
// pool-level check above): a packet is PERMANENTLY unselectable when
|
|
993
|
+
// every confirmed pool is exhausted, in the packet's 413 skip-set, or
|
|
994
|
+
// fit-excluded by a declared context cap — none of which reset. Without
|
|
995
|
+
// this, a fit-excluded packet on otherwise-healthy pools spins the 50ms
|
|
996
|
+
// wait tick forever (the pool-level guard never fires for a per-packet
|
|
997
|
+
// condition). A merely PAUSED pool that fits still counts as
|
|
998
|
+
// future-possible — pauses reset, caps don't.
|
|
999
|
+
const neverDispatchable = state.pendingQueue.filter((p) => {
|
|
1000
|
+
const skipSet = state.oversizedPacketPools.get(p.id);
|
|
1001
|
+
return confirmedPools.every((pool) => state.exhaustedPoolIds.has(pool.id) ||
|
|
1002
|
+
(skipSet?.has(pool.id) ?? false) ||
|
|
1003
|
+
(pool.contextCapTokens != null &&
|
|
1004
|
+
p.estimatedTokens + AGENTIC_WORKER_HARNESS_OVERHEAD_TOKENS >
|
|
1005
|
+
pool.contextCapTokens));
|
|
1006
|
+
});
|
|
1007
|
+
if (neverDispatchable.length > 0) {
|
|
1008
|
+
for (const p of neverDispatchable) {
|
|
1009
|
+
if (!state.completedIds.has(p.id))
|
|
1010
|
+
state.strandedIds.add(p.id);
|
|
1011
|
+
}
|
|
1012
|
+
const strandedIdSet = new Set(neverDispatchable.map((p) => p.id));
|
|
1013
|
+
state.pendingQueue = state.pendingQueue.filter((p) => !strandedIdSet.has(p.id));
|
|
1014
|
+
try {
|
|
1015
|
+
process.stderr.write(JSON.stringify({
|
|
1016
|
+
ts: new Date().toISOString(),
|
|
1017
|
+
kind: "rolling_dispatch_stranded_no_fitting_pool",
|
|
1018
|
+
packet_ids: [...strandedIdSet].sort(),
|
|
1019
|
+
}) + "\n");
|
|
1020
|
+
}
|
|
1021
|
+
catch {
|
|
1022
|
+
// Observability must never abort a run.
|
|
1023
|
+
}
|
|
1024
|
+
continue;
|
|
1025
|
+
}
|
|
856
1026
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
857
1027
|
continue;
|
|
858
1028
|
}
|