@aiwg/cli 2026.10.8 → 2026.10.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/providers/model-capabilities.v1.json +197 -3
- package/bin/aiwg.mjs +8 -0
- package/dist/src/artifacts/cli.js +9 -5
- package/dist/src/auth/config.js +3 -8
- package/dist/src/auth/resource-credentials.js +35 -2
- package/dist/src/cli/handlers/diagnose.js +13 -0
- package/dist/src/cli/handlers/mc-admission.js +39 -0
- package/dist/src/cli/handlers/mc.js +2 -31
- package/dist/src/cli/handlers/models-help.js +52 -0
- package/dist/src/cli/handlers/models.js +11 -81
- package/dist/src/cli/handlers/resource-versions.js +8 -5
- package/dist/src/cli/handlers/script-runner.js +13 -1
- package/dist/src/cli/handlers/use-addon-profile.js +72 -0
- package/dist/src/cli/handlers/use.js +209 -233
- package/dist/src/cli/ui.js +1 -0
- package/dist/src/config/aiwg-config.js +38 -184
- package/dist/src/config/cli-help.js +54 -0
- package/dist/src/config/cli.js +5 -69
- package/dist/src/config/config-normalization.js +10 -0
- package/dist/src/config/worker-activation-gate.js +26 -0
- package/dist/src/config/worker-adoption-cli.js +102 -0
- package/dist/src/config/worker-adoption.js +368 -0
- package/dist/src/config/worker-config-digest.js +14 -0
- package/dist/src/config/worker-group-templates.js +150 -0
- package/dist/src/config/worker-guidance-warning.js +28 -0
- package/dist/src/config/worker-owner-lock.js +152 -0
- package/dist/src/config/worker-parallelism.js +154 -0
- package/dist/src/config/worker-write.js +75 -0
- package/dist/src/decision/telemetry/exporter.js +2 -1
- package/dist/src/decision/telemetry/index.js +1 -0
- package/dist/src/decision/telemetry/mapping.js +43 -0
- package/dist/src/decision/telemetry/otlp-http.js +5 -2
- package/dist/src/decision/telemetry/redaction.js +6 -1
- package/dist/src/decision/telemetry/types.js +3 -0
- package/dist/src/decision/telemetry/worker.js +69 -0
- package/dist/src/lint/rule-activation.js +15 -7
- package/dist/src/lint/runner.js +2 -2
- package/dist/src/mcp/server.mjs +2 -11
- package/dist/src/mcp/stdio-lifecycle.mjs +27 -0
- package/dist/src/mcp/tools/worker-failure-diagnostics.mjs +9 -0
- package/dist/src/mcp/tools/worker-invoke.mjs +6 -6
- package/dist/src/mission-protocol/codecs.js +8 -38
- package/dist/src/mission-protocol/worker-contract-diagnostics.js +62 -0
- package/dist/src/mission-protocol/worker-invocation-types.js +2 -0
- package/dist/src/models/conductor-data-policy.d.mts +8 -0
- package/dist/src/models/conductor-data-policy.mjs +190 -0
- package/dist/src/models/config-loader.js +19 -3
- package/dist/src/models/config-writer.js +27 -0
- package/dist/src/models/model-capabilities.v1.json +197 -3
- package/dist/src/models/provider-policy.js +7 -155
- package/dist/src/models/resolver.js +2 -293
- package/dist/src/models/worker-artifact-binding.mjs +12 -0
- package/dist/src/models/worker-direct-readiness-policy.js +155 -0
- package/dist/src/models/worker-direct-readiness.js +9 -218
- package/dist/src/models/worker-health-readiness.js +105 -26
- package/dist/src/models/worker-installation.mjs +12 -14
- package/dist/src/models/worker-member-references.js +64 -0
- package/dist/src/models/worker-plan.js +317 -0
- package/dist/src/models/worker-selection-policy.js +217 -0
- package/dist/src/models/wrapper-deployment.js +1 -1
- package/dist/src/resources/web-release.js +76 -65
- package/dist/src/serve/shared-host-admission-checks.js +116 -0
- package/dist/src/serve/shared-host-scheduler.js +25 -107
- package/dist/src/serve/shared-host-starvation.js +73 -0
- package/dist/src/serve/worker-artifacts.d.mts +4 -0
- package/dist/src/serve/worker-artifacts.mjs +56 -0
- package/dist/src/serve/worker-command-request.js +68 -0
- package/dist/src/serve/worker-invocation-paths.js +144 -0
- package/dist/src/serve/worker-invocation-preparation.js +169 -0
- package/dist/src/serve/worker-invocation.js +24 -291
- package/dist/src/serve/worker-stdin-frame.d.mts +2 -2
- package/dist/src/serve/worker-stdin-frame.mjs +6 -6
- package/dist/src/smiths/context-pipeline/parallelism-section.js +19 -7
- package/dist/src/storage/backends/fortemi.js +33 -29
- package/dist/src/storage/backends/logseq.js +16 -3
- package/dist/src/storage/backends/postgres.js +74 -33
- package/dist/src/storage/backends/postgrest.js +37 -13
- package/dist/src/storage/config.js +6 -1
- package/dist/src/uhp/config.js +4 -0
- package/package.json +9 -1
- package/schemas/mission-v1.schema.json +20 -19
- package/schemas/models/model-config-compatibility.v1.schema.json +16 -0
- package/tools/agents/providers/claude-post-deploy.mjs +35 -0
- package/tools/agents/providers/claude.mjs +3 -30
- package/tools/agents/providers/codex-post-deploy.mjs +13 -0
- package/tools/agents/providers/codex.mjs +2 -8
- package/tools/agents/providers/conductor-data-guard.mjs +61 -0
- package/tools/agents/providers/rule-delivery.mjs +3 -16
- package/tools/agents/providers/rule-presentation.mjs +38 -0
- package/tools/plugin/package-plugins.mjs +1 -1
- package/tools/ralph-external/lib/claude-adapter.mjs +2 -2
- package/tools/ralph-external/lib/codex-adapter.mjs +3 -3
- package/tools/ralph-external/lib/managed-worker-attempt.mjs +86 -0
- package/tools/ralph-external/lib/provider-adapter.mjs +11 -260
- package/tools/ralph-external/lib/session-attempt.mjs +85 -0
- package/tools/ralph-external/lib/strict-worker-json.mjs +190 -0
- package/tools/ralph-external/lib/worker-artifacts.mjs +209 -0
- package/tools/ralph-external/lib/worker-final.mjs +143 -0
- package/tools/ralph-external/lib/worker-inputs.mjs +55 -0
- package/tools/ralph-external/lib/worker-isolation.mjs +2 -44
- package/tools/ralph-external/lib/worker-launch-cli.mjs +47 -0
- package/tools/ralph-external/lib/worker-native-sandbox.mjs +83 -0
- package/tools/ralph-external/session-launcher.mjs +73 -217
- package/vscode-extension/schemas/aiwg.config.v1.json +161 -2
|
@@ -433,6 +433,9 @@ async function fetchBytes(fetcher, url, label, maxBytes, bearerToken) {
|
|
|
433
433
|
catch (error) {
|
|
434
434
|
if (timedOut)
|
|
435
435
|
throw new Error(`${label} request timed out after ${RESOURCE_FETCH_TIMEOUT_MS}ms`);
|
|
436
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
437
|
+
if (bearerToken && message.includes(bearerToken))
|
|
438
|
+
throw new Error(message.split(bearerToken).join('[REDACTED]'));
|
|
436
439
|
throw error;
|
|
437
440
|
}
|
|
438
441
|
finally {
|
|
@@ -1072,80 +1075,88 @@ export async function resolveWebRelease(options = {}) {
|
|
|
1072
1075
|
const headers = { ...(init.headers || {}), ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}) };
|
|
1073
1076
|
return (options.fetcher ?? globalThis.fetch)(input, { ...init, headers });
|
|
1074
1077
|
};
|
|
1075
|
-
|
|
1076
|
-
if (
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
if (
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1078
|
+
try {
|
|
1079
|
+
if (selector.kind === "exact") {
|
|
1080
|
+
if (options.offline)
|
|
1081
|
+
return resolveOfflineExact(cacheRoot, selector, selector.value, publicKeyPem, base);
|
|
1082
|
+
const fetcher = authorize;
|
|
1083
|
+
if (!fetcher)
|
|
1084
|
+
throw new Error("No fetch implementation is available for AIWG web resources");
|
|
1085
|
+
return await fetchAndCacheRelease(base, fetcher, cacheRoot, selector, selector.value, publicKeyPem);
|
|
1086
|
+
}
|
|
1087
|
+
if (selector.kind === "range" || selector.kind === "digest") {
|
|
1088
|
+
if (!options.offline && selector.kind === "digest") {
|
|
1089
|
+
for (const version of cachedReleaseVersions(cacheRoot)) {
|
|
1090
|
+
if (!cachedDigests(cacheRoot, version).includes(selector.digest))
|
|
1091
|
+
continue;
|
|
1092
|
+
try {
|
|
1093
|
+
return verifyCachedGeneration(cacheRoot, version, selector.digest, selector, publicKeyPem, base, selector.digest);
|
|
1094
|
+
}
|
|
1095
|
+
catch {
|
|
1096
|
+
// A corrupt immutable generation cannot bypass signed index resolution.
|
|
1097
|
+
}
|
|
1093
1098
|
}
|
|
1094
1099
|
}
|
|
1100
|
+
const fetcher = authorize;
|
|
1101
|
+
const index = await resolveVersionIndex(base, fetcher, cacheRoot, publicKeyPem, options.offline, options.onDiagnostic);
|
|
1102
|
+
const selected = selectVersionFromIndex(index, selector);
|
|
1103
|
+
if (options.offline) {
|
|
1104
|
+
return resolveOfflineExact(cacheRoot, selector, selected.version, publicKeyPem, base, selected.releaseManifestSha256);
|
|
1105
|
+
}
|
|
1106
|
+
if (!fetcher)
|
|
1107
|
+
throw new Error("No fetch implementation is available for AIWG web resources");
|
|
1108
|
+
return await fetchAndCacheRelease(base, fetcher, cacheRoot, selector, selected.version, publicKeyPem, selected.releaseManifestSha256);
|
|
1095
1109
|
}
|
|
1096
|
-
const fetcher = authorize;
|
|
1097
|
-
const index = await resolveVersionIndex(base, fetcher, cacheRoot, publicKeyPem, options.offline, options.onDiagnostic);
|
|
1098
|
-
const selected = selectVersionFromIndex(index, selector);
|
|
1099
1110
|
if (options.offline) {
|
|
1100
|
-
|
|
1111
|
+
const cached = readCachedChannel(cacheRoot, selector.value, publicKeyPem);
|
|
1112
|
+
if (!cached)
|
|
1113
|
+
throw new Error(`AIWG resource channel ${selector.value} is not cached; offline mode cannot fetch it`);
|
|
1114
|
+
const release = resolveOfflineExact(cacheRoot, selector, cached.manifest.version, publicKeyPem, base, cached.manifest.releaseManifestSha256, cached.manifest.sequence);
|
|
1115
|
+
return cached.manifest.expiresAt ? { ...release, channelExpiresAt: cached.manifest.expiresAt } : release;
|
|
1101
1116
|
}
|
|
1117
|
+
const fetcher = authorize;
|
|
1102
1118
|
if (!fetcher)
|
|
1103
1119
|
throw new Error("No fetch implementation is available for AIWG web resources");
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
+
const channelPrefix = `resources/channels/${selector.value}`;
|
|
1121
|
+
let prior = null;
|
|
1122
|
+
try {
|
|
1123
|
+
prior = readCachedChannel(cacheRoot, selector.value, publicKeyPem);
|
|
1124
|
+
}
|
|
1125
|
+
catch {
|
|
1126
|
+
prior = null;
|
|
1127
|
+
}
|
|
1128
|
+
const fetched = await fetchMetadata(fetcher, resourceUrl(base, `${channelPrefix}.json`), `channel ${selector.value}`, MAX_SIGNED_METADATA_BYTES, prior?.validator);
|
|
1129
|
+
if (fetched.notModified) {
|
|
1130
|
+
cacheChannel(cacheRoot, prior.manifest, prior.bytes, prior.signatureBytes, prior.digest, fetched.validator);
|
|
1131
|
+
options.onDiagnostic?.({ resource: "channel", outcome: "conditional-hit", validator: prior.validator.etag ? "etag" : "last-modified" });
|
|
1132
|
+
const release = await fetchAndCacheRelease(base, fetcher, cacheRoot, selector, prior.manifest.version, publicKeyPem, prior.manifest.releaseManifestSha256, prior.manifest.sequence);
|
|
1133
|
+
return prior.manifest.expiresAt ? { ...release, channelExpiresAt: prior.manifest.expiresAt } : release;
|
|
1134
|
+
}
|
|
1135
|
+
const channelBytes = fetched.bytes;
|
|
1136
|
+
const channelSignatureBytes = await fetchBytes(fetcher, resourceUrl(base, `${channelPrefix}.sig`), `channel ${selector.value} signature`, MAX_SIGNATURE_BYTES);
|
|
1137
|
+
const channelDigest = verifySignedResourceBytes(channelBytes, channelSignatureBytes, publicKeyPem, `channel ${selector.value}`);
|
|
1138
|
+
const channel = validateChannelManifest(parseJson(channelBytes, `channel ${selector.value}`), selector.value);
|
|
1139
|
+
if (prior && channel.sequence < prior.manifest.sequence) {
|
|
1140
|
+
throw new Error(`channel ${selector.value} sequence rollback detected (${channel.sequence} < ${prior.manifest.sequence})`);
|
|
1141
|
+
}
|
|
1142
|
+
if (prior &&
|
|
1143
|
+
channel.sequence === prior.manifest.sequence &&
|
|
1144
|
+
(channelDigest !== prior.digest ||
|
|
1145
|
+
channel.version !== prior.manifest.version ||
|
|
1146
|
+
channel.releaseManifestSha256 !== prior.manifest.releaseManifestSha256)) {
|
|
1147
|
+
throw new Error(`channel ${selector.value} sequence ${channel.sequence} has conflicting signed metadata`);
|
|
1148
|
+
}
|
|
1149
|
+
const release = await fetchAndCacheRelease(base, fetcher, cacheRoot, selector, channel.version, publicKeyPem, channel.releaseManifestSha256, channel.sequence);
|
|
1150
|
+
cacheChannel(cacheRoot, channel, channelBytes, channelSignatureBytes, channelDigest, fetched.validator);
|
|
1151
|
+
options.onDiagnostic?.({ resource: "channel", outcome: prior?.validator ? "revalidated" : "unconditional", validator: fetched.validator?.etag ? "etag" : fetched.validator?.lastModified ? "last-modified" : "none" });
|
|
1152
|
+
return channel.expiresAt ? { ...release, channelExpiresAt: channel.expiresAt } : release;
|
|
1120
1153
|
}
|
|
1121
|
-
catch {
|
|
1122
|
-
|
|
1154
|
+
catch (error) {
|
|
1155
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1156
|
+
if (bearerToken && message.includes(bearerToken))
|
|
1157
|
+
throw new Error(message.split(bearerToken).join('[REDACTED]'));
|
|
1158
|
+
throw error;
|
|
1123
1159
|
}
|
|
1124
|
-
const fetched = await fetchMetadata(fetcher, resourceUrl(base, `${channelPrefix}.json`), `channel ${selector.value}`, MAX_SIGNED_METADATA_BYTES, prior?.validator);
|
|
1125
|
-
if (fetched.notModified) {
|
|
1126
|
-
cacheChannel(cacheRoot, prior.manifest, prior.bytes, prior.signatureBytes, prior.digest, fetched.validator);
|
|
1127
|
-
options.onDiagnostic?.({ resource: "channel", outcome: "conditional-hit", validator: prior.validator.etag ? "etag" : "last-modified" });
|
|
1128
|
-
const release = await fetchAndCacheRelease(base, fetcher, cacheRoot, selector, prior.manifest.version, publicKeyPem, prior.manifest.releaseManifestSha256, prior.manifest.sequence);
|
|
1129
|
-
return prior.manifest.expiresAt ? { ...release, channelExpiresAt: prior.manifest.expiresAt } : release;
|
|
1130
|
-
}
|
|
1131
|
-
const channelBytes = fetched.bytes;
|
|
1132
|
-
const channelSignatureBytes = await fetchBytes(fetcher, resourceUrl(base, `${channelPrefix}.sig`), `channel ${selector.value} signature`, MAX_SIGNATURE_BYTES);
|
|
1133
|
-
const channelDigest = verifySignedResourceBytes(channelBytes, channelSignatureBytes, publicKeyPem, `channel ${selector.value}`);
|
|
1134
|
-
const channel = validateChannelManifest(parseJson(channelBytes, `channel ${selector.value}`), selector.value);
|
|
1135
|
-
if (prior && channel.sequence < prior.manifest.sequence) {
|
|
1136
|
-
throw new Error(`channel ${selector.value} sequence rollback detected (${channel.sequence} < ${prior.manifest.sequence})`);
|
|
1137
|
-
}
|
|
1138
|
-
if (prior &&
|
|
1139
|
-
channel.sequence === prior.manifest.sequence &&
|
|
1140
|
-
(channelDigest !== prior.digest ||
|
|
1141
|
-
channel.version !== prior.manifest.version ||
|
|
1142
|
-
channel.releaseManifestSha256 !== prior.manifest.releaseManifestSha256)) {
|
|
1143
|
-
throw new Error(`channel ${selector.value} sequence ${channel.sequence} has conflicting signed metadata`);
|
|
1144
|
-
}
|
|
1145
|
-
const release = await fetchAndCacheRelease(base, fetcher, cacheRoot, selector, channel.version, publicKeyPem, channel.releaseManifestSha256, channel.sequence);
|
|
1146
|
-
cacheChannel(cacheRoot, channel, channelBytes, channelSignatureBytes, channelDigest, fetched.validator);
|
|
1147
|
-
options.onDiagnostic?.({ resource: "channel", outcome: prior?.validator ? "revalidated" : "unconditional", validator: fetched.validator?.etag ? "etag" : fetched.validator?.lastModified ? "last-modified" : "none" });
|
|
1148
|
-
return channel.expiresAt ? { ...release, channelExpiresAt: channel.expiresAt } : release;
|
|
1149
1160
|
}
|
|
1150
1161
|
export async function fetchVerifiedRawResource(release, resourcePath, options = {}) {
|
|
1151
1162
|
assertSafeRelativePath(resourcePath, "raw resource path");
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
export function attemptBlock(snapshot, candidate, policy, now) {
|
|
2
|
+
const accounting = candidate.accounting;
|
|
3
|
+
const deny = (cause, kind = 'policy-exclusion') => ({ kind, cause, terminal: true });
|
|
4
|
+
const attempts = Object.values(snapshot.records).filter(record => record.accounting?.attemptNumber !== undefined);
|
|
5
|
+
const taskAttempts = attempts.filter(record => record.accounting.taskKey === accounting.taskKey);
|
|
6
|
+
if (taskAttempts.length) {
|
|
7
|
+
const previous = taskAttempts.sort((a, b) => a.accounting.attemptNumber - b.accounting.attemptNumber).at(-1);
|
|
8
|
+
if (candidate.allowance?.previousRequestId !== previous.requestId)
|
|
9
|
+
return deny('new attempt must reference the latest task attempt');
|
|
10
|
+
if (previous.state !== 'settled')
|
|
11
|
+
return deny('previous attempt requires proof-checked reconciliation before replacement');
|
|
12
|
+
const failure = previous.accounting.failure;
|
|
13
|
+
const purpose = candidate.allowance?.purpose ?? 'retry';
|
|
14
|
+
if (purpose === 'retry' || purpose === 'fallback' || failure?.class === 'rate-limit') {
|
|
15
|
+
if (!failure || !failure.retryable || !['rate-limit', 'unavailable', 'timeout'].includes(failure.class))
|
|
16
|
+
return deny('previous failure does not permit automatic retry');
|
|
17
|
+
if (!(policy.managed.retry?.on ?? ['rate-limit', 'unavailable', 'timeout']).includes(failure.class))
|
|
18
|
+
return deny(`${failure.class} retries are disabled`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
else if (candidate.allowance?.previousRequestId)
|
|
22
|
+
return deny('unknown previous task attempt');
|
|
23
|
+
for (const key of [accounting.runKey, accounting.taskKey]) {
|
|
24
|
+
const scope = snapshot.allowances[key];
|
|
25
|
+
if (Date.parse(scope.deadlineAt ?? '') <= now)
|
|
26
|
+
return deny('cumulative allowance deadline elapsed');
|
|
27
|
+
const prior = attempts.filter(record => record.accounting.runKey === key || record.accounting.taskKey === key);
|
|
28
|
+
if (scope.limits.maxAttempts !== undefined && prior.reduce((sum, record) => sum + record.accounting.attemptCount, 0) + accounting.attemptCount > scope.limits.maxAttempts)
|
|
29
|
+
return deny('cumulative attempt allowance exhausted', 'budget-denial');
|
|
30
|
+
for (const [dimension, limit] of [['tokens', scope.limits.maxTokens], ['costUsd', scope.limits.maxCostUsd]]) {
|
|
31
|
+
if (limit === undefined)
|
|
32
|
+
continue;
|
|
33
|
+
let used = 0;
|
|
34
|
+
for (const record of prior) {
|
|
35
|
+
const value = chargedUsage(record, dimension);
|
|
36
|
+
if (value === null)
|
|
37
|
+
return deny(`unknown ${dimension} charge has no finite reservation`, 'budget-denial');
|
|
38
|
+
used += value;
|
|
39
|
+
}
|
|
40
|
+
const reserved = accounting.reservation[dimension];
|
|
41
|
+
if (reserved === null || reserved === 0 || used + reserved > limit)
|
|
42
|
+
return deny(`cumulative ${dimension} reservation exceeds allowance`, 'budget-denial');
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (taskAttempts.some(record => Date.parse(record.accounting.notBefore ?? '') > now))
|
|
46
|
+
return { kind: 'policy-exclusion', cause: 'retry-backoff', terminal: false };
|
|
47
|
+
if (policy.managed.budgets?.unknownCost === 'deny' && accounting.reservation.costUsd === null)
|
|
48
|
+
return deny('unknown cost is denied by policy', 'budget-denial');
|
|
49
|
+
const quotaScope = candidate.managed?.cli && snapshot.hostBinding
|
|
50
|
+
? policy.managed.upstream[candidate.managed.cli].quotaScope : candidate.managed?.quotaScope;
|
|
51
|
+
const quotas = quotaScope ? [`quota/${encodeURIComponent(quotaScope)}`]
|
|
52
|
+
: Object.keys(snapshot.cooldowns ?? {});
|
|
53
|
+
if (quotas.some(key => Date.parse(snapshot.cooldowns?.[key]?.until ?? '') > now))
|
|
54
|
+
return { kind: 'policy-exclusion', cause: 'quota-cooldown', terminal: false };
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
function chargedUsage(record, dimension) {
|
|
58
|
+
const accounting = record.accounting;
|
|
59
|
+
const usage = accounting.usage;
|
|
60
|
+
const measured = !usage || usage.quality === 'unknown' ? null : dimension === 'costUsd' ? usage.costUsd
|
|
61
|
+
: usage.inputTokens === null || usage.outputTokens === null || usage.cacheTokens === null || usage.reasoningTokens === null ? null
|
|
62
|
+
: usage.inputTokens + usage.outputTokens + usage.cacheTokens + usage.reasoningTokens;
|
|
63
|
+
const reservation = accounting.reservation[dimension];
|
|
64
|
+
// Uncertain execution can still accrue spend; estimates never refund a reserve.
|
|
65
|
+
if (record.state !== 'settled' || usage?.quality !== 'reported') {
|
|
66
|
+
return reservation === null || measured === null ? reservation : Math.max(reservation, measured);
|
|
67
|
+
}
|
|
68
|
+
return measured ?? reservation;
|
|
69
|
+
}
|
|
70
|
+
export function capacityBlock(snapshot, candidate, policy, now = -Infinity) {
|
|
71
|
+
const slots = policy.managed ? candidate.reservedSlots ?? 1 : 1;
|
|
72
|
+
const active = Object.values(snapshot.records).filter(record => record.state === 'admitted' || (policy.managed && record.state === 'uncertain'));
|
|
73
|
+
const limits = [
|
|
74
|
+
{ dimension: 'global', cap: policy.maxConcurrent, matches: () => true },
|
|
75
|
+
];
|
|
76
|
+
if (policy.managed)
|
|
77
|
+
for (const key of candidate.dimensions ?? []) {
|
|
78
|
+
const cap = policy.managed.dimensionCaps[key];
|
|
79
|
+
if (cap !== undefined)
|
|
80
|
+
limits.push({ dimension: key, cap, matches: record => !record.dimensions || record.dimensions.includes(key) });
|
|
81
|
+
}
|
|
82
|
+
for (const [field, quotas] of [
|
|
83
|
+
['environment', policy.environmentQuotas], ['provider', policy.providerQuotas],
|
|
84
|
+
['runtimeKind', { host: policy.defaultHostQuota ?? 1, ...policy.runtimeQuotas }],
|
|
85
|
+
]) {
|
|
86
|
+
const cap = quotas?.[candidate[field]];
|
|
87
|
+
if (cap !== undefined)
|
|
88
|
+
limits.push({ dimension: `${field}/${candidate[field]}`, cap: policy.managed ? cap : Math.ceil(cap), matches: record => record[field] === candidate[field] });
|
|
89
|
+
}
|
|
90
|
+
const impossible = limits.find(limit => slots > limit.cap);
|
|
91
|
+
if (impossible)
|
|
92
|
+
return { kind: 'policy-exclusion', cause: 'request-exceeds-cap', dimension: impossible.dimension, terminal: false };
|
|
93
|
+
const used = (records, limit) => records.filter(limit.matches)
|
|
94
|
+
.reduce((sum, record) => sum + (policy.managed ? record.reservedSlots ?? 1 : 1), 0);
|
|
95
|
+
const blocked = limits.filter(limit => used(active, limit) + slots > limit.cap);
|
|
96
|
+
if (!blocked.length)
|
|
97
|
+
return undefined;
|
|
98
|
+
const dimensions = blocked.map(limit => limit.dimension);
|
|
99
|
+
const certain = active.filter(record => record.state === 'admitted' && !(Date.parse(record.leaseExpiresAt ?? '') <= now));
|
|
100
|
+
if (policy.managed && blocked.some(limit => used(certain, limit) + slots <= limit.cap)) {
|
|
101
|
+
return { kind: 'safety-blocked', cause: 'uncertain-lease', dimension: dimensions[0], dimensions, terminal: false };
|
|
102
|
+
}
|
|
103
|
+
return { kind: 'capacity-wait', cause: 'capacity-cap', dimension: dimensions[0], dimensions, terminal: false };
|
|
104
|
+
}
|
|
105
|
+
/** Past retry/cooldown exclusion does not count toward an eligible service gap. */
|
|
106
|
+
export function attemptReadyAt(snapshot, candidate, policy) {
|
|
107
|
+
const retry = Object.values(snapshot.records).filter(record => record.accounting?.attemptNumber !== undefined
|
|
108
|
+
&& record.accounting.taskKey === candidate.accounting?.taskKey)
|
|
109
|
+
.reduce((latest, record) => Math.max(latest, Date.parse(record.accounting.notBefore ?? '') || 0), 0);
|
|
110
|
+
const quota = candidate.managed?.cli && snapshot.hostBinding
|
|
111
|
+
? policy.managed.upstream[candidate.managed.cli].quotaScope : candidate.managed?.quotaScope;
|
|
112
|
+
const cooldown = Object.entries(snapshot.cooldowns ?? {}).filter(([key]) => !quota || key === `quota/${encodeURIComponent(quota)}`)
|
|
113
|
+
.reduce((latest, [, value]) => Math.max(latest, Date.parse(value.until) || 0), 0);
|
|
114
|
+
return Math.max(retry, cooldown);
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=shared-host-admission-checks.js.map
|
|
@@ -14,6 +14,9 @@ import { hostname } from 'node:os';
|
|
|
14
14
|
import { createHash } from 'node:crypto';
|
|
15
15
|
import { advanceProgressWatermark, assertMissionWorkerHandoff, assertValidWorkerPayload, decideLeaseMapping, projectHealthTrace } from '../mission-protocol/codecs.js';
|
|
16
16
|
import { isCanonicalWorkerModelKey } from '../models/provider-policy.js';
|
|
17
|
+
import { attemptBlock, capacityBlock } from './shared-host-admission-checks.js';
|
|
18
|
+
import { projectAdmissionDiagnostics } from './shared-host-starvation.js';
|
|
19
|
+
import { resolveParallelism } from '../config/aiwg-config.js';
|
|
17
20
|
/** Translate existing owner ceilings without creating another budget contract. */
|
|
18
21
|
export function managedAllowanceFromOwner(runId, taskId, owner) {
|
|
19
22
|
const budgets = owner.kind === 'mission' ? owner.budgets : owner.kind === 'ralph' ? {
|
|
@@ -57,8 +60,8 @@ export function managedDimensionKeys(projectId, worker) {
|
|
|
57
60
|
...(worker.quotaScope ? [`quota/${encodeURIComponent(worker.quotaScope)}`] : []),
|
|
58
61
|
];
|
|
59
62
|
}
|
|
60
|
-
/** Build the project policy for the existing config/activation owner. */
|
|
61
63
|
export function managedProjectAdmissionPolicy(projectRoot, parallelism, policyDigest, maxConcurrent) {
|
|
64
|
+
parallelism = resolveParallelism(parallelism);
|
|
62
65
|
const { projectId } = projectAdmissionAuthority(projectRoot);
|
|
63
66
|
const admission = parallelism.admission;
|
|
64
67
|
if ((admission?.authorityRef ?? 'project') !== 'project')
|
|
@@ -90,6 +93,7 @@ export function managedProjectAdmissionPolicy(projectRoot, parallelism, policyDi
|
|
|
90
93
|
add({ kind: 'native', quotaScope }, cap);
|
|
91
94
|
return {
|
|
92
95
|
maxConcurrent, leaseTtlMs: (admission?.leaseSeconds ?? 60) * 1000,
|
|
96
|
+
starvationThresholdMs: (admission?.starvationSeconds ?? 120) * 1000,
|
|
93
97
|
agingIntervalMs: 30_000, allowPreemption: false, defaultHostQuota: maxConcurrent,
|
|
94
98
|
managed: {
|
|
95
99
|
authorityRef: 'project', projectId, policyDigest, maxQueued: admission?.maxQueued ?? 128,
|
|
@@ -97,7 +101,7 @@ export function managedProjectAdmissionPolicy(projectRoot, parallelism, policyDi
|
|
|
97
101
|
budgets: parallelism.budgets, retry: parallelism.retry,
|
|
98
102
|
installations: Object.fromEntries(Object.entries(parallelism.clis ?? {}).filter(([, cli]) => cli.enabled !== false)
|
|
99
103
|
.map(([id, cli]) => [id, { accountScope: cli.accountScope, quotaScope: cli.quotaScope }])),
|
|
100
|
-
members: Object.fromEntries(Object.entries(parallelism.groups ?? {}).flatMap(([group, definition]) => definition.members.filter(member => member.enabled !== false && parallelism.clis?.[member.cli]?.enabled !== false)
|
|
104
|
+
members: Object.fromEntries(Object.entries(parallelism.groups ?? {}).flatMap(([group, definition]) => (definition.members ?? []).filter(member => member.enabled !== false && parallelism.clis?.[member.cli]?.enabled !== false)
|
|
101
105
|
.map(member => [`${group}/${member.id}`, { cli: member.cli }]))),
|
|
102
106
|
},
|
|
103
107
|
};
|
|
@@ -256,16 +260,21 @@ export class SharedHostScheduler {
|
|
|
256
260
|
store;
|
|
257
261
|
policy;
|
|
258
262
|
clock;
|
|
259
|
-
|
|
263
|
+
checkActivation;
|
|
264
|
+
constructor(store, policy, clock = Date.now, checkActivation) {
|
|
260
265
|
this.store = store;
|
|
261
266
|
this.policy = policy;
|
|
262
267
|
this.clock = clock;
|
|
268
|
+
this.checkActivation = checkActivation;
|
|
263
269
|
if (!Number.isInteger(policy.maxConcurrent) || policy.maxConcurrent < 1) {
|
|
264
270
|
throw new Error('maxConcurrent must be a positive integer');
|
|
265
271
|
}
|
|
266
272
|
if (policy.leaseTtlMs < 1 || policy.agingIntervalMs < 1) {
|
|
267
273
|
throw new Error('leaseTtlMs and agingIntervalMs must be positive');
|
|
268
274
|
}
|
|
275
|
+
if (!Number.isSafeInteger(policy.starvationThresholdMs ?? 120_000) || (policy.starvationThresholdMs ?? 120_000) < 1) {
|
|
276
|
+
throw new Error('starvationThresholdMs must be a positive safe integer');
|
|
277
|
+
}
|
|
269
278
|
if (policy.managed)
|
|
270
279
|
this.policy = structuredClone(policy);
|
|
271
280
|
const managed = this.policy.managed;
|
|
@@ -347,6 +356,7 @@ export class SharedHostScheduler {
|
|
|
347
356
|
const record = snapshot.records[requestKey];
|
|
348
357
|
record.state = 'denied';
|
|
349
358
|
record.reason = 'managed admission queue is full';
|
|
359
|
+
record.denial = { kind: 'policy-exclusion', cause: record.reason, terminal: true };
|
|
350
360
|
record.finishedAt = new Date(this.clock()).toISOString();
|
|
351
361
|
record.revision += 1;
|
|
352
362
|
}
|
|
@@ -609,6 +619,7 @@ export class SharedHostScheduler {
|
|
|
609
619
|
snapshot.allowances = Object.fromEntries(Object.entries(snapshot.allowances ?? {})
|
|
610
620
|
.filter(([key]) => JSON.parse(key)[0] === projectId));
|
|
611
621
|
}
|
|
622
|
+
snapshot.diagnostics = projectAdmissionDiagnostics(value, record => this.policyFor(value, record), this.clock(), this.policy.managed?.authorityRef.startsWith('host:') ? this.policy.managed.projectId : undefined);
|
|
612
623
|
return snapshot;
|
|
613
624
|
}
|
|
614
625
|
reconcile(snapshot) {
|
|
@@ -717,66 +728,17 @@ export class SharedHostScheduler {
|
|
|
717
728
|
};
|
|
718
729
|
}
|
|
719
730
|
allowAttempt(snapshot, candidate, now) {
|
|
720
|
-
const
|
|
721
|
-
|
|
731
|
+
const block = attemptBlock(snapshot, candidate, this.policyFor(snapshot, candidate), now);
|
|
732
|
+
if (!block)
|
|
733
|
+
return true;
|
|
734
|
+
if (block.terminal) {
|
|
722
735
|
candidate.state = 'denied';
|
|
723
|
-
candidate.reason =
|
|
736
|
+
candidate.reason = block.cause;
|
|
737
|
+
candidate.denial = block;
|
|
724
738
|
candidate.finishedAt = new Date(now).toISOString();
|
|
725
739
|
candidate.revision += 1;
|
|
726
|
-
return false;
|
|
727
|
-
};
|
|
728
|
-
const attempts = Object.values(snapshot.records).filter(record => record.accounting?.attemptNumber !== undefined);
|
|
729
|
-
const taskAttempts = attempts.filter(record => record.accounting.taskKey === accounting.taskKey);
|
|
730
|
-
if (taskAttempts.length) {
|
|
731
|
-
const previous = taskAttempts.sort((a, b) => a.accounting.attemptNumber - b.accounting.attemptNumber).at(-1);
|
|
732
|
-
if (candidate.allowance?.previousRequestId !== previous.requestId)
|
|
733
|
-
return deny('new attempt must reference the latest task attempt');
|
|
734
|
-
if (previous.state !== 'settled')
|
|
735
|
-
return deny('previous attempt requires proof-checked reconciliation before replacement');
|
|
736
|
-
const failure = previous.accounting.failure;
|
|
737
|
-
const purpose = candidate.allowance?.purpose ?? 'retry';
|
|
738
|
-
if (purpose === 'retry' || purpose === 'fallback' || failure?.class === 'rate-limit') {
|
|
739
|
-
if (!failure || !failure.retryable || !['rate-limit', 'unavailable', 'timeout'].includes(failure.class))
|
|
740
|
-
return deny('previous failure does not permit automatic retry');
|
|
741
|
-
if (!(this.policyFor(snapshot, candidate).managed.retry?.on ?? ['rate-limit', 'unavailable', 'timeout']).includes(failure.class))
|
|
742
|
-
return deny(`${failure.class} retries are disabled`);
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
else if (candidate.allowance?.previousRequestId)
|
|
746
|
-
return deny('unknown previous task attempt');
|
|
747
|
-
for (const key of [accounting.runKey, accounting.taskKey]) {
|
|
748
|
-
const scope = snapshot.allowances[key];
|
|
749
|
-
if (Date.parse(scope.deadlineAt ?? '') <= now)
|
|
750
|
-
return deny('cumulative allowance deadline elapsed');
|
|
751
|
-
const prior = attempts.filter(record => record.accounting.runKey === key || record.accounting.taskKey === key);
|
|
752
|
-
if (scope.limits.maxAttempts !== undefined && prior.reduce((sum, record) => sum + record.accounting.attemptCount, 0) + accounting.attemptCount > scope.limits.maxAttempts)
|
|
753
|
-
return deny('cumulative attempt allowance exhausted');
|
|
754
|
-
for (const [dimension, limit] of [['tokens', scope.limits.maxTokens], ['costUsd', scope.limits.maxCostUsd]]) {
|
|
755
|
-
if (limit === undefined)
|
|
756
|
-
continue;
|
|
757
|
-
let used = 0;
|
|
758
|
-
for (const record of prior) {
|
|
759
|
-
const value = chargedUsage(record, dimension);
|
|
760
|
-
if (value === null)
|
|
761
|
-
return deny(`unknown ${dimension} charge has no finite reservation`);
|
|
762
|
-
used += value;
|
|
763
|
-
}
|
|
764
|
-
const reserved = accounting.reservation[dimension];
|
|
765
|
-
if (reserved === null || reserved === 0 || used + reserved > limit)
|
|
766
|
-
return deny(`cumulative ${dimension} reservation exceeds allowance`);
|
|
767
|
-
}
|
|
768
740
|
}
|
|
769
|
-
|
|
770
|
-
return false;
|
|
771
|
-
if (this.policyFor(snapshot, candidate).managed.budgets?.unknownCost === 'deny' && accounting.reservation.costUsd === null)
|
|
772
|
-
return deny('unknown cost is denied by policy');
|
|
773
|
-
const quotaScope = candidate.managed?.cli && snapshot.hostBinding
|
|
774
|
-
? this.policyFor(snapshot, candidate).managed.upstream[candidate.managed.cli].quotaScope : candidate.managed?.quotaScope;
|
|
775
|
-
const quotas = quotaScope ? [`quota/${encodeURIComponent(quotaScope)}`]
|
|
776
|
-
: Object.keys(snapshot.cooldowns ?? {});
|
|
777
|
-
if (quotas.some(key => Date.parse(snapshot.cooldowns?.[key]?.until ?? '') > now))
|
|
778
|
-
return false;
|
|
779
|
-
return true;
|
|
741
|
+
return false;
|
|
780
742
|
}
|
|
781
743
|
tryPreempt(snapshot, candidate, now) {
|
|
782
744
|
const victims = Object.values(snapshot.records)
|
|
@@ -800,31 +762,7 @@ export class SharedHostScheduler {
|
|
|
800
762
|
victim.revision += 1;
|
|
801
763
|
}
|
|
802
764
|
hasCapacity(snapshot, candidate) {
|
|
803
|
-
|
|
804
|
-
const active = Object.values(snapshot.records).filter(record => record.state === 'admitted' || (policy.managed && record.state === 'uncertain'));
|
|
805
|
-
const slots = policy.managed ? candidate.reservedSlots ?? 1 : 1;
|
|
806
|
-
if (active.reduce((sum, record) => sum + (policy.managed ? record.reservedSlots ?? 1 : 1), 0) + slots > policy.maxConcurrent)
|
|
807
|
-
return false;
|
|
808
|
-
if (policy.managed) {
|
|
809
|
-
for (const key of candidate.dimensions ?? []) {
|
|
810
|
-
const cap = policy.managed.dimensionCaps[key];
|
|
811
|
-
if (cap === undefined)
|
|
812
|
-
continue;
|
|
813
|
-
const used = active.filter(record => !record.dimensions || record.dimensions.includes(key))
|
|
814
|
-
.reduce((sum, record) => sum + (record.reservedSlots ?? 1), 0);
|
|
815
|
-
if (used + slots > cap)
|
|
816
|
-
return false;
|
|
817
|
-
}
|
|
818
|
-
}
|
|
819
|
-
if (!belowQuota(active, 'environment', candidate.environment, policy.environmentQuotas, slots, !!policy.managed))
|
|
820
|
-
return false;
|
|
821
|
-
if (!belowQuota(active, 'provider', candidate.provider, policy.providerQuotas, slots, !!policy.managed))
|
|
822
|
-
return false;
|
|
823
|
-
const runtimeQuotas = {
|
|
824
|
-
host: policy.defaultHostQuota ?? 1,
|
|
825
|
-
...policy.runtimeQuotas,
|
|
826
|
-
};
|
|
827
|
-
return belowQuota(active, 'runtimeKind', candidate.runtimeKind, runtimeQuotas, slots, !!policy.managed);
|
|
765
|
+
return !capacityBlock(snapshot, candidate, this.policyFor(snapshot, candidate));
|
|
828
766
|
}
|
|
829
767
|
effectivePriority(record, now) {
|
|
830
768
|
const waited = Math.max(0, now - Date.parse(record.submittedAt));
|
|
@@ -927,6 +865,7 @@ export class SharedHostScheduler {
|
|
|
927
865
|
}
|
|
928
866
|
}
|
|
929
867
|
assertSubmissionsOpen(snapshot) {
|
|
868
|
+
this.checkActivation?.();
|
|
930
869
|
if (snapshot.migration || snapshot.authorityRedirect)
|
|
931
870
|
throw new Error('admission authority is frozen for migration');
|
|
932
871
|
}
|
|
@@ -1028,6 +967,7 @@ export function hostProjectAdmissionPolicy(snapshot, projectId) {
|
|
|
1028
967
|
return {
|
|
1029
968
|
...structuredClone(host.policy),
|
|
1030
969
|
leaseTtlMs: Math.min(host.policy.leaseTtlMs, project.policy.leaseTtlMs),
|
|
970
|
+
starvationThresholdMs: Math.min(host.policy.starvationThresholdMs ?? 120_000, project.policy.starvationThresholdMs ?? 120_000),
|
|
1031
971
|
managed: {
|
|
1032
972
|
...structuredClone(managed), authorityRef: host.authorityRef, hostPolicyRevision: host.revision,
|
|
1033
973
|
authorityGeneration: generation, maxQueued: Math.min(host.maxQueued, managed.maxQueued),
|
|
@@ -1271,18 +1211,9 @@ function isCanonicalManagedDimension(projectId, key) {
|
|
|
1271
1211
|
return false;
|
|
1272
1212
|
}
|
|
1273
1213
|
function sameManagedRequest(record, request) {
|
|
1274
|
-
const { state: _state, revision: _revision, reason: _reason, admittedAt: _admittedAt, leaseExpiresAt: _leaseExpiresAt, finishedAt: _finishedAt, preemptedBy: _preemptedBy, projectId: _project, attemptGeneration: _attempt, leaseGeneration: _lease, dimensions: _dimensions, reservedSlots: _slots, accounting: _accounting, launchClaimedAt: _launch, execution: _execution, health: _health, lastProgressAt: _progress, finalEvidence: _final, ...original } = record;
|
|
1214
|
+
const { state: _state, revision: _revision, reason: _reason, denial: _denial, admittedAt: _admittedAt, leaseExpiresAt: _leaseExpiresAt, finishedAt: _finishedAt, preemptedBy: _preemptedBy, projectId: _project, attemptGeneration: _attempt, leaseGeneration: _lease, dimensions: _dimensions, reservedSlots: _slots, accounting: _accounting, launchClaimedAt: _launch, execution: _execution, health: _health, lastProgressAt: _progress, finalEvidence: _final, ...original } = record;
|
|
1275
1215
|
return canonical(original) === canonical(request);
|
|
1276
1216
|
}
|
|
1277
|
-
function belowQuota(active, field, value, quotas, slots = 1, managed = false) {
|
|
1278
|
-
const quota = quotas?.[String(value)];
|
|
1279
|
-
if (quota === undefined)
|
|
1280
|
-
return true;
|
|
1281
|
-
if (!managed)
|
|
1282
|
-
return active.filter(record => record[field] === value).length < quota;
|
|
1283
|
-
return active.filter(record => record[field] === value)
|
|
1284
|
-
.reduce((sum, record) => sum + (record.reservedSlots ?? 1), 0) + slots <= quota;
|
|
1285
|
-
}
|
|
1286
1217
|
function sameRequest(record, request) {
|
|
1287
1218
|
return record.orchestratorId === request.orchestratorId
|
|
1288
1219
|
&& record.environment === request.environment
|
|
@@ -1311,19 +1242,6 @@ function intersectBudgets(...budgets) {
|
|
|
1311
1242
|
}
|
|
1312
1243
|
return result;
|
|
1313
1244
|
}
|
|
1314
|
-
function chargedUsage(record, dimension) {
|
|
1315
|
-
const accounting = record.accounting;
|
|
1316
|
-
const usage = accounting.usage;
|
|
1317
|
-
const measured = !usage || usage.quality === 'unknown' ? null : dimension === 'costUsd' ? usage.costUsd
|
|
1318
|
-
: usage.inputTokens === null || usage.outputTokens === null || usage.cacheTokens === null || usage.reasoningTokens === null ? null
|
|
1319
|
-
: usage.inputTokens + usage.outputTokens + usage.cacheTokens + usage.reasoningTokens;
|
|
1320
|
-
const reservation = accounting.reservation[dimension];
|
|
1321
|
-
// Uncertain execution can still accrue spend; estimates never refund a reserve.
|
|
1322
|
-
if (record.state !== 'settled' || usage?.quality !== 'reported') {
|
|
1323
|
-
return reservation === null || measured === null ? reservation : Math.max(reservation, measured);
|
|
1324
|
-
}
|
|
1325
|
-
return measured ?? reservation;
|
|
1326
|
-
}
|
|
1327
1245
|
function canRefineUsage(previous, next) {
|
|
1328
1246
|
const rank = { unknown: 0, estimated: 1, reported: 2 };
|
|
1329
1247
|
if (rank[next.quality] < rank[previous.quality])
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// Project bounded group/run service diagnostics without changing admission decisions.
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { attemptBlock, attemptReadyAt, capacityBlock } from './shared-host-admission-checks.js';
|
|
4
|
+
const maximumFindings = 64;
|
|
5
|
+
const label = (value) => Buffer.byteLength(value, 'utf8') <= 256 ? value
|
|
6
|
+
: `${Buffer.from(value).subarray(0, 180).toString('utf8')}#${createHash('sha256').update(value).digest('hex')}`;
|
|
7
|
+
const run = (record) => record.allowance?.runId ?? record.orchestratorId;
|
|
8
|
+
const elapsed = (now, since) => Math.max(0, now - since);
|
|
9
|
+
export function projectAdmissionDiagnostics(snapshot, policyFor, now, projectId) {
|
|
10
|
+
const records = Object.values(snapshot.records);
|
|
11
|
+
const findings = new Map();
|
|
12
|
+
// Current waits take precedence over historical denials. Stable ties make
|
|
13
|
+
// truncation repeatable across independent readers of the durable snapshot.
|
|
14
|
+
const candidates = records.filter(record => record.managed && (!projectId || record.projectId === projectId)
|
|
15
|
+
&& (record.state === 'queued' || (record.state === 'denied' && record.denial)))
|
|
16
|
+
.sort((a, b) => Number(a.state === 'denied') - Number(b.state === 'denied')
|
|
17
|
+
|| (a.state === 'denied' ? Date.parse(b.finishedAt) - Date.parse(a.finishedAt) : Date.parse(a.submittedAt) - Date.parse(b.submittedAt))
|
|
18
|
+
|| a.requestId.localeCompare(b.requestId));
|
|
19
|
+
for (const candidate of candidates) {
|
|
20
|
+
const submitted = Date.parse(candidate.submittedAt);
|
|
21
|
+
if (candidate.state === 'queued' && submitted + candidate.queueTimeoutMs <= now)
|
|
22
|
+
continue;
|
|
23
|
+
const policy = policyFor(candidate);
|
|
24
|
+
const thresholdMs = policy.starvationThresholdMs ?? 120_000;
|
|
25
|
+
const waitMs = elapsed(candidate.state === 'denied' ? Date.parse(candidate.finishedAt) : now, submitted);
|
|
26
|
+
if (candidate.state === 'queued' && waitMs < thresholdMs)
|
|
27
|
+
continue;
|
|
28
|
+
let block = candidate.denial;
|
|
29
|
+
if (candidate.state === 'queued') {
|
|
30
|
+
block = snapshot.migration || snapshot.authorityRedirect
|
|
31
|
+
? { kind: 'policy-exclusion', cause: 'authority-frozen', terminal: false }
|
|
32
|
+
: !candidate.accounting
|
|
33
|
+
? { kind: 'safety-blocked', cause: 'allowance-reconciliation-required', terminal: false }
|
|
34
|
+
: attemptBlock(snapshot, candidate, policy, now) ?? capacityBlock(snapshot, candidate, policy, now);
|
|
35
|
+
}
|
|
36
|
+
if (!block)
|
|
37
|
+
continue;
|
|
38
|
+
const scopes = [
|
|
39
|
+
...(candidate.managed?.group ? [['group', candidate.managed.group]] : []), ['run', run(candidate)],
|
|
40
|
+
];
|
|
41
|
+
for (const [scope, scopeId] of scopes) {
|
|
42
|
+
const sameScope = (record) => record.projectId === candidate.projectId
|
|
43
|
+
&& (scope === 'group' ? record.managed?.group === scopeId : run(record) === scopeId);
|
|
44
|
+
const lastService = records.filter(sameScope).reduce((latest, record) => Math.max(latest, Date.parse(record.admittedAt ?? '') || 0), Math.max(submitted, attemptReadyAt(snapshot, candidate, policy)));
|
|
45
|
+
const serviceWaitMs = elapsed(now, lastService);
|
|
46
|
+
// Admissions from other projects also compete at host-wide dimensions;
|
|
47
|
+
// only counts, never other projects' identities, enter the public view.
|
|
48
|
+
const consumes = (record, dimension) => dimension === 'global'
|
|
49
|
+
|| (dimension.startsWith('environment/') ? record.environment === candidate.environment
|
|
50
|
+
: dimension.startsWith('provider/') ? record.provider === candidate.provider
|
|
51
|
+
: dimension.startsWith('runtimeKind/') ? record.runtimeKind === candidate.runtimeKind
|
|
52
|
+
: !record.dimensions || record.dimensions.includes(dimension));
|
|
53
|
+
// Every independently binding cap must see competing service. A full
|
|
54
|
+
// global queue cannot mask a local group cap that never frees capacity.
|
|
55
|
+
const competingAdmissions = records.filter(record => record.admittedAt && !sameScope(record)
|
|
56
|
+
&& Date.parse(record.admittedAt) > lastService && Date.parse(record.admittedAt) <= now
|
|
57
|
+
&& (block.dimensions ?? (block.dimension ? [block.dimension] : [])).every(dimension => consumes(record, dimension))).length;
|
|
58
|
+
const kind = block.kind === 'capacity-wait' && serviceWaitMs >= thresholdMs && competingAdmissions >= 2
|
|
59
|
+
? 'service-starvation' : block.kind;
|
|
60
|
+
const finding = {
|
|
61
|
+
kind, scope, scopeId: label(scopeId), projectId: candidate.projectId ?? policy.managed?.projectId,
|
|
62
|
+
requestId: label(candidate.requestId), group: candidate.managed?.group, runId: label(run(candidate)),
|
|
63
|
+
waitMs, serviceWaitMs, thresholdMs, blockingCause: block.cause,
|
|
64
|
+
...(block.dimension ? { dimension: label(block.dimension) } : {}), competingAdmissions,
|
|
65
|
+
};
|
|
66
|
+
const key = JSON.stringify([finding.projectId, scope, scopeId, kind, block.cause, block.dimension]);
|
|
67
|
+
if (!findings.has(key))
|
|
68
|
+
findings.set(key, finding);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return { findings: [...findings.values()].slice(0, maximumFindings), truncated: Math.max(0, findings.size - maximumFindings) };
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=shared-host-starvation.js.map
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Pure validation shared by the typed renderer and installed execution owner. */
|
|
2
|
+
import type { WorkerResolvedInvocation } from '../mission-protocol/types.js';
|
|
3
|
+
export function validateWorkerArtifacts(invocation: Pick<WorkerResolvedInvocation,
|
|
4
|
+
'artifacts' | 'invocationId' | 'cwd' | 'briefPath' | 'stdoutPath' | 'stderrPath' | 'finalPath'>): NonNullable<WorkerResolvedInvocation['artifacts']>;
|