@aiwg/cli 2026.10.6 → 2026.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/aiwg.mjs +4 -2
- package/dist/src/a2a/agent-card.js +11 -0
- package/dist/src/api/index.d.ts +1 -0
- package/dist/src/api/index.js +1 -0
- package/dist/src/artifacts/fortemi-shard-contract/2.2.0/contract.json +103 -0
- package/dist/src/artifacts/fortemi-shard-contract/2.2.0/full-v1-embedding-config.schema.json +212 -0
- package/dist/src/artifacts/fortemi-shard-contract/2.2.0/full-v1-embedding-set.schema.json +240 -0
- package/dist/src/artifacts/fortemi-shard-contract/2.2.0/full-v1-manifest.schema.json +451 -0
- package/dist/src/artifacts/fortemi-shard-export.js +284 -0
- package/dist/src/artifacts/index-preparation.js +2 -0
- package/dist/src/auth/client.js +5 -1
- package/dist/src/auth/credential-store.js +5 -1
- package/dist/src/auth/sensitive-input.js +151 -0
- package/dist/src/channel/manager.mjs +3 -0
- package/dist/src/cli/handlers/agentcard.js +47 -25
- package/dist/src/cli/handlers/auth.js +22 -2
- package/dist/src/cli/handlers/cost-report.js +7 -1
- package/dist/src/cli/handlers/decision.js +36 -10
- package/dist/src/cli/handlers/effect.js +4 -84
- package/dist/src/cli/handlers/guided-setup.js +1 -1
- package/dist/src/cli/handlers/init.js +1 -1
- package/dist/src/cli/handlers/install.js +6 -0
- package/dist/src/cli/handlers/local-executor.js +33 -6
- package/dist/src/cli/handlers/marketplace.js +15 -4
- package/dist/src/cli/handlers/setup.js +24 -20
- package/dist/src/cli/handlers/use.js +57 -15
- package/dist/src/cli/prompt-utils.js +73 -0
- package/dist/src/config/gitignore.js +4 -0
- package/dist/src/config/workspace.js +2 -1
- package/dist/src/cost/fleet-report.js +20 -22
- package/dist/src/decision/adapters/jev.js +20 -31
- package/dist/src/decision/ask.js +6 -1
- package/dist/src/decision/benchmark.js +19 -6
- package/dist/src/decision/driver.js +2 -2
- package/dist/src/decision/jev-credentials.js +18 -5
- package/dist/src/decision/ledger.js +230 -34
- package/dist/src/decision/qualification/gate-evidence.js +6 -4
- package/dist/src/decision/qualification/load-staged.js +47 -10
- package/dist/src/decision/qualification/runner.js +7 -3
- package/dist/src/discovery/interview.js +10 -5
- package/dist/src/discovery/plan-apply.js +26 -30
- package/dist/src/discovery/plan-validation.js +2 -2
- package/dist/src/effects/host.js +91 -0
- package/dist/src/extensions/commands/definitions.js +2 -2
- package/dist/src/extensions/project-local-gitignore.js +2 -0
- package/dist/src/issues/cli.js +16 -5
- package/dist/src/issues/live.js +44 -12
- package/dist/src/marketplace/exchange.js +18 -6
- package/dist/src/marketplace/provenance.js +3 -2
- package/dist/src/mcp/adapters/codex-runtime.js +46 -76
- package/dist/src/mcp/cli.mjs +122 -109
- package/dist/src/mcp/credentials.d.mts +4 -0
- package/dist/src/mcp/credentials.mjs +44 -3
- package/dist/src/mcp/grok-build-config.mjs +4 -1
- package/dist/src/mcp/muse-native-config.mjs +2 -1
- package/dist/src/mcp/omp-config.mjs +2 -0
- package/dist/src/mcp/registry.js +19 -15
- package/dist/src/mcp/registry.mjs +2 -18
- package/dist/src/mcp/tools/agentic-sandbox.mjs +46 -28
- package/dist/src/mcp/tools/worker-invoke.mjs +13 -2
- package/dist/src/memory/project-registry.js +2 -1
- package/dist/src/mission-protocol/worker-status.js +91 -1
- package/dist/src/models/grok-build-models.js +20 -0
- package/dist/src/models/worker-installation.d.mts +1 -1
- package/dist/src/models/worker-installation.mjs +65 -4
- package/dist/src/packages/adapters/git.js +99 -23
- package/dist/src/packages/adapters/gitea.js +8 -27
- package/dist/src/packages/adapters/github.js +7 -25
- package/dist/src/packages/registry.js +4 -4
- package/dist/src/process/cli-command.d.mts +5 -0
- package/dist/src/process/cli-command.mjs +42 -0
- package/dist/src/providers/provider-definitions.js +2 -2
- package/dist/src/serve/shared-host-scheduler.js +4 -1
- package/dist/src/serve/worker-invocation.js +10 -0
- package/dist/src/smiths/context-pipeline/claude-hook.js +9 -9
- package/dist/src/smiths/context-pipeline/context-file-safety.js +330 -9
- package/dist/src/smiths/context-pipeline/finalization.js +3 -2
- package/dist/src/smiths/context-pipeline/generator.js +7 -19
- package/dist/src/smiths/context-pipeline/legacy-inject.js +3 -10
- package/dist/src/smiths/context-pipeline/managed-hook.js +5 -5
- package/dist/src/smiths/context-pipeline/workspace-context.js +4 -12
- package/dist/src/storage/backends/fortemi.js +107 -49
- package/dist/src/tracker/project-inputs.js +2 -0
- package/dist/src/update/notifier.mjs +2 -4
- package/package.json +1 -1
- package/tools/agents/providers/base.mjs +5 -1
- package/tools/agents/providers/claude.mjs +5 -2
- package/tools/agents/providers/codex.mjs +4 -1
- package/tools/agents/providers/context-write.mjs +28 -0
- package/tools/agents/providers/copilot.mjs +5 -2
- package/tools/agents/providers/hermes.mjs +5 -2
- package/tools/agents/providers/omp.mjs +3 -1
- package/tools/agents/providers/rule-delivery.mjs +29 -12
- package/tools/agents/providers/warp.mjs +4 -1
- package/tools/agents/providers/windsurf.mjs +6 -3
- package/tools/ralph-external/lib/consecutive-failure-circuit-breaker.mjs +26 -0
- package/tools/ralph-external/lib/launcher-recovery-evidence.mjs +7 -1
- package/tools/ralph-external/lib/recovery-probe.mjs +14 -4
- package/tools/ralph-external/process-monitor.mjs +42 -4
- package/tools/ralph-external/session-launcher.mjs +125 -3
|
@@ -308,6 +308,9 @@ const HOST_CODES = new Set(['RESOURCE', 'NOT_FOUND', 'SANDBOX']);
|
|
|
308
308
|
const PROVIDER_CODES = new Set(['NETWORK']);
|
|
309
309
|
const INSTALLATION_CODES = new Set(['AUTH']);
|
|
310
310
|
const MODEL_CODES = new Set(['MODEL']);
|
|
311
|
+
const STATUS_REASON_CODES = new Set(['NOT_FOUND', 'SPAWN_REFUSED', 'RATE_LIMIT', 'RESOURCE', 'SANDBOX',
|
|
312
|
+
'NETWORK', 'AUTH', 'MODEL', 'UNKNOWN']);
|
|
313
|
+
export const RECOVERY_STATUS_LIMIT = 32;
|
|
311
314
|
|
|
312
315
|
export const scopeKeyFor = Object.freeze({
|
|
313
316
|
invocation: (id) => `invocation:${id ?? 'unknown'}`,
|
|
@@ -511,6 +514,7 @@ export class ScopedFailureBreaker extends EventEmitter {
|
|
|
511
514
|
this._scopes = new Map();
|
|
512
515
|
this._outstanding = new Map();
|
|
513
516
|
this._hostFailureFacts = [];
|
|
517
|
+
this._transitions = [];
|
|
514
518
|
this._canarySeq = 0;
|
|
515
519
|
this._observedInvocations = new Set();
|
|
516
520
|
this.canariesObserved = 0;
|
|
@@ -1022,6 +1026,10 @@ export class ScopedFailureBreaker extends EventEmitter {
|
|
|
1022
1026
|
const from = rec.status;
|
|
1023
1027
|
if (from === to) return;
|
|
1024
1028
|
rec.status = to;
|
|
1029
|
+
if (rec.scopeKey.length <= 256) this._transitions.push({ scopeKey: rec.scopeKey, scope: rec.scope, from, to,
|
|
1030
|
+
reasons: [...rec.evidence.keys()].filter(code => STATUS_REASON_CODES.has(code)).slice(0, 8),
|
|
1031
|
+
atMs: this.nowMs(), nextEligibleAtMs: to === 'closed' ? null : rec.nextProbeAtMs });
|
|
1032
|
+
if (this._transitions.length > RECOVERY_STATUS_LIMIT) this._transitions.shift();
|
|
1025
1033
|
this.emit('transition', { scopeKey: rec.scopeKey, from, to, action,
|
|
1026
1034
|
distinctFailures: new Set(rec.failures.map(failure => failure.invocationId)).size, reasons: [...rec.evidence.keys()],
|
|
1027
1035
|
nextEligibleAtMs: rec.nextProbeAtMs, resetAtMs: rec.resetAtMs,
|
|
@@ -1454,6 +1462,23 @@ export class ScopedFailureBreaker extends EventEmitter {
|
|
|
1454
1462
|
};
|
|
1455
1463
|
}
|
|
1456
1464
|
|
|
1465
|
+
/** Pure status projection: never sweep reservations, probe, or advance state. */
|
|
1466
|
+
recoveryStatus() {
|
|
1467
|
+
const recentKeys = [...new Set(this._transitions.map(event => event.scopeKey).reverse())];
|
|
1468
|
+
const active = [...this._scopes.values()].filter(rec => rec.status !== 'closed');
|
|
1469
|
+
const keys = [...new Set([...active.map(rec => rec.scopeKey), ...recentKeys])];
|
|
1470
|
+
const scopes = keys.filter(key => key.length <= 256).slice(0, RECOVERY_STATUS_LIMIT).map(scopeKey => {
|
|
1471
|
+
const rec = this._scopes.get(scopeKey);
|
|
1472
|
+
const last = this._transitions.findLast(event => event.scopeKey === scopeKey);
|
|
1473
|
+
return { scopeKey, scope: rec?.scope ?? last.scope, state: rec?.status ?? last.to,
|
|
1474
|
+
reasons: rec ? [...rec.evidence.keys()].filter(code => STATUS_REASON_CODES.has(code)).slice(0, 8) : [...last.reasons],
|
|
1475
|
+
nextEligibleAtMs: !rec || rec.status === 'closed' ? null : rec.nextProbeAtMs,
|
|
1476
|
+
canaryInFlight: rec?.activeCanary != null };
|
|
1477
|
+
});
|
|
1478
|
+
return { observedAtMs: this.nowMs(), scopes, omittedScopeCount: Math.max(0, keys.length - scopes.length),
|
|
1479
|
+
transitions: this._transitions.map(event => ({ ...event, reasons: [...event.reasons] })) };
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1457
1482
|
/** Correlate launcher-observed host faults across resolved inference backends.
|
|
1458
1483
|
* Replay the first fault once corroborated, preserving the three-distinct
|
|
1459
1484
|
* threshold even when the first backend alone could not establish scope.
|
|
@@ -1550,6 +1575,7 @@ export class ScopedFailureBreaker extends EventEmitter {
|
|
|
1550
1575
|
|
|
1551
1576
|
destroy() {
|
|
1552
1577
|
this._hostFailureFacts = [];
|
|
1578
|
+
this._transitions = [];
|
|
1553
1579
|
for (const rec of this._scopes.values()) rec.breaker.destroy();
|
|
1554
1580
|
this._scopes.clear();
|
|
1555
1581
|
this._outstanding.clear();
|
|
@@ -3,6 +3,7 @@ import { runRecoveryProbe } from './recovery-probe.mjs';
|
|
|
3
3
|
/** Recovery evidence observed outside worker output by the launch owner. */
|
|
4
4
|
const observations = new WeakMap();
|
|
5
5
|
const failureHints = new WeakMap();
|
|
6
|
+
const assessed = new WeakSet();
|
|
6
7
|
|
|
7
8
|
export function recordLauncherRecoveryHint(result, reasonCode) {
|
|
8
9
|
failureHints.set(result, reasonCode);
|
|
@@ -23,9 +24,10 @@ export function launcherRecoveryEvidence(result) {
|
|
|
23
24
|
*/
|
|
24
25
|
export async function observeLauncherRecoveryOutcome({ invocation, result, installation, breaker,
|
|
25
26
|
failureHint = '', beforeRequest, onDispatch, fetchImpl = globalThis.fetch, wallNowMs = Date.now, timeoutMs = 10_000 } = {}) {
|
|
26
|
-
if (!result || result.attemptOutcome !== 'failed' || result.cancelled) return;
|
|
27
|
+
if (!result || result.attemptOutcome !== 'failed' || result.cancelled || assessed.has(result)) return;
|
|
27
28
|
const backend = invocation?.selected?.inferenceProvider;
|
|
28
29
|
if (HOST_ERRORS.has(result.errorCode) || HOST_RESULTS.has(result.nativeResult)) {
|
|
30
|
+
assessed.add(result);
|
|
29
31
|
const corroboratedBackends = breaker?.observeHostFailure({ invocationId: result.invocationId,
|
|
30
32
|
inferenceProvider: backend, reasonCode: 'RESOURCE' }) ?? [];
|
|
31
33
|
if (corroboratedBackends.length >= 2) observations.set(result, {
|
|
@@ -43,6 +45,10 @@ export async function observeLauncherRecoveryOutcome({ invocation, result, insta
|
|
|
43
45
|
if (!await beforeRequest()) return;
|
|
44
46
|
const requestTimeoutMs = remaining();
|
|
45
47
|
if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs <= 0) return;
|
|
48
|
+
// One diagnostic request per attempt, including unsuccessful diagnostics.
|
|
49
|
+
// Re-reading a result cannot consume allowance or create fresh evidence.
|
|
50
|
+
if (assessed.has(result)) return;
|
|
51
|
+
assessed.add(result);
|
|
46
52
|
const observed = await runRecoveryProbe({ driver: { qualification: { installation } }, invocation: invocation ?? { transport: 'shell' },
|
|
47
53
|
timeoutMs: Math.min(10_000, requestTimeoutMs), maxOutputBytes: 65_536, onDispatch, fetchImpl, wallNowMs });
|
|
48
54
|
if (observed.reason === 'probe-rate-limited') {
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { RecoveryPolicy } from './consecutive-failure-circuit-breaker.mjs';
|
|
2
|
+
|
|
1
3
|
/** A recovery probe is a conductor-owned request, never an uncontained worker. */
|
|
2
4
|
export async function runRecoveryProbe({ driver, invocation, timeoutMs = 10_000, maxOutputBytes = 65_536,
|
|
3
|
-
onDispatch, fetchImpl = globalThis.fetch, wallNowMs = Date.now }) {
|
|
5
|
+
onDispatch, fetchImpl = globalThis.fetch, wallNowMs = Date.now, nowMs = () => performance.now() }) {
|
|
4
6
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 10_000
|
|
5
7
|
|| !Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 1 || maxOutputBytes > 65_536) {
|
|
6
8
|
throw new RangeError('Recovery probe exceeds the 10s/64KiB bounds');
|
|
@@ -19,6 +21,7 @@ export async function runRecoveryProbe({ driver, invocation, timeoutMs = 10_000,
|
|
|
19
21
|
return failed('probe-endpoint-unavailable', { operatorIntervention: true });
|
|
20
22
|
}
|
|
21
23
|
const controller = new AbortController();
|
|
24
|
+
const deadlineMs = nowMs() + timeoutMs;
|
|
22
25
|
const timer = setTimeout(() => controller.abort(), Math.max(1, Math.floor(timeoutMs)));
|
|
23
26
|
let aborted;
|
|
24
27
|
const timeout = new Promise((_, reject) => {
|
|
@@ -30,14 +33,19 @@ export async function runRecoveryProbe({ driver, invocation, timeoutMs = 10_000,
|
|
|
30
33
|
timeout.catch(() => {});
|
|
31
34
|
try {
|
|
32
35
|
await onDispatch?.();
|
|
36
|
+
if (nowMs() >= deadlineMs) controller.abort();
|
|
33
37
|
if (controller.signal.aborted) return failed('probe-timeout');
|
|
34
38
|
try {
|
|
35
39
|
// HEAD has no task, credentials, inference body or worker filesystem.
|
|
36
40
|
// A successful endpoint check permits only one subsequent task canary.
|
|
37
41
|
const response = await Promise.race([fetchImpl(endpoint, { method: 'HEAD', redirect: 'error', signal: controller.signal }), timeout]);
|
|
38
42
|
let headerBytes = 16 + Buffer.byteLength(response.statusText ?? '');
|
|
39
|
-
for (const [name, value] of response.headers) headerBytes += Buffer.byteLength(name) + Buffer.byteLength(value) + 4;
|
|
40
43
|
try {
|
|
44
|
+
if (nowMs() >= deadlineMs) { controller.abort(); return failed('probe-timeout'); }
|
|
45
|
+
for (const [name, value] of response.headers) {
|
|
46
|
+
headerBytes += Buffer.byteLength(name) + Buffer.byteLength(value) + 4;
|
|
47
|
+
if (headerBytes > maxOutputBytes) return failed('probe-output-limit');
|
|
48
|
+
}
|
|
41
49
|
if (headerBytes > maxOutputBytes) return failed('probe-output-limit');
|
|
42
50
|
if (response.body) return failed('probe-body-unexpected');
|
|
43
51
|
const success = response.status >= 200 && response.status <= 299;
|
|
@@ -45,7 +53,8 @@ export async function runRecoveryProbe({ driver, invocation, timeoutMs = 10_000,
|
|
|
45
53
|
if (response.status === 429) {
|
|
46
54
|
const after = response.headers.get('retry-after');
|
|
47
55
|
const seconds = after != null && /^\d+(?:\.\d+)?$/.test(after.trim()) ? Number(after) : null;
|
|
48
|
-
const date = seconds === null &&
|
|
56
|
+
const date = seconds === null && /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(after ?? '')
|
|
57
|
+
? Date.parse(after) : NaN;
|
|
49
58
|
const reset = response.headers.get('x-ratelimit-reset');
|
|
50
59
|
const resetEpoch = reset != null && /^\d+(?:\.\d+)?$/.test(reset.trim()) ? Number(reset) * 1000 : NaN;
|
|
51
60
|
const resetDelay = response.headers.get('ratelimit-reset');
|
|
@@ -53,7 +62,8 @@ export async function runRecoveryProbe({ driver, invocation, timeoutMs = 10_000,
|
|
|
53
62
|
Number.isFinite(resetEpoch) ? (resetEpoch - wallNowMs()) / 1000 : null,
|
|
54
63
|
resetDelay != null && /^\d+(?:\.\d+)?$/.test(resetDelay.trim()) ? Number(resetDelay) : null]
|
|
55
64
|
.filter(value => Number.isFinite(value) && value >= 0);
|
|
56
|
-
return failed('probe-rate-limited', { retryAfterSeconds: delays.length
|
|
65
|
+
return failed('probe-rate-limited', { retryAfterSeconds: delays.length
|
|
66
|
+
? Math.min(RecoveryPolicy.maxRetryAfterSeconds, Math.max(...delays)) : null });
|
|
57
67
|
}
|
|
58
68
|
return failed('probe-failed', { status: response.status });
|
|
59
69
|
} finally {
|
|
@@ -15,7 +15,7 @@ import { hostname } from 'node:os';
|
|
|
15
15
|
import { join, dirname } from 'path';
|
|
16
16
|
import { constants } from 'node:fs';
|
|
17
17
|
import { open, opendir, rename, unlink } from 'node:fs/promises';
|
|
18
|
-
import { randomUUID } from 'node:crypto';
|
|
18
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
19
19
|
import path from 'node:path';
|
|
20
20
|
import { createWorkerHealthObserver } from './lib/worker-health.mjs';
|
|
21
21
|
|
|
@@ -38,6 +38,9 @@ import { createWorkerHealthObserver } from './lib/worker-health.mjs';
|
|
|
38
38
|
// error, a thrown assertion, ...) and silently report every live-process
|
|
39
39
|
// health query as unavailable instead of surfacing the actual failure.
|
|
40
40
|
let boundedExecFileSync = null;
|
|
41
|
+
let projectRecoveryStatus = null;
|
|
42
|
+
let mergeRecoveryStatuses = null;
|
|
43
|
+
let projectStartupCorrection = null;
|
|
41
44
|
let processHealthUnavailableReason = null;
|
|
42
45
|
let importImpl;
|
|
43
46
|
try {
|
|
@@ -58,6 +61,11 @@ if (importImpl) {
|
|
|
58
61
|
if (!['AIWG_IMPLEMENTATION_MISSING', 'AIWG_PACKAGE_ROOT_MISSING'].includes(error?.code)) throw error;
|
|
59
62
|
processHealthUnavailableReason = `bounded-child-process helper unavailable: implementation missing: ${error.message}`;
|
|
60
63
|
}
|
|
64
|
+
try {
|
|
65
|
+
({ projectRecoveryStatus, mergeRecoveryStatuses, projectStartupCorrection } = await importImpl(import.meta.url, 'mission-protocol/worker-status.js'));
|
|
66
|
+
} catch (error) {
|
|
67
|
+
if (!['AIWG_IMPLEMENTATION_MISSING', 'AIWG_PACKAGE_ROOT_MISSING'].includes(error?.code)) throw error;
|
|
68
|
+
}
|
|
61
69
|
}
|
|
62
70
|
|
|
63
71
|
/** Side-effect-free Linux sample. Unknown identity never grants signal authority. */
|
|
@@ -848,8 +856,10 @@ export async function sampleWorkerStatus(options, memory = new Map()) {
|
|
|
848
856
|
const monotonicNow = options.monotonicNow ?? monotonicMs;
|
|
849
857
|
const began = monotonicNow();
|
|
850
858
|
const report = { version: 1, sampledAt, scope: 'direct-conductor', observerCoverage: 'unobserved',
|
|
851
|
-
workers: [], breakers: [],
|
|
859
|
+
workers: [], breakers: [], recovery: { source: 'unavailable', observedAt: null, observedAtMs: null,
|
|
860
|
+
scopes: [], transitions: [], omittedScopeCount: 0, truncated: false }, nextCheckAt: null, complete: true, findings: [], exitCode: 0 };
|
|
852
861
|
let root;
|
|
862
|
+
const recoveryOwners = new Map();
|
|
853
863
|
try {
|
|
854
864
|
root = await openStatusDirectory(options.runDir ?? options.runsRoot);
|
|
855
865
|
const listing = options.runDir ? { entries: [{ name: null }], complete: true }
|
|
@@ -870,6 +880,11 @@ export async function sampleWorkerStatus(options, memory = new Map()) {
|
|
|
870
880
|
process: await statusRecord(directory, 'process.json', true), health: await statusRecord(directory, 'health.json'),
|
|
871
881
|
exit: await statusRecord(directory, 'exit.json'), scratch: await statusRecord(directory, 'last-observed.json') };
|
|
872
882
|
const ownerRecord = await statusRecord(directory, '.status-observer');
|
|
883
|
+
const supervisor = records.process.supervisor;
|
|
884
|
+
const ownerHost = safeLabel(supervisor?.host), ownerBoot = safeLabel(supervisor?.bootId), ownerStart = safeLabel(supervisor?.startTicks);
|
|
885
|
+
const launcherOwnerRecorded = records.process.origin === 'launcher'
|
|
886
|
+
&& supervisor?.uid === process.getuid() && Number.isSafeInteger(supervisor.pid) && supervisor.pid > 0
|
|
887
|
+
&& ownerHost && ownerBoot && ownerStart;
|
|
873
888
|
const expected = records.process.worker;
|
|
874
889
|
// classifyWorkerStatus() also ignores an unverified worker for this
|
|
875
890
|
// comparison; skip the resample too so this never samples an
|
|
@@ -884,6 +899,22 @@ export async function sampleWorkerStatus(options, memory = new Map()) {
|
|
|
884
899
|
? (options.sampleIdentity ?? sampleWorkerIdentity)(ownerIdentity.pid) : null;
|
|
885
900
|
const row = classifyWorkerStatus(records, { now, monotonicNow: sampleTime, bootId, actual, counters,
|
|
886
901
|
previous: memory.get(key), ownerRecord, actualObserver });
|
|
902
|
+
if (projectRecoveryStatus && launcherOwnerRecorded && records.process.recovery) {
|
|
903
|
+
const recovery = projectRecoveryStatus(records.process.recovery, 'recorded');
|
|
904
|
+
const ownerKey = JSON.stringify([ownerHost, ownerBoot, supervisor.pid, ownerStart]);
|
|
905
|
+
const previous = recoveryOwners.get(ownerKey);
|
|
906
|
+
if (recovery.observedAt && (!previous || Date.parse(recovery.observedAt) > Date.parse(previous.observedAt)
|
|
907
|
+
|| recovery.observedAt === previous.observedAt && recovery.transitions.length >= previous.transitions.length)) {
|
|
908
|
+
const digest = createHash('sha256').update(ownerKey).digest('hex').slice(0, 12);
|
|
909
|
+
const owner = `${supervisor.pid}@${ownerHost.slice(0, 80)}:${ownerStart.slice(0, 32)}:${digest}`;
|
|
910
|
+
for (const scope of recovery.scopes) scope.owner = owner;
|
|
911
|
+
for (const transition of recovery.transitions) transition.owner = owner;
|
|
912
|
+
recoveryOwners.set(ownerKey, recovery);
|
|
913
|
+
report.recovery = mergeRecoveryStatuses([...recoveryOwners.values()]);
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
const correction = launcherOwnerRecorded && projectStartupCorrection?.(records.process.startupCorrection);
|
|
917
|
+
if (correction) row.startupCorrection = correction;
|
|
887
918
|
report.findings.push(...row.findings.map(finding => `${row.invocationId}: ${finding}`));
|
|
888
919
|
if (row.observerOwnership === 'unavailable') report.findings.push('OBSERVER_OWNER_UNAVAILABLE: reconcile reservation before observing');
|
|
889
920
|
memory.set(key, { cpuTicks: actual?.cpuTicks ?? null, counters, health: row.health });
|
|
@@ -975,12 +1006,19 @@ export function createStatusObservationOwner() {
|
|
|
975
1006
|
export function formatWorkerStatus(report) {
|
|
976
1007
|
const label = value => value == null ? '?' : String(value);
|
|
977
1008
|
const seconds = value => value == null ? '?' : `${Math.round(value * 1000) / 1000}s`;
|
|
1009
|
+
const recovery = report.recovery;
|
|
1010
|
+
const recoveryLines = !recovery || recovery.source === 'unavailable'
|
|
1011
|
+
? ['Recovery scope status unavailable: no recorded recovery owner observation']
|
|
1012
|
+
: [`Recovery scopes (${recovery.source}) observed=${label(recovery.observedAt)} omitted=${recovery.omittedScopeCount} truncated=${recovery.truncated}`,
|
|
1013
|
+
...recovery.scopes.map(scope => ` ${scope.scopeKey} state=${scope.state} reasons=${scope.reasons.join(',') || 'UNKNOWN'} next eligible=${label(scope.nextEligibleAt ?? scope.nextEligibleAtMs)} canary in flight=${scope.canaryInFlight}${scope.owner ? ` owner=${scope.owner}` : ''}`),
|
|
1014
|
+
...recovery.transitions.map(transition => ` ${transition.scopeKey}: ${transition.from} -> ${transition.to} at=${label(transition.at ?? transition.atMs)} reasons=${transition.reasons.join(',') || 'UNKNOWN'} next eligible=${label(transition.nextEligibleAt ?? transition.nextEligibleAtMs)}${transition.owner ? ` owner=${transition.owner}` : ''}`)];
|
|
978
1015
|
return [`Workers (${report.scope}) coverage=${report.observerCoverage} complete=${report.complete} breakers=${report.breakers.length} (no shared Tier 0 breaker authority)`,
|
|
979
1016
|
'invocation | member | state | coverage | age | last progress | remaining | cause | next action',
|
|
980
1017
|
...report.workers.map(row => row.error ? `? | ? | unreadable | ? | ? | ? | ? | ${row.error.code} | ${row.nextAction}`
|
|
981
1018
|
: `${row.invocationId} | ${label(row.member)} | ${row.health.state} | ${row.health.coverage} | ${seconds(row.elapsedSeconds)} | ${seconds(row.ages.progressSeconds)} | ${seconds(row.remainingSeconds)} | ${row.health.reasonCode} | ${row.nextAction}\n`
|
|
982
|
-
+ ` thresholds=${JSON.stringify(row.thresholds)} termination=${row.termination} verification=${label(row.verification)} observer-gap=${row.health.maxCheckGapSeconds}s`
|
|
983
|
-
|
|
1019
|
+
+ ` thresholds=${JSON.stringify(row.thresholds)} termination=${row.termination} verification=${label(row.verification)} observer-gap=${row.health.maxCheckGapSeconds}s`
|
|
1020
|
+
+ (row.startupCorrection ? `\n startup correction=${row.startupCorrection.action} state=${row.startupCorrection.state} reason=${row.startupCorrection.reasonCode} stop=${label(row.startupCorrection.stopCode)}` : '')),
|
|
1021
|
+
...recoveryLines, ...report.findings, `next check=${label(report.nextCheckAt)}`].join('\n');
|
|
984
1022
|
}
|
|
985
1023
|
|
|
986
1024
|
/** One bounded foreground loop. Timeout only stops watching, never the worker. */
|
|
@@ -487,6 +487,10 @@ function workerLaunchRefusal(outcome) {
|
|
|
487
487
|
: 'Worker launch did not return an attached child');
|
|
488
488
|
}
|
|
489
489
|
|
|
490
|
+
// Only the admitted direct owner can supply inputs for an automatic correction.
|
|
491
|
+
// Neither worker output nor a caller-supplied result can populate this context.
|
|
492
|
+
const directStartupCorrectionContext = new WeakMap();
|
|
493
|
+
|
|
490
494
|
async function launchOneWorker(owner, options) {
|
|
491
495
|
const { inspectWorkerPath: inspect, revalidateWorkerInvocationPlan: validate } = await importImpl(import.meta.url, 'serve/worker-invocation.js');
|
|
492
496
|
const { qualificationDigest } = await importImpl(import.meta.url, 'models/qualification-digest.mjs');
|
|
@@ -664,6 +668,7 @@ async function launchOneWorker(owner, options) {
|
|
|
664
668
|
// worker-writable captureRoot fell back to the in-process ledger) is
|
|
665
669
|
// folded in up front so every exit path reports it.
|
|
666
670
|
const ledgerDiagnostics = admission.ledgerDiagnostic ? [admission.ledgerDiagnostic] : [];
|
|
671
|
+
const startupQualification = structuredClone(inputs.drivers[invocation.selected.cli].qualification);
|
|
667
672
|
const settleLedger = async (outcome) => {
|
|
668
673
|
const diagnostic = await settleDirectWorker(admission, outcome);
|
|
669
674
|
if (diagnostic) ledgerDiagnostics.push(diagnostic);
|
|
@@ -747,11 +752,22 @@ async function launchOneWorker(owner, options) {
|
|
|
747
752
|
// writes; `.catch(() => {})` keeps a prior failure from poisoning it, and
|
|
748
753
|
// the per-call promise returned to the caller still reflects this write's
|
|
749
754
|
// own outcome.
|
|
755
|
+
let latestProcessRecord = null;
|
|
750
756
|
const publish = (name, value) => {
|
|
757
|
+
if (name === 'process.json') {
|
|
758
|
+
value = { ...value, recovery: { ...admission.breaker.recoveryStatus(), observedAt: new Date().toISOString() } };
|
|
759
|
+
if (options.startupCorrectionRecord) value.startupCorrection = options.startupCorrectionRecord;
|
|
760
|
+
latestProcessRecord = value;
|
|
761
|
+
}
|
|
751
762
|
const write = publishing.catch(() => {}).then(() => writeRecord(runDir, name, value, inspect));
|
|
752
763
|
publishing = write.catch(error => { publishFailures.push({ name, error: error?.message ?? String(error) }); });
|
|
753
764
|
return write;
|
|
754
765
|
};
|
|
766
|
+
const publishRecovery = () => {
|
|
767
|
+
if (latestProcessRecord) return publish('process.json', latestProcessRecord);
|
|
768
|
+
};
|
|
769
|
+
const onScopeTransition = () => { publishRecovery()?.catch(() => {}); };
|
|
770
|
+
admission.breaker.on('transition', onScopeTransition);
|
|
755
771
|
const handles = [];
|
|
756
772
|
let outcome;
|
|
757
773
|
let artifacts;
|
|
@@ -914,6 +930,8 @@ async function launchOneWorker(owner, options) {
|
|
|
914
930
|
// accumulated across this attempt's lifecycle on the returned result,
|
|
915
931
|
// mirroring `publishFailures` — never silently discarded.
|
|
916
932
|
if (ledgerDiagnostics.length && result) result.ledgerDiagnostics = ledgerDiagnostics.slice();
|
|
933
|
+
directStartupCorrectionContext.set(owner, { saved, invocation, loadInputs, clock, dispatchGuard,
|
|
934
|
+
ledger: admission.ledger, inspect, qualification: startupQualification });
|
|
917
935
|
return result;
|
|
918
936
|
} catch (error) {
|
|
919
937
|
if (!outcome?.child) {
|
|
@@ -985,6 +1003,8 @@ async function launchOneWorker(owner, options) {
|
|
|
985
1003
|
if (ledgerDiagnostics.length) error.ledgerDiagnostics = ledgerDiagnostics;
|
|
986
1004
|
throw error;
|
|
987
1005
|
} finally {
|
|
1006
|
+
admission.breaker.removeListener('transition', onScopeTransition);
|
|
1007
|
+
await publishing;
|
|
988
1008
|
try { await artifacts?.cleanup(); } finally {
|
|
989
1009
|
// Never chmod a worker-replaced pathname; only the inodes we opened.
|
|
990
1010
|
for (const handle of handles) { await handle.chmod(0o600).catch(() => {}); await handle.close().catch(() => {}); }
|
|
@@ -3916,6 +3936,102 @@ export function classifyAttemptForSettlement(invocation, result, { quotaAccount
|
|
|
3916
3936
|
inferenceProvider: selected.inferenceProvider ?? null, quotaAccount, installationQualifier };
|
|
3917
3937
|
}
|
|
3918
3938
|
|
|
3939
|
+
/** Refresh only a demonstrated stale executable binding. Installation, login,
|
|
3940
|
+
* permissions, models, working directories and policy are never repaired here.
|
|
3941
|
+
* One retry shares the original task clock and the existing atomic ledgers. */
|
|
3942
|
+
async function correctDirectWorkerStartup(owner, options, previous) {
|
|
3943
|
+
const context = directStartupCorrectionContext.get(owner);
|
|
3944
|
+
if (!context || options.correctStartup !== undefined || previous?.errorCode !== 'ENOENT'
|
|
3945
|
+
|| previous.termination !== 'confirmed' || previous.exitCode != null || previous.signal != null
|
|
3946
|
+
|| previous.cancelled || previous.timedOut || previous.publishFailures?.length
|
|
3947
|
+
|| classifyLaunchOutcomeForRecovery(previous).reasonCode !== 'NOT_FOUND') return previous;
|
|
3948
|
+
const { saved, invocation, loadInputs, clock, dispatchGuard, ledger, inspect, qualification } = context;
|
|
3949
|
+
const record = { action: 'refresh-executable-binding', reasonCode: 'NOT_FOUND', state: 'stopped',
|
|
3950
|
+
previousInvocationId: previous.invocationId };
|
|
3951
|
+
const recordAt = async (runDir, invocationId) => {
|
|
3952
|
+
const processRecord = JSON.parse((await boundedPrivateRead(join(runDir, 'process.json'), 65536, inspect)).toString('utf8'));
|
|
3953
|
+
if (processRecord.origin !== 'launcher' || processRecord.invocationId !== invocationId) throw new Error('Correction record identity changed');
|
|
3954
|
+
await (options.publishRecord ?? publishWorkerRecord)(runDir, 'process.json', { ...processRecord, startupCorrection: { ...record } }, inspect);
|
|
3955
|
+
};
|
|
3956
|
+
const stop = async code => {
|
|
3957
|
+
record.stopCode = code;
|
|
3958
|
+
previous.startupCorrection = { ...record };
|
|
3959
|
+
await recordAt(options.runDir, previous.invocationId);
|
|
3960
|
+
return previous;
|
|
3961
|
+
};
|
|
3962
|
+
const guardStop = () => dispatchGuard?.isCancelled?.() === true ? 'CANCELLED'
|
|
3963
|
+
: dispatchGuard?.deadlineAtMs != null && (dispatchGuard.nowMs?.() ?? Date.now()) >= dispatchGuard.deadlineAtMs ? 'DEADLINE' : null;
|
|
3964
|
+
if (ledger?.n >= ledger?.maxAttempts) return stop('ATTEMPT_LIMIT');
|
|
3965
|
+
if (guardStop()) return stop(guardStop());
|
|
3966
|
+
let inputs;
|
|
3967
|
+
let plan;
|
|
3968
|
+
const retryId = randomUUID();
|
|
3969
|
+
const finalPath = `${invocation.finalPath}.recovery-${retryId}`;
|
|
3970
|
+
try {
|
|
3971
|
+
inputs = await loadInputs();
|
|
3972
|
+
const { resolveWorkerInvocation } = await importImpl(import.meta.url, 'serve/worker-invocation.js');
|
|
3973
|
+
plan = await resolveWorkerInvocation(inputs.snapshot, { ...saved.request, final_output: finalPath }, inputs.drivers, clock());
|
|
3974
|
+
} catch { return stop('QUALIFICATION_REFUSED'); }
|
|
3975
|
+
if (guardStop()) return stop(guardStop());
|
|
3976
|
+
if (plan?.status !== 'eligible' || plan.mode !== 'direct' || plan.invocation?.readiness !== 'launchable') return stop('QUALIFICATION_REFUSED');
|
|
3977
|
+
if (plan.invocation.argv[0] === invocation.argv[0]) return stop('UNCHANGED_EXECUTABLE');
|
|
3978
|
+
// A freshly requalified installation may have new binary/proof digests.
|
|
3979
|
+
// Preserve every effective adapter declaration and installation boundary.
|
|
3980
|
+
const effectiveQualification = value => {
|
|
3981
|
+
const copy = structuredClone(value);
|
|
3982
|
+
const stripEvidence = evidence => {
|
|
3983
|
+
if (evidence) for (const key of ['scopeDigest', 'fixtureDigest', 'verifiedAt', 'evidenceRef']) delete evidence[key];
|
|
3984
|
+
};
|
|
3985
|
+
stripEvidence(copy.adapter?.qualification);
|
|
3986
|
+
for (const profile of Object.values(copy.adapter?.invocationProfiles ?? {})) stripEvidence(profile.installationQualification);
|
|
3987
|
+
if (copy.installation) {
|
|
3988
|
+
delete copy.installation.executable;
|
|
3989
|
+
delete copy.installation.evidence;
|
|
3990
|
+
}
|
|
3991
|
+
if (copy.stdinFramerBinding) delete copy.stdinFramerBinding.storeDigest;
|
|
3992
|
+
return JSON.stringify(copy);
|
|
3993
|
+
};
|
|
3994
|
+
if (effectiveQualification(qualification) !== effectiveQualification(inputs.drivers[invocation.selected.cli]?.qualification)) return stop('BINDING_CHANGED');
|
|
3995
|
+
// The resolver is authoritative for the replacement executable. Everything
|
|
3996
|
+
// else must remain identical, including qualified selection, policy, limits,
|
|
3997
|
+
// framing, environment, controls, prompt digest and workspace. Only fresh
|
|
3998
|
+
// output names and expiration are expected to differ between attempts.
|
|
3999
|
+
const binding = value => {
|
|
4000
|
+
const copy = structuredClone(value);
|
|
4001
|
+
const replacements = ['stderrPath', 'stdoutPath', 'finalPath'].map(key => [copy[key], `<${key}>`]);
|
|
4002
|
+
const normalize = value => replacements.find(([path]) => path === value)?.[1] ?? value;
|
|
4003
|
+
copy.argv = copy.argv.map((value, index) => index === 0 ? '<executable>' : normalize(value));
|
|
4004
|
+
delete copy.selected.qualificationDigest;
|
|
4005
|
+
for (const key of ['stdoutPath', 'stderrPath', 'finalPath', 'expiresAt', 'shellCommand', 'profileDigest']) delete copy[key];
|
|
4006
|
+
return JSON.stringify(copy);
|
|
4007
|
+
};
|
|
4008
|
+
if (binding(plan.invocation) !== binding(invocation)) return stop('BINDING_CHANGED');
|
|
4009
|
+
const invocationPath = join(options.runDir, `recovery-plan-${retryId}.json`);
|
|
4010
|
+
const runDir = `${options.runDir}.recovery-${retryId}`;
|
|
4011
|
+
if (guardStop()) return stop(guardStop());
|
|
4012
|
+
let retry;
|
|
4013
|
+
try {
|
|
4014
|
+
const planHandle = await withWorkerPath(invocationPath, target => open(target,
|
|
4015
|
+
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600));
|
|
4016
|
+
try { await planHandle.writeFile(JSON.stringify(plan)); await planHandle.sync(); } finally { await planHandle.close(); }
|
|
4017
|
+
// Publication happens before any retry launch. A record failure stops work.
|
|
4018
|
+
record.state = 'applied';
|
|
4019
|
+
await recordAt(options.runDir, previous.invocationId);
|
|
4020
|
+
retry = await launchOneWorker(owner, { ...options, invocationPath, runDir, dispatchGuard,
|
|
4021
|
+
inputs: undefined, loadInputs, retryOf: { invocationId: previous.invocationId, runDir: options.runDir },
|
|
4022
|
+
correctStartup: { reasonCode: 'NOT_FOUND' }, startupCorrectionRecord: { ...record } });
|
|
4023
|
+
} catch {
|
|
4024
|
+
record.state = 'stopped';
|
|
4025
|
+
return stop(guardStop() ?? 'RETRY_REFUSED');
|
|
4026
|
+
}
|
|
4027
|
+
record.retryInvocationId = retry.invocationId;
|
|
4028
|
+
if (retry.attemptOutcome !== 'succeeded') { record.state = 'stopped'; record.stopCode = 'RETRY_FAILED'; }
|
|
4029
|
+
retry.startupCorrection = { ...record };
|
|
4030
|
+
await recordAt(options.runDir, previous.invocationId);
|
|
4031
|
+
await recordAt(runDir, retry.invocationId);
|
|
4032
|
+
return retry;
|
|
4033
|
+
}
|
|
4034
|
+
|
|
3919
4035
|
// #2915 round 8 (D6, "lazy public fields still differ after mixed use"):
|
|
3920
4036
|
// `lastAttemptResult`/`lastAttemptInvocationId` are informational
|
|
3921
4037
|
// bookkeeping only (#2915 step 4 — the retry gate no longer reads them;
|
|
@@ -3987,8 +4103,9 @@ export class SessionLauncher extends EventEmitter {
|
|
|
3987
4103
|
* `options.correctStartup: { reasonCode, correctionsUsed }` enforces the
|
|
3988
4104
|
* one-qualified-startup-correction budget on a conductor-declared
|
|
3989
4105
|
* correction retry (startupCorrectionGate): unqualified causes and second
|
|
3990
|
-
* corrections are refused, within the task's existing allowance.
|
|
3991
|
-
*
|
|
4106
|
+
* corrections are refused, within the task's existing allowance. The only
|
|
4107
|
+
* automatic correction refreshes a changed qualified executable binding
|
|
4108
|
+
* after a launcher-observed ENOENT with confirmed stop, once per task chain.
|
|
3992
4109
|
*
|
|
3993
4110
|
* Both checks are gated recovery behavior, not a general launch
|
|
3994
4111
|
* precondition: enforcement happens entirely INSIDE `launchOneWorker`,
|
|
@@ -4015,8 +4132,13 @@ export class SessionLauncher extends EventEmitter {
|
|
|
4015
4132
|
// `options.retryOf`-driven and disk-verified every time) is set inside
|
|
4016
4133
|
// `launchOneWorker` itself, gated on whether recovery actually admitted
|
|
4017
4134
|
// this attempt (#2915 review 6, finding 6) — never here unconditionally.
|
|
4018
|
-
|
|
4135
|
+
directStartupCorrectionContext.delete(this);
|
|
4136
|
+
try {
|
|
4137
|
+
const result = await launchOneWorker(this, options);
|
|
4138
|
+
return await correctDirectWorkerStartup(this, options, result);
|
|
4139
|
+
}
|
|
4019
4140
|
finally {
|
|
4141
|
+
directStartupCorrectionContext.delete(this);
|
|
4020
4142
|
_workerRecoverySubscriptions.get(this)?.();
|
|
4021
4143
|
_workerRecoverySubscriptions.delete(this);
|
|
4022
4144
|
this.workerLaunchPending = false;
|