@voybio/ace-swarm 0.2.4 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -1
- package/README.md +20 -13
- package/assets/.agents/skills/eval-harness/SKILL.md +14 -0
- package/assets/.agents/skills/handoff-lint/SKILL.md +14 -0
- package/assets/.agents/skills/incident-commander/SKILL.md +14 -0
- package/assets/.agents/skills/memory-curator/SKILL.md +14 -0
- package/assets/.agents/skills/release-sentry/SKILL.md +14 -0
- package/assets/.agents/skills/risk-quant/SKILL.md +14 -0
- package/assets/.agents/skills/schema-forge/SKILL.md +14 -0
- package/assets/.agents/skills/state-auditor/SKILL.md +14 -0
- package/assets/agent-state/EVIDENCE_LOG.md +1 -1
- package/assets/agent-state/MODULES/gates/gate-correctness.json +1 -1
- package/assets/agent-state/MODULES/roles/capability-framework.json +41 -0
- package/assets/agent-state/MODULES/roles/capability-git.json +33 -0
- package/assets/agent-state/MODULES/roles/capability-safety.json +37 -0
- package/assets/agent-state/MODULES/schemas/ACE_RUNTIME_PROFILE.schema.json +21 -0
- package/assets/agent-state/MODULES/schemas/RUNTIME_EXECUTOR_SESSION_REGISTRY.schema.json +43 -0
- package/assets/agent-state/MODULES/schemas/WORKSPACE_SESSION_REGISTRY.schema.json +11 -0
- package/assets/agent-state/STATUS.md +2 -2
- package/assets/scripts/ace-hook-dispatch.mjs +70 -6
- package/assets/scripts/render-mcp-configs.sh +19 -5
- package/dist/ace-context.js +22 -1
- package/dist/ace-server-instructions.js +3 -3
- package/dist/ace-state-resolver.js +5 -3
- package/dist/astgrep-index.d.ts +9 -1
- package/dist/astgrep-index.js +14 -3
- package/dist/cli.js +52 -20
- package/dist/handoff-registry.js +5 -5
- package/dist/helpers/artifacts.d.ts +19 -0
- package/dist/helpers/artifacts.js +152 -0
- package/dist/helpers/bootstrap.d.ts +24 -0
- package/dist/helpers/bootstrap.js +894 -0
- package/dist/helpers/constants.d.ts +53 -0
- package/dist/helpers/constants.js +288 -0
- package/dist/helpers/drift.d.ts +13 -0
- package/dist/helpers/drift.js +45 -0
- package/dist/helpers/path-utils.d.ts +17 -0
- package/dist/helpers/path-utils.js +104 -0
- package/dist/helpers/store-resolution.d.ts +19 -0
- package/dist/helpers/store-resolution.js +301 -0
- package/dist/helpers/workspace-root.d.ts +3 -0
- package/dist/helpers/workspace-root.js +80 -0
- package/dist/helpers.d.ts +8 -123
- package/dist/helpers.js +8 -1747
- package/dist/job-scheduler.js +3 -3
- package/dist/local-model-runtime.js +12 -1
- package/dist/model-bridge.d.ts +7 -0
- package/dist/model-bridge.js +75 -5
- package/dist/orchestrator-supervisor.d.ts +14 -0
- package/dist/orchestrator-supervisor.js +72 -1
- package/dist/run-ledger.js +3 -3
- package/dist/runtime-command.d.ts +8 -0
- package/dist/runtime-command.js +38 -6
- package/dist/runtime-executor.d.ts +14 -0
- package/dist/runtime-executor.js +669 -171
- package/dist/runtime-profile.d.ts +32 -0
- package/dist/runtime-profile.js +89 -13
- package/dist/runtime-tool-specs.d.ts +21 -0
- package/dist/runtime-tool-specs.js +78 -3
- package/dist/safe-edit.d.ts +7 -0
- package/dist/safe-edit.js +163 -37
- package/dist/schemas.js +19 -0
- package/dist/shared.d.ts +2 -2
- package/dist/status-events.js +9 -6
- package/dist/store/ace-packed-store.d.ts +3 -2
- package/dist/store/ace-packed-store.js +188 -110
- package/dist/store/bootstrap-store.d.ts +1 -1
- package/dist/store/bootstrap-store.js +94 -81
- package/dist/store/cache-workspace.d.ts +22 -0
- package/dist/store/cache-workspace.js +149 -0
- package/dist/store/materializers/context-snapshot-materializer.js +6 -7
- package/dist/store/materializers/hook-context-materializer.d.ts +6 -9
- package/dist/store/materializers/hook-context-materializer.js +11 -21
- package/dist/store/materializers/host-file-materializer.js +6 -0
- package/dist/store/materializers/projection-manager.d.ts +0 -1
- package/dist/store/materializers/projection-manager.js +5 -13
- package/dist/store/materializers/scheduler-projection-materializer.js +1 -1
- package/dist/store/materializers/vericify-projector.d.ts +7 -7
- package/dist/store/materializers/vericify-projector.js +11 -11
- package/dist/store/repositories/local-model-runtime-repository.d.ts +120 -3
- package/dist/store/repositories/local-model-runtime-repository.js +242 -6
- package/dist/store/skills-install.d.ts +4 -0
- package/dist/store/skills-install.js +21 -12
- package/dist/store/state-reader.d.ts +2 -0
- package/dist/store/state-reader.js +20 -0
- package/dist/store/store-artifacts.d.ts +7 -0
- package/dist/store/store-artifacts.js +27 -1
- package/dist/store/store-authority-audit.d.ts +18 -1
- package/dist/store/store-authority-audit.js +115 -5
- package/dist/store/store-snapshot.d.ts +3 -0
- package/dist/store/store-snapshot.js +22 -2
- package/dist/store/workspace-store-paths.d.ts +39 -0
- package/dist/store/workspace-store-paths.js +94 -0
- package/dist/store/write-coordinator.d.ts +65 -0
- package/dist/store/write-coordinator.js +386 -0
- package/dist/todo-state.js +5 -5
- package/dist/tools-agent.js +319 -34
- package/dist/tools-discovery.js +1 -1
- package/dist/tools-files.d.ts +7 -0
- package/dist/tools-files.js +299 -10
- package/dist/tools-framework.js +107 -27
- package/dist/tools-handoff.js +2 -2
- package/dist/tools-lifecycle.js +4 -4
- package/dist/tools-memory.js +6 -6
- package/dist/tools-todo.js +2 -2
- package/dist/tracker-adapters.d.ts +1 -1
- package/dist/tracker-adapters.js +13 -18
- package/dist/tracker-sync.js +5 -3
- package/dist/tui/agent-runner.js +3 -1
- package/dist/tui/chat.js +103 -7
- package/dist/tui/dashboard.d.ts +1 -0
- package/dist/tui/dashboard.js +43 -0
- package/dist/tui/layout.d.ts +20 -0
- package/dist/tui/layout.js +31 -1
- package/dist/tui/local-model-contract.d.ts +6 -2
- package/dist/tui/local-model-contract.js +16 -3
- package/dist/vericify-bridge.d.ts +5 -0
- package/dist/vericify-bridge.js +27 -3
- package/dist/workspace-manager.d.ts +30 -3
- package/dist/workspace-manager.js +257 -27
- package/package.json +1 -2
- package/dist/internal-tool-runtime.d.ts +0 -21
- package/dist/internal-tool-runtime.js +0 -136
- package/dist/store/workspace-snapshot.d.ts +0 -26
- package/dist/store/workspace-snapshot.js +0 -107
package/dist/runtime-executor.js
CHANGED
|
@@ -1,27 +1,121 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, mkdirSync, mkdtempSync, readFileSync,
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync, } from "node:fs";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join, resolve } from "node:path";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
-
import { resolveWorkspaceArtifactPath as resolveWorkspaceArtifactPathHelper, resolveWorkspaceRoot, safeRead,
|
|
6
|
+
import { resolveWorkspaceArtifactPath as resolveWorkspaceArtifactPathHelper, resolveWorkspaceRoot, safeRead, safeWriteAsync, withFileLock, } from "./helpers.js";
|
|
7
7
|
import { appendRunLedgerEntrySafe } from "./run-ledger.js";
|
|
8
8
|
import { runShellCommand } from "./runtime-command.js";
|
|
9
|
-
import {
|
|
9
|
+
import { loadRuntimeProfile, resolveEffectiveSurgicalReadBudget, renderRuntimePromptFromSnapshot, getLivenessDefaults, } from "./runtime-profile.js";
|
|
10
10
|
import { renderAceContinuityPromptBlock, renderAceRecallPromptBlock, renderAceSnapshotPromptBlock, } from "./ace-context.js";
|
|
11
|
-
import { executeRuntimeTool,
|
|
11
|
+
import { executeRuntimeTool, buildFilteredToolCatalog, SMALL_LOCAL_PRIORITY_TOOL_NAMES, } from "./runtime-tool-specs.js";
|
|
12
12
|
import { validateRuntimeExecutorSessionRegistryPayload, } from "./schemas.js";
|
|
13
13
|
import { appendStatusEventSafe } from "./status-events.js";
|
|
14
14
|
import { appendVericifyProcessPostSafe, deriveVericifyRunRef, isVericifyBridgeEnabled, refreshVericifyBridgeSnapshotSafe, } from "./vericify-bridge.js";
|
|
15
|
-
import {
|
|
15
|
+
import { createWorkspaceSessionAsync, removeWorkspaceSession, runWorkspaceSessionHook, } from "./workspace-manager.js";
|
|
16
16
|
import { buildAceContinuityPacket, buildAceRecallContext, normalizeAutonomyPolicy, normalizeContinuityPolicy, } from "./ace-autonomy.js";
|
|
17
17
|
import { openStore } from "./store/ace-packed-store.js";
|
|
18
|
-
import { getWorkspaceStorePath, storeExistsSync,
|
|
18
|
+
import { getWorkspaceStorePath, storeExistsSync, } from "./store/store-snapshot.js";
|
|
19
19
|
import { operationalArtifactVirtualPath } from "./store/store-artifacts.js";
|
|
20
|
-
import {
|
|
20
|
+
import { withStoreWriteCoordinator } from "./store/write-coordinator.js";
|
|
21
|
+
import { LocalModelRuntimeRepository, withLocalModelRuntimeRepository, } from "./store/repositories/local-model-runtime-repository.js";
|
|
21
22
|
export const RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH = "agent-state/runtime-executor-sessions.json";
|
|
22
23
|
export const RUNTIME_EXECUTOR_SESSION_SCHEMA_REL_PATH = "agent-state/MODULES/schemas/RUNTIME_EXECUTOR_SESSION_REGISTRY.schema.json";
|
|
23
24
|
export const RUNTIME_EXECUTOR_SESSION_SCHEMA_NAME = "runtime-executor-session-registry@1.0.0";
|
|
25
|
+
const WORKSPACE_ROOT = resolveWorkspaceRoot();
|
|
26
|
+
let activeWorkspaceRoot = WORKSPACE_ROOT;
|
|
27
|
+
export const DEFAULT_TURN_OUTPUT_POLICY = {
|
|
28
|
+
emit_to: ["tui", "vericify"],
|
|
29
|
+
silent_unless_blocked: false,
|
|
30
|
+
require_approval_before_emit: false,
|
|
31
|
+
};
|
|
32
|
+
function buildToolChoiceAuditRecord(input) {
|
|
33
|
+
const priority_tools = SMALL_LOCAL_PRIORITY_TOOL_NAMES.filter((name) => input.filteredCatalog.entries.some((entry) => entry.name === name));
|
|
34
|
+
const demoted_tools = [
|
|
35
|
+
...(input.model_class === "small_local"
|
|
36
|
+
? input.filteredCatalog.entries
|
|
37
|
+
.filter((entry) => entry.cost_class === "heavy")
|
|
38
|
+
.map((entry) => ({
|
|
39
|
+
name: entry.name,
|
|
40
|
+
reason_code: "heavy_for_small_local",
|
|
41
|
+
}))
|
|
42
|
+
: []),
|
|
43
|
+
...input.filteredCatalog.unavailable_tools
|
|
44
|
+
.filter((entry) => entry.reason_code === "workspace_disabled" || entry.reason_code === "policy_disabled")
|
|
45
|
+
.map((entry) => ({
|
|
46
|
+
name: entry.name,
|
|
47
|
+
reason_code: entry.reason_code,
|
|
48
|
+
})),
|
|
49
|
+
];
|
|
50
|
+
return {
|
|
51
|
+
audit_id: randomUUID(),
|
|
52
|
+
session_id: input.session_id,
|
|
53
|
+
turn_number: input.turn_number,
|
|
54
|
+
captured_at: input.captured_at,
|
|
55
|
+
model_class: input.model_class,
|
|
56
|
+
priority_tools,
|
|
57
|
+
demoted_tools,
|
|
58
|
+
selected_budget: input.selected_budget,
|
|
59
|
+
capability_snapshot_id: input.capability_snapshot_id,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
const sessionSidecarStates = new Map();
|
|
63
|
+
function initSessionSidecars(sessionId) {
|
|
64
|
+
let release;
|
|
65
|
+
const gate = new Promise((resolve) => {
|
|
66
|
+
release = resolve;
|
|
67
|
+
});
|
|
68
|
+
sessionSidecarStates.set(sessionId, { chain: gate, release });
|
|
69
|
+
}
|
|
70
|
+
function scheduleRuntimeSidecar(sessionId, _label, fn) {
|
|
71
|
+
const state = sessionSidecarStates.get(sessionId);
|
|
72
|
+
if (!state)
|
|
73
|
+
return;
|
|
74
|
+
const next = state.chain.then(() => fn()).catch(() => undefined);
|
|
75
|
+
state.chain = next;
|
|
76
|
+
}
|
|
77
|
+
function releaseSidecarsAndForget(sessionId) {
|
|
78
|
+
const state = sessionSidecarStates.get(sessionId);
|
|
79
|
+
if (!state)
|
|
80
|
+
return;
|
|
81
|
+
if (state.release) {
|
|
82
|
+
state.release();
|
|
83
|
+
state.release = undefined;
|
|
84
|
+
}
|
|
85
|
+
// Let sidecars run in background; clean up state when chain completes.
|
|
86
|
+
void state.chain.finally(() => {
|
|
87
|
+
sessionSidecarStates.delete(sessionId);
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
async function waitForSessionSidecars(sessionId) {
|
|
91
|
+
const state = sessionSidecarStates.get(sessionId);
|
|
92
|
+
if (!state)
|
|
93
|
+
return;
|
|
94
|
+
await state.chain.catch(() => undefined);
|
|
95
|
+
}
|
|
96
|
+
function setSessionPhase(sessionId, phase, checkpoint) {
|
|
97
|
+
const active = activeSessions.get(sessionId);
|
|
98
|
+
if (active) {
|
|
99
|
+
active.phase = phase;
|
|
100
|
+
active.phase_started_at = Date.now();
|
|
101
|
+
if (checkpoint !== undefined)
|
|
102
|
+
active.last_checkpoint = checkpoint;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
24
105
|
const activeSessions = new Map();
|
|
106
|
+
async function readLocalModelRuntimeStatus(sessionId) {
|
|
107
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
108
|
+
return undefined;
|
|
109
|
+
const storePath = getWorkspaceStorePath(workspaceRoot());
|
|
110
|
+
const store = await openStore(storePath);
|
|
111
|
+
try {
|
|
112
|
+
const repo = new LocalModelRuntimeRepository(store);
|
|
113
|
+
return await repo.getRuntimeStatus(sessionId);
|
|
114
|
+
}
|
|
115
|
+
finally {
|
|
116
|
+
await store.close();
|
|
117
|
+
}
|
|
118
|
+
}
|
|
25
119
|
const turnResponseSchema = z
|
|
26
120
|
.object({
|
|
27
121
|
status: z.enum(["continue", "done", "blocked", "failed"]),
|
|
@@ -45,14 +139,27 @@ function defaultRegistry() {
|
|
|
45
139
|
};
|
|
46
140
|
}
|
|
47
141
|
function workspaceRoot() {
|
|
48
|
-
return
|
|
142
|
+
return WORKSPACE_ROOT;
|
|
143
|
+
}
|
|
144
|
+
function runtimeProfilePath() {
|
|
145
|
+
return resolve(WORKSPACE_ROOT, "agent-state", "ACE_WORKFLOW.md");
|
|
146
|
+
}
|
|
147
|
+
function loadCurrentRuntimeProfile() {
|
|
148
|
+
const workspaceProfilePath = runtimeProfilePath();
|
|
149
|
+
if (existsSync(workspaceProfilePath)) {
|
|
150
|
+
const result = loadRuntimeProfile(workspaceProfilePath);
|
|
151
|
+
if (result.ok)
|
|
152
|
+
return result;
|
|
153
|
+
throw new Error(`ACE workflow runtime profile is invalid: ${result.errors.join("; ")}`);
|
|
154
|
+
}
|
|
155
|
+
const packaged = loadRuntimeProfile();
|
|
156
|
+
if (packaged.ok)
|
|
157
|
+
return packaged;
|
|
158
|
+
throw new Error(`ACE runtime profile is invalid: ${packaged.errors.join("; ")}`);
|
|
49
159
|
}
|
|
50
160
|
function resolveWorkspaceArtifactPath(filePath) {
|
|
51
161
|
return resolveWorkspaceArtifactPathHelper(filePath, "write");
|
|
52
162
|
}
|
|
53
|
-
function safeWriteWorkspaceFile(filePath, content) {
|
|
54
|
-
return safeWrite(filePath, content);
|
|
55
|
-
}
|
|
56
163
|
function parseRegistry(raw) {
|
|
57
164
|
let parsed;
|
|
58
165
|
try {
|
|
@@ -69,17 +176,19 @@ function parseRegistry(raw) {
|
|
|
69
176
|
}
|
|
70
177
|
function readRegistry() {
|
|
71
178
|
const raw = safeRead(RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH);
|
|
72
|
-
if (raw.startsWith("[FILE NOT FOUND]") ||
|
|
179
|
+
if (raw.startsWith("[FILE NOT FOUND]") ||
|
|
180
|
+
raw.startsWith("[ACCESS DENIED]") ||
|
|
181
|
+
raw.includes("\x00")) {
|
|
73
182
|
return defaultRegistry();
|
|
74
183
|
}
|
|
75
184
|
return parseRegistry(raw);
|
|
76
185
|
}
|
|
77
|
-
function writeRegistry(registry) {
|
|
186
|
+
async function writeRegistry(registry) {
|
|
78
187
|
const validation = validateRuntimeExecutorSessionRegistryPayload(registry);
|
|
79
188
|
if (!validation.ok) {
|
|
80
189
|
throw new Error(`Runtime executor session registry failed validation (${validation.schema}): ${validation.errors.join("; ")}`);
|
|
81
190
|
}
|
|
82
|
-
return
|
|
191
|
+
return safeWriteAsync(RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH, JSON.stringify(registry, null, 2));
|
|
83
192
|
}
|
|
84
193
|
function registryPath() {
|
|
85
194
|
if (storeExistsSync(workspaceRoot())) {
|
|
@@ -92,7 +201,7 @@ async function mutateRegistry(updater) {
|
|
|
92
201
|
const registry = readRegistry();
|
|
93
202
|
const result = await updater(registry);
|
|
94
203
|
registry.updated_at = new Date().toISOString();
|
|
95
|
-
writeRegistry(registry);
|
|
204
|
+
await writeRegistry(registry);
|
|
96
205
|
return result;
|
|
97
206
|
});
|
|
98
207
|
}
|
|
@@ -245,6 +354,24 @@ async function emitVericifyProcessPostForSession(session, kind, summary, toolRef
|
|
|
245
354
|
}).catch(() => undefined);
|
|
246
355
|
refreshVericifyBridgeSnapshotSafe();
|
|
247
356
|
}
|
|
357
|
+
async function touchRuntimeProgress(sessionId, _reason, at = Date.now()) {
|
|
358
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
359
|
+
return;
|
|
360
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
361
|
+
const currentStatus = await repo.getRuntimeStatus(sessionId);
|
|
362
|
+
if (!currentStatus)
|
|
363
|
+
return;
|
|
364
|
+
await repo.upsertRuntimeStatusWithTransition({
|
|
365
|
+
...currentStatus,
|
|
366
|
+
sleep_state: currentStatus.sleep_state === "stalled" ? "awake" : currentStatus.sleep_state,
|
|
367
|
+
last_progress_at: at,
|
|
368
|
+
}, {
|
|
369
|
+
from: currentStatus.bridge_status,
|
|
370
|
+
reason: _reason,
|
|
371
|
+
evidence_refs: [`session:${sessionId}`, `progress_at:${at}`],
|
|
372
|
+
});
|
|
373
|
+
}).catch(() => undefined);
|
|
374
|
+
}
|
|
248
375
|
async function emitExecutorEvent(eventType, status, summary, payload) {
|
|
249
376
|
await appendStatusEventSafe({
|
|
250
377
|
source_module: "capability-framework",
|
|
@@ -300,7 +427,10 @@ async function persistTurnArtifacts(paths, prompt, requestPayload, responseRaw)
|
|
|
300
427
|
};
|
|
301
428
|
}
|
|
302
429
|
const storePath = getWorkspaceStorePath(workspaceRoot());
|
|
303
|
-
|
|
430
|
+
writeFileSync(paths.tempPromptPath, prompt, "utf-8");
|
|
431
|
+
writeFileSync(paths.tempRequestPath, JSON.stringify(requestPayload, null, 2), "utf-8");
|
|
432
|
+
writeFileSync(paths.tempResponsePath, responseRaw, "utf-8");
|
|
433
|
+
await withStoreWriteCoordinator(storePath, async () => {
|
|
304
434
|
const store = await openStore(storePath);
|
|
305
435
|
try {
|
|
306
436
|
await store.setBlob(paths.promptStoreKey, prompt);
|
|
@@ -311,27 +441,31 @@ async function persistTurnArtifacts(paths, prompt, requestPayload, responseRaw)
|
|
|
311
441
|
finally {
|
|
312
442
|
await store.close();
|
|
313
443
|
}
|
|
314
|
-
});
|
|
444
|
+
}, { operation_label: "runtime-executor-turn-artifacts" });
|
|
315
445
|
return {
|
|
316
|
-
promptPath:
|
|
317
|
-
requestPath:
|
|
318
|
-
responsePath:
|
|
446
|
+
promptPath: paths.tempPromptPath,
|
|
447
|
+
requestPath: paths.tempRequestPath,
|
|
448
|
+
responsePath: paths.tempResponsePath,
|
|
319
449
|
};
|
|
320
450
|
}
|
|
321
|
-
async function runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook) {
|
|
451
|
+
async function runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook, runtimeProfilePath) {
|
|
322
452
|
let cleanupError;
|
|
323
453
|
let cleanupStatus = "pending";
|
|
324
454
|
if (shouldRunAfterHook) {
|
|
325
|
-
const afterRun = runWorkspaceSessionHook({
|
|
455
|
+
const afterRun = await runWorkspaceSessionHook({
|
|
326
456
|
session_id: workspaceSessionId,
|
|
327
457
|
kind: "after_run",
|
|
458
|
+
runtime_profile_path: runtimeProfilePath,
|
|
328
459
|
});
|
|
329
460
|
if (!afterRun.ok) {
|
|
330
461
|
cleanupError = afterRun.error ?? "after_run hook failed";
|
|
331
462
|
cleanupStatus = "failed";
|
|
332
463
|
}
|
|
333
464
|
}
|
|
334
|
-
const removal = removeWorkspaceSession({
|
|
465
|
+
const removal = await removeWorkspaceSession({
|
|
466
|
+
session_id: workspaceSessionId,
|
|
467
|
+
runtime_profile_path: runtimeProfilePath,
|
|
468
|
+
});
|
|
335
469
|
if (removal.ok && removal.session) {
|
|
336
470
|
cleanupStatus =
|
|
337
471
|
removal.session.status === "archived" ? "archived" : "removed";
|
|
@@ -340,6 +474,8 @@ async function runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook) {
|
|
|
340
474
|
cleanupError = removal.error ?? "Workspace session cleanup failed";
|
|
341
475
|
cleanupStatus = "failed";
|
|
342
476
|
}
|
|
477
|
+
// Update the registry first (critical lane) then emit the transition record as a sidecar.
|
|
478
|
+
setSessionPhase(sessionId, "cleanup_registry_update");
|
|
343
479
|
await mutateRegistry((registry) => {
|
|
344
480
|
const session = findSession(registry, sessionId);
|
|
345
481
|
if (!session)
|
|
@@ -352,15 +488,54 @@ async function runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook) {
|
|
|
352
488
|
if (cleanupError)
|
|
353
489
|
session.cleanup_error = cleanupError;
|
|
354
490
|
});
|
|
491
|
+
const capturedCleanupStatus = cleanupStatus;
|
|
492
|
+
const capturedCleanupError = cleanupError;
|
|
493
|
+
const capturedWorkspaceSessionId = workspaceSessionId;
|
|
494
|
+
scheduleRuntimeSidecar(sessionId, "cleanup-transition", async () => {
|
|
495
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
496
|
+
return;
|
|
497
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
498
|
+
await repo.appendTransitionRecord({
|
|
499
|
+
session_id: sessionId,
|
|
500
|
+
subject_kind: "workspace_session",
|
|
501
|
+
subject_id: capturedWorkspaceSessionId,
|
|
502
|
+
from: "active",
|
|
503
|
+
to: capturedCleanupStatus === "archived" ? "archived" : capturedCleanupStatus === "removed" ? "removed" : "failed",
|
|
504
|
+
reason: capturedCleanupError
|
|
505
|
+
? `Workspace session ${capturedWorkspaceSessionId} cleanup failed: ${capturedCleanupError}`
|
|
506
|
+
: `Workspace session ${capturedWorkspaceSessionId} cleanup completed.`,
|
|
507
|
+
reason_code: capturedCleanupError ? "cleanup_failed" : "cleanup_complete",
|
|
508
|
+
evidence_refs: [capturedWorkspaceSessionId],
|
|
509
|
+
});
|
|
510
|
+
}).catch(() => undefined);
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
function classifyTurnOutcome(responseStatus, policy, toolCallCount) {
|
|
514
|
+
if (responseStatus === "blocked" || responseStatus === "failed") {
|
|
515
|
+
return {
|
|
516
|
+
outcome: "escalation_blocker",
|
|
517
|
+
reason: `Turn ended with status=${responseStatus}; external action required`,
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
if (policy.silent_unless_blocked && toolCallCount === 0) {
|
|
521
|
+
return {
|
|
522
|
+
outcome: "no_op_success",
|
|
523
|
+
reason: "silent_unless_blocked=true and turn produced no tool calls",
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
return {
|
|
527
|
+
outcome: "meaningful_completion",
|
|
528
|
+
reason: `Turn completed normally with ${toolCallCount} tool call(s)`,
|
|
529
|
+
};
|
|
355
530
|
}
|
|
356
|
-
async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
531
|
+
async function runSessionLoop(sessionId, context, autoCleanup, runtimeProfileSnapshot) {
|
|
357
532
|
const active = activeSessions.get(sessionId);
|
|
358
533
|
let shouldRunAfterHook = false;
|
|
359
534
|
let workspaceSessionId = "";
|
|
360
535
|
try {
|
|
361
|
-
const
|
|
362
|
-
const
|
|
363
|
-
|
|
536
|
+
const autonomyPolicy = normalizeAutonomyPolicy(runtimeProfileSnapshot.profile.autonomy);
|
|
537
|
+
const continuityPolicy = normalizeContinuityPolicy(runtimeProfileSnapshot.profile.continuity);
|
|
538
|
+
setSessionPhase(sessionId, "registry_start");
|
|
364
539
|
const registryStart = await mutateRegistry((registry) => {
|
|
365
540
|
const session = findSession(registry, sessionId);
|
|
366
541
|
if (!session)
|
|
@@ -374,6 +549,7 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
374
549
|
if (!registryStart) {
|
|
375
550
|
throw new Error(`Failed to start unattended session: ${sessionId}`);
|
|
376
551
|
}
|
|
552
|
+
setSessionPhase(sessionId, "preflight");
|
|
377
553
|
const preflight = runAutonomyChecks("before_run", autonomyPolicy, context);
|
|
378
554
|
if (!preflight.ok) {
|
|
379
555
|
const summary = `Autonomy preflight blocked execution: ${preflight.summary}`;
|
|
@@ -391,9 +567,11 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
391
567
|
}
|
|
392
568
|
return;
|
|
393
569
|
}
|
|
394
|
-
|
|
570
|
+
setSessionPhase(sessionId, "workspace_before_run");
|
|
571
|
+
const beforeRun = await runWorkspaceSessionHook({
|
|
395
572
|
session_id: registryStart.workspace_session_id,
|
|
396
573
|
kind: "before_run",
|
|
574
|
+
runtime_profile_path: runtimeProfileSnapshot.path,
|
|
397
575
|
});
|
|
398
576
|
if (!beforeRun.ok) {
|
|
399
577
|
const summary = beforeRun.error ?? "before_run hook failed";
|
|
@@ -411,6 +589,49 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
411
589
|
shouldRunAfterHook = true;
|
|
412
590
|
let currentTask = registryStart.current_task;
|
|
413
591
|
let previousToolCalls = [];
|
|
592
|
+
const runtimeStatus = storeExistsSync(workspaceRoot())
|
|
593
|
+
? await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => repo.getRuntimeStatus(sessionId))
|
|
594
|
+
: undefined;
|
|
595
|
+
const effectiveModelClass = runtimeStatus?.model_class ?? "frontier";
|
|
596
|
+
// Stall restart state — session-level, reset on successful turn.
|
|
597
|
+
// Seeded runtime status is authoritative when present.
|
|
598
|
+
let consecutiveStallRestarts = runtimeStatus?.stall_restart_count ?? 0;
|
|
599
|
+
let isStallRestart = false;
|
|
600
|
+
let stallRestartContext = "";
|
|
601
|
+
// Liveness defaults are explicit runtime policy. The resolved model class
|
|
602
|
+
// only informs the default budget profile when the caller has not already
|
|
603
|
+
// supplied a workspace/session budget or status-seeded runtime budget.
|
|
604
|
+
const livenessDefaults = getLivenessDefaults(effectiveModelClass);
|
|
605
|
+
const effectiveTurnBudgetMs = runtimeStatus?.turn_budget_ms ?? registryStart.turn_timeout_ms ?? livenessDefaults.turn_budget_ms;
|
|
606
|
+
const effectiveStallWindowMs = runtimeStatus?.stall_window_ms ?? livenessDefaults.stall_window_ms;
|
|
607
|
+
const maxStallRestarts = livenessDefaults.max_stall_restarts;
|
|
608
|
+
const initialBackoffMs = livenessDefaults.initial_backoff_ms;
|
|
609
|
+
const maxRetryBackoffMs = livenessDefaults.max_retry_backoff_ms;
|
|
610
|
+
let lastProgressAt = Date.now();
|
|
611
|
+
let queuedProgress;
|
|
612
|
+
let progressWrite;
|
|
613
|
+
const pumpProgressWrite = () => {
|
|
614
|
+
if (progressWrite)
|
|
615
|
+
return progressWrite;
|
|
616
|
+
progressWrite = (async () => {
|
|
617
|
+
while (queuedProgress) {
|
|
618
|
+
const next = queuedProgress;
|
|
619
|
+
queuedProgress = undefined;
|
|
620
|
+
await touchRuntimeProgress(sessionId, next.reason, next.at).catch(() => undefined);
|
|
621
|
+
}
|
|
622
|
+
})().finally(() => {
|
|
623
|
+
progressWrite = undefined;
|
|
624
|
+
});
|
|
625
|
+
return progressWrite;
|
|
626
|
+
};
|
|
627
|
+
const noteProgress = (reason, at = Date.now()) => {
|
|
628
|
+
lastProgressAt = at;
|
|
629
|
+
queuedProgress = { reason, at };
|
|
630
|
+
void pumpProgressWrite();
|
|
631
|
+
};
|
|
632
|
+
const flushProgress = async () => {
|
|
633
|
+
await pumpProgressWrite().catch(() => undefined);
|
|
634
|
+
};
|
|
414
635
|
for (let turnNumber = 1; turnNumber <= registryStart.max_turns; turnNumber += 1) {
|
|
415
636
|
if (active?.stop_requested) {
|
|
416
637
|
const stopped = await finalizeSession(sessionId, "stopped", `Unattended session ${sessionId} stopped by request.`);
|
|
@@ -423,6 +644,7 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
423
644
|
}
|
|
424
645
|
return;
|
|
425
646
|
}
|
|
647
|
+
setSessionPhase(sessionId, "turn_preparing", `Turn ${turnNumber}`);
|
|
426
648
|
const sessionSnapshot = await mutateRegistry((registry) => {
|
|
427
649
|
const session = findSession(registry, sessionId);
|
|
428
650
|
if (!session)
|
|
@@ -435,10 +657,14 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
435
657
|
throw new Error(`Missing unattended session ${sessionId}`);
|
|
436
658
|
const turnStartedAt = new Date().toISOString();
|
|
437
659
|
const paths = buildTurnPaths(sessionId, sessionSnapshot.workspace_path, turnNumber);
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
660
|
+
// Build capability-filtered tool catalog in-memory from the resolved model class.
|
|
661
|
+
const filteredCatalog = buildFilteredToolCatalog(effectiveModelClass);
|
|
662
|
+
const capabilitySnapshotId = randomUUID();
|
|
663
|
+
const selectedBudget = resolveEffectiveSurgicalReadBudget(runtimeProfileSnapshot.profile, effectiveModelClass);
|
|
664
|
+
const toolCatalog = filteredCatalog.entries.map((e) => ({
|
|
665
|
+
name: e.name,
|
|
666
|
+
description: e.description,
|
|
667
|
+
input_schema: e.input_schema,
|
|
442
668
|
}));
|
|
443
669
|
const turnRecall = autonomyPolicy.recall_context
|
|
444
670
|
? buildAceRecallContext({
|
|
@@ -452,8 +678,11 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
452
678
|
snapshot_name: resolveSnapshotName(context),
|
|
453
679
|
})
|
|
454
680
|
: undefined;
|
|
455
|
-
const
|
|
456
|
-
task: currentTask
|
|
681
|
+
const taskWithStallHint = isStallRestart
|
|
682
|
+
? `${stallRestartContext}\n\nOriginal task: ${currentTask}`
|
|
683
|
+
: currentTask;
|
|
684
|
+
const prompt = renderRuntimePromptFromSnapshot(runtimeProfileSnapshot, {
|
|
685
|
+
task: taskWithStallHint,
|
|
457
686
|
session_id: sessionId,
|
|
458
687
|
turn_number: turnNumber,
|
|
459
688
|
workspace_path: sessionSnapshot.workspace_path,
|
|
@@ -493,13 +722,106 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
493
722
|
};
|
|
494
723
|
writeFileSync(paths.tempPromptPath, prompt, "utf-8");
|
|
495
724
|
writeFileSync(paths.tempRequestPath, JSON.stringify(requestPayload, null, 2), "utf-8");
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
725
|
+
// Sidecar: persist capability snapshot, tool-choice audit, turn snapshot, and TURN_STARTED event.
|
|
726
|
+
// These are audit evidence — the shell must not wait for them before starting.
|
|
727
|
+
const capturedSnapshotId = capabilitySnapshotId;
|
|
728
|
+
const capturedTurnNumber = turnNumber;
|
|
729
|
+
const capturedSnapshot = sessionSnapshot;
|
|
730
|
+
const capturedRecall = turnRecall;
|
|
731
|
+
const capturedContinuity = turnContinuity;
|
|
732
|
+
const capturedBudget = selectedBudget;
|
|
733
|
+
const capturedCatalog = filteredCatalog;
|
|
734
|
+
scheduleRuntimeSidecar(sessionId, `capability-snapshot-turn-${turnNumber}`, async () => {
|
|
735
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
736
|
+
return;
|
|
737
|
+
let toolChoiceAuditId;
|
|
738
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
739
|
+
const toolChoiceAudit = buildToolChoiceAuditRecord({
|
|
740
|
+
session_id: sessionId,
|
|
741
|
+
turn_number: capturedTurnNumber,
|
|
742
|
+
captured_at: Date.now(),
|
|
743
|
+
model_class: effectiveModelClass,
|
|
744
|
+
capability_snapshot_id: capturedSnapshotId,
|
|
745
|
+
filteredCatalog: capturedCatalog,
|
|
746
|
+
selected_budget: { read_file_lines_max_lines: capturedBudget.read_file_lines_max_lines },
|
|
747
|
+
});
|
|
748
|
+
toolChoiceAuditId = toolChoiceAudit.audit_id;
|
|
749
|
+
await repo.saveToolChoiceAudit(toolChoiceAudit);
|
|
750
|
+
await repo.saveCapabilitySnapshot({
|
|
751
|
+
snapshot_id: capturedSnapshotId,
|
|
752
|
+
session_id: sessionId,
|
|
753
|
+
turn_number: capturedTurnNumber,
|
|
754
|
+
captured_at: Date.now(),
|
|
755
|
+
model_class: effectiveModelClass,
|
|
756
|
+
allowed_tools: capturedCatalog.entries.map((e) => e.name),
|
|
757
|
+
unavailable_tools: capturedCatalog.unavailable_tools,
|
|
758
|
+
tool_cost_class: capturedCatalog.tool_cost_class,
|
|
759
|
+
});
|
|
760
|
+
await repo.appendTransitionRecord({
|
|
761
|
+
session_id: sessionId,
|
|
762
|
+
subject_kind: "capability",
|
|
763
|
+
subject_id: `${sessionId}/${capturedTurnNumber}`,
|
|
764
|
+
from: "previous_turn",
|
|
765
|
+
to: `model_class:${effectiveModelClass}`,
|
|
766
|
+
reason: `Capability snapshot captured for turn ${capturedTurnNumber}`,
|
|
767
|
+
evidence_refs: [capturedSnapshotId],
|
|
768
|
+
});
|
|
769
|
+
}).catch(() => undefined);
|
|
770
|
+
const snapStorePath = getWorkspaceStorePath(workspaceRoot());
|
|
771
|
+
await withStoreWriteCoordinator(snapStorePath, async () => {
|
|
772
|
+
const snapStore = await openStore(snapStorePath);
|
|
773
|
+
try {
|
|
774
|
+
const snapRepo = new LocalModelRuntimeRepository(snapStore);
|
|
775
|
+
const currentStatus = await snapRepo.getRuntimeStatus(sessionId);
|
|
776
|
+
const updatedStatus = await snapRepo.upsertRuntimeStatusWithTransition({
|
|
777
|
+
...(currentStatus ?? {
|
|
778
|
+
session_id: sessionId,
|
|
779
|
+
process_id: process.pid,
|
|
780
|
+
turn_count: capturedTurnNumber,
|
|
781
|
+
bridge_status: "running",
|
|
782
|
+
preflight_state: "ready",
|
|
783
|
+
surface_kind: "unattended_runtime",
|
|
784
|
+
}),
|
|
785
|
+
active_workspace_path: capturedSnapshot.workspace_path,
|
|
786
|
+
active_workspace_session_id: registryStart.workspace_session_id,
|
|
787
|
+
workspace_hook_health: "ok",
|
|
788
|
+
last_progress_at: Date.now(),
|
|
789
|
+
}, {
|
|
790
|
+
from: currentStatus?.bridge_status,
|
|
791
|
+
reason: `Turn ${capturedTurnNumber} snapshot captured with capability and prompt assembly state`,
|
|
792
|
+
evidence_refs: [capturedSnapshotId, toolChoiceAuditId].filter((ref) => typeof ref === "string" && ref.length > 0),
|
|
793
|
+
});
|
|
794
|
+
const turnSnapshot = {
|
|
795
|
+
snapshot_id: randomUUID(),
|
|
796
|
+
session_id: sessionId,
|
|
797
|
+
turn_number: capturedTurnNumber,
|
|
798
|
+
captured_at: Date.now(),
|
|
799
|
+
preflight: preflight,
|
|
800
|
+
recall: (capturedRecall ?? null),
|
|
801
|
+
continuity: (capturedContinuity ?? null),
|
|
802
|
+
capability_snapshot_id: capturedSnapshotId,
|
|
803
|
+
tool_choice_audit_id: toolChoiceAuditId ?? null,
|
|
804
|
+
surface_kind: updatedStatus.surface_kind ?? currentStatus?.surface_kind ?? "tui_interactive",
|
|
805
|
+
};
|
|
806
|
+
await snapRepo.saveTurnSnapshot(turnSnapshot);
|
|
807
|
+
await snapStore.commit();
|
|
808
|
+
}
|
|
809
|
+
finally {
|
|
810
|
+
await snapStore.close();
|
|
811
|
+
}
|
|
812
|
+
}, { operation_label: "runtime-executor-turn-snapshot" });
|
|
501
813
|
});
|
|
502
|
-
|
|
814
|
+
scheduleRuntimeSidecar(sessionId, `turn-started-event-${turnNumber}`, async () => {
|
|
815
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_TURN_STARTED", "in_progress", `Unattended session ${sessionId} turn ${capturedTurnNumber} started.`, {
|
|
816
|
+
session_id: sessionId,
|
|
817
|
+
turn_number: capturedTurnNumber,
|
|
818
|
+
workspace_path: capturedSnapshot.workspace_path,
|
|
819
|
+
...vericifyPayloadForSession(capturedSnapshot),
|
|
820
|
+
});
|
|
821
|
+
});
|
|
822
|
+
noteProgress(`Turn ${turnNumber} started.`);
|
|
823
|
+
setSessionPhase(sessionId, "turn_executing", `Turn ${turnNumber} shell`);
|
|
824
|
+
const shellResult = await runShellCommand(runtimeProfileSnapshot.profile.executor.command ?? "", {
|
|
503
825
|
cwd: sessionSnapshot.workspace_path,
|
|
504
826
|
env: {
|
|
505
827
|
...process.env,
|
|
@@ -510,31 +832,114 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
510
832
|
ACE_RUNTIME_REQUEST_FILE: paths.tempRequestPath,
|
|
511
833
|
ACE_RUNTIME_RESPONSE_FILE: paths.tempResponsePath,
|
|
512
834
|
},
|
|
513
|
-
timeout_ms:
|
|
835
|
+
timeout_ms: effectiveTurnBudgetMs,
|
|
836
|
+
stall_timeout_ms: effectiveStallWindowMs,
|
|
514
837
|
on_spawn: (child) => {
|
|
515
838
|
if (active)
|
|
516
839
|
active.child = child;
|
|
517
840
|
},
|
|
841
|
+
on_progress: (event) => {
|
|
842
|
+
noteProgress(`Executor ${event.source} progress during turn ${turnNumber}.`, event.at);
|
|
843
|
+
},
|
|
518
844
|
});
|
|
519
845
|
if (active)
|
|
520
846
|
active.child = undefined;
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
847
|
+
lastProgressAt = shellResult.last_progress_at;
|
|
848
|
+
await flushProgress();
|
|
849
|
+
setSessionPhase(sessionId, "turn_persisting", `Turn ${turnNumber} response`);
|
|
850
|
+
// Stall detection is based on the inter-progress window, not the total turn budget.
|
|
851
|
+
if (shellResult.timed_out && shellResult.timeout_reason === "stall") {
|
|
852
|
+
consecutiveStallRestarts += 1;
|
|
853
|
+
// Update runtime status: sleep_state = "stalled"
|
|
526
854
|
if (storeExistsSync(workspaceRoot())) {
|
|
527
|
-
|
|
855
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
856
|
+
const currentStatus = await repo.getRuntimeStatus(sessionId);
|
|
857
|
+
if (currentStatus) {
|
|
858
|
+
await repo.upsertRuntimeStatusWithTransition({
|
|
859
|
+
...currentStatus,
|
|
860
|
+
sleep_state: "stalled",
|
|
861
|
+
stall_restart_count: (currentStatus.stall_restart_count ?? 0) + 1,
|
|
862
|
+
last_progress_at: lastProgressAt,
|
|
863
|
+
}, {
|
|
864
|
+
from: currentStatus.bridge_status,
|
|
865
|
+
reason: `Turn ${turnNumber} exceeded stall_window_ms (${effectiveStallWindowMs}ms) without progress. Stall restart ${consecutiveStallRestarts}/${maxStallRestarts}.`,
|
|
866
|
+
evidence_refs: [
|
|
867
|
+
`turn:${turnNumber}`,
|
|
868
|
+
`stall_window_ms:${effectiveStallWindowMs}`,
|
|
869
|
+
`last_progress_at:${lastProgressAt}`,
|
|
870
|
+
],
|
|
871
|
+
});
|
|
872
|
+
await repo.appendTransitionRecord({
|
|
873
|
+
session_id: sessionId,
|
|
874
|
+
subject_kind: "runtime_status",
|
|
875
|
+
subject_id: sessionId,
|
|
876
|
+
from: "running",
|
|
877
|
+
to: "stalled",
|
|
878
|
+
reason: `Turn ${turnNumber} exceeded stall_window_ms (${effectiveStallWindowMs}ms) without progress. Stall restart ${consecutiveStallRestarts}/${maxStallRestarts}.`,
|
|
879
|
+
evidence_refs: [
|
|
880
|
+
`turn:${turnNumber}`,
|
|
881
|
+
`stall_window_ms:${effectiveStallWindowMs}`,
|
|
882
|
+
`last_progress_at:${lastProgressAt}`,
|
|
883
|
+
],
|
|
884
|
+
});
|
|
885
|
+
}
|
|
886
|
+
}).catch(() => undefined);
|
|
528
887
|
}
|
|
529
|
-
|
|
530
|
-
|
|
888
|
+
if (consecutiveStallRestarts > maxStallRestarts) {
|
|
889
|
+
// Exhausted restarts — escalate to blocked
|
|
890
|
+
const exhaustedSummary = `Stall restart limit exhausted after ${maxStallRestarts} attempts. Session blocked.`;
|
|
891
|
+
if (storeExistsSync(workspaceRoot())) {
|
|
892
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
893
|
+
const currentStatus = await repo.getRuntimeStatus(sessionId);
|
|
894
|
+
if (currentStatus) {
|
|
895
|
+
await repo.upsertRuntimeStatusWithTransition({
|
|
896
|
+
...currentStatus,
|
|
897
|
+
bridge_status: "blocked",
|
|
898
|
+
blocked_reason: "stall_restart_exhausted",
|
|
899
|
+
sleep_state: "stalled",
|
|
900
|
+
}, {
|
|
901
|
+
from: currentStatus.bridge_status,
|
|
902
|
+
reason: exhaustedSummary,
|
|
903
|
+
reason_code: "stall_restart_exhausted",
|
|
904
|
+
evidence_refs: [`stall_restarts:${consecutiveStallRestarts}`],
|
|
905
|
+
});
|
|
906
|
+
await repo.appendTransitionRecord({
|
|
907
|
+
session_id: sessionId,
|
|
908
|
+
subject_kind: "runtime_status",
|
|
909
|
+
subject_id: sessionId,
|
|
910
|
+
from: "stalled",
|
|
911
|
+
to: "blocked",
|
|
912
|
+
reason: exhaustedSummary,
|
|
913
|
+
reason_code: "stall_restart_exhausted",
|
|
914
|
+
evidence_refs: [`stall_restarts:${consecutiveStallRestarts}`],
|
|
915
|
+
});
|
|
916
|
+
}
|
|
917
|
+
}).catch(() => undefined);
|
|
918
|
+
}
|
|
919
|
+
await finalizeSession(sessionId, "blocked", exhaustedSummary, exhaustedSummary);
|
|
920
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_FAILED", "blocked", exhaustedSummary, {
|
|
921
|
+
session_id: sessionId,
|
|
922
|
+
turn_number: turnNumber,
|
|
923
|
+
blocked_reason: "stall_restart_exhausted",
|
|
924
|
+
...vericifyPayloadForSession(sessionSnapshot),
|
|
925
|
+
});
|
|
926
|
+
return;
|
|
927
|
+
}
|
|
928
|
+
// Backoff before restart
|
|
929
|
+
const backoffMs = Math.min(initialBackoffMs * Math.pow(2, consecutiveStallRestarts - 1), maxRetryBackoffMs, effectiveStallWindowMs);
|
|
930
|
+
await new Promise((resolve) => setTimeout(resolve, backoffMs));
|
|
931
|
+
// Set up restart context for the next turn
|
|
932
|
+
isStallRestart = true;
|
|
933
|
+
stallRestartContext = `The previous turn (turn ${turnNumber}) stalled after ${effectiveStallWindowMs}ms without progress inside turn_budget_ms=${effectiveTurnBudgetMs}ms; resume from current workspace state — do not restart from scratch.`;
|
|
934
|
+
continue;
|
|
935
|
+
}
|
|
936
|
+
// Reset stall state on successful turn completion
|
|
937
|
+
consecutiveStallRestarts = 0;
|
|
938
|
+
isStallRestart = false;
|
|
939
|
+
stallRestartContext = "";
|
|
531
940
|
if (active?.stop_requested) {
|
|
532
941
|
const turnEndedAt = new Date().toISOString();
|
|
533
|
-
const
|
|
534
|
-
status: "failed",
|
|
535
|
-
summary: `Turn ${turnNumber} stopped by request.`,
|
|
536
|
-
tool_calls: [],
|
|
537
|
-
}, null, 2));
|
|
942
|
+
const stoppedSummary = `Turn ${turnNumber} stopped by request.`;
|
|
538
943
|
await mutateRegistry((registry) => {
|
|
539
944
|
const session = findSession(registry, sessionId);
|
|
540
945
|
if (!session)
|
|
@@ -546,10 +951,10 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
546
951
|
ended_at: turnEndedAt,
|
|
547
952
|
status: "stopped",
|
|
548
953
|
response_status: "failed",
|
|
549
|
-
summary:
|
|
550
|
-
prompt_path:
|
|
551
|
-
request_path:
|
|
552
|
-
response_path:
|
|
954
|
+
summary: stoppedSummary,
|
|
955
|
+
prompt_path: paths.tempPromptPath,
|
|
956
|
+
request_path: paths.tempRequestPath,
|
|
957
|
+
response_path: paths.tempResponsePath,
|
|
553
958
|
exit_code: shellResult.exit_code,
|
|
554
959
|
stdout: clipOutput(shellResult.stdout),
|
|
555
960
|
stderr: clipOutput(shellResult.stderr),
|
|
@@ -559,13 +964,12 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
559
964
|
session.turn_count = session.turns.length;
|
|
560
965
|
});
|
|
561
966
|
const stopped = await finalizeSession(sessionId, "stopped", `Unattended session ${sessionId} stopped by request.`);
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
});
|
|
967
|
+
scheduleRuntimeSidecar(sessionId, "session-stopped-event", async () => {
|
|
968
|
+
if (!stopped)
|
|
969
|
+
return;
|
|
970
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_STOPPED", "done", stopped.result_summary ?? `Unattended session ${sessionId} stopped by request.`, { session_id: sessionId, ...vericifyPayloadForSession(stopped) });
|
|
567
971
|
await emitVericifyProcessPostForSession(stopped, "progress", stopped.result_summary ?? `Unattended session ${sessionId} stopped by request.`, ["runtime-executor"]);
|
|
568
|
-
}
|
|
972
|
+
});
|
|
569
973
|
return;
|
|
570
974
|
}
|
|
571
975
|
let response;
|
|
@@ -575,11 +979,6 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
575
979
|
catch (error) {
|
|
576
980
|
const message = error instanceof Error ? error.message : String(error);
|
|
577
981
|
const turnEndedAt = new Date().toISOString();
|
|
578
|
-
const artifactPaths = await finalizeTurnArtifacts(JSON.stringify({
|
|
579
|
-
status: "failed",
|
|
580
|
-
summary: message,
|
|
581
|
-
tool_calls: [],
|
|
582
|
-
}, null, 2));
|
|
583
982
|
await mutateRegistry((registry) => {
|
|
584
983
|
const session = findSession(registry, sessionId);
|
|
585
984
|
if (!session)
|
|
@@ -592,9 +991,9 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
592
991
|
status: "failed",
|
|
593
992
|
response_status: "failed",
|
|
594
993
|
summary: message,
|
|
595
|
-
prompt_path:
|
|
596
|
-
request_path:
|
|
597
|
-
response_path:
|
|
994
|
+
prompt_path: paths.tempPromptPath,
|
|
995
|
+
request_path: paths.tempRequestPath,
|
|
996
|
+
response_path: paths.tempResponsePath,
|
|
598
997
|
exit_code: shellResult.exit_code,
|
|
599
998
|
stdout: clipOutput(shellResult.stdout),
|
|
600
999
|
stderr: clipOutput(shellResult.stderr),
|
|
@@ -604,28 +1003,30 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
604
1003
|
session.turn_count = session.turns.length;
|
|
605
1004
|
});
|
|
606
1005
|
const failed = await finalizeSession(sessionId, "failed", message, message);
|
|
607
|
-
|
|
1006
|
+
scheduleRuntimeSidecar(sessionId, "session-parse-failed-event", async () => {
|
|
1007
|
+
if (!failed)
|
|
1008
|
+
return;
|
|
608
1009
|
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_FAILED", "fail", message, {
|
|
609
1010
|
session_id: sessionId,
|
|
610
1011
|
turn_number: turnNumber,
|
|
611
1012
|
...vericifyPayloadForSession(failed),
|
|
612
1013
|
});
|
|
613
|
-
await emitVericifyProcessPostForSession(failed, "blocker", message, [
|
|
614
|
-
|
|
615
|
-
]);
|
|
616
|
-
}
|
|
1014
|
+
await emitVericifyProcessPostForSession(failed, "blocker", message, ["runtime-executor"]);
|
|
1015
|
+
});
|
|
617
1016
|
return;
|
|
618
1017
|
}
|
|
619
1018
|
const normalizedSummary = normalizeWorkspaceSummary(response.summary, sessionSnapshot.workspace_path);
|
|
620
|
-
const artifactPaths = await finalizeTurnArtifacts(readFileSync(paths.tempResponsePath, "utf-8"));
|
|
621
1019
|
const toolCalls = [];
|
|
622
1020
|
for (const toolCall of response.tool_calls) {
|
|
623
1021
|
const toolStartedAt = new Date().toISOString();
|
|
1022
|
+
noteProgress(`Runtime tool ${toolCall.name} started during turn ${turnNumber}.`);
|
|
624
1023
|
const toolResult = await executeRuntimeTool(toolCall.name, toolCall.input, {
|
|
625
1024
|
session_id: sessionId,
|
|
626
1025
|
workspace_path: sessionSnapshot.workspace_path,
|
|
627
1026
|
turn_number: turnNumber,
|
|
628
1027
|
});
|
|
1028
|
+
noteProgress(`Runtime tool ${toolCall.name} finished during turn ${turnNumber}.`);
|
|
1029
|
+
await flushProgress();
|
|
629
1030
|
toolCalls.push(mapToolCallResult(toolResult, toolStartedAt));
|
|
630
1031
|
}
|
|
631
1032
|
previousToolCalls = toolCalls;
|
|
@@ -635,6 +1036,9 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
635
1036
|
: response.status === "failed"
|
|
636
1037
|
? "failed"
|
|
637
1038
|
: "completed";
|
|
1039
|
+
const effectivePolicy = sessionSnapshot.output_policy ?? DEFAULT_TURN_OUTPUT_POLICY;
|
|
1040
|
+
const { outcome: turnOutcome, reason: outcomeReason } = classifyTurnOutcome(response.status, effectivePolicy, toolCalls.length);
|
|
1041
|
+
// Use temp file paths for the registry turn record; the ACEPACK artifact write is a sidecar.
|
|
638
1042
|
await mutateRegistry((registry) => {
|
|
639
1043
|
const session = findSession(registry, sessionId);
|
|
640
1044
|
if (!session)
|
|
@@ -647,27 +1051,62 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
647
1051
|
status: turnStatus,
|
|
648
1052
|
response_status: response.status,
|
|
649
1053
|
summary: normalizedSummary,
|
|
650
|
-
prompt_path:
|
|
651
|
-
request_path:
|
|
652
|
-
response_path:
|
|
1054
|
+
prompt_path: paths.tempPromptPath,
|
|
1055
|
+
request_path: paths.tempRequestPath,
|
|
1056
|
+
response_path: paths.tempResponsePath,
|
|
653
1057
|
exit_code: shellResult.exit_code,
|
|
654
1058
|
stdout: clipOutput(shellResult.stdout),
|
|
655
1059
|
stderr: clipOutput(shellResult.stderr),
|
|
656
1060
|
tool_calls: toolCalls,
|
|
1061
|
+
turn_outcome: turnOutcome,
|
|
1062
|
+
outcome_reason: outcomeReason,
|
|
657
1063
|
};
|
|
658
1064
|
session.turns.push(turnRecord);
|
|
659
1065
|
session.turn_count = session.turns.length;
|
|
660
1066
|
session.current_task = response.next_task?.trim() || currentTask;
|
|
661
1067
|
session.updated_at = turnEndedAt;
|
|
662
1068
|
});
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
1069
|
+
// Sidecar: persist turn artifacts to ACEPACK, emit outcome transition and events.
|
|
1070
|
+
const capturedTurnStatus = turnStatus;
|
|
1071
|
+
const capturedTurnOutcome = turnOutcome;
|
|
1072
|
+
const capturedOutcomeReason = outcomeReason;
|
|
1073
|
+
const capturedToolCalls = toolCalls;
|
|
1074
|
+
const capturedNormalizedSummary = normalizedSummary;
|
|
1075
|
+
const capturedArtifactPaths = { promptPath: paths.tempPromptPath, requestPath: paths.tempRequestPath, responsePath: paths.tempResponsePath };
|
|
1076
|
+
const capturedSnapshot2 = sessionSnapshot;
|
|
1077
|
+
scheduleRuntimeSidecar(sessionId, `turn-artifacts-${turnNumber}`, async () => {
|
|
1078
|
+
const responseRaw = existsSync(paths.tempResponsePath)
|
|
1079
|
+
? readFileSync(paths.tempResponsePath, "utf-8")
|
|
1080
|
+
: JSON.stringify({ status: capturedTurnStatus, summary: capturedNormalizedSummary, tool_calls: [] }, null, 2);
|
|
1081
|
+
await persistTurnArtifacts(paths, prompt, requestPayload, responseRaw).catch(() => undefined);
|
|
1082
|
+
});
|
|
1083
|
+
scheduleRuntimeSidecar(sessionId, `turn-outcome-transition-${turnNumber}`, async () => {
|
|
1084
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
1085
|
+
return;
|
|
1086
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
1087
|
+
await repo.appendTransitionRecord({
|
|
1088
|
+
session_id: sessionId,
|
|
1089
|
+
subject_kind: "turn",
|
|
1090
|
+
subject_id: `${sessionId}/${capturedTurnNumber}`,
|
|
1091
|
+
from: "in_progress",
|
|
1092
|
+
to: capturedTurnOutcome,
|
|
1093
|
+
reason: capturedOutcomeReason,
|
|
1094
|
+
evidence_refs: capturedToolCalls.slice(0, 3).map((tc) => tc.tool_name),
|
|
1095
|
+
});
|
|
1096
|
+
}).catch(() => undefined);
|
|
1097
|
+
await emitExecutorEvent(capturedTurnStatus === "failed" ? "RUNTIME_EXECUTOR_TURN_FAILED" : "RUNTIME_EXECUTOR_TURN_COMPLETED", capturedTurnStatus === "blocked" ? "blocked" : capturedTurnStatus === "failed" ? "fail" : "done", capturedNormalizedSummary, {
|
|
1098
|
+
session_id: sessionId,
|
|
1099
|
+
turn_number: capturedTurnNumber,
|
|
1100
|
+
response_status: capturedTurnStatus,
|
|
1101
|
+
tool_call_count: capturedToolCalls.length,
|
|
1102
|
+
...vericifyPayloadForSession(capturedSnapshot2),
|
|
1103
|
+
});
|
|
1104
|
+
const vericifyKind = capturedTurnOutcome === "escalation_blocker"
|
|
1105
|
+
? "blocker"
|
|
1106
|
+
: capturedTurnOutcome === "meaningful_completion"
|
|
1107
|
+
? "completion"
|
|
1108
|
+
: "progress";
|
|
1109
|
+
await emitVericifyProcessPostForSession(capturedSnapshot2, vericifyKind, `[${capturedTurnOutcome}] turn ${capturedTurnNumber}: ${capturedNormalizedSummary}`, capturedToolCalls.slice(0, 5).map((tc) => tc.tool_name), [String(capturedTurnNumber)]);
|
|
671
1110
|
});
|
|
672
1111
|
if (response.status === "continue") {
|
|
673
1112
|
currentTask = response.next_task?.trim() || currentTask;
|
|
@@ -675,69 +1114,81 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
675
1114
|
}
|
|
676
1115
|
if (response.status === "blocked") {
|
|
677
1116
|
await finalizeSession(sessionId, "blocked", normalizedSummary, normalizedSummary);
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
1117
|
+
scheduleRuntimeSidecar(sessionId, "session-blocked-event", async () => {
|
|
1118
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_BLOCKED", "blocked", capturedNormalizedSummary, {
|
|
1119
|
+
session_id: sessionId,
|
|
1120
|
+
turn_number: capturedTurnNumber,
|
|
1121
|
+
...vericifyPayloadForSession(capturedSnapshot2),
|
|
1122
|
+
});
|
|
1123
|
+
await emitVericifyProcessPostForSession(capturedSnapshot2, "blocker", capturedNormalizedSummary, [
|
|
1124
|
+
"runtime-executor",
|
|
1125
|
+
]);
|
|
682
1126
|
});
|
|
683
|
-
await emitVericifyProcessPostForSession(sessionSnapshot, "blocker", normalizedSummary, [
|
|
684
|
-
"runtime-executor",
|
|
685
|
-
]);
|
|
686
1127
|
return;
|
|
687
1128
|
}
|
|
688
1129
|
if (response.status === "failed") {
|
|
689
1130
|
const failed = await finalizeSession(sessionId, "failed", normalizedSummary, normalizedSummary);
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
1131
|
+
scheduleRuntimeSidecar(sessionId, "session-failed-event", async () => {
|
|
1132
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_FAILED", "fail", capturedNormalizedSummary, {
|
|
1133
|
+
session_id: sessionId,
|
|
1134
|
+
turn_number: capturedTurnNumber,
|
|
1135
|
+
...(failed ? vericifyPayloadForSession(failed) : vericifyPayloadForSession(capturedSnapshot2)),
|
|
1136
|
+
});
|
|
1137
|
+
await emitVericifyProcessPostForSession(failed ?? capturedSnapshot2, "blocker", capturedNormalizedSummary, ["runtime-executor"]);
|
|
694
1138
|
});
|
|
695
|
-
await emitVericifyProcessPostForSession(failed ?? sessionSnapshot, "blocker", normalizedSummary, ["runtime-executor"]);
|
|
696
1139
|
return;
|
|
697
1140
|
}
|
|
698
1141
|
const stopCheck = runAutonomyChecks("stop", autonomyPolicy, context);
|
|
699
1142
|
if (!stopCheck.ok) {
|
|
700
1143
|
const summary = `Autonomy stop checks failed: ${stopCheck.summary}`;
|
|
701
1144
|
const blocked = await finalizeSession(sessionId, "blocked", summary, summary);
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
1145
|
+
scheduleRuntimeSidecar(sessionId, "session-stop-check-blocked-event", async () => {
|
|
1146
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_BLOCKED", "blocked", summary, {
|
|
1147
|
+
session_id: sessionId,
|
|
1148
|
+
turn_number: capturedTurnNumber,
|
|
1149
|
+
executed_checks: stopCheck.executed_checks,
|
|
1150
|
+
skipped_checks: stopCheck.skipped_checks,
|
|
1151
|
+
...(blocked ? vericifyPayloadForSession(blocked) : vericifyPayloadForSession(capturedSnapshot2)),
|
|
1152
|
+
});
|
|
1153
|
+
await emitVericifyProcessPostForSession(blocked ?? capturedSnapshot2, "blocker", summary, ["runtime-executor"]);
|
|
708
1154
|
});
|
|
709
|
-
await emitVericifyProcessPostForSession(blocked ?? sessionSnapshot, "blocker", summary, ["runtime-executor"]);
|
|
710
1155
|
return;
|
|
711
1156
|
}
|
|
1157
|
+
// Critical: finalize session status immediately so waitForUnattendedSession resolves correctly.
|
|
712
1158
|
const completed = await finalizeSession(sessionId, "completed", normalizedSummary);
|
|
713
|
-
|
|
714
|
-
await
|
|
715
|
-
tool: "runtime-executor",
|
|
716
|
-
category: "major_update",
|
|
717
|
-
message: normalizedSummary,
|
|
718
|
-
artifacts: [
|
|
719
|
-
RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH,
|
|
720
|
-
artifactPaths.promptPath,
|
|
721
|
-
artifactPaths.requestPath,
|
|
722
|
-
artifactPaths.responsePath,
|
|
723
|
-
],
|
|
724
|
-
metadata: {
|
|
725
|
-
session_id: sessionId,
|
|
726
|
-
turn_number: turnNumber,
|
|
727
|
-
workspace_session_id: completed.workspace_session_id,
|
|
728
|
-
...vericifyPayloadForSession(completed),
|
|
729
|
-
},
|
|
730
|
-
}).catch(() => undefined);
|
|
731
|
-
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_COMPLETED", "done", normalizedSummary, {
|
|
1159
|
+
scheduleRuntimeSidecar(sessionId, "session-completed-events", async () => {
|
|
1160
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_COMPLETED", "done", capturedNormalizedSummary, {
|
|
732
1161
|
session_id: sessionId,
|
|
733
|
-
turn_number:
|
|
734
|
-
workspace_session_id:
|
|
735
|
-
...vericifyPayloadForSession(
|
|
1162
|
+
turn_number: capturedTurnNumber,
|
|
1163
|
+
workspace_session_id: capturedSnapshot2.workspace_session_id,
|
|
1164
|
+
...vericifyPayloadForSession(capturedSnapshot2),
|
|
736
1165
|
});
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
1166
|
+
if (completed) {
|
|
1167
|
+
await appendRunLedgerEntrySafe({
|
|
1168
|
+
tool: "runtime-executor",
|
|
1169
|
+
category: "major_update",
|
|
1170
|
+
message: capturedNormalizedSummary,
|
|
1171
|
+
artifacts: [
|
|
1172
|
+
RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH,
|
|
1173
|
+
capturedArtifactPaths.promptPath,
|
|
1174
|
+
capturedArtifactPaths.requestPath,
|
|
1175
|
+
capturedArtifactPaths.responsePath,
|
|
1176
|
+
],
|
|
1177
|
+
metadata: {
|
|
1178
|
+
session_id: sessionId,
|
|
1179
|
+
turn_number: capturedTurnNumber,
|
|
1180
|
+
workspace_session_id: completed.workspace_session_id,
|
|
1181
|
+
...vericifyPayloadForSession(completed),
|
|
1182
|
+
},
|
|
1183
|
+
}).catch(() => undefined);
|
|
1184
|
+
await emitVericifyProcessPostForSession(completed, "completion", capturedNormalizedSummary, [
|
|
1185
|
+
"runtime-executor",
|
|
1186
|
+
]);
|
|
1187
|
+
if (isVericifyBridgeEnabled()) {
|
|
1188
|
+
refreshVericifyBridgeSnapshotSafe();
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
});
|
|
741
1192
|
return;
|
|
742
1193
|
}
|
|
743
1194
|
const exhausted = `Unattended session ${sessionId} exceeded max_turns=${registryStart.max_turns}`;
|
|
@@ -790,10 +1241,15 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
790
1241
|
}
|
|
791
1242
|
finally {
|
|
792
1243
|
if (workspaceSessionId && autoCleanup) {
|
|
793
|
-
|
|
1244
|
+
setSessionPhase(sessionId, "cleanup_capturing");
|
|
1245
|
+
await runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook, runtimeProfileSnapshot.path).catch(() => undefined);
|
|
794
1246
|
}
|
|
1247
|
+
// Release sidecars to run in background — do not await them.
|
|
1248
|
+
// Critical-path registry and workspace writes are already complete.
|
|
1249
|
+
releaseSidecarsAndForget(sessionId);
|
|
795
1250
|
const active = activeSessions.get(sessionId);
|
|
796
1251
|
if (active) {
|
|
1252
|
+
active.phase = "completed";
|
|
797
1253
|
active.child = undefined;
|
|
798
1254
|
activeSessions.delete(sessionId);
|
|
799
1255
|
}
|
|
@@ -821,18 +1277,29 @@ export function getUnattendedSession(sessionId) {
|
|
|
821
1277
|
export function getRuntimeExecutorSessionRegistryPath() {
|
|
822
1278
|
return registryPath();
|
|
823
1279
|
}
|
|
824
|
-
export function startUnattendedSession(input) {
|
|
1280
|
+
export async function startUnattendedSession(input) {
|
|
825
1281
|
const registryPathValue = registryPath();
|
|
826
|
-
|
|
1282
|
+
let runtimeProfileSnapshot;
|
|
1283
|
+
try {
|
|
1284
|
+
runtimeProfileSnapshot = loadCurrentRuntimeProfile();
|
|
1285
|
+
}
|
|
1286
|
+
catch (error) {
|
|
1287
|
+
return Promise.resolve({
|
|
1288
|
+
ok: false,
|
|
1289
|
+
registry_path: registryPathValue,
|
|
1290
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1291
|
+
});
|
|
1292
|
+
}
|
|
827
1293
|
const storeBackedWorkspace = storeExistsSync(workspaceRoot());
|
|
828
|
-
if (
|
|
1294
|
+
if (runtimeProfileSnapshot.profile.runtime.mode !== "unattended") {
|
|
829
1295
|
return Promise.resolve({
|
|
830
1296
|
ok: false,
|
|
831
1297
|
registry_path: registryPathValue,
|
|
832
1298
|
error: "ACE_WORKFLOW.md runtime.mode must be \"unattended\" before starting an unattended session.",
|
|
833
1299
|
});
|
|
834
1300
|
}
|
|
835
|
-
if (!
|
|
1301
|
+
if (!runtimeProfileSnapshot.profile.executor.command ||
|
|
1302
|
+
runtimeProfileSnapshot.profile.executor.command.trim().length === 0) {
|
|
836
1303
|
return Promise.resolve({
|
|
837
1304
|
ok: false,
|
|
838
1305
|
registry_path: registryPathValue,
|
|
@@ -848,10 +1315,11 @@ export function startUnattendedSession(input) {
|
|
|
848
1315
|
error: `Unattended session id already exists: ${sessionId}`,
|
|
849
1316
|
});
|
|
850
1317
|
}
|
|
851
|
-
const workspace =
|
|
1318
|
+
const workspace = await createWorkspaceSessionAsync({
|
|
852
1319
|
source: "executor",
|
|
853
1320
|
workspace_name: input.workspace_name,
|
|
854
1321
|
workspace_path: input.workspace_path,
|
|
1322
|
+
runtime_profile_path: runtimeProfileSnapshot.path,
|
|
855
1323
|
objective_id: input.objective_id,
|
|
856
1324
|
tracker_item_id: input.tracker_item_id,
|
|
857
1325
|
});
|
|
@@ -862,56 +1330,73 @@ export function startUnattendedSession(input) {
|
|
|
862
1330
|
error: workspace.error ?? "Failed to create managed workspace session.",
|
|
863
1331
|
});
|
|
864
1332
|
}
|
|
865
|
-
const
|
|
866
|
-
const
|
|
1333
|
+
const workspaceSession = workspace.session;
|
|
1334
|
+
const maxTurns = Math.max(1, input.max_turns ?? runtimeProfileSnapshot.profile.executor.max_turns ?? 6);
|
|
1335
|
+
const turnTimeout = Math.max(1_000, input.turn_timeout_ms ?? runtimeProfileSnapshot.profile.executor.turn_timeout_ms ?? 300_000);
|
|
867
1336
|
const now = new Date().toISOString();
|
|
868
1337
|
const sessionRecord = {
|
|
869
1338
|
session_id: sessionId,
|
|
870
1339
|
status: "starting",
|
|
871
1340
|
task: input.task,
|
|
872
1341
|
current_task: input.task,
|
|
873
|
-
runtime_profile_path:
|
|
1342
|
+
runtime_profile_path: runtimeProfileSnapshot.path,
|
|
874
1343
|
workspace_session_id: workspace.session.session_id,
|
|
875
1344
|
workspace_path: workspace.session.workspace_path,
|
|
876
1345
|
objective_id: input.objective_id,
|
|
877
1346
|
tracker_item_id: input.tracker_item_id,
|
|
878
|
-
command:
|
|
1347
|
+
command: runtimeProfileSnapshot.profile.executor.command,
|
|
879
1348
|
max_turns: maxTurns,
|
|
880
1349
|
turn_timeout_ms: turnTimeout,
|
|
881
1350
|
turn_count: 0,
|
|
882
1351
|
created_at: now,
|
|
883
1352
|
updated_at: now,
|
|
884
1353
|
workspace_cleanup_status: input.auto_cleanup === false ? "pending" : "pending",
|
|
1354
|
+
output_policy: input.emit_to || input.silent_unless_blocked || input.require_approval_before_emit
|
|
1355
|
+
? {
|
|
1356
|
+
emit_to: input.emit_to ?? DEFAULT_TURN_OUTPUT_POLICY.emit_to,
|
|
1357
|
+
silent_unless_blocked: input.silent_unless_blocked ?? false,
|
|
1358
|
+
require_approval_before_emit: input.require_approval_before_emit ?? false,
|
|
1359
|
+
}
|
|
1360
|
+
: undefined,
|
|
885
1361
|
turns: [],
|
|
886
1362
|
};
|
|
887
|
-
|
|
1363
|
+
await mutateRegistry((registry) => {
|
|
888
1364
|
registry.sessions.push(sessionRecord);
|
|
889
|
-
})
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
1365
|
+
});
|
|
1366
|
+
// Init sidecar gate BEFORE starting the loop so sidecars can be scheduled immediately.
|
|
1367
|
+
// Sidecars are blocked behind a release signal and only run after the critical path completes.
|
|
1368
|
+
initSessionSidecars(sessionId);
|
|
1369
|
+
const active = {
|
|
1370
|
+
stop_requested: false,
|
|
1371
|
+
phase: "starting",
|
|
1372
|
+
phase_started_at: Date.now(),
|
|
1373
|
+
};
|
|
1374
|
+
activeSessions.set(sessionId, active);
|
|
1375
|
+
active.completion = runSessionLoop(sessionId, input.context, storeBackedWorkspace || input.auto_cleanup !== false, runtimeProfileSnapshot).finally(() => {
|
|
1376
|
+
activeSessions.delete(sessionId);
|
|
1377
|
+
});
|
|
1378
|
+
// Schedule session-started events as sidecars; they run after the critical lane completes.
|
|
1379
|
+
const capturedRecord = sessionRecord;
|
|
1380
|
+
const capturedSession = workspace.session;
|
|
1381
|
+
scheduleRuntimeSidecar(sessionId, "session-started-event", async () => {
|
|
1382
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_STARTED", "started", `Unattended session ${sessionId} created.`, {
|
|
898
1383
|
session_id: sessionId,
|
|
899
|
-
workspace_session_id:
|
|
900
|
-
workspace_path:
|
|
1384
|
+
workspace_session_id: capturedSession?.session_id,
|
|
1385
|
+
workspace_path: capturedSession?.workspace_path,
|
|
901
1386
|
max_turns: maxTurns,
|
|
902
|
-
...vericifyPayloadForSession(
|
|
1387
|
+
...vericifyPayloadForSession(capturedRecord),
|
|
903
1388
|
});
|
|
904
|
-
|
|
1389
|
+
await emitVericifyProcessPostForSession(capturedRecord, "intent", `Unattended session ${sessionId} created for task: ${input.task}`, ["runtime-executor"]);
|
|
905
1390
|
if (isVericifyBridgeEnabled()) {
|
|
906
1391
|
refreshVericifyBridgeSnapshotSafe();
|
|
907
1392
|
}
|
|
908
|
-
return {
|
|
909
|
-
ok: true,
|
|
910
|
-
registry_path: registryPathValue,
|
|
911
|
-
session: sessionRecord,
|
|
912
|
-
workspace: workspace.session,
|
|
913
|
-
};
|
|
914
1393
|
});
|
|
1394
|
+
return {
|
|
1395
|
+
ok: true,
|
|
1396
|
+
registry_path: registryPathValue,
|
|
1397
|
+
session: sessionRecord,
|
|
1398
|
+
workspace: workspace.session,
|
|
1399
|
+
};
|
|
915
1400
|
}
|
|
916
1401
|
export async function waitForUnattendedSession(sessionId, timeoutMs = 30_000) {
|
|
917
1402
|
const registryPathValue = registryPath();
|
|
@@ -926,14 +1411,22 @@ export async function waitForUnattendedSession(sessionId, timeoutMs = 30_000) {
|
|
|
926
1411
|
error: `Unknown unattended session: ${sessionId}`,
|
|
927
1412
|
};
|
|
928
1413
|
}
|
|
1414
|
+
if (terminalStatus(session.status)) {
|
|
1415
|
+
await waitForSessionSidecars(sessionId);
|
|
1416
|
+
const freshSession = getUnattendedSession(sessionId);
|
|
1417
|
+
return {
|
|
1418
|
+
ok: terminalStatus(freshSession?.status ?? session.status),
|
|
1419
|
+
timed_out: false,
|
|
1420
|
+
registry_path: registryPathValue,
|
|
1421
|
+
session: freshSession ?? session,
|
|
1422
|
+
};
|
|
1423
|
+
}
|
|
929
1424
|
return {
|
|
930
|
-
ok:
|
|
1425
|
+
ok: false,
|
|
931
1426
|
timed_out: false,
|
|
932
1427
|
registry_path: registryPathValue,
|
|
933
1428
|
session,
|
|
934
|
-
error:
|
|
935
|
-
? undefined
|
|
936
|
-
: `Session ${sessionId} is not active but has not reached a terminal state.`,
|
|
1429
|
+
error: `Session ${sessionId} is not active but has not reached a terminal state.`,
|
|
937
1430
|
};
|
|
938
1431
|
}
|
|
939
1432
|
const timeoutPromise = new Promise((resolve) => {
|
|
@@ -943,19 +1436,24 @@ export async function waitForUnattendedSession(sessionId, timeoutMs = 30_000) {
|
|
|
943
1436
|
const outcome = await Promise.race([completion, timeoutPromise]);
|
|
944
1437
|
const session = getUnattendedSession(sessionId);
|
|
945
1438
|
if (outcome === "timeout") {
|
|
1439
|
+
const phaseInfo = active.phase
|
|
1440
|
+
? ` (phase=${active.phase}${active.last_checkpoint ? `, checkpoint=${active.last_checkpoint}` : ""})`
|
|
1441
|
+
: "";
|
|
946
1442
|
return {
|
|
947
1443
|
ok: false,
|
|
948
1444
|
timed_out: true,
|
|
949
1445
|
registry_path: registryPathValue,
|
|
950
1446
|
session,
|
|
951
|
-
error: `Timed out waiting for unattended session ${sessionId}`,
|
|
1447
|
+
error: `Timed out waiting for unattended session ${sessionId}${phaseInfo}`,
|
|
952
1448
|
};
|
|
953
1449
|
}
|
|
1450
|
+
await waitForSessionSidecars(sessionId);
|
|
1451
|
+
const completedSession = getUnattendedSession(sessionId);
|
|
954
1452
|
return {
|
|
955
|
-
ok: !!
|
|
1453
|
+
ok: !!completedSession && terminalStatus(completedSession.status),
|
|
956
1454
|
timed_out: false,
|
|
957
1455
|
registry_path: registryPathValue,
|
|
958
|
-
session,
|
|
1456
|
+
session: completedSession ?? session,
|
|
959
1457
|
};
|
|
960
1458
|
}
|
|
961
1459
|
export async function stopUnattendedSession(sessionId) {
|