@voybio/ace-swarm 0.2.5 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -1
- package/README.md +21 -13
- package/assets/.agents/ACE/agent-qa/instructions.md +11 -0
- package/assets/agent-state/EVIDENCE_LOG.md +1 -1
- package/assets/agent-state/MODULES/roles/capability-framework.json +41 -0
- package/assets/agent-state/MODULES/roles/capability-git.json +33 -0
- package/assets/agent-state/MODULES/roles/capability-safety.json +37 -0
- package/assets/agent-state/MODULES/schemas/ACE_RUNTIME_PROFILE.schema.json +21 -0
- package/assets/agent-state/MODULES/schemas/RUNTIME_EXECUTOR_SESSION_REGISTRY.schema.json +43 -0
- package/assets/agent-state/MODULES/schemas/RUNTIME_TOOL_SPEC_REGISTRY.schema.json +43 -0
- package/assets/agent-state/MODULES/schemas/WORKSPACE_SESSION_REGISTRY.schema.json +11 -0
- package/assets/agent-state/STATUS.md +2 -2
- package/assets/agent-state/runtime-tool-specs.json +70 -2
- package/assets/instructions/ACE_Coder.instructions.md +13 -0
- package/assets/instructions/ACE_UI.instructions.md +11 -0
- package/assets/scripts/ace-hook-dispatch.mjs +70 -6
- package/assets/scripts/render-mcp-configs.sh +19 -5
- package/dist/ace-context.js +91 -11
- package/dist/ace-internal-tools.d.ts +3 -1
- package/dist/ace-internal-tools.js +10 -2
- package/dist/ace-server-instructions.js +3 -3
- package/dist/ace-state-resolver.js +5 -3
- package/dist/agent-runtime/role-adapters.d.ts +18 -1
- package/dist/agent-runtime/role-adapters.js +49 -5
- package/dist/astgrep-index.d.ts +57 -1
- package/dist/astgrep-index.js +140 -4
- package/dist/cli.js +232 -35
- package/dist/discovery-runtime-wrappers.d.ts +108 -0
- package/dist/discovery-runtime-wrappers.js +615 -0
- package/dist/handoff-registry.js +5 -5
- package/dist/helpers/artifacts.d.ts +19 -0
- package/dist/helpers/artifacts.js +152 -0
- package/dist/helpers/bootstrap.d.ts +24 -0
- package/dist/helpers/bootstrap.js +894 -0
- package/dist/helpers/constants.d.ts +53 -0
- package/dist/helpers/constants.js +295 -0
- package/dist/helpers/drift.d.ts +13 -0
- package/dist/helpers/drift.js +45 -0
- package/dist/helpers/path-utils.d.ts +24 -0
- package/dist/helpers/path-utils.js +123 -0
- package/dist/helpers/store-resolution.d.ts +19 -0
- package/dist/helpers/store-resolution.js +305 -0
- package/dist/helpers/workspace-root.d.ts +3 -0
- package/dist/helpers/workspace-root.js +80 -0
- package/dist/helpers.d.ts +8 -125
- package/dist/helpers.js +8 -1768
- package/dist/job-scheduler.js +33 -7
- package/dist/json-sanitizer.d.ts +16 -0
- package/dist/json-sanitizer.js +26 -0
- package/dist/local-model-policy.d.ts +27 -0
- package/dist/local-model-policy.js +84 -0
- package/dist/local-model-runtime.d.ts +6 -0
- package/dist/local-model-runtime.js +33 -21
- package/dist/model-bridge.d.ts +13 -1
- package/dist/model-bridge.js +410 -23
- package/dist/orchestrator-supervisor.d.ts +56 -0
- package/dist/orchestrator-supervisor.js +179 -1
- package/dist/plan-proposal.d.ts +115 -0
- package/dist/plan-proposal.js +1073 -0
- package/dist/run-ledger.js +3 -3
- package/dist/runtime-command.d.ts +8 -0
- package/dist/runtime-command.js +38 -6
- package/dist/runtime-executor.d.ts +20 -1
- package/dist/runtime-executor.js +737 -172
- package/dist/runtime-profile.d.ts +32 -0
- package/dist/runtime-profile.js +89 -13
- package/dist/runtime-tool-specs.d.ts +39 -0
- package/dist/runtime-tool-specs.js +144 -28
- package/dist/safe-edit.d.ts +7 -0
- package/dist/safe-edit.js +163 -37
- package/dist/schemas.js +48 -1
- package/dist/server.js +51 -0
- package/dist/shared.d.ts +3 -2
- package/dist/shared.js +2 -0
- package/dist/status-events.js +9 -6
- package/dist/store/ace-packed-store.d.ts +3 -2
- package/dist/store/ace-packed-store.js +188 -110
- package/dist/store/bootstrap-store.d.ts +2 -1
- package/dist/store/bootstrap-store.js +102 -83
- package/dist/store/cache-workspace.js +11 -5
- package/dist/store/materializers/context-snapshot-materializer.js +6 -2
- package/dist/store/materializers/hook-context-materializer.d.ts +6 -9
- package/dist/store/materializers/hook-context-materializer.js +11 -21
- package/dist/store/materializers/host-file-materializer.js +6 -0
- package/dist/store/materializers/projection-manager.d.ts +0 -1
- package/dist/store/materializers/projection-manager.js +5 -13
- package/dist/store/materializers/scheduler-projection-materializer.js +1 -1
- package/dist/store/materializers/vericify-projector.d.ts +7 -7
- package/dist/store/materializers/vericify-projector.js +11 -11
- package/dist/store/repositories/local-model-runtime-repository.d.ts +120 -3
- package/dist/store/repositories/local-model-runtime-repository.js +242 -6
- package/dist/store/repositories/vericify-repository.d.ts +1 -1
- package/dist/store/skills-install.d.ts +4 -0
- package/dist/store/skills-install.js +21 -12
- package/dist/store/state-reader.d.ts +2 -0
- package/dist/store/state-reader.js +20 -0
- package/dist/store/store-artifacts.d.ts +7 -0
- package/dist/store/store-artifacts.js +27 -1
- package/dist/store/store-authority-audit.d.ts +18 -1
- package/dist/store/store-authority-audit.js +115 -5
- package/dist/store/store-snapshot.d.ts +3 -0
- package/dist/store/store-snapshot.js +22 -2
- package/dist/store/workspace-store-paths.d.ts +39 -0
- package/dist/store/workspace-store-paths.js +94 -0
- package/dist/store/write-coordinator.d.ts +65 -0
- package/dist/store/write-coordinator.js +386 -0
- package/dist/todo-state.js +5 -5
- package/dist/tools-agent.d.ts +20 -0
- package/dist/tools-agent.js +789 -25
- package/dist/tools-discovery.js +136 -1
- package/dist/tools-files.d.ts +7 -0
- package/dist/tools-files.js +1002 -11
- package/dist/tools-framework.js +105 -66
- package/dist/tools-handoff.js +2 -2
- package/dist/tools-lifecycle.js +4 -4
- package/dist/tools-memory.js +6 -6
- package/dist/tools-todo.js +2 -2
- package/dist/tracker-adapters.d.ts +1 -1
- package/dist/tracker-adapters.js +13 -18
- package/dist/tracker-sync.js +5 -3
- package/dist/tui/agent-runner.js +3 -1
- package/dist/tui/chat.js +103 -7
- package/dist/tui/dashboard.d.ts +1 -0
- package/dist/tui/dashboard.js +43 -0
- package/dist/tui/index.js +10 -1
- package/dist/tui/layout.d.ts +20 -0
- package/dist/tui/layout.js +31 -1
- package/dist/tui/local-model-contract.d.ts +6 -2
- package/dist/tui/local-model-contract.js +16 -3
- package/dist/tui/ollama.d.ts +8 -1
- package/dist/tui/ollama.js +53 -12
- package/dist/tui/openai-compatible.d.ts +13 -0
- package/dist/tui/openai-compatible.js +305 -5
- package/dist/tui/provider-discovery.d.ts +1 -0
- package/dist/tui/provider-discovery.js +35 -11
- package/dist/vericify-bridge.d.ts +6 -1
- package/dist/vericify-bridge.js +27 -3
- package/dist/workspace-manager.d.ts +30 -3
- package/dist/workspace-manager.js +257 -27
- package/package.json +1 -2
- package/dist/internal-tool-runtime.d.ts +0 -21
- package/dist/internal-tool-runtime.js +0 -136
- package/dist/store/workspace-snapshot.d.ts +0 -26
- package/dist/store/workspace-snapshot.js +0 -107
package/dist/runtime-executor.js
CHANGED
|
@@ -1,27 +1,137 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, mkdirSync, mkdtempSync, readFileSync,
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync, } from "node:fs";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join, resolve } from "node:path";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
-
import { resolveWorkspaceArtifactPath as resolveWorkspaceArtifactPathHelper, resolveWorkspaceRoot, safeRead,
|
|
6
|
+
import { resolveWorkspaceArtifactPath as resolveWorkspaceArtifactPathHelper, resolveWorkspaceRoot, safeRead, safeWriteAsync, withFileLock, } from "./helpers.js";
|
|
7
7
|
import { appendRunLedgerEntrySafe } from "./run-ledger.js";
|
|
8
8
|
import { runShellCommand } from "./runtime-command.js";
|
|
9
|
-
import {
|
|
9
|
+
import { loadRuntimeProfile, resolveEffectiveSurgicalReadBudget, renderRuntimePromptFromSnapshot, getLivenessDefaults, } from "./runtime-profile.js";
|
|
10
|
+
import { resolveLocalModelExecutionPolicy } from "./local-model-policy.js";
|
|
10
11
|
import { renderAceContinuityPromptBlock, renderAceRecallPromptBlock, renderAceSnapshotPromptBlock, } from "./ace-context.js";
|
|
11
|
-
import { executeRuntimeTool,
|
|
12
|
+
import { executeRuntimeTool, buildFilteredToolCatalog, SMALL_LOCAL_PRIORITY_TOOL_NAMES, } from "./runtime-tool-specs.js";
|
|
12
13
|
import { validateRuntimeExecutorSessionRegistryPayload, } from "./schemas.js";
|
|
13
14
|
import { appendStatusEventSafe } from "./status-events.js";
|
|
14
15
|
import { appendVericifyProcessPostSafe, deriveVericifyRunRef, isVericifyBridgeEnabled, refreshVericifyBridgeSnapshotSafe, } from "./vericify-bridge.js";
|
|
15
|
-
import {
|
|
16
|
+
import { createWorkspaceSessionAsync, removeWorkspaceSession, runWorkspaceSessionHook, } from "./workspace-manager.js";
|
|
16
17
|
import { buildAceContinuityPacket, buildAceRecallContext, normalizeAutonomyPolicy, normalizeContinuityPolicy, } from "./ace-autonomy.js";
|
|
17
18
|
import { openStore } from "./store/ace-packed-store.js";
|
|
18
|
-
import { getWorkspaceStorePath, storeExistsSync,
|
|
19
|
+
import { getWorkspaceStorePath, storeExistsSync, } from "./store/store-snapshot.js";
|
|
19
20
|
import { operationalArtifactVirtualPath } from "./store/store-artifacts.js";
|
|
20
|
-
import {
|
|
21
|
+
import { withStoreWriteCoordinator } from "./store/write-coordinator.js";
|
|
22
|
+
import { LocalModelRuntimeRepository, withLocalModelRuntimeRepository, } from "./store/repositories/local-model-runtime-repository.js";
|
|
23
|
+
import { proposePlanDeterministic, validatePlan } from "./plan-proposal.js";
|
|
21
24
|
export const RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH = "agent-state/runtime-executor-sessions.json";
|
|
22
25
|
export const RUNTIME_EXECUTOR_SESSION_SCHEMA_REL_PATH = "agent-state/MODULES/schemas/RUNTIME_EXECUTOR_SESSION_REGISTRY.schema.json";
|
|
23
26
|
export const RUNTIME_EXECUTOR_SESSION_SCHEMA_NAME = "runtime-executor-session-registry@1.0.0";
|
|
27
|
+
const WORKSPACE_ROOT = resolveWorkspaceRoot();
|
|
28
|
+
let activeWorkspaceRoot = WORKSPACE_ROOT;
|
|
29
|
+
export const DEFAULT_TURN_OUTPUT_POLICY = {
|
|
30
|
+
emit_to: ["tui", "vericify"],
|
|
31
|
+
silent_unless_blocked: false,
|
|
32
|
+
require_approval_before_emit: false,
|
|
33
|
+
};
|
|
34
|
+
function buildToolChoiceAuditRecord(input) {
|
|
35
|
+
const priority_tools = SMALL_LOCAL_PRIORITY_TOOL_NAMES.filter((name) => input.filteredCatalog.entries.some((entry) => entry.name === name));
|
|
36
|
+
const demoted_tools = [
|
|
37
|
+
...(input.model_class === "small_local"
|
|
38
|
+
? input.filteredCatalog.entries
|
|
39
|
+
.filter((entry) => entry.cost_class === "heavy")
|
|
40
|
+
.map((entry) => ({
|
|
41
|
+
name: entry.name,
|
|
42
|
+
reason_code: "heavy_for_small_local",
|
|
43
|
+
}))
|
|
44
|
+
: []),
|
|
45
|
+
...input.filteredCatalog.unavailable_tools
|
|
46
|
+
.filter((entry) => entry.reason_code === "workspace_disabled" || entry.reason_code === "policy_disabled")
|
|
47
|
+
.map((entry) => ({
|
|
48
|
+
name: entry.name,
|
|
49
|
+
reason_code: entry.reason_code,
|
|
50
|
+
})),
|
|
51
|
+
];
|
|
52
|
+
return {
|
|
53
|
+
audit_id: randomUUID(),
|
|
54
|
+
session_id: input.session_id,
|
|
55
|
+
turn_number: input.turn_number,
|
|
56
|
+
captured_at: input.captured_at,
|
|
57
|
+
model_class: input.model_class,
|
|
58
|
+
priority_tools,
|
|
59
|
+
demoted_tools,
|
|
60
|
+
selected_budget: input.selected_budget,
|
|
61
|
+
capability_snapshot_id: input.capability_snapshot_id,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
const sessionSidecarStates = new Map();
|
|
65
|
+
function initSessionSidecars(sessionId) {
|
|
66
|
+
let release;
|
|
67
|
+
const gate = new Promise((resolve) => {
|
|
68
|
+
release = resolve;
|
|
69
|
+
});
|
|
70
|
+
sessionSidecarStates.set(sessionId, { chain: gate, release });
|
|
71
|
+
}
|
|
72
|
+
function scheduleRuntimeSidecar(sessionId, _label, fn) {
|
|
73
|
+
const state = sessionSidecarStates.get(sessionId);
|
|
74
|
+
if (!state)
|
|
75
|
+
return;
|
|
76
|
+
const next = state.chain.then(() => fn()).catch(() => undefined);
|
|
77
|
+
state.chain = next;
|
|
78
|
+
}
|
|
79
|
+
function releaseSidecarsAndForget(sessionId) {
|
|
80
|
+
const state = sessionSidecarStates.get(sessionId);
|
|
81
|
+
if (!state)
|
|
82
|
+
return;
|
|
83
|
+
if (state.release) {
|
|
84
|
+
state.release();
|
|
85
|
+
state.release = undefined;
|
|
86
|
+
}
|
|
87
|
+
// Let sidecars run in background; clean up state when chain completes.
|
|
88
|
+
void state.chain.finally(() => {
|
|
89
|
+
sessionSidecarStates.delete(sessionId);
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
async function waitForSessionSidecars(sessionId) {
|
|
93
|
+
const state = sessionSidecarStates.get(sessionId);
|
|
94
|
+
if (!state)
|
|
95
|
+
return;
|
|
96
|
+
await state.chain.catch(() => undefined);
|
|
97
|
+
}
|
|
98
|
+
async function delaySessionCompletedSidecarIfRequested(sessionId) {
|
|
99
|
+
const rawDelayMs = process.env.ACE_RUNTIME_EXECUTOR_TEST_SIDECAR_DELAY_MS;
|
|
100
|
+
if (!rawDelayMs)
|
|
101
|
+
return;
|
|
102
|
+
const delayMs = Number(rawDelayMs);
|
|
103
|
+
if (!Number.isFinite(delayMs) || delayMs <= 0)
|
|
104
|
+
return;
|
|
105
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
106
|
+
const markerDir = process.env.ACE_RUNTIME_EXECUTOR_TEST_SIDECAR_MARKER_DIR;
|
|
107
|
+
if (markerDir && markerDir.trim().length > 0) {
|
|
108
|
+
mkdirSync(markerDir, { recursive: true });
|
|
109
|
+
writeFileSync(join(markerDir, `${sessionId}.done`), new Date().toISOString(), "utf-8");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function setSessionPhase(sessionId, phase, checkpoint) {
|
|
113
|
+
const active = activeSessions.get(sessionId);
|
|
114
|
+
if (active) {
|
|
115
|
+
active.phase = phase;
|
|
116
|
+
active.phase_started_at = Date.now();
|
|
117
|
+
if (checkpoint !== undefined)
|
|
118
|
+
active.last_checkpoint = checkpoint;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
24
121
|
const activeSessions = new Map();
|
|
122
|
+
async function readLocalModelRuntimeStatus(sessionId) {
|
|
123
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
124
|
+
return undefined;
|
|
125
|
+
const storePath = getWorkspaceStorePath(workspaceRoot());
|
|
126
|
+
const store = await openStore(storePath);
|
|
127
|
+
try {
|
|
128
|
+
const repo = new LocalModelRuntimeRepository(store);
|
|
129
|
+
return await repo.getRuntimeStatus(sessionId);
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
await store.close();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
25
135
|
const turnResponseSchema = z
|
|
26
136
|
.object({
|
|
27
137
|
status: z.enum(["continue", "done", "blocked", "failed"]),
|
|
@@ -45,14 +155,27 @@ function defaultRegistry() {
|
|
|
45
155
|
};
|
|
46
156
|
}
|
|
47
157
|
function workspaceRoot() {
|
|
48
|
-
return
|
|
158
|
+
return WORKSPACE_ROOT;
|
|
159
|
+
}
|
|
160
|
+
function runtimeProfilePath() {
|
|
161
|
+
return resolve(WORKSPACE_ROOT, "agent-state", "ACE_WORKFLOW.md");
|
|
162
|
+
}
|
|
163
|
+
function loadCurrentRuntimeProfile() {
|
|
164
|
+
const workspaceProfilePath = runtimeProfilePath();
|
|
165
|
+
if (existsSync(workspaceProfilePath)) {
|
|
166
|
+
const result = loadRuntimeProfile(workspaceProfilePath);
|
|
167
|
+
if (result.ok)
|
|
168
|
+
return result;
|
|
169
|
+
throw new Error(`ACE workflow runtime profile is invalid: ${result.errors.join("; ")}`);
|
|
170
|
+
}
|
|
171
|
+
const packaged = loadRuntimeProfile();
|
|
172
|
+
if (packaged.ok)
|
|
173
|
+
return packaged;
|
|
174
|
+
throw new Error(`ACE runtime profile is invalid: ${packaged.errors.join("; ")}`);
|
|
49
175
|
}
|
|
50
176
|
function resolveWorkspaceArtifactPath(filePath) {
|
|
51
177
|
return resolveWorkspaceArtifactPathHelper(filePath, "write");
|
|
52
178
|
}
|
|
53
|
-
function safeWriteWorkspaceFile(filePath, content) {
|
|
54
|
-
return safeWrite(filePath, content);
|
|
55
|
-
}
|
|
56
179
|
function parseRegistry(raw) {
|
|
57
180
|
let parsed;
|
|
58
181
|
try {
|
|
@@ -69,17 +192,19 @@ function parseRegistry(raw) {
|
|
|
69
192
|
}
|
|
70
193
|
function readRegistry() {
|
|
71
194
|
const raw = safeRead(RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH);
|
|
72
|
-
if (raw.startsWith("[FILE NOT FOUND]") ||
|
|
195
|
+
if (raw.startsWith("[FILE NOT FOUND]") ||
|
|
196
|
+
raw.startsWith("[ACCESS DENIED]") ||
|
|
197
|
+
raw.includes("\x00")) {
|
|
73
198
|
return defaultRegistry();
|
|
74
199
|
}
|
|
75
200
|
return parseRegistry(raw);
|
|
76
201
|
}
|
|
77
|
-
function writeRegistry(registry) {
|
|
202
|
+
async function writeRegistry(registry) {
|
|
78
203
|
const validation = validateRuntimeExecutorSessionRegistryPayload(registry);
|
|
79
204
|
if (!validation.ok) {
|
|
80
205
|
throw new Error(`Runtime executor session registry failed validation (${validation.schema}): ${validation.errors.join("; ")}`);
|
|
81
206
|
}
|
|
82
|
-
return
|
|
207
|
+
return safeWriteAsync(RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH, JSON.stringify(registry, null, 2));
|
|
83
208
|
}
|
|
84
209
|
function registryPath() {
|
|
85
210
|
if (storeExistsSync(workspaceRoot())) {
|
|
@@ -92,7 +217,7 @@ async function mutateRegistry(updater) {
|
|
|
92
217
|
const registry = readRegistry();
|
|
93
218
|
const result = await updater(registry);
|
|
94
219
|
registry.updated_at = new Date().toISOString();
|
|
95
|
-
writeRegistry(registry);
|
|
220
|
+
await writeRegistry(registry);
|
|
96
221
|
return result;
|
|
97
222
|
});
|
|
98
223
|
}
|
|
@@ -245,6 +370,24 @@ async function emitVericifyProcessPostForSession(session, kind, summary, toolRef
|
|
|
245
370
|
}).catch(() => undefined);
|
|
246
371
|
refreshVericifyBridgeSnapshotSafe();
|
|
247
372
|
}
|
|
373
|
+
async function touchRuntimeProgress(sessionId, _reason, at = Date.now()) {
|
|
374
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
375
|
+
return;
|
|
376
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
377
|
+
const currentStatus = await repo.getRuntimeStatus(sessionId);
|
|
378
|
+
if (!currentStatus)
|
|
379
|
+
return;
|
|
380
|
+
await repo.upsertRuntimeStatusWithTransition({
|
|
381
|
+
...currentStatus,
|
|
382
|
+
sleep_state: currentStatus.sleep_state === "stalled" ? "awake" : currentStatus.sleep_state,
|
|
383
|
+
last_progress_at: at,
|
|
384
|
+
}, {
|
|
385
|
+
from: currentStatus.bridge_status,
|
|
386
|
+
reason: _reason,
|
|
387
|
+
evidence_refs: [`session:${sessionId}`, `progress_at:${at}`],
|
|
388
|
+
});
|
|
389
|
+
}).catch(() => undefined);
|
|
390
|
+
}
|
|
248
391
|
async function emitExecutorEvent(eventType, status, summary, payload) {
|
|
249
392
|
await appendStatusEventSafe({
|
|
250
393
|
source_module: "capability-framework",
|
|
@@ -300,7 +443,10 @@ async function persistTurnArtifacts(paths, prompt, requestPayload, responseRaw)
|
|
|
300
443
|
};
|
|
301
444
|
}
|
|
302
445
|
const storePath = getWorkspaceStorePath(workspaceRoot());
|
|
303
|
-
|
|
446
|
+
writeFileSync(paths.tempPromptPath, prompt, "utf-8");
|
|
447
|
+
writeFileSync(paths.tempRequestPath, JSON.stringify(requestPayload, null, 2), "utf-8");
|
|
448
|
+
writeFileSync(paths.tempResponsePath, responseRaw, "utf-8");
|
|
449
|
+
await withStoreWriteCoordinator(storePath, async () => {
|
|
304
450
|
const store = await openStore(storePath);
|
|
305
451
|
try {
|
|
306
452
|
await store.setBlob(paths.promptStoreKey, prompt);
|
|
@@ -311,27 +457,31 @@ async function persistTurnArtifacts(paths, prompt, requestPayload, responseRaw)
|
|
|
311
457
|
finally {
|
|
312
458
|
await store.close();
|
|
313
459
|
}
|
|
314
|
-
});
|
|
460
|
+
}, { operation_label: "runtime-executor-turn-artifacts" });
|
|
315
461
|
return {
|
|
316
|
-
promptPath:
|
|
317
|
-
requestPath:
|
|
318
|
-
responsePath:
|
|
462
|
+
promptPath: paths.tempPromptPath,
|
|
463
|
+
requestPath: paths.tempRequestPath,
|
|
464
|
+
responsePath: paths.tempResponsePath,
|
|
319
465
|
};
|
|
320
466
|
}
|
|
321
|
-
async function runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook) {
|
|
467
|
+
async function runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook, runtimeProfilePath) {
|
|
322
468
|
let cleanupError;
|
|
323
469
|
let cleanupStatus = "pending";
|
|
324
470
|
if (shouldRunAfterHook) {
|
|
325
|
-
const afterRun = runWorkspaceSessionHook({
|
|
471
|
+
const afterRun = await runWorkspaceSessionHook({
|
|
326
472
|
session_id: workspaceSessionId,
|
|
327
473
|
kind: "after_run",
|
|
474
|
+
runtime_profile_path: runtimeProfilePath,
|
|
328
475
|
});
|
|
329
476
|
if (!afterRun.ok) {
|
|
330
477
|
cleanupError = afterRun.error ?? "after_run hook failed";
|
|
331
478
|
cleanupStatus = "failed";
|
|
332
479
|
}
|
|
333
480
|
}
|
|
334
|
-
const removal = removeWorkspaceSession({
|
|
481
|
+
const removal = await removeWorkspaceSession({
|
|
482
|
+
session_id: workspaceSessionId,
|
|
483
|
+
runtime_profile_path: runtimeProfilePath,
|
|
484
|
+
});
|
|
335
485
|
if (removal.ok && removal.session) {
|
|
336
486
|
cleanupStatus =
|
|
337
487
|
removal.session.status === "archived" ? "archived" : "removed";
|
|
@@ -340,6 +490,8 @@ async function runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook) {
|
|
|
340
490
|
cleanupError = removal.error ?? "Workspace session cleanup failed";
|
|
341
491
|
cleanupStatus = "failed";
|
|
342
492
|
}
|
|
493
|
+
// Update the registry first (critical lane) then emit the transition record as a sidecar.
|
|
494
|
+
setSessionPhase(sessionId, "cleanup_registry_update");
|
|
343
495
|
await mutateRegistry((registry) => {
|
|
344
496
|
const session = findSession(registry, sessionId);
|
|
345
497
|
if (!session)
|
|
@@ -352,15 +504,54 @@ async function runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook) {
|
|
|
352
504
|
if (cleanupError)
|
|
353
505
|
session.cleanup_error = cleanupError;
|
|
354
506
|
});
|
|
507
|
+
const capturedCleanupStatus = cleanupStatus;
|
|
508
|
+
const capturedCleanupError = cleanupError;
|
|
509
|
+
const capturedWorkspaceSessionId = workspaceSessionId;
|
|
510
|
+
scheduleRuntimeSidecar(sessionId, "cleanup-transition", async () => {
|
|
511
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
512
|
+
return;
|
|
513
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
514
|
+
await repo.appendTransitionRecord({
|
|
515
|
+
session_id: sessionId,
|
|
516
|
+
subject_kind: "workspace_session",
|
|
517
|
+
subject_id: capturedWorkspaceSessionId,
|
|
518
|
+
from: "active",
|
|
519
|
+
to: capturedCleanupStatus === "archived" ? "archived" : capturedCleanupStatus === "removed" ? "removed" : "failed",
|
|
520
|
+
reason: capturedCleanupError
|
|
521
|
+
? `Workspace session ${capturedWorkspaceSessionId} cleanup failed: ${capturedCleanupError}`
|
|
522
|
+
: `Workspace session ${capturedWorkspaceSessionId} cleanup completed.`,
|
|
523
|
+
reason_code: capturedCleanupError ? "cleanup_failed" : "cleanup_complete",
|
|
524
|
+
evidence_refs: [capturedWorkspaceSessionId],
|
|
525
|
+
});
|
|
526
|
+
}).catch(() => undefined);
|
|
527
|
+
});
|
|
355
528
|
}
|
|
356
|
-
|
|
529
|
+
function classifyTurnOutcome(responseStatus, policy, toolCallCount) {
|
|
530
|
+
if (responseStatus === "blocked" || responseStatus === "failed") {
|
|
531
|
+
return {
|
|
532
|
+
outcome: "escalation_blocker",
|
|
533
|
+
reason: `Turn ended with status=${responseStatus}; external action required`,
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
if (policy.silent_unless_blocked && toolCallCount === 0) {
|
|
537
|
+
return {
|
|
538
|
+
outcome: "no_op_success",
|
|
539
|
+
reason: "silent_unless_blocked=true and turn produced no tool calls",
|
|
540
|
+
};
|
|
541
|
+
}
|
|
542
|
+
return {
|
|
543
|
+
outcome: "meaningful_completion",
|
|
544
|
+
reason: `Turn completed normally with ${toolCallCount} tool call(s)`,
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
async function runSessionLoop(sessionId, context, autoCleanup, runtimeProfileSnapshot) {
|
|
357
548
|
const active = activeSessions.get(sessionId);
|
|
358
549
|
let shouldRunAfterHook = false;
|
|
359
550
|
let workspaceSessionId = "";
|
|
360
551
|
try {
|
|
361
|
-
const
|
|
362
|
-
const
|
|
363
|
-
|
|
552
|
+
const autonomyPolicy = normalizeAutonomyPolicy(runtimeProfileSnapshot.profile.autonomy);
|
|
553
|
+
const continuityPolicy = normalizeContinuityPolicy(runtimeProfileSnapshot.profile.continuity);
|
|
554
|
+
setSessionPhase(sessionId, "registry_start");
|
|
364
555
|
const registryStart = await mutateRegistry((registry) => {
|
|
365
556
|
const session = findSession(registry, sessionId);
|
|
366
557
|
if (!session)
|
|
@@ -374,6 +565,7 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
374
565
|
if (!registryStart) {
|
|
375
566
|
throw new Error(`Failed to start unattended session: ${sessionId}`);
|
|
376
567
|
}
|
|
568
|
+
setSessionPhase(sessionId, "preflight");
|
|
377
569
|
const preflight = runAutonomyChecks("before_run", autonomyPolicy, context);
|
|
378
570
|
if (!preflight.ok) {
|
|
379
571
|
const summary = `Autonomy preflight blocked execution: ${preflight.summary}`;
|
|
@@ -391,9 +583,36 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
391
583
|
}
|
|
392
584
|
return;
|
|
393
585
|
}
|
|
394
|
-
|
|
586
|
+
// ── Plan validation preflight (Task 6) ───────────────────────────────
|
|
587
|
+
{
|
|
588
|
+
const currentValidatedPlanId = registryStart.validated_plan_id;
|
|
589
|
+
if (!currentValidatedPlanId) {
|
|
590
|
+
const task = registryStart.current_task || registryStart.task;
|
|
591
|
+
try {
|
|
592
|
+
const proposal = proposePlanDeterministic(task);
|
|
593
|
+
const verdict = await validatePlan({ proposal, sessionId });
|
|
594
|
+
if (verdict.ok) {
|
|
595
|
+
// Persist validated_plan_id opportunistically. This is a readiness hint,
|
|
596
|
+
// not a hard gate on unattended execution.
|
|
597
|
+
await mutateRegistry((registry) => {
|
|
598
|
+
const session = findSession(registry, sessionId);
|
|
599
|
+
if (session)
|
|
600
|
+
session.validated_plan_id = verdict.plan_id;
|
|
601
|
+
}).catch(() => undefined);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
catch {
|
|
605
|
+
// Preflight validation is best-effort. Unattended runtime should still run
|
|
606
|
+
// even when the planner/model path or trace lookup is unavailable.
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
// ── End plan validation preflight ────────────────────────────────────
|
|
611
|
+
setSessionPhase(sessionId, "workspace_before_run");
|
|
612
|
+
const beforeRun = await runWorkspaceSessionHook({
|
|
395
613
|
session_id: registryStart.workspace_session_id,
|
|
396
614
|
kind: "before_run",
|
|
615
|
+
runtime_profile_path: runtimeProfileSnapshot.path,
|
|
397
616
|
});
|
|
398
617
|
if (!beforeRun.ok) {
|
|
399
618
|
const summary = beforeRun.error ?? "before_run hook failed";
|
|
@@ -411,6 +630,56 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
411
630
|
shouldRunAfterHook = true;
|
|
412
631
|
let currentTask = registryStart.current_task;
|
|
413
632
|
let previousToolCalls = [];
|
|
633
|
+
const runtimeStatus = storeExistsSync(workspaceRoot())
|
|
634
|
+
? await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => repo.getRuntimeStatus(sessionId))
|
|
635
|
+
: undefined;
|
|
636
|
+
const executionPolicy = resolveLocalModelExecutionPolicy({
|
|
637
|
+
provider: "",
|
|
638
|
+
model: "",
|
|
639
|
+
role: "orchestrator",
|
|
640
|
+
task: currentTask,
|
|
641
|
+
requested_model_class: runtimeStatus?.model_class,
|
|
642
|
+
});
|
|
643
|
+
const effectiveModelClass = executionPolicy.model_class;
|
|
644
|
+
// Stall restart state — session-level, reset on successful turn.
|
|
645
|
+
// Seeded runtime status is authoritative when present.
|
|
646
|
+
let consecutiveStallRestarts = runtimeStatus?.stall_restart_count ?? 0;
|
|
647
|
+
let isStallRestart = false;
|
|
648
|
+
let stallRestartContext = "";
|
|
649
|
+
// Liveness defaults are explicit runtime policy. The resolved model class
|
|
650
|
+
// only informs the default budget profile when the caller has not already
|
|
651
|
+
// supplied a workspace/session budget or status-seeded runtime budget.
|
|
652
|
+
const livenessDefaults = getLivenessDefaults(effectiveModelClass);
|
|
653
|
+
const effectiveTurnBudgetMs = runtimeStatus?.turn_budget_ms ?? registryStart.turn_timeout_ms ?? livenessDefaults.turn_budget_ms;
|
|
654
|
+
const effectiveStallWindowMs = runtimeStatus?.stall_window_ms ?? livenessDefaults.stall_window_ms;
|
|
655
|
+
const maxStallRestarts = livenessDefaults.max_stall_restarts;
|
|
656
|
+
const initialBackoffMs = livenessDefaults.initial_backoff_ms;
|
|
657
|
+
const maxRetryBackoffMs = livenessDefaults.max_retry_backoff_ms;
|
|
658
|
+
let lastProgressAt = Date.now();
|
|
659
|
+
let queuedProgress;
|
|
660
|
+
let progressWrite;
|
|
661
|
+
const pumpProgressWrite = () => {
|
|
662
|
+
if (progressWrite)
|
|
663
|
+
return progressWrite;
|
|
664
|
+
progressWrite = (async () => {
|
|
665
|
+
while (queuedProgress) {
|
|
666
|
+
const next = queuedProgress;
|
|
667
|
+
queuedProgress = undefined;
|
|
668
|
+
await touchRuntimeProgress(sessionId, next.reason, next.at).catch(() => undefined);
|
|
669
|
+
}
|
|
670
|
+
})().finally(() => {
|
|
671
|
+
progressWrite = undefined;
|
|
672
|
+
});
|
|
673
|
+
return progressWrite;
|
|
674
|
+
};
|
|
675
|
+
const noteProgress = (reason, at = Date.now()) => {
|
|
676
|
+
lastProgressAt = at;
|
|
677
|
+
queuedProgress = { reason, at };
|
|
678
|
+
void pumpProgressWrite();
|
|
679
|
+
};
|
|
680
|
+
const flushProgress = async () => {
|
|
681
|
+
await pumpProgressWrite().catch(() => undefined);
|
|
682
|
+
};
|
|
414
683
|
for (let turnNumber = 1; turnNumber <= registryStart.max_turns; turnNumber += 1) {
|
|
415
684
|
if (active?.stop_requested) {
|
|
416
685
|
const stopped = await finalizeSession(sessionId, "stopped", `Unattended session ${sessionId} stopped by request.`);
|
|
@@ -423,6 +692,7 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
423
692
|
}
|
|
424
693
|
return;
|
|
425
694
|
}
|
|
695
|
+
setSessionPhase(sessionId, "turn_preparing", `Turn ${turnNumber}`);
|
|
426
696
|
const sessionSnapshot = await mutateRegistry((registry) => {
|
|
427
697
|
const session = findSession(registry, sessionId);
|
|
428
698
|
if (!session)
|
|
@@ -435,10 +705,14 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
435
705
|
throw new Error(`Missing unattended session ${sessionId}`);
|
|
436
706
|
const turnStartedAt = new Date().toISOString();
|
|
437
707
|
const paths = buildTurnPaths(sessionId, sessionSnapshot.workspace_path, turnNumber);
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
708
|
+
// Build capability-filtered tool catalog in-memory from the resolved model class.
|
|
709
|
+
const filteredCatalog = buildFilteredToolCatalog(effectiveModelClass);
|
|
710
|
+
const capabilitySnapshotId = randomUUID();
|
|
711
|
+
const selectedBudget = resolveEffectiveSurgicalReadBudget(runtimeProfileSnapshot.profile, effectiveModelClass);
|
|
712
|
+
const toolCatalog = filteredCatalog.entries.map((e) => ({
|
|
713
|
+
name: e.name,
|
|
714
|
+
description: e.description,
|
|
715
|
+
input_schema: e.input_schema,
|
|
442
716
|
}));
|
|
443
717
|
const turnRecall = autonomyPolicy.recall_context
|
|
444
718
|
? buildAceRecallContext({
|
|
@@ -452,8 +726,11 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
452
726
|
snapshot_name: resolveSnapshotName(context),
|
|
453
727
|
})
|
|
454
728
|
: undefined;
|
|
455
|
-
const
|
|
456
|
-
task: currentTask
|
|
729
|
+
const taskWithStallHint = isStallRestart
|
|
730
|
+
? `${stallRestartContext}\n\nOriginal task: ${currentTask}`
|
|
731
|
+
: currentTask;
|
|
732
|
+
let prompt = renderRuntimePromptFromSnapshot(runtimeProfileSnapshot, {
|
|
733
|
+
task: taskWithStallHint,
|
|
457
734
|
session_id: sessionId,
|
|
458
735
|
turn_number: turnNumber,
|
|
459
736
|
workspace_path: sessionSnapshot.workspace_path,
|
|
@@ -466,6 +743,16 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
466
743
|
ace_continuity_packet_md: renderAceContinuityPromptBlock(turnContinuity),
|
|
467
744
|
ace_context_snapshot_md: renderAceSnapshotPromptBlock(turnRecall),
|
|
468
745
|
});
|
|
746
|
+
if (effectiveModelClass === "small_local") {
|
|
747
|
+
prompt += [
|
|
748
|
+
"",
|
|
749
|
+
"## ACE-Owned Preflight Packet",
|
|
750
|
+
`- status: ${preflight.ok ? "passed" : "blocked"}`,
|
|
751
|
+
`- summary: ${preflight.summary}`,
|
|
752
|
+
`- evidence_refs: agent-state/TASK.md, agent-state/STATUS.md, agent-state/QUALITY_GATES.md`,
|
|
753
|
+
"- model_visible_tools: route_task or run_orchestrator only when delegation is needed; ACE owns recall_context, validate_framework, and get_framework_status.",
|
|
754
|
+
].join("\n");
|
|
755
|
+
}
|
|
469
756
|
const requestPayload = {
|
|
470
757
|
session_id: sessionId,
|
|
471
758
|
turn_number: turnNumber,
|
|
@@ -493,13 +780,106 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
493
780
|
};
|
|
494
781
|
writeFileSync(paths.tempPromptPath, prompt, "utf-8");
|
|
495
782
|
writeFileSync(paths.tempRequestPath, JSON.stringify(requestPayload, null, 2), "utf-8");
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
783
|
+
// Sidecar: persist capability snapshot, tool-choice audit, turn snapshot, and TURN_STARTED event.
|
|
784
|
+
// These are audit evidence — the shell must not wait for them before starting.
|
|
785
|
+
const capturedSnapshotId = capabilitySnapshotId;
|
|
786
|
+
const capturedTurnNumber = turnNumber;
|
|
787
|
+
const capturedSnapshot = sessionSnapshot;
|
|
788
|
+
const capturedRecall = turnRecall;
|
|
789
|
+
const capturedContinuity = turnContinuity;
|
|
790
|
+
const capturedBudget = selectedBudget;
|
|
791
|
+
const capturedCatalog = filteredCatalog;
|
|
792
|
+
scheduleRuntimeSidecar(sessionId, `capability-snapshot-turn-${turnNumber}`, async () => {
|
|
793
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
794
|
+
return;
|
|
795
|
+
let toolChoiceAuditId;
|
|
796
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
797
|
+
const toolChoiceAudit = buildToolChoiceAuditRecord({
|
|
798
|
+
session_id: sessionId,
|
|
799
|
+
turn_number: capturedTurnNumber,
|
|
800
|
+
captured_at: Date.now(),
|
|
801
|
+
model_class: effectiveModelClass,
|
|
802
|
+
capability_snapshot_id: capturedSnapshotId,
|
|
803
|
+
filteredCatalog: capturedCatalog,
|
|
804
|
+
selected_budget: { read_file_lines_max_lines: capturedBudget.read_file_lines_max_lines },
|
|
805
|
+
});
|
|
806
|
+
toolChoiceAuditId = toolChoiceAudit.audit_id;
|
|
807
|
+
await repo.saveToolChoiceAudit(toolChoiceAudit);
|
|
808
|
+
await repo.saveCapabilitySnapshot({
|
|
809
|
+
snapshot_id: capturedSnapshotId,
|
|
810
|
+
session_id: sessionId,
|
|
811
|
+
turn_number: capturedTurnNumber,
|
|
812
|
+
captured_at: Date.now(),
|
|
813
|
+
model_class: effectiveModelClass,
|
|
814
|
+
allowed_tools: capturedCatalog.entries.map((e) => e.name),
|
|
815
|
+
unavailable_tools: capturedCatalog.unavailable_tools,
|
|
816
|
+
tool_cost_class: capturedCatalog.tool_cost_class,
|
|
817
|
+
});
|
|
818
|
+
await repo.appendTransitionRecord({
|
|
819
|
+
session_id: sessionId,
|
|
820
|
+
subject_kind: "capability",
|
|
821
|
+
subject_id: `${sessionId}/${capturedTurnNumber}`,
|
|
822
|
+
from: "previous_turn",
|
|
823
|
+
to: `model_class:${effectiveModelClass}`,
|
|
824
|
+
reason: `Capability snapshot captured for turn ${capturedTurnNumber}`,
|
|
825
|
+
evidence_refs: [capturedSnapshotId],
|
|
826
|
+
});
|
|
827
|
+
}).catch(() => undefined);
|
|
828
|
+
const snapStorePath = getWorkspaceStorePath(workspaceRoot());
|
|
829
|
+
await withStoreWriteCoordinator(snapStorePath, async () => {
|
|
830
|
+
const snapStore = await openStore(snapStorePath);
|
|
831
|
+
try {
|
|
832
|
+
const snapRepo = new LocalModelRuntimeRepository(snapStore);
|
|
833
|
+
const currentStatus = await snapRepo.getRuntimeStatus(sessionId);
|
|
834
|
+
const updatedStatus = await snapRepo.upsertRuntimeStatusWithTransition({
|
|
835
|
+
...(currentStatus ?? {
|
|
836
|
+
session_id: sessionId,
|
|
837
|
+
process_id: process.pid,
|
|
838
|
+
turn_count: capturedTurnNumber,
|
|
839
|
+
bridge_status: "running",
|
|
840
|
+
preflight_state: "ready",
|
|
841
|
+
surface_kind: "unattended_runtime",
|
|
842
|
+
}),
|
|
843
|
+
active_workspace_path: capturedSnapshot.workspace_path,
|
|
844
|
+
active_workspace_session_id: registryStart.workspace_session_id,
|
|
845
|
+
workspace_hook_health: "ok",
|
|
846
|
+
last_progress_at: Date.now(),
|
|
847
|
+
}, {
|
|
848
|
+
from: currentStatus?.bridge_status,
|
|
849
|
+
reason: `Turn ${capturedTurnNumber} snapshot captured with capability and prompt assembly state`,
|
|
850
|
+
evidence_refs: [capturedSnapshotId, toolChoiceAuditId].filter((ref) => typeof ref === "string" && ref.length > 0),
|
|
851
|
+
});
|
|
852
|
+
const turnSnapshot = {
|
|
853
|
+
snapshot_id: randomUUID(),
|
|
854
|
+
session_id: sessionId,
|
|
855
|
+
turn_number: capturedTurnNumber,
|
|
856
|
+
captured_at: Date.now(),
|
|
857
|
+
preflight: preflight,
|
|
858
|
+
recall: (capturedRecall ?? null),
|
|
859
|
+
continuity: (capturedContinuity ?? null),
|
|
860
|
+
capability_snapshot_id: capturedSnapshotId,
|
|
861
|
+
tool_choice_audit_id: toolChoiceAuditId ?? null,
|
|
862
|
+
surface_kind: updatedStatus.surface_kind ?? currentStatus?.surface_kind ?? "tui_interactive",
|
|
863
|
+
};
|
|
864
|
+
await snapRepo.saveTurnSnapshot(turnSnapshot);
|
|
865
|
+
await snapStore.commit();
|
|
866
|
+
}
|
|
867
|
+
finally {
|
|
868
|
+
await snapStore.close();
|
|
869
|
+
}
|
|
870
|
+
}, { operation_label: "runtime-executor-turn-snapshot" });
|
|
501
871
|
});
|
|
502
|
-
|
|
872
|
+
scheduleRuntimeSidecar(sessionId, `turn-started-event-${turnNumber}`, async () => {
|
|
873
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_TURN_STARTED", "in_progress", `Unattended session ${sessionId} turn ${capturedTurnNumber} started.`, {
|
|
874
|
+
session_id: sessionId,
|
|
875
|
+
turn_number: capturedTurnNumber,
|
|
876
|
+
workspace_path: capturedSnapshot.workspace_path,
|
|
877
|
+
...vericifyPayloadForSession(capturedSnapshot),
|
|
878
|
+
});
|
|
879
|
+
});
|
|
880
|
+
noteProgress(`Turn ${turnNumber} started.`);
|
|
881
|
+
setSessionPhase(sessionId, "turn_executing", `Turn ${turnNumber} shell`);
|
|
882
|
+
const shellResult = await runShellCommand(runtimeProfileSnapshot.profile.executor.command ?? "", {
|
|
503
883
|
cwd: sessionSnapshot.workspace_path,
|
|
504
884
|
env: {
|
|
505
885
|
...process.env,
|
|
@@ -510,31 +890,114 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
510
890
|
ACE_RUNTIME_REQUEST_FILE: paths.tempRequestPath,
|
|
511
891
|
ACE_RUNTIME_RESPONSE_FILE: paths.tempResponsePath,
|
|
512
892
|
},
|
|
513
|
-
timeout_ms:
|
|
893
|
+
timeout_ms: effectiveTurnBudgetMs,
|
|
894
|
+
stall_timeout_ms: effectiveStallWindowMs,
|
|
514
895
|
on_spawn: (child) => {
|
|
515
896
|
if (active)
|
|
516
897
|
active.child = child;
|
|
517
898
|
},
|
|
899
|
+
on_progress: (event) => {
|
|
900
|
+
noteProgress(`Executor ${event.source} progress during turn ${turnNumber}.`, event.at);
|
|
901
|
+
},
|
|
518
902
|
});
|
|
519
903
|
if (active)
|
|
520
904
|
active.child = undefined;
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
905
|
+
lastProgressAt = shellResult.last_progress_at;
|
|
906
|
+
await flushProgress();
|
|
907
|
+
setSessionPhase(sessionId, "turn_persisting", `Turn ${turnNumber} response`);
|
|
908
|
+
// Stall detection is based on the inter-progress window, not the total turn budget.
|
|
909
|
+
if (shellResult.timed_out && shellResult.timeout_reason === "stall") {
|
|
910
|
+
consecutiveStallRestarts += 1;
|
|
911
|
+
// Update runtime status: sleep_state = "stalled"
|
|
526
912
|
if (storeExistsSync(workspaceRoot())) {
|
|
527
|
-
|
|
913
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
914
|
+
const currentStatus = await repo.getRuntimeStatus(sessionId);
|
|
915
|
+
if (currentStatus) {
|
|
916
|
+
await repo.upsertRuntimeStatusWithTransition({
|
|
917
|
+
...currentStatus,
|
|
918
|
+
sleep_state: "stalled",
|
|
919
|
+
stall_restart_count: (currentStatus.stall_restart_count ?? 0) + 1,
|
|
920
|
+
last_progress_at: lastProgressAt,
|
|
921
|
+
}, {
|
|
922
|
+
from: currentStatus.bridge_status,
|
|
923
|
+
reason: `Turn ${turnNumber} exceeded stall_window_ms (${effectiveStallWindowMs}ms) without progress. Stall restart ${consecutiveStallRestarts}/${maxStallRestarts}.`,
|
|
924
|
+
evidence_refs: [
|
|
925
|
+
`turn:${turnNumber}`,
|
|
926
|
+
`stall_window_ms:${effectiveStallWindowMs}`,
|
|
927
|
+
`last_progress_at:${lastProgressAt}`,
|
|
928
|
+
],
|
|
929
|
+
});
|
|
930
|
+
await repo.appendTransitionRecord({
|
|
931
|
+
session_id: sessionId,
|
|
932
|
+
subject_kind: "runtime_status",
|
|
933
|
+
subject_id: sessionId,
|
|
934
|
+
from: "running",
|
|
935
|
+
to: "stalled",
|
|
936
|
+
reason: `Turn ${turnNumber} exceeded stall_window_ms (${effectiveStallWindowMs}ms) without progress. Stall restart ${consecutiveStallRestarts}/${maxStallRestarts}.`,
|
|
937
|
+
evidence_refs: [
|
|
938
|
+
`turn:${turnNumber}`,
|
|
939
|
+
`stall_window_ms:${effectiveStallWindowMs}`,
|
|
940
|
+
`last_progress_at:${lastProgressAt}`,
|
|
941
|
+
],
|
|
942
|
+
});
|
|
943
|
+
}
|
|
944
|
+
}).catch(() => undefined);
|
|
528
945
|
}
|
|
529
|
-
|
|
530
|
-
|
|
946
|
+
if (consecutiveStallRestarts > maxStallRestarts) {
|
|
947
|
+
// Exhausted restarts — escalate to blocked
|
|
948
|
+
const exhaustedSummary = `Stall restart limit exhausted after ${maxStallRestarts} attempts. Session blocked.`;
|
|
949
|
+
if (storeExistsSync(workspaceRoot())) {
|
|
950
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
951
|
+
const currentStatus = await repo.getRuntimeStatus(sessionId);
|
|
952
|
+
if (currentStatus) {
|
|
953
|
+
await repo.upsertRuntimeStatusWithTransition({
|
|
954
|
+
...currentStatus,
|
|
955
|
+
bridge_status: "blocked",
|
|
956
|
+
blocked_reason: "stall_restart_exhausted",
|
|
957
|
+
sleep_state: "stalled",
|
|
958
|
+
}, {
|
|
959
|
+
from: currentStatus.bridge_status,
|
|
960
|
+
reason: exhaustedSummary,
|
|
961
|
+
reason_code: "stall_restart_exhausted",
|
|
962
|
+
evidence_refs: [`stall_restarts:${consecutiveStallRestarts}`],
|
|
963
|
+
});
|
|
964
|
+
await repo.appendTransitionRecord({
|
|
965
|
+
session_id: sessionId,
|
|
966
|
+
subject_kind: "runtime_status",
|
|
967
|
+
subject_id: sessionId,
|
|
968
|
+
from: "stalled",
|
|
969
|
+
to: "blocked",
|
|
970
|
+
reason: exhaustedSummary,
|
|
971
|
+
reason_code: "stall_restart_exhausted",
|
|
972
|
+
evidence_refs: [`stall_restarts:${consecutiveStallRestarts}`],
|
|
973
|
+
});
|
|
974
|
+
}
|
|
975
|
+
}).catch(() => undefined);
|
|
976
|
+
}
|
|
977
|
+
await finalizeSession(sessionId, "blocked", exhaustedSummary, exhaustedSummary);
|
|
978
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_FAILED", "blocked", exhaustedSummary, {
|
|
979
|
+
session_id: sessionId,
|
|
980
|
+
turn_number: turnNumber,
|
|
981
|
+
blocked_reason: "stall_restart_exhausted",
|
|
982
|
+
...vericifyPayloadForSession(sessionSnapshot),
|
|
983
|
+
});
|
|
984
|
+
return;
|
|
985
|
+
}
|
|
986
|
+
// Backoff before restart
|
|
987
|
+
const backoffMs = Math.min(initialBackoffMs * Math.pow(2, consecutiveStallRestarts - 1), maxRetryBackoffMs, effectiveStallWindowMs);
|
|
988
|
+
await new Promise((resolve) => setTimeout(resolve, backoffMs));
|
|
989
|
+
// Set up restart context for the next turn
|
|
990
|
+
isStallRestart = true;
|
|
991
|
+
stallRestartContext = `The previous turn (turn ${turnNumber}) stalled after ${effectiveStallWindowMs}ms without progress inside turn_budget_ms=${effectiveTurnBudgetMs}ms; resume from current workspace state — do not restart from scratch.`;
|
|
992
|
+
continue;
|
|
993
|
+
}
|
|
994
|
+
// Reset stall state on successful turn completion
|
|
995
|
+
consecutiveStallRestarts = 0;
|
|
996
|
+
isStallRestart = false;
|
|
997
|
+
stallRestartContext = "";
|
|
531
998
|
if (active?.stop_requested) {
|
|
532
999
|
const turnEndedAt = new Date().toISOString();
|
|
533
|
-
const
|
|
534
|
-
status: "failed",
|
|
535
|
-
summary: `Turn ${turnNumber} stopped by request.`,
|
|
536
|
-
tool_calls: [],
|
|
537
|
-
}, null, 2));
|
|
1000
|
+
const stoppedSummary = `Turn ${turnNumber} stopped by request.`;
|
|
538
1001
|
await mutateRegistry((registry) => {
|
|
539
1002
|
const session = findSession(registry, sessionId);
|
|
540
1003
|
if (!session)
|
|
@@ -546,10 +1009,10 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
546
1009
|
ended_at: turnEndedAt,
|
|
547
1010
|
status: "stopped",
|
|
548
1011
|
response_status: "failed",
|
|
549
|
-
summary:
|
|
550
|
-
prompt_path:
|
|
551
|
-
request_path:
|
|
552
|
-
response_path:
|
|
1012
|
+
summary: stoppedSummary,
|
|
1013
|
+
prompt_path: paths.tempPromptPath,
|
|
1014
|
+
request_path: paths.tempRequestPath,
|
|
1015
|
+
response_path: paths.tempResponsePath,
|
|
553
1016
|
exit_code: shellResult.exit_code,
|
|
554
1017
|
stdout: clipOutput(shellResult.stdout),
|
|
555
1018
|
stderr: clipOutput(shellResult.stderr),
|
|
@@ -559,13 +1022,12 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
559
1022
|
session.turn_count = session.turns.length;
|
|
560
1023
|
});
|
|
561
1024
|
const stopped = await finalizeSession(sessionId, "stopped", `Unattended session ${sessionId} stopped by request.`);
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
});
|
|
1025
|
+
scheduleRuntimeSidecar(sessionId, "session-stopped-event", async () => {
|
|
1026
|
+
if (!stopped)
|
|
1027
|
+
return;
|
|
1028
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_STOPPED", "done", stopped.result_summary ?? `Unattended session ${sessionId} stopped by request.`, { session_id: sessionId, ...vericifyPayloadForSession(stopped) });
|
|
567
1029
|
await emitVericifyProcessPostForSession(stopped, "progress", stopped.result_summary ?? `Unattended session ${sessionId} stopped by request.`, ["runtime-executor"]);
|
|
568
|
-
}
|
|
1030
|
+
});
|
|
569
1031
|
return;
|
|
570
1032
|
}
|
|
571
1033
|
let response;
|
|
@@ -575,11 +1037,6 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
575
1037
|
catch (error) {
|
|
576
1038
|
const message = error instanceof Error ? error.message : String(error);
|
|
577
1039
|
const turnEndedAt = new Date().toISOString();
|
|
578
|
-
const artifactPaths = await finalizeTurnArtifacts(JSON.stringify({
|
|
579
|
-
status: "failed",
|
|
580
|
-
summary: message,
|
|
581
|
-
tool_calls: [],
|
|
582
|
-
}, null, 2));
|
|
583
1040
|
await mutateRegistry((registry) => {
|
|
584
1041
|
const session = findSession(registry, sessionId);
|
|
585
1042
|
if (!session)
|
|
@@ -592,9 +1049,9 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
592
1049
|
status: "failed",
|
|
593
1050
|
response_status: "failed",
|
|
594
1051
|
summary: message,
|
|
595
|
-
prompt_path:
|
|
596
|
-
request_path:
|
|
597
|
-
response_path:
|
|
1052
|
+
prompt_path: paths.tempPromptPath,
|
|
1053
|
+
request_path: paths.tempRequestPath,
|
|
1054
|
+
response_path: paths.tempResponsePath,
|
|
598
1055
|
exit_code: shellResult.exit_code,
|
|
599
1056
|
stdout: clipOutput(shellResult.stdout),
|
|
600
1057
|
stderr: clipOutput(shellResult.stderr),
|
|
@@ -604,28 +1061,30 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
604
1061
|
session.turn_count = session.turns.length;
|
|
605
1062
|
});
|
|
606
1063
|
const failed = await finalizeSession(sessionId, "failed", message, message);
|
|
607
|
-
|
|
1064
|
+
scheduleRuntimeSidecar(sessionId, "session-parse-failed-event", async () => {
|
|
1065
|
+
if (!failed)
|
|
1066
|
+
return;
|
|
608
1067
|
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_FAILED", "fail", message, {
|
|
609
1068
|
session_id: sessionId,
|
|
610
1069
|
turn_number: turnNumber,
|
|
611
1070
|
...vericifyPayloadForSession(failed),
|
|
612
1071
|
});
|
|
613
|
-
await emitVericifyProcessPostForSession(failed, "blocker", message, [
|
|
614
|
-
|
|
615
|
-
]);
|
|
616
|
-
}
|
|
1072
|
+
await emitVericifyProcessPostForSession(failed, "blocker", message, ["runtime-executor"]);
|
|
1073
|
+
});
|
|
617
1074
|
return;
|
|
618
1075
|
}
|
|
619
1076
|
const normalizedSummary = normalizeWorkspaceSummary(response.summary, sessionSnapshot.workspace_path);
|
|
620
|
-
const artifactPaths = await finalizeTurnArtifacts(readFileSync(paths.tempResponsePath, "utf-8"));
|
|
621
1077
|
const toolCalls = [];
|
|
622
1078
|
for (const toolCall of response.tool_calls) {
|
|
623
1079
|
const toolStartedAt = new Date().toISOString();
|
|
1080
|
+
noteProgress(`Runtime tool ${toolCall.name} started during turn ${turnNumber}.`);
|
|
624
1081
|
const toolResult = await executeRuntimeTool(toolCall.name, toolCall.input, {
|
|
625
1082
|
session_id: sessionId,
|
|
626
1083
|
workspace_path: sessionSnapshot.workspace_path,
|
|
627
1084
|
turn_number: turnNumber,
|
|
628
1085
|
});
|
|
1086
|
+
noteProgress(`Runtime tool ${toolCall.name} finished during turn ${turnNumber}.`);
|
|
1087
|
+
await flushProgress();
|
|
629
1088
|
toolCalls.push(mapToolCallResult(toolResult, toolStartedAt));
|
|
630
1089
|
}
|
|
631
1090
|
previousToolCalls = toolCalls;
|
|
@@ -635,6 +1094,9 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
635
1094
|
: response.status === "failed"
|
|
636
1095
|
? "failed"
|
|
637
1096
|
: "completed";
|
|
1097
|
+
const effectivePolicy = sessionSnapshot.output_policy ?? DEFAULT_TURN_OUTPUT_POLICY;
|
|
1098
|
+
const { outcome: turnOutcome, reason: outcomeReason } = classifyTurnOutcome(response.status, effectivePolicy, toolCalls.length);
|
|
1099
|
+
// Use temp file paths for the registry turn record; the ACEPACK artifact write is a sidecar.
|
|
638
1100
|
await mutateRegistry((registry) => {
|
|
639
1101
|
const session = findSession(registry, sessionId);
|
|
640
1102
|
if (!session)
|
|
@@ -647,27 +1109,62 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
647
1109
|
status: turnStatus,
|
|
648
1110
|
response_status: response.status,
|
|
649
1111
|
summary: normalizedSummary,
|
|
650
|
-
prompt_path:
|
|
651
|
-
request_path:
|
|
652
|
-
response_path:
|
|
1112
|
+
prompt_path: paths.tempPromptPath,
|
|
1113
|
+
request_path: paths.tempRequestPath,
|
|
1114
|
+
response_path: paths.tempResponsePath,
|
|
653
1115
|
exit_code: shellResult.exit_code,
|
|
654
1116
|
stdout: clipOutput(shellResult.stdout),
|
|
655
1117
|
stderr: clipOutput(shellResult.stderr),
|
|
656
1118
|
tool_calls: toolCalls,
|
|
1119
|
+
turn_outcome: turnOutcome,
|
|
1120
|
+
outcome_reason: outcomeReason,
|
|
657
1121
|
};
|
|
658
1122
|
session.turns.push(turnRecord);
|
|
659
1123
|
session.turn_count = session.turns.length;
|
|
660
1124
|
session.current_task = response.next_task?.trim() || currentTask;
|
|
661
1125
|
session.updated_at = turnEndedAt;
|
|
662
1126
|
});
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
1127
|
+
// Sidecar: persist turn artifacts to ACEPACK, emit outcome transition and events.
|
|
1128
|
+
const capturedTurnStatus = turnStatus;
|
|
1129
|
+
const capturedTurnOutcome = turnOutcome;
|
|
1130
|
+
const capturedOutcomeReason = outcomeReason;
|
|
1131
|
+
const capturedToolCalls = toolCalls;
|
|
1132
|
+
const capturedNormalizedSummary = normalizedSummary;
|
|
1133
|
+
const capturedArtifactPaths = { promptPath: paths.tempPromptPath, requestPath: paths.tempRequestPath, responsePath: paths.tempResponsePath };
|
|
1134
|
+
const capturedSnapshot2 = sessionSnapshot;
|
|
1135
|
+
scheduleRuntimeSidecar(sessionId, `turn-artifacts-${turnNumber}`, async () => {
|
|
1136
|
+
const responseRaw = existsSync(paths.tempResponsePath)
|
|
1137
|
+
? readFileSync(paths.tempResponsePath, "utf-8")
|
|
1138
|
+
: JSON.stringify({ status: capturedTurnStatus, summary: capturedNormalizedSummary, tool_calls: [] }, null, 2);
|
|
1139
|
+
await persistTurnArtifacts(paths, prompt, requestPayload, responseRaw).catch(() => undefined);
|
|
1140
|
+
});
|
|
1141
|
+
scheduleRuntimeSidecar(sessionId, `turn-outcome-transition-${turnNumber}`, async () => {
|
|
1142
|
+
if (!storeExistsSync(workspaceRoot()))
|
|
1143
|
+
return;
|
|
1144
|
+
await withLocalModelRuntimeRepository(workspaceRoot(), async (repo) => {
|
|
1145
|
+
await repo.appendTransitionRecord({
|
|
1146
|
+
session_id: sessionId,
|
|
1147
|
+
subject_kind: "turn",
|
|
1148
|
+
subject_id: `${sessionId}/${capturedTurnNumber}`,
|
|
1149
|
+
from: "in_progress",
|
|
1150
|
+
to: capturedTurnOutcome,
|
|
1151
|
+
reason: capturedOutcomeReason,
|
|
1152
|
+
evidence_refs: capturedToolCalls.slice(0, 3).map((tc) => tc.tool_name),
|
|
1153
|
+
});
|
|
1154
|
+
}).catch(() => undefined);
|
|
1155
|
+
await emitExecutorEvent(capturedTurnStatus === "failed" ? "RUNTIME_EXECUTOR_TURN_FAILED" : "RUNTIME_EXECUTOR_TURN_COMPLETED", capturedTurnStatus === "blocked" ? "blocked" : capturedTurnStatus === "failed" ? "fail" : "done", capturedNormalizedSummary, {
|
|
1156
|
+
session_id: sessionId,
|
|
1157
|
+
turn_number: capturedTurnNumber,
|
|
1158
|
+
response_status: capturedTurnStatus,
|
|
1159
|
+
tool_call_count: capturedToolCalls.length,
|
|
1160
|
+
...vericifyPayloadForSession(capturedSnapshot2),
|
|
1161
|
+
});
|
|
1162
|
+
const vericifyKind = capturedTurnOutcome === "escalation_blocker"
|
|
1163
|
+
? "blocker"
|
|
1164
|
+
: capturedTurnOutcome === "meaningful_completion"
|
|
1165
|
+
? "completion"
|
|
1166
|
+
: "progress";
|
|
1167
|
+
await emitVericifyProcessPostForSession(capturedSnapshot2, vericifyKind, `[${capturedTurnOutcome}] turn ${capturedTurnNumber}: ${capturedNormalizedSummary}`, capturedToolCalls.slice(0, 5).map((tc) => tc.tool_name), [String(capturedTurnNumber)]);
|
|
671
1168
|
});
|
|
672
1169
|
if (response.status === "continue") {
|
|
673
1170
|
currentTask = response.next_task?.trim() || currentTask;
|
|
@@ -675,69 +1172,82 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
675
1172
|
}
|
|
676
1173
|
if (response.status === "blocked") {
|
|
677
1174
|
await finalizeSession(sessionId, "blocked", normalizedSummary, normalizedSummary);
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
1175
|
+
scheduleRuntimeSidecar(sessionId, "session-blocked-event", async () => {
|
|
1176
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_BLOCKED", "blocked", capturedNormalizedSummary, {
|
|
1177
|
+
session_id: sessionId,
|
|
1178
|
+
turn_number: capturedTurnNumber,
|
|
1179
|
+
...vericifyPayloadForSession(capturedSnapshot2),
|
|
1180
|
+
});
|
|
1181
|
+
await emitVericifyProcessPostForSession(capturedSnapshot2, "blocker", capturedNormalizedSummary, [
|
|
1182
|
+
"runtime-executor",
|
|
1183
|
+
]);
|
|
682
1184
|
});
|
|
683
|
-
await emitVericifyProcessPostForSession(sessionSnapshot, "blocker", normalizedSummary, [
|
|
684
|
-
"runtime-executor",
|
|
685
|
-
]);
|
|
686
1185
|
return;
|
|
687
1186
|
}
|
|
688
1187
|
if (response.status === "failed") {
|
|
689
1188
|
const failed = await finalizeSession(sessionId, "failed", normalizedSummary, normalizedSummary);
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
1189
|
+
scheduleRuntimeSidecar(sessionId, "session-failed-event", async () => {
|
|
1190
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_FAILED", "fail", capturedNormalizedSummary, {
|
|
1191
|
+
session_id: sessionId,
|
|
1192
|
+
turn_number: capturedTurnNumber,
|
|
1193
|
+
...(failed ? vericifyPayloadForSession(failed) : vericifyPayloadForSession(capturedSnapshot2)),
|
|
1194
|
+
});
|
|
1195
|
+
await emitVericifyProcessPostForSession(failed ?? capturedSnapshot2, "blocker", capturedNormalizedSummary, ["runtime-executor"]);
|
|
694
1196
|
});
|
|
695
|
-
await emitVericifyProcessPostForSession(failed ?? sessionSnapshot, "blocker", normalizedSummary, ["runtime-executor"]);
|
|
696
1197
|
return;
|
|
697
1198
|
}
|
|
698
1199
|
const stopCheck = runAutonomyChecks("stop", autonomyPolicy, context);
|
|
699
1200
|
if (!stopCheck.ok) {
|
|
700
1201
|
const summary = `Autonomy stop checks failed: ${stopCheck.summary}`;
|
|
701
1202
|
const blocked = await finalizeSession(sessionId, "blocked", summary, summary);
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
1203
|
+
scheduleRuntimeSidecar(sessionId, "session-stop-check-blocked-event", async () => {
|
|
1204
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_BLOCKED", "blocked", summary, {
|
|
1205
|
+
session_id: sessionId,
|
|
1206
|
+
turn_number: capturedTurnNumber,
|
|
1207
|
+
executed_checks: stopCheck.executed_checks,
|
|
1208
|
+
skipped_checks: stopCheck.skipped_checks,
|
|
1209
|
+
...(blocked ? vericifyPayloadForSession(blocked) : vericifyPayloadForSession(capturedSnapshot2)),
|
|
1210
|
+
});
|
|
1211
|
+
await emitVericifyProcessPostForSession(blocked ?? capturedSnapshot2, "blocker", summary, ["runtime-executor"]);
|
|
708
1212
|
});
|
|
709
|
-
await emitVericifyProcessPostForSession(blocked ?? sessionSnapshot, "blocker", summary, ["runtime-executor"]);
|
|
710
1213
|
return;
|
|
711
1214
|
}
|
|
1215
|
+
// Critical: finalize session status immediately so waitForUnattendedSession resolves correctly.
|
|
712
1216
|
const completed = await finalizeSession(sessionId, "completed", normalizedSummary);
|
|
713
|
-
|
|
714
|
-
await
|
|
715
|
-
|
|
716
|
-
category: "major_update",
|
|
717
|
-
message: normalizedSummary,
|
|
718
|
-
artifacts: [
|
|
719
|
-
RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH,
|
|
720
|
-
artifactPaths.promptPath,
|
|
721
|
-
artifactPaths.requestPath,
|
|
722
|
-
artifactPaths.responsePath,
|
|
723
|
-
],
|
|
724
|
-
metadata: {
|
|
725
|
-
session_id: sessionId,
|
|
726
|
-
turn_number: turnNumber,
|
|
727
|
-
workspace_session_id: completed.workspace_session_id,
|
|
728
|
-
...vericifyPayloadForSession(completed),
|
|
729
|
-
},
|
|
730
|
-
}).catch(() => undefined);
|
|
731
|
-
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_COMPLETED", "done", normalizedSummary, {
|
|
1217
|
+
scheduleRuntimeSidecar(sessionId, "session-completed-events", async () => {
|
|
1218
|
+
await delaySessionCompletedSidecarIfRequested(sessionId);
|
|
1219
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_COMPLETED", "done", capturedNormalizedSummary, {
|
|
732
1220
|
session_id: sessionId,
|
|
733
|
-
turn_number:
|
|
734
|
-
workspace_session_id:
|
|
735
|
-
...vericifyPayloadForSession(
|
|
1221
|
+
turn_number: capturedTurnNumber,
|
|
1222
|
+
workspace_session_id: capturedSnapshot2.workspace_session_id,
|
|
1223
|
+
...vericifyPayloadForSession(capturedSnapshot2),
|
|
736
1224
|
});
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
1225
|
+
if (completed) {
|
|
1226
|
+
await appendRunLedgerEntrySafe({
|
|
1227
|
+
tool: "runtime-executor",
|
|
1228
|
+
category: "major_update",
|
|
1229
|
+
message: capturedNormalizedSummary,
|
|
1230
|
+
artifacts: [
|
|
1231
|
+
RUNTIME_EXECUTOR_SESSION_REGISTRY_REL_PATH,
|
|
1232
|
+
capturedArtifactPaths.promptPath,
|
|
1233
|
+
capturedArtifactPaths.requestPath,
|
|
1234
|
+
capturedArtifactPaths.responsePath,
|
|
1235
|
+
],
|
|
1236
|
+
metadata: {
|
|
1237
|
+
session_id: sessionId,
|
|
1238
|
+
turn_number: capturedTurnNumber,
|
|
1239
|
+
workspace_session_id: completed.workspace_session_id,
|
|
1240
|
+
...vericifyPayloadForSession(completed),
|
|
1241
|
+
},
|
|
1242
|
+
}).catch(() => undefined);
|
|
1243
|
+
await emitVericifyProcessPostForSession(completed, "completion", capturedNormalizedSummary, [
|
|
1244
|
+
"runtime-executor",
|
|
1245
|
+
]);
|
|
1246
|
+
if (isVericifyBridgeEnabled()) {
|
|
1247
|
+
refreshVericifyBridgeSnapshotSafe();
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
});
|
|
741
1251
|
return;
|
|
742
1252
|
}
|
|
743
1253
|
const exhausted = `Unattended session ${sessionId} exceeded max_turns=${registryStart.max_turns}`;
|
|
@@ -790,10 +1300,15 @@ async function runSessionLoop(sessionId, context, autoCleanup) {
|
|
|
790
1300
|
}
|
|
791
1301
|
finally {
|
|
792
1302
|
if (workspaceSessionId && autoCleanup) {
|
|
793
|
-
|
|
1303
|
+
setSessionPhase(sessionId, "cleanup_capturing");
|
|
1304
|
+
await runCleanup(sessionId, workspaceSessionId, shouldRunAfterHook, runtimeProfileSnapshot.path).catch(() => undefined);
|
|
794
1305
|
}
|
|
1306
|
+
// Release sidecars to run in background — do not await them.
|
|
1307
|
+
// Critical-path registry and workspace writes are already complete.
|
|
1308
|
+
releaseSidecarsAndForget(sessionId);
|
|
795
1309
|
const active = activeSessions.get(sessionId);
|
|
796
1310
|
if (active) {
|
|
1311
|
+
active.phase = "completed";
|
|
797
1312
|
active.child = undefined;
|
|
798
1313
|
activeSessions.delete(sessionId);
|
|
799
1314
|
}
|
|
@@ -821,18 +1336,29 @@ export function getUnattendedSession(sessionId) {
|
|
|
821
1336
|
export function getRuntimeExecutorSessionRegistryPath() {
|
|
822
1337
|
return registryPath();
|
|
823
1338
|
}
|
|
824
|
-
export function startUnattendedSession(input) {
|
|
1339
|
+
export async function startUnattendedSession(input) {
|
|
825
1340
|
const registryPathValue = registryPath();
|
|
826
|
-
|
|
1341
|
+
let runtimeProfileSnapshot;
|
|
1342
|
+
try {
|
|
1343
|
+
runtimeProfileSnapshot = loadCurrentRuntimeProfile();
|
|
1344
|
+
}
|
|
1345
|
+
catch (error) {
|
|
1346
|
+
return Promise.resolve({
|
|
1347
|
+
ok: false,
|
|
1348
|
+
registry_path: registryPathValue,
|
|
1349
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1350
|
+
});
|
|
1351
|
+
}
|
|
827
1352
|
const storeBackedWorkspace = storeExistsSync(workspaceRoot());
|
|
828
|
-
if (
|
|
1353
|
+
if (runtimeProfileSnapshot.profile.runtime.mode !== "unattended") {
|
|
829
1354
|
return Promise.resolve({
|
|
830
1355
|
ok: false,
|
|
831
1356
|
registry_path: registryPathValue,
|
|
832
1357
|
error: "ACE_WORKFLOW.md runtime.mode must be \"unattended\" before starting an unattended session.",
|
|
833
1358
|
});
|
|
834
1359
|
}
|
|
835
|
-
if (!
|
|
1360
|
+
if (!runtimeProfileSnapshot.profile.executor.command ||
|
|
1361
|
+
runtimeProfileSnapshot.profile.executor.command.trim().length === 0) {
|
|
836
1362
|
return Promise.resolve({
|
|
837
1363
|
ok: false,
|
|
838
1364
|
registry_path: registryPathValue,
|
|
@@ -848,10 +1374,11 @@ export function startUnattendedSession(input) {
|
|
|
848
1374
|
error: `Unattended session id already exists: ${sessionId}`,
|
|
849
1375
|
});
|
|
850
1376
|
}
|
|
851
|
-
const workspace =
|
|
1377
|
+
const workspace = await createWorkspaceSessionAsync({
|
|
852
1378
|
source: "executor",
|
|
853
1379
|
workspace_name: input.workspace_name,
|
|
854
1380
|
workspace_path: input.workspace_path,
|
|
1381
|
+
runtime_profile_path: runtimeProfileSnapshot.path,
|
|
855
1382
|
objective_id: input.objective_id,
|
|
856
1383
|
tracker_item_id: input.tracker_item_id,
|
|
857
1384
|
});
|
|
@@ -862,59 +1389,80 @@ export function startUnattendedSession(input) {
|
|
|
862
1389
|
error: workspace.error ?? "Failed to create managed workspace session.",
|
|
863
1390
|
});
|
|
864
1391
|
}
|
|
865
|
-
const
|
|
866
|
-
const
|
|
1392
|
+
const workspaceSession = workspace.session;
|
|
1393
|
+
const maxTurns = Math.max(1, input.max_turns ?? runtimeProfileSnapshot.profile.executor.max_turns ?? 6);
|
|
1394
|
+
const turnTimeout = Math.max(1_000, input.turn_timeout_ms ?? runtimeProfileSnapshot.profile.executor.turn_timeout_ms ?? 300_000);
|
|
867
1395
|
const now = new Date().toISOString();
|
|
868
1396
|
const sessionRecord = {
|
|
869
1397
|
session_id: sessionId,
|
|
870
1398
|
status: "starting",
|
|
871
1399
|
task: input.task,
|
|
872
1400
|
current_task: input.task,
|
|
873
|
-
runtime_profile_path:
|
|
1401
|
+
runtime_profile_path: runtimeProfileSnapshot.path,
|
|
874
1402
|
workspace_session_id: workspace.session.session_id,
|
|
875
1403
|
workspace_path: workspace.session.workspace_path,
|
|
876
1404
|
objective_id: input.objective_id,
|
|
877
1405
|
tracker_item_id: input.tracker_item_id,
|
|
878
|
-
command:
|
|
1406
|
+
command: runtimeProfileSnapshot.profile.executor.command,
|
|
879
1407
|
max_turns: maxTurns,
|
|
880
1408
|
turn_timeout_ms: turnTimeout,
|
|
881
1409
|
turn_count: 0,
|
|
882
1410
|
created_at: now,
|
|
883
1411
|
updated_at: now,
|
|
884
1412
|
workspace_cleanup_status: input.auto_cleanup === false ? "pending" : "pending",
|
|
1413
|
+
output_policy: input.emit_to || input.silent_unless_blocked || input.require_approval_before_emit
|
|
1414
|
+
? {
|
|
1415
|
+
emit_to: input.emit_to ?? DEFAULT_TURN_OUTPUT_POLICY.emit_to,
|
|
1416
|
+
silent_unless_blocked: input.silent_unless_blocked ?? false,
|
|
1417
|
+
require_approval_before_emit: input.require_approval_before_emit ?? false,
|
|
1418
|
+
}
|
|
1419
|
+
: undefined,
|
|
885
1420
|
turns: [],
|
|
886
1421
|
};
|
|
887
|
-
|
|
1422
|
+
await mutateRegistry((registry) => {
|
|
888
1423
|
registry.sessions.push(sessionRecord);
|
|
889
|
-
})
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
1424
|
+
});
|
|
1425
|
+
// Init sidecar gate BEFORE starting the loop so sidecars can be scheduled immediately.
|
|
1426
|
+
// Sidecars are blocked behind a release signal and only run after the critical path completes.
|
|
1427
|
+
initSessionSidecars(sessionId);
|
|
1428
|
+
const active = {
|
|
1429
|
+
stop_requested: false,
|
|
1430
|
+
phase: "starting",
|
|
1431
|
+
phase_started_at: Date.now(),
|
|
1432
|
+
};
|
|
1433
|
+
activeSessions.set(sessionId, active);
|
|
1434
|
+
active.completion = runSessionLoop(sessionId, input.context, storeBackedWorkspace || input.auto_cleanup !== false, runtimeProfileSnapshot).finally(() => {
|
|
1435
|
+
activeSessions.delete(sessionId);
|
|
1436
|
+
});
|
|
1437
|
+
// Schedule session-started events as sidecars; they run after the critical lane completes.
|
|
1438
|
+
const capturedRecord = sessionRecord;
|
|
1439
|
+
const capturedSession = workspace.session;
|
|
1440
|
+
scheduleRuntimeSidecar(sessionId, "session-started-event", async () => {
|
|
1441
|
+
await emitExecutorEvent("RUNTIME_EXECUTOR_SESSION_STARTED", "started", `Unattended session ${sessionId} created.`, {
|
|
898
1442
|
session_id: sessionId,
|
|
899
|
-
workspace_session_id:
|
|
900
|
-
workspace_path:
|
|
1443
|
+
workspace_session_id: capturedSession?.session_id,
|
|
1444
|
+
workspace_path: capturedSession?.workspace_path,
|
|
901
1445
|
max_turns: maxTurns,
|
|
902
|
-
...vericifyPayloadForSession(
|
|
1446
|
+
...vericifyPayloadForSession(capturedRecord),
|
|
903
1447
|
});
|
|
904
|
-
|
|
1448
|
+
await emitVericifyProcessPostForSession(capturedRecord, "intent", `Unattended session ${sessionId} created for task: ${input.task}`, ["runtime-executor"]);
|
|
905
1449
|
if (isVericifyBridgeEnabled()) {
|
|
906
1450
|
refreshVericifyBridgeSnapshotSafe();
|
|
907
1451
|
}
|
|
908
|
-
return {
|
|
909
|
-
ok: true,
|
|
910
|
-
registry_path: registryPathValue,
|
|
911
|
-
session: sessionRecord,
|
|
912
|
-
workspace: workspace.session,
|
|
913
|
-
};
|
|
914
1452
|
});
|
|
1453
|
+
return {
|
|
1454
|
+
ok: true,
|
|
1455
|
+
registry_path: registryPathValue,
|
|
1456
|
+
session: sessionRecord,
|
|
1457
|
+
workspace: workspace.session,
|
|
1458
|
+
};
|
|
915
1459
|
}
|
|
916
|
-
export async function waitForUnattendedSession(sessionId,
|
|
1460
|
+
export async function waitForUnattendedSession(sessionId, timeoutMsOrOptions = 30_000, options) {
|
|
917
1461
|
const registryPathValue = registryPath();
|
|
1462
|
+
const timeoutMs = typeof timeoutMsOrOptions === "number" ? timeoutMsOrOptions : 30_000;
|
|
1463
|
+
const flushSidecars = typeof timeoutMsOrOptions === "number"
|
|
1464
|
+
? options?.flush_sidecars !== false
|
|
1465
|
+
: timeoutMsOrOptions.flush_sidecars !== false;
|
|
918
1466
|
const active = activeSessions.get(sessionId);
|
|
919
1467
|
if (!active?.completion) {
|
|
920
1468
|
const session = getUnattendedSession(sessionId);
|
|
@@ -926,14 +1474,24 @@ export async function waitForUnattendedSession(sessionId, timeoutMs = 30_000) {
|
|
|
926
1474
|
error: `Unknown unattended session: ${sessionId}`,
|
|
927
1475
|
};
|
|
928
1476
|
}
|
|
1477
|
+
if (terminalStatus(session.status)) {
|
|
1478
|
+
if (flushSidecars) {
|
|
1479
|
+
await waitForSessionSidecars(sessionId);
|
|
1480
|
+
}
|
|
1481
|
+
const freshSession = getUnattendedSession(sessionId);
|
|
1482
|
+
return {
|
|
1483
|
+
ok: terminalStatus(freshSession?.status ?? session.status),
|
|
1484
|
+
timed_out: false,
|
|
1485
|
+
registry_path: registryPathValue,
|
|
1486
|
+
session: freshSession ?? session,
|
|
1487
|
+
};
|
|
1488
|
+
}
|
|
929
1489
|
return {
|
|
930
|
-
ok:
|
|
1490
|
+
ok: false,
|
|
931
1491
|
timed_out: false,
|
|
932
1492
|
registry_path: registryPathValue,
|
|
933
1493
|
session,
|
|
934
|
-
error:
|
|
935
|
-
? undefined
|
|
936
|
-
: `Session ${sessionId} is not active but has not reached a terminal state.`,
|
|
1494
|
+
error: `Session ${sessionId} is not active but has not reached a terminal state.`,
|
|
937
1495
|
};
|
|
938
1496
|
}
|
|
939
1497
|
const timeoutPromise = new Promise((resolve) => {
|
|
@@ -943,19 +1501,26 @@ export async function waitForUnattendedSession(sessionId, timeoutMs = 30_000) {
|
|
|
943
1501
|
const outcome = await Promise.race([completion, timeoutPromise]);
|
|
944
1502
|
const session = getUnattendedSession(sessionId);
|
|
945
1503
|
if (outcome === "timeout") {
|
|
1504
|
+
const phaseInfo = active.phase
|
|
1505
|
+
? ` (phase=${active.phase}${active.last_checkpoint ? `, checkpoint=${active.last_checkpoint}` : ""})`
|
|
1506
|
+
: "";
|
|
946
1507
|
return {
|
|
947
1508
|
ok: false,
|
|
948
1509
|
timed_out: true,
|
|
949
1510
|
registry_path: registryPathValue,
|
|
950
1511
|
session,
|
|
951
|
-
error: `Timed out waiting for unattended session ${sessionId}`,
|
|
1512
|
+
error: `Timed out waiting for unattended session ${sessionId}${phaseInfo}`,
|
|
952
1513
|
};
|
|
953
1514
|
}
|
|
1515
|
+
if (flushSidecars) {
|
|
1516
|
+
await waitForSessionSidecars(sessionId);
|
|
1517
|
+
}
|
|
1518
|
+
const completedSession = getUnattendedSession(sessionId);
|
|
954
1519
|
return {
|
|
955
|
-
ok: !!
|
|
1520
|
+
ok: !!completedSession && terminalStatus(completedSession.status),
|
|
956
1521
|
timed_out: false,
|
|
957
1522
|
registry_path: registryPathValue,
|
|
958
|
-
session,
|
|
1523
|
+
session: completedSession ?? session,
|
|
959
1524
|
};
|
|
960
1525
|
}
|
|
961
1526
|
export async function stopUnattendedSession(sessionId) {
|