@kylecheng3146/agent-ops 0.9.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/packages/cli/src/args.js +58 -14
- package/dist/packages/cli/src/bin.js +10 -3
- package/dist/packages/cli/src/cli.js +30 -6
- package/dist/packages/cli/src/commands/advance.js +2 -1
- package/dist/packages/cli/src/commands/escapes.js +109 -0
- package/dist/packages/cli/src/commands/parallel.js +1 -1
- package/dist/packages/cli/src/commands/review-show.js +1 -1
- package/dist/packages/cli/src/commands/review.js +63 -11
- package/dist/packages/cli/src/commands/run.js +10 -3
- package/dist/packages/cli/src/commands/task.js +41 -3
- package/dist/packages/cli/src/commands/update.js +16 -4
- package/dist/packages/cli/src/commands/verify.js +4 -1
- package/dist/packages/cli/src/context.js +22 -4
- package/dist/packages/cli/src/debug-intake.js +1 -1
- package/dist/packages/cli/src/office-entry.js +8 -4
- package/dist/packages/cli/src/owned-run-step.js +4 -4
- package/dist/packages/cli/src/parallel-deps.js +0 -2
- package/dist/packages/cli/src/run-deps.js +12 -9
- package/dist/packages/cli/src/run-entry.js +76 -17
- package/dist/packages/cli/src/run-setup-entry.js +2 -2
- package/dist/runtime/src/escape/link.js +317 -0
- package/dist/runtime/src/escape/origin.js +20 -0
- package/dist/runtime/src/escape/rules.js +252 -0
- package/dist/runtime/src/fs/temporary.js +10 -0
- package/dist/runtime/src/install/harness.js +7 -4
- package/dist/runtime/src/logging/local-log.js +44 -1
- package/dist/runtime/src/office/scene.js +1 -1
- package/dist/runtime/src/office/server.js +43 -16
- package/dist/runtime/src/office/snapshot.js +1 -1
- package/dist/runtime/src/parallel/finish.js +139 -65
- package/dist/runtime/src/parallel/manage.js +51 -1
- package/dist/runtime/src/parallel/receipt.js +3 -1
- package/dist/runtime/src/parallel/service.js +5 -1
- package/dist/runtime/src/review/execute.js +20 -12
- package/dist/runtime/src/review/packet.js +20 -7
- package/dist/runtime/src/review/probe.js +3 -2
- package/dist/runtime/src/review/runner.js +26 -3
- package/dist/runtime/src/review/slots.js +1 -1
- package/dist/runtime/src/run/controls.js +99 -1
- package/dist/runtime/src/run/hosts/codex.js +28 -1
- package/dist/runtime/src/run/hosts/protocol.js +5 -2
- package/dist/runtime/src/run/integration.js +35 -13
- package/dist/runtime/src/run/policy.js +65 -10
- package/dist/runtime/src/run/scheduler.js +10 -5
- package/dist/runtime/src/run/service.js +68 -21
- package/dist/runtime/src/run/supervisor.js +11 -2
- package/dist/runtime/src/run/transport.js +1 -0
- package/dist/runtime/src/schema/validate.js +6 -1
- package/dist/runtime/src/security/permissions.js +85 -0
- package/dist/runtime/src/surface/discovery.js +79 -2
- package/dist/runtime/src/surface/schema.js +43 -2
- package/dist/runtime/src/task/completion.js +20 -0
- package/dist/runtime/src/task/contract.js +3 -1
- package/dist/runtime/src/task/debug-state.js +9 -3
- package/dist/runtime/src/task/debug-user-event.js +82 -17
- package/dist/runtime/src/task/debug.js +59 -16
- package/dist/runtime/src/task/service.js +58 -6
- package/dist/runtime/src/verify/acceptance-replay.js +24 -11
- package/docs/en/guides/configuration.md +24 -5
- package/docs/en/guides/task-debug.md +26 -2
- package/docs/en/spec/acceptance-and-evidence.md +104 -2
- package/docs/en/spec/harness-adapters.md +26 -1
- package/docs/en/spec/loop-engineering.md +34 -6
- package/docs/en/spec/review.md +6 -0
- package/docs/zh-TW/guides/configuration.md +20 -4
- package/docs/zh-TW/guides/task-debug.md +19 -2
- package/docs/zh-TW/spec/acceptance-and-evidence.md +84 -4
- package/docs/zh-TW/spec/harness-adapters.md +21 -1
- package/docs/zh-TW/spec/loop-engineering.md +9 -3
- package/docs/zh-TW/spec/review.md +5 -0
- package/package.json +1 -1
- package/schemas/task.schema.json +8 -1
|
@@ -1,10 +1,108 @@
|
|
|
1
1
|
import { sha256 } from "../fs/hash.js";
|
|
2
2
|
import { AgentOpsError } from "../fs/paths.js";
|
|
3
|
-
import { canonicalJson } from "../config/hash.js";
|
|
3
|
+
import { canonicalJson, calculateConfigHash } from "../config/hash.js";
|
|
4
|
+
import { readRunPolicy } from "./policy.js";
|
|
4
5
|
import { join } from "node:path";
|
|
5
6
|
import { readPrivateFile, writePrivateFile } from "../security/permissions.js";
|
|
6
7
|
import { redactSecrets } from "../security/redact.js";
|
|
7
8
|
const MAX_REASON_LENGTH = 4_096;
|
|
9
|
+
const RETRY_APPROVAL = "retry-approval:";
|
|
10
|
+
const AUTH_ID = /^auth-[a-f0-9]{64}$/u;
|
|
11
|
+
const DENIAL_DIAGNOSTIC = "Native authorization could not be accepted; no alternate permission path is allowed.";
|
|
12
|
+
/** A manual resume covers initial recovery once, never a subsequent supervisor restart. */
|
|
13
|
+
export function nativeResumeTime(state, now = Date.now()) {
|
|
14
|
+
const control = state.controls.at(-1);
|
|
15
|
+
if (control?.action !== "resume" || control.actor !== "user" || control.appliedAt !== null)
|
|
16
|
+
return undefined;
|
|
17
|
+
const time = Date.parse(control.requestedAt);
|
|
18
|
+
if (!Number.isFinite(time) || !Number.isFinite(now) || time > now ||
|
|
19
|
+
state.events.some(e => e.code === "RUN_SUPERVISOR_START" && (!Number.isFinite(Date.parse(e.at)) || Date.parse(e.at) >= time)))
|
|
20
|
+
return undefined;
|
|
21
|
+
return control.requestedAt;
|
|
22
|
+
}
|
|
23
|
+
/** An explicit retry acknowledges a failed reviewer, never a command permission. */
|
|
24
|
+
export async function approvalRetryEvents(state, requested) {
|
|
25
|
+
const fail = () => { throw error("RUN_NOT_RESUMABLE", "Approval retry requires the original, intact Codex usage-limit review failure."); };
|
|
26
|
+
if (!Array.isArray(requested) || requested.some(id => typeof id !== "string" || !AUTH_ID.test(id)) || new Set(requested).size !== requested.length)
|
|
27
|
+
fail();
|
|
28
|
+
const prior = new Set();
|
|
29
|
+
for (const control of state.controls) {
|
|
30
|
+
if (control.action !== "resume" || control.actor !== "user" || !control.reason?.startsWith(RETRY_APPROVAL))
|
|
31
|
+
continue;
|
|
32
|
+
let ids;
|
|
33
|
+
try {
|
|
34
|
+
ids = JSON.parse(control.reason.slice(RETRY_APPROVAL.length));
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return fail();
|
|
38
|
+
}
|
|
39
|
+
if (!Array.isArray(ids) || ids.some(id => typeof id !== "string" || !AUTH_ID.test(id)))
|
|
40
|
+
fail();
|
|
41
|
+
for (const id of ids)
|
|
42
|
+
prior.add(id);
|
|
43
|
+
}
|
|
44
|
+
const ignored = new Set();
|
|
45
|
+
for (const id of new Set([...prior, ...requested])) {
|
|
46
|
+
const index = state.events.findIndex(e => e.id === id);
|
|
47
|
+
const event = state.events[index];
|
|
48
|
+
if (event?.type !== "control" || event.code !== "RUN_COMMAND_DENIED")
|
|
49
|
+
fail();
|
|
50
|
+
const path = join(state.commonDir, "agent-ops/runs", state.runId, "authorizations", id + ".json");
|
|
51
|
+
let ref;
|
|
52
|
+
let authorization;
|
|
53
|
+
try {
|
|
54
|
+
ref = JSON.parse(event.detail);
|
|
55
|
+
const source = await readPrivateFile(path, state.commonDir);
|
|
56
|
+
if (ref.artifact !== path || ref.result !== "denied" || ref.source !== "codex-auto-review" || source === null ||
|
|
57
|
+
Buffer.byteLength(source) > 128 * 1024 || sha256(source) !== ref.digest)
|
|
58
|
+
fail();
|
|
59
|
+
authorization = JSON.parse(source);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return fail();
|
|
63
|
+
}
|
|
64
|
+
const a = authorization;
|
|
65
|
+
if (a === null || typeof a !== "object")
|
|
66
|
+
fail();
|
|
67
|
+
const worker = state.workers.find(w => w.workerId === event.workerId);
|
|
68
|
+
if (a.runId !== state.runId || a.workerId !== event.workerId || worker === undefined || event.taskId !== worker.taskId ||
|
|
69
|
+
!Number.isSafeInteger(a.generation) || a.generation < 1 || a.generation > worker.generation ||
|
|
70
|
+
(!prior.has(id) && (a.generation !== worker.generation || a.nativeSessionId !== worker.nativeSessionId)) ||
|
|
71
|
+
a.source !== "codex-auto-review" || a.result !== "denied" || a.operation !== "command" ||
|
|
72
|
+
typeof a.reason !== "string" || !/^Automatic approval review failed: You[’']ve hit your usage limit(?:[.!]|$)/u.test(a.reason) ||
|
|
73
|
+
typeof a.command !== "string" || typeof a.resource !== "string" ||
|
|
74
|
+
!/^[a-f0-9]{64}$/u.test(a.commandDigest) || !/^[a-f0-9]{64}$/u.test(a.resourceDigest) ||
|
|
75
|
+
typeof a.authorizationId !== "string" || a.authorizationId.length === 0 || a.authorizationId.length > 256 ||
|
|
76
|
+
!(a.nativeSessionId === null || typeof a.nativeSessionId === "string") ||
|
|
77
|
+
id !== "auth-" + sha256(canonicalJson({ runId: state.runId, workerId: a.workerId, generation: a.generation,
|
|
78
|
+
nativeSessionId: a.nativeSessionId, source: a.source, id: a.authorizationId })))
|
|
79
|
+
fail();
|
|
80
|
+
// Command/resource strings were redacted when saved; their original digests cannot be recomputed.
|
|
81
|
+
let binding = state.policyBinding;
|
|
82
|
+
let matched = false;
|
|
83
|
+
for (let depth = 0; binding !== undefined && depth < 128; depth++) {
|
|
84
|
+
const policy = await readRunPolicy({ ...state, policyBinding: binding }, binding.runtimeHash, Date.now(), true);
|
|
85
|
+
if (binding.artifactDigest === a.policyArtifactDigest) {
|
|
86
|
+
matched = binding.configHash === a.policyConfigHash && binding.runtimeHash === a.runtimeHash && binding.expiresAt === a.expiresAt;
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
if (policy?.previousDigest == null)
|
|
90
|
+
break;
|
|
91
|
+
const source = await readPrivateFile(join(state.commonDir, "agent-ops/runs", state.runId, "policies", policy.previousDigest + ".json"), state.commonDir);
|
|
92
|
+
if (source === null || sha256(source) !== policy.previousDigest)
|
|
93
|
+
fail();
|
|
94
|
+
const previous = JSON.parse(source);
|
|
95
|
+
binding = { artifactDigest: policy.previousDigest, configHash: calculateConfigHash(previous.config), runtimeHash: previous.runtimeHash, expiresAt: previous.expiresAt };
|
|
96
|
+
}
|
|
97
|
+
if (!matched)
|
|
98
|
+
fail();
|
|
99
|
+
ignored.add(id);
|
|
100
|
+
const next = state.events.slice(index + 1).find(e => e.workerId === event.workerId && e.code === "RUN_COMMAND_DENIED");
|
|
101
|
+
if (next?.type === "diagnostic" && next.taskId === null && next.detail === DENIAL_DIAGNOSTIC)
|
|
102
|
+
ignored.add(next.id);
|
|
103
|
+
}
|
|
104
|
+
return ignored;
|
|
105
|
+
}
|
|
8
106
|
function error(code, message) {
|
|
9
107
|
return new AgentOpsError(code, message);
|
|
10
108
|
}
|
|
@@ -41,6 +41,7 @@ export class CodexGoalHost {
|
|
|
41
41
|
kind = "codex";
|
|
42
42
|
sessions = new WeakMap();
|
|
43
43
|
turns = new WeakMap();
|
|
44
|
+
activations = new WeakMap();
|
|
44
45
|
now;
|
|
45
46
|
constructor(options = {}) {
|
|
46
47
|
this.options = options;
|
|
@@ -67,12 +68,14 @@ export class CodexGoalHost {
|
|
|
67
68
|
throw new AgentOpsError("NATIVE_GOAL_INVALID", "Codex goal objective must be 1–4000 characters.");
|
|
68
69
|
}
|
|
69
70
|
const session = this.session(handle);
|
|
71
|
+
this.activations.get(handle).started = true;
|
|
70
72
|
const goal = await session.request("thread/goal/set", {
|
|
71
73
|
threadId: handle.threadId,
|
|
72
74
|
objective,
|
|
73
75
|
status: "active",
|
|
74
76
|
tokenBudget
|
|
75
77
|
});
|
|
78
|
+
this.assertActive(goal);
|
|
76
79
|
const turn = await session.request("turn/start", {
|
|
77
80
|
threadId: handle.threadId,
|
|
78
81
|
input: [{ type: "text", text: objective }],
|
|
@@ -89,12 +92,14 @@ export class CodexGoalHost {
|
|
|
89
92
|
assertContext(handle, update);
|
|
90
93
|
assertProcessRunning(handle.process);
|
|
91
94
|
const session = this.session(handle);
|
|
95
|
+
this.activations.get(handle).started = true;
|
|
92
96
|
const goal = await session.request("thread/goal/set", {
|
|
93
97
|
threadId: handle.threadId,
|
|
94
98
|
objective: update.objective,
|
|
95
99
|
status: "active",
|
|
96
100
|
tokenBudget: update.tokenBudget ?? null
|
|
97
101
|
});
|
|
102
|
+
this.assertActive(goal);
|
|
98
103
|
const turnId = this.turns.get(handle);
|
|
99
104
|
let turn;
|
|
100
105
|
if (turnId === undefined) {
|
|
@@ -176,7 +181,23 @@ export class CodexGoalHost {
|
|
|
176
181
|
async open(input, resume, threadId) {
|
|
177
182
|
const parts = commandParts(this.options, "codex", ["app-server", "--listen", "stdio://"]);
|
|
178
183
|
const process = spawnNative(parts.command, parts.args, input.cwd, input.env, this.options);
|
|
179
|
-
const
|
|
184
|
+
const activation = { started: false };
|
|
185
|
+
let ownedThreadId = threadId;
|
|
186
|
+
const resumeTime = Date.parse(input.manualResumeAt ?? "");
|
|
187
|
+
const channel = new JsonRpcSession(process, input.timeoutMs ?? this.options.requestTimeoutMs ?? 30_000, message => {
|
|
188
|
+
if (message.method !== "thread/goal/updated" && message.method !== "thread/goal/cleared")
|
|
189
|
+
return true;
|
|
190
|
+
const payload = nestedRecord(message, "params");
|
|
191
|
+
const scopedThread = stringField(payload, "threadId");
|
|
192
|
+
if (ownedThreadId !== null && scopedThread !== null && scopedThread !== ownedThreadId)
|
|
193
|
+
return false;
|
|
194
|
+
const goal = nestedRecord(payload, "goal");
|
|
195
|
+
const updatedAt = goal?.updatedAt;
|
|
196
|
+
// Classify at receipt: queued snapshots may be observed after activation starts.
|
|
197
|
+
return !(resume && !activation.started && scopedThread === ownedThreadId && scopedThread !== null &&
|
|
198
|
+
["budgetLimited", "usageLimited"].includes(statusField(goal)) && Number.isFinite(resumeTime) && this.now().getTime() >= resumeTime &&
|
|
199
|
+
typeof updatedAt === "number" && Number.isSafeInteger(updatedAt) && updatedAt >= 0 && updatedAt < Math.floor(resumeTime / 1000));
|
|
200
|
+
});
|
|
180
201
|
try {
|
|
181
202
|
if (process.pid !== undefined)
|
|
182
203
|
await input.registerProcess?.(process.pid);
|
|
@@ -203,6 +224,7 @@ export class CodexGoalHost {
|
|
|
203
224
|
});
|
|
204
225
|
nativeThreadId = resultThreadId(result);
|
|
205
226
|
}
|
|
227
|
+
ownedThreadId = nativeThreadId;
|
|
206
228
|
const handle = {
|
|
207
229
|
runId: input.runId,
|
|
208
230
|
workerId: input.workerId,
|
|
@@ -219,6 +241,7 @@ export class CodexGoalHost {
|
|
|
219
241
|
goalStatus: "unknown"
|
|
220
242
|
};
|
|
221
243
|
this.sessions.set(handle, channel);
|
|
244
|
+
this.activations.set(handle, activation);
|
|
222
245
|
return handle;
|
|
223
246
|
}
|
|
224
247
|
catch (cause) {
|
|
@@ -248,6 +271,10 @@ export class CodexGoalHost {
|
|
|
248
271
|
}
|
|
249
272
|
return session;
|
|
250
273
|
}
|
|
274
|
+
assertActive(result) {
|
|
275
|
+
if (statusField(nestedRecord(result, "goal") ?? result) !== "active")
|
|
276
|
+
throw new AgentOpsError("NATIVE_GOAL_NOT_ACTIVE", "Codex did not activate the goal; no turn was started or steered.");
|
|
277
|
+
}
|
|
251
278
|
mapEvent(handle, message) {
|
|
252
279
|
const method = typeof message.method === "string" ? message.method : "message";
|
|
253
280
|
const payload = isRecord(message.params) ? message.params : message;
|
|
@@ -20,15 +20,17 @@ function parseLine(line, source) {
|
|
|
20
20
|
export class JsonRpcSession {
|
|
21
21
|
child;
|
|
22
22
|
requestTimeoutMs;
|
|
23
|
+
acceptNotification;
|
|
23
24
|
lines;
|
|
24
25
|
pending = new Map();
|
|
25
26
|
events = [];
|
|
26
27
|
waiters = [];
|
|
27
28
|
nextId = 1;
|
|
28
29
|
closed = null;
|
|
29
|
-
constructor(child, requestTimeoutMs = 30_000) {
|
|
30
|
+
constructor(child, requestTimeoutMs = 30_000, acceptNotification = () => true) {
|
|
30
31
|
this.child = child;
|
|
31
32
|
this.requestTimeoutMs = requestTimeoutMs;
|
|
33
|
+
this.acceptNotification = acceptNotification;
|
|
32
34
|
this.lines = createInterface({ input: child.stdout });
|
|
33
35
|
this.lines.on("line", (line) => this.receive(line, "stdout"));
|
|
34
36
|
child.once("error", (error) => this.close(error));
|
|
@@ -135,7 +137,8 @@ export class JsonRpcSession {
|
|
|
135
137
|
}
|
|
136
138
|
return;
|
|
137
139
|
}
|
|
138
|
-
this.
|
|
140
|
+
if (id !== undefined || this.acceptNotification(message))
|
|
141
|
+
this.enqueue(message);
|
|
139
142
|
}
|
|
140
143
|
write(message) {
|
|
141
144
|
this.child.stdin.write(`${JSON.stringify(message)}\n`);
|
|
@@ -4,6 +4,7 @@ import { canonicalJson } from "../config/hash.js";
|
|
|
4
4
|
import { sha256 } from "../fs/hash.js";
|
|
5
5
|
import { AgentOpsError } from "../fs/paths.js";
|
|
6
6
|
import { readPrivateFile, writePrivateFile } from "../security/permissions.js";
|
|
7
|
+
import { parseWorktreeListPorcelain, readWorktreeRecord } from "../parallel/service.js";
|
|
7
8
|
import { FileRunRepository } from "./service.js";
|
|
8
9
|
function ledger(commonDir) { return new FileRunRepository(join(commonDir, "agent-ops", "runs"), commonDir); }
|
|
9
10
|
function proofPath(commonDir, runId) { return join(commonDir, "agent-ops", "runs", runId, "integration-proof.json"); }
|
|
@@ -84,6 +85,11 @@ export async function readRunIntegrationProof(commonDir, record) {
|
|
|
84
85
|
return null;
|
|
85
86
|
if (state.ownerSessionId !== record.sessionId || state.coordinatorId !== record.workerId)
|
|
86
87
|
throw new AgentOpsError("RUN_INTEGRATION_OWNERSHIP", "Recovery coordinator does not match the sealed transaction.");
|
|
88
|
+
const coordinator = state.workers.find(w => w.workerId === state.coordinatorId);
|
|
89
|
+
if (record.mainRoot !== state.root || record.ownerSessionId !== state.ownerSessionId ||
|
|
90
|
+
(state.workers.length > 0 && (coordinator === undefined || coordinator.worktree !== record.path ||
|
|
91
|
+
coordinator.generation !== record.workerGeneration || coordinator.ownerSessionId !== record.ownerSessionId)))
|
|
92
|
+
throw new AgentOpsError("RUN_INTEGRATION_OWNERSHIP", "Recovery requires the current coordinator worktree owner and generation.");
|
|
87
93
|
return await readSealedProof(commonDir, record.runId, state.integration);
|
|
88
94
|
}
|
|
89
95
|
/** A failed pre-merge CAS may be retried only after archiving the unused sealed transaction. */
|
|
@@ -133,17 +139,15 @@ export async function markRunIntegration(commonDir, record, status, step, receip
|
|
|
133
139
|
await ledger(commonDir).mutate(record.runId, current => {
|
|
134
140
|
if (current.integration === null || current.ownerSessionId !== record.sessionId || current.coordinatorId !== record.workerId)
|
|
135
141
|
throw new AgentOpsError("RUN_INTEGRATION_OWNERSHIP", "The integration transaction is missing or belongs to another coordinator.");
|
|
142
|
+
if (current.status !== "active" || current.disableRestart || current.awaitingResume)
|
|
143
|
+
throw new AgentOpsError("RUN_RESUME_DISABLED", "A paused, stopped or disabled run cannot advance integration steps.");
|
|
136
144
|
return { ...current, integration: { ...current.integration, status, updatedAt: new Date().toISOString(),
|
|
137
145
|
completedSteps: [...new Set([...current.integration.completedSteps, step])],
|
|
138
146
|
...(receipt === undefined ? {} : { receiptPath: receipt.path, receiptDigest: receipt.digest }) } };
|
|
139
147
|
});
|
|
140
148
|
}
|
|
141
|
-
/**
|
|
142
|
-
|
|
143
|
-
* sealed artifacts and advances the ledger; it never recreates a checkout or
|
|
144
|
-
* starts a native writer.
|
|
145
|
-
*/
|
|
146
|
-
export async function recoverRunIntegrationAfterCleanup(commonDir, runId, git) {
|
|
149
|
+
/** Validate the sealed transaction before any remaining cleanup or final bookkeeping. */
|
|
150
|
+
export async function validateRunIntegrationCleanup(commonDir, runId, git, allowMissingNote = false) {
|
|
147
151
|
const repository = ledger(commonDir);
|
|
148
152
|
const state = await repository.read(runId);
|
|
149
153
|
if (state === null || state.integration === null)
|
|
@@ -154,7 +158,7 @@ export async function recoverRunIntegrationAfterCleanup(commonDir, runId, git) {
|
|
|
154
158
|
await readSealedProof(commonDir, runId, integration);
|
|
155
159
|
if (integration.status !== "cleaned" &&
|
|
156
160
|
(integration.status !== "receipt-written" || !integration.completedSteps.includes("receipt") ||
|
|
157
|
-
!integration.completedSteps.includes("
|
|
161
|
+
!integration.completedSteps.includes("target") || !integration.completedSteps.includes("tasks"))) {
|
|
158
162
|
throw new AgentOpsError("RUN_INTEGRATION_RECOVERY_REQUIRED", "Recovery requires a recorded receipt and Git note before cleanup can be completed.");
|
|
159
163
|
}
|
|
160
164
|
if (integration.receiptPath === null || integration.receiptDigest === null)
|
|
@@ -177,9 +181,10 @@ export async function recoverRunIntegrationAfterCleanup(commonDir, runId, git) {
|
|
|
177
181
|
validateRunIntegrationReceipt(integration, receipt);
|
|
178
182
|
const note = await git(state.root, ["notes", "--ref=agent-ops", "show", integration.candidate]);
|
|
179
183
|
const noteText = note.stdout;
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
184
|
+
const noteBound = note.exitCode === 0 && noteText.split("\n").includes(`session: ${state.ownerSessionId}`) &&
|
|
185
|
+
noteText.split("\n").includes(`receipt: ${integration.receiptPath}`) &&
|
|
186
|
+
noteText.split("\n").includes(`receipt-sha256: ${integration.receiptDigest}`);
|
|
187
|
+
if (!noteBound && !(allowMissingNote && !integration.completedSteps.includes("note") && note.exitCode === 1)) {
|
|
183
188
|
throw new AgentOpsError("RUN_INTEGRATION_RECOVERY_REQUIRED", "The sealed receipt is not bound by the Git note for the candidate.");
|
|
184
189
|
}
|
|
185
190
|
const target = await git(state.root, ["rev-parse", state.targetBranch + "^{commit}"]);
|
|
@@ -189,10 +194,27 @@ export async function recoverRunIntegrationAfterCleanup(commonDir, runId, git) {
|
|
|
189
194
|
(await git(state.root, ["merge-base", "--is-ancestor", integration.candidate, target.stdout.trim()])).exitCode === 0;
|
|
190
195
|
if (!exact && !historic)
|
|
191
196
|
throw new AgentOpsError("RUN_INTEGRATION_TARGET_CHANGED", "The target is inconsistent with the sealed, receipt-bound integration history.");
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
197
|
+
return { state, integration, receipt: receipt, noteBound, observedTarget: target.stdout.trim() };
|
|
198
|
+
}
|
|
199
|
+
/** After owned checkouts vanish, advance bookkeeping without executing product proof or native writers. */
|
|
200
|
+
export async function recoverRunIntegrationAfterCleanup(commonDir, runId, git) {
|
|
201
|
+
const { state, integration, observedTarget } = await validateRunIntegrationCleanup(commonDir, runId, git);
|
|
202
|
+
const listing = await git(state.root, ["worktree", "list", "--porcelain"]);
|
|
203
|
+
if (listing.exitCode !== 0)
|
|
204
|
+
throw new AgentOpsError("RUN_INTEGRATION_RECOVERY_REQUIRED", "Cannot confirm that integration checkouts were cleaned.");
|
|
205
|
+
for (const checkout of parseWorktreeListPorcelain(listing.stdout)) {
|
|
206
|
+
const record = await readWorktreeRecord(checkout.path);
|
|
207
|
+
if (record?.runId === runId || state.workers.some(worker => worker.worktree === checkout.path))
|
|
208
|
+
throw new AgentOpsError("RUN_INTEGRATION_CLEANUP_PENDING", "An owned integration checkout still exists; finish its missing cleanup steps before marking the run cleaned.");
|
|
209
|
+
}
|
|
210
|
+
const repository = ledger(commonDir);
|
|
211
|
+
await repository.mutate(runId, async (current) => {
|
|
212
|
+
const target = await git(state.root, ["rev-parse", state.targetBranch + "^{commit}"]);
|
|
213
|
+
if (current.revision !== state.revision || canonicalJson(current.integration) !== canonicalJson(integration) ||
|
|
214
|
+
target.exitCode !== 0 || target.stdout.trim() !== observedTarget)
|
|
195
215
|
throw new AgentOpsError("RUN_INTEGRATION_STALE", "The integration changed during recovery.");
|
|
216
|
+
if (current.integration === null)
|
|
217
|
+
throw new AgentOpsError("RUN_INTEGRATION_REQUIRED", "The integration disappeared during recovery.");
|
|
196
218
|
if (current.integration.status === "cleaned")
|
|
197
219
|
return current;
|
|
198
220
|
if (current.integration.status !== "receipt-written")
|
|
@@ -9,7 +9,9 @@ import { evaluateGuardrail } from "../guardrails/evaluate.js";
|
|
|
9
9
|
import { validateConfig } from "../schema/validate.js";
|
|
10
10
|
import { readPrivateFile, writePrivateFile } from "../security/permissions.js";
|
|
11
11
|
import { redactSecrets } from "../security/redact.js";
|
|
12
|
-
import {
|
|
12
|
+
import { remainingRunTimeMs } from "./scheduler.js";
|
|
13
|
+
import { MAX_EVENTS } from "./service.js";
|
|
14
|
+
export const RUN_POLICY_TTL_MS = 60 * 60 * 1000;
|
|
13
15
|
function fail(code, message) { throw new AgentOpsError(code, message); }
|
|
14
16
|
const plain = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
15
17
|
function validApprovals(value) {
|
|
@@ -62,13 +64,14 @@ export async function runPolicyAssessment(state, policy, capabilityId) {
|
|
|
62
64
|
const assessment = current.approvals.find(a => a.capabilityId === capabilityId);
|
|
63
65
|
if (assessment !== undefined && canonicalJson(capabilities(current.config).get(capabilityId) ?? null) === canonicalJson(definition))
|
|
64
66
|
return assessment;
|
|
65
|
-
|
|
67
|
+
const digest = current.assessmentDigest ?? current.previousDigest;
|
|
68
|
+
if (digest === null)
|
|
66
69
|
break;
|
|
67
|
-
const source = await readPrivateFile(join(state.commonDir, "agent-ops/runs", state.runId, "policies",
|
|
68
|
-
if (source === null || sha256(source) !==
|
|
70
|
+
const source = await readPrivateFile(join(state.commonDir, "agent-ops/runs", state.runId, "policies", digest + ".json"), state.commonDir);
|
|
71
|
+
if (source === null || sha256(source) !== digest)
|
|
69
72
|
fail("RUN_POLICY_CHANGED", "Historical capability assessment is missing or changed.");
|
|
70
73
|
const previous = JSON.parse(source);
|
|
71
|
-
current = (await readRunPolicy({ ...state, policyBinding: { artifactDigest:
|
|
74
|
+
current = (await readRunPolicy({ ...state, policyBinding: { artifactDigest: digest, configHash: calculateConfigHash(previous.config),
|
|
72
75
|
runtimeHash: previous.runtimeHash, expiresAt: previous.expiresAt } }, previous.runtimeHash, Date.now(), true));
|
|
73
76
|
}
|
|
74
77
|
return fail("RUN_POLICY_APPROVAL_REQUIRED", "The current capability lacks a matching historical run assessment.");
|
|
@@ -107,6 +110,8 @@ export function validateRunPolicyChange(previous, candidate, approvals) {
|
|
|
107
110
|
return config;
|
|
108
111
|
}
|
|
109
112
|
export async function readRunPolicy(state, runtimeHash, now = Date.now(), allowExpired = false) {
|
|
113
|
+
if (!Number.isSafeInteger(now) || now < 0 || now > 8.64e15 - RUN_POLICY_TTL_MS)
|
|
114
|
+
fail("RUN_CLOCK_INVALID", "Policy clock must be a valid nonnegative timestamp with room for a finite grant.");
|
|
110
115
|
const binding = state.policyBinding;
|
|
111
116
|
if (binding === undefined)
|
|
112
117
|
return null;
|
|
@@ -121,11 +126,19 @@ export async function readRunPolicy(state, runtimeHash, now = Date.now(), allowE
|
|
|
121
126
|
catch {
|
|
122
127
|
return fail("RUN_POLICY_INVALID", "Run policy JSON is invalid.");
|
|
123
128
|
}
|
|
124
|
-
if (!plain(parsed) || Object.keys(parsed).some(key => !["schemaVersion", "runId", "config", "baseConfig", "baseTrustBinding", "runtimeHash", "expiresAt", "approvals", "previousDigest"].includes(key)) ||
|
|
129
|
+
if (!plain(parsed) || Object.keys(parsed).some(key => !["schemaVersion", "runId", "config", "baseConfig", "baseTrustBinding", "runtimeHash", "expiresAt", "issuedAt", "approvals", "previousDigest", "assessmentDigest"].includes(key)) ||
|
|
125
130
|
!validBaseBinding(parsed.baseTrustBinding, state.root) || !validApprovals(parsed.approvals) ||
|
|
126
|
-
!(parsed.previousDigest === null || (typeof parsed.previousDigest === "string" && /^[a-f0-9]{64}$/u.test(parsed.previousDigest)))
|
|
131
|
+
!(parsed.previousDigest === null || (typeof parsed.previousDigest === "string" && /^[a-f0-9]{64}$/u.test(parsed.previousDigest))) ||
|
|
132
|
+
(parsed.assessmentDigest !== undefined && (typeof parsed.assessmentDigest !== "string" || !/^[a-f0-9]{64}$/u.test(parsed.assessmentDigest))))
|
|
127
133
|
fail("RUN_POLICY_INVALID", "Run policy artifact has an invalid structure or trust origin.");
|
|
128
134
|
const artifact = parsed;
|
|
135
|
+
const expires = Date.parse(artifact.expiresAt), issued = Date.parse(artifact.issuedAt ?? state.createdAt);
|
|
136
|
+
if (typeof artifact.expiresAt !== "string" || (artifact.issuedAt !== undefined && typeof artifact.issuedAt !== "string") ||
|
|
137
|
+
!Number.isFinite(expires) || !Number.isFinite(issued) || expires <= issued ||
|
|
138
|
+
(artifact.issuedAt !== undefined && expires - issued > RUN_POLICY_TTL_MS))
|
|
139
|
+
fail("RUN_POLICY_INVALID", "Policy requires a valid finite authorization window.");
|
|
140
|
+
if (now < issued)
|
|
141
|
+
fail("RUN_CLOCK_INVALID", "Policy clock moved behind its issuance; manual reconciliation is required.");
|
|
129
142
|
if (artifact.schemaVersion !== 1 || artifact.runId !== state.runId || !validateConfig(artifact.config).ok ||
|
|
130
143
|
!validateConfig(artifact.baseConfig).ok || calculateConfigHash(artifact.config) !== binding.configHash ||
|
|
131
144
|
artifact.runtimeHash !== binding.runtimeHash || artifact.expiresAt !== binding.expiresAt ||
|
|
@@ -143,7 +156,7 @@ export async function bindRunPolicy(repository, runId, input) {
|
|
|
143
156
|
if (state === null)
|
|
144
157
|
return fail("RUN_NOT_FOUND", "Run does not exist.");
|
|
145
158
|
const now = input.now ?? Date.now();
|
|
146
|
-
if (!Number.
|
|
159
|
+
if (!Number.isSafeInteger(now) || now < 0 || !validBaseBinding(input.baseTrustBinding, state.root))
|
|
147
160
|
fail("RUN_POLICY_INVALID", "Policy requires the original canonical repository trust binding and a valid clock.");
|
|
148
161
|
const previous = await readRunPolicy(state, input.resumeRuntime === true ? state.policyBinding?.runtimeHash ?? input.runtimeHash : input.runtimeHash, now, true);
|
|
149
162
|
if ((state.policyBinding?.artifactDigest ?? undefined) !== input.expectedDigest)
|
|
@@ -159,12 +172,13 @@ export async function bindRunPolicy(repository, runId, input) {
|
|
|
159
172
|
if (!validateConfig(config).ok || !/^[a-f0-9]{64}$/u.test(input.runtimeHash) ||
|
|
160
173
|
(previous === null && input.baseTrustBinding.configHash !== calculateConfigHash(config)))
|
|
161
174
|
fail("RUN_POLICY_INVALID", "Run policy or runtime identity is invalid.");
|
|
162
|
-
const remaining =
|
|
175
|
+
const remaining = remainingRunTimeMs(state, now);
|
|
163
176
|
if (remaining <= 0)
|
|
164
177
|
fail("RUN_BUDGET_EXHAUSTED", "No active execution budget remains.");
|
|
165
178
|
const artifact = { schemaVersion: 1, runId, config, baseConfig: previous?.baseConfig ?? config,
|
|
166
179
|
baseTrustBinding: previous?.baseTrustBinding ?? input.baseTrustBinding, runtimeHash: input.runtimeHash,
|
|
167
|
-
|
|
180
|
+
issuedAt: new Date(now).toISOString(), expiresAt: new Date(now + Math.min(remaining, RUN_POLICY_TTL_MS)).toISOString(),
|
|
181
|
+
approvals: (input.approvals ?? []).map(a => ({ ...a, reason: redactSecrets(a.reason) })), previousDigest: previous === null ? null : state.policyBinding.artifactDigest };
|
|
168
182
|
const content = canonicalJson(artifact), digest = sha256(content);
|
|
169
183
|
await writePrivateFile(join(state.commonDir, "agent-ops", "runs", runId, "policies", digest + ".json"), content, state.commonDir);
|
|
170
184
|
return await repository.mutate(runId, current => {
|
|
@@ -174,3 +188,44 @@ export async function bindRunPolicy(repository, runId, input) {
|
|
|
174
188
|
artifactDigest: digest, expiresAt: artifact.expiresAt } };
|
|
175
189
|
});
|
|
176
190
|
}
|
|
191
|
+
/** Renew the same finite grant without changing native identities, scope or trust. Reads never renew. */
|
|
192
|
+
export async function renewRunPolicy(repository, runId, input) {
|
|
193
|
+
const state = await repository.read(runId);
|
|
194
|
+
if (state === null)
|
|
195
|
+
return fail("RUN_NOT_FOUND", "Run does not exist.");
|
|
196
|
+
if (state.status !== "active" || state.disableRestart || state.awaitingResume || state.events.some(e => e.code === "RUN_COMMAND_DENIED"))
|
|
197
|
+
return fail("RUN_POLICY_DISABLED", "Renewal cannot awaken a stopped run or erase a denial.");
|
|
198
|
+
const now = input.now ?? Date.now();
|
|
199
|
+
if (now < Date.parse(state.budget.lastObservedAt))
|
|
200
|
+
return fail("RUN_CLOCK_INVALID", "Policy clock moved behind the last observed accounting time.");
|
|
201
|
+
const authority = await input.authority();
|
|
202
|
+
const previous = await readRunPolicy(state, authority.runtimeHash, now, true);
|
|
203
|
+
if (previous === null)
|
|
204
|
+
return fail("RUN_POLICY_INVALID", "Renewal requires an existing bound authorization.");
|
|
205
|
+
const assertAuthority = (current) => {
|
|
206
|
+
if (!current.trusted || current.runtimeHash !== previous.runtimeHash ||
|
|
207
|
+
canonicalJson(current.baseTrustBinding) !== canonicalJson(previous.baseTrustBinding) ||
|
|
208
|
+
calculateConfigHash(current.config) !== calculateConfigHash(previous.baseConfig))
|
|
209
|
+
fail("RUN_POLICY_AUTHORITY_CHANGED", "Original trust, repository configuration or executable runtime changed; obtain genuine authorization before resume.");
|
|
210
|
+
};
|
|
211
|
+
assertAuthority(authority);
|
|
212
|
+
if (Date.parse(previous.expiresAt) > now)
|
|
213
|
+
return state;
|
|
214
|
+
if (remainingRunTimeMs(state, now) <= 0)
|
|
215
|
+
return fail("RUN_BUDGET_EXHAUSTED", "Legacy finite runs require manual resume before renewal.");
|
|
216
|
+
const artifact = { ...previous, issuedAt: new Date(now).toISOString(),
|
|
217
|
+
expiresAt: new Date(now + Math.min(RUN_POLICY_TTL_MS, remainingRunTimeMs(state, now))).toISOString(),
|
|
218
|
+
assessmentDigest: previous.assessmentDigest ?? state.policyBinding.artifactDigest,
|
|
219
|
+
previousDigest: state.policyBinding.artifactDigest };
|
|
220
|
+
const content = canonicalJson(artifact), digest = sha256(content);
|
|
221
|
+
await writePrivateFile(join(state.commonDir, "agent-ops", "runs", runId, "policies", digest + ".json"), content, state.commonDir);
|
|
222
|
+
assertAuthority(await input.authority());
|
|
223
|
+
return await repository.mutate(runId, current => {
|
|
224
|
+
if (current.revision !== state.revision || current.status !== "active" || current.disableRestart || current.awaitingResume ||
|
|
225
|
+
current.policyBinding?.artifactDigest !== state.policyBinding.artifactDigest || current.events.some(e => e.code === "RUN_COMMAND_DENIED"))
|
|
226
|
+
return fail("RUN_POLICY_STALE", "Run stop, denial or scope changed during renewal.");
|
|
227
|
+
return { ...current, policyBinding: { ...state.policyBinding, artifactDigest: digest, expiresAt: artifact.expiresAt },
|
|
228
|
+
events: [...current.events, { id: `renewal-${digest.slice(0, 32)}`, at: artifact.issuedAt, type: "diagnostic",
|
|
229
|
+
code: "RUN_POLICY_RENEWED", workerId: null, taskId: null, detail: `Same-authority finite grant renewed from ${state.policyBinding.artifactDigest}.` }].slice(-MAX_EVENTS) };
|
|
230
|
+
});
|
|
231
|
+
}
|
|
@@ -63,16 +63,17 @@ export function planSchedule(state) {
|
|
|
63
63
|
const running = state.tasks.filter((task) => task.status === "running" || task.status === "awaiting-delivery");
|
|
64
64
|
const blocked = state.tasks.filter((task) => task.status === "blocked");
|
|
65
65
|
const ready = state.tasks.filter((task) => ["planned", "ready"].includes(task.status) && dependencyState(state.tasks, task) === "ready");
|
|
66
|
+
const slotConsumers = state.workers.filter(worker => ["assigned", "starting", "running", "idle", "handing-off"].includes(worker.status));
|
|
66
67
|
return {
|
|
67
|
-
ready,
|
|
68
|
+
ready: ready.filter(task => task.taskId !== state.rootTaskId),
|
|
68
69
|
running,
|
|
69
70
|
blocked,
|
|
70
|
-
availableSlots: Math.max(0, Math.min(state.jobs, MAX_ACTIVE_WORKERS) -
|
|
71
|
+
availableSlots: Math.max(0, Math.min(state.jobs, MAX_ACTIVE_WORKERS) - slotConsumers.length)
|
|
71
72
|
};
|
|
72
73
|
}
|
|
73
74
|
/**
|
|
74
75
|
* Compute whole-run active wall time as the union of intervals. This prevents
|
|
75
|
-
* two parallel writers from doubling
|
|
76
|
+
* two parallel writers from doubling measured consumption.
|
|
76
77
|
*/
|
|
77
78
|
export function activeWallTimeMs(intervals, nowMs) {
|
|
78
79
|
if (!Number.isFinite(nowMs) || nowMs < 0)
|
|
@@ -101,7 +102,11 @@ export function activeWallTimeMs(intervals, nowMs) {
|
|
|
101
102
|
return start === null ? 0 : total + end - start;
|
|
102
103
|
}
|
|
103
104
|
export function budgetExpired(state, now = Date.now()) {
|
|
104
|
-
return activeWallTimeMs(state.budget.activeIntervals, now) >= state.budget.limitMs;
|
|
105
|
+
return state.budget.limitMs !== null && activeWallTimeMs(state.budget.activeIntervals, now) >= state.budget.limitMs;
|
|
106
|
+
}
|
|
107
|
+
/** Infinity is internal only: callers retain their finite command/review timeouts. */
|
|
108
|
+
export function remainingRunTimeMs(state, now = Date.now()) {
|
|
109
|
+
return state.budget.limitMs === null ? Infinity : state.budget.limitMs - activeWallTimeMs(state.budget.activeIntervals, now);
|
|
105
110
|
}
|
|
106
111
|
/** Two consecutive identical failure rounds without a useful change stop one child. */
|
|
107
112
|
export function noProgressForTwoRounds(rounds) {
|
|
@@ -249,7 +254,7 @@ export class RunScheduler {
|
|
|
249
254
|
throw error("RUN_CLOCK_INVALID", "Budget clock is invalid.");
|
|
250
255
|
return await this.#repository.mutate(runId, (current) => {
|
|
251
256
|
const accumulatedMs = activeWallTimeMs(current.budget.activeIntervals, nowMs);
|
|
252
|
-
const expired = accumulatedMs >= current.budget.limitMs;
|
|
257
|
+
const expired = current.budget.limitMs !== null && accumulatedMs >= current.budget.limitMs;
|
|
253
258
|
return {
|
|
254
259
|
...current,
|
|
255
260
|
status: expired && ["active", "awaiting-input"].includes(current.status) ? "budget-limited" : current.status,
|