@fitlab-ai/agent-infra 0.9.6 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/internal-cli.js +5 -0
- package/dist/lib/agent-clients/adapters/codex-lifecycle/app-server.js +45 -4
- package/dist/lib/agent-clients/adapters/codex-lifecycle/build-identity.js +162 -0
- package/dist/lib/agent-clients/adapters/codex-lifecycle/capability-store.js +244 -0
- package/dist/lib/agent-clients/adapters/codex-lifecycle/sandbox-controller.js +370 -0
- package/dist/lib/agent-clients/adapters/codex-lifecycle/store.js +21 -5
- package/dist/lib/defaults.json +10 -2
- package/dist/lib/git/workflow.js +30 -3
- package/dist/lib/internal/codex-lifecycle.js +69 -1
- package/dist/lib/internal/codex-sandbox-controller.js +106 -0
- package/dist/lib/internal/task-orchestration.js +37 -5
- package/dist/lib/platform/context.js +6 -4
- package/dist/lib/platform/github-client.js +55 -2
- package/dist/lib/process-data/checkpoint.js +319 -0
- package/dist/lib/process-data/index.js +128 -60
- package/dist/lib/process-data/materialize.js +84 -0
- package/dist/lib/process-data/normalize.js +34 -1
- package/dist/lib/process-data/reconcile.js +55 -1
- package/dist/lib/process-data/sources.js +422 -1
- package/dist/lib/process-data/store.js +341 -11
- package/dist/lib/sandbox/commands/create.js +103 -51
- package/dist/lib/sandbox/commands/rm.js +64 -15
- package/dist/lib/sandbox/commands/start.js +4 -3
- package/dist/lib/sandbox/control/container-identity.js +64 -0
- package/dist/lib/sandbox/control/executor.js +13 -7
- package/dist/lib/sandbox/control/lifecycle.js +238 -29
- package/dist/lib/sandbox/control/protocol.js +7 -0
- package/dist/lib/sandbox/control/server.js +202 -26
- package/dist/lib/sandbox/control/state.js +25 -15
- package/dist/lib/sandbox/control/timing.js +8 -0
- package/dist/lib/sandbox/mounts.js +5 -13
- package/dist/lib/sandbox/recovery.js +59 -42
- package/dist/lib/sandbox/workspace-view.js +57 -40
- package/dist/lib/server/daemon.js +1 -1
- package/dist/lib/server/process-control.js +11 -1
- package/dist/lib/server/process-state.js +57 -20
- package/dist/lib/task/artifact-lifecycle.js +82 -11
- package/dist/lib/task/artifact-receipts.js +128 -0
- package/dist/lib/task/codex-orchestration.js +81 -6
- package/dist/lib/task/commands/validate.js +1 -1
- package/dist/lib/task/create.js +10 -2
- package/dist/lib/task/delegation-receipts.js +84 -2
- package/dist/lib/task/events.js +139 -2
- package/dist/lib/task/ledger.js +13 -1
- package/dist/lib/task/orchestration.js +235 -14
- package/dist/lib/task/sections.js +5 -1
- package/dist/lib/task/task-execution-lock.js +5 -4
- package/dist/lib/task/verification-engine.js +40 -8
- package/dist/lifecycle-build-manifest.json +6 -0
- package/lib/agent-clients/adapters/codex-lifecycle/app-server.ts +60 -3
- package/lib/agent-clients/adapters/codex-lifecycle/build-identity.ts +227 -0
- package/lib/agent-clients/adapters/codex-lifecycle/capability-store.ts +321 -0
- package/lib/agent-clients/adapters/codex-lifecycle/manifest-files.json +26 -0
- package/lib/agent-clients/adapters/codex-lifecycle/sandbox-controller.ts +453 -0
- package/lib/agent-clients/adapters/codex-lifecycle/store.ts +33 -4
- package/lib/defaults.json +10 -2
- package/lib/git/workflow.ts +25 -2
- package/lib/internal/codex-lifecycle.ts +68 -1
- package/lib/internal/codex-sandbox-controller.ts +109 -0
- package/lib/internal/task-orchestration.ts +31 -4
- package/lib/platform/context.ts +8 -5
- package/lib/platform/github-client.ts +61 -3
- package/lib/process-data/checkpoint.ts +335 -0
- package/lib/process-data/index.ts +112 -60
- package/lib/process-data/materialize.ts +83 -0
- package/lib/process-data/normalize.ts +29 -1
- package/lib/process-data/reconcile.ts +64 -2
- package/lib/process-data/sources.ts +449 -2
- package/lib/process-data/store.ts +304 -11
- package/lib/process-data/types.ts +124 -2
- package/lib/sandbox/commands/create.ts +56 -7
- package/lib/sandbox/commands/rm.ts +64 -14
- package/lib/sandbox/commands/start.ts +4 -3
- package/lib/sandbox/control/container-identity.ts +93 -0
- package/lib/sandbox/control/executor.ts +14 -10
- package/lib/sandbox/control/lifecycle.ts +264 -33
- package/lib/sandbox/control/protocol.ts +31 -5
- package/lib/sandbox/control/server.ts +184 -24
- package/lib/sandbox/control/state.ts +34 -17
- package/lib/sandbox/control/timing.ts +12 -0
- package/lib/sandbox/mounts.ts +5 -15
- package/lib/sandbox/recovery.ts +71 -42
- package/lib/sandbox/workspace-view.ts +61 -40
- package/lib/server/daemon.ts +1 -1
- package/lib/server/process-control.ts +13 -3
- package/lib/server/process-state.ts +82 -27
- package/lib/task/artifact-lifecycle.ts +73 -10
- package/lib/task/artifact-receipts.ts +161 -0
- package/lib/task/codex-orchestration.ts +91 -5
- package/lib/task/commands/validate.ts +1 -1
- package/lib/task/create.ts +11 -2
- package/lib/task/delegation-receipts.ts +160 -8
- package/lib/task/events.ts +122 -1
- package/lib/task/ledger.ts +16 -1
- package/lib/task/orchestration.ts +356 -17
- package/lib/task/sections.ts +6 -2
- package/lib/task/task-execution-lock.ts +11 -9
- package/lib/task/verification-engine.ts +51 -16
- package/package.json +1 -1
- package/templates/.agents/README.en.md +7 -0
- package/templates/.agents/README.zh-CN.md +5 -0
- package/templates/.agents/hooks/lifecycle-delegation.js +25 -1
- package/templates/.agents/rules/lifecycle-orchestration.en.md +8 -2
- package/templates/.agents/rules/lifecycle-orchestration.zh-CN.md +8 -2
- package/templates/.agents/rules/pr-checks-commands.github.en.md +1 -1
- package/templates/.agents/rules/pr-checks-commands.github.zh-CN.md +1 -1
- package/templates/.agents/rules/review-handshake.en.md +3 -1
- package/templates/.agents/rules/review-handshake.zh-CN.md +3 -1
- package/templates/.agents/skills/code-task/reference/dual-mode.en.md +3 -1
- package/templates/.agents/skills/code-task/reference/dual-mode.zh-CN.md +3 -1
- package/templates/.agents/skills/complete-task/SKILL.en.md +4 -0
- package/templates/.agents/skills/complete-task/SKILL.zh-CN.md +4 -0
- package/templates/.agents/skills/run-manual-validation/SKILL.en.md +6 -6
- package/templates/.agents/skills/run-manual-validation/SKILL.zh-CN.md +6 -6
- package/templates/.agents/skills/run-manual-validation/config/verify.en.json +14 -3
- package/templates/.agents/skills/run-manual-validation/config/verify.zh-CN.json +14 -3
- package/templates/.agents/skills/run-manual-validation/reference/discovery-and-execution.en.md +67 -0
- package/templates/.agents/skills/run-manual-validation/reference/discovery-and-execution.zh-CN.md +67 -0
- package/templates/.agents/skills/run-manual-validation/reference/report-template.en.md +25 -0
- package/templates/.agents/skills/run-manual-validation/reference/report-template.zh-CN.md +25 -0
- package/templates/.agents/skills/run-task/SKILL.en.md +9 -7
- package/templates/.agents/skills/run-task/SKILL.zh-CN.md +9 -7
- package/templates/.agents/skills/run-task/reference/host-validation.en.md +2 -0
- package/templates/.agents/skills/run-task/reference/host-validation.zh-CN.md +2 -0
- package/templates/.agents/skills/update-agent-infra/scripts/sync-templates.js +75 -3
- package/templates/.agents/workspace/README.en.md +0 -26
- package/templates/.agents/workspace/README.zh-CN.md +0 -26
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
3
4
|
import { getProcessStartTime } from '../../server/process-state.ts';
|
|
4
5
|
import { assertGitWorktreeBinding } from '../../git/worktree-identity.ts';
|
|
5
6
|
import {
|
|
6
7
|
controlError,
|
|
7
|
-
|
|
8
|
+
DEFAULT_SANDBOX_CONTROL_TIMING,
|
|
8
9
|
type SandboxControlManifest,
|
|
9
10
|
type SandboxControlRequest,
|
|
10
|
-
type SandboxControlResponse
|
|
11
|
+
type SandboxControlResponse,
|
|
12
|
+
type SandboxControlTimingPolicy
|
|
11
13
|
} from './protocol.ts';
|
|
12
14
|
import { prepareSandboxControlExecution, type PreparedSandboxControlExecution, type SandboxControlExecutionResult } from './executor.ts';
|
|
13
15
|
import {
|
|
@@ -21,11 +23,15 @@ import {
|
|
|
21
23
|
writeSandboxControlStatus
|
|
22
24
|
} from './state.ts';
|
|
23
25
|
import { validateSandboxControlRequest } from './protocol.ts';
|
|
26
|
+
import { inspectSandboxControlContainer, type ContainerObservation } from './container-identity.ts';
|
|
24
27
|
import {
|
|
25
28
|
acquireSandboxControlBrokerStartup,
|
|
29
|
+
garbageCollectSandboxControlRoot,
|
|
26
30
|
isSandboxControlRootQuiescing,
|
|
27
31
|
readSandboxControlManifest
|
|
28
32
|
} from './lifecycle.ts';
|
|
33
|
+
import type { BrokerOwner } from './lifecycle.ts';
|
|
34
|
+
import { nextSandboxControlBackoff } from './timing.ts';
|
|
29
35
|
|
|
30
36
|
type ActiveExecution = {
|
|
31
37
|
request: SandboxControlRequest;
|
|
@@ -105,9 +111,10 @@ function bindingReason(manifest: SandboxControlManifest): string | null {
|
|
|
105
111
|
}
|
|
106
112
|
}
|
|
107
113
|
|
|
108
|
-
function recoverProcessing(manifest: SandboxControlManifest):
|
|
114
|
+
function recoverProcessing(manifest: SandboxControlManifest, brokerOwns: () => boolean): boolean {
|
|
109
115
|
for (const entry of fs.readdirSync(manifest.processingDir, { withFileTypes: true })) {
|
|
110
116
|
if (!entry.isDirectory() || !/^[a-f0-9-]{16,64}$/.test(entry.name)) continue;
|
|
117
|
+
if (!brokerOwns()) return false;
|
|
111
118
|
const descriptor = executionPath(manifest, entry.name);
|
|
112
119
|
const existingResponse = responsePath(manifest, entry.name);
|
|
113
120
|
if (fs.existsSync(descriptor)) {
|
|
@@ -115,6 +122,7 @@ function recoverProcessing(manifest: SandboxControlManifest): void {
|
|
|
115
122
|
if (!terminateSandboxControlExecution(execution)) {
|
|
116
123
|
throw new Error(`SANDBOX_CONTROL_EXECUTION_STILL_RUNNING: ${entry.name}`);
|
|
117
124
|
}
|
|
125
|
+
if (!brokerOwns()) return false;
|
|
118
126
|
appendSandboxControlAudit(manifest, 'orphan-tree-terminated', { requestId: entry.name });
|
|
119
127
|
let terminal = false;
|
|
120
128
|
if (fs.existsSync(existingResponse)) {
|
|
@@ -126,12 +134,18 @@ function recoverProcessing(manifest: SandboxControlManifest): void {
|
|
|
126
134
|
terminal = false;
|
|
127
135
|
}
|
|
128
136
|
}
|
|
129
|
-
if (!terminal)
|
|
137
|
+
if (!terminal) {
|
|
138
|
+
if (!brokerOwns()) return false;
|
|
139
|
+
writeResponse(manifest, unknown(entry.name));
|
|
140
|
+
}
|
|
130
141
|
} else {
|
|
142
|
+
if (!brokerOwns()) return false;
|
|
131
143
|
writeResponse(manifest, notExecuted(entry.name));
|
|
132
144
|
}
|
|
145
|
+
if (!brokerOwns()) return false;
|
|
133
146
|
fs.rmSync(path.join(manifest.processingDir, entry.name), { recursive: true, force: true });
|
|
134
147
|
}
|
|
148
|
+
return true;
|
|
135
149
|
}
|
|
136
150
|
|
|
137
151
|
function delay(ms: number): Promise<void> {
|
|
@@ -144,19 +158,42 @@ export function sandboxControlSafeEnv(env: NodeJS.ProcessEnv = process.env): Nod
|
|
|
144
158
|
|
|
145
159
|
export async function serveSandboxControl(
|
|
146
160
|
manifestPath: string,
|
|
147
|
-
signal: AbortSignal = new AbortController().signal
|
|
161
|
+
signal: AbortSignal = new AbortController().signal,
|
|
162
|
+
options: {
|
|
163
|
+
timing?: SandboxControlTimingPolicy;
|
|
164
|
+
inspectContainer?: (manifest: SandboxControlManifest) => Promise<ContainerObservation>;
|
|
165
|
+
bindingCheck?: (manifest: SandboxControlManifest) => string | null;
|
|
166
|
+
} = {}
|
|
148
167
|
): Promise<void> {
|
|
168
|
+
const timing = options.timing ?? DEFAULT_SANDBOX_CONTROL_TIMING;
|
|
169
|
+
const inspectContainer = options.inspectContainer ?? ((value: SandboxControlManifest) => inspectSandboxControlContainer(value));
|
|
170
|
+
const bindingCheck = options.bindingCheck ?? bindingReason;
|
|
149
171
|
const manifest = readSandboxControlManifest(manifestPath);
|
|
150
172
|
const root = path.dirname(manifestPath);
|
|
151
173
|
const startTime = getProcessStartTime(process.pid);
|
|
152
174
|
if (!startTime) throw new Error('SANDBOX_CONTROL_BROKER_IDENTITY_UNAVAILABLE');
|
|
153
|
-
const
|
|
175
|
+
const brokerId = randomUUID();
|
|
176
|
+
const releaseStartup = await acquireSandboxControlBrokerStartup(root, { pid: process.pid, startTime, brokerId });
|
|
154
177
|
const requestsDir = path.join(manifest.channelDir, 'requests');
|
|
155
178
|
const responsesDir = path.join(manifest.channelDir, 'responses');
|
|
156
179
|
const consumedDir = path.join(root, 'consumed');
|
|
157
|
-
const broker = {
|
|
180
|
+
const broker: BrokerOwner = {
|
|
181
|
+
version: 3,
|
|
182
|
+
pid: process.pid,
|
|
183
|
+
startTime,
|
|
184
|
+
brokerId,
|
|
185
|
+
token: manifest.token,
|
|
186
|
+
generation: manifest.generation
|
|
187
|
+
};
|
|
158
188
|
const brokerPath = path.join(root, 'broker.json');
|
|
159
|
-
const brokerRecord = `${JSON.stringify(
|
|
189
|
+
const brokerRecord = `${JSON.stringify(broker)}\n`;
|
|
190
|
+
const brokerOwns = (): boolean => {
|
|
191
|
+
try {
|
|
192
|
+
return fs.readFileSync(brokerPath, 'utf8') === brokerRecord;
|
|
193
|
+
} catch {
|
|
194
|
+
return false;
|
|
195
|
+
}
|
|
196
|
+
};
|
|
160
197
|
try {
|
|
161
198
|
for (const directory of [manifest.channelDir, requestsDir, responsesDir, consumedDir, manifest.publicStatusDir, manifest.processingDir]) {
|
|
162
199
|
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
@@ -178,11 +215,20 @@ export async function serveSandboxControl(
|
|
|
178
215
|
let active: ActiveExecution | null = null;
|
|
179
216
|
let lastState = '';
|
|
180
217
|
let lastStatusAt = 0;
|
|
218
|
+
let nextBindingCheckAt = 0;
|
|
219
|
+
let nextContainerHeartbeatAt = Date.now() + timing.containerHeartbeatMs;
|
|
220
|
+
let containerReasonCode: string | null = null;
|
|
221
|
+
let containerBackoffMs = timing.parkedBindingInitialMs;
|
|
222
|
+
let bindingReasonCode: string | null = null;
|
|
223
|
+
let bindingBackoffMs: number = timing.parkedBindingInitialMs;
|
|
181
224
|
try {
|
|
225
|
+
if (!brokerOwns()) return;
|
|
182
226
|
writeSandboxControlStatus(manifest, broker, 'starting', null, null);
|
|
227
|
+
if (!brokerOwns()) return;
|
|
183
228
|
appendSandboxControlAudit(manifest, 'broker-start', { pid: broker.pid });
|
|
184
|
-
recoverProcessing(manifest);
|
|
229
|
+
if (!recoverProcessing(manifest, brokerOwns)) return;
|
|
185
230
|
while (!signal.aborted) {
|
|
231
|
+
if (!brokerOwns()) break;
|
|
186
232
|
let current: SandboxControlManifest;
|
|
187
233
|
try {
|
|
188
234
|
current = readSandboxControlManifest(manifestPath);
|
|
@@ -190,54 +236,123 @@ export async function serveSandboxControl(
|
|
|
190
236
|
break;
|
|
191
237
|
}
|
|
192
238
|
if (current.token !== manifest.token || current.generation !== manifest.generation) break;
|
|
239
|
+
const heartbeatNow = Date.now();
|
|
240
|
+
if (heartbeatNow >= nextContainerHeartbeatAt) {
|
|
241
|
+
let observation: ContainerObservation;
|
|
242
|
+
try {
|
|
243
|
+
observation = await inspectContainer(manifest);
|
|
244
|
+
} catch (error) {
|
|
245
|
+
observation = { state: 'unknown', reason: controlError(error).code };
|
|
246
|
+
}
|
|
247
|
+
if (!brokerOwns()) break;
|
|
248
|
+
if (observation.state === 'found') {
|
|
249
|
+
containerReasonCode = null;
|
|
250
|
+
containerBackoffMs = timing.parkedBindingInitialMs;
|
|
251
|
+
nextContainerHeartbeatAt = Date.now() + timing.containerHeartbeatMs;
|
|
252
|
+
} else if (observation.state === 'unknown') {
|
|
253
|
+
containerReasonCode = 'SANDBOX_CONTROL_CONTAINER_UNKNOWN';
|
|
254
|
+
appendSandboxControlAudit(manifest, 'container-heartbeat-unknown', { reason: observation.reason });
|
|
255
|
+
const backoff = nextSandboxControlBackoff(containerBackoffMs, timing.slowCheckMs);
|
|
256
|
+
nextContainerHeartbeatAt = Date.now() + backoff.delayMs;
|
|
257
|
+
containerBackoffMs = backoff.nextDelayMs;
|
|
258
|
+
} else {
|
|
259
|
+
try {
|
|
260
|
+
await garbageCollectSandboxControlRoot(root, {
|
|
261
|
+
timeoutMs: timing.quiesceDeadlineMs,
|
|
262
|
+
inspectContainer: (timeoutMs) => inspectContainer(manifest),
|
|
263
|
+
selfOwner: broker
|
|
264
|
+
});
|
|
265
|
+
active = null;
|
|
266
|
+
return;
|
|
267
|
+
} catch (error) {
|
|
268
|
+
if (!brokerOwns()) break;
|
|
269
|
+
containerReasonCode = 'SANDBOX_CONTROL_CONTAINER_ABSENT_GC_FAILED';
|
|
270
|
+
appendSandboxControlAudit(manifest, 'container-gc-failed', { reason: controlError(error).code });
|
|
271
|
+
const backoff = nextSandboxControlBackoff(containerBackoffMs, timing.slowCheckMs);
|
|
272
|
+
nextContainerHeartbeatAt = Date.now() + backoff.delayMs;
|
|
273
|
+
containerBackoffMs = backoff.nextDelayMs;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
193
277
|
|
|
194
278
|
if (active?.settled) {
|
|
279
|
+
if (!brokerOwns()) break;
|
|
195
280
|
if (active.result) {
|
|
281
|
+
if (!brokerOwns()) break;
|
|
196
282
|
writeResponse(manifest, {
|
|
197
283
|
version: 2, id: active.request.id, phase: 'completed', exitCode: active.result.exitCode,
|
|
198
284
|
stdout: active.result.stdout, stderr: active.result.stderr, error: null
|
|
199
285
|
});
|
|
200
286
|
} else {
|
|
287
|
+
if (!brokerOwns()) break;
|
|
201
288
|
writeResponse(manifest, unknown(active.request.id));
|
|
202
289
|
}
|
|
290
|
+
if (!brokerOwns()) break;
|
|
203
291
|
fs.rmSync(path.join(manifest.processingDir, active.request.id), { recursive: true, force: true });
|
|
204
292
|
active = null;
|
|
205
293
|
}
|
|
206
294
|
|
|
207
295
|
let reasonCode: string | null = null;
|
|
208
296
|
try {
|
|
297
|
+
if (!brokerOwns()) break;
|
|
209
298
|
if (cleanupStaleSandboxControlLease(manifest)) {
|
|
299
|
+
if (!brokerOwns()) break;
|
|
210
300
|
appendSandboxControlAudit(manifest, 'lease-stale-cleanup');
|
|
211
301
|
}
|
|
212
302
|
if (readActiveLease(manifest)) reasonCode = 'SANDBOX_CONTROL_HANDOFF_ACTIVE';
|
|
213
303
|
} catch {
|
|
214
304
|
reasonCode = 'SANDBOX_CONTROL_HANDOFF_ACTIVE';
|
|
215
305
|
}
|
|
216
|
-
|
|
306
|
+
const now = Date.now();
|
|
307
|
+
if (now >= nextBindingCheckAt) {
|
|
308
|
+
bindingReasonCode = bindingCheck(manifest);
|
|
309
|
+
if (bindingReasonCode) {
|
|
310
|
+
const backoff = nextSandboxControlBackoff(bindingBackoffMs, timing.slowCheckMs);
|
|
311
|
+
nextBindingCheckAt = now + backoff.delayMs;
|
|
312
|
+
bindingBackoffMs = backoff.nextDelayMs;
|
|
313
|
+
} else {
|
|
314
|
+
nextBindingCheckAt = now + timing.slowCheckMs;
|
|
315
|
+
bindingBackoffMs = timing.parkedBindingInitialMs;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
reasonCode ??= containerReasonCode;
|
|
319
|
+
reasonCode ??= bindingReasonCode;
|
|
217
320
|
const state = reasonCode ? 'parked' : active ? 'busy' : 'healthy';
|
|
218
321
|
const stateKey = `${state}:${reasonCode ?? ''}:${active?.request.id ?? ''}`;
|
|
219
|
-
|
|
220
|
-
if (stateKey !== lastState || now - lastStatusAt >= SANDBOX_CONTROL_STATUS_INTERVAL_MS) {
|
|
322
|
+
if (brokerOwns() && (stateKey !== lastState || now - lastStatusAt >= timing.controlTickMs)) {
|
|
221
323
|
writeSandboxControlStatus(manifest, broker, state, reasonCode, active?.request.id ?? null, now);
|
|
222
324
|
lastStatusAt = now;
|
|
223
325
|
}
|
|
224
|
-
if (stateKey !== lastState) {
|
|
326
|
+
if (brokerOwns() && stateKey !== lastState) {
|
|
225
327
|
appendSandboxControlAudit(manifest, 'broker-state', { state, reasonCode, requestId: active?.request.id ?? null });
|
|
226
328
|
lastState = stateKey;
|
|
227
329
|
}
|
|
228
330
|
|
|
331
|
+
let retiring = false;
|
|
229
332
|
for (const name of fs.readdirSync(requestsDir).sort()) {
|
|
230
333
|
if (!/^[a-f0-9-]{16,64}\.json$/.test(name)) continue;
|
|
231
334
|
const id = name.slice(0, -5);
|
|
232
335
|
const source = path.join(requestsDir, name);
|
|
233
336
|
let claimed: string | null = null;
|
|
234
337
|
try {
|
|
338
|
+
if (!brokerOwns()) {
|
|
339
|
+
retiring = true;
|
|
340
|
+
break;
|
|
341
|
+
}
|
|
235
342
|
claimed = claimRequest(manifest, source, id);
|
|
343
|
+
if (!brokerOwns()) {
|
|
344
|
+
retiring = true;
|
|
345
|
+
break;
|
|
346
|
+
}
|
|
236
347
|
consumeRequest(consumedDir, id);
|
|
348
|
+
if (!brokerOwns()) {
|
|
349
|
+
retiring = true;
|
|
350
|
+
break;
|
|
351
|
+
}
|
|
237
352
|
if (reasonCode) throw new Error(reasonCode);
|
|
238
353
|
if (active) throw new Error('SANDBOX_CONTROL_BUSY');
|
|
239
354
|
const request = validateSandboxControlRequest(JSON.parse(fs.readFileSync(claimed, 'utf8')), manifest);
|
|
240
|
-
if (
|
|
355
|
+
if (bindingCheck(manifest)) throw new Error('SANDBOX_WORKTREE_BINDING_LOST');
|
|
241
356
|
if (readActiveLease(manifest)) throw new Error('SANDBOX_CONTROL_HANDOFF_ACTIVE');
|
|
242
357
|
const prepared = await prepareSandboxControlExecution({
|
|
243
358
|
manifest, manifestPath, request, requestPath: claimed, internalCliPath: process.argv[1]!
|
|
@@ -248,37 +363,82 @@ export async function serveSandboxControl(
|
|
|
248
363
|
(result) => { execution.result = result; execution.settled = true; },
|
|
249
364
|
(error) => { execution.failure = error; execution.settled = true; }
|
|
250
365
|
);
|
|
366
|
+
if (!brokerOwns()) {
|
|
367
|
+
prepared.terminate(false);
|
|
368
|
+
active = null;
|
|
369
|
+
retiring = true;
|
|
370
|
+
continue;
|
|
371
|
+
}
|
|
372
|
+
if (!brokerOwns()) {
|
|
373
|
+
prepared.terminate(false);
|
|
374
|
+
active = null;
|
|
375
|
+
retiring = true;
|
|
376
|
+
continue;
|
|
377
|
+
}
|
|
251
378
|
writeResponse(manifest, {
|
|
252
379
|
version: 2, id, phase: 'accepted', exitCode: null, stdout: '', stderr: '', error: null
|
|
253
380
|
});
|
|
381
|
+
if (!brokerOwns()) {
|
|
382
|
+
prepared.terminate(false);
|
|
383
|
+
active = null;
|
|
384
|
+
retiring = true;
|
|
385
|
+
break;
|
|
386
|
+
}
|
|
254
387
|
try {
|
|
255
|
-
prepared.start();
|
|
388
|
+
prepared.start(brokerOwns);
|
|
256
389
|
} catch (error) {
|
|
257
|
-
|
|
390
|
+
const owned = brokerOwns();
|
|
391
|
+
prepared.terminate(owned);
|
|
392
|
+
if (!owned || !brokerOwns()) {
|
|
393
|
+
active = null;
|
|
394
|
+
retiring = true;
|
|
395
|
+
break;
|
|
396
|
+
}
|
|
258
397
|
writeResponse(manifest, unknown(id));
|
|
398
|
+
if (!brokerOwns()) {
|
|
399
|
+
active = null;
|
|
400
|
+
retiring = true;
|
|
401
|
+
break;
|
|
402
|
+
}
|
|
259
403
|
fs.rmSync(path.join(manifest.processingDir, id), { recursive: true, force: true });
|
|
260
404
|
active = null;
|
|
261
405
|
claimed = null;
|
|
406
|
+
if (!brokerOwns()) {
|
|
407
|
+
retiring = true;
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
262
410
|
appendSandboxControlAudit(manifest, 'executor-gate-failed', { requestId: id });
|
|
263
411
|
continue;
|
|
264
412
|
}
|
|
265
413
|
} catch (error) {
|
|
414
|
+
if (!brokerOwns()) {
|
|
415
|
+
retiring = true;
|
|
416
|
+
break;
|
|
417
|
+
}
|
|
266
418
|
writeResponse(manifest, rejected(id, error));
|
|
419
|
+
if (!brokerOwns()) {
|
|
420
|
+
retiring = true;
|
|
421
|
+
break;
|
|
422
|
+
}
|
|
267
423
|
if (claimed) fs.rmSync(path.dirname(claimed), { recursive: true, force: true });
|
|
268
424
|
}
|
|
269
425
|
}
|
|
270
|
-
|
|
426
|
+
if (retiring || !brokerOwns()) break;
|
|
427
|
+
await delay(timing.controlTickMs);
|
|
271
428
|
}
|
|
272
429
|
} finally {
|
|
273
430
|
if (active) {
|
|
274
|
-
|
|
275
|
-
|
|
431
|
+
const owned = brokerOwns();
|
|
432
|
+
active.prepared.terminate(owned);
|
|
433
|
+
if (owned && brokerOwns()) writeResponse(manifest, unknown(active.request.id));
|
|
276
434
|
}
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
435
|
+
if (brokerOwns()) {
|
|
436
|
+
appendSandboxControlAudit(manifest, 'broker-stop', { pid: broker.pid, brokerId: broker.brokerId });
|
|
437
|
+
try {
|
|
438
|
+
if (fs.readFileSync(brokerPath, 'utf8') === brokerRecord) fs.unlinkSync(brokerPath);
|
|
439
|
+
} catch {
|
|
440
|
+
// A newer owner or recovery path owns the record.
|
|
441
|
+
}
|
|
282
442
|
}
|
|
283
443
|
}
|
|
284
444
|
}
|
|
@@ -4,6 +4,7 @@ import path from 'node:path';
|
|
|
4
4
|
import { randomUUID } from 'node:crypto';
|
|
5
5
|
import { buildProcessTreeStopCommand } from '../../server/process-control.ts';
|
|
6
6
|
import { processIdentityMatches } from '../../server/process-state.ts';
|
|
7
|
+
import type { ProcessIdentity } from '../../server/process-state.ts';
|
|
7
8
|
import type {
|
|
8
9
|
SandboxControlExecution,
|
|
9
10
|
SandboxControlLease,
|
|
@@ -41,14 +42,14 @@ export function statusPath(manifest: SandboxControlManifest): string {
|
|
|
41
42
|
|
|
42
43
|
export function writeSandboxControlStatus(
|
|
43
44
|
manifest: SandboxControlManifest,
|
|
44
|
-
broker:
|
|
45
|
+
broker: ProcessIdentity & { brokerId: string },
|
|
45
46
|
state: SandboxControlStatus['state'],
|
|
46
47
|
reasonCode: string | null,
|
|
47
48
|
activeRequestId: string | null,
|
|
48
49
|
now = Date.now()
|
|
49
50
|
): SandboxControlStatus {
|
|
50
51
|
const status: SandboxControlStatus = {
|
|
51
|
-
version:
|
|
52
|
+
version: 2, generation: manifest.generation, broker, state, reasonCode, activeRequestId, updatedAt: now
|
|
52
53
|
};
|
|
53
54
|
atomicWriteJson(statusPath(manifest), status, 0o600, false);
|
|
54
55
|
return status;
|
|
@@ -56,8 +57,10 @@ export function writeSandboxControlStatus(
|
|
|
56
57
|
|
|
57
58
|
export function parseSandboxControlStatus(value: unknown): SandboxControlStatus {
|
|
58
59
|
const status = value as Partial<SandboxControlStatus> | null;
|
|
59
|
-
if (!status || status.version !==
|
|
60
|
-
|| !status.broker || !Number.isSafeInteger(status.broker.pid) || typeof status.broker.startTime !== '
|
|
60
|
+
if (!status || status.version !== 2 || typeof status.generation !== 'string'
|
|
61
|
+
|| !status.broker || !Number.isSafeInteger(status.broker.pid) || typeof status.broker.startTime !== 'number'
|
|
62
|
+
|| !Number.isSafeInteger(status.broker.startTime) || typeof status.broker.brokerId !== 'string'
|
|
63
|
+
|| status.broker.brokerId.length === 0
|
|
61
64
|
|| !['starting', 'healthy', 'busy', 'parked'].includes(status.state ?? '')
|
|
62
65
|
|| (status.reasonCode !== null && typeof status.reasonCode !== 'string')
|
|
63
66
|
|| (status.activeRequestId !== null && typeof status.activeRequestId !== 'string')
|
|
@@ -73,9 +76,10 @@ export function readActiveLease(manifest: SandboxControlManifest, now = Date.now
|
|
|
73
76
|
const filePath = path.join(path.dirname(manifest.publicStatusDir), 'lease.json');
|
|
74
77
|
if (!fs.existsSync(filePath)) return null;
|
|
75
78
|
const lease = readJsonFile(filePath) as Partial<SandboxControlLease> | null;
|
|
76
|
-
if (!lease || lease.version !==
|
|
79
|
+
if (!lease || lease.version !== 2
|
|
77
80
|
|| typeof lease.nonce !== 'string' || !lease.owner || !Number.isSafeInteger(lease.owner.pid)
|
|
78
|
-
|| typeof lease.owner.startTime !== '
|
|
81
|
+
|| typeof lease.owner.startTime !== 'number' || !Number.isSafeInteger(lease.owner.startTime)
|
|
82
|
+
|| !Number.isSafeInteger(lease.issuedAt)
|
|
79
83
|
|| !Number.isSafeInteger(lease.expiresAt) || typeof lease.generation !== 'string'
|
|
80
84
|
|| (lease.taskId !== null && typeof lease.taskId !== 'string')
|
|
81
85
|
|| typeof lease.branch !== 'string' || typeof lease.reason !== 'string') {
|
|
@@ -91,9 +95,10 @@ export function cleanupStaleSandboxControlLease(manifest: SandboxControlManifest
|
|
|
91
95
|
if (!fs.existsSync(filePath)) return false;
|
|
92
96
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
93
97
|
const lease = JSON.parse(raw) as Partial<SandboxControlLease> | null;
|
|
94
|
-
if (!lease || lease.version !==
|
|
98
|
+
if (!lease || lease.version !== 2
|
|
95
99
|
|| typeof lease.nonce !== 'string' || !lease.owner || !Number.isSafeInteger(lease.owner.pid)
|
|
96
|
-
|| typeof lease.owner.startTime !== '
|
|
100
|
+
|| typeof lease.owner.startTime !== 'number' || !Number.isSafeInteger(lease.owner.startTime)
|
|
101
|
+
|| !Number.isSafeInteger(lease.issuedAt)
|
|
97
102
|
|| !Number.isSafeInteger(lease.expiresAt) || typeof lease.generation !== 'string'
|
|
98
103
|
|| (lease.taskId !== null && typeof lease.taskId !== 'string')
|
|
99
104
|
|| typeof lease.branch !== 'string' || typeof lease.reason !== 'string') {
|
|
@@ -101,7 +106,7 @@ export function cleanupStaleSandboxControlLease(manifest: SandboxControlManifest
|
|
|
101
106
|
}
|
|
102
107
|
const stale = lease.generation !== manifest.generation || lease.taskId !== manifest.taskId
|
|
103
108
|
|| lease.branch !== manifest.branch || lease.expiresAt! <= now || !processIdentityMatches({
|
|
104
|
-
|
|
109
|
+
pid: lease.owner.pid!, startTime: lease.owner.startTime!
|
|
105
110
|
});
|
|
106
111
|
if (!stale) return false;
|
|
107
112
|
if (fs.readFileSync(filePath, 'utf8') !== raw) throw new Error('SANDBOX_CONTROL_LEASE_OWNERSHIP_LOST');
|
|
@@ -131,9 +136,10 @@ export function executionPath(manifest: SandboxControlManifest, requestId: strin
|
|
|
131
136
|
|
|
132
137
|
export function readExecution(filePath: string): SandboxControlExecution {
|
|
133
138
|
const execution = readJsonFile(filePath) as Partial<SandboxControlExecution> | null;
|
|
134
|
-
if (!execution || execution.version !==
|
|
139
|
+
if (!execution || execution.version !== 2 || typeof execution.generation !== 'string'
|
|
135
140
|
|| typeof execution.requestId !== 'string' || typeof execution.nonce !== 'string'
|
|
136
|
-
|| !execution.child || !Number.isSafeInteger(execution.child.pid) || typeof execution.child.startTime !== '
|
|
141
|
+
|| !execution.child || !Number.isSafeInteger(execution.child.pid) || typeof execution.child.startTime !== 'number'
|
|
142
|
+
|| !Number.isSafeInteger(execution.child.startTime)
|
|
137
143
|
|| !['prepared', 'running', 'terminating'].includes(execution.phase ?? '')
|
|
138
144
|
|| !Number.isSafeInteger(execution.updatedAt)) throw new Error('SANDBOX_CONTROL_EXECUTION_INVALID');
|
|
139
145
|
return execution as SandboxControlExecution;
|
|
@@ -141,7 +147,13 @@ export function readExecution(filePath: string): SandboxControlExecution {
|
|
|
141
147
|
|
|
142
148
|
export function terminateSandboxControlExecution(
|
|
143
149
|
execution: SandboxControlExecution,
|
|
144
|
-
options: {
|
|
150
|
+
options: {
|
|
151
|
+
platform?: NodeJS.Platform;
|
|
152
|
+
timeoutMs?: number;
|
|
153
|
+
deadlineAt?: number;
|
|
154
|
+
forceAt?: number;
|
|
155
|
+
allowForce?: boolean;
|
|
156
|
+
} = {}
|
|
145
157
|
): boolean {
|
|
146
158
|
const platform = options.platform ?? process.platform;
|
|
147
159
|
const processGroupId = execution.child.processGroupId;
|
|
@@ -163,7 +175,7 @@ export function terminateSandboxControlExecution(
|
|
|
163
175
|
};
|
|
164
176
|
const treeAlive = (): boolean => {
|
|
165
177
|
if (platform === 'win32' || !processGroupId) {
|
|
166
|
-
return processIdentityMatches({
|
|
178
|
+
return processIdentityMatches({ pid: execution.child.pid, startTime: execution.child.startTime });
|
|
167
179
|
}
|
|
168
180
|
return processGroupAlive(processGroupId);
|
|
169
181
|
};
|
|
@@ -175,18 +187,23 @@ export function terminateSandboxControlExecution(
|
|
|
175
187
|
} catch {
|
|
176
188
|
// The identity check below distinguishes an already-exited tree from a failed termination.
|
|
177
189
|
}
|
|
178
|
-
const deadline = Date.now() + (options.timeoutMs ?? SANDBOX_CONTROL_EXECUTION_STOP_MS);
|
|
179
|
-
|
|
190
|
+
const deadline = options.deadlineAt ?? (Date.now() + (options.timeoutMs ?? SANDBOX_CONTROL_EXECUTION_STOP_MS));
|
|
191
|
+
const forceAt = options.forceAt ?? (options.deadlineAt === undefined
|
|
192
|
+
? deadline
|
|
193
|
+
: Math.min(deadline, Date.now() + Math.floor(Math.max(0, deadline - Date.now()) / 2)));
|
|
194
|
+
while (Date.now() < forceAt) {
|
|
180
195
|
if (!treeAlive()) return true;
|
|
181
196
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
|
|
182
197
|
}
|
|
183
|
-
if (platform !== 'win32'
|
|
198
|
+
if (platform !== 'win32' && options.allowForce !== false
|
|
199
|
+
&& (options.deadlineAt === undefined || Date.now() < deadline)) {
|
|
184
200
|
try {
|
|
185
201
|
process.kill(processGroupId ? -processGroupId : execution.child.pid, 'SIGKILL');
|
|
186
202
|
} catch {
|
|
187
203
|
// The final identity check handles an already-exited tree.
|
|
188
204
|
}
|
|
189
|
-
const killDeadline =
|
|
205
|
+
const killDeadline = options.deadlineAt
|
|
206
|
+
?? (Date.now() + (options.timeoutMs ?? SANDBOX_CONTROL_EXECUTION_STOP_MS));
|
|
190
207
|
while (Date.now() < killDeadline) {
|
|
191
208
|
if (!treeAlive()) return true;
|
|
192
209
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export type SandboxControlBackoffStep = Readonly<{
|
|
2
|
+
delayMs: number;
|
|
3
|
+
nextDelayMs: number;
|
|
4
|
+
}>;
|
|
5
|
+
|
|
6
|
+
export function nextSandboxControlBackoff(currentDelayMs: number, capMs: number): SandboxControlBackoffStep {
|
|
7
|
+
const delayMs = Math.min(currentDelayMs, capMs);
|
|
8
|
+
return {
|
|
9
|
+
delayMs,
|
|
10
|
+
nextDelayMs: Math.min(capMs, delayMs * 2)
|
|
11
|
+
};
|
|
12
|
+
}
|
package/lib/sandbox/mounts.ts
CHANGED
|
@@ -6,7 +6,6 @@ import {
|
|
|
6
6
|
worktreeDirCandidates
|
|
7
7
|
} from './constants.ts';
|
|
8
8
|
import type { SandboxConfig } from './config.ts';
|
|
9
|
-
import { sandboxWorkspaceViewStatePaths } from './workspace-view.ts';
|
|
10
9
|
|
|
11
10
|
export type SandboxBindMountDeclaration = {
|
|
12
11
|
hostPaths: string[];
|
|
@@ -31,19 +30,6 @@ export function sandboxCoreBindMounts(
|
|
|
31
30
|
}
|
|
32
31
|
): SandboxBindMountDeclaration[] {
|
|
33
32
|
const taskBound = Boolean(overrides.taskSource && overrides.taskId);
|
|
34
|
-
const workspaceMounts = sandboxWorkspaceViewStatePaths(overrides.workspaceViewRoot).map(
|
|
35
|
-
({ state, hostPath }) => taskBound && state === 'active'
|
|
36
|
-
? {
|
|
37
|
-
hostPaths: [path.join(hostPath, '.short-ids.json')],
|
|
38
|
-
containerPath: '/workspace/.agents/workspace/active/.short-ids.json',
|
|
39
|
-
readOnly: true
|
|
40
|
-
}
|
|
41
|
-
: {
|
|
42
|
-
hostPaths: [hostPath],
|
|
43
|
-
containerPath: path.posix.join('/workspace/.agents/workspace', state),
|
|
44
|
-
readOnly: true
|
|
45
|
-
}
|
|
46
|
-
);
|
|
47
33
|
const mounts: SandboxBindMountDeclaration[] = [
|
|
48
34
|
{
|
|
49
35
|
hostPaths: overrides.worktree
|
|
@@ -52,7 +38,11 @@ export function sandboxCoreBindMounts(
|
|
|
52
38
|
containerPath: '/workspace',
|
|
53
39
|
readOnly: false
|
|
54
40
|
},
|
|
55
|
-
|
|
41
|
+
{
|
|
42
|
+
hostPaths: [overrides.workspaceViewRoot],
|
|
43
|
+
containerPath: '/workspace/.agents/workspace',
|
|
44
|
+
readOnly: true
|
|
45
|
+
},
|
|
56
46
|
];
|
|
57
47
|
if (taskBound) {
|
|
58
48
|
mounts.push({
|