@fitlab-ai/agent-infra 0.9.6 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/internal-cli.js +5 -0
- package/dist/lib/agent-clients/adapters/codex-lifecycle/app-server.js +45 -4
- package/dist/lib/agent-clients/adapters/codex-lifecycle/build-identity.js +162 -0
- package/dist/lib/agent-clients/adapters/codex-lifecycle/capability-store.js +244 -0
- package/dist/lib/agent-clients/adapters/codex-lifecycle/sandbox-controller.js +370 -0
- package/dist/lib/agent-clients/adapters/codex-lifecycle/store.js +21 -5
- package/dist/lib/defaults.json +10 -2
- package/dist/lib/git/workflow.js +30 -3
- package/dist/lib/internal/codex-lifecycle.js +69 -1
- package/dist/lib/internal/codex-sandbox-controller.js +106 -0
- package/dist/lib/internal/task-orchestration.js +37 -5
- package/dist/lib/platform/context.js +6 -4
- package/dist/lib/platform/github-client.js +55 -2
- package/dist/lib/process-data/checkpoint.js +319 -0
- package/dist/lib/process-data/index.js +128 -60
- package/dist/lib/process-data/materialize.js +84 -0
- package/dist/lib/process-data/normalize.js +34 -1
- package/dist/lib/process-data/reconcile.js +55 -1
- package/dist/lib/process-data/sources.js +422 -1
- package/dist/lib/process-data/store.js +341 -11
- package/dist/lib/sandbox/commands/create.js +103 -51
- package/dist/lib/sandbox/commands/rm.js +64 -15
- package/dist/lib/sandbox/commands/start.js +4 -3
- package/dist/lib/sandbox/control/container-identity.js +64 -0
- package/dist/lib/sandbox/control/executor.js +13 -7
- package/dist/lib/sandbox/control/lifecycle.js +238 -29
- package/dist/lib/sandbox/control/protocol.js +7 -0
- package/dist/lib/sandbox/control/server.js +202 -26
- package/dist/lib/sandbox/control/state.js +25 -15
- package/dist/lib/sandbox/control/timing.js +8 -0
- package/dist/lib/sandbox/mounts.js +5 -13
- package/dist/lib/sandbox/recovery.js +59 -42
- package/dist/lib/sandbox/workspace-view.js +57 -40
- package/dist/lib/server/daemon.js +1 -1
- package/dist/lib/server/process-control.js +11 -1
- package/dist/lib/server/process-state.js +57 -20
- package/dist/lib/task/artifact-lifecycle.js +82 -11
- package/dist/lib/task/artifact-receipts.js +128 -0
- package/dist/lib/task/codex-orchestration.js +81 -6
- package/dist/lib/task/commands/validate.js +1 -1
- package/dist/lib/task/create.js +10 -2
- package/dist/lib/task/delegation-receipts.js +84 -2
- package/dist/lib/task/events.js +139 -2
- package/dist/lib/task/ledger.js +13 -1
- package/dist/lib/task/orchestration.js +235 -14
- package/dist/lib/task/sections.js +5 -1
- package/dist/lib/task/task-execution-lock.js +5 -4
- package/dist/lib/task/verification-engine.js +40 -8
- package/dist/lifecycle-build-manifest.json +6 -0
- package/lib/agent-clients/adapters/codex-lifecycle/app-server.ts +60 -3
- package/lib/agent-clients/adapters/codex-lifecycle/build-identity.ts +227 -0
- package/lib/agent-clients/adapters/codex-lifecycle/capability-store.ts +321 -0
- package/lib/agent-clients/adapters/codex-lifecycle/manifest-files.json +26 -0
- package/lib/agent-clients/adapters/codex-lifecycle/sandbox-controller.ts +453 -0
- package/lib/agent-clients/adapters/codex-lifecycle/store.ts +33 -4
- package/lib/defaults.json +10 -2
- package/lib/git/workflow.ts +25 -2
- package/lib/internal/codex-lifecycle.ts +68 -1
- package/lib/internal/codex-sandbox-controller.ts +109 -0
- package/lib/internal/task-orchestration.ts +31 -4
- package/lib/platform/context.ts +8 -5
- package/lib/platform/github-client.ts +61 -3
- package/lib/process-data/checkpoint.ts +335 -0
- package/lib/process-data/index.ts +112 -60
- package/lib/process-data/materialize.ts +83 -0
- package/lib/process-data/normalize.ts +29 -1
- package/lib/process-data/reconcile.ts +64 -2
- package/lib/process-data/sources.ts +449 -2
- package/lib/process-data/store.ts +304 -11
- package/lib/process-data/types.ts +124 -2
- package/lib/sandbox/commands/create.ts +56 -7
- package/lib/sandbox/commands/rm.ts +64 -14
- package/lib/sandbox/commands/start.ts +4 -3
- package/lib/sandbox/control/container-identity.ts +93 -0
- package/lib/sandbox/control/executor.ts +14 -10
- package/lib/sandbox/control/lifecycle.ts +264 -33
- package/lib/sandbox/control/protocol.ts +31 -5
- package/lib/sandbox/control/server.ts +184 -24
- package/lib/sandbox/control/state.ts +34 -17
- package/lib/sandbox/control/timing.ts +12 -0
- package/lib/sandbox/mounts.ts +5 -15
- package/lib/sandbox/recovery.ts +71 -42
- package/lib/sandbox/workspace-view.ts +61 -40
- package/lib/server/daemon.ts +1 -1
- package/lib/server/process-control.ts +13 -3
- package/lib/server/process-state.ts +82 -27
- package/lib/task/artifact-lifecycle.ts +73 -10
- package/lib/task/artifact-receipts.ts +161 -0
- package/lib/task/codex-orchestration.ts +91 -5
- package/lib/task/commands/validate.ts +1 -1
- package/lib/task/create.ts +11 -2
- package/lib/task/delegation-receipts.ts +160 -8
- package/lib/task/events.ts +122 -1
- package/lib/task/ledger.ts +16 -1
- package/lib/task/orchestration.ts +356 -17
- package/lib/task/sections.ts +6 -2
- package/lib/task/task-execution-lock.ts +11 -9
- package/lib/task/verification-engine.ts +51 -16
- package/package.json +1 -1
- package/templates/.agents/README.en.md +7 -0
- package/templates/.agents/README.zh-CN.md +5 -0
- package/templates/.agents/hooks/lifecycle-delegation.js +25 -1
- package/templates/.agents/rules/lifecycle-orchestration.en.md +8 -2
- package/templates/.agents/rules/lifecycle-orchestration.zh-CN.md +8 -2
- package/templates/.agents/rules/pr-checks-commands.github.en.md +1 -1
- package/templates/.agents/rules/pr-checks-commands.github.zh-CN.md +1 -1
- package/templates/.agents/rules/review-handshake.en.md +3 -1
- package/templates/.agents/rules/review-handshake.zh-CN.md +3 -1
- package/templates/.agents/skills/code-task/reference/dual-mode.en.md +3 -1
- package/templates/.agents/skills/code-task/reference/dual-mode.zh-CN.md +3 -1
- package/templates/.agents/skills/complete-task/SKILL.en.md +4 -0
- package/templates/.agents/skills/complete-task/SKILL.zh-CN.md +4 -0
- package/templates/.agents/skills/run-manual-validation/SKILL.en.md +6 -6
- package/templates/.agents/skills/run-manual-validation/SKILL.zh-CN.md +6 -6
- package/templates/.agents/skills/run-manual-validation/config/verify.en.json +14 -3
- package/templates/.agents/skills/run-manual-validation/config/verify.zh-CN.json +14 -3
- package/templates/.agents/skills/run-manual-validation/reference/discovery-and-execution.en.md +67 -0
- package/templates/.agents/skills/run-manual-validation/reference/discovery-and-execution.zh-CN.md +67 -0
- package/templates/.agents/skills/run-manual-validation/reference/report-template.en.md +25 -0
- package/templates/.agents/skills/run-manual-validation/reference/report-template.zh-CN.md +25 -0
- package/templates/.agents/skills/run-task/SKILL.en.md +9 -7
- package/templates/.agents/skills/run-task/SKILL.zh-CN.md +9 -7
- package/templates/.agents/skills/run-task/reference/host-validation.en.md +2 -0
- package/templates/.agents/skills/run-task/reference/host-validation.zh-CN.md +2 -0
- package/templates/.agents/skills/update-agent-infra/scripts/sync-templates.js +75 -3
- package/templates/.agents/workspace/README.en.md +0 -26
- package/templates/.agents/workspace/README.zh-CN.md +0 -26
|
@@ -3,24 +3,52 @@ import fs from 'node:fs';
|
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { buildProcessTreeStopCommand } from '../../server/process-control.ts';
|
|
5
5
|
import { processIdentityMatches } from '../../server/process-state.ts';
|
|
6
|
+
import type { ProcessIdentity } from '../../server/process-state.ts';
|
|
6
7
|
import type {
|
|
7
8
|
SandboxControlExecution,
|
|
8
9
|
SandboxControlManifest,
|
|
9
|
-
SandboxControlStatus
|
|
10
|
+
SandboxControlStatus,
|
|
11
|
+
SandboxControlTimingPolicy
|
|
10
12
|
} from './protocol.ts';
|
|
13
|
+
import { DEFAULT_SANDBOX_CONTROL_TIMING } from './protocol.ts';
|
|
14
|
+
import { inspectSandboxControlContainer, type ContainerObservation } from './container-identity.ts';
|
|
11
15
|
import {
|
|
12
16
|
parseSandboxControlStatus,
|
|
13
17
|
readExecution,
|
|
14
18
|
terminateSandboxControlExecution
|
|
15
19
|
} from './state.ts';
|
|
16
20
|
|
|
17
|
-
type OwnerIdentity = Readonly<{
|
|
18
|
-
type BrokerOwner =
|
|
21
|
+
type OwnerIdentity = ProcessIdentity & Readonly<{ brokerId?: string }>;
|
|
22
|
+
export type BrokerOwner = ProcessIdentity & Readonly<{ version: 3; brokerId: string; token: string; generation: string }>;
|
|
19
23
|
|
|
20
|
-
const DEFAULT_QUIESCE_TIMEOUT_MS =
|
|
24
|
+
const DEFAULT_QUIESCE_TIMEOUT_MS = DEFAULT_SANDBOX_CONTROL_TIMING.quiesceDeadlineMs;
|
|
21
25
|
const QUIESCING_FILE = 'quiescing.json';
|
|
22
26
|
const BROKER_STARTING_FILE = 'broker-starting.json';
|
|
23
27
|
|
|
28
|
+
async function awaitWithDeadline<T>(
|
|
29
|
+
operation: () => Promise<T>,
|
|
30
|
+
deadlineAt: number,
|
|
31
|
+
errorCode: string
|
|
32
|
+
): Promise<T> {
|
|
33
|
+
const remaining = deadlineAt - Date.now();
|
|
34
|
+
if (remaining <= 0) throw new Error(errorCode);
|
|
35
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
36
|
+
try {
|
|
37
|
+
return await Promise.race([
|
|
38
|
+
operation(),
|
|
39
|
+
new Promise<T>((_, reject) => {
|
|
40
|
+
timer = setTimeout(() => reject(new Error(errorCode)), remaining);
|
|
41
|
+
})
|
|
42
|
+
]);
|
|
43
|
+
} finally {
|
|
44
|
+
if (timer) clearTimeout(timer);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function forceDeadline(deadlineAt: number, timeoutMs: number): number {
|
|
49
|
+
return Math.max(Date.now(), deadlineAt - Math.max(1, Math.floor(timeoutMs / 2)));
|
|
50
|
+
}
|
|
51
|
+
|
|
24
52
|
function regularFile(filePath: string): boolean {
|
|
25
53
|
try {
|
|
26
54
|
const stat = fs.lstatSync(filePath);
|
|
@@ -54,8 +82,8 @@ function readStartupOwner(filePath: string): { raw: string; owner: OwnerIdentity
|
|
|
54
82
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
55
83
|
try {
|
|
56
84
|
const record = JSON.parse(raw) as Partial<OwnerIdentity> & { version?: unknown };
|
|
57
|
-
if (record.version !==
|
|
58
|
-
|| typeof record.startTime !== '
|
|
85
|
+
if (record.version !== 2 || !Number.isSafeInteger(record.pid) || (record.pid ?? 0) <= 0
|
|
86
|
+
|| typeof record.startTime !== 'number' || !Number.isSafeInteger(record.startTime)) return null;
|
|
59
87
|
return { raw, owner: { pid: record.pid!, startTime: record.startTime } };
|
|
60
88
|
} catch {
|
|
61
89
|
return null;
|
|
@@ -88,7 +116,7 @@ export async function acquireSandboxControlBrokerStartup(
|
|
|
88
116
|
): Promise<() => void> {
|
|
89
117
|
const resolvedRoot = path.resolve(root);
|
|
90
118
|
const transitionPath = path.join(resolvedRoot, BROKER_STARTING_FILE);
|
|
91
|
-
const record = `${JSON.stringify({ version:
|
|
119
|
+
const record = `${JSON.stringify({ version: 2, ...owner })}\n`;
|
|
92
120
|
const deadline = Date.now() + timeoutMs;
|
|
93
121
|
while (Date.now() < deadline) {
|
|
94
122
|
if (isSandboxControlRootQuiescing(resolvedRoot)) throw new Error('SANDBOX_CONTROL_QUIESCING');
|
|
@@ -134,11 +162,17 @@ export function readSandboxControlManifest(manifestPath: string): SandboxControl
|
|
|
134
162
|
} catch {
|
|
135
163
|
throw new Error('SANDBOX_CONTROL_MANIFEST_INVALID');
|
|
136
164
|
}
|
|
137
|
-
if (manifest.version !==
|
|
138
|
-
throw new Error('SANDBOX_CONTROL_MANIFEST_VERSION_INVALID: expected version
|
|
165
|
+
if (manifest.version !== 4) {
|
|
166
|
+
throw new Error('SANDBOX_CONTROL_MANIFEST_VERSION_INVALID: expected version 4; container-only recreation is required');
|
|
139
167
|
}
|
|
140
|
-
if (typeof manifest.
|
|
168
|
+
if (typeof manifest.engine !== 'string' || manifest.engine.length === 0
|
|
169
|
+
|| typeof manifest.repoRoot !== 'string' || typeof manifest.worktreeRoot !== 'string'
|
|
141
170
|
|| typeof manifest.project !== 'string' || typeof manifest.container !== 'string'
|
|
171
|
+
|| !manifest.containerIdentity || typeof manifest.containerIdentity.id !== 'string'
|
|
172
|
+
|| manifest.containerIdentity.id.length === 0 || !manifest.containerIdentity.labels
|
|
173
|
+
|| typeof manifest.containerIdentity.labels !== 'object'
|
|
174
|
+
|| Array.isArray(manifest.containerIdentity.labels)
|
|
175
|
+
|| Object.values(manifest.containerIdentity.labels).some((value) => typeof value !== 'string')
|
|
142
176
|
|| typeof manifest.branch !== 'string' || !['task-bound', 'branch-only'].includes(manifest.mode ?? '')
|
|
143
177
|
|| (manifest.taskId !== null && typeof manifest.taskId !== 'string')
|
|
144
178
|
|| typeof manifest.channelDir !== 'string' || typeof manifest.publicStatusDir !== 'string'
|
|
@@ -162,9 +196,10 @@ function readBrokerOwner(filePath: string): BrokerOwner | null {
|
|
|
162
196
|
} catch {
|
|
163
197
|
throw new Error('SANDBOX_CONTROL_OWNER_MISMATCH');
|
|
164
198
|
}
|
|
165
|
-
if (!owner || owner.version !==
|
|
166
|
-
|| typeof owner.startTime !== '
|
|
167
|
-
|| typeof owner.
|
|
199
|
+
if (!owner || owner.version !== 3 || !Number.isSafeInteger(owner.pid) || (owner.pid ?? 0) <= 0
|
|
200
|
+
|| typeof owner.startTime !== 'number' || !Number.isSafeInteger(owner.startTime)
|
|
201
|
+
|| typeof owner.brokerId !== 'string' || owner.brokerId.length === 0
|
|
202
|
+
|| typeof owner.token !== 'string' || typeof owner.generation !== 'string') throw new Error('SANDBOX_CONTROL_OWNER_MISMATCH');
|
|
168
203
|
return owner as BrokerOwner;
|
|
169
204
|
}
|
|
170
205
|
|
|
@@ -179,11 +214,12 @@ function readStatusOwner(filePath: string): SandboxControlStatus | null {
|
|
|
179
214
|
}
|
|
180
215
|
|
|
181
216
|
function sameOwner(left: OwnerIdentity, right: OwnerIdentity): boolean {
|
|
182
|
-
return left.pid === right.pid && left.startTime === right.startTime
|
|
217
|
+
return left.pid === right.pid && left.startTime === right.startTime
|
|
218
|
+
&& (!left.brokerId || !right.brokerId || left.brokerId === right.brokerId);
|
|
183
219
|
}
|
|
184
220
|
|
|
185
221
|
function ownerLive(owner: OwnerIdentity): boolean {
|
|
186
|
-
return processIdentityMatches({
|
|
222
|
+
return processIdentityMatches({ pid: owner.pid, startTime: owner.startTime });
|
|
187
223
|
}
|
|
188
224
|
|
|
189
225
|
function readExecutions(manifest: SandboxControlManifest): SandboxControlExecution[] {
|
|
@@ -234,11 +270,11 @@ function executionAlive(execution: SandboxControlExecution, platform: NodeJS.Pla
|
|
|
234
270
|
if (platform !== 'win32' && execution.child.processGroupId) {
|
|
235
271
|
return processGroupAlive(execution.child.processGroupId);
|
|
236
272
|
}
|
|
237
|
-
return processIdentityMatches({
|
|
273
|
+
return processIdentityMatches({ pid: execution.child.pid, startTime: execution.child.startTime });
|
|
238
274
|
}
|
|
239
275
|
|
|
240
|
-
async function waitForExit(owner: OwnerIdentity, timeoutMs: number): Promise<boolean> {
|
|
241
|
-
const deadline = Date.now() + timeoutMs;
|
|
276
|
+
async function waitForExit(owner: OwnerIdentity, timeoutMs: number, deadlineAt?: number): Promise<boolean> {
|
|
277
|
+
const deadline = deadlineAt ?? (Date.now() + timeoutMs);
|
|
242
278
|
while (Date.now() < deadline) {
|
|
243
279
|
if (!ownerLive(owner)) return true;
|
|
244
280
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
@@ -267,7 +303,7 @@ function mergeExecutions(...groups: readonly SandboxControlExecution[][]): Sandb
|
|
|
267
303
|
|
|
268
304
|
export async function quiesceSandboxControlRoot(
|
|
269
305
|
root: string,
|
|
270
|
-
options: { platform?: NodeJS.Platform; timeoutMs?: number } = {}
|
|
306
|
+
options: { platform?: NodeJS.Platform; timeoutMs?: number; timing?: SandboxControlTimingPolicy } = {}
|
|
271
307
|
): Promise<'missing' | 'stale' | 'stopped'> {
|
|
272
308
|
const resolvedRoot = path.resolve(root);
|
|
273
309
|
if (!fs.existsSync(resolvedRoot)) return 'missing';
|
|
@@ -280,14 +316,17 @@ export async function quiesceSandboxControlRoot(
|
|
|
280
316
|
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('SANDBOX_CONTROL_CHANNEL_INVALID');
|
|
281
317
|
}
|
|
282
318
|
|
|
319
|
+
const timeoutMs = options.timeoutMs ?? options.timing?.quiesceDeadlineMs ?? DEFAULT_QUIESCE_TIMEOUT_MS;
|
|
320
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
321
|
+
const forceAt = forceDeadline(deadlineAt, timeoutMs);
|
|
283
322
|
markSandboxControlRootQuiescing(resolvedRoot);
|
|
284
|
-
await waitForStartupTransition(resolvedRoot,
|
|
323
|
+
await waitForStartupTransition(resolvedRoot, Math.max(0, forceAt - Date.now()));
|
|
285
324
|
|
|
286
325
|
const manifestPath = path.join(resolvedRoot, 'manifest.json');
|
|
287
326
|
const manifest = fs.existsSync(manifestPath) ? readSandboxControlManifest(manifestPath) : null;
|
|
288
327
|
const broker = readBrokerOwner(path.join(resolvedRoot, 'broker.json'));
|
|
289
328
|
const status = readStatusOwner(path.join(resolvedRoot, 'public', 'status.json'));
|
|
290
|
-
const statusOwner = status ?
|
|
329
|
+
const statusOwner = status ? status.broker : null;
|
|
291
330
|
const brokerLive = broker ? ownerLive(broker) : false;
|
|
292
331
|
const statusLive = statusOwner ? ownerLive(statusOwner) : false;
|
|
293
332
|
|
|
@@ -303,13 +342,24 @@ export async function quiesceSandboxControlRoot(
|
|
|
303
342
|
if (!manifest && brokerLive) throw new Error('SANDBOX_CONTROL_OWNER_MISMATCH');
|
|
304
343
|
|
|
305
344
|
const platform = options.platform ?? process.platform;
|
|
306
|
-
const timeoutMs = options.timeoutMs ?? DEFAULT_QUIESCE_TIMEOUT_MS;
|
|
307
345
|
let executions = manifest ? readExecutions(manifest) : [];
|
|
308
346
|
const owner: OwnerIdentity | null = brokerLive ? broker : statusLive ? statusOwner : null;
|
|
309
347
|
if (!owner) {
|
|
310
348
|
if (manifest && !broker && !status) throw new Error('SANDBOX_CONTROL_OWNER_EVIDENCE_MISSING');
|
|
311
349
|
for (const execution of executions) {
|
|
312
|
-
|
|
350
|
+
terminateSandboxControlExecution(execution, { platform, timeoutMs: 0, allowForce: false });
|
|
351
|
+
}
|
|
352
|
+
const softDeadlineAt = Math.min(forceAt, Date.now() + Math.floor(Math.max(0, forceAt - Date.now()) / 2));
|
|
353
|
+
while (Date.now() < softDeadlineAt) {
|
|
354
|
+
if (!executions.some((execution) => executionAlive(execution, platform))) break;
|
|
355
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
356
|
+
}
|
|
357
|
+
if (manifest) executions = mergeExecutions(executions, readExecutions(manifest));
|
|
358
|
+
if (Date.now() >= deadlineAt) throw new Error('SANDBOX_CONTROL_QUIESCE_DEADLINE_EXCEEDED');
|
|
359
|
+
for (const execution of executions) {
|
|
360
|
+
if (!terminateSandboxControlExecution(execution, {
|
|
361
|
+
platform, timeoutMs: Math.max(0, deadlineAt - Date.now()), deadlineAt, forceAt: Date.now()
|
|
362
|
+
})) {
|
|
313
363
|
throw new Error(`SANDBOX_CONTROL_EXECUTION_STILL_RUNNING: ${execution.requestId}`);
|
|
314
364
|
}
|
|
315
365
|
}
|
|
@@ -319,24 +369,205 @@ export async function quiesceSandboxControlRoot(
|
|
|
319
369
|
return broker || status ? 'stale' : 'missing';
|
|
320
370
|
}
|
|
321
371
|
|
|
372
|
+
const remaining = (): number => Math.max(0, deadlineAt - Date.now());
|
|
322
373
|
signalOwner(owner, platform, false);
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
374
|
+
for (const execution of executions) {
|
|
375
|
+
terminateSandboxControlExecution(execution, { platform, timeoutMs: 0, allowForce: false });
|
|
376
|
+
}
|
|
377
|
+
const softDeadlineAt = Math.min(forceAt, Date.now() + Math.floor(Math.max(0, forceAt - Date.now()) / 2));
|
|
378
|
+
while (Date.now() < softDeadlineAt) {
|
|
379
|
+
if (!ownerLive(owner) && !executions.some((execution) => executionAlive(execution, platform))) break;
|
|
380
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
381
|
+
}
|
|
382
|
+
if (!ownerLive(owner) && !executions.some((execution) => executionAlive(execution, platform))) {
|
|
327
383
|
return 'stopped';
|
|
328
384
|
}
|
|
329
|
-
|
|
330
385
|
if (manifest) executions = mergeExecutions(executions, readExecutions(manifest));
|
|
386
|
+
if (Date.now() >= deadlineAt) throw new Error('SANDBOX_CONTROL_QUIESCE_DEADLINE_EXCEEDED');
|
|
331
387
|
for (const execution of executions) {
|
|
332
|
-
|
|
388
|
+
terminateSandboxControlExecution(execution, { platform, timeoutMs: 0, deadlineAt, forceAt: Date.now() });
|
|
389
|
+
}
|
|
390
|
+
signalOwner(owner, platform, true);
|
|
391
|
+
while (Date.now() < deadlineAt) {
|
|
392
|
+
if (!ownerLive(owner) && !executions.some((execution) => executionAlive(execution, platform))) return 'stopped';
|
|
393
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
394
|
+
}
|
|
395
|
+
if (ownerLive(owner)) throw new Error('SANDBOX_CONTROL_BROKER_STILL_RUNNING');
|
|
396
|
+
if (executions.some((execution) => executionAlive(execution, platform))) throw new Error('SANDBOX_CONTROL_EXECUTION_STILL_RUNNING');
|
|
397
|
+
return 'stopped';
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
export type RemoveSandboxControlOptions = Readonly<{
|
|
401
|
+
platform?: NodeJS.Platform;
|
|
402
|
+
timeoutMs?: number;
|
|
403
|
+
timing?: SandboxControlTimingPolicy;
|
|
404
|
+
inspectContainer?: (timeoutMs: number) => Promise<ContainerObservation>;
|
|
405
|
+
removeContainer: (timeoutMs: number) => Promise<void>;
|
|
406
|
+
requireAbsent?: boolean;
|
|
407
|
+
selfOwner?: BrokerOwner;
|
|
408
|
+
}>;
|
|
409
|
+
|
|
410
|
+
export async function removeSandboxControlRoot(
|
|
411
|
+
root: string,
|
|
412
|
+
options: RemoveSandboxControlOptions
|
|
413
|
+
): Promise<void> {
|
|
414
|
+
const resolvedRoot = path.resolve(root);
|
|
415
|
+
if (!fs.existsSync(resolvedRoot)) return;
|
|
416
|
+
const stat = fs.lstatSync(resolvedRoot);
|
|
417
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('SANDBOX_CONTROL_CHANNEL_INVALID');
|
|
418
|
+
const manifestPath = path.join(resolvedRoot, 'manifest.json');
|
|
419
|
+
const manifest = readSandboxControlManifest(manifestPath);
|
|
420
|
+
const inspectContainer = options.inspectContainer
|
|
421
|
+
?? ((timeoutMs) => inspectSandboxControlContainer(manifest, { timeoutMs }));
|
|
422
|
+
const timeoutMs = options.timeoutMs ?? options.timing?.quiesceDeadlineMs ?? DEFAULT_QUIESCE_TIMEOUT_MS;
|
|
423
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
424
|
+
const forceAt = forceDeadline(deadlineAt, timeoutMs);
|
|
425
|
+
const remaining = (): number => Math.max(0, deadlineAt - Date.now());
|
|
426
|
+
const observation = await awaitWithDeadline(
|
|
427
|
+
() => inspectContainer(remaining()), deadlineAt, 'SANDBOX_CONTROL_REMOVE_DEADLINE_EXCEEDED'
|
|
428
|
+
);
|
|
429
|
+
if (observation.state === 'unknown') throw new Error(`SANDBOX_CONTROL_CONTAINER_UNKNOWN: ${observation.reason}`);
|
|
430
|
+
if (options.requireAbsent && observation.state !== 'absent') {
|
|
431
|
+
throw new Error('SANDBOX_CONTROL_CONTAINER_REAPPEARED');
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
markSandboxControlRootQuiescing(resolvedRoot);
|
|
435
|
+
await waitForStartupTransition(resolvedRoot, Math.max(0, forceAt - Date.now()));
|
|
436
|
+
const brokerPath = path.join(resolvedRoot, 'broker.json');
|
|
437
|
+
const statusPath = path.join(resolvedRoot, 'public', 'status.json');
|
|
438
|
+
const broker = readBrokerOwner(brokerPath);
|
|
439
|
+
const status = readStatusOwner(statusPath);
|
|
440
|
+
if (broker && (broker.token !== manifest.token || broker.generation !== manifest.generation)) {
|
|
441
|
+
throw new Error('SANDBOX_CONTROL_OWNER_MISMATCH');
|
|
442
|
+
}
|
|
443
|
+
if (status && status.generation !== manifest.generation) {
|
|
444
|
+
throw new Error('SANDBOX_CONTROL_OWNER_MISMATCH');
|
|
445
|
+
}
|
|
446
|
+
if (broker && status && !sameOwner(broker, status.broker)
|
|
447
|
+
&& (ownerLive(broker) || ownerLive(status.broker))) {
|
|
448
|
+
throw new Error('SANDBOX_CONTROL_OWNER_MISMATCH');
|
|
449
|
+
}
|
|
450
|
+
const platform = options.platform ?? process.platform;
|
|
451
|
+
const owner = broker && ownerLive(broker) ? broker
|
|
452
|
+
: status && ownerLive(status.broker) ? status.broker : null;
|
|
453
|
+
const selfOwner = options.selfOwner;
|
|
454
|
+
const selfOwned = Boolean(selfOwner && broker && sameOwner(broker, selfOwner));
|
|
455
|
+
if (selfOwner && (!broker || !selfOwned || broker.token !== manifest.token || broker.generation !== manifest.generation)) {
|
|
456
|
+
throw new Error('SANDBOX_CONTROL_OWNER_TRANSITION');
|
|
457
|
+
}
|
|
458
|
+
let executions = readExecutions(manifest);
|
|
459
|
+
if (owner && !selfOwned) signalOwner(owner, platform, false);
|
|
460
|
+
for (const execution of executions) {
|
|
461
|
+
terminateSandboxControlExecution(execution, { platform, timeoutMs: 0, allowForce: false });
|
|
462
|
+
}
|
|
463
|
+
const softDeadlineAt = Math.min(forceAt, Date.now() + Math.floor(Math.max(0, forceAt - Date.now()) / 2));
|
|
464
|
+
while (Date.now() < softDeadlineAt) {
|
|
465
|
+
if ((!owner || selfOwned || !ownerLive(owner)) && !executions.some((execution) => executionAlive(execution, platform))) break;
|
|
466
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if (observation.state === 'found') {
|
|
470
|
+
if (Date.now() >= deadlineAt) throw new Error('SANDBOX_CONTROL_REMOVE_DEADLINE_EXCEEDED');
|
|
471
|
+
await awaitWithDeadline(
|
|
472
|
+
() => options.removeContainer(remaining()),
|
|
473
|
+
deadlineAt,
|
|
474
|
+
'SANDBOX_CONTROL_REMOVE_DEADLINE_EXCEEDED'
|
|
475
|
+
);
|
|
476
|
+
const afterRemoval = await awaitWithDeadline(
|
|
477
|
+
() => inspectContainer(remaining()),
|
|
478
|
+
deadlineAt,
|
|
479
|
+
'SANDBOX_CONTROL_REMOVE_DEADLINE_EXCEEDED'
|
|
480
|
+
);
|
|
481
|
+
if (afterRemoval.state !== 'absent') {
|
|
482
|
+
throw new Error(afterRemoval.state === 'unknown'
|
|
483
|
+
? `SANDBOX_CONTROL_CONTAINER_UNKNOWN: ${afterRemoval.reason}`
|
|
484
|
+
: 'SANDBOX_CONTROL_CONTAINER_STILL_EXISTS');
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
const currentManifest = readSandboxControlManifest(manifestPath);
|
|
489
|
+
if (currentManifest.token !== manifest.token || currentManifest.generation !== manifest.generation
|
|
490
|
+
|| !isSandboxControlRootQuiescing(resolvedRoot)) {
|
|
491
|
+
throw new Error('SANDBOX_CONTROL_OWNER_TRANSITION');
|
|
492
|
+
}
|
|
493
|
+
const currentBroker = readBrokerOwner(brokerPath);
|
|
494
|
+
if (selfOwned && (!currentBroker || !selfOwner || !sameOwner(selfOwner, currentBroker)
|
|
495
|
+
|| currentBroker.token !== manifest.token || currentBroker.generation !== manifest.generation)) {
|
|
496
|
+
throw new Error('SANDBOX_CONTROL_OWNER_REPLACED');
|
|
497
|
+
}
|
|
498
|
+
if (owner && currentBroker && !sameOwner(owner, currentBroker)) {
|
|
499
|
+
throw new Error('SANDBOX_CONTROL_OWNER_REPLACED');
|
|
500
|
+
}
|
|
501
|
+
if (!owner && currentBroker && ownerLive(currentBroker)) {
|
|
502
|
+
throw new Error('SANDBOX_CONTROL_OWNER_REPLACED');
|
|
503
|
+
}
|
|
504
|
+
if (owner && ownerLive(owner) && !currentBroker) {
|
|
505
|
+
throw new Error('SANDBOX_CONTROL_OWNER_EVIDENCE_MISSING');
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
executions = mergeExecutions(executions, readExecutions(manifest));
|
|
509
|
+
if (Date.now() >= deadlineAt) throw new Error('SANDBOX_CONTROL_REMOVE_DEADLINE_EXCEEDED');
|
|
510
|
+
for (const execution of executions) {
|
|
511
|
+
if (!terminateSandboxControlExecution(execution, {
|
|
512
|
+
platform, timeoutMs: remaining(), deadlineAt, forceAt: Date.now()
|
|
513
|
+
})) {
|
|
333
514
|
throw new Error(`SANDBOX_CONTROL_EXECUTION_STILL_RUNNING: ${execution.requestId}`);
|
|
334
515
|
}
|
|
335
516
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
517
|
+
if (owner && currentBroker && ownerLive(currentBroker) && !selfOwned) {
|
|
518
|
+
if (remaining() <= 0) throw new Error('SANDBOX_CONTROL_REMOVE_DEADLINE_EXCEEDED');
|
|
519
|
+
signalOwner(currentBroker, platform, true);
|
|
520
|
+
if (remaining() <= 0 || !await waitForExit(currentBroker, remaining(), deadlineAt)) {
|
|
521
|
+
throw new Error('SANDBOX_CONTROL_BROKER_STILL_RUNNING');
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
const finalExecutions = mergeExecutions(executions, readExecutions(manifest));
|
|
526
|
+
if (finalExecutions.some((execution) => executionAlive(execution, platform))) {
|
|
339
527
|
throw new Error('SANDBOX_CONTROL_EXECUTION_STILL_RUNNING');
|
|
340
528
|
}
|
|
341
|
-
|
|
529
|
+
|
|
530
|
+
const finalBrokerRaw = fs.existsSync(brokerPath) ? fs.readFileSync(brokerPath, 'utf8') : null;
|
|
531
|
+
if (finalBrokerRaw !== null) {
|
|
532
|
+
const finalBroker = readBrokerOwner(brokerPath);
|
|
533
|
+
if (!finalBroker) throw new Error('SANDBOX_CONTROL_OWNER_MISMATCH');
|
|
534
|
+
const finalSelfOwner = Boolean(selfOwner && sameOwner(finalBroker, selfOwner)
|
|
535
|
+
&& finalBroker.token === manifest.token && finalBroker.generation === manifest.generation);
|
|
536
|
+
if (ownerLive(finalBroker) && !finalSelfOwner) throw new Error('SANDBOX_CONTROL_BROKER_STILL_RUNNING');
|
|
537
|
+
if (fs.readFileSync(brokerPath, 'utf8') !== finalBrokerRaw) {
|
|
538
|
+
throw new Error('SANDBOX_CONTROL_OWNER_TRANSITION');
|
|
539
|
+
}
|
|
540
|
+
fs.unlinkSync(brokerPath);
|
|
541
|
+
}
|
|
542
|
+
const finalStatusRaw = fs.existsSync(statusPath) ? fs.readFileSync(statusPath, 'utf8') : null;
|
|
543
|
+
if (finalStatusRaw !== null) {
|
|
544
|
+
const finalStatus = readStatusOwner(statusPath);
|
|
545
|
+
if (!finalStatus) throw new Error('SANDBOX_CONTROL_OWNER_MISMATCH');
|
|
546
|
+
if (ownerLive(finalStatus.broker) && !(selfOwner && sameOwner(finalStatus.broker, selfOwner))) {
|
|
547
|
+
throw new Error('SANDBOX_CONTROL_BROKER_STILL_RUNNING');
|
|
548
|
+
}
|
|
549
|
+
if (fs.readFileSync(statusPath, 'utf8') !== finalStatusRaw) {
|
|
550
|
+
throw new Error('SANDBOX_CONTROL_OWNER_TRANSITION');
|
|
551
|
+
}
|
|
552
|
+
fs.unlinkSync(statusPath);
|
|
553
|
+
}
|
|
554
|
+
if (fs.existsSync(brokerPath) || fs.existsSync(statusPath)) {
|
|
555
|
+
throw new Error('SANDBOX_CONTROL_OWNER_EVIDENCE_REMAINS');
|
|
556
|
+
}
|
|
557
|
+
fs.rmSync(resolvedRoot, { recursive: true, force: true });
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
export type GarbageCollectSandboxControlOptions = Omit<RemoveSandboxControlOptions, 'removeContainer' | 'requireAbsent'>;
|
|
561
|
+
|
|
562
|
+
export async function garbageCollectSandboxControlRoot(
|
|
563
|
+
root: string,
|
|
564
|
+
options: GarbageCollectSandboxControlOptions = {}
|
|
565
|
+
): Promise<void> {
|
|
566
|
+
await removeSandboxControlRoot(path.resolve(root), {
|
|
567
|
+
...options,
|
|
568
|
+
requireAbsent: true,
|
|
569
|
+
removeContainer: async () => {
|
|
570
|
+
throw new Error('SANDBOX_CONTROL_CONTAINER_REAPPEARED');
|
|
571
|
+
}
|
|
572
|
+
});
|
|
342
573
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { validateTaskCreateCandidate, type TaskCreateCandidateV1 } from '../../task/create.ts';
|
|
2
|
+
import type { ProcessIdentity } from '../../server/process-state.ts';
|
|
2
3
|
|
|
3
4
|
export const SANDBOX_CONTROL_MAX_BYTES = 64 * 1024;
|
|
4
5
|
export const SANDBOX_CONTROL_ADMISSION_WINDOW_MS = 2_000;
|
|
@@ -6,13 +7,38 @@ export const SANDBOX_CONTROL_STATUS_INTERVAL_MS = 250;
|
|
|
6
7
|
export const SANDBOX_CONTROL_STATUS_STALE_MS = 1_500;
|
|
7
8
|
export const SANDBOX_CONTROL_FUTURE_SKEW_MS = 1_000;
|
|
8
9
|
export const SANDBOX_CONTROL_FAMILIES = ['task-lifecycle', 'task-orchestration', 'task-create'] as const;
|
|
10
|
+
export type SandboxControlTimingPolicy = Readonly<{
|
|
11
|
+
controlTickMs: number;
|
|
12
|
+
parkedBindingInitialMs: number;
|
|
13
|
+
slowCheckMs: number;
|
|
14
|
+
containerHeartbeatMs: number;
|
|
15
|
+
quiesceDeadlineMs: number;
|
|
16
|
+
}>;
|
|
17
|
+
export const DEFAULT_SANDBOX_CONTROL_TIMING: SandboxControlTimingPolicy = Object.freeze({
|
|
18
|
+
controlTickMs: 250,
|
|
19
|
+
parkedBindingInitialMs: 1_000,
|
|
20
|
+
slowCheckMs: 5_000,
|
|
21
|
+
containerHeartbeatMs: 5_000,
|
|
22
|
+
quiesceDeadlineMs: 7_000
|
|
23
|
+
} as const);
|
|
9
24
|
|
|
10
25
|
export type SandboxControlFamily = typeof SANDBOX_CONTROL_FAMILIES[number];
|
|
26
|
+
export type SandboxControlContainerIdentity = Readonly<{
|
|
27
|
+
id: string;
|
|
28
|
+
labels: Readonly<Record<string, string>>;
|
|
29
|
+
}>;
|
|
11
30
|
export type SandboxControlManifest = Readonly<{
|
|
12
|
-
version:
|
|
31
|
+
version: 4; engine: string; repoRoot: string; worktreeRoot: string; project: string; container: string;
|
|
32
|
+
containerIdentity: SandboxControlContainerIdentity;
|
|
13
33
|
branch: string; mode: 'task-bound' | 'branch-only'; taskId: string | null; token: string;
|
|
14
34
|
generation: string; channelDir: string; publicStatusDir: string; processingDir: string;
|
|
15
35
|
}>;
|
|
36
|
+
export type SandboxControlBrokerOwner = ProcessIdentity & Readonly<{
|
|
37
|
+
version: 3;
|
|
38
|
+
brokerId: string;
|
|
39
|
+
token: string;
|
|
40
|
+
generation: string;
|
|
41
|
+
}>;
|
|
16
42
|
type RequestBase = Readonly<{
|
|
17
43
|
version: 2; id: string; token: string; generation: string; issuedAt: number; expiresAt: number;
|
|
18
44
|
}>;
|
|
@@ -29,17 +55,17 @@ export type SandboxControlResponse = Readonly<{
|
|
|
29
55
|
stdout: string; stderr: string; error: SandboxControlError | null;
|
|
30
56
|
}>;
|
|
31
57
|
export type SandboxControlStatus = Readonly<{
|
|
32
|
-
version:
|
|
58
|
+
version: 2; generation: string; broker: ProcessIdentity & { brokerId: string };
|
|
33
59
|
state: 'starting' | 'healthy' | 'busy' | 'parked'; reasonCode: string | null;
|
|
34
60
|
activeRequestId: string | null; updatedAt: number;
|
|
35
61
|
}>;
|
|
36
62
|
export type SandboxControlLease = Readonly<{
|
|
37
|
-
version:
|
|
63
|
+
version: 2; generation: string; nonce: string; owner: ProcessIdentity;
|
|
38
64
|
issuedAt: number; expiresAt: number; taskId: string | null; branch: string; reason: string;
|
|
39
65
|
}>;
|
|
40
66
|
export type SandboxControlExecution = Readonly<{
|
|
41
|
-
version:
|
|
42
|
-
child:
|
|
67
|
+
version: 2; generation: string; requestId: string; nonce: string;
|
|
68
|
+
child: ProcessIdentity & { processGroupId: number | null };
|
|
43
69
|
phase: 'prepared' | 'running' | 'terminating'; updatedAt: number;
|
|
44
70
|
}>;
|
|
45
71
|
|