@zq-silk/yui 0.15.0 → 0.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -9
- package/dist/cli/commandCatalog.js +5 -5
- package/dist/cli.js +5 -3
- package/dist/commands/agentCommands.js +13 -6
- package/dist/commands/globalRoleCommands.js +11 -3
- package/dist/commands/roleConfiguration.js +7 -0
- package/dist/commands/roleRuntimeGuard.js +30 -0
- package/dist/commands/taskCommands.js +10 -3
- package/dist/context/sessionBootstrapManifest.js +24 -9
- package/dist/controller/fileSchedulerStoreAdapter.js +4 -4
- package/dist/controller/jobClient.js +1 -0
- package/dist/controller/jobControl.js +137 -10
- package/dist/controller/jobSupervisor.js +89 -75
- package/dist/controller/runtime.js +19 -28
- package/dist/controller/runtimeLaunchCoordinator.js +9 -30
- package/dist/controller/sessionNotify.js +5 -0
- package/dist/core/boundedRpc.js +3 -1
- package/dist/executor/agentExecutor.js +8 -11
- package/dist/executor/effectiveLaunch.js +34 -17
- package/dist/executor/fileRoleLaunchPlanner.js +11 -8
- package/dist/job/durableJob.js +68 -6
- package/dist/kernel/callAuthority.js +24 -0
- package/dist/kernel/instanceHost.js +97 -0
- package/dist/kernel/kernelPorts.js +44 -0
- package/dist/kernel/operationFacts.js +32 -0
- package/dist/runtime/agentHost.js +7 -0
- package/dist/runtime/codexInteractiveHost.js +191 -0
- package/dist/runtime/exactControlPlane.js +8 -10
- package/dist/runtime/structuredProviderHost.js +35 -0
- package/dist/runtime/tmuxAdapters.js +51 -9
- package/dist/scheduler/activeRoleTurnDelivery.js +4 -4
- package/dist/scheduler/leaderWakeupProcessor.js +3 -4
- package/dist/storage/sqliteSchema.js +28 -0
- package/dist/storage/sqliteStore.js +11 -3
- package/dist/storage/storageVersions.js +1 -1
- package/dist/storage/storeRpc.js +1 -1
- package/dist/storage/taskStore.js +6 -1
- package/dist/storage/upgrade/upgradeOrchestrator.js +3 -0
- package/dist/tmux/tmuxManager.js +43 -28
- package/i18n/README.zh-CN.md +7 -0
- package/package.json +1 -1
|
@@ -12,7 +12,10 @@ import { createHash } from "node:crypto";
|
|
|
12
12
|
import { resolve, sep } from "node:path";
|
|
13
13
|
import { acknowledgeUnknownDurableJob, createDurableJob, durableJobIdempotencyKey, isDurableJobTerminal, requestDurableJobCancel, retryDurableJobIdempotencyKey } from "../job/durableJob.js";
|
|
14
14
|
import { activeLiveRoleAgentSession } from "../executor/agentExecutor.js";
|
|
15
|
+
import { CallAuthority } from "../kernel/callAuthority.js";
|
|
16
|
+
import { redactLaunchText } from "../runtime/launchDiagnostics.js";
|
|
15
17
|
export function createDurableJobControl(store) {
|
|
18
|
+
const authority = createJobCallAuthority(store);
|
|
16
19
|
return {
|
|
17
20
|
startJob(params, now) {
|
|
18
21
|
// rr4/finding-3: The entire create path — validation, idempotency
|
|
@@ -20,7 +23,8 @@ export function createDurableJobControl(store) {
|
|
|
20
23
|
// between the idempotency check and the save lets a concurrent
|
|
21
24
|
// startJob with the same key create a duplicate job.
|
|
22
25
|
return store.transaction((tx) => {
|
|
23
|
-
|
|
26
|
+
const context = authority.authenticate(Object.freeze({ ...params.caller }), params.taskId);
|
|
27
|
+
assertNonSecretJobInput(params);
|
|
24
28
|
const baseKey = durableJobIdempotencyKey({
|
|
25
29
|
owner: params.owner,
|
|
26
30
|
projectId: params.projectId,
|
|
@@ -29,12 +33,48 @@ export function createDurableJobControl(store) {
|
|
|
29
33
|
workspace: params.workspace,
|
|
30
34
|
env: params.env
|
|
31
35
|
});
|
|
32
|
-
const
|
|
36
|
+
const inputDigest = params.retryOf === undefined
|
|
33
37
|
? baseKey
|
|
34
38
|
: retryDurableJobIdempotencyKey(baseKey, params.retryOf);
|
|
39
|
+
const requestId = params.requestId === undefined ? inputDigest
|
|
40
|
+
: requiredId(params.requestId, "job.start requestId");
|
|
41
|
+
// An IntegrationAttempt already is a durable operation identity.
|
|
42
|
+
// Recovery by another authorized Role must find its original Job,
|
|
43
|
+
// including the window before Integration persisted the returned id.
|
|
44
|
+
if (params.owner.kind === "integration-attempt") {
|
|
45
|
+
const integrationId = params.owner.integrationAttemptId;
|
|
46
|
+
const owned = tx.listDurableJobs(params.taskId).filter((job) => (job.owner.kind === "integration-attempt"
|
|
47
|
+
&& job.owner.integrationAttemptId === integrationId));
|
|
48
|
+
if (owned.length > 1) {
|
|
49
|
+
throw jobDomainError("IntegrationAttempt has multiple Jobs; inspect its existing records.");
|
|
50
|
+
}
|
|
51
|
+
const original = owned[0];
|
|
52
|
+
if (original !== undefined) {
|
|
53
|
+
if (original.operation.inputDigest !== inputDigest
|
|
54
|
+
|| original.operation.targetId !== params.workspace) {
|
|
55
|
+
throw jobDomainError(`Integration Job input conflicts with its original request: ${original.id}.`);
|
|
56
|
+
}
|
|
57
|
+
return { job: original, created: false };
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
const key = createHash("sha256").update(JSON.stringify([
|
|
61
|
+
context.actorId, requestId
|
|
62
|
+
])).digest("hex");
|
|
35
63
|
const existing = tx.findDurableJobByIdempotencyKey(params.taskId, key);
|
|
36
|
-
if (existing !== null)
|
|
64
|
+
if (existing !== null) {
|
|
65
|
+
if (existing.operation.inputDigest !== inputDigest || existing.operation.targetId !== params.workspace) {
|
|
66
|
+
throw jobDomainError(`Job request identity conflicts with its original input: ${existing.id}.`);
|
|
67
|
+
}
|
|
37
68
|
return { job: existing, created: false };
|
|
69
|
+
}
|
|
70
|
+
// A historical content-addressed request has no attributable caller.
|
|
71
|
+
// Do not silently execute it again under a newly attributed identity.
|
|
72
|
+
const historical = tx.findDurableJobByIdempotencyKey(params.taskId, inputDigest);
|
|
73
|
+
if (params.requestId === undefined && historical !== null) {
|
|
74
|
+
throw jobDomainError(`Historical request already exists: ${historical.id}; inspect it or select an explicit new requestId.`);
|
|
75
|
+
}
|
|
76
|
+
authority.authorize(context, params.taskId);
|
|
77
|
+
validateStartParams(tx, params);
|
|
38
78
|
const id = tx.nextDurableJobId(params.taskId);
|
|
39
79
|
const job = createDurableJob({
|
|
40
80
|
id,
|
|
@@ -45,6 +85,10 @@ export function createDurableJobControl(store) {
|
|
|
45
85
|
workspace: params.workspace,
|
|
46
86
|
env: params.env,
|
|
47
87
|
steps: params.steps,
|
|
88
|
+
operation: {
|
|
89
|
+
requestId, inputDigest, actorId: context.actorId,
|
|
90
|
+
authorityRef: jobAuthorityBinding(tx, params.caller.scope, params.caller.role, params.taskId)
|
|
91
|
+
},
|
|
48
92
|
artifactsLocator: `artifacts/jobs/${params.taskId}/${id}`,
|
|
49
93
|
...(params.retryOf === undefined ? {} : { retryOf: params.retryOf })
|
|
50
94
|
}, now);
|
|
@@ -83,6 +127,78 @@ export function createDurableJobControl(store) {
|
|
|
83
127
|
}
|
|
84
128
|
};
|
|
85
129
|
}
|
|
130
|
+
/** T02 ingress adapter. The existing Job boundary remains the authority and
|
|
131
|
+
* semantic writer; this does not authorize arbitrary plugin or resource work.
|
|
132
|
+
*/
|
|
133
|
+
export function createJobCallAuthority(store) {
|
|
134
|
+
return new CallAuthority((caller, taskId) => {
|
|
135
|
+
assertCallerAuthorized(store, caller, taskId);
|
|
136
|
+
return caller.scope === "task"
|
|
137
|
+
? `task:${taskId}/role:${caller.role}`
|
|
138
|
+
: `global:${caller.role}`;
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
/** The collector does not use this gate: late results belong to the original
|
|
142
|
+
* Job even when its management binding is revoked. Only a new spawn checks it.
|
|
143
|
+
*/
|
|
144
|
+
export function authorizeJobStart(store, job) {
|
|
145
|
+
const taskPrefix = `task:${job.taskId}/role:`;
|
|
146
|
+
const actor = job.operation.actorId;
|
|
147
|
+
const scope = actor.startsWith(taskPrefix) ? "task" : actor.startsWith("global:") ? "global" : undefined;
|
|
148
|
+
if (scope === undefined)
|
|
149
|
+
throw jobDomainError("Job caller binding is unavailable; no execution was started.");
|
|
150
|
+
const role = actor.slice(scope === "task" ? taskPrefix.length : "global:".length);
|
|
151
|
+
if (jobAuthorityBinding(store, scope, role, job.taskId) !== job.operation.authorityRef) {
|
|
152
|
+
throw jobDomainError("Job caller binding was revoked; no execution was started.");
|
|
153
|
+
}
|
|
154
|
+
validateJobTarget(store, job);
|
|
155
|
+
}
|
|
156
|
+
function jobAuthorityBinding(store, scope, roleName, taskId) {
|
|
157
|
+
// A Host detach/reattach preserves the native Session and its queued work.
|
|
158
|
+
// Authenticate the live launch at ingress, but bind accepted Jobs to the
|
|
159
|
+
// caller's durable Session identity, not its disposable Host generation.
|
|
160
|
+
if (scope === "task") {
|
|
161
|
+
const role = store.getRole(taskId, roleName);
|
|
162
|
+
const sessions = store.getTaskRoleSessionSet(taskId, roleName);
|
|
163
|
+
const session = activeLiveRoleAgentSession(sessions);
|
|
164
|
+
const hash = role === null ? null : store.getJobCallerKeyHash(taskId, roleName, role.activeAgentId);
|
|
165
|
+
if (hash === null || role === null || sessions?.activeAgentId !== role.activeAgentId
|
|
166
|
+
|| session === null || session.agentId !== role.activeAgentId) {
|
|
167
|
+
throw jobDomainError("Current Job caller Session is unavailable.");
|
|
168
|
+
}
|
|
169
|
+
return createHash("sha256").update(JSON.stringify([
|
|
170
|
+
hash, session.agentId, session.adapterId, session.nativeSessionId
|
|
171
|
+
])).digest("hex");
|
|
172
|
+
}
|
|
173
|
+
const role = store.getGlobalRole(roleName);
|
|
174
|
+
const session = activeLiveRoleAgentSession(store.getGlobalRoleSessionSet(roleName));
|
|
175
|
+
if (role === null || session === null)
|
|
176
|
+
throw jobDomainError("Current Job caller binding is unavailable.");
|
|
177
|
+
return createHash("sha256").update(JSON.stringify([
|
|
178
|
+
role.activeAgentId, session.agentId, session.nativeSessionId
|
|
179
|
+
])).digest("hex");
|
|
180
|
+
}
|
|
181
|
+
function assertNonSecretJobInput(params) {
|
|
182
|
+
// Existing Jobs persist commands and environments. This boundary accepts
|
|
183
|
+
// non-secret executable specifications only; credential resolution is not a
|
|
184
|
+
// Job feature. Never hash a known secret then call the digest sanitized.
|
|
185
|
+
const secretKey = /api[_-]?key|private[_-]?key|token|secret|password|passwd|cookie|credential|authorization/i;
|
|
186
|
+
for (const env of [params.env, ...params.steps.map((step) => step.env ?? {})]) {
|
|
187
|
+
if (Object.keys(env).some((key) => secretKey.test(key))) {
|
|
188
|
+
throw jobDomainError("Job input cannot persist credentials; use a non-secret specification.");
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const input = JSON.stringify({
|
|
192
|
+
env: params.env, steps: params.steps, requestId: params.requestId
|
|
193
|
+
});
|
|
194
|
+
// Reject recognizable key material regardless of the parameter name, and
|
|
195
|
+
// URL userinfo before either hashing or persisting the specification.
|
|
196
|
+
const privateKey = /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----/;
|
|
197
|
+
const urlCredential = /[a-z][a-z0-9+.-]*:\/\/[^\s/"<>]+:[^\s/"<>]*@/i;
|
|
198
|
+
if (privateKey.test(input) || urlCredential.test(input) || redactLaunchText(input) !== input) {
|
|
199
|
+
throw jobDomainError("Job input cannot persist credentials; use a non-secret specification.");
|
|
200
|
+
}
|
|
201
|
+
}
|
|
86
202
|
/**
|
|
87
203
|
* Persisted-boundary validation for `job.start`. The owner must resolve to a
|
|
88
204
|
* live Task record, the Task must be active, the workspace must be the exact
|
|
@@ -94,6 +210,10 @@ export function createDurableJobControl(store) {
|
|
|
94
210
|
* managed workspace, verifies write access, and requires an active Task.
|
|
95
211
|
*/
|
|
96
212
|
function validateStartParams(store, params) {
|
|
213
|
+
validateJobTarget(store, params);
|
|
214
|
+
assertCallerAuthorized(store, params.caller, params.taskId);
|
|
215
|
+
}
|
|
216
|
+
function validateJobTarget(store, params) {
|
|
97
217
|
// The Task must be active — a terminal Task cannot run jobs.
|
|
98
218
|
const task = store.getTask(params.taskId);
|
|
99
219
|
if (task === null) {
|
|
@@ -179,10 +299,6 @@ function validateStartParams(store, params) {
|
|
|
179
299
|
throw jobDomainError(`Retry original job must be terminal: ${params.retryOf} is ${original.status}.`);
|
|
180
300
|
}
|
|
181
301
|
}
|
|
182
|
-
// rr8: Bind the declared owner to the caller's managed identity. A
|
|
183
|
-
// Role is not an authorization boundary. Scope and exact managed Session
|
|
184
|
-
// identity are verified independently below.
|
|
185
|
-
assertCallerAuthorized(store, params.caller, params.taskId);
|
|
186
302
|
}
|
|
187
303
|
/**
|
|
188
304
|
* rr8/rr12: Bind the declared job owner to the caller's managed identity. The
|
|
@@ -262,9 +378,17 @@ function assertCallerAuthorized(store, caller, taskId) {
|
|
|
262
378
|
// frozen environment, and its own claim would add nothing the store does
|
|
263
379
|
// not already own.
|
|
264
380
|
const run = caller.role === undefined ? null : store.getActiveTurn(taskId, caller.role);
|
|
265
|
-
|
|
381
|
+
const currentRole = caller.role === undefined ? null : store.getRole(taskId, caller.role);
|
|
382
|
+
if (run === null || run.status !== "active" || currentRole === null
|
|
383
|
+
|| currentRole.activeAgentId !== run.effective.agentId) {
|
|
266
384
|
throw jobControlError("UNAUTHORIZED", "A managed Task Session's Role is not bound to an active Turn.");
|
|
267
385
|
}
|
|
386
|
+
const sessions = store.getTaskRoleSessionSet(taskId, currentRole.name);
|
|
387
|
+
const session = activeLiveRoleAgentSession(sessions);
|
|
388
|
+
if (sessions?.activeAgentId !== currentRole.activeAgentId || session === null
|
|
389
|
+
|| session.agentId !== run.effective.agentId || session.adapterId !== run.effective.adapterId) {
|
|
390
|
+
throw jobControlError("UNAUTHORIZED", "DurableJob control requires the current live Task Session.");
|
|
391
|
+
}
|
|
268
392
|
// rr13: Verify the non-replayable per-Session caller key. The key is injected
|
|
269
393
|
// at native Session launch and never persisted in plaintext; only its SHA-256
|
|
270
394
|
// hash is durable. A client with database read access can see the hash but cannot
|
|
@@ -272,7 +396,7 @@ function assertCallerAuthorized(store, caller, taskId) {
|
|
|
272
396
|
if (caller.callerKey === undefined) {
|
|
273
397
|
throw jobControlError("UNAUTHORIZED", "job.start/job.cancel requires a managed Session caller key.");
|
|
274
398
|
}
|
|
275
|
-
const expectedHash = store.getJobCallerKeyHash(taskId, caller.role ?? "",
|
|
399
|
+
const expectedHash = store.getJobCallerKeyHash(taskId, caller.role ?? "", currentRole.activeAgentId);
|
|
276
400
|
if (expectedHash === null) {
|
|
277
401
|
throw jobControlError("UNAUTHORIZED", "The managed Session has no durable caller key; it must be relaunched.");
|
|
278
402
|
}
|
|
@@ -337,7 +461,7 @@ export function parseDurableJobStartParams(value) {
|
|
|
337
461
|
const record = value;
|
|
338
462
|
const allowed = new Set([
|
|
339
463
|
"taskId", "owner", "projectId", "head", "workspace", "env", "steps",
|
|
340
|
-
"retryOf", "caller"
|
|
464
|
+
"retryOf", "caller", "requestId"
|
|
341
465
|
]);
|
|
342
466
|
for (const key of Object.keys(record)) {
|
|
343
467
|
if (!allowed.has(key)) {
|
|
@@ -367,6 +491,9 @@ export function parseDurableJobStartParams(value) {
|
|
|
367
491
|
env,
|
|
368
492
|
steps,
|
|
369
493
|
caller,
|
|
494
|
+
...(record.requestId === undefined ? {} : {
|
|
495
|
+
requestId: requiredId(record.requestId, "job.start requestId")
|
|
496
|
+
}),
|
|
370
497
|
...(retryOf === undefined ? {} : { retryOf })
|
|
371
498
|
};
|
|
372
499
|
}
|
|
@@ -16,9 +16,10 @@ import { chmodSync, existsSync, mkdirSync, openSync, closeSync, readFileSync, st
|
|
|
16
16
|
import { join } from "node:path";
|
|
17
17
|
import { fileURLToPath } from "node:url";
|
|
18
18
|
import { writeTextFileAtomically } from "../storage/durableFile.js";
|
|
19
|
-
import { cancelQueuedDurableJob, completeDurableJob, isDurableJobTerminal, markDurableJobUnknown, markDurableJobWakeupNotified, startDurableJob, touchDurableJobHeartbeat } from "../job/durableJob.js";
|
|
19
|
+
import { cancelQueuedDurableJob, completeDurableJob, isDurableJobTerminal, markDurableJobUnknown, markDurableJobWakeupNotified, rejectQueuedDurableJob, startDurableJob, touchDurableJobHeartbeat } from "../job/durableJob.js";
|
|
20
20
|
import { readLinuxProcessStartIdentity } from "./domainIdentity.js";
|
|
21
21
|
import { wakeReason } from "../scheduler/wakeReason.js";
|
|
22
|
+
import { recordOperationEvidence } from "../kernel/operationFacts.js";
|
|
22
23
|
const DEFAULT_STEP_TIMEOUT_MS = 30 * 60_000;
|
|
23
24
|
const HEARTBEAT_STALE_MS = 2 * 60_000;
|
|
24
25
|
const SIGKILL_GRACE_MS = 30_000;
|
|
@@ -29,6 +30,7 @@ export class DurableJobSupervisor {
|
|
|
29
30
|
#terminalEvents;
|
|
30
31
|
#wake;
|
|
31
32
|
#onError;
|
|
33
|
+
#authorizeStart;
|
|
32
34
|
// f5: Composite key (taskId/jobId) because job IDs are Task-local — every
|
|
33
35
|
// Task has a job-1, so a Task-local key would cross-kill healthy runners.
|
|
34
36
|
#sigkillAt = new Map();
|
|
@@ -39,6 +41,7 @@ export class DurableJobSupervisor {
|
|
|
39
41
|
this.#terminalEvents = options.terminalEvents;
|
|
40
42
|
this.#wake = options.wake ?? (() => undefined);
|
|
41
43
|
this.#onError = options.onError ?? (() => undefined);
|
|
44
|
+
this.#authorizeStart = options.authorizeStart;
|
|
42
45
|
}
|
|
43
46
|
reconcile(now) {
|
|
44
47
|
const jobs = this.#store.listActiveDurableJobs();
|
|
@@ -67,90 +70,42 @@ export class DurableJobSupervisor {
|
|
|
67
70
|
#sigkillKey(job) {
|
|
68
71
|
return `${job.taskId}/${job.id}`;
|
|
69
72
|
}
|
|
70
|
-
/**
|
|
71
|
-
* Reconcile a queued job.
|
|
72
|
-
*
|
|
73
|
-
* f4: A queued job with a cancel request converges to `cancelled` without
|
|
74
|
-
* spawning a runner — but only if no runner was already spawned. If a
|
|
75
|
-
* start marker proves a runner exists (real pid, or pending marker +
|
|
76
|
-
* ready.json), the job is adopted to running first; the running-cancel
|
|
77
|
-
* path then fences and signals it. Cancelling a spawned job from queued
|
|
78
|
-
* would orphan the runner.
|
|
79
|
-
*
|
|
80
|
-
* f3: The normal path writes a pending start marker, spawns the runner, and
|
|
81
|
-
* lets the runner's own `ready.json` handshake prove it started before any
|
|
82
|
-
* side effect. On recovery the supervisor adopts queued→running first
|
|
83
|
-
* (harvest/unknown require `running`), then harvests exit or handles a
|
|
84
|
-
* dead process — never calling complete/unknown directly from `queued`.
|
|
85
|
-
*/
|
|
73
|
+
/** Adopt observed execution, preserve unknown, or start an unattempted request. */
|
|
86
74
|
#reconcileQueued(job, now) {
|
|
87
75
|
const marker = this.#artifacts.readStartMarker(job.taskId, job.id);
|
|
88
|
-
|
|
89
|
-
// spawned runner. Check spawn evidence before converging to cancelled.
|
|
76
|
+
const spawned = this.#spawnedProcessFromEvidence(job, marker);
|
|
90
77
|
if (job.cancelRequestedAt !== undefined) {
|
|
91
78
|
this.#artifacts.writeCancelFence(job.taskId, job.id);
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
// f1/rr5: A runner was spawned. Signal it in THIS reconcile pass —
|
|
95
|
-
// not adopt to running and wait for the next pass. The signal is
|
|
96
|
-
// sent before the adoption so the runner begins draining
|
|
97
|
-
// immediately; the adoption preserves evidence (exit.json / dead-
|
|
98
|
-
// process handling converges the job on this or the next pass).
|
|
99
|
-
this.#process.signalIfOwned(spawnedProcess.pid, spawnedProcess.startIdentity, "SIGTERM");
|
|
100
|
-
this.#adoptAndContinue(job, spawnedProcess, now);
|
|
101
|
-
return;
|
|
79
|
+
if (spawned !== null) {
|
|
80
|
+
this.#process.signalIfOwned(spawned.pid, spawned.startIdentity, "SIGTERM");
|
|
102
81
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
// or may never have started. Do NOT terminalize without signaling.
|
|
107
|
-
// Re-spawn: the new runner writes ready.json, observes the cancel
|
|
108
|
-
// fence, and exits as cancelled without side effects. The next
|
|
109
|
-
// pass harvests the cancelled exit.json.
|
|
110
|
-
this.#startJob(job, now);
|
|
111
|
-
return;
|
|
112
|
-
}
|
|
113
|
-
// No spawn attempted — safe to cancel from queued.
|
|
114
|
-
// f6/rr5: The terminal transition and the Leader wakeup must be
|
|
115
|
-
// atomic (same transaction). Compose cancel + wakeupNotified and
|
|
116
|
-
// pass the wakeup param so the adapter enqueues the Leader mailbox
|
|
117
|
-
// entry in the same transaction.
|
|
118
|
-
const terminal = this.#store.transitionDurableJob(job.taskId, job.id, (current) => markDurableJobWakeupNotified(cancelQueuedDurableJob(current, now), now), now, { reason: wakeReason("job-finished"), refs: wakeupRefs(job) });
|
|
119
|
-
this.#deliverTerminalEvent(terminal);
|
|
82
|
+
}
|
|
83
|
+
if (spawned !== null) {
|
|
84
|
+
this.#adoptAndContinue(job, spawned, now);
|
|
120
85
|
return;
|
|
121
86
|
}
|
|
122
|
-
if (
|
|
123
|
-
|
|
87
|
+
if (job.operation.effect !== "none") {
|
|
88
|
+
// A send intent was committed but no acceptance can be proved. Absence
|
|
89
|
+
// of a file is not proof of no external effect. Never respawn unknown.
|
|
90
|
+
const terminal = this.#store.transitionDurableJob(job.taskId, job.id, (current) => markDurableJobWakeupNotified(markDurableJobUnknown(current, "runner acceptance is unknown; inspect the original request", current.checkpoint?.completedSteps ?? [], now), now), now, { reason: wakeReason("job-finished"), refs: wakeupRefs(job) });
|
|
91
|
+
this.#deliverTerminalEvent(terminal);
|
|
124
92
|
return;
|
|
125
93
|
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
if (ready === null) {
|
|
131
|
-
// The runner either never started or died before writing ready.
|
|
132
|
-
// No side effects could have occurred — re-spawn safely.
|
|
133
|
-
this.#startJob(job, now);
|
|
134
|
-
return;
|
|
135
|
-
}
|
|
136
|
-
// Runner proved it started: adopt and continue from evidence.
|
|
137
|
-
// rr4/finding-4: Use the runner's own startIdentity from ready.json,
|
|
138
|
-
// not a fresh /proc read (which could return a reused PID's identity).
|
|
139
|
-
this.#adoptAndContinue(job, { pid: ready.pid, startIdentity: ready.startIdentity }, now);
|
|
94
|
+
if (job.cancelRequestedAt !== undefined) {
|
|
95
|
+
// No effect attempted. Cancellation and its wake commit atomically.
|
|
96
|
+
const terminal = this.#store.transitionDurableJob(job.taskId, job.id, (current) => markDurableJobWakeupNotified(cancelQueuedDurableJob(current, now), now), now, { reason: wakeReason("job-finished"), refs: wakeupRefs(job) });
|
|
97
|
+
this.#deliverTerminalEvent(terminal);
|
|
140
98
|
return;
|
|
141
99
|
}
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
this.#adoptAndContinue(job, spawned ?? { pid: marker.pid, startIdentity: marker.startIdentity }, now);
|
|
100
|
+
if (marker !== null)
|
|
101
|
+
throw new Error("Job start marker contradicts its unattempted operation.");
|
|
102
|
+
this.#startJob(job, now);
|
|
146
103
|
}
|
|
147
104
|
/**
|
|
148
105
|
* Determine whether a runner was spawned for this job, based on durable
|
|
149
106
|
* evidence. Returns the process identity if spawned, null otherwise.
|
|
150
107
|
*/
|
|
151
108
|
#spawnedProcessFromEvidence(job, marker) {
|
|
152
|
-
if (marker === null)
|
|
153
|
-
return null;
|
|
154
109
|
// f3/rr5: ready.json is the runner's own record of its actual OS start.
|
|
155
110
|
// When both the start marker and ready.json exist, ready.json is
|
|
156
111
|
// authoritative: the marker is the Controller's declared intent (written
|
|
@@ -161,7 +116,7 @@ export class DurableJobSupervisor {
|
|
|
161
116
|
if (ready !== null) {
|
|
162
117
|
return { pid: ready.pid, startIdentity: ready.startIdentity };
|
|
163
118
|
}
|
|
164
|
-
if (marker.startIdentity !== "pending") {
|
|
119
|
+
if (marker !== null && marker.startIdentity !== "pending") {
|
|
165
120
|
// Real start marker with a pid — the runner was spawned.
|
|
166
121
|
return { pid: marker.pid, startIdentity: marker.startIdentity };
|
|
167
122
|
}
|
|
@@ -170,8 +125,7 @@ export class DurableJobSupervisor {
|
|
|
170
125
|
}
|
|
171
126
|
/**
|
|
172
127
|
* f3: Adopt a queued job to running, then harvest exit or handle a dead
|
|
173
|
-
* process.
|
|
174
|
-
* complete/unknown require `running`.
|
|
128
|
+
* process. A request with unobserved acceptance instead stays unknown.
|
|
175
129
|
*/
|
|
176
130
|
#adoptAndContinue(job, process, now) {
|
|
177
131
|
this.#store.transitionDurableJob(job.taskId, job.id, (current) => startDurableJob(current, process, now), now);
|
|
@@ -190,6 +144,13 @@ export class DurableJobSupervisor {
|
|
|
190
144
|
}
|
|
191
145
|
}
|
|
192
146
|
#startJob(job, now) {
|
|
147
|
+
try {
|
|
148
|
+
this.#authorizeStart(job);
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
this.#rejectStart(job, error, now);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
193
154
|
const spec = {
|
|
194
155
|
jobId: job.id,
|
|
195
156
|
taskId: job.taskId,
|
|
@@ -201,9 +162,28 @@ export class DurableJobSupervisor {
|
|
|
201
162
|
head: job.head
|
|
202
163
|
};
|
|
203
164
|
const specPath = this.#artifacts.writeSpec(job.taskId, job.id, spec);
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
165
|
+
let attempted;
|
|
166
|
+
try {
|
|
167
|
+
attempted = this.#store.transitionDurableJob(job.taskId, job.id, (current) => {
|
|
168
|
+
if (current.status !== "queued" || current.operation.effect !== "none") {
|
|
169
|
+
throw new Error("Job already attempted; inspect the original request.");
|
|
170
|
+
}
|
|
171
|
+
this.#authorizeStart(current);
|
|
172
|
+
return {
|
|
173
|
+
...current,
|
|
174
|
+
operation: recordOperationEvidence(current.operation, { effect: "possible" }),
|
|
175
|
+
updatedAt: now.toISOString()
|
|
176
|
+
};
|
|
177
|
+
}, now);
|
|
178
|
+
}
|
|
179
|
+
catch (error) {
|
|
180
|
+
this.#rejectStart(job, error, now);
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
if (attempted === null)
|
|
184
|
+
return;
|
|
185
|
+
// Both the request and possible-effect boundary precede spawn. A missing
|
|
186
|
+
// acceptance after this point is unknown, never permission to respawn.
|
|
207
187
|
this.#artifacts.writeStartMarker(job.taskId, job.id, {
|
|
208
188
|
pid: 0,
|
|
209
189
|
startIdentity: "pending",
|
|
@@ -229,6 +209,14 @@ export class DurableJobSupervisor {
|
|
|
229
209
|
spawned.onExit?.(() => this.#wake(job.taskId));
|
|
230
210
|
this.#wake(job.taskId);
|
|
231
211
|
}
|
|
212
|
+
#rejectStart(job, error, now) {
|
|
213
|
+
// Only an explicit domain refusal is a known failure. Storage/CAS/runtime
|
|
214
|
+
// errors still propagate; never infer rejection from an unavailable read.
|
|
215
|
+
if (!(error instanceof Error) || error.name !== "CoreJobError")
|
|
216
|
+
throw error;
|
|
217
|
+
const terminal = this.#store.transitionDurableJob(job.taskId, job.id, (current) => markDurableJobWakeupNotified(rejectQueuedDurableJob(current, error.message, now), now), now, { reason: wakeReason("job-finished"), refs: wakeupRefs(job) });
|
|
218
|
+
this.#deliverTerminalEvent(terminal);
|
|
219
|
+
}
|
|
232
220
|
#superviseRunning(job, now) {
|
|
233
221
|
// 1. Harvest exit.json if present.
|
|
234
222
|
const exit = this.#artifacts.readExitJson(job.taskId, job.id);
|
|
@@ -278,6 +266,19 @@ export class DurableJobSupervisor {
|
|
|
278
266
|
}
|
|
279
267
|
}
|
|
280
268
|
#harvestExit(job, exit, now) {
|
|
269
|
+
// Persist the original receipt locator before interpreting its output.
|
|
270
|
+
// A bad schema cannot erase evidence of an already executed runner.
|
|
271
|
+
this.#store.transitionDurableJob(job.taskId, job.id, (current) => ({
|
|
272
|
+
...current,
|
|
273
|
+
operation: recordOperationEvidence(current.operation, {
|
|
274
|
+
effect: "confirmed",
|
|
275
|
+
receiptRefs: [`${current.artifactsLocator}/exit.json`],
|
|
276
|
+
partialResultRefs: Array.isArray(exit.steps)
|
|
277
|
+
? exit.steps.flatMap((step) => typeof step?.logPath === "string" && step.logPath.trim()
|
|
278
|
+
? [step.logPath] : []) : []
|
|
279
|
+
}),
|
|
280
|
+
updatedAt: now.toISOString()
|
|
281
|
+
}), now);
|
|
281
282
|
const result = {
|
|
282
283
|
outcome: exit.outcome,
|
|
283
284
|
exitCode: exit.exitCode,
|
|
@@ -291,7 +292,20 @@ export class DurableJobSupervisor {
|
|
|
291
292
|
// the adapter enqueues the Leader mailbox entry in the same transaction.
|
|
292
293
|
// A separate #notifyWakeup pass would lose the wakeup if the Controller
|
|
293
294
|
// died between the terminal write and the flag flip.
|
|
294
|
-
const terminal = this.#store.transitionDurableJob(job.taskId, job.id, (current) =>
|
|
295
|
+
const terminal = this.#store.transitionDurableJob(job.taskId, job.id, (current) => {
|
|
296
|
+
let completed;
|
|
297
|
+
try {
|
|
298
|
+
completed = completeDurableJob(current, result, now);
|
|
299
|
+
}
|
|
300
|
+
catch {
|
|
301
|
+
completed = completeDurableJob(current, {
|
|
302
|
+
outcome: "failed", exitCode: null, signal: null,
|
|
303
|
+
unknownReason: "runner output is invalid; original receipt is retained",
|
|
304
|
+
steps: []
|
|
305
|
+
}, now);
|
|
306
|
+
}
|
|
307
|
+
return markDurableJobWakeupNotified(completed, now);
|
|
308
|
+
}, now, { reason: wakeReason("job-finished"), refs: wakeupRefs(job) });
|
|
295
309
|
this.#deliverTerminalEvent(terminal);
|
|
296
310
|
this.#sigkillAt.delete(this.#sigkillKey(job));
|
|
297
311
|
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { reconciliationIntervalMilliseconds, resolveAgentLaunchInactivityTimeoutSeconds, resolveControllerTaskConcurrency, resolveDeliveryTimeoutSeconds, resolveRuntimeHealth, resolveTmuxBin, resolveTmuxHistoryLimit } from "../config/yuiConfig.js";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
3
2
|
import { resolve } from "node:path";
|
|
4
3
|
import { isDeepStrictEqual } from "node:util";
|
|
5
4
|
import { controllerSocketPath } from "../core/controllerEndpoint.js";
|
|
@@ -7,9 +6,9 @@ import { AGENT_OPERATIONAL_ENVIRONMENT_NAMES, nativeAgentEnvironmentNames, YUI_M
|
|
|
7
6
|
import { hasRuntimeCleanupObligation, runtimeLifecycleSignalKey, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
8
7
|
import { agentProcessReadinessProbe, ExecutorRegistry } from "../executor/executorRegistry.js";
|
|
9
8
|
import { activeLiveRoleAgentSession, roleAgentSessionResumeMode } from "../executor/agentExecutor.js";
|
|
10
|
-
import {
|
|
9
|
+
import { roleSessionMayContinue, effectiveLaunchConfig, resolveEffectiveLaunch } from "../executor/effectiveLaunch.js";
|
|
11
10
|
import { AgentConfigurationCatalogService, validateAgentLaunchConfiguration } from "../executor/agentConfigurationCatalog.js";
|
|
12
|
-
import { isTaskOwnedWorkspace,
|
|
11
|
+
import { isTaskOwnedWorkspace, sameManagedWorkspaceIdentity } from "../worktree/managedWorkspace.js";
|
|
13
12
|
import { FileRoleLaunchPlanner } from "../executor/fileRoleLaunchPlanner.js";
|
|
14
13
|
import { openCurrentTaskStore } from "../storage/currentTaskStore.js";
|
|
15
14
|
import { SqliteTaskStore } from "../storage/sqliteStore.js";
|
|
@@ -22,7 +21,8 @@ import { startFileTaskController } from "./controller.js";
|
|
|
22
21
|
import { AgentHostProviderTurnFenceError, FileSchedulerStoreAdapter } from "./fileSchedulerStoreAdapter.js";
|
|
23
22
|
import { openSchedulerTelemetry } from "../telemetry/telemetryWiring.js";
|
|
24
23
|
import { createFileArtifactPort, createLinuxProcessPort, DurableJobSupervisor } from "./jobSupervisor.js";
|
|
25
|
-
import {
|
|
24
|
+
import { authorizeJobStart } from "./jobControl.js";
|
|
25
|
+
import { createKernelPorts } from "../kernel/kernelPorts.js";
|
|
26
26
|
import { FileRuntimeEventInbox } from "./runtimeEventInbox.js";
|
|
27
27
|
import { AgentRuntimeObserver } from "./agentRuntimeObserver.js";
|
|
28
28
|
import { AsyncRuntimeEventProcessor, FileRuntimeEventProcessor, createAsyncRuntimeObserver, } from "./runtimeEventProcessor.js";
|
|
@@ -216,7 +216,6 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
216
216
|
assertCurrent: (request) => {
|
|
217
217
|
assertRuntimeLaunchRequestCurrent(store, request);
|
|
218
218
|
},
|
|
219
|
-
launchFingerprint: (request) => (runtimeLaunchFingerprint(store, request)),
|
|
220
219
|
onCleanupRequired: signalRuntimeCleanup,
|
|
221
220
|
runtimeIsolation
|
|
222
221
|
});
|
|
@@ -357,10 +356,12 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
357
356
|
// supervisor enqueues a durable-job-terminal event; the processor drains it
|
|
358
357
|
// on the next pass, waking the Controller immediately instead of waiting for
|
|
359
358
|
// the poll interval.
|
|
359
|
+
const kernel = createKernelPorts(store, createLinuxProcessPort());
|
|
360
360
|
const jobSupervisor = new DurableJobSupervisor({
|
|
361
361
|
store: schedulerStore,
|
|
362
|
-
process:
|
|
362
|
+
process: kernel.runner,
|
|
363
363
|
artifacts: createFileArtifactPort(home),
|
|
364
|
+
authorizeStart: (job) => authorizeJobStart(store, job),
|
|
364
365
|
// rr6/f1: Bounded supervision wake. The supervisor signals the Controller
|
|
365
366
|
// after spawning a runner (queued→running adoption) and when a runner
|
|
366
367
|
// exits (terminal harvest), so a quick job converges without waiting for
|
|
@@ -395,7 +396,7 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
395
396
|
},
|
|
396
397
|
onError: options.onError
|
|
397
398
|
});
|
|
398
|
-
const jobControl =
|
|
399
|
+
const jobControl = kernel.jobs;
|
|
399
400
|
const continuationReconciler = options.continuationMetadata === undefined
|
|
400
401
|
? undefined
|
|
401
402
|
: new ProviderContinuationReconciliationService(store, schedulerStore, options.continuationMetadata);
|
|
@@ -464,6 +465,7 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
464
465
|
let resourceClose;
|
|
465
466
|
const closeResources = () => {
|
|
466
467
|
resourceClose ??= Promise.all([
|
|
468
|
+
kernel.close(),
|
|
467
469
|
asyncStoreClient?.close() ?? Promise.resolve(),
|
|
468
470
|
inventoryClient?.close() ?? Promise.resolve()
|
|
469
471
|
]).then(() => undefined);
|
|
@@ -472,6 +474,7 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
472
474
|
const closed = running.closed.then(closeResources);
|
|
473
475
|
return {
|
|
474
476
|
...running,
|
|
477
|
+
kernel,
|
|
475
478
|
closed,
|
|
476
479
|
close: async () => {
|
|
477
480
|
try {
|
|
@@ -501,7 +504,6 @@ export function createRuntimeLifecycleDispatcher(store, schedulerStore, sessionH
|
|
|
501
504
|
assertCurrent: (request) => {
|
|
502
505
|
assertRuntimeLaunchRequestCurrent(store, request);
|
|
503
506
|
},
|
|
504
|
-
launchFingerprint: (request) => (runtimeLaunchFingerprint(store, request)),
|
|
505
507
|
onCleanupRequired
|
|
506
508
|
});
|
|
507
509
|
const lifecycleTails = new Map();
|
|
@@ -862,11 +864,15 @@ function assertRuntimeLaunchRequestCurrent(store, request) {
|
|
|
862
864
|
|| session.nativeSessionId !== request.nativeSessionId) {
|
|
863
865
|
throw new Error(`Native session changed: ${request.owner.roleName}.`);
|
|
864
866
|
}
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
if (
|
|
869
|
-
|
|
867
|
+
// The Role's durable Session record is the only authority for which Host
|
|
868
|
+
// activation may be restored. Targeting anything else would revive a
|
|
869
|
+
// historical activation.
|
|
870
|
+
if (request.hostActivationId !== undefined
|
|
871
|
+
&& session.runtimeGenerationId !== request.hostActivationId) {
|
|
872
|
+
throw new Error(`Session restore does not target the Role's current Host activation: ${request.owner.roleName}.`);
|
|
873
|
+
}
|
|
874
|
+
if (!roleSessionMayContinue(session.effective, request.effective)) {
|
|
875
|
+
throw new Error(`Native session cannot continue under this launch: ${request.owner.roleName}.`);
|
|
870
876
|
}
|
|
871
877
|
}
|
|
872
878
|
}
|
|
@@ -888,21 +894,6 @@ function currentDesiredEffective(store, request, role) {
|
|
|
888
894
|
...(item === null ? {} : { workItemWriteProjectIds: item.writeProjectIds })
|
|
889
895
|
});
|
|
890
896
|
}
|
|
891
|
-
function runtimeLaunchFingerprint(store, request) {
|
|
892
|
-
const agent = store.getConfiguredAgent(request.effective.agentId);
|
|
893
|
-
if (agent === null) {
|
|
894
|
-
throw new Error(`Agent no longer exists: ${request.effective.agentId}.`);
|
|
895
|
-
}
|
|
896
|
-
return createHash("sha256").update(JSON.stringify([
|
|
897
|
-
request.owner,
|
|
898
|
-
request.effective,
|
|
899
|
-
request.managedWorkspace === undefined
|
|
900
|
-
? undefined
|
|
901
|
-
: managedWorkspaceIdentity(request.managedWorkspace),
|
|
902
|
-
request.runtimePolicy,
|
|
903
|
-
agent
|
|
904
|
-
])).digest("hex");
|
|
905
|
-
}
|
|
906
897
|
function currentDesiredManagedWorkspace(store, taskId, roleName) {
|
|
907
898
|
const item = store.listWorkItems(taskId).find((candidate) => (candidate.assignee === roleName
|
|
908
899
|
&& !["completed", "failed", "retired"].includes(candidate.status))) ?? null;
|