@zq-silk/yui 2.17.0 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commandCatalog.js +2 -1
- package/dist/commands/controllerCommands.js +14 -4
- package/dist/commands/durableJobCommands.js +50 -3
- package/dist/commands/globalRoleCommands.js +18 -12
- package/dist/commands/inboxCommands.js +1 -0
- package/dist/commands/operatorCommands.js +10 -2
- package/dist/controlPlaneCli.js +21 -4
- package/dist/controller/clientRuntime.js +34 -13
- package/dist/controller/controller.js +45 -22
- package/dist/controller/diagnosticJournal.js +214 -0
- package/dist/controller/fileSchedulerStoreAdapter.js +143 -101
- package/dist/controller/globalInputDelivery.js +6 -2
- package/dist/controller/globalRuntimeAttention.js +11 -3
- package/dist/controller/jobControl.js +1 -1
- package/dist/controller/jobDefinitionControl.js +24 -13
- package/dist/controller/jobSupervisor.js +4 -0
- package/dist/controller/messageHealth.js +354 -0
- package/dist/controller/runtime.js +75 -5
- package/dist/controller/taskHostTerminalRecovery.js +11 -0
- package/dist/controller/workerSchedulerStore.js +24 -0
- package/dist/core/boundedRpc.js +4 -0
- package/dist/core/controllerServer.js +1 -1
- package/dist/core/controllerTelemetry.js +92 -19
- package/dist/job/globalShell.js +51 -0
- package/dist/job/jobBusinessResult.js +4 -3
- package/dist/job/jobDefinition.js +6 -1
- package/dist/job/jobRunner.js +9 -0
- package/dist/job/scheduleService.js +2 -0
- package/dist/message/globalInboxRecovery.js +30 -19
- package/dist/message/globalProviderRetry.js +7 -1
- package/dist/runtime/builtinAgentErrorMappers.js +4 -0
- package/dist/runtime/globalSessionHealth.js +39 -0
- package/dist/scheduler/roleRunStall.js +10 -10
- package/dist/storage/contextRecords.js +16 -0
- package/dist/storage/persistenceWorker.js +41 -19
- package/dist/storage/sqliteSchema.js +35 -11
- package/dist/storage/sqliteStore.js +102 -8
- package/dist/storage/storageVersions.js +1 -1
- package/dist/storage/storeRpc.js +25 -11
- package/dist/storage/taskReadRevisionSchema.js +39 -0
- package/dist/web/assets/client/app/catalog.js +0 -29
- package/dist/web/assets/client/app/jobs.js +115 -0
- package/dist/web/assets/client/app/main.js +17 -6
- package/dist/web/assets/client/app/manage.js +1 -0
- package/dist/web/assets/client/app/settings.js +1 -5
- package/dist/web/assets/client/i18n/en.js +203 -18
- package/dist/web/assets/client/i18n/zh-CN.js +202 -17
- package/dist/web/assets/client/index.js +2 -0
- package/dist/web/assets/client/layout/manageShell.js +61 -15
- package/dist/web/assets/client/layout/router.js +2 -1
- package/dist/web/assets/client/lib/prefs.js +1 -1
- package/dist/web/assets/client/views/jobs.js +1042 -160
- package/dist/web/assets/client/views/sidebar.js +3 -0
- package/dist/web/assets/shared/manageNav.js +3 -1
- package/dist/web/assets/shell/appRail.js +2 -1
- package/dist/web/assets/shell/manage.js +7 -2
- package/dist/web/assets/shell/models.js +4 -3
- package/dist/web/assets/shell/settingsContent.js +0 -3
- package/dist/web/assets/shell/sidebar.js +0 -1
- package/dist/web/assets/styles/views/jobs.js +301 -12
- package/dist/web/assets/styles/views/sidebar.js +1 -2
- package/dist/web/webServer.js +28 -1
- package/docs/global-shell-jobs.md +78 -0
- package/docs/job-runs.md +6 -3
- package/docs/managed-turn-and-session-runtime.md +170 -0
- package/docs/managed-turn-and-session-runtime.zh-CN.md +116 -0
- package/docs/release-workflow.md +63 -0
- package/docs/release-workflow.zh-CN.md +44 -0
- package/docs/storage-baseline.md +14 -1
- package/docs/storage-baseline.zh-CN.md +6 -1
- package/docs/testing/verification-levels.md +2 -2
- package/docs/testing/verification-levels.zh-CN.md +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keep maintenance read/check/write closures together on the persistence
|
|
3
|
+
* worker. These operations recheck their exact durable condition in one
|
|
4
|
+
* transaction and are idempotent under the worker's existing replay contract.
|
|
5
|
+
* Do not add admission/claim operations here: their result and unknown-effect
|
|
6
|
+
* contracts need a separate audit before moving across this boundary.
|
|
7
|
+
*/
|
|
8
|
+
const WORKER_MAINTENANCE = new Set([
|
|
9
|
+
"resolveExpiredInputRecommendations",
|
|
10
|
+
"recordRoleRunProgress",
|
|
11
|
+
"recordRoleRunDiagnostic",
|
|
12
|
+
"recordRoleRunStall"
|
|
13
|
+
]);
|
|
14
|
+
export function workerSchedulerStore(store, client) {
|
|
15
|
+
return new Proxy(store, {
|
|
16
|
+
get(target, property) {
|
|
17
|
+
if (WORKER_MAINTENANCE.has(property)) {
|
|
18
|
+
return (...args) => client.invokeObserver(String(property), args);
|
|
19
|
+
}
|
|
20
|
+
const value = Reflect.get(target, property);
|
|
21
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
}
|
package/dist/core/boundedRpc.js
CHANGED
|
@@ -48,6 +48,10 @@ export function serializeError(error) {
|
|
|
48
48
|
export function deserializeError(serialized) {
|
|
49
49
|
const error = new Error(serialized.message);
|
|
50
50
|
error.name = serialized.name;
|
|
51
|
+
if (serialized.code !== undefined)
|
|
52
|
+
Object.assign(error, { code: serialized.code });
|
|
53
|
+
if (serialized.stack !== undefined)
|
|
54
|
+
error.stack = serialized.stack;
|
|
51
55
|
return error;
|
|
52
56
|
}
|
|
53
57
|
// -- Bounded slot pool (backpressure) ----------------------------------------
|
|
@@ -311,7 +311,7 @@ async function routeRequest(socket, line, token, dispatcher, stop, status, telem
|
|
|
311
311
|
// observation completes when the response is handed to the socket, so a
|
|
312
312
|
// later status snapshot sees prior built-in completions and the dispatcher
|
|
313
313
|
// service-time metric stays a distinct observation.
|
|
314
|
-
const observation = telemetry.commandObserver.start();
|
|
314
|
+
const observation = telemetry.commandObserver.start(request.method, request.id);
|
|
315
315
|
const send = (response, onFlushed) => {
|
|
316
316
|
observation.complete(isBuiltinControllerMethod(request.method) ? "builtin" : "dispatched", response.ok ? "success" : "failure");
|
|
317
317
|
sendResponse(socket, response, onFlushed);
|
|
@@ -1,13 +1,69 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
/** Fixed-size retained diagnostics; live bookkeeping follows in-flight work.
|
|
3
|
+
* Not a log or scheduling authority. No params,
|
|
4
|
+
* prompts, paths, failure text or credentials are retained. Target/request
|
|
5
|
+
* hashes correlate an observation without publishing arbitrary input. */
|
|
6
|
+
export class ControllerWorkObserver {
|
|
7
|
+
clock;
|
|
8
|
+
#active = new Map();
|
|
9
|
+
#recent = [];
|
|
10
|
+
#completed = 0;
|
|
11
|
+
#maximumMs = 0;
|
|
12
|
+
#slowest = null;
|
|
13
|
+
constructor(clock = monotonicMilliseconds) {
|
|
14
|
+
this.clock = clock;
|
|
15
|
+
}
|
|
16
|
+
start(phase, target) {
|
|
17
|
+
const token = {};
|
|
18
|
+
const entry = { phase, target: target === undefined ? null
|
|
19
|
+
: createHash("sha256").update(target).digest("hex").slice(0, 16), startedAt: this.clock() };
|
|
20
|
+
this.#active.set(token, entry);
|
|
21
|
+
return (failed = false) => {
|
|
22
|
+
if (!this.#active.delete(token))
|
|
23
|
+
return;
|
|
24
|
+
const elapsedMs = Math.max(0, this.clock() - entry.startedAt);
|
|
25
|
+
this.#completed++;
|
|
26
|
+
this.#maximumMs = Math.max(this.#maximumMs, elapsedMs);
|
|
27
|
+
const completed = { phase, target: entry.target, startedAtMs: entry.startedAt, elapsedMs, failed };
|
|
28
|
+
if (this.#slowest === null || elapsedMs > this.#slowest.elapsedMs)
|
|
29
|
+
this.#slowest = completed;
|
|
30
|
+
this.#recent.push(completed);
|
|
31
|
+
if (this.#recent.length > 16)
|
|
32
|
+
this.#recent.shift();
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
async observe(phase, target, action) {
|
|
36
|
+
const finish = this.start(phase, target);
|
|
37
|
+
try {
|
|
38
|
+
const value = await action();
|
|
39
|
+
finish();
|
|
40
|
+
return value;
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
finish(true);
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
snapshot() {
|
|
48
|
+
const active = [];
|
|
49
|
+
for (const entry of this.#active.values()) {
|
|
50
|
+
active.push({ phase: entry.phase, target: entry.target, startedAtMs: entry.startedAt,
|
|
51
|
+
ageMs: Math.max(0, this.clock() - entry.startedAt) });
|
|
52
|
+
if (active.length === 16)
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
completed: this.#completed, maximumMs: this.#maximumMs, inFlight: this.#active.size,
|
|
57
|
+
active, slowest: this.#slowest,
|
|
58
|
+
recent: [...this.#recent]
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Small worker-response observation; no historical arrays on the RPC wire. */
|
|
63
|
+
export function compactWorkObservation(observer) {
|
|
64
|
+
const { completed, maximumMs, slowest, recent } = observer.snapshot();
|
|
65
|
+
return { completed, maximumMs, slowest, latest: recent.at(-1) ?? null };
|
|
66
|
+
}
|
|
11
67
|
const CONTROLLER_DELAY_BUCKETS_MS = [10, 50, 100, 250, 500, 1_000, 3_000];
|
|
12
68
|
const DEFAULT_EVENT_LOOP_DELAY_INTERVAL_MS = 50;
|
|
13
69
|
const BUILTIN_METHODS = new Set([
|
|
@@ -19,6 +75,12 @@ const BUILTIN_METHODS = new Set([
|
|
|
19
75
|
"controller.rollback-handover",
|
|
20
76
|
"controller.handover-state"
|
|
21
77
|
]);
|
|
78
|
+
const OBSERVED_METHODS = new Set([...BUILTIN_METHODS,
|
|
79
|
+
"scheduler.signal", "scheduler.scan", "scheduler.configure",
|
|
80
|
+
"runtime.ensure-role-session", "runtime.stop-role-session",
|
|
81
|
+
"capability.search", "capability.describe", "capability.call",
|
|
82
|
+
"schedule.control", "job.definition", "job.start", "job.get", "job.cancel", "job.acknowledge"
|
|
83
|
+
]);
|
|
22
84
|
export function isBuiltinControllerMethod(method) {
|
|
23
85
|
return BUILTIN_METHODS.has(method);
|
|
24
86
|
}
|
|
@@ -47,19 +109,23 @@ export class ControllerCommandObserver {
|
|
|
47
109
|
#completed = 0;
|
|
48
110
|
#failed = 0;
|
|
49
111
|
#inFlight = 0;
|
|
50
|
-
/**
|
|
51
|
-
#inFlightStarted =
|
|
112
|
+
/** Insertion order preserves arrival order; completion removes its own token. */
|
|
113
|
+
#inFlightStarted = new Map();
|
|
52
114
|
#builtinCompleted = 0;
|
|
53
115
|
#builtinFailed = 0;
|
|
54
116
|
#dispatchedCompleted = 0;
|
|
55
117
|
#dispatchedFailed = 0;
|
|
118
|
+
#observations;
|
|
56
119
|
constructor(clock = monotonicMilliseconds) {
|
|
57
120
|
this.#clock = clock;
|
|
121
|
+
this.#observations = new ControllerWorkObserver(clock);
|
|
58
122
|
}
|
|
59
|
-
start() {
|
|
123
|
+
start(method = "unknown", requestId) {
|
|
60
124
|
this.#received += 1;
|
|
61
125
|
this.#inFlight += 1;
|
|
62
|
-
|
|
126
|
+
const token = {};
|
|
127
|
+
this.#inFlightStarted.set(token, this.#clock());
|
|
128
|
+
const finish = this.#observations.start(OBSERVED_METHODS.has(method) ? method : "other", requestId);
|
|
63
129
|
let completed = false;
|
|
64
130
|
return {
|
|
65
131
|
complete: (kind, outcome) => {
|
|
@@ -67,7 +133,8 @@ export class ControllerCommandObserver {
|
|
|
67
133
|
return;
|
|
68
134
|
completed = true;
|
|
69
135
|
this.#inFlight = Math.max(0, this.#inFlight - 1);
|
|
70
|
-
this.#inFlightStarted.
|
|
136
|
+
this.#inFlightStarted.delete(token);
|
|
137
|
+
finish(outcome === "failure");
|
|
71
138
|
this.#completed += 1;
|
|
72
139
|
if (outcome === "failure")
|
|
73
140
|
this.#failed += 1;
|
|
@@ -87,9 +154,9 @@ export class ControllerCommandObserver {
|
|
|
87
154
|
};
|
|
88
155
|
}
|
|
89
156
|
snapshot() {
|
|
90
|
-
const oldest = this.#inFlightStarted.
|
|
157
|
+
const oldest = this.#inFlightStarted.size === 0
|
|
91
158
|
? null
|
|
92
|
-
: Math.max(0, this.#clock() - this.#inFlightStarted
|
|
159
|
+
: Math.max(0, this.#clock() - this.#inFlightStarted.values().next().value);
|
|
93
160
|
return {
|
|
94
161
|
received: this.#received,
|
|
95
162
|
completed: this.#completed,
|
|
@@ -103,7 +170,8 @@ export class ControllerCommandObserver {
|
|
|
103
170
|
dispatched: {
|
|
104
171
|
completed: this.#dispatchedCompleted,
|
|
105
172
|
failed: this.#dispatchedFailed
|
|
106
|
-
}
|
|
173
|
+
},
|
|
174
|
+
observations: this.#observations.snapshot()
|
|
107
175
|
};
|
|
108
176
|
}
|
|
109
177
|
}
|
|
@@ -122,6 +190,7 @@ export class ControllerEventLoopDelay {
|
|
|
122
190
|
#expectedAt = 0;
|
|
123
191
|
#samples = 0;
|
|
124
192
|
#maximumLagMs = 0;
|
|
193
|
+
#maximumObservedAtMs = null;
|
|
125
194
|
#buckets = new Map(CONTROLLER_DELAY_BUCKETS_MS.map((threshold) => [threshold, 0]));
|
|
126
195
|
#stopped = false;
|
|
127
196
|
constructor(clock = monotonicMilliseconds, scheduler = productionTelemetryScheduler, intervalMs = DEFAULT_EVENT_LOOP_DELAY_INTERVAL_MS) {
|
|
@@ -142,7 +211,10 @@ export class ControllerEventLoopDelay {
|
|
|
142
211
|
this.#expectedAt = this.#clock() + this.#intervalMs;
|
|
143
212
|
this.#samples += 1;
|
|
144
213
|
const bounded = Math.ceil(lag);
|
|
145
|
-
this.#
|
|
214
|
+
if (this.#maximumObservedAtMs === null || bounded > this.#maximumLagMs) {
|
|
215
|
+
this.#maximumLagMs = bounded;
|
|
216
|
+
this.#maximumObservedAtMs = this.#clock();
|
|
217
|
+
}
|
|
146
218
|
for (const threshold of CONTROLLER_DELAY_BUCKETS_MS) {
|
|
147
219
|
if (bounded <= threshold) {
|
|
148
220
|
this.#buckets.set(threshold, (this.#buckets.get(threshold) ?? 0) + 1);
|
|
@@ -158,6 +230,7 @@ export class ControllerEventLoopDelay {
|
|
|
158
230
|
return {
|
|
159
231
|
samples: this.#samples,
|
|
160
232
|
maximumLagMs: this.#maximumLagMs,
|
|
233
|
+
maximumObservedAtMs: this.#maximumObservedAtMs,
|
|
161
234
|
lagBuckets: Object.fromEntries(CONTROLLER_DELAY_BUCKETS_MS.map((threshold) => [
|
|
162
235
|
`le${threshold}ms`,
|
|
163
236
|
this.#buckets.get(threshold) ?? 0
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { requireKnownFields } from "../domain/validation.js";
|
|
5
|
+
import { validateJobExecutionConfiguration } from "./durableJob.js";
|
|
6
|
+
export function validateJobScriptFiles(files) {
|
|
7
|
+
if (!Array.isArray(files) || files.length > 64)
|
|
8
|
+
throw new Error("Job supports at most 64 script files.");
|
|
9
|
+
const paths = new Set();
|
|
10
|
+
let bytes = 0;
|
|
11
|
+
for (const file of files) {
|
|
12
|
+
requireKnownFields(file, ["path", "content"], "Job script file");
|
|
13
|
+
if (typeof file.path !== "string" || file.path.length > 240
|
|
14
|
+
|| !/^[a-zA-Z0-9_-][a-zA-Z0-9._/-]*$/u.test(file.path)
|
|
15
|
+
|| file.path.split("/").some((part) => !part || part === "." || part === "..")) {
|
|
16
|
+
throw new Error("Script paths must be safe relative paths, for example scripts/check.sh.");
|
|
17
|
+
}
|
|
18
|
+
if (paths.has(file.path) || [...paths].some(path => path.startsWith(file.path + "/") || file.path.startsWith(path + "/"))) {
|
|
19
|
+
throw new Error("Script paths must be unique and cannot overlap directories.");
|
|
20
|
+
}
|
|
21
|
+
if (typeof file.content !== "string" || file.content.includes("\0"))
|
|
22
|
+
throw new Error("Scripts must be UTF-8 text without NUL.");
|
|
23
|
+
bytes += Buffer.byteLength(file.content);
|
|
24
|
+
if (bytes > 512 * 1024)
|
|
25
|
+
throw new Error("Job scripts exceed 512 KiB.");
|
|
26
|
+
paths.add(file.path);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export function prepareGlobalShell(configuration, workspace) {
|
|
30
|
+
requireKnownFields(configuration, ["kind", "env", "steps", "files"], "Global Shell Job");
|
|
31
|
+
if (configuration.kind !== "shell")
|
|
32
|
+
throw new Error("Not a Shell Job.");
|
|
33
|
+
validateJobScriptFiles(configuration.files);
|
|
34
|
+
// No repository applies. This hash identifies executable configuration,
|
|
35
|
+
// never a Project commit or an Integration candidate.
|
|
36
|
+
const head = createHash("sha1").update(JSON.stringify(configuration)).digest("hex");
|
|
37
|
+
const spec = { workspace, env: configuration.env, steps: configuration.steps, head };
|
|
38
|
+
validateJobExecutionConfiguration(spec);
|
|
39
|
+
return spec;
|
|
40
|
+
}
|
|
41
|
+
/** A fresh per-Run directory; never overwrite another run's files or follow
|
|
42
|
+
* a pre-existing workspace. The runner's exclusive ready receipt fences replay. */
|
|
43
|
+
export function materializeJobScripts(workspace, files) {
|
|
44
|
+
validateJobScriptFiles(files);
|
|
45
|
+
mkdirSync(workspace, { mode: 0o700 });
|
|
46
|
+
for (const file of files) {
|
|
47
|
+
const path = join(workspace, file.path);
|
|
48
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
49
|
+
writeFileSync(path, file.content, { flag: "wx", mode: 0o700 });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -62,9 +62,10 @@ export function jobRunSnapshot(run) {
|
|
|
62
62
|
...runReference(run), definitionId: run.definitionId, definitionRevision: run.definitionRevision,
|
|
63
63
|
owner: "owner" in run ? run.owner : null,
|
|
64
64
|
trigger: run.trigger, retryOf: run.retryOf ?? null,
|
|
65
|
-
executor: run.delegation ? { kind: "task", delegation: run.delegation } : "taskId" in run ? { kind: "shell" }
|
|
66
|
-
kind: "
|
|
67
|
-
|
|
65
|
+
executor: run.delegation ? { kind: "task", delegation: run.delegation } : "taskId" in run ? { kind: "shell" }
|
|
66
|
+
: run.action === "shell.exec" ? { kind: "shell", configuration: run.actionInput } : {
|
|
67
|
+
kind: "builtin", action: run.action, input: run.actionInput
|
|
68
|
+
},
|
|
68
69
|
target: run.delegation ? { taskId: run.delegation.taskId, workItemId: run.delegation.workItemId ?? null }
|
|
69
70
|
: { projectId: run.projectId ?? null, workspace: run.workspace, head: run.head },
|
|
70
71
|
execution: run.delegation ? null : { env: run.env, steps: run.steps, implementation: run.operation.implementation }
|
|
@@ -2,6 +2,7 @@ import { requireIdentity, requireKnownFields, requirePositiveInteger, requireTex
|
|
|
2
2
|
import { validateJobExecutionConfiguration, validateDurableJobOwner } from "./durableJob.js";
|
|
3
3
|
import { validateAgentJobConfiguration } from "./agentJob.js";
|
|
4
4
|
import { validateTaskJobConfiguration } from "./taskDelegation.js";
|
|
5
|
+
import { prepareGlobalShell } from "./globalShell.js";
|
|
5
6
|
export function jobScopeKey(scope) {
|
|
6
7
|
if (scope.kind === "global")
|
|
7
8
|
return "global";
|
|
@@ -44,6 +45,10 @@ export function validateJobDefinition(job) {
|
|
|
44
45
|
validateTaskJobConfiguration(config);
|
|
45
46
|
}
|
|
46
47
|
else if (job.scope.kind === "global") {
|
|
48
|
+
if (config.kind === "shell") {
|
|
49
|
+
prepareGlobalShell(config, "/job-validation");
|
|
50
|
+
return job;
|
|
51
|
+
}
|
|
47
52
|
if (config.kind !== "builtin")
|
|
48
53
|
throw new Error("Global Jobs require a controlled builtin action.");
|
|
49
54
|
requireKnownFields(config, ["kind", "action", "projectId", "input"], "Builtin Job");
|
|
@@ -54,7 +59,7 @@ export function validateJobDefinition(job) {
|
|
|
54
59
|
throw new Error("Builtin Job input is required.");
|
|
55
60
|
}
|
|
56
61
|
else {
|
|
57
|
-
if (config.kind !== "shell")
|
|
62
|
+
if (config.kind !== "shell" || !("owner" in config))
|
|
58
63
|
throw new Error("Task Jobs require a managed shell configuration.");
|
|
59
64
|
requireKnownFields(config, ["kind", "owner", "projectId", "head", "workspace", "env", "steps"], "Task Job");
|
|
60
65
|
requireIdentity(config.projectId, "Job Project id");
|
package/dist/job/jobRunner.js
CHANGED
|
@@ -13,6 +13,7 @@ import { join, resolve } from "node:path";
|
|
|
13
13
|
import { fileURLToPath } from "node:url";
|
|
14
14
|
import { jobStepDirectory } from "./stepDirectory.js";
|
|
15
15
|
import { prepareHttpPreview, isHttpPreview } from "./httpPreview.js";
|
|
16
|
+
import { materializeJobScripts, validateJobScriptFiles } from "./globalShell.js";
|
|
16
17
|
import { writeTextFileAtomically } from "../storage/durableFile.js";
|
|
17
18
|
import { currentProcessStartIdentity } from "../core/fileLockOwner.js";
|
|
18
19
|
const DEFAULT_STEP_TIMEOUT_MS = 30 * 60_000;
|
|
@@ -112,6 +113,8 @@ export async function runDurableJobRunner(specPath) {
|
|
|
112
113
|
let failedStep;
|
|
113
114
|
let preview;
|
|
114
115
|
try {
|
|
116
|
+
if (spec.files !== undefined)
|
|
117
|
+
materializeJobScripts(spec.workspace, spec.files);
|
|
115
118
|
for (let index = 0; index < spec.steps.length; index += 1) {
|
|
116
119
|
if (cancelled || existsSync(cancelPath)) {
|
|
117
120
|
cancelled = true;
|
|
@@ -278,6 +281,12 @@ function readSpec(specPath) {
|
|
|
278
281
|
if (typeof spec.artifactDir !== "string" || !spec.artifactDir.startsWith("/")) {
|
|
279
282
|
throw new Error("DurableJob spec artifactDir must be an absolute path.");
|
|
280
283
|
}
|
|
284
|
+
if (spec.files !== undefined) {
|
|
285
|
+
if (spec.scope !== "global" || spec.workspace !== join(spec.artifactDir, "workspace")) {
|
|
286
|
+
throw new Error("Job script snapshots require their own global Run workspace.");
|
|
287
|
+
}
|
|
288
|
+
validateJobScriptFiles(spec.files);
|
|
289
|
+
}
|
|
281
290
|
if (typeof spec.head !== "string" || spec.head.length === 0) {
|
|
282
291
|
throw new Error("DurableJob spec is missing head.");
|
|
283
292
|
}
|
|
@@ -4,6 +4,8 @@ import { requireProjectSyncTarget } from "./projectSync.js";
|
|
|
4
4
|
/** Internal compiled registration, not a caller identity accepted by RPC. */
|
|
5
5
|
const actions = new Set(["system.noop", "project.refresh"]);
|
|
6
6
|
export function requireServiceTarget(job) {
|
|
7
|
+
if (job.scope.kind === "global" && job.configuration.kind === "shell")
|
|
8
|
+
return;
|
|
7
9
|
if (job.scope.kind !== "global" || job.configuration.kind !== "builtin"
|
|
8
10
|
|| !actions.has(job.configuration.action))
|
|
9
11
|
throw new Error("Schedule service is not registered for this scope/action.");
|
|
@@ -4,28 +4,39 @@ import { hasRuntimeCleanupObligation, runtimeLifecycleTarget } from "../runtime/
|
|
|
4
4
|
import { currentProviderConversation } from "../runtime/providerRuntimeIdentity.js";
|
|
5
5
|
import { INBOX_CHECK_PREFIX } from "./inboxTurn.js";
|
|
6
6
|
const requestId = (agentId, nativeSessionId) => `${INBOX_CHECK_PREFIX}recovery:${createHash("sha256").update(JSON.stringify([agentId, nativeSessionId])).digest("hex")}`;
|
|
7
|
+
function recoveryCandidate(store, roleName) {
|
|
8
|
+
const sessions = store.getGlobalRoleSessionSet(roleName);
|
|
9
|
+
const session = sessions?.sessions[sessions.activeAgentId];
|
|
10
|
+
const binding = sessions?.providerBinding;
|
|
11
|
+
if (!session || session.status !== "active" || !session.nativeSessionId || !binding
|
|
12
|
+
|| binding.authority.owner !== "controller"
|
|
13
|
+
|| currentProviderConversation(binding).conversationId !== session.nativeSessionId
|
|
14
|
+
|| binding.run && ["submitting", "accepted", "delivery-unknown"].includes(binding.run.status)
|
|
15
|
+
|| hasRuntimeCleanupObligation(store.getWorkMailbox(runtimeLifecycleTarget({ scope: "global", roleName }))))
|
|
16
|
+
return;
|
|
17
|
+
if (store.queryGlobalRoleMessages(roleName, { handling: true, handlingExceptSession: {
|
|
18
|
+
agentId: session.agentId, nativeSessionId: session.nativeSessionId
|
|
19
|
+
}, limit: 0 }).count === 0)
|
|
20
|
+
return;
|
|
21
|
+
const id = requestId(session.agentId, session.nativeSessionId);
|
|
22
|
+
return { id, agentId: session.agentId, nativeSessionId: session.nativeSessionId,
|
|
23
|
+
existing: store.queryGlobalRoleMessages(roleName, { requestId: id, limit: 1 }).messages[0] };
|
|
24
|
+
}
|
|
7
25
|
/** Derive a single initialization notice from existing retained claims. New
|
|
8
26
|
* Sessions are created only by the existing authorized lifecycle path. A
|
|
9
27
|
* restart can finish notice creation, but cannot select or replace a Session. */
|
|
10
28
|
export function globalInboxRecovery(store, roleName, now) {
|
|
29
|
+
// The common idle/already-notified path must not reserve SQLite's writer.
|
|
30
|
+
// This snapshot is only a preflight; creation rechecks all authority, cleanup
|
|
31
|
+
// and retained-claim facts in the original atomic transaction.
|
|
32
|
+
const candidate = store.readTransaction(tx => recoveryCandidate(tx, roleName));
|
|
33
|
+
if (!candidate || candidate.existing)
|
|
34
|
+
return candidate?.existing;
|
|
11
35
|
return store.transaction(tx => {
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|| binding.authority.owner !== "controller"
|
|
17
|
-
|| currentProviderConversation(binding).conversationId !== session.nativeSessionId
|
|
18
|
-
|| binding.run && ["submitting", "accepted", "delivery-unknown"].includes(binding.run.status)
|
|
19
|
-
|| hasRuntimeCleanupObligation(tx.getWorkMailbox(runtimeLifecycleTarget({ scope: "global", roleName }))))
|
|
20
|
-
return;
|
|
21
|
-
if (tx.queryGlobalRoleMessages(roleName, { handling: true, handlingExceptSession: {
|
|
22
|
-
agentId: session.agentId, nativeSessionId: session.nativeSessionId
|
|
23
|
-
}, limit: 0 }).count === 0)
|
|
24
|
-
return;
|
|
25
|
-
const id = requestId(session.agentId, session.nativeSessionId);
|
|
26
|
-
const existing = tx.queryGlobalRoleMessages(roleName, { requestId: id, limit: 1 }).messages[0];
|
|
27
|
-
if (existing)
|
|
28
|
-
return existing;
|
|
36
|
+
const latest = recoveryCandidate(tx, roleName);
|
|
37
|
+
if (!latest || latest.existing)
|
|
38
|
+
return latest?.existing;
|
|
39
|
+
const { id, agentId, nativeSessionId } = latest;
|
|
29
40
|
const notice = {
|
|
30
41
|
...createGlobalRoleMessage(tx.nextGlobalRoleMessageId(), roleName, [
|
|
31
42
|
"Yui-Summary: This successor has retained inbox handling obligations; inspect original evidence before settlement.",
|
|
@@ -34,9 +45,9 @@ export function globalInboxRecovery(store, roleName, now) {
|
|
|
34
45
|
"After handling under current authority, explicitly acknowledge the original entry without replacing its claim.",
|
|
35
46
|
"Uncertain external effects require verification or a real user decision. New queued input remains separate."
|
|
36
47
|
].join("\n"), "system", { type: "system" }, now, {
|
|
37
|
-
inputControl: { action: "queue", requestId: id, expectedSessionId:
|
|
48
|
+
inputControl: { action: "queue", requestId: id, expectedSessionId: nativeSessionId }
|
|
38
49
|
}),
|
|
39
|
-
deliveryTarget: { agentId
|
|
50
|
+
deliveryTarget: { agentId, nativeSessionId }
|
|
40
51
|
};
|
|
41
52
|
tx.saveGlobalRoleMessage(notice);
|
|
42
53
|
return notice;
|
|
@@ -2,6 +2,12 @@ import { markGlobalRoleMessageNotDelivered } from "./message.js";
|
|
|
2
2
|
/** Settle the original queue entry when recovery ends with exact nondelivery.
|
|
3
3
|
* Cancellation of a submitted/unknown request is not negative Provider proof. */
|
|
4
4
|
export function settleGlobalRetryInput(store, roleName, binding, now) {
|
|
5
|
+
const settlement = globalRetryInputSettlement(store, roleName, binding, now);
|
|
6
|
+
if (settlement !== undefined)
|
|
7
|
+
store.updateGlobalRoleMessage(settlement);
|
|
8
|
+
}
|
|
9
|
+
/** Read-only proposal; callers applying it must recheck under their transaction. */
|
|
10
|
+
export function globalRetryInputSettlement(store, roleName, binding, now) {
|
|
5
11
|
const retry = binding?.retry;
|
|
6
12
|
if (retry === undefined || retry.input.kind !== "message" || retry.input.roleName !== roleName
|
|
7
13
|
|| !["cancelled", "exhausted", "needs-attention"].includes(retry.status)
|
|
@@ -11,5 +17,5 @@ export function settleGlobalRetryInput(store, roleName, binding, now) {
|
|
|
11
17
|
const message = store.listGlobalRoleMessages(roleName).find(m => m.id === messageId);
|
|
12
18
|
if (message === undefined || message.delivery !== undefined || message.notDelivered !== undefined)
|
|
13
19
|
return;
|
|
14
|
-
|
|
20
|
+
return markGlobalRoleMessageNotDelivered(message, `provider-retry-${retry.status}: ${retry.reason ?? retry.failureRef}`, now);
|
|
15
21
|
}
|
|
@@ -30,6 +30,10 @@ export function mapCodexAgentError(input) {
|
|
|
30
30
|
}
|
|
31
31
|
function mapCodexDisplayError(input) {
|
|
32
32
|
const text = `${input.message}\n${input.raw}`;
|
|
33
|
+
if (/Codex Session availability is systemError; no input was submitted\./u.test(text)) {
|
|
34
|
+
return { category: "session", code: "provider.session-system-error",
|
|
35
|
+
inputDisposition: "not-accepted", sessionDisposition: "unknown" };
|
|
36
|
+
}
|
|
33
37
|
if (/^server_error$/iu.test(input.message)) {
|
|
34
38
|
return recoverable("availability", "provider.server-error");
|
|
35
39
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { currentProviderConversation } from "./providerRuntimeIdentity.js";
|
|
2
|
+
import { providerRetrySummary } from "./providerRetry.js";
|
|
3
|
+
/** Read-only provenance. Writer selection is not live Host/transport health. */
|
|
4
|
+
export function globalSessionHealth(role, sessions) {
|
|
5
|
+
const session = sessions?.sessions[role.activeAgentId];
|
|
6
|
+
const binding = sessions?.providerBinding;
|
|
7
|
+
const matches = session !== undefined && binding != null
|
|
8
|
+
&& sessions?.activeAgentId === role.activeAgentId
|
|
9
|
+
&& currentProviderConversation(binding).conversationId === session.nativeSessionId;
|
|
10
|
+
// Conversation replacement retains the terminal Turn as audit evidence while
|
|
11
|
+
// advancing writer authority. Matching the new conversation alone is insufficient.
|
|
12
|
+
const turnMatches = matches && binding.run?.authorityEpoch === binding.authority.epoch;
|
|
13
|
+
const turn = turnMatches ? binding.run : null;
|
|
14
|
+
const retry = matches && binding.retry?.nativeSessionId === session?.nativeSessionId
|
|
15
|
+
&& binding.retry?.authorityEpoch === binding.authority.epoch
|
|
16
|
+
? providerRetrySummary(binding) : null;
|
|
17
|
+
const failure = turn?.status === "failed" || turn?.status === "rejected" ? turn.failure : undefined;
|
|
18
|
+
return {
|
|
19
|
+
desiredRevision: role.launchRevision,
|
|
20
|
+
effectiveRevision: session?.effective.sourceDesiredRevision ?? null,
|
|
21
|
+
configurationDrift: session === undefined ? null
|
|
22
|
+
: session.effective.sourceDesiredRevision !== role.launchRevision,
|
|
23
|
+
nativeSessionId: session?.nativeSessionId ?? null,
|
|
24
|
+
nativeTurn: turn == null ? null : {
|
|
25
|
+
attemptId: turn.attemptId, nativeTurnId: turn.nativeTurnId, status: turn.status
|
|
26
|
+
},
|
|
27
|
+
currentFailure: failure === undefined ? null : {
|
|
28
|
+
ref: failure.ref, source: failure.error.source, phase: failure.error.phase,
|
|
29
|
+
category: failure.error.category, code: failure.error.code,
|
|
30
|
+
inputDisposition: failure.error.inputDisposition,
|
|
31
|
+
sessionDisposition: failure.error.sessionDisposition
|
|
32
|
+
},
|
|
33
|
+
retry,
|
|
34
|
+
historicalBinding: binding != null && (!matches || (binding.run !== null && !turnMatches)),
|
|
35
|
+
hostObservation: "unobserved",
|
|
36
|
+
note: "Selected/active writer state is not delivery or terminal-observation health. "
|
|
37
|
+
+ "Inspect exact Host capabilities and runtime diagnostics; Controller restart does not refresh a live Host."
|
|
38
|
+
};
|
|
39
|
+
}
|
|
@@ -460,9 +460,9 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
460
460
|
if (stallCandidates.length === 0)
|
|
461
461
|
return [];
|
|
462
462
|
const diagnosticStartedAt = now.toISOString();
|
|
463
|
-
const finishDiagnostics = (outcome) => {
|
|
463
|
+
const finishDiagnostics = async (outcome) => {
|
|
464
464
|
for (const { task, role, run } of stallCandidates) {
|
|
465
|
-
store.recordRoleRunDiagnostic({
|
|
465
|
+
await store.recordRoleRunDiagnostic({
|
|
466
466
|
taskId: task.id,
|
|
467
467
|
roleName: role.name,
|
|
468
468
|
runId: run.id,
|
|
@@ -523,7 +523,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
523
523
|
// Health inspection is advisory. Unknown host state must leave the exact
|
|
524
524
|
// AgentRun/Session fence untouched. Closing this window schedules the next
|
|
525
525
|
// bounded diagnostic thirty minutes later instead of hot-looping.
|
|
526
|
-
finishDiagnostics("observation-error");
|
|
526
|
+
await finishDiagnostics("observation-error");
|
|
527
527
|
return [];
|
|
528
528
|
}
|
|
529
529
|
}
|
|
@@ -535,7 +535,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
535
535
|
catch {
|
|
536
536
|
// A malformed/partial provider snapshot is not evidence of a stall.
|
|
537
537
|
// Leave all fences untouched and let the next full pass retry.
|
|
538
|
-
finishDiagnostics("observation-error");
|
|
538
|
+
await finishDiagnostics("observation-error");
|
|
539
539
|
return [];
|
|
540
540
|
}
|
|
541
541
|
}
|
|
@@ -550,7 +550,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
550
550
|
let live;
|
|
551
551
|
try {
|
|
552
552
|
if (liveStatuses !== undefined && !liveStatuses.has(key)) {
|
|
553
|
-
finishDiagnostics("observation-error");
|
|
553
|
+
await finishDiagnostics("observation-error");
|
|
554
554
|
return [];
|
|
555
555
|
}
|
|
556
556
|
live = liveStatuses !== undefined
|
|
@@ -571,7 +571,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
571
571
|
// Without a complete live snapshot, especially for downstream AgentRuns,
|
|
572
572
|
// Leader classification would be unsafe. Treat the whole advisory pass
|
|
573
573
|
// as unknown instead of escalating a false stall.
|
|
574
|
-
finishDiagnostics("observation-error");
|
|
574
|
+
await finishDiagnostics("observation-error");
|
|
575
575
|
return [];
|
|
576
576
|
}
|
|
577
577
|
// A missing process is handled by the existing fenced liveness pass. This
|
|
@@ -669,7 +669,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
669
669
|
// A new semantic progress point closes the previous episode first. It
|
|
670
670
|
// may itself already be older than the window, in which case the same
|
|
671
671
|
// pass records the next AgentRun+progressAt episode below.
|
|
672
|
-
store.recordRoleRunProgress({
|
|
672
|
+
await store.recordRoleRunProgress({
|
|
673
673
|
taskId: candidate.task.id,
|
|
674
674
|
roleName: candidate.role.name,
|
|
675
675
|
runId: candidate.run.id,
|
|
@@ -702,7 +702,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
702
702
|
// diagnostic context, not duplicated as another Task-level alert.
|
|
703
703
|
if (previous?.progressAt === progressAt)
|
|
704
704
|
continue;
|
|
705
|
-
const persisted = store.recordRoleRunStall({
|
|
705
|
+
const persisted = await store.recordRoleRunStall({
|
|
706
706
|
taskId: candidate.task.id,
|
|
707
707
|
roleName: candidate.role.name,
|
|
708
708
|
runId: candidate.run.id,
|
|
@@ -738,7 +738,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
738
738
|
const previous = store.getRunProgressFacts(candidate.task.id, candidate.run.id)?.latestStall;
|
|
739
739
|
if (previous !== undefined
|
|
740
740
|
&& Date.parse(progressAt) > Date.parse(previous.progressAt)) {
|
|
741
|
-
store.recordRoleRunProgress({
|
|
741
|
+
await store.recordRoleRunProgress({
|
|
742
742
|
taskId: candidate.task.id,
|
|
743
743
|
roleName: candidate.role.name,
|
|
744
744
|
runId: candidate.run.id,
|
|
@@ -748,7 +748,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
748
748
|
});
|
|
749
749
|
}
|
|
750
750
|
}
|
|
751
|
-
finishDiagnostics("observed");
|
|
751
|
+
await finishDiagnostics("observed");
|
|
752
752
|
return raised;
|
|
753
753
|
}
|
|
754
754
|
function isStallCandidate(run, now, windowMs, lastDiagnosticFinishedAt, latestProgressAt) {
|
|
@@ -54,6 +54,22 @@ export function queryContextRecords(db, taskId, query) {
|
|
|
54
54
|
const from = `${table} r${candidate ? ", json_each(r.payload, '$.candidates') candidate" : ""}`;
|
|
55
55
|
const where = ["r.task_id = @taskId"];
|
|
56
56
|
const params = { taskId };
|
|
57
|
+
if (query.eventAttemptId !== undefined) {
|
|
58
|
+
if (query.family !== "task-event")
|
|
59
|
+
throw new Error("Attempt selection requires Events.");
|
|
60
|
+
where.push("json_extract(r.payload, '$.payload.attemptId') = @eventAttemptId");
|
|
61
|
+
params.eventAttemptId = query.eventAttemptId;
|
|
62
|
+
}
|
|
63
|
+
if (query.queuedForRole !== undefined) {
|
|
64
|
+
if (query.family !== "task-message")
|
|
65
|
+
throw new Error("Queued Role selection requires Messages.");
|
|
66
|
+
where.push(`json_extract(r.payload, '$.recipient.roleName') = @queuedRole
|
|
67
|
+
AND json_extract(r.payload, '$.recipient.ownerRunId') IS NOT NULL
|
|
68
|
+
AND json_extract(r.payload, '$.inbox') IS NULL
|
|
69
|
+
AND json_extract(r.payload, '$.continuation.runId') IS NULL
|
|
70
|
+
AND COALESCE(json_extract(r.payload, '$.inputControl.action'), '') != 'steer'`);
|
|
71
|
+
params.queuedRole = query.queuedForRole;
|
|
72
|
+
}
|
|
57
73
|
if (query.ids !== undefined) {
|
|
58
74
|
where.push(`${id} IN (SELECT value FROM json_each(@ids))`);
|
|
59
75
|
params.ids = JSON.stringify(query.ids);
|