@zq-silk/yui 2.17.0 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commandCatalog.js +2 -1
- package/dist/commands/controllerCommands.js +13 -4
- package/dist/commands/durableJobCommands.js +50 -3
- package/dist/commands/globalRoleCommands.js +18 -12
- package/dist/commands/inboxCommands.js +1 -0
- package/dist/commands/operatorCommands.js +10 -2
- package/dist/controlPlaneCli.js +21 -4
- package/dist/controller/clientRuntime.js +34 -13
- package/dist/controller/controller.js +35 -22
- package/dist/controller/diagnosticJournal.js +214 -0
- package/dist/controller/fileSchedulerStoreAdapter.js +139 -101
- package/dist/controller/globalInputDelivery.js +6 -2
- package/dist/controller/globalRuntimeAttention.js +11 -3
- package/dist/controller/jobControl.js +1 -1
- package/dist/controller/jobDefinitionControl.js +24 -13
- package/dist/controller/jobSupervisor.js +4 -0
- package/dist/controller/runtime.js +66 -4
- package/dist/controller/workerSchedulerStore.js +24 -0
- package/dist/core/boundedRpc.js +4 -0
- package/dist/core/controllerServer.js +1 -1
- package/dist/core/controllerTelemetry.js +92 -19
- package/dist/job/globalShell.js +51 -0
- package/dist/job/jobBusinessResult.js +4 -3
- package/dist/job/jobDefinition.js +6 -1
- package/dist/job/jobRunner.js +9 -0
- package/dist/job/scheduleService.js +2 -0
- package/dist/message/globalInboxRecovery.js +30 -19
- package/dist/message/globalProviderRetry.js +7 -1
- package/dist/runtime/builtinAgentErrorMappers.js +4 -0
- package/dist/runtime/globalSessionHealth.js +39 -0
- package/dist/scheduler/roleRunStall.js +10 -10
- package/dist/storage/persistenceWorker.js +41 -19
- package/dist/storage/sqliteSchema.js +35 -11
- package/dist/storage/sqliteStore.js +86 -8
- package/dist/storage/storageVersions.js +1 -1
- package/dist/storage/storeRpc.js +25 -11
- package/dist/storage/taskReadRevisionSchema.js +39 -0
- package/dist/web/assets/client/app/catalog.js +0 -29
- package/dist/web/assets/client/app/jobs.js +115 -0
- package/dist/web/assets/client/app/main.js +17 -6
- package/dist/web/assets/client/app/manage.js +1 -0
- package/dist/web/assets/client/app/settings.js +1 -5
- package/dist/web/assets/client/i18n/en.js +203 -18
- package/dist/web/assets/client/i18n/zh-CN.js +202 -17
- package/dist/web/assets/client/index.js +2 -0
- package/dist/web/assets/client/layout/manageShell.js +61 -15
- package/dist/web/assets/client/layout/router.js +2 -1
- package/dist/web/assets/client/lib/prefs.js +1 -1
- package/dist/web/assets/client/views/jobs.js +1042 -160
- package/dist/web/assets/client/views/sidebar.js +3 -0
- package/dist/web/assets/shared/manageNav.js +3 -1
- package/dist/web/assets/shell/appRail.js +2 -1
- package/dist/web/assets/shell/manage.js +7 -2
- package/dist/web/assets/shell/models.js +4 -3
- package/dist/web/assets/shell/settingsContent.js +0 -3
- package/dist/web/assets/shell/sidebar.js +0 -1
- package/dist/web/assets/styles/views/jobs.js +301 -12
- package/dist/web/assets/styles/views/sidebar.js +1 -2
- package/dist/web/webServer.js +28 -1
- package/docs/global-shell-jobs.md +78 -0
- package/docs/job-runs.md +6 -3
- package/docs/managed-turn-and-session-runtime.md +74 -0
- package/docs/managed-turn-and-session-runtime.zh-CN.md +51 -0
- package/docs/release-workflow.md +43 -0
- package/docs/release-workflow.zh-CN.md +29 -0
- package/docs/storage-baseline.md +14 -1
- package/docs/storage-baseline.zh-CN.md +6 -1
- package/docs/testing/verification-levels.md +2 -2
- package/docs/testing/verification-levels.zh-CN.md +2 -2
- package/package.json +1 -1
|
@@ -4,28 +4,39 @@ import { hasRuntimeCleanupObligation, runtimeLifecycleTarget } from "../runtime/
|
|
|
4
4
|
import { currentProviderConversation } from "../runtime/providerRuntimeIdentity.js";
|
|
5
5
|
import { INBOX_CHECK_PREFIX } from "./inboxTurn.js";
|
|
6
6
|
const requestId = (agentId, nativeSessionId) => `${INBOX_CHECK_PREFIX}recovery:${createHash("sha256").update(JSON.stringify([agentId, nativeSessionId])).digest("hex")}`;
|
|
7
|
+
function recoveryCandidate(store, roleName) {
|
|
8
|
+
const sessions = store.getGlobalRoleSessionSet(roleName);
|
|
9
|
+
const session = sessions?.sessions[sessions.activeAgentId];
|
|
10
|
+
const binding = sessions?.providerBinding;
|
|
11
|
+
if (!session || session.status !== "active" || !session.nativeSessionId || !binding
|
|
12
|
+
|| binding.authority.owner !== "controller"
|
|
13
|
+
|| currentProviderConversation(binding).conversationId !== session.nativeSessionId
|
|
14
|
+
|| binding.run && ["submitting", "accepted", "delivery-unknown"].includes(binding.run.status)
|
|
15
|
+
|| hasRuntimeCleanupObligation(store.getWorkMailbox(runtimeLifecycleTarget({ scope: "global", roleName }))))
|
|
16
|
+
return;
|
|
17
|
+
if (store.queryGlobalRoleMessages(roleName, { handling: true, handlingExceptSession: {
|
|
18
|
+
agentId: session.agentId, nativeSessionId: session.nativeSessionId
|
|
19
|
+
}, limit: 0 }).count === 0)
|
|
20
|
+
return;
|
|
21
|
+
const id = requestId(session.agentId, session.nativeSessionId);
|
|
22
|
+
return { id, agentId: session.agentId, nativeSessionId: session.nativeSessionId,
|
|
23
|
+
existing: store.queryGlobalRoleMessages(roleName, { requestId: id, limit: 1 }).messages[0] };
|
|
24
|
+
}
|
|
7
25
|
/** Derive a single initialization notice from existing retained claims. New
|
|
8
26
|
* Sessions are created only by the existing authorized lifecycle path. A
|
|
9
27
|
* restart can finish notice creation, but cannot select or replace a Session. */
|
|
10
28
|
export function globalInboxRecovery(store, roleName, now) {
|
|
29
|
+
// The common idle/already-notified path must not reserve SQLite's writer.
|
|
30
|
+
// This snapshot is only a preflight; creation rechecks all authority, cleanup
|
|
31
|
+
// and retained-claim facts in the original atomic transaction.
|
|
32
|
+
const candidate = store.readTransaction(tx => recoveryCandidate(tx, roleName));
|
|
33
|
+
if (!candidate || candidate.existing)
|
|
34
|
+
return candidate?.existing;
|
|
11
35
|
return store.transaction(tx => {
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|| binding.authority.owner !== "controller"
|
|
17
|
-
|| currentProviderConversation(binding).conversationId !== session.nativeSessionId
|
|
18
|
-
|| binding.run && ["submitting", "accepted", "delivery-unknown"].includes(binding.run.status)
|
|
19
|
-
|| hasRuntimeCleanupObligation(tx.getWorkMailbox(runtimeLifecycleTarget({ scope: "global", roleName }))))
|
|
20
|
-
return;
|
|
21
|
-
if (tx.queryGlobalRoleMessages(roleName, { handling: true, handlingExceptSession: {
|
|
22
|
-
agentId: session.agentId, nativeSessionId: session.nativeSessionId
|
|
23
|
-
}, limit: 0 }).count === 0)
|
|
24
|
-
return;
|
|
25
|
-
const id = requestId(session.agentId, session.nativeSessionId);
|
|
26
|
-
const existing = tx.queryGlobalRoleMessages(roleName, { requestId: id, limit: 1 }).messages[0];
|
|
27
|
-
if (existing)
|
|
28
|
-
return existing;
|
|
36
|
+
const latest = recoveryCandidate(tx, roleName);
|
|
37
|
+
if (!latest || latest.existing)
|
|
38
|
+
return latest?.existing;
|
|
39
|
+
const { id, agentId, nativeSessionId } = latest;
|
|
29
40
|
const notice = {
|
|
30
41
|
...createGlobalRoleMessage(tx.nextGlobalRoleMessageId(), roleName, [
|
|
31
42
|
"Yui-Summary: This successor has retained inbox handling obligations; inspect original evidence before settlement.",
|
|
@@ -34,9 +45,9 @@ export function globalInboxRecovery(store, roleName, now) {
|
|
|
34
45
|
"After handling under current authority, explicitly acknowledge the original entry without replacing its claim.",
|
|
35
46
|
"Uncertain external effects require verification or a real user decision. New queued input remains separate."
|
|
36
47
|
].join("\n"), "system", { type: "system" }, now, {
|
|
37
|
-
inputControl: { action: "queue", requestId: id, expectedSessionId:
|
|
48
|
+
inputControl: { action: "queue", requestId: id, expectedSessionId: nativeSessionId }
|
|
38
49
|
}),
|
|
39
|
-
deliveryTarget: { agentId
|
|
50
|
+
deliveryTarget: { agentId, nativeSessionId }
|
|
40
51
|
};
|
|
41
52
|
tx.saveGlobalRoleMessage(notice);
|
|
42
53
|
return notice;
|
|
@@ -2,6 +2,12 @@ import { markGlobalRoleMessageNotDelivered } from "./message.js";
|
|
|
2
2
|
/** Settle the original queue entry when recovery ends with exact nondelivery.
|
|
3
3
|
* Cancellation of a submitted/unknown request is not negative Provider proof. */
|
|
4
4
|
export function settleGlobalRetryInput(store, roleName, binding, now) {
|
|
5
|
+
const settlement = globalRetryInputSettlement(store, roleName, binding, now);
|
|
6
|
+
if (settlement !== undefined)
|
|
7
|
+
store.updateGlobalRoleMessage(settlement);
|
|
8
|
+
}
|
|
9
|
+
/** Read-only proposal; callers applying it must recheck under their transaction. */
|
|
10
|
+
export function globalRetryInputSettlement(store, roleName, binding, now) {
|
|
5
11
|
const retry = binding?.retry;
|
|
6
12
|
if (retry === undefined || retry.input.kind !== "message" || retry.input.roleName !== roleName
|
|
7
13
|
|| !["cancelled", "exhausted", "needs-attention"].includes(retry.status)
|
|
@@ -11,5 +17,5 @@ export function settleGlobalRetryInput(store, roleName, binding, now) {
|
|
|
11
17
|
const message = store.listGlobalRoleMessages(roleName).find(m => m.id === messageId);
|
|
12
18
|
if (message === undefined || message.delivery !== undefined || message.notDelivered !== undefined)
|
|
13
19
|
return;
|
|
14
|
-
|
|
20
|
+
return markGlobalRoleMessageNotDelivered(message, `provider-retry-${retry.status}: ${retry.reason ?? retry.failureRef}`, now);
|
|
15
21
|
}
|
|
@@ -30,6 +30,10 @@ export function mapCodexAgentError(input) {
|
|
|
30
30
|
}
|
|
31
31
|
function mapCodexDisplayError(input) {
|
|
32
32
|
const text = `${input.message}\n${input.raw}`;
|
|
33
|
+
if (/Codex Session availability is systemError; no input was submitted\./u.test(text)) {
|
|
34
|
+
return { category: "session", code: "provider.session-system-error",
|
|
35
|
+
inputDisposition: "not-accepted", sessionDisposition: "unknown" };
|
|
36
|
+
}
|
|
33
37
|
if (/^server_error$/iu.test(input.message)) {
|
|
34
38
|
return recoverable("availability", "provider.server-error");
|
|
35
39
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { currentProviderConversation } from "./providerRuntimeIdentity.js";
|
|
2
|
+
import { providerRetrySummary } from "./providerRetry.js";
|
|
3
|
+
/** Read-only provenance. Writer selection is not live Host/transport health. */
|
|
4
|
+
export function globalSessionHealth(role, sessions) {
|
|
5
|
+
const session = sessions?.sessions[role.activeAgentId];
|
|
6
|
+
const binding = sessions?.providerBinding;
|
|
7
|
+
const matches = session !== undefined && binding != null
|
|
8
|
+
&& sessions?.activeAgentId === role.activeAgentId
|
|
9
|
+
&& currentProviderConversation(binding).conversationId === session.nativeSessionId;
|
|
10
|
+
// Conversation replacement retains the terminal Turn as audit evidence while
|
|
11
|
+
// advancing writer authority. Matching the new conversation alone is insufficient.
|
|
12
|
+
const turnMatches = matches && binding.run?.authorityEpoch === binding.authority.epoch;
|
|
13
|
+
const turn = turnMatches ? binding.run : null;
|
|
14
|
+
const retry = matches && binding.retry?.nativeSessionId === session?.nativeSessionId
|
|
15
|
+
&& binding.retry?.authorityEpoch === binding.authority.epoch
|
|
16
|
+
? providerRetrySummary(binding) : null;
|
|
17
|
+
const failure = turn?.status === "failed" || turn?.status === "rejected" ? turn.failure : undefined;
|
|
18
|
+
return {
|
|
19
|
+
desiredRevision: role.launchRevision,
|
|
20
|
+
effectiveRevision: session?.effective.sourceDesiredRevision ?? null,
|
|
21
|
+
configurationDrift: session === undefined ? null
|
|
22
|
+
: session.effective.sourceDesiredRevision !== role.launchRevision,
|
|
23
|
+
nativeSessionId: session?.nativeSessionId ?? null,
|
|
24
|
+
nativeTurn: turn == null ? null : {
|
|
25
|
+
attemptId: turn.attemptId, nativeTurnId: turn.nativeTurnId, status: turn.status
|
|
26
|
+
},
|
|
27
|
+
currentFailure: failure === undefined ? null : {
|
|
28
|
+
ref: failure.ref, source: failure.error.source, phase: failure.error.phase,
|
|
29
|
+
category: failure.error.category, code: failure.error.code,
|
|
30
|
+
inputDisposition: failure.error.inputDisposition,
|
|
31
|
+
sessionDisposition: failure.error.sessionDisposition
|
|
32
|
+
},
|
|
33
|
+
retry,
|
|
34
|
+
historicalBinding: binding != null && (!matches || (binding.run !== null && !turnMatches)),
|
|
35
|
+
hostObservation: "unobserved",
|
|
36
|
+
note: "Selected/active writer state is not delivery or terminal-observation health. "
|
|
37
|
+
+ "Inspect exact Host capabilities and runtime diagnostics; Controller restart does not refresh a live Host."
|
|
38
|
+
};
|
|
39
|
+
}
|
|
@@ -460,9 +460,9 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
460
460
|
if (stallCandidates.length === 0)
|
|
461
461
|
return [];
|
|
462
462
|
const diagnosticStartedAt = now.toISOString();
|
|
463
|
-
const finishDiagnostics = (outcome) => {
|
|
463
|
+
const finishDiagnostics = async (outcome) => {
|
|
464
464
|
for (const { task, role, run } of stallCandidates) {
|
|
465
|
-
store.recordRoleRunDiagnostic({
|
|
465
|
+
await store.recordRoleRunDiagnostic({
|
|
466
466
|
taskId: task.id,
|
|
467
467
|
roleName: role.name,
|
|
468
468
|
runId: run.id,
|
|
@@ -523,7 +523,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
523
523
|
// Health inspection is advisory. Unknown host state must leave the exact
|
|
524
524
|
// AgentRun/Session fence untouched. Closing this window schedules the next
|
|
525
525
|
// bounded diagnostic thirty minutes later instead of hot-looping.
|
|
526
|
-
finishDiagnostics("observation-error");
|
|
526
|
+
await finishDiagnostics("observation-error");
|
|
527
527
|
return [];
|
|
528
528
|
}
|
|
529
529
|
}
|
|
@@ -535,7 +535,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
535
535
|
catch {
|
|
536
536
|
// A malformed/partial provider snapshot is not evidence of a stall.
|
|
537
537
|
// Leave all fences untouched and let the next full pass retry.
|
|
538
|
-
finishDiagnostics("observation-error");
|
|
538
|
+
await finishDiagnostics("observation-error");
|
|
539
539
|
return [];
|
|
540
540
|
}
|
|
541
541
|
}
|
|
@@ -550,7 +550,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
550
550
|
let live;
|
|
551
551
|
try {
|
|
552
552
|
if (liveStatuses !== undefined && !liveStatuses.has(key)) {
|
|
553
|
-
finishDiagnostics("observation-error");
|
|
553
|
+
await finishDiagnostics("observation-error");
|
|
554
554
|
return [];
|
|
555
555
|
}
|
|
556
556
|
live = liveStatuses !== undefined
|
|
@@ -571,7 +571,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
571
571
|
// Without a complete live snapshot, especially for downstream AgentRuns,
|
|
572
572
|
// Leader classification would be unsafe. Treat the whole advisory pass
|
|
573
573
|
// as unknown instead of escalating a false stall.
|
|
574
|
-
finishDiagnostics("observation-error");
|
|
574
|
+
await finishDiagnostics("observation-error");
|
|
575
575
|
return [];
|
|
576
576
|
}
|
|
577
577
|
// A missing process is handled by the existing fenced liveness pass. This
|
|
@@ -669,7 +669,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
669
669
|
// A new semantic progress point closes the previous episode first. It
|
|
670
670
|
// may itself already be older than the window, in which case the same
|
|
671
671
|
// pass records the next AgentRun+progressAt episode below.
|
|
672
|
-
store.recordRoleRunProgress({
|
|
672
|
+
await store.recordRoleRunProgress({
|
|
673
673
|
taskId: candidate.task.id,
|
|
674
674
|
roleName: candidate.role.name,
|
|
675
675
|
runId: candidate.run.id,
|
|
@@ -702,7 +702,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
702
702
|
// diagnostic context, not duplicated as another Task-level alert.
|
|
703
703
|
if (previous?.progressAt === progressAt)
|
|
704
704
|
continue;
|
|
705
|
-
const persisted = store.recordRoleRunStall({
|
|
705
|
+
const persisted = await store.recordRoleRunStall({
|
|
706
706
|
taskId: candidate.task.id,
|
|
707
707
|
roleName: candidate.role.name,
|
|
708
708
|
runId: candidate.run.id,
|
|
@@ -738,7 +738,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
738
738
|
const previous = store.getRunProgressFacts(candidate.task.id, candidate.run.id)?.latestStall;
|
|
739
739
|
if (previous !== undefined
|
|
740
740
|
&& Date.parse(progressAt) > Date.parse(previous.progressAt)) {
|
|
741
|
-
store.recordRoleRunProgress({
|
|
741
|
+
await store.recordRoleRunProgress({
|
|
742
742
|
taskId: candidate.task.id,
|
|
743
743
|
roleName: candidate.role.name,
|
|
744
744
|
runId: candidate.run.id,
|
|
@@ -748,7 +748,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
748
748
|
});
|
|
749
749
|
}
|
|
750
750
|
}
|
|
751
|
-
finishDiagnostics("observed");
|
|
751
|
+
await finishDiagnostics("observed");
|
|
752
752
|
return raised;
|
|
753
753
|
}
|
|
754
754
|
function isStallCandidate(run, now, windowMs, lastDiagnosticFinishedAt, latestProgressAt) {
|
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
import { runRpcWorker } from "../core/boundedRpc.js";
|
|
34
34
|
import { SqliteTaskStore } from "./sqliteStore.js";
|
|
35
35
|
import { StorageRecordError } from "./taskStore.js";
|
|
36
|
+
import { randomUUID } from "node:crypto";
|
|
37
|
+
import { ControllerWorkObserver, compactWorkObservation } from "../core/controllerTelemetry.js";
|
|
36
38
|
// -- method invocation -------------------------------------------------------
|
|
37
39
|
function invokeMethod(target, method, args) {
|
|
38
40
|
const fn = target[method];
|
|
@@ -41,6 +43,13 @@ function invokeMethod(target, method, args) {
|
|
|
41
43
|
}
|
|
42
44
|
return fn.apply(target, args);
|
|
43
45
|
}
|
|
46
|
+
const workerInstance = randomUUID();
|
|
47
|
+
const work = new ControllerWorkObserver();
|
|
48
|
+
function observation() {
|
|
49
|
+
return { instance: workerInstance, work: compactWorkObservation(work),
|
|
50
|
+
database: state.writer?.storageObservationMetrics() ?? null,
|
|
51
|
+
reads: state.observer?.compactReadObservationMetrics?.() ?? null };
|
|
52
|
+
}
|
|
44
53
|
/**
|
|
45
54
|
* Sentinel returned by a handler when the durable outbox already records the
|
|
46
55
|
* request's effect (§5.4). The host's `result` builder maps it to an
|
|
@@ -80,7 +89,10 @@ async function handleInit(request) {
|
|
|
80
89
|
state.observer = {
|
|
81
90
|
invoke(method, args) {
|
|
82
91
|
return invokeMethod(host, method, args);
|
|
83
|
-
}
|
|
92
|
+
},
|
|
93
|
+
...(host.compactReadObservationMetrics === undefined ? {} : {
|
|
94
|
+
compactReadObservationMetrics: () => host.compactReadObservationMetrics()
|
|
95
|
+
})
|
|
84
96
|
};
|
|
85
97
|
}
|
|
86
98
|
}
|
|
@@ -162,21 +174,31 @@ runRpcWorker({
|
|
|
162
174
|
},
|
|
163
175
|
init: (request) => handleInit(request),
|
|
164
176
|
handle: async (request) => {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
177
|
+
const finish = work.start(request.kind, "requestId" in request ? request.requestId : undefined);
|
|
178
|
+
try {
|
|
179
|
+
switch (request.kind) {
|
|
180
|
+
case "call":
|
|
181
|
+
return await handleCall(request);
|
|
182
|
+
case "transaction":
|
|
183
|
+
return await handleTransaction(request);
|
|
184
|
+
case "observer":
|
|
185
|
+
return await handleObserver(request);
|
|
186
|
+
case "telemetry":
|
|
187
|
+
if (state.writer === undefined)
|
|
188
|
+
throw new Error("Worker not initialized.");
|
|
189
|
+
return state.writer.flushTelemetry(request.entries, request.runCap);
|
|
190
|
+
case "init":
|
|
191
|
+
case "cancel":
|
|
192
|
+
case "shutdown":
|
|
193
|
+
throw new Error(`Unexpected request kind for handle: ${request.kind}`);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
finish(true);
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
finally {
|
|
201
|
+
finish();
|
|
180
202
|
}
|
|
181
203
|
},
|
|
182
204
|
cancel: (requestId) => {
|
|
@@ -196,7 +218,7 @@ runRpcWorker({
|
|
|
196
218
|
},
|
|
197
219
|
ready: () => ({ kind: "ready" }),
|
|
198
220
|
result: (requestId, value) => (value === ALREADY_APPLIED
|
|
199
|
-
? { kind: "already-applied", requestId }
|
|
200
|
-
: { kind: "result", requestId, result: value }),
|
|
201
|
-
error: (requestId, error) => ({ kind: "error", requestId, error })
|
|
221
|
+
? { kind: "already-applied", requestId, observation: observation() }
|
|
222
|
+
: { kind: "result", requestId, result: value, observation: observation() }),
|
|
223
|
+
error: (requestId, error) => ({ kind: "error", requestId, error, observation: observation() })
|
|
202
224
|
});
|
|
@@ -5,6 +5,7 @@ import { migrateTaskMainWorkspaces } from "./upgrade/taskMainWorkspaceMigration.
|
|
|
5
5
|
import { migrateProjectKnowledge } from "./upgrade/projectKnowledgeMigration.js";
|
|
6
6
|
import { JOB_SCHEMA_SQL } from "./jobSchema.js";
|
|
7
7
|
import { AGENT_JOB_SCHEMA_SQL } from "./agentJobSchema.js";
|
|
8
|
+
import { TASK_READ_REVISION_SCHEMA_SQL } from "./taskReadRevisionSchema.js";
|
|
8
9
|
import { SCHEDULE_SCHEMA_SQL } from "./scheduleSchema.js";
|
|
9
10
|
import { GLOBAL_SESSION_TABLE_SQL, SESSION_HISTORY_UPGRADE_SQL } from "./sessionHistorySchema.js";
|
|
10
11
|
import { migrateJobDefinitions } from "./upgrade/jobDefinitionMigration.js";
|
|
@@ -13,7 +14,7 @@ import { migrateScheduleDispatch } from "./upgrade/scheduleDispatchMigration.js"
|
|
|
13
14
|
import { migrateProjectSyncClock } from "./upgrade/projectSyncClockMigration.js";
|
|
14
15
|
import { CURRENT_STORAGE_VERSION, MIN_SUPPORTED_STORAGE_VERSION, STORAGE_FORMAT, isMinorStorageUpgrade, isStorageVersion, storageVersionParts } from "./storageVersions.js";
|
|
15
16
|
export { BASELINE_SCHEMA_SQL };
|
|
16
|
-
export const SQLITE_SCHEMA_TABLES = [...BASELINE_SCHEMA_TABLES, "job_definitions", "global_job_runs", "schedules", "agent_job_runs"];
|
|
17
|
+
export const SQLITE_SCHEMA_TABLES = [...BASELINE_SCHEMA_TABLES, "job_definitions", "global_job_runs", "schedules", "agent_job_runs", "task_read_revisions"];
|
|
17
18
|
export const TELEMETRY_KEEP_PER_RUN = 200;
|
|
18
19
|
export const TELEMETRY_RUN_CAP = 50_000;
|
|
19
20
|
export const BASELINE_SCHEMA_CHECKSUM = createHash("sha256").update(BASELINE_SCHEMA_SQL).digest("hex");
|
|
@@ -23,7 +24,9 @@ export const SCHEDULE_SCHEMA_CHECKSUM = createHash("sha256").update(SCHEDULE_SCH
|
|
|
23
24
|
// Fresh Homes create the final layout directly; published baseline SQL stays immutable.
|
|
24
25
|
const SESSION_SCHEMA_SQL = SCHEDULE_SCHEMA_SQL_COMPLETE.replace(/CREATE TABLE global_role_session_sets \([\s\S]*?\n\);/, GLOBAL_SESSION_TABLE_SQL);
|
|
25
26
|
const SESSION_SCHEMA_CHECKSUM = createHash("sha256").update(SESSION_SCHEMA_SQL).digest("hex");
|
|
26
|
-
export const
|
|
27
|
+
export const PRE_READ_REVISION_SCHEMA_SQL = SESSION_SCHEMA_SQL + AGENT_JOB_SCHEMA_SQL;
|
|
28
|
+
const PRE_READ_REVISION_SCHEMA_CHECKSUM = createHash("sha256").update(PRE_READ_REVISION_SCHEMA_SQL).digest("hex");
|
|
29
|
+
export const CURRENT_SCHEMA_SQL = PRE_READ_REVISION_SCHEMA_SQL + TASK_READ_REVISION_SCHEMA_SQL;
|
|
27
30
|
export const CURRENT_SCHEMA_CHECKSUM = createHash("sha256").update(CURRENT_SCHEMA_SQL).digest("hex");
|
|
28
31
|
export class SqliteSchemaError extends Error {
|
|
29
32
|
code;
|
|
@@ -230,7 +233,7 @@ const MINOR_UPGRADES = Object.freeze([{
|
|
|
230
233
|
name: "job-agent-execution",
|
|
231
234
|
introducedIn: "next",
|
|
232
235
|
sourceChecksum: SESSION_SCHEMA_CHECKSUM,
|
|
233
|
-
targetChecksum:
|
|
236
|
+
targetChecksum: PRE_READ_REVISION_SCHEMA_CHECKSUM,
|
|
234
237
|
// Existing records and immutable artifacts remain byte-for-byte unchanged.
|
|
235
238
|
// Declares Agent definitions/Runs and optional Job ownership of a non-Task Role.
|
|
236
239
|
sql: AGENT_JOB_SCHEMA_SQL
|
|
@@ -239,8 +242,8 @@ const MINOR_UPGRADES = Object.freeze([{
|
|
|
239
242
|
toVersion: "1.21",
|
|
240
243
|
name: "project-sync-clock",
|
|
241
244
|
introducedIn: "next",
|
|
242
|
-
sourceChecksum:
|
|
243
|
-
targetChecksum:
|
|
245
|
+
sourceChecksum: PRE_READ_REVISION_SCHEMA_CHECKSUM,
|
|
246
|
+
targetChecksum: PRE_READ_REVISION_SCHEMA_CHECKSUM,
|
|
244
247
|
sql: "",
|
|
245
248
|
dataMigration: "project-sync-clock"
|
|
246
249
|
}, {
|
|
@@ -248,11 +251,31 @@ const MINOR_UPGRADES = Object.freeze([{
|
|
|
248
251
|
toVersion: "1.22",
|
|
249
252
|
name: "role-profile-availability",
|
|
250
253
|
introducedIn: "next",
|
|
251
|
-
sourceChecksum:
|
|
252
|
-
targetChecksum:
|
|
254
|
+
sourceChecksum: PRE_READ_REVISION_SCHEMA_CHECKSUM,
|
|
255
|
+
targetChecksum: PRE_READ_REVISION_SCHEMA_CHECKSUM,
|
|
253
256
|
// Optional enabled flag; absence means enabled. Preserve all historical
|
|
254
257
|
// Role/Profile bytes, frozen contexts and Session snapshots.
|
|
255
258
|
sql: ""
|
|
259
|
+
}, {
|
|
260
|
+
fromVersion: "1.22",
|
|
261
|
+
toVersion: "1.23",
|
|
262
|
+
name: "global-shell-scripts",
|
|
263
|
+
introducedIn: "next",
|
|
264
|
+
sourceChecksum: PRE_READ_REVISION_SCHEMA_CHECKSUM,
|
|
265
|
+
targetChecksum: PRE_READ_REVISION_SCHEMA_CHECKSUM,
|
|
266
|
+
// Additive global Shell definitions with revisioned script text and exact
|
|
267
|
+
// Run snapshots. Historical definitions/Runs stay byte-for-byte unchanged.
|
|
268
|
+
sql: ""
|
|
269
|
+
}, {
|
|
270
|
+
fromVersion: "1.23",
|
|
271
|
+
toVersion: "1.24",
|
|
272
|
+
name: "task-read-revisions",
|
|
273
|
+
introducedIn: "next",
|
|
274
|
+
sourceChecksum: PRE_READ_REVISION_SCHEMA_CHECKSUM,
|
|
275
|
+
targetChecksum: CURRENT_SCHEMA_CHECKSUM,
|
|
276
|
+
// No payload rewriting or historical revision invention. A newly opened
|
|
277
|
+
// reader starts empty; zero is the initial token until the first mutation.
|
|
278
|
+
sql: TASK_READ_REVISION_SCHEMA_SQL
|
|
256
279
|
}]);
|
|
257
280
|
export function storageMinorUpgradePlan(from) {
|
|
258
281
|
if (from === CURRENT_STORAGE_VERSION)
|
|
@@ -282,10 +305,11 @@ export function validateSqliteSchemaObjects(db, version = CURRENT_STORAGE_VERSIO
|
|
|
282
305
|
const checksum = version === CURRENT_STORAGE_VERSION
|
|
283
306
|
? CURRENT_SCHEMA_CHECKSUM : storageMinorUpgradePlan(version)?.[0]?.sourceChecksum;
|
|
284
307
|
const sql = checksum === CURRENT_SCHEMA_CHECKSUM ? CURRENT_SCHEMA_SQL
|
|
285
|
-
: checksum ===
|
|
286
|
-
: checksum ===
|
|
287
|
-
: checksum ===
|
|
288
|
-
: checksum ===
|
|
308
|
+
: checksum === PRE_READ_REVISION_SCHEMA_CHECKSUM ? PRE_READ_REVISION_SCHEMA_SQL
|
|
309
|
+
: checksum === SESSION_SCHEMA_CHECKSUM ? SESSION_SCHEMA_SQL
|
|
310
|
+
: checksum === SCHEDULE_SCHEMA_CHECKSUM ? SCHEDULE_SCHEMA_SQL_COMPLETE
|
|
311
|
+
: checksum === JOB_SCHEMA_CHECKSUM ? BASELINE_SCHEMA_SQL + JOB_SCHEMA_SQL
|
|
312
|
+
: checksum === BASELINE_SCHEMA_CHECKSUM ? BASELINE_SCHEMA_SQL : undefined;
|
|
289
313
|
if (sql === undefined || checksum === undefined)
|
|
290
314
|
throw new SqliteSchemaError(`unsupported schema definition for storage ${version}.`);
|
|
291
315
|
let expectedObjects = schemaObjectsByChecksum.get(checksum);
|
|
@@ -34,6 +34,7 @@ import { validateAgentJobRun, assertAgentJobRunTransition } from "../job/agentJo
|
|
|
34
34
|
* in-process contract or deferred validation phase.
|
|
35
35
|
*/
|
|
36
36
|
import Database from "../sqlite/driver.js";
|
|
37
|
+
import { ControllerWorkObserver, compactWorkObservation } from "../core/controllerTelemetry.js";
|
|
37
38
|
import { findNativeSessionSets } from "./nativeSessionLookup.js";
|
|
38
39
|
import { existsSync, mkdirSync } from "node:fs";
|
|
39
40
|
import { join } from "node:path";
|
|
@@ -141,10 +142,37 @@ function assertRunAdmission(task, run) {
|
|
|
141
142
|
}
|
|
142
143
|
}
|
|
143
144
|
export class SqliteTaskStore {
|
|
145
|
+
#payloadReads = new ControllerWorkObserver();
|
|
146
|
+
#transactions = {
|
|
147
|
+
"write-lock": new ControllerWorkObserver(),
|
|
148
|
+
"write-body": new ControllerWorkObserver(),
|
|
149
|
+
"transaction-commit": new ControllerWorkObserver()
|
|
150
|
+
};
|
|
151
|
+
storageObservationMetrics() {
|
|
152
|
+
return {
|
|
153
|
+
lock: compactWorkObservation(this.#transactions["write-lock"]),
|
|
154
|
+
body: compactWorkObservation(this.#transactions["write-body"]),
|
|
155
|
+
commit: compactWorkObservation(this.#transactions["transaction-commit"]),
|
|
156
|
+
payloadReads: compactWorkObservation(this.#payloadReads)
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
#observeTransaction(phase, action) {
|
|
160
|
+
const finish = this.#transactions[phase].start(phase);
|
|
161
|
+
try {
|
|
162
|
+
const result = action();
|
|
163
|
+
finish();
|
|
164
|
+
return result;
|
|
165
|
+
}
|
|
166
|
+
catch (error) {
|
|
167
|
+
finish(true);
|
|
168
|
+
throw error;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
144
171
|
#db;
|
|
145
172
|
#rootDir;
|
|
146
173
|
#openedSchemaHead;
|
|
147
174
|
#inTransaction = false;
|
|
175
|
+
#writeTransaction = false;
|
|
148
176
|
#dirty = false;
|
|
149
177
|
constructor(rootDir, _options = {}) {
|
|
150
178
|
this.#rootDir = rootDir;
|
|
@@ -261,6 +289,13 @@ export class SqliteTaskStore {
|
|
|
261
289
|
listEventsByType(taskId, types) {
|
|
262
290
|
return this.#sortById(this.#listPayload("events", "task_id = ? AND type IN (SELECT value FROM json_each(?))", [taskId, JSON.stringify(types)]), event => event.id);
|
|
263
291
|
}
|
|
292
|
+
hasRuntimeObservation(taskId, identity) {
|
|
293
|
+
const byEvent = "eventId" in identity;
|
|
294
|
+
return this.#db.prepare(`SELECT 1 FROM events WHERE task_id = ?
|
|
295
|
+
AND type = 'runtime.observation'
|
|
296
|
+
AND json_extract(payload, '${byEvent ? "$.payload.eventId" : "$.payload.semanticKey"}') = ?
|
|
297
|
+
LIMIT 1`).get(taskId, byEvent ? identity.eventId : identity.semanticKey) !== undefined;
|
|
298
|
+
}
|
|
264
299
|
readTaskUsageFacts(taskId) {
|
|
265
300
|
// Bounded source selection, not a scan/deserialization of Task history.
|
|
266
301
|
// Strip result bodies before they cross the SQLite boundary.
|
|
@@ -348,13 +383,15 @@ export class SqliteTaskStore {
|
|
|
348
383
|
writeTelemetryBatch(this.#db, entries, runCap);
|
|
349
384
|
}
|
|
350
385
|
#begin() {
|
|
351
|
-
this.#db.exec("BEGIN IMMEDIATE");
|
|
386
|
+
this.#observeTransaction("write-lock", () => this.#db.exec("BEGIN IMMEDIATE"));
|
|
352
387
|
this.#inTransaction = true;
|
|
388
|
+
this.#writeTransaction = true;
|
|
353
389
|
this.#dirty = false;
|
|
354
390
|
}
|
|
355
391
|
#commit() {
|
|
356
|
-
this.#db.exec("COMMIT");
|
|
392
|
+
this.#observeTransaction("transaction-commit", () => this.#db.exec("COMMIT"));
|
|
357
393
|
this.#inTransaction = false;
|
|
394
|
+
this.#writeTransaction = false;
|
|
358
395
|
this.#dirty = false;
|
|
359
396
|
}
|
|
360
397
|
#rollback() {
|
|
@@ -363,6 +400,7 @@ export class SqliteTaskStore {
|
|
|
363
400
|
}
|
|
364
401
|
catch { /* already closed */ }
|
|
365
402
|
this.#inTransaction = false;
|
|
403
|
+
this.#writeTransaction = false;
|
|
366
404
|
this.#dirty = false;
|
|
367
405
|
}
|
|
368
406
|
/** Verify the SQLite schema head at the single write boundary. */
|
|
@@ -393,7 +431,7 @@ export class SqliteTaskStore {
|
|
|
393
431
|
this.#begin();
|
|
394
432
|
try {
|
|
395
433
|
this.#prepareWrite();
|
|
396
|
-
const result = fn
|
|
434
|
+
const result = this.#observeTransaction("write-body", fn);
|
|
397
435
|
this.#bumpRevision();
|
|
398
436
|
this.#commit();
|
|
399
437
|
return result;
|
|
@@ -413,7 +451,7 @@ export class SqliteTaskStore {
|
|
|
413
451
|
// This prevents a long-lived Store from executing SQL after the database
|
|
414
452
|
// schema head has changed underneath it.
|
|
415
453
|
this.#prepareWrite();
|
|
416
|
-
const result = run(this);
|
|
454
|
+
const result = this.#observeTransaction("write-body", () => run(this));
|
|
417
455
|
if (this.#dirty) {
|
|
418
456
|
if (options?.requestId !== undefined) {
|
|
419
457
|
this.#insertOutbox(options.requestId, options.outboxCommand ?? null);
|
|
@@ -438,7 +476,7 @@ export class SqliteTaskStore {
|
|
|
438
476
|
this.#begin();
|
|
439
477
|
try {
|
|
440
478
|
this.#prepareWrite();
|
|
441
|
-
const result = await execute(this);
|
|
479
|
+
const result = await this.#transactions["write-body"].observe("write-body", undefined, () => execute(this));
|
|
442
480
|
if (this.#dirty)
|
|
443
481
|
this.#bumpRevision();
|
|
444
482
|
this.#commit();
|
|
@@ -472,7 +510,7 @@ export class SqliteTaskStore {
|
|
|
472
510
|
if (current !== expectedRevision) {
|
|
473
511
|
throw new StorageConflictError(`Storage revision conflict (expected ${expectedRevision}, found ${current}).`, current);
|
|
474
512
|
}
|
|
475
|
-
const result = execute(this);
|
|
513
|
+
const result = this.#observeTransaction("write-body", () => execute(this));
|
|
476
514
|
if (this.#dirty) {
|
|
477
515
|
this.#prepareWrite();
|
|
478
516
|
if (options?.requestId !== undefined) {
|
|
@@ -496,6 +534,13 @@ export class SqliteTaskStore {
|
|
|
496
534
|
return row.revision;
|
|
497
535
|
}
|
|
498
536
|
getStateRevision() { return this.getRevision(); }
|
|
537
|
+
getTaskReadRevision(taskId) {
|
|
538
|
+
// Uncommitted writes have no durable cache identity, even if rolled back.
|
|
539
|
+
if (this.#writeTransaction)
|
|
540
|
+
return Number.NaN;
|
|
541
|
+
return this.#db.prepare("SELECT revision FROM task_read_revisions WHERE task_id = ?")
|
|
542
|
+
.get(taskId)?.revision ?? 0;
|
|
543
|
+
}
|
|
499
544
|
/**
|
|
500
545
|
* Run an ordered command batch inside one `BEGIN IMMEDIATE … COMMIT`, yielding
|
|
501
546
|
* to the event loop between commands so a cancellation signal can interrupt
|
|
@@ -656,8 +701,19 @@ export class SqliteTaskStore {
|
|
|
656
701
|
return row === undefined ? null : this.#validateTaskPayload(table, row.payload);
|
|
657
702
|
}
|
|
658
703
|
#listPayload(table, where, params) {
|
|
659
|
-
|
|
660
|
-
|
|
704
|
+
// Table names are fixed internal callers; no SQL, parameters or payload
|
|
705
|
+
// text enters diagnostics. Include parsing/validation, not only SQLite IO.
|
|
706
|
+
const finish = this.#payloadReads.start(`list-${table}`);
|
|
707
|
+
try {
|
|
708
|
+
const rows = this.#db.prepare(`SELECT payload FROM ${table} WHERE ${where}`).all(...params);
|
|
709
|
+
const values = rows.map((row) => this.#validateTaskPayload(table, row.payload));
|
|
710
|
+
finish();
|
|
711
|
+
return values;
|
|
712
|
+
}
|
|
713
|
+
catch (error) {
|
|
714
|
+
finish(true);
|
|
715
|
+
throw error;
|
|
716
|
+
}
|
|
661
717
|
}
|
|
662
718
|
#validateTaskPayload(table, payload) {
|
|
663
719
|
const record = this.#parse(payload);
|
|
@@ -1019,6 +1075,17 @@ export class SqliteTaskStore {
|
|
|
1019
1075
|
OR json_extract(payload, '$.interruptThen') IS NOT NULL)`);
|
|
1020
1076
|
if (query.interruptThen)
|
|
1021
1077
|
where.push("json_extract(payload, '$.interruptThen') IS NOT NULL");
|
|
1078
|
+
if (query.reservedThen !== undefined) {
|
|
1079
|
+
where.push(`json_extract(payload, '$.delivery') IS NULL
|
|
1080
|
+
AND json_extract(payload, '$.notDelivered') IS NULL
|
|
1081
|
+
AND json_extract(payload, '$.interruptThen.targetNativeSessionId') = @thenSession
|
|
1082
|
+
AND json_extract(payload, '$.interruptThen.targetAgentId') = @thenAgent
|
|
1083
|
+
AND json_extract(payload, '$.interruptThen.targetAuthorityEpoch') = @thenEpoch
|
|
1084
|
+
AND json_extract(payload, '$.interruptThen.targetAuthorityHolderId') = @thenHolder`);
|
|
1085
|
+
Object.assign(params, { thenSession: query.reservedThen.nativeSessionId,
|
|
1086
|
+
thenAgent: query.reservedThen.agentId, thenEpoch: query.reservedThen.authorityEpoch,
|
|
1087
|
+
thenHolder: query.reservedThen.holderId });
|
|
1088
|
+
}
|
|
1022
1089
|
if (query.operatorNotice)
|
|
1023
1090
|
where.push("json_extract(payload, '$.inputControl.requestId') LIKE 'operator-notice:%'");
|
|
1024
1091
|
if (query.afterSequence !== undefined) {
|
|
@@ -1335,6 +1402,17 @@ export class SqliteTaskStore {
|
|
|
1335
1402
|
listJobDefinitions(scope) {
|
|
1336
1403
|
return this.#listPayload("job_definitions", "scope_key=?", [jobScopeKey(scope)]);
|
|
1337
1404
|
}
|
|
1405
|
+
listJobScopeSummaries() {
|
|
1406
|
+
const rows = this.#db.prepare(`SELECT scope_key, SUM(definitions) AS definitions, SUM(schedules) AS schedules,
|
|
1407
|
+
SUM(enabled) AS enabled FROM (
|
|
1408
|
+
SELECT scope_key, 1 AS definitions, 0 AS schedules, 0 AS enabled FROM job_definitions
|
|
1409
|
+
UNION ALL SELECT scope_key, 0, 1, CASE WHEN state='enabled' THEN 1 ELSE 0 END FROM schedules)
|
|
1410
|
+
GROUP BY scope_key`).all();
|
|
1411
|
+
return rows.map(row => ({
|
|
1412
|
+
scope: row.scope_key === "global" ? { kind: "global" } : { kind: "task", taskId: row.scope_key },
|
|
1413
|
+
definitions: Number(row.definitions), schedules: Number(row.schedules), enabledSchedules: Number(row.enabled)
|
|
1414
|
+
}));
|
|
1415
|
+
}
|
|
1338
1416
|
saveGlobalJobRun(run) {
|
|
1339
1417
|
validateDurableJob(run);
|
|
1340
1418
|
if ("taskId" in run || run.scope !== "global")
|