@zq-silk/yui 2.18.0 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/controllerCommands.js +1 -0
- package/dist/controller/controller.js +10 -0
- package/dist/controller/fileSchedulerStoreAdapter.js +4 -0
- package/dist/controller/messageHealth.js +354 -0
- package/dist/controller/runtime.js +9 -1
- package/dist/controller/taskHostTerminalRecovery.js +11 -0
- package/dist/storage/contextRecords.js +16 -0
- package/dist/storage/sqliteStore.js +16 -0
- package/docs/managed-turn-and-session-runtime.md +96 -0
- package/docs/managed-turn-and-session-runtime.zh-CN.md +65 -0
- package/docs/release-workflow.md +20 -0
- package/docs/release-workflow.zh-CN.md +15 -0
- package/package.json +1 -1
|
@@ -187,6 +187,7 @@ export function parseControllerRuntimeSnapshot(result, droppedEvents) {
|
|
|
187
187
|
reads: (runtime.reads ?? null),
|
|
188
188
|
persistence: (runtime.persistence ?? null),
|
|
189
189
|
diagnostics: (runtime.diagnostics ?? null),
|
|
190
|
+
messageHealth: (runtime.messageHealth ?? null),
|
|
190
191
|
scheduler: (runtime.scheduler ?? null),
|
|
191
192
|
preRoutingWait: "unmeasured; event-loop lag is sampled, not per-request queue time",
|
|
192
193
|
clock: "process-monotonic milliseconds; target/request labels are sha256 prefixes"
|
|
@@ -693,6 +693,8 @@ export class FileTaskController {
|
|
|
693
693
|
#jobSupervisor;
|
|
694
694
|
#schedules;
|
|
695
695
|
#globalInputDelivery;
|
|
696
|
+
#messageHealth;
|
|
697
|
+
#lastMessageHealth = null;
|
|
696
698
|
#providerRetry;
|
|
697
699
|
#continuationReconciler;
|
|
698
700
|
#hostTerminalRecovery;
|
|
@@ -753,6 +755,7 @@ export class FileTaskController {
|
|
|
753
755
|
this.#continuationReconciler = options.continuationReconciler;
|
|
754
756
|
this.#hostTerminalRecovery = options.hostTerminalRecovery;
|
|
755
757
|
this.#globalInputDelivery = options.globalInputDelivery;
|
|
758
|
+
this.#messageHealth = options.messageHealth;
|
|
756
759
|
this.#providerRetry = options.providerRetry;
|
|
757
760
|
this.#signalScheduler = new MailboxScheduler(async (keys) => { await this.#requestPass({ kind: "dirty", keys }); }, {
|
|
758
761
|
windowMs: options.signalWindowMs ?? DEFAULT_SIGNAL_WINDOW_MS,
|
|
@@ -789,6 +792,7 @@ export class FileTaskController {
|
|
|
789
792
|
runtimeMetrics() {
|
|
790
793
|
const metrics = this.#lastRuntimeDrain?.metrics ?? ZERO_DRAIN_METRICS;
|
|
791
794
|
return {
|
|
795
|
+
messageHealth: this.#lastMessageHealth,
|
|
792
796
|
diagnostics: this.#diagnosticMetrics?.() ?? null,
|
|
793
797
|
phases: this.#phases.snapshot(),
|
|
794
798
|
persistence: this.#persistenceMetrics?.() ?? null,
|
|
@@ -1015,6 +1019,12 @@ export class FileTaskController {
|
|
|
1015
1019
|
if (scope.kind === "full" && this.#hostTerminalRecovery !== undefined) {
|
|
1016
1020
|
await this.#observeMaintenance("Host terminal recovery", this.#hostTerminalRecovery);
|
|
1017
1021
|
}
|
|
1022
|
+
if (scope.kind === "full" && this.#messageHealth !== undefined) {
|
|
1023
|
+
// Consume already collected Host evidence, never another live probe.
|
|
1024
|
+
const health = await this.#observeMaintenance("message-health", () => this.#messageHealth(this.#now()));
|
|
1025
|
+
if (health !== undefined)
|
|
1026
|
+
this.#lastMessageHealth = health;
|
|
1027
|
+
}
|
|
1018
1028
|
await this.#phases.observe("provider-retry", "home", () => this.#providerRetry?.reconcile());
|
|
1019
1029
|
await this.#phases.observe("global-input", "home", () => this.#globalInputDelivery?.({ reconcile: scope.kind === "full", now: this.#now() }));
|
|
1020
1030
|
for (const taskId of runtimeTaskFailureIds(firstRuntimeDrain)) {
|
|
@@ -35,6 +35,7 @@ import { recordTaskAgentError } from "./taskAgentError.js";
|
|
|
35
35
|
import { recordGlobalRuntimeAttention, recordGlobalRuntimeFailure, recordGlobalRuntimeRecovery } from "./globalRuntimeAttention.js";
|
|
36
36
|
import { checkInboxTurn } from "./inboxTurnCheck.js";
|
|
37
37
|
import { queueOperatorNotification } from "./operatorNotification.js";
|
|
38
|
+
import { reconcileMessageHealth } from "./messageHealth.js";
|
|
38
39
|
import { reconcileProjectSync } from "../job/projectSync.js";
|
|
39
40
|
import { serializeError } from "../core/boundedRpc.js";
|
|
40
41
|
import { claimOperatorAttention } from "../scheduler/operatorEvent.js";
|
|
@@ -95,6 +96,9 @@ export class FileSchedulerStoreAdapter {
|
|
|
95
96
|
this.drivers = drivers;
|
|
96
97
|
this.snapshotExecutionLaneWorkspace = snapshotExecutionLaneWorkspace;
|
|
97
98
|
}
|
|
99
|
+
reconcileMessageHealth(now, hosts) {
|
|
100
|
+
return reconcileMessageHealth(this.store, now, hosts);
|
|
101
|
+
}
|
|
98
102
|
/** Registry maintenance only; this never executes a Job or starts a process. */
|
|
99
103
|
reconcileProjectSync(now) {
|
|
100
104
|
const errors = [];
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
import { createTaskEvent } from "../event/taskEvent.js";
|
|
2
|
+
import { createGlobalRoleMessage } from "../message/message.js";
|
|
3
|
+
import { hasRuntimeCleanupObligation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
4
|
+
import { providerRetryPending } from "../runtime/providerRetry.js";
|
|
5
|
+
import { globalInputBacklog } from "../message/globalInputBacklog.js";
|
|
6
|
+
// These are observation thresholds, not leases or authority to release a writer.
|
|
7
|
+
export const MESSAGE_HEALTH_WARNING_MS = 5 * 60_000;
|
|
8
|
+
export const MESSAGE_HEALTH_CRITICAL_MS = 30 * 60_000;
|
|
9
|
+
const HEADER = "Yui message health v1\n";
|
|
10
|
+
const NOTICE = "operator-notice:message-health:";
|
|
11
|
+
export function classifyMessageHealth(facts, now) {
|
|
12
|
+
const ageMs = facts.oldestAt === null ? 0 : Math.max(0, +now - Date.parse(facts.oldestAt));
|
|
13
|
+
const occupancyMs = facts.processingAt === null ? 0 : Math.max(0, +now - Date.parse(facts.processingAt));
|
|
14
|
+
let state;
|
|
15
|
+
let reason;
|
|
16
|
+
if (facts.stopped) {
|
|
17
|
+
state = "paused";
|
|
18
|
+
reason = "explicit-stop-pause-or-lifecycle";
|
|
19
|
+
}
|
|
20
|
+
else if (facts.count === 0 && facts.handling) {
|
|
21
|
+
state = "busy";
|
|
22
|
+
reason = "claimed-input-handling-is-not-delivery-proof";
|
|
23
|
+
}
|
|
24
|
+
else if (facts.count === 0 && facts.processingAt === null) {
|
|
25
|
+
state = "clear";
|
|
26
|
+
reason = "no-awaiting-delivery";
|
|
27
|
+
}
|
|
28
|
+
else if (facts.turnStatus === "accepted" && !facts.blocked) {
|
|
29
|
+
state = "busy";
|
|
30
|
+
reason = "accepted-turn-duration-is-not-a-fault";
|
|
31
|
+
}
|
|
32
|
+
else if (facts.retryDeadline !== undefined && +now <= Date.parse(facts.retryDeadline)) {
|
|
33
|
+
state = "recovering";
|
|
34
|
+
reason = "bounded-provider-retry-owned";
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
reason = facts.retryDeadline !== undefined ? "retry-budget-expired"
|
|
38
|
+
: facts.blocked ? "retained-delivery-or-processing-barrier"
|
|
39
|
+
: facts.turnStatus === "delivery-unknown" ? "delivery-uncertain"
|
|
40
|
+
: facts.turnStatus === "submitting" ? "submission-unsettled" : "no-delivery-progress";
|
|
41
|
+
state = Math.max(ageMs, occupancyMs) >= MESSAGE_HEALTH_CRITICAL_MS ? "critical"
|
|
42
|
+
: Math.max(ageMs, occupancyMs) >= MESSAGE_HEALTH_WARNING_MS ? "warning" : "waiting";
|
|
43
|
+
}
|
|
44
|
+
return { state, reason, ageMs, occupancyMs };
|
|
45
|
+
}
|
|
46
|
+
// Display-only shell arguments; guidance never executes these commands.
|
|
47
|
+
function arg(value) {
|
|
48
|
+
return /^[a-zA-Z0-9_.:/-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
|
|
49
|
+
}
|
|
50
|
+
function recoveryGuidance(o) {
|
|
51
|
+
const { taskId, roleName } = o.target;
|
|
52
|
+
const owner = taskId === undefined ? "Operator (or the user if Operator itself is unavailable)"
|
|
53
|
+
: roleName === "leader" ? "Operator; the affected Leader must not synchronously stop itself"
|
|
54
|
+
: "the authorized Task Leader, or Operator";
|
|
55
|
+
const retryRead = taskId === undefined ? `yui session retry ${arg(roleName)} show`
|
|
56
|
+
: `yui task role session retry ${arg(taskId)} ${arg(roleName)} show`;
|
|
57
|
+
const lines = [
|
|
58
|
+
`Responsible: ${owner}. Guidance is not authorization and executes nothing.`,
|
|
59
|
+
`Snapshot ${o.observedAt}; Current state: ${o.state}; reason: ${o.reason}. Refresh yui controller status --verbose --json before acting; an old notice is not a current instruction.`,
|
|
60
|
+
`Exact target: Session=${o.session ?? "unreported"}, attempt=${o.attemptId ?? "unreported"}, batch=${o.batchId ?? "none"}, wake=${o.wakeId ?? "none"}.`,
|
|
61
|
+
`Read first: ${[...o.reads, retryRead, ...(o.deliveryEvidence ? [o.deliveryEvidence.read] : [])].join("; ")}.`,
|
|
62
|
+
"Check current CLI help and reported Host capabilities as the responsible caller. Missing capability, inaccessible evidence or an identity change blocks the proposed action; do not guess another command or target."
|
|
63
|
+
];
|
|
64
|
+
if (o.state === "recovering") {
|
|
65
|
+
lines.push("Existing bounded retry owns recovery. Observe its exact attempt/deadline; do not duplicate or interrupt it. Escalate its retained failure only if exhausted or needs-attention.");
|
|
66
|
+
}
|
|
67
|
+
else if (o.state === "busy" || o.state === "waiting") {
|
|
68
|
+
lines.push("No intervention is indicated: accepted work or claimed handling can be slow. Inspect actual progress and delivery evidence; silence or a missing ACK is not permission to interrupt.");
|
|
69
|
+
}
|
|
70
|
+
else if (o.state === "paused") {
|
|
71
|
+
lines.push("Respect the deliberate stop/pause, human ownership, matching InputRequest or pending lifecycle cleanup. Inspect that existing obligation; this notice does not authorize resumption or another stop.");
|
|
72
|
+
}
|
|
73
|
+
else if (o.state === "clear") {
|
|
74
|
+
lines.push("Only delivery backlog clearance was observed. No management action is indicated; inspect original inputs and handling before concluding business recovery.");
|
|
75
|
+
}
|
|
76
|
+
else if (taskId === undefined) {
|
|
77
|
+
lines.push("No per-GlobalRole Host-stop command is established by this candidate. Read the original Global Message/control and retry evidence; use current caller capability discovery for a supported targeted lifecycle action. Do not translate Task commands to a GlobalRole or use a stop-all fallback.");
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
const target = `${arg(taskId)} ${arg(roleName)}`;
|
|
81
|
+
lines.push("Choose only the path matching refreshed evidence, not a stop/replace/retry sequence. Prefer correction plus guarded retry for a proven rejection when sufficient; stop is for an independently established residual Host barrier, and replacement is not the default.");
|
|
82
|
+
lines.push(`Observed barrier: delivery=${o.deliveryEvidence?.outcome ?? "unreported"}; detail=${o.deliveryEvidence?.detail ?? "unreported"}; Host=${o.hostObservation.state}/${o.hostObservation.reconciliation ?? "unreported"}; retry=${o.retry?.status ?? "unreported"}. These observations do not establish quiescence or prove Goal is the cause; do not clear Goal merely because it says stalled.`);
|
|
83
|
+
if (o.session !== null && o.blocked) {
|
|
84
|
+
lines.push(`For confirmed residual Host ownership, the responsible supervisor may consider the existing targeted management path without waiting for a full repair package or every historical projection to align. Template: yui task role session stop ${target} --reason '<verified target, authority and stop rationale>'. First refresh the exact Session/Host identity, native/background work and event persistence; normal busy work, owned retry, intentional pause, unpersisted Host events or unknown cleanup consequences block a routine stop. The CLI captures and fences the current target; this snapshot cannot pin a later invocation.`);
|
|
85
|
+
lines.push("A stop request being saved is not physical stop. Require the exact Host/resource cleanup result; if cleanup fails or volatile facts remain, retain that process and evidence and escalate the precise boundary, without force-kill or claiming it stopped. After proven stop, let existing authorized delivery/lifecycle reuse the retained conversation when supported; verify the actual native Session identity, never infer reuse from Host ready.");
|
|
86
|
+
lines.push(`Only if reuse is unsuitable and deliberate replacement is justified, preserve durable progress and use the separately authorized template: yui task role session new ${target} --reason '<why replacement is necessary>'. This requests fresh Session selection after fenced cleanup, not same-conversation reuse. A self-replacing Leader must end its turn. Retained inputs/claims still require inspection, not replay.`);
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
lines.push("A current controlled Session/residual Host barrier is not established. Inspect the missing runtime or Assignment evidence first; no targeted stop is recommended from this snapshot.");
|
|
90
|
+
}
|
|
91
|
+
if (o.wakeId !== null && roleName === "leader") {
|
|
92
|
+
const wakeTarget = `${arg(taskId)} ${arg(o.wakeId)}`;
|
|
93
|
+
if (o.deliveryEvidence?.outcome === "rejected") {
|
|
94
|
+
lines.push(`After correcting the rejection cause, and only if this exact wake is still unconsumed, its original claim matches and native acceptance/activity are proven absent: yui task wake retry ${wakeTarget} --reason '<verified correction and quiescence>'. Queued is not accepted. Do not retry accepted/unknown input. This candidate does not establish resolve support for rejected wakes; do not assume another recovery package is installed.`);
|
|
95
|
+
}
|
|
96
|
+
else if (o.deliveryEvidence?.outcome === "unknown") {
|
|
97
|
+
lines.push(`Do not replay unknown input. After exact shared-native/background/writer quiescence and persisted evidence, with the original unknown claim still current: yui task wake resolve ${wakeTarget} --reason '<exact quiescence evidence>'. This releases only scheduling without replay, ACK or acceptance. The rightful handler must inspect original effects and settle still-valid obligations separately.`);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
lines.push("Deferred, accepted or unreported delivery does not meet this candidate's rejected-retry/unknown-resolve preconditions. Keep the original wake and inspect the runtime barrier instead of mechanically invoking either action.");
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
lines.push("Verify separately: management request saved; exact physical stop if requested; retained conversation reused or explicit replacement recorded; subsequent original wake/message actually accepted by Provider; real handling/progress and remaining obligations. Host ready, pending=0, command success or alert ACK alone proves none of the later stages. If evidence is missing, recovery remains unverified.");
|
|
105
|
+
lines.push("If an action is refused or fails, retain its exact target, receipt/error, original input and workspace; report the smallest missing capability, authority or cleanup evidence to the responsible supervisor. No private retry loop, forced ACK, queue clearing, raw DB edits, unknown-effect replay, model/account/permission changes or whole-Controller/all-Session restart.");
|
|
106
|
+
return lines.join("\n");
|
|
107
|
+
}
|
|
108
|
+
function eventType(roleName) { return `message.health.${roleName}`; }
|
|
109
|
+
function prefix(target) { return `${HEADER}${target.roleName}\n`; }
|
|
110
|
+
function noticeId(saved) { return NOTICE + encodeURIComponent(saved.id); }
|
|
111
|
+
function latest(store, target) {
|
|
112
|
+
if (target.taskId !== undefined) {
|
|
113
|
+
const event = store.queryContextRecords(target.taskId, { family: "task-event",
|
|
114
|
+
eventTypes: [eventType(target.roleName)], descending: true, limit: 1 }).records[0]?.value;
|
|
115
|
+
return event ? { id: `${target.taskId}/${event.id}`,
|
|
116
|
+
read: `yui task event show ${arg(target.taskId)} ${arg(event.id)}`,
|
|
117
|
+
evidence: JSON.parse(event.payload.evidence) } : null;
|
|
118
|
+
}
|
|
119
|
+
const message = store.queryGlobalRoleMessages(target.roleName, {
|
|
120
|
+
systemBodyPrefix: prefix(target), descending: true, limit: 1
|
|
121
|
+
}).messages[0];
|
|
122
|
+
return message ? { id: `${target.roleName}/${message.id}`,
|
|
123
|
+
read: `yui role message show ${arg(target.roleName)} ${arg(message.id)}`,
|
|
124
|
+
evidence: JSON.parse(message.body.slice(prefix(target).length)) } : null;
|
|
125
|
+
}
|
|
126
|
+
function record(store, target, evidence, now) {
|
|
127
|
+
if (target.taskId !== undefined) {
|
|
128
|
+
const event = createTaskEvent(store.nextEventId(target.taskId), target.taskId, eventType(target.roleName), { evidence: JSON.stringify(evidence) }, now);
|
|
129
|
+
store.saveEvent(target.taskId, event);
|
|
130
|
+
return { id: `${target.taskId}/${event.id}`, read: `yui task event show ${arg(target.taskId)} ${arg(event.id)}`, evidence };
|
|
131
|
+
}
|
|
132
|
+
const message = createGlobalRoleMessage(store.nextGlobalRoleMessageId(), target.roleName, prefix(target) + JSON.stringify(evidence), "system", { type: "system" }, now);
|
|
133
|
+
store.saveGlobalRoleMessage(message);
|
|
134
|
+
return { id: `${target.roleName}/${message.id}`, read: `yui role message show ${arg(target.roleName)} ${arg(message.id)}`, evidence };
|
|
135
|
+
}
|
|
136
|
+
function alertStatus(store, saved) {
|
|
137
|
+
if (!saved)
|
|
138
|
+
return { stage: "not-generated", reason: null, messageId: null };
|
|
139
|
+
const message = store.queryGlobalRoleMessages("operator", {
|
|
140
|
+
requestId: noticeId(saved), limit: 1
|
|
141
|
+
}).messages[0];
|
|
142
|
+
if (message?.inbox?.ack)
|
|
143
|
+
return { stage: "handled", reason: null, messageId: message.id };
|
|
144
|
+
if (message?.delivery)
|
|
145
|
+
return { stage: message.delivery.via === "provider" ? "delivered" : "transport-accepted",
|
|
146
|
+
reason: message.delivery.via === "provider" ? null : "provider-acceptance-unconfirmed", messageId: message.id };
|
|
147
|
+
if (message)
|
|
148
|
+
return { stage: "saved", messageId: message.id,
|
|
149
|
+
reason: message.notDelivered?.reason ?? message.control?.outcome
|
|
150
|
+
?? (store.getGlobalRole("operator")?.enabled === false ? "operator-disabled"
|
|
151
|
+
: globalInputBacklog([message], store.getGlobalRoleSessionSet("operator")).reason) };
|
|
152
|
+
return { stage: "generated", messageId: null,
|
|
153
|
+
reason: store.getGlobalRole("operator") === null ? "operator-unconfigured"
|
|
154
|
+
: operatorAvailable(store) ? "operator-notice-pending" : "operator-unavailable" };
|
|
155
|
+
}
|
|
156
|
+
function operatorAvailable(store) {
|
|
157
|
+
const sessions = store.getGlobalRoleSessionSet("operator");
|
|
158
|
+
return sessions?.sessions[sessions.activeAgentId]?.status === "active" && sessions.providerBinding != null;
|
|
159
|
+
}
|
|
160
|
+
/** Same Global Message queue and receipt contract as ordinary Operator notices.
|
|
161
|
+
* No Task mailbox is involved. A busy/unavailable Operator never loses evidence.
|
|
162
|
+
* At most one unsent engineering notice; original failed notices are not replayed. */
|
|
163
|
+
function notify(store, saved, observation, now) {
|
|
164
|
+
if (store.getGlobalRole("operator") === null
|
|
165
|
+
|| !operatorAvailable(store)
|
|
166
|
+
|| store.queryGlobalRoleMessages("operator", { requestId: noticeId(saved), limit: 1 }).count
|
|
167
|
+
|| store.queryGlobalRoleMessages("operator", { pending: true, operatorNotice: true, limit: 1 }).count)
|
|
168
|
+
return;
|
|
169
|
+
const { phase } = saved.evidence;
|
|
170
|
+
const sessions = store.getGlobalRoleSessionSet("operator");
|
|
171
|
+
const session = sessions.sessions[sessions.activeAgentId];
|
|
172
|
+
const message = createGlobalRoleMessage(store.nextGlobalRoleMessageId(), "operator", [
|
|
173
|
+
`Yui-Summary: Message delivery ${observation.state}: ${observation.target.taskId ?? "global"}/${observation.target.roleName} (retained ${phase} evidence).`,
|
|
174
|
+
`Inspect: ${saved.read}`,
|
|
175
|
+
`Original diagnostic phase=${phase}, first=${saved.evidence.firstAt}, latest evidence=${saved.evidence.at}; guidance below uses the current observation.`,
|
|
176
|
+
"Independent diagnostics: yui controller status --verbose --json",
|
|
177
|
+
`Waiting=${observation.count}, pending=${observation.pendingCount}, processing=${observation.processingCount}, ageMs=${observation.ageMs}, occupancyMs=${observation.occupancyMs}; answered inputs=${observation.answeredInputs.join(", ") || "none observed"}.`,
|
|
178
|
+
observation.recovery,
|
|
179
|
+
"Do not ACK business inputs, release unknown writers, replay input, replace Sessions or change models automatically."
|
|
180
|
+
].join("\n"), "system", { type: "system" }, now, { inputControl: { action: "queue", requestId: noticeId(saved) } });
|
|
181
|
+
store.saveGlobalRoleMessage({ ...message,
|
|
182
|
+
deliveryTarget: { agentId: session.agentId, nativeSessionId: session.nativeSessionId } });
|
|
183
|
+
}
|
|
184
|
+
function oldest(times) {
|
|
185
|
+
return times.filter((time) => time != null).sort()[0] ?? null;
|
|
186
|
+
}
|
|
187
|
+
function observe(store, target, now, hosts) {
|
|
188
|
+
const { taskId, roleName } = target;
|
|
189
|
+
const owner = taskId === undefined ? { scope: "global", roleName }
|
|
190
|
+
: { scope: "task", taskId, roleName };
|
|
191
|
+
const task = taskId === undefined ? null : store.getTask(taskId);
|
|
192
|
+
const role = taskId === undefined ? store.getGlobalRole(roleName) : store.getRole(taskId, roleName);
|
|
193
|
+
const sessions = taskId === undefined ? store.getGlobalRoleSessionSet(roleName)
|
|
194
|
+
: store.getTaskRoleSessionSet(taskId, roleName);
|
|
195
|
+
const session = sessions?.sessions[sessions.activeAgentId];
|
|
196
|
+
const binding = sessions?.providerBinding;
|
|
197
|
+
const currentBinding = binding != null && session !== undefined
|
|
198
|
+
&& binding.conversations.some(conversation => conversation.status === "current"
|
|
199
|
+
&& conversation.epoch === binding.currentConversationEpoch && conversation.conversationId === session.nativeSessionId);
|
|
200
|
+
const turn = currentBinding && binding?.run?.authorityEpoch === binding.authority.epoch ? binding.run : null;
|
|
201
|
+
const mailbox = taskId === undefined
|
|
202
|
+
? roleName === "operator" ? store.getWorkMailbox({ kind: "operator" }) : null
|
|
203
|
+
: store.getWorkMailbox({ kind: "role", taskId, roleName });
|
|
204
|
+
const handlingOnly = mailbox?.processing?.owner.startsWith("agent-inbox:") ?? false;
|
|
205
|
+
const batches = [mailbox?.pending, handlingOnly ? null : mailbox?.processing?.batch].filter(batch => batch != null);
|
|
206
|
+
const refs = batches.flatMap(batch => [...batch.refs]);
|
|
207
|
+
const wakeId = mailbox?.processing?.owner.startsWith("leader-notification:")
|
|
208
|
+
? mailbox.processing.owner.slice("leader-notification:".length) : null;
|
|
209
|
+
const wake = wakeId && taskId ? store.getTaskWake(taskId, wakeId) : null;
|
|
210
|
+
const delivery = taskId && mailbox?.processing ? store.queryContextRecords(taskId, {
|
|
211
|
+
family: "task-event", eventTypes: ["notification.delivery"],
|
|
212
|
+
eventAttemptId: mailbox.processing.batchId, descending: true, limit: 1
|
|
213
|
+
}).records[0]?.value : undefined;
|
|
214
|
+
// Delivery failures stay visible even when ordinary pending excludes notDelivered.
|
|
215
|
+
// Handling-only historical receipts never become new queue work.
|
|
216
|
+
const globalPage = taskId === undefined ? store.queryGlobalRoleMessages(roleName, {
|
|
217
|
+
awaitingDelivery: true, unclaimed: true, excludeHealthNotices: true, limit: 8
|
|
218
|
+
}) : null;
|
|
219
|
+
const handling = handlingOnly || taskId === undefined && store.queryGlobalRoleMessages(roleName, {
|
|
220
|
+
awaitingDelivery: true, handling: true, excludeHealthNotices: true, limit: 1
|
|
221
|
+
}).count > 0;
|
|
222
|
+
const messages = globalPage?.messages ?? [];
|
|
223
|
+
const taskPage = taskId === undefined ? null : store.queryContextRecords(taskId, {
|
|
224
|
+
family: "task-message", queuedForRole: roleName, unhandledMessages: true, limit: 8
|
|
225
|
+
});
|
|
226
|
+
const taskMessages = taskPage?.records.map(record => record.value) ?? [];
|
|
227
|
+
const host = hosts.find(item => item.taskId === taskId && item.roleName === roleName
|
|
228
|
+
&& item.attemptId === turn?.attemptId && item.nativeSessionId === session?.nativeSessionId);
|
|
229
|
+
let hostObservation = host
|
|
230
|
+
? { state: host.state, observedAt: host.observedAt, reconciliation: host.reconciliation }
|
|
231
|
+
: { state: "unobserved", observedAt: null, reconciliation: null };
|
|
232
|
+
if (taskId === undefined && turn?.status === "accepted") {
|
|
233
|
+
// Reuse existing Global Host failure evidence, never probe a second time.
|
|
234
|
+
const failure = store.queryGlobalRoleMessages(roleName, {
|
|
235
|
+
systemBodyPrefix: "Yui Global failure evidence ", descending: true, limit: 1
|
|
236
|
+
}).messages[0];
|
|
237
|
+
if (failure) {
|
|
238
|
+
const facts = JSON.parse(failure.body.slice(failure.body.indexOf("\n") + 1));
|
|
239
|
+
if (facts.reason === "host-terminal-unobservable" && facts.attemptId === turn.attemptId
|
|
240
|
+
&& facts.nativeSessionId === session?.nativeSessionId && failure.createdAt >= turn.updatedAt) {
|
|
241
|
+
hostObservation = { state: typeof facts.hostState === "string" ? facts.hostState : "unobservable",
|
|
242
|
+
observedAt: failure.createdAt, reconciliation: facts.cause ?? null };
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
const count = batches.reduce((n, batch) => n + batch.requestCount, 0)
|
|
247
|
+
+ (globalPage?.count ?? 0) + (taskPage?.count ?? 0);
|
|
248
|
+
const answeredInputs = refs.flatMap(ref => {
|
|
249
|
+
if (ref.type !== "input")
|
|
250
|
+
return [];
|
|
251
|
+
return store.getInputRequest(ref.taskId, ref.id)?.status === "answered" ? [`${ref.taskId}/${ref.id}`] : [];
|
|
252
|
+
});
|
|
253
|
+
// Only explicit matching blocked refs constitute a user wait. An unrelated
|
|
254
|
+
// open InputRequest must not hide an answered input stuck in the same Task.
|
|
255
|
+
const waitingInputs = taskId === undefined || roleName === "leader" ? [] :
|
|
256
|
+
store.listOpenInputRequests([taskId]).filter(request => request.blockedRefs.some(ref => ref.type === "run" && refs.some(queued => queued.type === "run" && queued.id === ref.id)
|
|
257
|
+
|| ref.type === "work-item" && taskMessages.some(message => message.recipient?.workItemId === ref.id)))
|
|
258
|
+
.map(request => request.id);
|
|
259
|
+
const facts = {
|
|
260
|
+
count, handling, oldestAt: oldest([...batches.map(batch => batch.firstQueuedAt), globalPage?.oldestCreatedAt,
|
|
261
|
+
taskMessages[0]?.createdAt]),
|
|
262
|
+
processingAt: handlingOnly && count === 0 ? null : mailbox?.processing?.startedAt ?? null,
|
|
263
|
+
blocked: mailbox?.processing != null && wake?.status !== "consumed"
|
|
264
|
+
&& turn?.status !== "accepted"
|
|
265
|
+
|| messages.some(message => message.notDelivered !== undefined
|
|
266
|
+
|| message.control?.outcome === "delivery-unknown" || message.control?.outcome === "pending")
|
|
267
|
+
|| taskMessages.some(message => message.continuation?.notDeliveredReason !== undefined)
|
|
268
|
+
|| ["idle", "failed", "exited", "unobservable"].includes(hostObservation.state)
|
|
269
|
+
&& hostObservation.reconciliation !== "terminal-republished",
|
|
270
|
+
stopped: role?.enabled === false || session?.endReason === "stopped"
|
|
271
|
+
|| task !== null && (!["active", "draft"].includes(task.status) || task.executionGate.state !== "enabled")
|
|
272
|
+
|| binding?.authority.owner === "human" || binding?.goal?.status === "paused"
|
|
273
|
+
|| waitingInputs.length > 0
|
|
274
|
+
|| hasRuntimeCleanupObligation(store.getWorkMailbox(runtimeLifecycleTarget(owner))),
|
|
275
|
+
turnStatus: turn?.status,
|
|
276
|
+
...(currentBinding && providerRetryPending(binding)
|
|
277
|
+
&& binding.retry.nativeSessionId === session.nativeSessionId
|
|
278
|
+
&& binding.retry.authorityEpoch === binding.authority.epoch ? { retryDeadline: binding.retry.deadline } : {})
|
|
279
|
+
};
|
|
280
|
+
const classification = classifyMessageHealth(facts, now);
|
|
281
|
+
const readBase = taskId ? `yui task role session inspect ${arg(taskId)} ${arg(roleName)}`
|
|
282
|
+
: `yui config role show ${arg(roleName)}`;
|
|
283
|
+
const observation = {
|
|
284
|
+
...facts, ...classification, target, observedAt: now.toISOString(),
|
|
285
|
+
session: session?.nativeSessionId ?? null, agentId: session?.agentId ?? null, hostObservation,
|
|
286
|
+
attemptId: turn?.attemptId ?? null, nativeTurnId: turn?.nativeTurnId ?? null,
|
|
287
|
+
host: binding?.authority.holderId ?? null, authority: binding?.authority.owner ?? null,
|
|
288
|
+
retry: binding?.retry ? { status: binding.retry.status, deadline: binding.retry.deadline,
|
|
289
|
+
attempts: binding.retry.attempts } : null,
|
|
290
|
+
wakeId, batchId: mailbox?.processing?.batchId ?? null, processingOwner: mailbox?.processing?.owner ?? null,
|
|
291
|
+
deliveryEvidence: delivery ? { outcome: delivery.payload.outcome ?? "unreported",
|
|
292
|
+
detail: delivery.payload.detail ?? null,
|
|
293
|
+
read: `yui task event show ${arg(taskId)} ${arg(delivery.id)}` } : null,
|
|
294
|
+
pendingCount: (mailbox?.pending?.requestCount ?? 0) + (globalPage?.count ?? 0) + (taskPage?.count ?? 0),
|
|
295
|
+
processingCount: mailbox?.processing?.batch.requestCount ?? 0,
|
|
296
|
+
pendingSince: mailbox?.pending?.firstQueuedAt ?? globalPage?.oldestCreatedAt ?? taskMessages[0]?.createdAt ?? null,
|
|
297
|
+
waitingInputs,
|
|
298
|
+
refs: [...refs.slice(0, 8), ...messages.map(message => ({
|
|
299
|
+
messageId: message.id, control: message.control ?? null, notDelivered: message.notDelivered ?? null
|
|
300
|
+
})), ...taskMessages.map(message => ({
|
|
301
|
+
messageId: message.id, recipient: message.recipient, continuation: message.continuation ?? null
|
|
302
|
+
}))], omittedRefs: Math.max(0, refs.length - 8) + Math.max(0, (globalPage?.count ?? 0) - messages.length)
|
|
303
|
+
+ Math.max(0, (taskPage?.count ?? 0) - taskMessages.length),
|
|
304
|
+
answeredInputs: answeredInputs.slice(0, 8),
|
|
305
|
+
reads: [readBase, ...(wakeId && taskId ? [`yui task wake show ${arg(taskId)} ${arg(wakeId)}`] : []),
|
|
306
|
+
...messages.map(message => `yui role message show ${arg(roleName)} ${arg(message.id)}`),
|
|
307
|
+
...taskMessages.map(message => `yui task message show ${arg(`${taskId}/${message.id}`)}`)]
|
|
308
|
+
};
|
|
309
|
+
return { ...observation, recovery: recoveryGuidance(observation) };
|
|
310
|
+
}
|
|
311
|
+
/** Low-rate Controller observation, not an Agent loop. All decisions use current
|
|
312
|
+
* records inside each short transaction; evidence + queue admission are atomic.
|
|
313
|
+
* Open Event/Message content carries audit facts; no new storage envelope/index
|
|
314
|
+
* or independently writable scheduler state is introduced. */
|
|
315
|
+
export function reconcileMessageHealth(store, now = new Date(), hosts = []) {
|
|
316
|
+
const targets = store.listTasks().filter(task => task.status !== "archived").flatMap(task => store.listRoles(task.id).map(role => ({ taskId: task.id, roleName: role.name })));
|
|
317
|
+
targets.push(...store.listGlobalRoles().map(role => ({ roleName: role.name })));
|
|
318
|
+
const items = [];
|
|
319
|
+
for (const target of targets) {
|
|
320
|
+
store.transaction(tx => {
|
|
321
|
+
const observation = observe(tx, target, now, hosts);
|
|
322
|
+
let saved = latest(tx, target);
|
|
323
|
+
const previous = saved?.evidence;
|
|
324
|
+
const active = previous !== undefined && previous.phase !== "recovered";
|
|
325
|
+
const growing = active && observation.state === "warning" && observation.ageMs >= 15 * 60_000
|
|
326
|
+
&& observation.count >= Math.max(8, previous.initialCount * 2);
|
|
327
|
+
const desired = growing ? "critical" : observation.state;
|
|
328
|
+
const phase = desired === "warning" || desired === "critical"
|
|
329
|
+
? (!active || desired === "critical" && previous.phase === "warning" ? desired : null)
|
|
330
|
+
: desired === "clear" && active ? "recovered" : null;
|
|
331
|
+
if (phase !== null) {
|
|
332
|
+
saved = record(tx, target, {
|
|
333
|
+
phase, firstAt: active ? previous.firstAt : now.toISOString(), at: now.toISOString(),
|
|
334
|
+
initialCount: active ? previous.initialCount : observation.count,
|
|
335
|
+
peakCount: Math.max(active ? previous.peakCount : 0, observation.count),
|
|
336
|
+
previous: saved?.id ?? null, observation
|
|
337
|
+
}, now);
|
|
338
|
+
}
|
|
339
|
+
if (saved !== null)
|
|
340
|
+
notify(tx, saved, observation, now);
|
|
341
|
+
if (observation.count > 0 || observation.processingAt !== null || saved !== null) {
|
|
342
|
+
items.push({ ...observation, evidence: saved?.read ?? null, alert: alertStatus(tx, saved),
|
|
343
|
+
incident: saved ? { phase: saved.evidence.phase, firstAt: saved.evidence.firstAt,
|
|
344
|
+
latestEvidenceAt: saved.evidence.at, initialCount: saved.evidence.initialCount } : null });
|
|
345
|
+
}
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
// Status is an O(1) cached bounded view; exact originals remain independently readable.
|
|
349
|
+
const priority = (state) => state === "critical" ? 0 : state === "warning" ? 1 : state === "clear" ? 3 : 2;
|
|
350
|
+
items.sort((a, b) => priority(a.state) - priority(b.state) || b.ageMs - a.ageMs);
|
|
351
|
+
return { observedAt: now.toISOString(), warningAfterMs: MESSAGE_HEALTH_WARNING_MS,
|
|
352
|
+
criticalAfterMs: MESSAGE_HEALTH_CRITICAL_MS, total: items.length, omitted: Math.max(0, items.length - 64),
|
|
353
|
+
items: items.slice(0, 64) };
|
|
354
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isDeepStrictEqual } from "node:util";
|
|
2
2
|
import { ControllerDiagnosticJournal } from "./diagnosticJournal.js";
|
|
3
|
+
import { reconcileMessageHealth } from "./messageHealth.js";
|
|
3
4
|
import { monotonicMilliseconds } from "../core/controllerTelemetry.js";
|
|
4
5
|
import { prepareNativeHostSteer } from "./nativeHostInput.js";
|
|
5
6
|
import { archiveLeaderTask } from "../task/leaderArchive.js";
|
|
@@ -583,6 +584,7 @@ async function createFileTaskControllerRuntime(home, options) {
|
|
|
583
584
|
const continuationReconciler = options.continuationMetadata === undefined
|
|
584
585
|
? undefined
|
|
585
586
|
: new ProviderContinuationReconciliationService(store, schedulerStore, options.continuationMetadata, options.onError);
|
|
587
|
+
let messageHealthHosts = [];
|
|
586
588
|
const running = await startFileTaskController(home, workerMaintenance
|
|
587
589
|
? workerSchedulerStore(schedulerStore, asyncStoreClient)
|
|
588
590
|
: schedulerStore, delivery, (method, params) => method === "task.role-capabilities"
|
|
@@ -627,7 +629,13 @@ async function createFileTaskControllerRuntime(home, options) {
|
|
|
627
629
|
}
|
|
628
630
|
}),
|
|
629
631
|
jobSupervisor,
|
|
630
|
-
|
|
632
|
+
messageHealth: async (now) => workerMaintenance
|
|
633
|
+
? await asyncStoreClient.invokeObserver("reconcileMessageHealth", [now, messageHealthHosts])
|
|
634
|
+
: reconcileMessageHealth(store, now, messageHealthHosts),
|
|
635
|
+
hostTerminalRecovery: async () => {
|
|
636
|
+
messageHealthHosts = [];
|
|
637
|
+
messageHealthHosts = await recoverTaskHostTerminals(home, store, options.onError ?? (() => undefined));
|
|
638
|
+
},
|
|
631
639
|
globalInputDelivery: (deliveryOptions) => deliverGlobalInputs(home, store, async (roleName) => {
|
|
632
640
|
await lifecycleDispatcher("runtime.ensure-role-session", { scope: "global", roleName });
|
|
633
641
|
}, options.onError ?? (() => undefined), deliveryOptions),
|
|
@@ -3,6 +3,7 @@ import { currentProviderConversation } from "../runtime/providerRuntimeIdentity.
|
|
|
3
3
|
import { providerRetryPending } from "../runtime/providerRetry.js";
|
|
4
4
|
/** Existing full reconciliation pass only; no new timer, input or Session. */
|
|
5
5
|
export async function recoverTaskHostTerminals(home, store, onError, reconcile = reconcileAgentHostTurn) {
|
|
6
|
+
const observations = [];
|
|
6
7
|
for (const candidate of store.listRuntimeSessionCandidates({ scope: "task" })) {
|
|
7
8
|
const owner = candidate.owner;
|
|
8
9
|
if (owner.scope !== "task")
|
|
@@ -20,14 +21,24 @@ export async function recoverTaskHostTerminals(home, store, onError, reconcile =
|
|
|
20
21
|
attemptId: turn.attemptId, nativeTurnId: turn.nativeTurnId,
|
|
21
22
|
authority: { epoch: binding.authority.epoch, owner: "controller", holderId: binding.authority.holderId }
|
|
22
23
|
} });
|
|
24
|
+
observations.push({ taskId: owner.taskId, roleName: owner.roleName,
|
|
25
|
+
nativeSessionId: currentProviderConversation(binding).conversationId, attemptId: turn.attemptId,
|
|
26
|
+
state: result.snapshot.state, reconciliation: result.reconciliation ?? null,
|
|
27
|
+
outcome: result.outcome, observedAt: new Date().toISOString() });
|
|
23
28
|
if (result.outcome === "rejected" || result.reconciliation === "unsupported") {
|
|
24
29
|
throw new Error(result.failure?.detail
|
|
25
30
|
?? "Host terminal reconciliation is unsupported; retain the original input and inspect Host/Inbox evidence.");
|
|
26
31
|
}
|
|
27
32
|
}
|
|
28
33
|
catch (error) {
|
|
34
|
+
if (!observations.some(item => item.taskId === owner.taskId && item.roleName === owner.roleName)) {
|
|
35
|
+
observations.push({ taskId: owner.taskId, roleName: owner.roleName,
|
|
36
|
+
nativeSessionId: currentProviderConversation(binding).conversationId, attemptId: turn.attemptId,
|
|
37
|
+
state: "unobservable", reconciliation: null, outcome: "unavailable", observedAt: new Date().toISOString() });
|
|
38
|
+
}
|
|
29
39
|
onError(new Error(`Task Host terminal recovery ${owner.taskId}/${owner.roleName}/${turn.attemptId}: `
|
|
30
40
|
+ `${error instanceof Error ? error.message : String(error)} No input was replayed.`, { cause: error }));
|
|
31
41
|
}
|
|
32
42
|
}
|
|
43
|
+
return observations;
|
|
33
44
|
}
|
|
@@ -54,6 +54,22 @@ export function queryContextRecords(db, taskId, query) {
|
|
|
54
54
|
const from = `${table} r${candidate ? ", json_each(r.payload, '$.candidates') candidate" : ""}`;
|
|
55
55
|
const where = ["r.task_id = @taskId"];
|
|
56
56
|
const params = { taskId };
|
|
57
|
+
if (query.eventAttemptId !== undefined) {
|
|
58
|
+
if (query.family !== "task-event")
|
|
59
|
+
throw new Error("Attempt selection requires Events.");
|
|
60
|
+
where.push("json_extract(r.payload, '$.payload.attemptId') = @eventAttemptId");
|
|
61
|
+
params.eventAttemptId = query.eventAttemptId;
|
|
62
|
+
}
|
|
63
|
+
if (query.queuedForRole !== undefined) {
|
|
64
|
+
if (query.family !== "task-message")
|
|
65
|
+
throw new Error("Queued Role selection requires Messages.");
|
|
66
|
+
where.push(`json_extract(r.payload, '$.recipient.roleName') = @queuedRole
|
|
67
|
+
AND json_extract(r.payload, '$.recipient.ownerRunId') IS NOT NULL
|
|
68
|
+
AND json_extract(r.payload, '$.inbox') IS NULL
|
|
69
|
+
AND json_extract(r.payload, '$.continuation.runId') IS NULL
|
|
70
|
+
AND COALESCE(json_extract(r.payload, '$.inputControl.action'), '') != 'steer'`);
|
|
71
|
+
params.queuedRole = query.queuedForRole;
|
|
72
|
+
}
|
|
57
73
|
if (query.ids !== undefined) {
|
|
58
74
|
where.push(`${id} IN (SELECT value FROM json_each(@ids))`);
|
|
59
75
|
params.ids = JSON.stringify(query.ids);
|
|
@@ -1088,6 +1088,22 @@ export class SqliteTaskStore {
|
|
|
1088
1088
|
}
|
|
1089
1089
|
if (query.operatorNotice)
|
|
1090
1090
|
where.push("json_extract(payload, '$.inputControl.requestId') LIKE 'operator-notice:%'");
|
|
1091
|
+
if (query.awaitingDelivery)
|
|
1092
|
+
where.push(`(json_extract(payload, '$.delivery') IS NULL
|
|
1093
|
+
OR json_extract(payload, '$.delivery.via') = 'transport')
|
|
1094
|
+
AND json_extract(payload, '$.inbox.ack') IS NULL
|
|
1095
|
+
AND json_extract(payload, '$.inbox.retirement') IS NULL
|
|
1096
|
+
AND (json_extract(payload, '$.inputControl.action') = 'queue'
|
|
1097
|
+
OR json_extract(payload, '$.interruptThen') IS NOT NULL
|
|
1098
|
+
OR json_extract(payload, '$.control.outcome') IN ('pending','delivery-unknown'))`);
|
|
1099
|
+
if (query.excludeHealthNotices)
|
|
1100
|
+
where.push("COALESCE(json_extract(payload, '$.inputControl.requestId'), '') NOT LIKE 'operator-notice:message-health:%'");
|
|
1101
|
+
if (query.systemBodyPrefix !== undefined) {
|
|
1102
|
+
where.push(`json_extract(payload, '$.kind') = 'system'
|
|
1103
|
+
AND json_extract(payload, '$.author.type') = 'system'
|
|
1104
|
+
AND substr(json_extract(payload, '$.body'), 1, length(@bodyPrefix)) = @bodyPrefix`);
|
|
1105
|
+
params.bodyPrefix = query.systemBodyPrefix;
|
|
1106
|
+
}
|
|
1091
1107
|
if (query.afterSequence !== undefined) {
|
|
1092
1108
|
where.push("seq > @after");
|
|
1093
1109
|
params.after = query.afterSequence;
|
|
@@ -643,3 +643,99 @@ Operator storage diagnosis/upgrade also reaches the dedicated migration entry
|
|
|
643
643
|
without first requiring the Home to already have the target storage version.
|
|
644
644
|
Completion, cancellation and archive have distinct authority and resource
|
|
645
645
|
boundaries. Storage changes use the single [upgrade contract](sqlite-control-plane-design.md).
|
|
646
|
+
|
|
647
|
+
## Message delivery health
|
|
648
|
+
|
|
649
|
+
The Controller's existing full reconciliation observes Task Role notification
|
|
650
|
+
mailboxes, Assignment continuations, Global Role input and the Operator attention
|
|
651
|
+
mailbox. It reads the real `pending` suffix separately from `processing`, never
|
|
652
|
+
the compatibility `getPendingWakeup` projection. It does not run an Agent polling
|
|
653
|
+
loop, acknowledge input, release a claim, resend an uncertain input or start an
|
|
654
|
+
unavailable Operator.
|
|
655
|
+
|
|
656
|
+
`yui controller status --verbose --json` exposes a cached, timestamped
|
|
657
|
+
`identity.runtime.observations.messageHealth` view (the Controller RPC exposes
|
|
658
|
+
`runtime.messageHealth`). It includes counts, oldest wait/occupancy, exact
|
|
659
|
+
Session/attempt/wake/message references, answered input references, recorded
|
|
660
|
+
delivery outcome, retry ownership and the existing Host reconciliation evidence.
|
|
661
|
+
`unobserved` is not proof of Host health. Inspect its exact Event/Message read
|
|
662
|
+
commands for durable originals. Status reads perform no database scans or probes.
|
|
663
|
+
The view shows at most 64 owners and reports omissions; records remain readable
|
|
664
|
+
through the existing Task event and Global Role message commands after restart.
|
|
665
|
+
|
|
666
|
+
An eligible stalled delivery warns after 5 minutes and escalates once at
|
|
667
|
+
30 minutes, or after 15 minutes if the queue at least doubles and reaches eight
|
|
668
|
+
items. A single old input is sufficient. Sampling is on the configured full-pass
|
|
669
|
+
interval (default 120 seconds, permitted 5–300 seconds): with a responsive
|
|
670
|
+
Controller, detection occurs on the first completed pass after the threshold.
|
|
671
|
+
These are not wall-clock delivery guarantees during downtime, blocked I/O or
|
|
672
|
+
unavailable Providers. Accepted long Turns without contrary exact Host evidence
|
|
673
|
+
are busy, not faults; explicit stops, disabled Roles, human ownership, native
|
|
674
|
+
pause and matching open input waits are distinct from blocked delivery. Valid
|
|
675
|
+
bounded retry remains owned by its existing recovery mechanism. Historical
|
|
676
|
+
missing ACKs without a new waiting suffix do not generate backlog alerts.
|
|
677
|
+
|
|
678
|
+
Each affected delivery owner has one open diagnostic episode: at most a warning,
|
|
679
|
+
one escalation and a backlog-clear observation. Concurrent arrivals coalesce;
|
|
680
|
+
they do not each call a model. Task evidence uses ordinary `message.health.<role>`
|
|
681
|
+
Events; Global evidence uses record-only system Messages. These are append-only
|
|
682
|
+
content in the existing open Event/Message formats, not a new storage envelope,
|
|
683
|
+
schema version or scheduling aggregate. The newest evidence reconstructs
|
|
684
|
+
notification deduplication after restart. Each notice reuses the ordinary
|
|
685
|
+
Operator Global Message queue with a stable request ID, independent of the
|
|
686
|
+
affected Task's occupied mailbox. Only one unsent engineering notice is admitted;
|
|
687
|
+
later evidence stays durable and is reconsidered on subsequent full passes.
|
|
688
|
+
|
|
689
|
+
If Operator is absent/stopped/uncontrolled, evidence remains generated but no
|
|
690
|
+
new Session is started. If it is busy, the notice stays saved; failures retain
|
|
691
|
+
their exact message and reason and are not replayed. Diagnostics distinguish
|
|
692
|
+
generated evidence, saved notice, transport acceptance, Provider delivery and
|
|
693
|
+
explicit handling. An alert ACK is never recovery. Only an observed empty
|
|
694
|
+
delivery backlog closes an episode; an explicit pause or merely releasing the
|
|
695
|
+
old processing claim while a pending suffix remains does not. Backlog clearance
|
|
696
|
+
itself proves neither business handling nor completion, and grants no recovery
|
|
697
|
+
authority. Moving all pending work to a new processing owner is not clearance:
|
|
698
|
+
retained processing with a deferred delivery remains observable, including its
|
|
699
|
+
exact attempt and rejection detail, even when the current Turn is completed.
|
|
700
|
+
|
|
701
|
+
### Management recovery guidance
|
|
702
|
+
|
|
703
|
+
Notices include the current observation time, exact identities, original read
|
|
704
|
+
commands and conditional recovery guidance, also available in the cached
|
|
705
|
+
`recovery` field. A notice delayed by Operator unavailability uses current facts
|
|
706
|
+
when it is finally saved, not the original incident's intervention advice.
|
|
707
|
+
Already-saved notices remain immutable evidence and are not resent merely to
|
|
708
|
+
update advice: always refresh status and exact Session/wake/message evidence
|
|
709
|
+
before acting. A command template is neither authorization nor proof that a
|
|
710
|
+
later invocation still targets the observed Session; current CLI fences apply.
|
|
711
|
+
|
|
712
|
+
An affected Leader is supervised by Operator; a subordinate Task Role by its
|
|
713
|
+
authorized Leader or Operator. Normal busy work, legitimate pause and owned
|
|
714
|
+
bounded retry receive non-intervention guidance. For an established retained
|
|
715
|
+
Task Host barrier, the supervisor may consider the existing targeted
|
|
716
|
+
`task role session stop` path without waiting for a complete engineering repair
|
|
717
|
+
or perfect historical projections. Inspect exact native/background execution,
|
|
718
|
+
event persistence, authority and cleanup consequences first. Unpersisted Host
|
|
719
|
+
facts must not be destroyed. A saved stop request is not successful physical
|
|
720
|
+
cleanup; retain failure evidence when cleanup cannot be proven. Existing
|
|
721
|
+
authorized lifecycle/delivery may reuse the retained conversation after proven
|
|
722
|
+
stop; verify its identity. Deliberate `task role session new` is a different,
|
|
723
|
+
explicit replacement decision, requiring durable progress and fenced cleanup.
|
|
724
|
+
Goal `stalled` alone neither establishes the cause nor warrants clearing Goal.
|
|
725
|
+
|
|
726
|
+
In this candidate, `task wake retry` requires a proven-rejected, unconsumed
|
|
727
|
+
exact claim, a corrected cause and no accepted/uncertain native execution.
|
|
728
|
+
`task wake resolve` supports an unknown exact claim after shared-native
|
|
729
|
+
quiescence; it releases scheduling without replay or semantic acknowledgement.
|
|
730
|
+
Deferred delivery meets neither precondition. Do not assume another recovery
|
|
731
|
+
package's extended behavior is installed. For Global Roles, this candidate
|
|
732
|
+
does not establish a dedicated targeted Host-stop command: inspect original
|
|
733
|
+
input/control and `session retry <role> show`, then discover current authorized
|
|
734
|
+
capabilities. There is no guessed Task equivalent or stop-all fallback.
|
|
735
|
+
|
|
736
|
+
On refusal or failure, preserve the exact receipt, original inputs and work,
|
|
737
|
+
and escalate only the missing authority, capability or cleanup evidence.
|
|
738
|
+
Verify management request, physical stop, conversation reuse/replacement,
|
|
739
|
+
actual subsequent Provider acceptance and business handling separately.
|
|
740
|
+
Missing evidence remains unverified; no automatic recovery, replay, forced
|
|
741
|
+
ACK, model/account/permission change or Controller-wide restart is introduced.
|
|
@@ -480,3 +480,68 @@ Task。Operator 诊断和 Controller 重启不先要求一个健康的 Controlle
|
|
|
480
480
|
也能到达专用的迁移入口,而不先要求 Home 已经处于目标存储版本。完成、取消和归档具有
|
|
481
481
|
彼此不同的权限和资源边界。存储变更使用唯一的
|
|
482
482
|
[升级合同](sqlite-control-plane-design.zh-CN.md)。
|
|
483
|
+
|
|
484
|
+
## 消息投递健康度
|
|
485
|
+
|
|
486
|
+
Controller 在既有 full reconciliation 中观察 Task Role 通知 mailbox、
|
|
487
|
+
Assignment continuation、Global Role 输入及 Operator attention mailbox。
|
|
488
|
+
分别读取真实 `pending` 后缀与 `processing`,不把 `getPendingWakeup` 的兼容投影
|
|
489
|
+
当作底层队列;不启动 Agent 私有轮询、不自动 ACK、释放占用或重放未知输入。
|
|
490
|
+
|
|
491
|
+
`yui controller status --verbose --json` 的
|
|
492
|
+
`identity.runtime.observations.messageHealth` 提供带采样时间的缓存诊断;Controller RPC 对应
|
|
493
|
+
`runtime.messageHealth`。可查看数量、最老等待与占用时长、精确 Session/attempt/wake/
|
|
494
|
+
message、已回答输入、投递证据、重试归属与既有 Host 核对结果。`unobserved` 不代表
|
|
495
|
+
Host 健康。按其中的 Event/Message 读取命令核对持久原始证据。状态读取不会扫描数据库
|
|
496
|
+
或发起探测;最多展示 64 个 owner 并显式报告省略数量,重启后原始记录仍可读取。
|
|
497
|
+
|
|
498
|
+
符合异常条件的投递等待在 5 分钟后预警,30 分钟后最多升级一次;若队列至少翻倍且达到
|
|
499
|
+
8 条,可在 15 分钟后提前升级。仅一条长期不推进的输入也会检测。采样复用 full-pass
|
|
500
|
+
间隔(默认 120 秒,可设 5–300 秒),健康响应的 Controller 在超过阈值后的首次完整
|
|
501
|
+
检查发现;停机、I/O 堵塞和 Provider 不可用期间不保证墙钟时限或即时送达。没有精确
|
|
502
|
+
Host 反证的 accepted 长 Turn 视为忙碌;显式停止、Role 禁用、human 所有权、原生
|
|
503
|
+
暂停、匹配的未回答输入等待单独区分。有效的有界重试仍由既有恢复机制拥有。只有历史
|
|
504
|
+
缺 ACK、没有新待投递后缀,不会生成积压告警。
|
|
505
|
+
|
|
506
|
+
按受影响的投递 owner 合并一个未关闭异常,最多保存预警、升级和积压消除三个阶段,
|
|
507
|
+
并发到达不会逐条调用模型。Task 使用普通 `message.health.<role>` Event,Global
|
|
508
|
+
使用 record-only system Message,都是既有开放内容格式中的追加证据,不新增存储
|
|
509
|
+
envelope、schema 或调度权威。最新证据用于重启后的去重。告警使用稳定 request ID,
|
|
510
|
+
走独立于受影响 Task 占用的 Operator Global Message 通道;最多保留一条未发送的
|
|
511
|
+
工程通知,其余待告警证据持久保留并在后续 full pass 检查。
|
|
512
|
+
|
|
513
|
+
Operator 缺席、已停止或没有受控 binding 时,不自动创建 Session;忙碌时保留已保存
|
|
514
|
+
通知,失败时保留原 Message 和原因、不重放。诊断区分已生成证据、已保存通知、传输
|
|
515
|
+
接受、Provider 送达与显式处置。告警 ACK 不关闭异常。只有观察到投递积压消失才记录
|
|
516
|
+
恢复;显式暂停、或仅释放旧 processing 但 pending 仍在,不算恢复。积压消除也不证明
|
|
517
|
+
业务处理完成、Task 验收或获得任何额外恢复权限。
|
|
518
|
+
pending 全部转入新 processing owner 也不算清除:即便当前 Turn 已完成,持续 deferred
|
|
519
|
+
的 processing 仍保留监测及精确 attempt、投递拒绝详情。
|
|
520
|
+
|
|
521
|
+
### 管理恢复引导
|
|
522
|
+
|
|
523
|
+
通知直接包含当前观测时间、精确身份、原始读取命令及有前提的恢复引导,缓存诊断的
|
|
524
|
+
`recovery` 字段也提供同一引导。Operator 不可用导致延迟保存的通知使用保存当时的
|
|
525
|
+
当前观测,不复制旧异常的操作建议;已保存通知不重写,也不为更新建议重复发送。
|
|
526
|
+
执行前必须刷新 status 和准确 Session/wake/message 原始证据。命令模板不是授权,
|
|
527
|
+
也不能固定未来调用的目标身份;实际操作仍受现有 CLI 围栏约束。
|
|
528
|
+
|
|
529
|
+
Leader 自身受阻由 Operator 处理,下属 Task Role 由有权限的 Leader 或 Operator
|
|
530
|
+
处理。正常忙碌、合法暂停、有界重试仍拥有恢复时,不推荐打断。确认 Task Host 遗留
|
|
531
|
+
占用后,可评估已有准确 `task role session stop` 管理路径,不必等待完整工程修复包
|
|
532
|
+
或所有历史投影对齐;先核对准确原生/后台执行、事件持久化、权限和清理影响,不能丢失
|
|
533
|
+
Host 中尚未持久化的事实。stop 请求保存不等于物理清理成功,失败时保留原证据。
|
|
534
|
+
证明停止后,既有授权生命周期/投递可在支持时复用原 conversation,须验证实际身份。
|
|
535
|
+
`task role session new` 则是另行明确选择新 Session,需保存进展并通过清理围栏。
|
|
536
|
+
仅显示 Goal stalled 不能确定根因,也不能据此要求清 Goal。
|
|
537
|
+
|
|
538
|
+
本候选的 `task wake retry` 仅适用已纠正原因、原准确 claim 仍未消费的明确 rejected,
|
|
539
|
+
且不能存在已接受或不确定原生执行;`task wake resolve` 仅支持准确 unknown claim
|
|
540
|
+
在共享原生静止证明后释放调度,不重放、不 ACK。deferred 不满足二者前提,不能假定
|
|
541
|
+
其他恢复包的扩展行为已经安装。GlobalRole 未确认有专用准确 Host-stop 命令时,读取
|
|
542
|
+
原输入/control 和 `session retry <role> show`,再发现当前权限下的能力;不臆造
|
|
543
|
+
Task 等价命令,也不退化为 stop-all。
|
|
544
|
+
|
|
545
|
+
动作拒绝或失败后,保留精确回执、原输入和工作区,只升级缺失的权限、能力或清理证据。
|
|
546
|
+
分别验证管理请求、物理停止、原会话复用/显式替换、后续实际 Provider 接受及业务处置;
|
|
547
|
+
缺证据仍标为未验证。不新增自动恢复、重放、强制 ACK、模型/账号/权限变更或全局重启。
|
package/docs/release-workflow.md
CHANGED
|
@@ -51,6 +51,26 @@ merely to avoid a version decision. Package versions remain distinct from protoc
|
|
|
51
51
|
and persistent-storage contracts: their compatibility checks and required migrations
|
|
52
52
|
are unchanged, and the storage rules below do not authorize a package release.
|
|
53
53
|
|
|
54
|
+
### 2.19.0 compatibility and migration notes
|
|
55
|
+
|
|
56
|
+
This minor adds message backlog and blocked-delivery monitoring with Operator
|
|
57
|
+
alerts (PR #492). Observations distinguish live work, paused execution and bounded
|
|
58
|
+
retry from stalled delivery, retain original evidence, and deduplicate alerts
|
|
59
|
+
across Controller restarts. Alerts include evidence-matched management guidance:
|
|
60
|
+
inspect the exact input and execution first, distinguish rejected retry from
|
|
61
|
+
unknown-result resolution, and require scoped stop proof or explicit replacement
|
|
62
|
+
authority where applicable. Monitoring never retries, stops or replaces execution
|
|
63
|
+
automatically; an alert is not recovery authority or proof of a unique root cause.
|
|
64
|
+
|
|
65
|
+
Storage remains **1.24**, and Controller/Host protocols remain v1. Existing
|
|
66
|
+
messages and events are preserved; this release adds no storage migration.
|
|
67
|
+
Older supported Homes still use the centralized contiguous migration chain.
|
|
68
|
+
With separate update authority, the Operator verifies the published artifact
|
|
69
|
+
then runs `yui update --version 2.19.0`. Compatible live Sessions are retained;
|
|
70
|
+
publishing alone does not update the installed program or recover other Tasks.
|
|
71
|
+
Exact-merge Core CI and the same tarball's fresh-install platform/Node matrix
|
|
72
|
+
remain release gates. These checks do not establish real-model behavior.
|
|
73
|
+
|
|
54
74
|
### 2.18.0 compatibility and migration notes
|
|
55
75
|
|
|
56
76
|
This minor includes the already-merged independent Jobs workspace and global
|
|
@@ -36,6 +36,21 @@ Agent 选择一个预先声明的计划,设施从持久状态驱动该计划
|
|
|
36
36
|
不能将不兼容冒称为兼容,也不为回避版本选择新增历史兼容机制。包版本与协议、
|
|
37
37
|
持久存储契约仍然独立:兼容性检查和必要迁移要求不变,下文存储规则不授予发版权限。
|
|
38
38
|
|
|
39
|
+
### 2.19.0 兼容性及迁移说明
|
|
40
|
+
|
|
41
|
+
本 minor 新增消息积压、投递阻塞监测及 Operator 告警(PR #492)。观测区分正常执行、
|
|
42
|
+
暂停、有界重试与投递停滞,保留原始证据,并跨 Controller 重启去重告警。
|
|
43
|
+
告警包含按证据匹配的管理恢复引导:先检查精确输入和执行状态,区分明确拒绝后的
|
|
44
|
+
retry 与未知结果的 resolve,并在需要时要求精确停止证明或显式替换授权。
|
|
45
|
+
监测不会自动重试、停止或替换执行;告警本身不是恢复授权,也不证明唯一根因。
|
|
46
|
+
|
|
47
|
+
存储仍为 **1.24**,Controller/Host 协议仍为 v1。已有消息与事件保留,本版本
|
|
48
|
+
不新增存储迁移;较早受支持 Home 仍通过集中式连续迁移链升级。
|
|
49
|
+
另行获得更新授权后,由 Operator 核验实际发布包,再执行
|
|
50
|
+
`yui update --version 2.19.0`。兼容的存活 Session 保留;发布本身不更新运行安装,
|
|
51
|
+
也不恢复其他 Task。精确 merge commit 的 Core CI 与同一 tarball 的全新安装
|
|
52
|
+
平台/Node 矩阵仍为发布门禁,这些验证不代表真实模型行为验证。
|
|
53
|
+
|
|
39
54
|
### 2.18.0 兼容性及迁移说明
|
|
40
55
|
|
|
41
56
|
本 minor 包含已合入的独立 Jobs 工作区和全局 Shell 脚本 Job(PR #490),以及
|