@sema-agent/server 7.101.0 → 7.101.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/stage-10-http-server.js +2 -0
- package/dist/fleet/fleet-bus.d.ts +30 -4
- package/dist/fleet/fleet-bus.js +20 -4
- package/dist/fleet/fleet-reconciler.d.ts +29 -6
- package/dist/fleet/fleet-reconciler.js +109 -79
- package/dist/observability/fail-open.d.ts +4 -0
- package/dist/observability/fail-open.js +4 -0
- package/dist/tool-approval.d.ts +9 -0
- package/dist/tool-approval.js +6 -0
- package/package.json +1 -1
|
@@ -186,6 +186,8 @@ export async function runStage10(ctx) {
|
|
|
186
186
|
isLiveOnThisReplica: server.isRunLiveOnThisReplica,
|
|
187
187
|
...(runStore ? { runs: runStore } : {}),
|
|
188
188
|
...(workflowRunStore ? { workflows: workflowRunStore } : {}),
|
|
189
|
+
agents: backgroundAgentStore,
|
|
190
|
+
hasPendingAsk: toolApproval ? (originTaskId) => toolApproval.hasLivePendingFrom(originTaskId) : undefined,
|
|
189
191
|
instanceId,
|
|
190
192
|
logger,
|
|
191
193
|
metrics,
|
|
@@ -248,13 +248,19 @@ export type FleetFrame = {
|
|
|
248
248
|
* durable 行压根不存在 —— 三形共同点是**没有终局可投影**,而本副本的行必然陈旧。此时造一个终态
|
|
249
249
|
* 就是编:判死的属主始终是 durable 侧的 reaper。所以只陈述「这条行离场了」,不逼消费端结算。
|
|
250
250
|
* (cli 侧裁量 [ref]:渲「离场」**不清行** —— 行保留、中性离场形、active 扣除、折叠排后。)
|
|
251
|
+
* 🔴 S-751:子代行的陈述级退场之前,对账腿先问过它的属主(bg registry 非终态 ∨ 有待决审批 ⇒ 留行);
|
|
252
|
+
* 本帧因此只在「静默满一周期 ∧ 属主不作保」时出现。而它**不封被退那一代**(见 {@link cycleSeq})——
|
|
253
|
+
* 之后同一代(或更大代)的帧到达 = 那一行其实还活着,行**复活**;更小的代照旧是前代迟到帧。
|
|
251
254
|
* ADDITIVE / tolerate-absent。
|
|
252
255
|
*/
|
|
253
256
|
removeReason?: "orphaned";
|
|
254
257
|
/** [ref] §2 判死×判代合成规则:退场帧带**被退那一代**的 {@link FleetTaskRow.cycleSeq}。
|
|
255
258
|
* 消费端因此能把 `retire(gen=N)` 之后的 `frame(gen=N+1)` 读成**复活**(行重生,退休不粘住新代),
|
|
256
|
-
*
|
|
257
|
-
*
|
|
259
|
+
* 而**结算级** `retire(gen=N)`(本帧无 `removeReason`)之后的 `frame(gen=N)` 读成**前代迟到帧,忽略**。
|
|
260
|
+
* 🔴 S-751:**陈述级**退场(`removeReason:"orphaned"`)之后的 `frame(gen=N)` = **复活**,不是迟到帧 ——
|
|
261
|
+
* 陈述级退场从不断言「这一代结束了」(它说的只是「本副本一个周期没听到这一行」),所以同代的下一帧
|
|
262
|
+
* 就是它还活着的证据(本仓 {@link FleetEventBus.removeTask} 对这一形不封代,消费端同读);更小的代照旧忽略。
|
|
263
|
+
* 缺席同 {@link FleetTaskRow.cycleSeq} ——「无此概念」,不是「第一代」。 */
|
|
258
264
|
cycleSeq?: number;
|
|
259
265
|
} | {
|
|
260
266
|
type: "workflow";
|
|
@@ -452,7 +458,10 @@ export declare class FleetEventBus {
|
|
|
452
458
|
private readonly tasks;
|
|
453
459
|
private readonly workflows;
|
|
454
460
|
/** [ref] / codex R1-F2:**代际墓碑** —— 已离场行的最后已知 `cycleSeq`(行本身已从 `tasks` 删掉,
|
|
455
|
-
* 代际状态却必须活得比行久一点,否则前代迟到帧会在空 Map 上把幽灵行重铸出来)。有界 LRU。
|
|
461
|
+
* 代际状态却必须活得比行久一点,否则前代迟到帧会在空 Map 上把幽灵行重铸出来)。有界 LRU。
|
|
462
|
+
* 🔴 S-751:碑上多记一位 `ended` = 「**这一代结束了**」是否被断言过。离场**恒**保住代际状态(`gen` 之前的帧
|
|
463
|
+
* 永远是前代迟到帧 —— 与行还在 Map 里时那道闸同一个判据);只有作「这一代结束了」断言的离场(结算级 / 投影级
|
|
464
|
+
* 常规清场)才把 `gen` **这一代本身**也封掉。陈述级退场(`removeReason:"orphaned"`)不作这个断言 ⇒ 同代帧 = 复活。 */
|
|
456
465
|
private readonly retiredGen;
|
|
457
466
|
private static readonly RETIRED_GEN_MAX;
|
|
458
467
|
constructor(now?: () => number);
|
|
@@ -472,7 +481,7 @@ export declare class FleetEventBus {
|
|
|
472
481
|
/** Drop a task row (terminal + swept) + fan out the removal. Idempotent (a no-op if already gone).
|
|
473
482
|
*
|
|
474
483
|
* `opts`([ref],两键均 ADDITIVE):`removeReason:"orphaned"` = **陈述级退场**(没有结算语义,见
|
|
475
|
-
* {@link FleetFrame} 的 `task_remove`
|
|
484
|
+
* {@link FleetFrame} 的 `task_remove` 臂;S-751 起**不封被退那一代**,同代帧 = 复活);`cycleSeq` = 被退那一代的代际号。发布方的常规清场
|
|
476
485
|
* (`onTerminal`/`onChildTerminal`)不传 opts ⇒ 帧逐字节与本改动之前相同。 */
|
|
477
486
|
removeTask(id: string, opts?: {
|
|
478
487
|
removeReason?: "orphaned";
|
|
@@ -489,6 +498,23 @@ export declare class FleetEventBus {
|
|
|
489
498
|
backgroundChildLaneRow(scope: string | undefined, childUuid: string): string | undefined;
|
|
490
499
|
/** [ref] 让位迁移料的读面:迁移「只补缺席位」需要看得见存活行现有什么(BCE 是权威车道,不覆盖)。 */
|
|
491
500
|
taskRow(id: string): Readonly<FleetTaskRow> | undefined;
|
|
501
|
+
/**
|
|
502
|
+
* S-751:一条子代行在它两只**属主账**上的键。三个 id 域互不相同,换算只有本类做得了(两种行 id 都是本文件
|
|
503
|
+
* 铸的,两域之间唯一的 join —— 让位登记簿 —— 也住在这里),所以换算只写这一处:
|
|
504
|
+
* · 行 id —— BCE 车道 = registry handle 本身(`e.taskId`);run-leg 车道 = `${runId} ${子代 task_progress taskId}`
|
|
505
|
+
* (`fleetRunPublisher` 的 `childId`);
|
|
506
|
+
* · `handle` —— core bg registry / `BackgroundAgentStore` 的行键,**只认** `DURABLE_AGENT_HANDLE_RE` 形
|
|
507
|
+
* (`wa*` 不在 bg 账上);
|
|
508
|
+
* · `sessionIds` —— 子代自己的 core session id(uuid 域)= 待决审批 `originTaskId` 的值域
|
|
509
|
+
* (core `AskRequest.sourceTaskId`:「the issuing task's session id」),**不是** handle。是**集合**:同一行名下
|
|
510
|
+
* 可以先后挂过几个子代 session(`wa*` 重试换 session、复活换 session),让位登记簿对每一个都留着 claim ——
|
|
511
|
+
* 只取其一(此前取最早那个)会漏掉当前这一次的卡(codex r1 F3)。
|
|
512
|
+
* 换不出的键就缺席 / 空(不猜):父链接行 / 顶层 run 行两者皆无;BCE 行在本车道学到子代 uuid 之前 `sessionIds` 为空。
|
|
513
|
+
*/
|
|
514
|
+
childOwnerKeys(row: Pick<FleetTaskRow, "id" | "scope">): {
|
|
515
|
+
handle?: string;
|
|
516
|
+
sessionIds: readonly string[];
|
|
517
|
+
};
|
|
492
518
|
/** Upsert a workflow row (MERGE by `id`) + fan out. */
|
|
493
519
|
publishWorkflow(delta: Partial<FleetWorkflowRow> & {
|
|
494
520
|
id: string;
|
package/dist/fleet/fleet-bus.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EventEmitter } from "node:events";
|
|
2
|
-
import { deriveAgentDisplayStatus, isDelegatedAgentTerminal } from "@sema-agent/core";
|
|
2
|
+
import { DURABLE_AGENT_HANDLE_RE, deriveAgentDisplayStatus, isDelegatedAgentTerminal } from "@sema-agent/core";
|
|
3
3
|
import { persistedPlaneOf } from "../terminal.js";
|
|
4
4
|
import { recordFailOpen } from "../observability/fail-open.js";
|
|
5
5
|
import { redactSecrets } from "../trace/redact.js";
|
|
@@ -74,7 +74,7 @@ export class FleetEventBus {
|
|
|
74
74
|
return;
|
|
75
75
|
if (existing === undefined && delta.retiredBy === undefined) {
|
|
76
76
|
const tomb = this.retiredGen.get(delta.id);
|
|
77
|
-
if (tomb !== undefined && delta.cycleSeq !== undefined && delta.cycleSeq
|
|
77
|
+
if (tomb !== undefined && delta.cycleSeq !== undefined && (delta.cycleSeq < tomb.gen || (tomb.ended && delta.cycleSeq === tomb.gen)))
|
|
78
78
|
return;
|
|
79
79
|
}
|
|
80
80
|
const merged = { ...(existing ?? { id: delta.id, status: "running" }), ...delta };
|
|
@@ -97,10 +97,12 @@ export class FleetEventBus {
|
|
|
97
97
|
if (existing !== undefined && opts?.cycleSeq !== undefined && existing.cycleSeq !== undefined && existing.cycleSeq > opts.cycleSeq)
|
|
98
98
|
return;
|
|
99
99
|
if (this.tasks.delete(id)) {
|
|
100
|
-
const
|
|
100
|
+
const prior = this.retiredGen.get(id);
|
|
101
|
+
const gen = existing?.cycleSeq ?? opts?.cycleSeq ?? prior?.gen;
|
|
101
102
|
if (gen !== undefined) {
|
|
103
|
+
const ended = opts?.removeReason !== "orphaned" || (prior !== undefined && prior.gen === gen && prior.ended);
|
|
102
104
|
this.retiredGen.delete(id);
|
|
103
|
-
this.retiredGen.set(id, gen);
|
|
105
|
+
this.retiredGen.set(id, { gen, ended });
|
|
104
106
|
if (this.retiredGen.size > FleetEventBus.RETIRED_GEN_MAX) {
|
|
105
107
|
const oldest = this.retiredGen.keys().next();
|
|
106
108
|
if (!oldest.done)
|
|
@@ -138,6 +140,20 @@ export class FleetEventBus {
|
|
|
138
140
|
taskRow(id) {
|
|
139
141
|
return this.tasks.get(id);
|
|
140
142
|
}
|
|
143
|
+
childOwnerKeys(row) {
|
|
144
|
+
const sp = row.id.lastIndexOf(" ");
|
|
145
|
+
if (sp > 0) {
|
|
146
|
+
const sessionId = row.id.slice(sp + 1);
|
|
147
|
+
const claimed = this.bceClaims.get(FleetEventBus.bceKey(row.scope, sessionId));
|
|
148
|
+
return { sessionIds: [sessionId], ...(claimed !== undefined && DURABLE_AGENT_HANDLE_RE.test(claimed) ? { handle: claimed } : {}) };
|
|
149
|
+
}
|
|
150
|
+
const prefix = FleetEventBus.bceKey(row.scope, "");
|
|
151
|
+
const sessionIds = [];
|
|
152
|
+
for (const [k, rowId] of this.bceClaims)
|
|
153
|
+
if (rowId === row.id && k.startsWith(prefix))
|
|
154
|
+
sessionIds.push(k.slice(prefix.length));
|
|
155
|
+
return { sessionIds, ...(DURABLE_AGENT_HANDLE_RE.test(row.id) ? { handle: row.id } : {}) };
|
|
156
|
+
}
|
|
141
157
|
publishWorkflow(delta) {
|
|
142
158
|
const merged = { ...(this.workflows.get(delta.id) ?? { id: delta.id, status: "running" }), ...delta };
|
|
143
159
|
this.workflows.set(delta.id, merged);
|
|
@@ -33,10 +33,17 @@
|
|
|
33
33
|
* 「**静默满一个对账周期**」(本腿订阅总线自记每行的最后帧时刻):还在出帧 = 活着的证据。
|
|
34
34
|
* durable 行**在场**的那三格没有这个歧义(`getRun` 命中 = 它就是一条 run 行),故照矩阵即时处置。
|
|
35
35
|
*
|
|
36
|
-
* ## 子代行(§1)
|
|
37
|
-
*
|
|
38
|
-
* bg 子代的 durable 真源在 core 侧 registry,server
|
|
39
|
-
*
|
|
36
|
+
* ## 子代行(§1;🔴 S-751 起改为「活性由属主说了算」)
|
|
37
|
+
* 子代行**不读 durable run 店**(它不是 run 行)。它的活性属主有两只,都在本仓可读 —— 读它们**不是**越界重判,
|
|
38
|
+
* 是问属主(S-751 之前此处写「bg 子代的 durable 真源在 core 侧 registry,server 不越界重判」,于是判据里只剩
|
|
39
|
+
* 「静默」这一条,而 registry 的读半场 `RunnerDeps.backgroundAgentStore` 一直就在本仓):
|
|
40
|
+
* ① core bg registry(`BackgroundAgentStore` 行 `status` 非终态 = `running` / `parked`);
|
|
41
|
+
* ② 本副本审批面的待决卡(子代发起、没人答的 ask —— 子代卡在那儿,当然不出帧)。
|
|
42
|
+
* **一条规则**:无父可循(父从未在 / 父本周期刚退休或退场)∧ 静默满一个对账周期 ∧ 两只属主都不作保 ⇒
|
|
43
|
+
* 陈述级退场。父离场**不再无条件同扫**子代(那是报案的一半病:父轮结束、子代一次工具调用 ≥60s 或审批卡
|
|
44
|
+
* 60s 没人答 ⇒ 活子代被扫);父离场只让子代「无父可循」,子代与既有孤儿走**同一只**判据、同一个循环。
|
|
45
|
+
* 同一问也挂在「durable 无行」那一格上:父链接未解析的 `a*` 行与顶层行同形(见上方收窄段),run 账不认识它
|
|
46
|
+
* ≠ 它死了。另一半病(陈述级退场立了代际墓碑、吞掉活子代之后的同代帧)的修在 `fleet-bus.ts` `removeTask`。
|
|
40
47
|
*
|
|
41
48
|
* ## workflow 行(§1)
|
|
42
49
|
* 只做同款「durable 终局投影」([ref] 的 snapshot pull 腿 + reapers 的 workflow recover 腿已在场),
|
|
@@ -77,7 +84,7 @@
|
|
|
77
84
|
* 与 `fleet-terminal-window.ts` 同一论证:两侧边界都由语义定死(比周期短 = 幽灵行仍在面板上,比周期长 =
|
|
78
85
|
* 拿对账腿当历史列表),旋钮只会让部署方去调一个没有正确取值的数;fleet 面现有零旋钮,从之。
|
|
79
86
|
*/
|
|
80
|
-
import { type WorkflowRun, type WorkflowRunStore } from "@sema-agent/core";
|
|
87
|
+
import { type BackgroundAgentRecord, type WorkflowRun, type WorkflowRunStore } from "@sema-agent/core";
|
|
81
88
|
import type { Logger } from "../observability/logger.js";
|
|
82
89
|
import type { Metrics } from "../observability/metrics.js";
|
|
83
90
|
import { type RunRecord } from "../plugins/store-contracts.js";
|
|
@@ -111,6 +118,12 @@ export interface FleetReconcileRunPort {
|
|
|
111
118
|
export interface FleetReconcileWorkflowPort {
|
|
112
119
|
get(id: string): Promise<WorkflowRun | null | undefined>;
|
|
113
120
|
}
|
|
121
|
+
/** S-751:core bg registry 账(`BackgroundAgentStore`)的只读窄口 —— 与 {@link FleetReconcileRunPort} 同一条
|
|
122
|
+
* 单写者纪律(接口上没有写方法)。生产 = `RunnerDeps.backgroundAgentStore` **同实例**(结构上就是它的 `get`
|
|
123
|
+
* 一面);`null` = 账上没有这一行。 */
|
|
124
|
+
export interface FleetReconcileAgentPort {
|
|
125
|
+
get(handle: string, scope: string): Promise<Pick<BackgroundAgentRecord, "status"> | null>;
|
|
126
|
+
}
|
|
114
127
|
export interface FleetReconcilerDeps {
|
|
115
128
|
bus: FleetEventBus;
|
|
116
129
|
/**
|
|
@@ -130,6 +143,14 @@ export interface FleetReconcilerDeps {
|
|
|
130
143
|
subscribeRunTerminal?: (fn: (rec: RunTerminalRecord) => void) => () => void;
|
|
131
144
|
runs?: FleetReconcileRunPort;
|
|
132
145
|
workflows?: FleetReconcileWorkflowPort | WorkflowRunStore;
|
|
146
|
+
/** S-751:子代活性的属主①(bg registry 账)。缺席 = 没有 bg 店的部署(纯内存 dev),这一问问不了。 */
|
|
147
|
+
agents?: FleetReconcileAgentPort;
|
|
148
|
+
/**
|
|
149
|
+
* S-751:子代活性的属主②:这只子代此刻有没有**待决**审批卡。只问是非(与 {@link isLiveOnThisReplica} 同姿势,
|
|
150
|
+
* 不搬 Map)。入参是 ask 发起者域(子代自己的 core session id,= 卡上的 `originTaskId`),**不是**行 id ——
|
|
151
|
+
* 行 id → 该域的换算只在 `FleetEventBus.childOwnerKeys`。生产 = `ToolApprovalCoordinator.hasLivePendingFrom`。
|
|
152
|
+
*/
|
|
153
|
+
hasPendingAsk?: (originTaskId: string) => boolean;
|
|
133
154
|
/** 本副本 id(与 `RunRecord.instanceId` 同域)。只进日志/读数 —— 两条 `running` 臂的**处置相同**
|
|
134
155
|
* (都是陈述级退场),分不分得出 claim 属谁不改变行为,故它不是判据。 */
|
|
135
156
|
instanceId?: string;
|
|
@@ -150,8 +171,10 @@ export interface FleetReconcileTally {
|
|
|
150
171
|
retired: number;
|
|
151
172
|
/** 陈述级退场的行数(顶层 + 孤儿子行)。 */
|
|
152
173
|
orphaned: number;
|
|
153
|
-
/**
|
|
174
|
+
/** 父本周期刚离场、自己又判成孤儿而陈述级退场的子行数(S-751 前 = 随父无条件同扫的数)。 */
|
|
154
175
|
childrenSwept: number;
|
|
176
|
+
/** S-751:静默满一周期、无父可循(或 run 账不认识),但属主作保而**留行**的子行数。 */
|
|
177
|
+
childrenKeptAlive: number;
|
|
155
178
|
/** durable 终局 ⇒ 投影退休的 workflow 行数。 */
|
|
156
179
|
workflowsRetired: number;
|
|
157
180
|
/** 本周期实际发出的 durable 点读次数。 */
|
|
@@ -11,7 +11,7 @@ export const FLEET_RECONCILE_COOLDOWN_MS = 60_000;
|
|
|
11
11
|
export const FLEET_RECONCILE_WATCH_MAX = 200;
|
|
12
12
|
export const FLEET_RECONCILE_WATCH_TTL_MS = 30 * 60_000;
|
|
13
13
|
const emptyTally = () => ({
|
|
14
|
-
scanned: 0, skippedLive: 0, skippedParked: 0, retired: 0, orphaned: 0, childrenSwept: 0, workflowsRetired: 0, reads: 0, cooling: false, failOpen: false,
|
|
14
|
+
scanned: 0, skippedLive: 0, skippedParked: 0, retired: 0, orphaned: 0, childrenSwept: 0, childrenKeptAlive: 0, workflowsRetired: 0, reads: 0, cooling: false, failOpen: false,
|
|
15
15
|
});
|
|
16
16
|
function rotate(rows, cursor) {
|
|
17
17
|
if (cursor === undefined || rows.length === 0)
|
|
@@ -22,6 +22,22 @@ function rotate(rows, cursor) {
|
|
|
22
22
|
return [...rows.slice(at), ...rows.slice(0, at)];
|
|
23
23
|
}
|
|
24
24
|
const SILENT_FOR_A_CYCLE = FLEET_RECONCILE_INTERVAL_MS;
|
|
25
|
+
function bgAgentVouches(status) {
|
|
26
|
+
switch (status) {
|
|
27
|
+
case "running":
|
|
28
|
+
case "parked":
|
|
29
|
+
return true;
|
|
30
|
+
case "completed":
|
|
31
|
+
case "failed":
|
|
32
|
+
case "killed":
|
|
33
|
+
return false;
|
|
34
|
+
default: {
|
|
35
|
+
const unhandled = status;
|
|
36
|
+
recordFailOpen("server.fleet.agent-status-unmapped", String(unhandled));
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
25
41
|
export function createFleetReconciler(deps) {
|
|
26
42
|
const now = deps.now ?? (() => Date.now());
|
|
27
43
|
const lastFrameAt = new Map();
|
|
@@ -86,11 +102,11 @@ export function createFleetReconciler(deps) {
|
|
|
86
102
|
coolingUntil = undefined;
|
|
87
103
|
const deadline = now() + FLEET_RECONCILE_READ_BUDGET_MS;
|
|
88
104
|
const budgetLeft = () => tally.reads < FLEET_RECONCILE_MAX_ROWS && now() < deadline;
|
|
89
|
-
const
|
|
105
|
+
const budgetedRead = async (read) => {
|
|
90
106
|
tally.reads++;
|
|
91
107
|
let t;
|
|
92
|
-
|
|
93
|
-
|
|
108
|
+
return await Promise.race([
|
|
109
|
+
read(),
|
|
94
110
|
new Promise((_r, reject) => {
|
|
95
111
|
t = setTimeout(() => reject(new ReadAborted(`fleet reconcile read exceeded ${FLEET_RECONCILE_READ_BUDGET_MS}ms`)), Math.max(0, deadline - now()));
|
|
96
112
|
if (typeof t.unref === "function")
|
|
@@ -100,30 +116,43 @@ export function createFleetReconciler(deps) {
|
|
|
100
116
|
if (t !== undefined)
|
|
101
117
|
clearTimeout(t);
|
|
102
118
|
});
|
|
103
|
-
return rec;
|
|
104
119
|
};
|
|
120
|
+
const readRun = (id) => budgetedRead(() => deps.runs.getRun(id));
|
|
105
121
|
const snap = deps.bus.snapshot();
|
|
106
122
|
for (const r of snap.tasks)
|
|
107
123
|
if (!lastFrameAt.has(r.id))
|
|
108
124
|
lastFrameAt.set(r.id, nowMs);
|
|
109
125
|
const idle = (id) => nowMs - (lastFrameAt.get(id) ?? nowMs) >= SILENT_FOR_A_CYCLE;
|
|
110
126
|
const present = new Set(snap.tasks.map((r) => r.id));
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
127
|
+
const parentsOnBus = new Set(present);
|
|
128
|
+
const stillSilent = (row) => {
|
|
129
|
+
const cur = deps.bus.taskRow(row.id);
|
|
130
|
+
return cur !== undefined && cur.cycleSeq === row.cycleSeq && idle(row.id);
|
|
131
|
+
};
|
|
132
|
+
const keptAlive = (row, why) => {
|
|
133
|
+
tally.childrenKeptAlive++;
|
|
134
|
+
deps.logger?.info("fleet_row_reconciled", { taskId: row.id, disposition: "child_kept_alive", why, instanceId: deps.instanceId ?? null });
|
|
135
|
+
};
|
|
136
|
+
const judgeSilentRow = async (row, exit) => {
|
|
137
|
+
const handle = deps.bus.childOwnerKeys(row).handle;
|
|
138
|
+
if (handle !== undefined && deps.agents !== undefined && row.scope !== undefined) {
|
|
139
|
+
if (!budgetLeft())
|
|
140
|
+
return "over_budget";
|
|
141
|
+
const scope = row.scope;
|
|
142
|
+
const rec = await budgetedRead(() => deps.agents.get(handle, scope));
|
|
143
|
+
if (rec !== null && bgAgentVouches(rec.status)) {
|
|
144
|
+
keptAlive(row, "bg_registry");
|
|
145
|
+
return "settled";
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const hasPendingAsk = deps.hasPendingAsk;
|
|
149
|
+
if (hasPendingAsk !== undefined && deps.bus.childOwnerKeys(row).sessionIds.some((id) => hasPendingAsk(id))) {
|
|
150
|
+
keptAlive(row, "pending_ask");
|
|
151
|
+
return "settled";
|
|
126
152
|
}
|
|
153
|
+
if (stillSilent(row))
|
|
154
|
+
exit();
|
|
155
|
+
return "settled";
|
|
127
156
|
};
|
|
128
157
|
const retire = (row, rec) => {
|
|
129
158
|
deps.bus.publishTask({
|
|
@@ -132,14 +161,14 @@ export function createFleetReconciler(deps) {
|
|
|
132
161
|
retiredBy: "reconcile",
|
|
133
162
|
...fleetRunResiduals(rec.result),
|
|
134
163
|
});
|
|
135
|
-
|
|
164
|
+
parentsOnBus.delete(row.id);
|
|
136
165
|
deps.bus.removeTask(row.id, { ...(row.cycleSeq !== undefined ? { cycleSeq: row.cycleSeq } : {}) });
|
|
137
166
|
lastFrameAt.delete(row.id);
|
|
138
167
|
tally.retired++;
|
|
139
168
|
deps.metrics?.inc("fleet_rows_reconciled_total", { disposition: "retired" });
|
|
140
169
|
};
|
|
141
170
|
const orphanExit = (row, why) => {
|
|
142
|
-
|
|
171
|
+
parentsOnBus.delete(row.id);
|
|
143
172
|
deps.bus.removeTask(row.id, { removeReason: "orphaned", ...(row.cycleSeq !== undefined ? { cycleSeq: row.cycleSeq } : {}) });
|
|
144
173
|
lastFrameAt.delete(row.id);
|
|
145
174
|
if (deps.runs && watch.size < FLEET_RECONCILE_WATCH_MAX)
|
|
@@ -148,6 +177,61 @@ export function createFleetReconciler(deps) {
|
|
|
148
177
|
deps.metrics?.inc("fleet_rows_reconciled_total", { disposition: "orphaned" });
|
|
149
178
|
deps.logger?.info("fleet_row_reconciled", { taskId: row.id, disposition: "orphaned", why, instanceId: deps.instanceId ?? null });
|
|
150
179
|
};
|
|
180
|
+
const childExit = (row, parentId) => {
|
|
181
|
+
deps.bus.removeTask(row.id, { removeReason: "orphaned", ...(row.cycleSeq !== undefined ? { cycleSeq: row.cycleSeq } : {}) });
|
|
182
|
+
lastFrameAt.delete(row.id);
|
|
183
|
+
parentsOnBus.delete(row.id);
|
|
184
|
+
if (present.has(parentId)) {
|
|
185
|
+
tally.childrenSwept++;
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
tally.orphaned++;
|
|
189
|
+
deps.metrics?.inc("fleet_rows_reconciled_total", { disposition: "orphaned_child" });
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
const judgeTopLevel = async (row) => {
|
|
193
|
+
tally.scanned++;
|
|
194
|
+
if (deps.isLiveOnThisReplica(row.id)) {
|
|
195
|
+
tally.skippedLive++;
|
|
196
|
+
return "settled";
|
|
197
|
+
}
|
|
198
|
+
if (!deps.runs)
|
|
199
|
+
return "settled";
|
|
200
|
+
if (!budgetLeft())
|
|
201
|
+
return "over_budget";
|
|
202
|
+
const rec = await readRun(row.id);
|
|
203
|
+
if (deps.isLiveOnThisReplica(row.id)) {
|
|
204
|
+
tally.skippedLive++;
|
|
205
|
+
return "settled";
|
|
206
|
+
}
|
|
207
|
+
const nowRow = deps.bus.taskRow(row.id);
|
|
208
|
+
if (nowRow === undefined)
|
|
209
|
+
return "settled";
|
|
210
|
+
if (nowRow.cycleSeq !== row.cycleSeq)
|
|
211
|
+
return "settled";
|
|
212
|
+
if (rec === undefined) {
|
|
213
|
+
if (!idle(row.id))
|
|
214
|
+
return "settled";
|
|
215
|
+
return judgeSilentRow(row, () => orphanExit(row, "durable_row_absent"));
|
|
216
|
+
}
|
|
217
|
+
if (isTerminalRunStatus(rec.status)) {
|
|
218
|
+
retire(row, rec);
|
|
219
|
+
return "settled";
|
|
220
|
+
}
|
|
221
|
+
if (isParkedRunStatus(rec.status)) {
|
|
222
|
+
tally.skippedParked++;
|
|
223
|
+
return "settled";
|
|
224
|
+
}
|
|
225
|
+
orphanExit(row, runProvenElsewhere(rec, deps.instanceId) ? "durable_running_other_replica" : "durable_running_no_local_leg");
|
|
226
|
+
return "settled";
|
|
227
|
+
};
|
|
228
|
+
const judgeChild = async (row, parentId) => {
|
|
229
|
+
if (parentsOnBus.has(parentId))
|
|
230
|
+
return "settled";
|
|
231
|
+
if (!idle(row.id))
|
|
232
|
+
return "settled";
|
|
233
|
+
return judgeSilentRow(row, () => childExit(row, parentId));
|
|
234
|
+
};
|
|
151
235
|
try {
|
|
152
236
|
for (const [id, w] of [...watch]) {
|
|
153
237
|
if (nowMs - w.at > FLEET_RECONCILE_WATCH_TTL_MS) {
|
|
@@ -171,53 +255,11 @@ export function createFleetReconciler(deps) {
|
|
|
171
255
|
tally.retired++;
|
|
172
256
|
}
|
|
173
257
|
for (const row of rotate(snap.tasks, taskCursor)) {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
tally.scanned++;
|
|
177
|
-
if (deps.isLiveOnThisReplica(row.id)) {
|
|
178
|
-
tally.skippedLive++;
|
|
179
|
-
continue;
|
|
180
|
-
}
|
|
181
|
-
if (!deps.runs)
|
|
182
|
-
continue;
|
|
183
|
-
if (!budgetLeft()) {
|
|
258
|
+
const step = row.parentId === undefined ? await judgeTopLevel(row) : await judgeChild(row, row.parentId);
|
|
259
|
+
if (step === "over_budget") {
|
|
184
260
|
taskCursor = row.id;
|
|
185
261
|
break;
|
|
186
262
|
}
|
|
187
|
-
const rec = await readRun(row.id);
|
|
188
|
-
if (deps.isLiveOnThisReplica(row.id)) {
|
|
189
|
-
tally.skippedLive++;
|
|
190
|
-
continue;
|
|
191
|
-
}
|
|
192
|
-
const nowRow = deps.bus.taskRow(row.id);
|
|
193
|
-
if (nowRow === undefined)
|
|
194
|
-
continue;
|
|
195
|
-
if (nowRow.cycleSeq !== row.cycleSeq)
|
|
196
|
-
continue;
|
|
197
|
-
if (rec === undefined) {
|
|
198
|
-
if (idle(row.id))
|
|
199
|
-
orphanExit(row, "durable_row_absent");
|
|
200
|
-
continue;
|
|
201
|
-
}
|
|
202
|
-
if (isTerminalRunStatus(rec.status)) {
|
|
203
|
-
retire(row, rec);
|
|
204
|
-
continue;
|
|
205
|
-
}
|
|
206
|
-
if (isParkedRunStatus(rec.status)) {
|
|
207
|
-
tally.skippedParked++;
|
|
208
|
-
continue;
|
|
209
|
-
}
|
|
210
|
-
orphanExit(row, runProvenElsewhere(rec, deps.instanceId) ? "durable_running_other_replica" : "durable_running_no_local_leg");
|
|
211
|
-
}
|
|
212
|
-
for (const row of snap.tasks) {
|
|
213
|
-
if (row.parentId === undefined || present.has(row.parentId))
|
|
214
|
-
continue;
|
|
215
|
-
if (!idle(row.id))
|
|
216
|
-
continue;
|
|
217
|
-
deps.bus.removeTask(row.id, { removeReason: "orphaned", ...(row.cycleSeq !== undefined ? { cycleSeq: row.cycleSeq } : {}) });
|
|
218
|
-
lastFrameAt.delete(row.id);
|
|
219
|
-
tally.orphaned++;
|
|
220
|
-
deps.metrics?.inc("fleet_rows_reconciled_total", { disposition: "orphaned_child" });
|
|
221
263
|
}
|
|
222
264
|
if (wfGet) {
|
|
223
265
|
for (const wf of rotate(snap.workflows, wfCursor)) {
|
|
@@ -227,19 +269,7 @@ export function createFleetReconciler(deps) {
|
|
|
227
269
|
wfCursor = wf.id;
|
|
228
270
|
break;
|
|
229
271
|
}
|
|
230
|
-
|
|
231
|
-
let wfTimer;
|
|
232
|
-
const run = await Promise.race([
|
|
233
|
-
wfGet(wf.id),
|
|
234
|
-
new Promise((_r, reject) => {
|
|
235
|
-
wfTimer = setTimeout(() => reject(new ReadAborted("fleet reconcile workflow read over budget")), Math.max(0, deadline - now()));
|
|
236
|
-
if (typeof wfTimer.unref === "function")
|
|
237
|
-
wfTimer.unref();
|
|
238
|
-
}),
|
|
239
|
-
]).finally(() => {
|
|
240
|
-
if (wfTimer !== undefined)
|
|
241
|
-
clearTimeout(wfTimer);
|
|
242
|
-
});
|
|
272
|
+
const run = await budgetedRead(() => wfGet(wf.id));
|
|
243
273
|
if (!run || !isTerminalWorkflowStatus(run.status))
|
|
244
274
|
continue;
|
|
245
275
|
if (wf.scope !== undefined && run.scope !== wf.scope)
|
|
@@ -173,6 +173,10 @@ export declare const FAIL_OPEN_TAGS: {
|
|
|
173
173
|
readonly cls: "F";
|
|
174
174
|
readonly note: "A-032 P1-④:`runStatusToFleet` 收到**词表外**的 run 状态词(闭集签名之外的唯一来路 = `rowToRun` 把无约束的 status 文本列裸 cast,滚动升级里更新的副本写的新词被旧副本读到)⇒ 折成 `idle`。放行的最坏后果 = 该行在 fleet 面板上**永久滞留**(removal 集 {completed,failed,killed} 收不到 idle),纯展示面、不影响 run 本身与任何执法判据,故 F 类。方向刻意不改(把 miss 折成终局会把一条还在跑的任务从面板上摘掉,误摘比多一行更伤)。留痕是承重的:2026-06-28 on-box e2e 逮到的 `blocked` 泄漏当年只能靠人眼在面板上发现——计数让下一次词表漂移在遥测里当场显形。编译期一侧已由闭集入参 + `never` 臂执法。";
|
|
175
175
|
};
|
|
176
|
+
readonly "server.fleet.agent-status-unmapped": {
|
|
177
|
+
readonly cls: "F";
|
|
178
|
+
readonly note: "S-751:fleet 对账腿问 bg registry「这只子代还活着吗」时,读到**词表外**的 `BackgroundAgentRecord.status`(闭集之外的唯一来路 = SQL 孪生把 status 文本列裸 cast 成联合,滚动升级里更新的 core 写下的新词被本副本读到)⇒ 按「属主作保」处置:子代行**留在面板上**。放行的最坏后果 = 一条其实已终局的子代行多留在面板上(它的终态通知仍会结算它;与 #261 之前「子代行永远不走」同形),纯展示面、不影响子代本身与任何执法判据,故 F 类。方向刻意不折成「死」:把不认识的词读成死会把一只正在跑的后台子代从面板上摘掉,而那正是 S-751 报案的症状(误摘比多一行更伤,与 `server.fleet.run-status-unmapped` 同一条判据)。编译期一侧由 `bgAgentVouches` 的穷举 switch + `never` 臂执法;留痕让词表漂移在遥测里当场显形。";
|
|
179
|
+
};
|
|
176
180
|
readonly "server.run-store.stale-claim-quarantine-unremoved": {
|
|
177
181
|
readonly cls: "F";
|
|
178
182
|
readonly note: "#213 陈旧 claim 接管:claim 已 rename 进 tmp/ 隔离名(接管本体已完成、正确性不受影响),但隔离件 rmSync 失败留在 tmp。放行的最坏后果 = tmp 里多一个死文件(不在 activeDir,hydrate 永不把它当 claim 复活);故 F 类。必须留痕:反复删不掉 = tmp 权限/fs 病灶,静默吞掉会让 tmp 无声膨胀。";
|
|
@@ -164,6 +164,10 @@ export const FAIL_OPEN_TAGS = {
|
|
|
164
164
|
cls: "F",
|
|
165
165
|
note: "A-032 P1-④:`runStatusToFleet` 收到**词表外**的 run 状态词(闭集签名之外的唯一来路 = `rowToRun` 把无约束的 status 文本列裸 cast,滚动升级里更新的副本写的新词被旧副本读到)⇒ 折成 `idle`。放行的最坏后果 = 该行在 fleet 面板上**永久滞留**(removal 集 {completed,failed,killed} 收不到 idle),纯展示面、不影响 run 本身与任何执法判据,故 F 类。方向刻意不改(把 miss 折成终局会把一条还在跑的任务从面板上摘掉,误摘比多一行更伤)。留痕是承重的:2026-06-28 on-box e2e 逮到的 `blocked` 泄漏当年只能靠人眼在面板上发现——计数让下一次词表漂移在遥测里当场显形。编译期一侧已由闭集入参 + `never` 臂执法。",
|
|
166
166
|
},
|
|
167
|
+
"server.fleet.agent-status-unmapped": {
|
|
168
|
+
cls: "F",
|
|
169
|
+
note: "S-751:fleet 对账腿问 bg registry「这只子代还活着吗」时,读到**词表外**的 `BackgroundAgentRecord.status`(闭集之外的唯一来路 = SQL 孪生把 status 文本列裸 cast 成联合,滚动升级里更新的 core 写下的新词被本副本读到)⇒ 按「属主作保」处置:子代行**留在面板上**。放行的最坏后果 = 一条其实已终局的子代行多留在面板上(它的终态通知仍会结算它;与 #261 之前「子代行永远不走」同形),纯展示面、不影响子代本身与任何执法判据,故 F 类。方向刻意不折成「死」:把不认识的词读成死会把一只正在跑的后台子代从面板上摘掉,而那正是 S-751 报案的症状(误摘比多一行更伤,与 `server.fleet.run-status-unmapped` 同一条判据)。编译期一侧由 `bgAgentVouches` 的穷举 switch + `never` 臂执法;留痕让词表漂移在遥测里当场显形。",
|
|
170
|
+
},
|
|
167
171
|
"server.run-store.stale-claim-quarantine-unremoved": {
|
|
168
172
|
cls: "F",
|
|
169
173
|
note: "#213 陈旧 claim 接管:claim 已 rename 进 tmp/ 隔离名(接管本体已完成、正确性不受影响),但隔离件 rmSync 失败留在 tmp。放行的最坏后果 = tmp 里多一个死文件(不在 activeDir,hydrate 永不把它当 claim 复活);故 F 类。必须留痕:反复删不掉 = tmp 权限/fs 病灶,静默吞掉会让 tmp 无声膨胀。",
|
package/dist/tool-approval.d.ts
CHANGED
|
@@ -1588,6 +1588,15 @@ export declare class ToolApprovalCoordinator {
|
|
|
1588
1588
|
* 宁少列不超范围;`"__none__"` 未鉴权哨兵自然恒空)。行的 `approvalId` 就是 `respond()` 收的 id
|
|
1589
1589
|
* (同源可达,[ref] 判据);`expiresAtMs` 照抄登记时单点铸定的窗死线([ref] 三键同一个数)。 */
|
|
1590
1590
|
listLivePending(scope: string | undefined): LivePendingRow[];
|
|
1591
|
+
/**
|
|
1592
|
+
* S-751:某只委派子代此刻有没有**待决**的活卡 —— fleet 对账腿「子代活性由属主说了算」的属主②读口(另一只是
|
|
1593
|
+
* bg registry 账)。子代卡在一张没人答的卡上时当然不出帧,只看「静默」就会把它当孤儿扫下面板(报案的真形)。
|
|
1594
|
+
* 只答是非、不交出 Map(与 `isRunLiveOnThisReplica` 同姿势);纯读,零锁零结算语义。
|
|
1595
|
+
* 键 = ask 发起者域(`originTaskId` = core `AskRequest.sourceTaskId` = 子代自己的 core session id),**不是**
|
|
1596
|
+
* fleet 行 id —— 换算在 `FleetEventBus.childOwnerKeys`。只看本副本的内存登记:子代跑在本副本,它的活卡就登记在
|
|
1597
|
+
* 本副本;耐久 park 的卡由 bg registry 的 `parked` 态作保(见 fleet-reconciler.ts `bgAgentVouches`)。
|
|
1598
|
+
*/
|
|
1599
|
+
hasLivePendingFrom(originTaskId: string): boolean;
|
|
1591
1600
|
/**
|
|
1592
1601
|
* 🔴 [ref](案A)追记 F4 —— **跨副本决议的执行通知**(悬挂行的一次终态轮询)。
|
|
1593
1602
|
*
|
package/dist/tool-approval.js
CHANGED
|
@@ -2114,6 +2114,12 @@ export class ToolApprovalCoordinator {
|
|
|
2114
2114
|
}
|
|
2115
2115
|
return rows;
|
|
2116
2116
|
}
|
|
2117
|
+
hasLivePendingFrom(originTaskId) {
|
|
2118
|
+
for (const e of this.pending.values())
|
|
2119
|
+
if (e.originTaskId === originTaskId)
|
|
2120
|
+
return true;
|
|
2121
|
+
return false;
|
|
2122
|
+
}
|
|
2117
2123
|
async pollSuspendedDecisions() {
|
|
2118
2124
|
const store = this.askStore;
|
|
2119
2125
|
if (!store)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sema-agent/server",
|
|
3
|
-
"version": "7.101.
|
|
3
|
+
"version": "7.101.1",
|
|
4
4
|
"description": "Sema Server — the server/API implementation layer for Sema, wiring core, registry, model providers, and cloud agent execution. Built on @sema-agent/core.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BUSL-1.1",
|