@sema-agent/server 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/budget-tracing.d.ts +48 -0
- package/dist/boot/budget-tracing.js +86 -0
- package/dist/boot/config-center.d.ts +62 -0
- package/dist/boot/config-center.js +995 -0
- package/dist/boot/coordinators.d.ts +33 -0
- package/dist/boot/coordinators.js +97 -0
- package/dist/boot/execution-env.d.ts +26 -0
- package/dist/boot/execution-env.js +370 -0
- package/dist/boot/leader.d.ts +27 -0
- package/dist/boot/leader.js +81 -0
- package/dist/boot/reapers.d.ts +53 -0
- package/dist/boot/reapers.js +252 -0
- package/dist/boot/resolve-spec.d.ts +70 -0
- package/dist/boot/resolve-spec.js +1072 -0
- package/dist/boot/runner-deps.d.ts +101 -0
- package/dist/boot/runner-deps.js +343 -0
- package/dist/boot/runtime-caps.d.ts +21 -0
- package/dist/boot/runtime-caps.js +62 -0
- package/dist/boot/session-faces.d.ts +57 -0
- package/dist/boot/session-faces.js +157 -0
- package/dist/boot/shutdown.d.ts +50 -0
- package/dist/boot/shutdown.js +129 -0
- package/dist/boot/stores.d.ts +32 -0
- package/dist/boot/stores.js +361 -0
- package/dist/boot/workflow-orchestration.d.ts +46 -0
- package/dist/boot/workflow-orchestration.js +150 -0
- package/dist/config-types.d.ts +40 -2
- package/dist/config.d.ts +14 -2
- package/dist/config.js +539 -361
- package/dist/main.js +163 -3798
- package/package.json +1 -1
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* design/158 A10:composition root 分段 —— session 只读/删除面(审计、watch 注册表、E21 purge 协调器、
|
|
3
|
+
* degenerate 仪表、plan-cache 探针)。
|
|
4
|
+
*
|
|
5
|
+
* 纯搬运:函数体逐字来自 `main.ts`(原 2967-3092 行),缩进不变;新增的只有 import 与包壳。
|
|
6
|
+
*
|
|
7
|
+
* ⚠️ **位置即契约**:
|
|
8
|
+
* 1. `sessionWatchRegistry` 必须在 `purgeSession` **之前**构造 —— purge 尾部的 `dropSession` 是
|
|
9
|
+
* 同副本租户轮转围栏(delete 即终结旧 owner 的 SSE 订阅),两者是一对。
|
|
10
|
+
* 2. `setLeafAdvanceListener(...)` 是**进程级单点**装载(leaf-bus 只有一个监听位),必须与
|
|
11
|
+
* registry 的构造同段发生,晚于它、早于任何会 append 的路由。
|
|
12
|
+
*/
|
|
13
|
+
import { buildSessionAudit, buildSessionAuditLocal, buildSessionAuditPg } from "../audit.js";
|
|
14
|
+
import { makeDegenerateInstrument } from "../degenerate-instrument.js";
|
|
15
|
+
import { PlanCacheProbe } from "../plan-cache-probe.js";
|
|
16
|
+
import { purgeScratchpadDir } from "../env-facts.js";
|
|
17
|
+
import { setLeafAdvanceListener } from "../session-leaf-bus.js";
|
|
18
|
+
import { SessionWatchRegistry, posIntEnv } from "../session-watch.js";
|
|
19
|
+
import { defaultTaskRegistry } from "@sema-agent/core";
|
|
20
|
+
export function createSessionFaces(ctx) {
|
|
21
|
+
const { config, logger, metrics, localRoot, backend, sessionStore, runStore, checkpointStore, toolResultStore, resumeAnchorStore, approvalExemptionStore, sessionPolicyStore, taskAttachmentStore, fileSnapshotStore, workflowCompletionInbox, } = ctx;
|
|
22
|
+
// Audit回溯 + degenerate-output instrument are TiDB-specific raw-SQL consumers (outside the store
|
|
23
|
+
// abstraction); they run only on the TiDB backend (the mysql2 pool), and are a no-op on PG until ported.
|
|
24
|
+
const mysqlPool = backend?.mysqlPool();
|
|
25
|
+
const auditPgPool = backend?.pgPool(); // the audit face now has a PG twin — web session area lights up on DB_BACKEND=pg
|
|
26
|
+
const auditLocalRoot = backend?.kind === "local" ? (config.localDataRoot ?? localRoot) : undefined; // the File leg
|
|
27
|
+
const ownerAware = sessionStore;
|
|
28
|
+
const sessionAudit = mysqlPool || auditPgPool || auditLocalRoot
|
|
29
|
+
? async (sessionId) => {
|
|
30
|
+
const audit = mysqlPool
|
|
31
|
+
? await buildSessionAudit(mysqlPool, sessionId)
|
|
32
|
+
: auditPgPool
|
|
33
|
+
? await buildSessionAuditPg(auditPgPool, sessionId)
|
|
34
|
+
: await buildSessionAuditLocal(auditLocalRoot, sessionId);
|
|
35
|
+
if (!audit)
|
|
36
|
+
return undefined;
|
|
37
|
+
return { ...audit, owner: (await ownerAware.ownerOf?.(sessionId)) ?? null };
|
|
38
|
+
}
|
|
39
|
+
: undefined;
|
|
40
|
+
// §0.5 E21 — DELETE /v1/sessions/:id purge coordinator. Deletes the session abstraction's CONVERSATION HISTORY
|
|
41
|
+
// (sessionStorage.deleteSession → session_meta/session_event) PLUS the service-owned operational rows for that
|
|
42
|
+
// session: the runs ledger (runStore.deleteBySession → task_run/task_active/task_event, owner-guarded), the
|
|
43
|
+
// durable checkpoints + resume-ctx (checkpointStore.deleteBySession, present only under DURABLE_APPROVAL), and
|
|
44
|
+
// offloaded tool results (toolResultStore.deleteBySession). Wired only when the session store can actually
|
|
45
|
+
// delete (a DB backend).
|
|
46
|
+
//
|
|
47
|
+
// 🔴 ORDERING (adversarial-review HIGH — privacy/right-to-delete + silent-success): the session HISTORY delete
|
|
48
|
+
// (session_meta) is the `ownerOf` gate the DELETE route consults to decide idempotency. It MUST be the LAST
|
|
49
|
+
// committed leg. If a privacy-relevant child leg (runs/checkpoint/tool-result) is deleted AFTER the meta row is
|
|
50
|
+
// gone and then FAILS, the idempotent retry sees `ownerOf===undefined` (history already gone) → the route
|
|
51
|
+
// short-circuits to 200 {deleted:false} and NEVER re-purges → those child rows are stranded while the API
|
|
52
|
+
// reports success. So we purge the child legs FIRST and PROPAGATE any error (do NOT swallow): on a child
|
|
53
|
+
// failure the history delete is SKIPPED, `session_meta` survives, the DELETE returns 500, and the idempotent
|
|
54
|
+
// retry's `ownerOf` still resolves → it re-runs the FULL purge to convergence. `deleted` is computed from the
|
|
55
|
+
// FINAL committed result (history-deleted || runsRemoved>0 — the privacy-relevant data that actually existed).
|
|
56
|
+
// [1196] session-watch registry(SSE 订阅共享探针面)。谓词与 /head 探针同门(getLeafId+ownerOf);
|
|
57
|
+
// env 旋钮经 posIntEnv 有界校验;getLeafId 箭头包一层=天然绑定宿主(SESSION_CACHE_TTL_SEC=0 裸类实例)。
|
|
58
|
+
const sessionWatchRegistry = ownerAware.getHead && ownerAware.ownerOf
|
|
59
|
+
? new SessionWatchRegistry(async (sid) => (await ownerAware.getHead(sid)) ?? { owner: undefined, leafId: null }, {
|
|
60
|
+
hotMs: posIntEnv(process.env.SESSION_EVENTS_HOT_MS, 200),
|
|
61
|
+
warmMs: posIntEnv(process.env.SESSION_EVENTS_WARM_MS, 2000),
|
|
62
|
+
maxWatchedSessions: posIntEnv(process.env.SESSION_EVENTS_MAX_PROBES, 512, 100_000),
|
|
63
|
+
})
|
|
64
|
+
: undefined;
|
|
65
|
+
// S2 fast path([1208]③ 兑现):同副本 append 落点(tidb/pg persist post-commit)经 leaf-bus 直推
|
|
66
|
+
// notifyLocal——单副本部署零延迟;owner 不随 bus 传(写点无廉价 owner 读),探针围栏仍是租户权威
|
|
67
|
+
// (session-leaf-bus 契约注)。local lane 无写钩,探针道照旧。
|
|
68
|
+
setLeafAdvanceListener(sessionWatchRegistry ? (sid, leaf, owner, seq) => sessionWatchRegistry.notifyLocal(sid, leaf, owner, seq) : undefined);
|
|
69
|
+
const purgeSession = ownerAware.deleteSession
|
|
70
|
+
? async (sessionId, owner) => {
|
|
71
|
+
// Runs-ledger FIRST: it transactionally re-asserts the no-active-run invariant (FOR UPDATE on task_active,
|
|
72
|
+
// serializing against a concurrent createRun). If a run claimed the session in the route's check→purge
|
|
73
|
+
// window it returns `{ active }` and deletes nothing → we bubble it up so the route 409s, BEFORE touching
|
|
74
|
+
// any other table (no partial purge of a live session). Each leg propagates its error so a child failure
|
|
75
|
+
// aborts the purge BEFORE the history-delete gate is removed (keeping the idempotent retry convergent).
|
|
76
|
+
const runs = runStore ? await runStore.deleteBySession(sessionId, owner) : { removed: 0 };
|
|
77
|
+
if ("active" in runs)
|
|
78
|
+
return { active: runs.active };
|
|
79
|
+
// checkpoint rows carry no owner column → owner-guarded via a session_meta.owner sub-select, so they MUST
|
|
80
|
+
// run while session_meta still exists (before the history delete below); tool_result is keyed only by
|
|
81
|
+
// `ref` so it stays route-guarded (see its deleteBySession doc).
|
|
82
|
+
if (checkpointStore)
|
|
83
|
+
await checkpointStore.deleteBySession(sessionId, owner);
|
|
84
|
+
if (toolResultStore)
|
|
85
|
+
await toolResultStore.deleteBySession?.(sessionId); // optional extra: SQL twins only (local file store has no per-session purge index; E21 purge is a durable-backend contract)
|
|
86
|
+
// E18 resume-at anchors are per-session privacy-relevant metadata → purge ALL of them with the session, scoped
|
|
87
|
+
// by session_id ALONE (the route already proved session ownership at the DELETE gate). A per-ROW owner guard
|
|
88
|
+
// here would LEAK: an anchor's owner is the per-run submitting principal, which diverges from the canonical
|
|
89
|
+
// session owner when an anonymous session is later attached by a principal (REQUIRE_PRINCIPAL=false) — those
|
|
90
|
+
// mixed-owner rows would survive the delete (a right-to-delete violation reported as success).
|
|
91
|
+
if (resumeAnchorStore)
|
|
92
|
+
await resumeAnchorStore.deleteBySession(sessionId);
|
|
93
|
+
// Approval exemptions are per-session operator decisions → purge with the session (same
|
|
94
|
+
// session_id-alone scoping rationale as the anchors above).
|
|
95
|
+
if (approvalExemptionStore)
|
|
96
|
+
await approvalExemptionStore.deleteBySession(sessionId);
|
|
97
|
+
// E6 session-policy rows are per-session operator rules → purge them too, scoped by session_id (the route
|
|
98
|
+
// owner-gated the session). Since core 1.423 the seam is on the INTERFACE (optional) and core's File/InMemory
|
|
99
|
+
// stores carry it too — so this fires on EVERY backend now. (The old note here claimed local rules were
|
|
100
|
+
// "process-ephemeral anyway" — false for the file-backed local store, whose policy rows survive restarts;
|
|
101
|
+
// that right-to-delete hole is what [1796]§三 → core 1.423 closed.)
|
|
102
|
+
if (sessionPolicyStore?.deleteBySession)
|
|
103
|
+
await sessionPolicyStore.deleteBySession(sessionId);
|
|
104
|
+
// D-1 附件随会话删(E21 级联;session_id 单键 scoping,与 anchors 同理由——路由已证会话所有权)。
|
|
105
|
+
if (taskAttachmentStore)
|
|
106
|
+
await taskAttachmentStore.deleteBySession(sessionId);
|
|
107
|
+
// E19 rewind-files snapshots are per-session working-tree state → purge them too (scoped by sessionId; the
|
|
108
|
+
// durable twins carry deleteBySession, local InMemory omits it → optional no-op on local).
|
|
109
|
+
if (fileSnapshotStore?.deleteBySession)
|
|
110
|
+
await fileSnapshotStore.deleteBySession(sessionId);
|
|
111
|
+
// P1 ①②: drop any pending async-workflow completions for this session (else they orphan when a later
|
|
112
|
+
// tenant reclaims the sessionId). Best-effort + non-fatal — a leftover entry is ALSO gated by the
|
|
113
|
+
// drain-time owner check, so a purge failure never leaks; it must not abort the session delete.
|
|
114
|
+
// `owner` scopes the purge fence (1.80): only the DELETED session's own late completions are fenced —
|
|
115
|
+
// a new tenant legitimately re-claiming this sessionId within the window keeps its push (review MED).
|
|
116
|
+
if (workflowCompletionInbox)
|
|
117
|
+
await workflowCompletionInbox.purge(sessionId, owner ?? null).catch((err) => logger.warn("workflow_completion_inbox_purge_failed", { sessionId, err: String(err) }));
|
|
118
|
+
// 修8(三路复审 absorb-2,接线 (a)):the session's scratchpad dir ([820]③, ensureScratchpadDir) rides
|
|
119
|
+
// the session lifecycle — purge it with the session. purgeScratchpadDir SELF-SWALLOWS (env-facts.ts
|
|
120
|
+
// contract: cleanup never faults the delete path; the periodic sweep below is the convergence backstop),
|
|
121
|
+
// so it cannot strand the E21 ordering invariant. Same root the ensure used (config.localDataRoot ?? localRoot).
|
|
122
|
+
await purgeScratchpadDir(config.localDataRoot ?? localRoot, sessionId);
|
|
123
|
+
// design/129 (core 1.239): reap this session's SESSION-scoped background children — they
|
|
124
|
+
// deliberately outlive turns (backgroundScope:"session"), so the session's DELETE is their lifecycle
|
|
125
|
+
// end (core's parent-teardown reap skips them by design; without this they run to their forced timeout).
|
|
126
|
+
// Sync + replica-local (the registry is in-process); best-effort — a reap of 0 on the wrong replica is
|
|
127
|
+
// covered by core's forced child-timeout cap.
|
|
128
|
+
// [1892]{core} 跨仓待办:scope 轴必传——core canAccess 对缺省 scope fail-closed(拒绝非通配,
|
|
129
|
+
// 1.441 复审①正过极性),不传 = 四类后台任务在会话删除时恒 0 回收(潜伏至今,core 复审顺手
|
|
130
|
+
// 核出)。铸值同源 runs.ts:25 契约:core 按 `spec.principal ?? "default"` 注册 ⇒ 此处
|
|
131
|
+
// owner(会话主)?? "default" 对齐;错 scope 不越租户由 canAccess 保证(session-reap-scope 钉)。
|
|
132
|
+
try {
|
|
133
|
+
const reaped = defaultTaskRegistry.reapSessionBackground(sessionId, owner ?? "default");
|
|
134
|
+
if (reaped > 0)
|
|
135
|
+
logger.info("session_background_reaped", { sessionId, reaped });
|
|
136
|
+
}
|
|
137
|
+
catch (err) {
|
|
138
|
+
logger.warn("session_background_reap_failed", { sessionId, err: String(err) });
|
|
139
|
+
}
|
|
140
|
+
// History/meta LAST: once committed, the `ownerOf` gate is gone and the delete is durably complete. The
|
|
141
|
+
// owner is threaded into the SQL guard (session_meta.owner) for data-layer defense-in-depth.
|
|
142
|
+
const historyDeleted = await ownerAware.deleteSession(sessionId, owner);
|
|
143
|
+
// [1196] 三轮复审(租户轮转围栏,同副本半场):session 删除即终结其 SSE 订阅者+watch 条目——
|
|
144
|
+
// 旧 owner 的活流不得跨越 delete/reclaim 边界收新租户的 head;跨副本半场=路由心跳期 owner 复核。
|
|
145
|
+
sessionWatchRegistry?.dropSession(sessionId);
|
|
146
|
+
return { deleted: historyDeleted || runs.removed > 0 };
|
|
147
|
+
}
|
|
148
|
+
: undefined;
|
|
149
|
+
// Degenerate-output a/b instrument: needs the durable session log to inspect turn history (design/39
|
|
150
|
+
// ② gating data). Only on the TiDB backend (mysql2 pool); env-only / PG deploys get no-op.
|
|
151
|
+
const instrumentDegenerate = mysqlPool ? makeDegenerateInstrument(mysqlPool, metrics, logger) : undefined;
|
|
152
|
+
// Plan-cache recurrence probe (core design/42, INSTRUMENT-FIRST): pure in-memory, no backend needed —
|
|
153
|
+
// counts per-scope task-objective recurrence so core can decide whether to build plan caching.
|
|
154
|
+
const planCacheProbe = new PlanCacheProbe(metrics, logger);
|
|
155
|
+
return { ownerAware, sessionAudit, sessionWatchRegistry, purgeSession, instrumentDegenerate, planCacheProbe };
|
|
156
|
+
}
|
|
157
|
+
//# sourceMappingURL=session-faces.js.map
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* design/158 A10:composition root 分段 —— 进程收尾(hardShutdown / drain / 三个信号处理器)。
|
|
3
|
+
*
|
|
4
|
+
* 纯搬运:函数体逐字来自 `main.ts`(原 4385-4488 行),缩进不变;新增的只有 import 与
|
|
5
|
+
* `installShutdownHandlers(ctx)` 包壳。
|
|
6
|
+
*
|
|
7
|
+
* ⚠️ **位置即契约**(搬运不得改次序):
|
|
8
|
+
* 1. 本段必须在 `server.listen()` **之后**调用 —— `process.on("SIGTERM"|"SIGINT"|"SIGHUP")` 的注册
|
|
9
|
+
* 时点即"从此刻起信号可被优雅处理";提前注册会让一个尚未 listen 的进程走 drain 分支。
|
|
10
|
+
* 2. 三个 `process.on` 的**相对次序**保持 SIGTERM → SIGINT → SIGHUP(与原文件逐字一致);
|
|
11
|
+
* 它们互不覆盖,但 handler 之间共享 `closing`/`draining` 两个 latch,次序变化会改变
|
|
12
|
+
* "第二个信号"的语义读数。
|
|
13
|
+
* 3. `clearInterval(reaper)` 必须是 hardShutdown 的第一件事(在 `server.close()` 之前),否则
|
|
14
|
+
* 收尾期还会有 reaper tick 打向正在关闭的池。
|
|
15
|
+
*/
|
|
16
|
+
import { Runner, type RunnerDeps } from "@sema-agent/core";
|
|
17
|
+
import type { ServiceConfig } from "../config.js";
|
|
18
|
+
import type { createHttpServer } from "../http/server.js";
|
|
19
|
+
import type { Logger } from "../observability/logger.js";
|
|
20
|
+
import type { startOtlpExporter } from "../observability/otel-exporter.js";
|
|
21
|
+
import type { CostQuota } from "../observability/cost-quota.js";
|
|
22
|
+
import type { RateLimiter } from "../observability/rate-limit.js";
|
|
23
|
+
import type { startFleetClientFromEnv } from "../fleet-client.js";
|
|
24
|
+
import type { BreakerStateStore, CostQuotaStore, RateLimiterStore, StoreBackend } from "../plugins/store-backend.js";
|
|
25
|
+
import type { WorkflowNotifyJournalStore } from "../orchestration/workflow-notify-journal.js";
|
|
26
|
+
export interface ShutdownCtx {
|
|
27
|
+
config: ServiceConfig;
|
|
28
|
+
logger: Logger;
|
|
29
|
+
server: ReturnType<typeof createHttpServer>;
|
|
30
|
+
reaper: NodeJS.Timeout;
|
|
31
|
+
otelExporter: ReturnType<typeof startOtlpExporter> | undefined;
|
|
32
|
+
breakerState: ReturnType<BreakerStateStore["startRefresh"]> | undefined;
|
|
33
|
+
costQuota: CostQuota | CostQuotaStore | undefined;
|
|
34
|
+
rateLimiter: RateLimiter | RateLimiterStore | undefined;
|
|
35
|
+
runner: Runner;
|
|
36
|
+
subRunner: Runner;
|
|
37
|
+
lspManager: RunnerDeps["lspManager"];
|
|
38
|
+
workflowNotifyJournal: WorkflowNotifyJournalStore | undefined;
|
|
39
|
+
fleetClient: ReturnType<typeof startFleetClientFromEnv>;
|
|
40
|
+
backend: StoreBackend | undefined;
|
|
41
|
+
drainState: {
|
|
42
|
+
draining: boolean;
|
|
43
|
+
since?: number;
|
|
44
|
+
inflight?: () => number;
|
|
45
|
+
lastActivityAt?: () => number;
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
/** 注册 SIGTERM/SIGINT/SIGHUP 收尾链。**必须在 listen 之后调用**(见文件头「位置即契约」)。 */
|
|
49
|
+
export declare function installShutdownHandlers(ctx: ShutdownCtx): void;
|
|
50
|
+
//# sourceMappingURL=shutdown.d.ts.map
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* design/158 A10:composition root 分段 —— 进程收尾(hardShutdown / drain / 三个信号处理器)。
|
|
3
|
+
*
|
|
4
|
+
* 纯搬运:函数体逐字来自 `main.ts`(原 4385-4488 行),缩进不变;新增的只有 import 与
|
|
5
|
+
* `installShutdownHandlers(ctx)` 包壳。
|
|
6
|
+
*
|
|
7
|
+
* ⚠️ **位置即契约**(搬运不得改次序):
|
|
8
|
+
* 1. 本段必须在 `server.listen()` **之后**调用 —— `process.on("SIGTERM"|"SIGINT"|"SIGHUP")` 的注册
|
|
9
|
+
* 时点即"从此刻起信号可被优雅处理";提前注册会让一个尚未 listen 的进程走 drain 分支。
|
|
10
|
+
* 2. 三个 `process.on` 的**相对次序**保持 SIGTERM → SIGINT → SIGHUP(与原文件逐字一致);
|
|
11
|
+
* 它们互不覆盖,但 handler 之间共享 `closing`/`draining` 两个 latch,次序变化会改变
|
|
12
|
+
* "第二个信号"的语义读数。
|
|
13
|
+
* 3. `clearInterval(reaper)` 必须是 hardShutdown 的第一件事(在 `server.close()` 之前),否则
|
|
14
|
+
* 收尾期还会有 reaper tick 打向正在关闭的池。
|
|
15
|
+
*/
|
|
16
|
+
import { Runner, defaultTaskRegistry } from "@sema-agent/core";
|
|
17
|
+
import { createSighupIdleHandler } from "../sighup-idle.js";
|
|
18
|
+
/** 注册 SIGTERM/SIGINT/SIGHUP 收尾链。**必须在 listen 之后调用**(见文件头「位置即契约」)。 */
|
|
19
|
+
export function installShutdownHandlers(ctx) {
|
|
20
|
+
const { config, logger, server, reaper, otelExporter, breakerState, costQuota, rateLimiter, runner, subRunner, lspManager, workflowNotifyJournal, fleetClient, backend, drainState, } = ctx;
|
|
21
|
+
let closing = false;
|
|
22
|
+
const hardShutdown = () => {
|
|
23
|
+
if (closing)
|
|
24
|
+
return;
|
|
25
|
+
closing = true;
|
|
26
|
+
clearInterval(reaper);
|
|
27
|
+
otelExporter?.stop();
|
|
28
|
+
breakerState?.stop();
|
|
29
|
+
if (costQuota && "stop" in costQuota)
|
|
30
|
+
costQuota.stop(); // the DB-backed quota/limiter (TiDB|PG) have a refresh loop to stop; the in-memory ones don't
|
|
31
|
+
if (rateLimiter && "stop" in rateLimiter)
|
|
32
|
+
rateLimiter.stop();
|
|
33
|
+
runner.sessions.dispose();
|
|
34
|
+
subRunner.sessions.dispose();
|
|
35
|
+
// TOC local LSP: NodeLspManager holds LOCAL child-process language servers — kill them on shutdown so a
|
|
36
|
+
// restart doesn't orphan a fleet of stdio servers. dispose() isn't on the LspServerManager interface (only the
|
|
37
|
+
// concrete managers have it), so probe it; the e2b/k8s bridge manager is a harmless no-op if it lacks one.
|
|
38
|
+
void lspManager?.dispose?.();
|
|
39
|
+
// SVC-1: release the File journal's append handle; the SQL twins hold no fd (probe — close isn't on the seam).
|
|
40
|
+
void workflowNotifyJournal?.close?.();
|
|
41
|
+
// core 1.270.1:hardShutdown 终点(drain 已完,in-flight 不再被破坏)全量收割 session 驻留
|
|
42
|
+
// bg bash(detached 进程组,引擎死后 reparent PID 1 残留=实测;retain-declared 例外在原语内)。
|
|
43
|
+
// TOB 容器形态同样调=无害且显式;SIGKILL 路径无钩=已知不可救记档。
|
|
44
|
+
try {
|
|
45
|
+
const reaped = defaultTaskRegistry.reapAllSessionBackground();
|
|
46
|
+
if (reaped > 0)
|
|
47
|
+
logger.info("shutdown_background_reaped", { reaped });
|
|
48
|
+
}
|
|
49
|
+
catch (err) {
|
|
50
|
+
logger.warn("shutdown_background_reap_failed", { err: String(err) });
|
|
51
|
+
}
|
|
52
|
+
void fleetClient?.stop(); // stop heartbeat + flush final usage window + best-effort DELETE deregistration
|
|
53
|
+
server.close(() => {
|
|
54
|
+
void (backend ? backend.close() : Promise.resolve()).finally(() => process.exit(0));
|
|
55
|
+
});
|
|
56
|
+
// 对抗复查 B-4(HIGH):the exit above only fires after backend.close() SETTLES — a wedged pool teardown
|
|
57
|
+
// (dead DB, hung socket) kept the "force-terminated" process alive indefinitely (the 10s closeAllConnections
|
|
58
|
+
// below unblocks server.close(), but nothing bounded backend.close). FINAL deadline: whatever is still
|
|
59
|
+
// holding after 15s, exit anyway (durable state is already flushed by then — checkpoint/terminal writes
|
|
60
|
+
// happen before hardShutdown; the pool teardown is best-effort cleanup, not correctness).
|
|
61
|
+
const finalExit = setTimeout(() => { logger.warn("hard_shutdown_final_deadline", { afterMs: 15_000 }); process.exit(0); }, 15_000);
|
|
62
|
+
finalExit.unref?.();
|
|
63
|
+
// `server.close()` waits for every connection to end — an open SSE stream (/v1/tasks/stream, /events) would
|
|
64
|
+
// otherwise block it forever, so pool.end()/exit never run and the process hangs until SIGKILL. Drop idle
|
|
65
|
+
// keep-alives now, then force-terminate any lingering connections after a grace period so `close()` can
|
|
66
|
+
// resolve and shutdown completes. The timer is unref'd but still fires while the server holds connections.
|
|
67
|
+
server.closeIdleConnections?.();
|
|
68
|
+
const forceClose = setTimeout(() => server.closeAllConnections?.(), 10_000);
|
|
69
|
+
forceClose.unref?.();
|
|
70
|
+
};
|
|
71
|
+
// SIGTERM = graceful DRAIN — flip `draining` (new billable submits 503+Retry-After, /health carries
|
|
72
|
+
// draining:true for readiness摘流), then wait for this instance's in-flight legs (live streams + bg/resume) to
|
|
73
|
+
// finish before the hard shutdown. Bounded by DRAIN_GRACE_MS (default 10min — an interactive turn is minutes;
|
|
74
|
+
// the previous behavior was a 10s hard-cut that killed long turns). A SECOND SIGTERM or SIGINT (dev Ctrl-C)
|
|
75
|
+
// skips the wait — k8s sends SIGKILL after terminationGracePeriodSeconds regardless, so the escape hatch is free.
|
|
76
|
+
let draining = false;
|
|
77
|
+
const drainThenShutdown = () => {
|
|
78
|
+
if (closing)
|
|
79
|
+
return;
|
|
80
|
+
if (draining) {
|
|
81
|
+
logger.info("drain_second_signal_hard_stop", {});
|
|
82
|
+
hardShutdown();
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
draining = true;
|
|
86
|
+
drainState.draining = true;
|
|
87
|
+
drainState.since = Date.now();
|
|
88
|
+
void fleetClient?.announceNow(); // draining 翻转即刻再 announce — 第一时间自摘流,比任何观测都快
|
|
89
|
+
const inflight = drainState.inflight?.() ?? 0;
|
|
90
|
+
logger.info("draining_started", { inflight, graceMs: config.drainGraceMs });
|
|
91
|
+
if (inflight === 0) {
|
|
92
|
+
hardShutdown();
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
// 修3(三路复审 B3-2 时钟):drain elapsed/截止计算走 performance.now() 单调域(挂钟跳变不得吞掉/拉长
|
|
96
|
+
// drainGraceMs 窗);对外报告的 drainState.since(/health drainingSince)保留 epoch(Date.now)。
|
|
97
|
+
const t0 = performance.now();
|
|
98
|
+
// NOT unref'd on purpose — this timer IS the shutdown driver (the open connections keep the loop alive anyway).
|
|
99
|
+
const tick = setInterval(() => {
|
|
100
|
+
const n = drainState.inflight?.() ?? 0;
|
|
101
|
+
const elapsed = Math.round(performance.now() - t0);
|
|
102
|
+
if (n === 0 || elapsed >= config.drainGraceMs) {
|
|
103
|
+
clearInterval(tick);
|
|
104
|
+
logger.info("drain_complete", { inflight: n, elapsedMs: elapsed, timedOut: n > 0 });
|
|
105
|
+
hardShutdown();
|
|
106
|
+
}
|
|
107
|
+
}, 1_000);
|
|
108
|
+
};
|
|
109
|
+
process.on("SIGTERM", drainThenShutdown);
|
|
110
|
+
process.on("SIGINT", hardShutdown);
|
|
111
|
+
// SIGHUP(壳窗口关闭把信号打到整个进程组)≠「没人在用这只引擎」:共享引擎形态(同 config 多壳会话,
|
|
112
|
+
// 引擎按 T11 防孤儿拍骑在首壳进程组里)下 peer 会话还活着——无条件 drain 会把 peer mid-turn 斩掉(499)
|
|
113
|
+
// 且窗内新提交 503(B3,壳侧 [771]② 取证)。改 hup-pending 空闲窗:不翻 draining(peer 零感知),
|
|
114
|
+
// inflight()(bg/resume legs + live streams)连续 sighupIdleGraceMs 为 0 才 hardShutdown,任何在飞 leg
|
|
115
|
+
// 重置窗口。真孤儿(最后一窗关掉)有界自灭=T11 保留;第二个 SIGHUP 升级走既有 drain 路径(硬梯子)。
|
|
116
|
+
// session 驻留 bash 的全量收割仍挂 hardShutdown 终点(reapAllSessionBackground)。
|
|
117
|
+
process.on("SIGHUP", createSighupIdleHandler({
|
|
118
|
+
inflight: () => drainState.inflight?.() ?? 0,
|
|
119
|
+
// ⚠️ 时基契约(三路复审修3):lastActivityAt 由 server.ts 以 performance.now() 单调域打点,与
|
|
120
|
+
// sighup-idle.ts 内部的 `now` seam(缺省 performance.now)同域——不得混入 Date.now 值。
|
|
121
|
+
lastActivityAt: () => drainState.lastActivityAt?.() ?? 0,
|
|
122
|
+
isStopped: () => closing || draining,
|
|
123
|
+
escalate: drainThenShutdown,
|
|
124
|
+
shutdown: hardShutdown,
|
|
125
|
+
idleGraceMs: config.sighupIdleGraceMs,
|
|
126
|
+
log: (event, fields) => logger.info(event, fields),
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
//# sourceMappingURL=shutdown.js.map
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { type MemoryBackend } from "@sema-agent/core";
|
|
2
|
+
import type { ServiceConfig } from "../config.js";
|
|
3
|
+
import { type MemorySyncRunner } from "../memory-sync-client.js";
|
|
4
|
+
import type { Logger } from "../observability/logger.js";
|
|
5
|
+
import type { Metrics } from "../observability/metrics.js";
|
|
6
|
+
import { type MemorySyncStore } from "../plugins/memory-sync-store-pg.js";
|
|
7
|
+
import { type TaskAttachmentStore } from "../plugins/task-attachment-store.js";
|
|
8
|
+
import { type StoreBackend } from "../plugins/store-backend.js";
|
|
9
|
+
export interface OpenStoresCtx {
|
|
10
|
+
config: ServiceConfig;
|
|
11
|
+
logger: Logger;
|
|
12
|
+
metrics: Metrics;
|
|
13
|
+
localRoot: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function openStores(ctx: OpenStoresCtx): Promise<{
|
|
16
|
+
backend: StoreBackend | undefined;
|
|
17
|
+
storeBackendDegraded: boolean;
|
|
18
|
+
memoryEngine: {
|
|
19
|
+
backend: MemoryBackend;
|
|
20
|
+
root: string;
|
|
21
|
+
} | undefined;
|
|
22
|
+
memorySyncCursors: MemorySyncStore | undefined;
|
|
23
|
+
rosterStore: import("@sema-agent/core").RosterStore | undefined;
|
|
24
|
+
backgroundAgentStore: import("@sema-agent/core").BackgroundAgentStore | undefined;
|
|
25
|
+
taskAttachmentStore: TaskAttachmentStore | undefined;
|
|
26
|
+
mailboxStore: import("@sema-agent/core").MailboxStore | undefined;
|
|
27
|
+
memoryExportBackend: MemoryBackend | undefined;
|
|
28
|
+
memorySyncRunner: MemorySyncRunner | undefined;
|
|
29
|
+
sessionStore: import("@sema-agent/core").SessionStore;
|
|
30
|
+
breakerState: import("../plugins/breaker-state-sql.js").TiDBBreakerState | import("../plugins/breaker-state-sql.js").PgBreakerState | undefined;
|
|
31
|
+
}>;
|
|
32
|
+
//# sourceMappingURL=stores.d.ts.map
|