@sema-agent/server 7.117.0 → 7.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +97 -0
- package/USAGE.md +37 -18
- package/dist/approval-card.d.ts +40 -10
- package/dist/approval-card.js +11 -1
- package/dist/approval.d.ts +4 -12
- package/dist/approval.js +4 -26
- package/dist/boot/parked-revive-gate.d.ts +1 -7
- package/dist/boot/parked-revive-gate.js +2 -12
- package/dist/boot/resolve-spec.js +8 -13
- package/dist/boot/runner-deps.js +2 -2
- package/dist/boot/stage-07-capability-layer.js +2 -2
- package/dist/capabilities/hands-lane.d.ts +25 -0
- package/dist/capabilities/hands-lane.js +1 -1
- package/dist/capabilities/mcp-probe.d.ts +2 -2
- package/dist/config.d.ts +14 -18
- package/dist/config.js +13 -24
- package/dist/deployment-governance.d.ts +2 -7
- package/dist/deployment-governance.js +3 -5
- package/dist/host-decision.d.ts +83 -25
- package/dist/host-decision.js +31 -1
- package/dist/http/admission.js +7 -1
- package/dist/http/approval-sweeps.js +1 -1
- package/dist/http/mcp-dial-deadline.d.ts +35 -0
- package/dist/http/mcp-dial-deadline.js +21 -0
- package/dist/http/memory-compliance-failure-reply.d.ts +11 -5
- package/dist/http/memory-compliance-failure-reply.js +17 -8
- package/dist/http/memory-store-refusal-reply.d.ts +14 -11
- package/dist/http/memory-store-refusal-reply.js +16 -5
- package/dist/http/resume-legs.js +7 -3
- package/dist/http/routes/approvals-assistant.js +21 -24
- package/dist/http/routes/capabilities.js +2 -0
- package/dist/http/routes/mcp-reconnect.d.ts +17 -0
- package/dist/http/routes/mcp-reconnect.js +6 -1
- package/dist/http/routes/memory-bundle.js +1 -1
- package/dist/http/routes/memory-compliance.js +1 -1
- package/dist/http/routes/memory-origin.js +1 -1
- package/dist/http/routes/runs.js +41 -33
- package/dist/http/routes/sessions.d.ts +1 -1
- package/dist/http/routes/sessions.js +3 -6
- package/dist/http/routes/workflows.js +2 -0
- package/dist/http/wire-types.d.ts +6 -1
- package/dist/memory-store-refusal.d.ts +42 -17
- package/dist/memory-store-refusal.js +25 -6
- package/dist/model-unreachable-notice.d.ts +1 -1
- package/dist/namespaced-rule-name.d.ts +45 -41
- package/dist/namespaced-rule-name.js +36 -22
- package/dist/observability/corrupt-read-seat.d.ts +4 -0
- package/dist/observability/corrupt-read-seat.js +1 -0
- package/dist/observability/fail-open.d.ts +14 -2
- package/dist/observability/fail-open.js +14 -2
- package/dist/operator-ask.d.ts +4 -1
- package/dist/orchestration/workflow-agent-session-index.d.ts +82 -59
- package/dist/orchestration/workflow-agent-session-index.js +80 -14
- package/dist/orchestration/workflow-completion-inbox.d.ts +5 -0
- package/dist/orchestration/workflow-completion-inbox.js +39 -17
- package/dist/orchestration/workflow-notify-journal.d.ts +1 -1
- package/dist/orchestration/workflow-notify-journal.js +7 -2
- package/dist/outbound-dispatcher.d.ts +29 -20
- package/dist/outbound-dispatcher.js +105 -16
- package/dist/parked-decide.d.ts +43 -48
- package/dist/parked-decide.js +54 -16
- package/dist/permission-rule-vocab.d.ts +4 -3
- package/dist/permission-rule-vocab.js +1 -1
- package/dist/plugins/approval-ask-store-sql.d.ts +17 -20
- package/dist/plugins/approval-ask-store-sql.js +0 -3
- package/dist/plugins/one-time-migrations.d.ts +5 -0
- package/dist/plugins/one-time-migrations.js +110 -4
- package/dist/plugins/pg-pool.js +1 -1
- package/dist/plugins/tidb-pool.js +1 -1
- package/dist/plugins/web-search.d.ts +20 -1
- package/dist/plugins/web-search.js +39 -10
- package/dist/plugins/workflow-run-store-sql.d.ts +7 -6
- package/dist/plugins/workflow-run-store-sql.js +5 -7
- package/dist/request-key-closure.js +1 -1
- package/dist/request-value-closure.d.ts +1 -1
- package/dist/rule-entry-list.d.ts +35 -46
- package/dist/rule-entry-list.js +17 -25
- package/dist/rules-consent.js +3 -0
- package/dist/run-local.js +1 -0
- package/dist/runs.d.ts +1 -1
- package/dist/session-policy-wire.d.ts +13 -12
- package/dist/session-policy-wire.js +11 -6
- package/dist/spec-fields.d.ts +7 -9
- package/dist/spec-fields.js +4 -7
- package/dist/standing-answer-grant.d.ts +14 -2
- package/dist/standing-answer-grant.js +8 -1
- package/dist/task-settings.js +8 -4
- package/dist/tool-approval.d.ts +53 -112
- package/dist/tool-approval.js +75 -55
- package/dist/trace/core-keyset-guard.d.ts +5 -5
- package/dist/trace/self-minted-wire.d.ts +16 -2
- package/dist/trace/self-minted-wire.js +7 -0
- package/dist/user-input-ledger.d.ts +24 -12
- package/dist/user-input-ledger.js +33 -14
- package/package.json +2 -2
- package/scripts/drain-check-local.mjs +74 -3
- package/scripts/drain-check.mysql.sql +43 -17
- package/scripts/drain-check.pg.sql +46 -20
|
@@ -6,27 +6,87 @@ export function readRunParks(run) {
|
|
|
6
6
|
if (typeof run !== "object" || run === null)
|
|
7
7
|
return undefined;
|
|
8
8
|
const v = run.parks;
|
|
9
|
+
if (v === undefined)
|
|
10
|
+
return [];
|
|
9
11
|
if (!Array.isArray(v))
|
|
10
12
|
return undefined;
|
|
11
13
|
const out = [];
|
|
12
14
|
for (const raw of v) {
|
|
13
15
|
if (typeof raw !== "object" || raw === null)
|
|
14
16
|
return undefined;
|
|
15
|
-
const { callKey, sessionId, token, originRunId } = raw;
|
|
17
|
+
const { callKey, sessionId, token, originRunId, originUnconfirmed, parkedBy, generation } = raw;
|
|
16
18
|
if (!nonEmptyString(callKey) || !nonEmptyString(sessionId) || !nonEmptyString(token) || !nonEmptyString(originRunId))
|
|
17
19
|
return undefined;
|
|
18
|
-
|
|
20
|
+
if (originUnconfirmed !== undefined && originUnconfirmed !== true)
|
|
21
|
+
return undefined;
|
|
22
|
+
const holder = parkedBy === undefined && generation === undefined
|
|
23
|
+
? {}
|
|
24
|
+
: nonEmptyString(parkedBy) && typeof generation === "number" && Number.isSafeInteger(generation) && generation >= 0
|
|
25
|
+
? { parkedBy, generation }
|
|
26
|
+
: undefined;
|
|
27
|
+
if (holder === undefined)
|
|
28
|
+
return undefined;
|
|
29
|
+
out.push({ callKey, sessionId, token, originRunId, ...(originUnconfirmed === true ? { originUnconfirmed } : {}), ...holder });
|
|
19
30
|
}
|
|
20
31
|
return out;
|
|
21
32
|
}
|
|
22
|
-
export function
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
33
|
+
export function heldParksOf(recordId, parks) {
|
|
34
|
+
return parks.filter((p) => p.parkedBy === recordId);
|
|
35
|
+
}
|
|
36
|
+
export function buildParkSessionRows(runId, run) {
|
|
37
|
+
const parks = readRunParks(run);
|
|
38
|
+
if (parks === undefined)
|
|
39
|
+
return undefined;
|
|
40
|
+
const bySession = new Map();
|
|
41
|
+
for (const p of parks)
|
|
42
|
+
if (!bySession.has(p.sessionId))
|
|
43
|
+
bySession.set(p.sessionId, { sessionId: p.sessionId, runId, agentKey: p.callKey, scope: run.scope });
|
|
44
|
+
return [...bySession.values()];
|
|
45
|
+
}
|
|
46
|
+
export function routeWorkflowPark(sessionId, candidates) {
|
|
47
|
+
const ordered = [...candidates].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
48
|
+
const readable = [];
|
|
49
|
+
let firstUnreadable;
|
|
50
|
+
for (const run of ordered) {
|
|
51
|
+
const parks = readRunParks(run);
|
|
52
|
+
if (parks === undefined)
|
|
53
|
+
firstUnreadable ??= run;
|
|
54
|
+
else
|
|
55
|
+
readable.push({ run, parks });
|
|
56
|
+
}
|
|
57
|
+
const top = highestHolders(sessionId, readable);
|
|
58
|
+
if (firstUnreadable !== undefined) {
|
|
59
|
+
const holder = top.length === 1 ? top[0].run.id : undefined;
|
|
60
|
+
return { kind: "unproven", cause: "unreadable", runId: firstUnreadable.id, run: firstUnreadable, ...(holder !== undefined ? { readableHolderRunId: holder } : {}) };
|
|
61
|
+
}
|
|
62
|
+
if (top.length === 1)
|
|
63
|
+
return { kind: "held", runId: top[0].run.id, run: top[0].run, park: top[0].park };
|
|
64
|
+
if (top.length > 1)
|
|
65
|
+
return { kind: "forked", runIds: top.map((t) => t.run.id) };
|
|
66
|
+
const older = readable.filter(({ parks }) => parks.some((p) => p.sessionId === sessionId && p.parkedBy === undefined));
|
|
67
|
+
if (older.length > 0) {
|
|
68
|
+
const latest = older.reduce((a, b) => (b.run.createdAt > a.run.createdAt || (b.run.createdAt === a.run.createdAt && b.run.id > a.run.id) ? b : a));
|
|
69
|
+
return { kind: "unproven", cause: "older_engine", runId: latest.run.id, run: latest.run };
|
|
70
|
+
}
|
|
71
|
+
return { kind: "none" };
|
|
72
|
+
}
|
|
73
|
+
function highestHolders(sessionId, readable) {
|
|
74
|
+
let top = [];
|
|
75
|
+
let topGeneration = -1;
|
|
76
|
+
for (const { run, parks } of readable) {
|
|
77
|
+
for (const park of heldParksOf(run.id, parks)) {
|
|
78
|
+
if (park.sessionId !== sessionId || park.generation === undefined)
|
|
79
|
+
continue;
|
|
80
|
+
if (park.generation > topGeneration) {
|
|
81
|
+
topGeneration = park.generation;
|
|
82
|
+
top = [{ run, park }];
|
|
83
|
+
}
|
|
84
|
+
else if (park.generation === topGeneration && !top.some((t) => t.run.id === run.id)) {
|
|
85
|
+
top.push({ run, park });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
28
88
|
}
|
|
29
|
-
return
|
|
89
|
+
return top;
|
|
30
90
|
}
|
|
31
91
|
export function sameAgentSessionRows(a, b) {
|
|
32
92
|
if (a.length !== b.length)
|
|
@@ -97,15 +157,21 @@ export class LocalWorkflowAgentSessionIndex {
|
|
|
97
157
|
}
|
|
98
158
|
apply(runId, rows) {
|
|
99
159
|
for (const old of this.byRun.get(runId) ?? []) {
|
|
100
|
-
|
|
160
|
+
const runs = this.bySession.get(old.sessionId);
|
|
161
|
+
runs?.delete(runId);
|
|
162
|
+
if (runs?.size === 0)
|
|
101
163
|
this.bySession.delete(old.sessionId);
|
|
102
164
|
}
|
|
103
165
|
if (rows.length === 0)
|
|
104
166
|
this.byRun.delete(runId);
|
|
105
167
|
else
|
|
106
168
|
this.byRun.set(runId, [...rows]);
|
|
107
|
-
for (const r of rows)
|
|
108
|
-
this.bySession.
|
|
169
|
+
for (const r of rows) {
|
|
170
|
+
let runs = this.bySession.get(r.sessionId);
|
|
171
|
+
if (runs === undefined)
|
|
172
|
+
this.bySession.set(r.sessionId, (runs = new Map()));
|
|
173
|
+
runs.set(runId, r);
|
|
174
|
+
}
|
|
109
175
|
}
|
|
110
176
|
async syncRun(runId, rows) {
|
|
111
177
|
if (this.ledgerPath !== undefined) {
|
|
@@ -114,8 +180,8 @@ export class LocalWorkflowAgentSessionIndex {
|
|
|
114
180
|
}
|
|
115
181
|
this.apply(runId, rows);
|
|
116
182
|
}
|
|
117
|
-
async
|
|
118
|
-
return this.bySession.get(sessionId) ??
|
|
183
|
+
async listByAgentSession(sessionId) {
|
|
184
|
+
return [...(this.bySession.get(sessionId)?.values() ?? [])];
|
|
119
185
|
}
|
|
120
186
|
}
|
|
121
187
|
//# sourceMappingURL=workflow-agent-session-index.js.map
|
|
@@ -177,7 +177,12 @@ export declare class FileWorkflowCompletionInbox extends InMemoryWorkflowComplet
|
|
|
177
177
|
private fd;
|
|
178
178
|
private appendsSinceCompact;
|
|
179
179
|
constructor(root: string, onWarn?: (msg: string, meta: Record<string, unknown>) => void, fsyncEnabled?: boolean);
|
|
180
|
+
/** 重放 = 本仓 JSONL 重放属主(登记座 `workflow-completion-inbox`,fail-open + F tag):每行过 {@link INBOX_RECORD_SHAPE},
|
|
181
|
+
* 读不出形(`null` / 非对象 / 缺 `op` / 缺 id …)⇒ 属主跳过 + 逐条披露,**不抛**;撕尾(崩在一次 append 中途)是崩溃的正常落盘形,
|
|
182
|
+
* 属主不计。好行按序折叠 —— {@link foldRecord} 与运行期各口同一套语义(重放是严格折叠)。 */
|
|
180
183
|
private replay;
|
|
184
|
+
/** 把一条已判过形的记录折进内存。穷尽 switch:加一种记录 ⇒ 编译红,由人处置(形判据与折叠同批改)。 */
|
|
185
|
+
private foldRecord;
|
|
181
186
|
private commit;
|
|
182
187
|
/** Compact iff we've appended enough since the last one. MUST be called AFTER the in-memory mutation the
|
|
183
188
|
* triggering commit records — compact() rewrites from `bySession`, so a compact fired BEFORE the caller
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { mkdirSync,
|
|
1
|
+
import { mkdirSync, openSync, writeSync, fsyncSync, closeSync, renameSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { redactSecrets } from "../trace/redact.js";
|
|
4
|
+
import { replayJsonlSkipping } from "../observability/corrupt-read-seat.js";
|
|
4
5
|
import { injectionTierOrAbsent } from "../trace/injection-tier.js";
|
|
5
6
|
import { semaProvenanceOrAbsent } from "../trace/sema-provenance.js";
|
|
6
7
|
import { resupplyFields, terminalFrameFacetFields } from "../trace/task-notification-facets.js";
|
|
@@ -37,6 +38,28 @@ export function foldKeyFamily(runId) {
|
|
|
37
38
|
return i > 0 && /^\d+$/.test(runId.slice(i + 1)) ? runId.slice(0, i) : runId;
|
|
38
39
|
}
|
|
39
40
|
const servedKey = (sessionId, runId) => JSON.stringify([sessionId, runId]);
|
|
41
|
+
const isLedgerId = (v) => typeof v === "string" && v.length > 0;
|
|
42
|
+
const INBOX_RECORD_SHAPE = {
|
|
43
|
+
what: 'an inbox record {op: "enqueue" (entry with sessionId + runId) | "ack" (sessionId + runId) | "drain" (sessionId)}',
|
|
44
|
+
read(record) {
|
|
45
|
+
const r = record;
|
|
46
|
+
switch (r.op) {
|
|
47
|
+
case "enqueue": {
|
|
48
|
+
const e = r.entry;
|
|
49
|
+
if (typeof e !== "object" || e === null || Array.isArray(e))
|
|
50
|
+
return undefined;
|
|
51
|
+
const { sessionId, runId } = e;
|
|
52
|
+
return isLedgerId(sessionId) && isLedgerId(runId) ? { op: "enqueue", entry: e } : undefined;
|
|
53
|
+
}
|
|
54
|
+
case "ack":
|
|
55
|
+
return isLedgerId(r.sessionId) && isLedgerId(r.runId) ? { op: "ack", sessionId: r.sessionId, runId: r.runId } : undefined;
|
|
56
|
+
case "drain":
|
|
57
|
+
return isLedgerId(r.sessionId) ? { op: "drain", sessionId: r.sessionId } : undefined;
|
|
58
|
+
default:
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
};
|
|
40
63
|
export class InMemoryWorkflowCompletionInbox {
|
|
41
64
|
onWarn;
|
|
42
65
|
bySession = new Map();
|
|
@@ -137,29 +160,28 @@ export class FileWorkflowCompletionInbox extends InMemoryWorkflowCompletionInbox
|
|
|
137
160
|
this.compact();
|
|
138
161
|
}
|
|
139
162
|
replay() {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
let rec;
|
|
147
|
-
try {
|
|
148
|
-
rec = JSON.parse(line);
|
|
149
|
-
}
|
|
150
|
-
catch {
|
|
151
|
-
continue;
|
|
152
|
-
}
|
|
153
|
-
if (rec.op === "enqueue" && rec.entry?.sessionId && rec.entry.runId)
|
|
163
|
+
for (const rec of replayJsonlSkipping(this.ledgerPath, "workflow-completion-inbox", INBOX_RECORD_SHAPE).records)
|
|
164
|
+
this.foldRecord(rec);
|
|
165
|
+
}
|
|
166
|
+
foldRecord(rec) {
|
|
167
|
+
switch (rec.op) {
|
|
168
|
+
case "enqueue":
|
|
154
169
|
this.applyEnqueue(rec.entry, false);
|
|
155
|
-
|
|
170
|
+
return;
|
|
171
|
+
case "ack": {
|
|
156
172
|
const q = this.bySession.get(rec.sessionId);
|
|
157
173
|
q?.delete(rec.runId);
|
|
158
174
|
if (q && q.size === 0)
|
|
159
175
|
this.bySession.delete(rec.sessionId);
|
|
176
|
+
return;
|
|
160
177
|
}
|
|
161
|
-
|
|
178
|
+
case "drain":
|
|
162
179
|
this.bySession.delete(rec.sessionId);
|
|
180
|
+
return;
|
|
181
|
+
default: {
|
|
182
|
+
const unreachable = rec;
|
|
183
|
+
return unreachable;
|
|
184
|
+
}
|
|
163
185
|
}
|
|
164
186
|
}
|
|
165
187
|
commit(rec) {
|
|
@@ -322,7 +322,7 @@ export declare class JournalingWorkflowRunStore implements WorkflowRunStore {
|
|
|
322
322
|
* **自吞错**:索引是观察面的派生物,它的故障绝不能把一次已成功的 run 持久化变成失败 —— 与
|
|
323
323
|
* `publishFleet` 的姿态同源。但自吞 ≠ 静默([ref]):留痕走**登记过的** fail-open tag
|
|
324
324
|
* `server.workflow-park-index.sync-failed`(P-DEBT;射程、后果与欠账的唯一属主 = 那条词表项的
|
|
325
|
-
* `note`,census 里有对应行)。 */
|
|
325
|
+
* `note`,census 里有对应行);记录的 `parks[]` 读不出而不同步的那一臂另走 `server.workflow-park-index.parks-unreadable`(F 类,见下)。 */
|
|
326
326
|
private syncAgentSessionIndex;
|
|
327
327
|
/** Derive + publish the MF-Fleet workflow row from a WorkflowRun (doneCount/totalCount from agents, tokens from
|
|
328
328
|
* stats); a terminal workflow LEAVES the fleet (publish terminal then remove), a running one stays/updates. */
|
|
@@ -2,7 +2,7 @@ import { join } from "node:path";
|
|
|
2
2
|
import { AppendLog, ensureDir } from "@sema-agent/core";
|
|
3
3
|
import { buildFleetWorkflowRow } from "../fleet/fleet-bus.js";
|
|
4
4
|
import { isTerminalWorkflowRowStatus, seedTerminalWorkflowRow } from "../fleet/fleet-terminal-window.js";
|
|
5
|
-
import {
|
|
5
|
+
import { buildParkSessionRows, sameAgentSessionRows } from "./workflow-agent-session-index.js";
|
|
6
6
|
import { recordFailOpen } from "../observability/fail-open.js";
|
|
7
7
|
import { replayJsonlSkipping } from "../observability/corrupt-read-seat.js";
|
|
8
8
|
const SUMMARY_MAX = 500;
|
|
@@ -278,7 +278,12 @@ export class JournalingWorkflowRunStore {
|
|
|
278
278
|
const index = this.agentSessionIndex;
|
|
279
279
|
if (index === undefined)
|
|
280
280
|
return;
|
|
281
|
-
const rows =
|
|
281
|
+
const rows = buildParkSessionRows(id, run);
|
|
282
|
+
if (rows === undefined) {
|
|
283
|
+
this.indexedRows.delete(id);
|
|
284
|
+
recordFailOpen("server.workflow-park-index.parks-unreadable", `runId=${id}`);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
282
287
|
const prior = this.indexedRows.get(id);
|
|
283
288
|
if (prior === undefined || !sameAgentSessionRows(prior, rows)) {
|
|
284
289
|
try {
|
|
@@ -45,14 +45,19 @@
|
|
|
45
45
|
* `CONNECT` 交换)都经它的 `connect`,不多不少。{@link ProxyDialPool} 在这里把两类结局收成一只错误 {@link OutboundProxyError}:
|
|
46
46
|
* · 拨号失败(拒连 / 解析不了 / 超时 / 代理 TLS / 收下就关 …)⇒ 包成 `OutboundProxyError`(`cause` = 原错误);
|
|
47
47
|
* · `CONNECT` 回非 200 ⇒ 关掉那条 socket(undici 自己也是这么做的),同样包成 `OutboundProxyError`(`cause` 说出状态码)。
|
|
48
|
-
* 🔴
|
|
48
|
+
* 🔴 **包装层的 `code` 只有一种来源:代理答了拒**(CONNECT 回非 200 / 绝对 URL 形回 407,唯一铸点 {@link proxyRefusal})⇒ 包装节点带引擎包根的
|
|
49
|
+
* `PROXY_TUNNEL_REFUSED` 与数字 `statusCode`(代理自己说的状态),节点之下只有本文件自己写的无码句子;**拨号失败不铸码**(下面那枚 errno / undici 码当家)。承重的两件事:
|
|
50
|
+
* ① undici 客户端在连接阶段只把 `UND_ERR_INFO` / `UND_ERR_SOCKET` 当可恢复错误重连(7.30.0 `client.js` `onError` 逐字判这两枚码),
|
|
49
51
|
* 而 `CONNECT` 恰在连接阶段 —— 代理收下 TCP 就关(Linux 上是 FIN ⇒ `UND_ERR_SOCKET`)时,裸 `ProxyAgent` / Node 24 `NODE_USE_ENV_PROXY`
|
|
50
|
-
* 都会重连死循环(修前实测 10 s 四五万条连接、请求挂到调用方超时、abort 之后仍在涨)
|
|
51
|
-
* ② core
|
|
52
|
+
* 都会重连死循环(修前实测 10 s 四五万条连接、请求挂到调用方超时、abort 之后仍在涨);包装节点的码不是那两枚(无码,或 `PROXY_TUNNEL_REFUSED`)⇒ 排队的请求当场失败。
|
|
53
|
+
* ② core 的连接失败判官(`readConnectFailure` / `classifyConnectFailure`)只看图上的 `code`:无码的包装节点对它透明(拨号失败的判据不变);带 `PROXY_TUNNEL_REFUSED`
|
|
54
|
+
* 的节点是确定的答案(短阶梯,终局句按 `statusCode` 选)—— 前提是节点之下没有别的带码节点(一枚外来码否决全图),所以拒绝的 `cause` 只能是本文件的无码句子,
|
|
55
|
+
* 不能盖在 undici 自己的拒(`UND_ERR_ABORTED`)之上。
|
|
52
56
|
* 修前读数(装了 dispatcher、未加归因;引擎终局句 = core `describeNetworkError` 原样):代理拒连 `[network] fetch failed (connect ECONNREFUSED
|
|
53
57
|
* 127.0.0.1:1)`;代理主机解析不了 `… (getaddrinfo ENOTFOUND proxy.s776.invalid)`;CONNECT 回 403 `… (Request was cancelled. ← UND_ERR_ABORTED:
|
|
54
58
|
* Proxy response (403) !== 200 when HTTP Tunneling)` —— 都不说「经代理」。修后:`… (via outbound proxy (from HTTPS_PROXY) ←
|
|
55
|
-
*
|
|
59
|
+
* ECONNREFUSED)`(整条链不带代理地址与凭据:包装句只说「经代理 + 哪只变量」,`cause` 链上只有码 —— 见 {@link projectHopFailure}、{@link OutboundProxyError})。core 的句子一字不改(它逐层展开 cause 链,
|
|
60
|
+
* 包装层就是链上多一节、`cause` 那一节换成同形副本)。隧道建好之后的失败(目标的 TLS、
|
|
56
61
|
* 目标的响应)不经这个接缝,不冠代理 —— 那是目标的事。
|
|
57
62
|
*
|
|
58
63
|
* ── http 目标:绝对 URL 形,不建隧道(S-804;test 黑盒验收 A1)──────────────────────────────────────────────────
|
|
@@ -77,7 +82,7 @@
|
|
|
77
82
|
* 代理 URL 的 userinfo 是凭据(undici 把它译成 `Proxy-Authorization: Basic …`)。它只进 dispatcher,不进任何观测面:姿态
|
|
78
83
|
* ({@link buildOutboundProxyPosture})、拒启句与归因句同一把尺 {@link displayProxyUrl}(形干净只出 `scheme//host[:port]`、过全仓那一只
|
|
79
84
|
* display-safe 铸点;形不干净整只占位);姿态只上启动日志一行
|
|
80
|
-
* `outbound_proxy`,不进 `/health`、不进公告、不进租户 wire(代理那一跳失败的终局句只说「经代理 +
|
|
85
|
+
* `outbound_proxy`,不进 `/health`、不进公告、不进租户 wire(代理那一跳失败的终局句只说「经代理 + 哪只变量 + 码」,整句无代理地址与凭据 —— {@link projectHopFailure};原句只进运维日志 `outbound_connect_failed.proxyHop`)。代理变量不可用(undici 构造即拒:无 scheme、`ftp:`、带路径 / 查询)⇒
|
|
81
86
|
* **拒启**(与 Node 24 `NODE_USE_ENV_PROXY=1` 遇坏值的行为同向)—— 静默直连 = 运维以为出网走了代理而实际没有,那是出网策略面的
|
|
82
87
|
* fail-open;拒因只回显 display-safe 值。
|
|
83
88
|
*
|
|
@@ -86,8 +91,8 @@
|
|
|
86
91
|
* 观察点 = undici 的**公开诊断通道**(`undici:request:create` / `:headers` / `:error`,进程级按名订阅,npm undici 与 Node 内建那一份
|
|
87
92
|
* 同名同发):不包 handler、不动 dispatch 的热路径。{@link installOutboundDispatcher} 装一次:
|
|
88
93
|
* ① 每条请求的建连期失败(码在闭集 {@link OUTBOUND_CONNECT_FAILURE_CODES} 里、且这条请求还没收到过响应头)⇒ 一条结构化 warn
|
|
89
|
-
* `outbound_connect_failed { targetHost, code, proxyInUse, proxyReadAtMs, sinceBootMs, hint }`(`proxyInUse` = 这一跳走的代理**变量名**,
|
|
90
|
-
* 不打值;直连 ⇒ `null`)—— 全进程的出网都算(模型、中心、工具 …),不只模型;
|
|
94
|
+
* `outbound_connect_failed { targetHost, code, proxyInUse, proxyHop?, proxyReadAtMs, sinceBootMs, hint }`(`proxyInUse` = 这一跳走的代理**变量名**,
|
|
95
|
+
* 不打值;直连 ⇒ `null`。`proxyHop` = 只在走了代理时在场的原句,含代理 host:port —— 运行时失败里代理地址**唯一**的去处,见 {@link projectHopFailure})—— 全进程的出网都算(模型、中心、工具 …),不只模型;
|
|
91
96
|
* ② 最近一次建连失败 + 生效代理的**名字**与读取时刻 ⇒ `GET /v1/diagnostics/wiring` 的 `outbound` 段({@link readOutboundWiring});
|
|
92
97
|
* ③ 请求若创建在一次**模型调用**的作用域里(`model-reach-scope.ts`),结局记到那次调用上 —— 模型调用终态时由
|
|
93
98
|
* `model-unreachable-notice.ts` 决定要不要给这条腿的用户铸 `server.model_unreachable`(只落日志不算闭环:运营用户看不到日志)。
|
|
@@ -95,6 +100,7 @@
|
|
|
95
100
|
* 目标请求上,那一条才知道「要去哪」。Bun 的原生 fetch 不经 undici ⇒ 通道不发、三件都没有(如实:与回环直连同一条 Bun 例外)。
|
|
96
101
|
*/
|
|
97
102
|
import { Dispatcher } from "undici";
|
|
103
|
+
import { PROXY_TUNNEL_REFUSED } from "@sema-agent/core";
|
|
98
104
|
import type { Logger } from "./observability/logger.js";
|
|
99
105
|
/** 三只变量的读值(boot 冻结;空串 = 未设)。 */
|
|
100
106
|
interface OutboundProxyEnv {
|
|
@@ -124,18 +130,25 @@ export declare function buildOutboundProxyPosture(env?: NodeJS.ProcessEnv): Outb
|
|
|
124
130
|
*/
|
|
125
131
|
export declare function isLoopbackOrigin(origin: string | URL | undefined): boolean;
|
|
126
132
|
/**
|
|
127
|
-
* 代理那一跳失败的**唯一**错误形(`cause` =
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
* core 的 `describeNetworkError` 读的是 `.message`,不受影响、不重复。
|
|
133
|
+
* 代理那一跳失败的**唯一**错误形(`cause` = 原错误经 {@link projectHopFailure} 的 wire 副本)。`proxy` = display-safe 代理地址,`from` = 哪只变量给的。
|
|
134
|
+
* `refusedWith` 在场 = 代理**答了**拒(只由 {@link proxyRefusal} 传)⇒ 本节点带 `code: PROXY_TUNNEL_REFUSED` + 数字 `statusCode`;缺席(拨号失败)⇒ 两键都不在(理由见顶注)。
|
|
135
|
+
* 🔴 **这句经 core 的终局句进租户可见的失败原因,所以整条链上没有代理地址、没有凭据**:message 只点明「经代理 + 哪只变量」,`cause` 链上只有码与本文件自己写的句子。
|
|
136
|
+
* 一条规则、不按姿态分叉;运维要地址 ⇒ 运维日志 `outbound_connect_failed` 的 `proxyHop`(原句)与启动日志 `outbound_proxy` 行(display-safe 代理值)。
|
|
137
|
+
* `proxy` 是**不可枚举**的自有属性(只能按名读;`JSON.stringify` / `util.inspect` 这类枚举自有属性的口读不到),原句只经 {@link OutboundProxyError.hopDetail} 取。
|
|
138
|
+
* 字符串化(`String(err)`)带出根因:只打第一层 cause 的日志腿(如 hook 的 `String(cause)`)也读得到 `ECONNREFUSED` 这类码;core 的 `describeNetworkError` 读的是 `.message`,不受影响、不重复。
|
|
133
139
|
*/
|
|
134
140
|
export declare class OutboundProxyError extends Error {
|
|
135
|
-
|
|
141
|
+
#private;
|
|
136
142
|
readonly from: string;
|
|
137
143
|
readonly name = "OutboundProxyError";
|
|
138
|
-
|
|
144
|
+
readonly proxy: string;
|
|
145
|
+
/** 代理答了拒 ⇒ 引擎的确定性连接码;拨号失败 ⇒ 不在场(`declare`:不铸 `undefined` 自有键,`"code" in` 与引擎的读法都读成缺席)。 */
|
|
146
|
+
readonly code?: typeof PROXY_TUNNEL_REFUSED;
|
|
147
|
+
/** 代理自己说的 HTTP 状态(与 `code` 同生同灭)。 */
|
|
148
|
+
readonly statusCode?: number;
|
|
149
|
+
constructor(proxy: string, from: string, cause: unknown, refusedWith?: number);
|
|
150
|
+
/** 代理那一跳的原句(含代理 host:port;凭据位已过脱敏器)。**只给运维日志**:不进 message、不进 `cause`、不进任何自有属性。 */
|
|
151
|
+
get hopDetail(): string;
|
|
139
152
|
toString(): string;
|
|
140
153
|
}
|
|
141
154
|
/** 代理变量不可用的拒启码(拼法随 `config.*` 家族,出现在拒句方括号里)。 */
|
|
@@ -170,11 +183,7 @@ export interface OutboundConnectFailure {
|
|
|
170
183
|
readonly code: OutboundConnectFailureCode;
|
|
171
184
|
readonly proxyInUse: string | null;
|
|
172
185
|
}
|
|
173
|
-
/**
|
|
174
|
-
* 一只请求错误是不是**建连期失败**:在错误图里找第一枚闭集码;路上遇到 {@link OutboundProxyError} ⇒ 这一跳走了代理,
|
|
175
|
-
* 记下它的变量名。找不到闭集码 ⇒ `undefined`(不是建连失败:TLS、中途断、取消、超时、代理 CONNECT 非 200 …)。
|
|
176
|
-
* 纯函数、不抛(只读 `code` / `cause` / `errors` 三个自有数据位)。
|
|
177
|
-
*/
|
|
186
|
+
/** 一只请求错误的建连期失败码与它走的代理变量名(`undefined` = 不是建连失败)。原句不在这里 —— 它只给运维日志。 */
|
|
178
187
|
export declare function classifyOutboundConnectFailure(error: unknown): {
|
|
179
188
|
readonly code: OutboundConnectFailureCode;
|
|
180
189
|
readonly proxyInUse: string | null;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Agent, Client, Dispatcher, EnvHttpProxyAgent, getGlobalDispatcher, Pool, setGlobalDispatcher } from "undici";
|
|
2
|
+
import { PROXY_TUNNEL_REFUSED } from "@sema-agent/core";
|
|
2
3
|
import { isIP } from "node:net";
|
|
3
4
|
import diagnosticsChannel from "node:diagnostics_channel";
|
|
4
|
-
import { buildDisplaySafeUrl, DISPLAY_SAFE_URL_TOKEN } from "./trace/redact.js";
|
|
5
|
+
import { buildDisplaySafeUrl, DISPLAY_SAFE_URL_TOKEN, redactSecrets } from "./trace/redact.js";
|
|
5
6
|
import { recordFailOpen } from "./observability/fail-open.js";
|
|
6
7
|
import { currentModelCallReach } from "./model-reach-scope.js";
|
|
7
8
|
function readOutboundProxyEnv(env = process.env) {
|
|
@@ -80,14 +81,95 @@ class OutboundDispatcher extends Dispatcher {
|
|
|
80
81
|
void done.then(cb, cb);
|
|
81
82
|
}
|
|
82
83
|
}
|
|
84
|
+
class ProxyHopNote extends Error {
|
|
85
|
+
}
|
|
86
|
+
const UNCLASSIFIED_HOP_FAILURE = "proxy hop failed (no error code)";
|
|
87
|
+
const CONNECT_ERROR_GRAPH_BUDGET = 16;
|
|
88
|
+
const HOP_CODE_SHAPE = /^[A-Za-z][A-Za-z0-9_]{0,63}$/;
|
|
89
|
+
const UNREADABLE_HOP_CODE = "HOP_CODE_UNREADABLE";
|
|
90
|
+
const HOP_DETAIL_CAP = 500;
|
|
91
|
+
const readMember = (value, key) => {
|
|
92
|
+
try {
|
|
93
|
+
return value === null || value === undefined ? undefined : value[key];
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
recordFailOpen("server.outbound.hop-member-unreadable");
|
|
97
|
+
return undefined;
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
const isErrorLike = (value) => {
|
|
101
|
+
try {
|
|
102
|
+
return value instanceof Error || Object.prototype.toString.call(value) === "[object Error]";
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
recordFailOpen("server.outbound.hop-member-unreadable");
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
function projectHopFailure(raw) {
|
|
110
|
+
const copies = new Map();
|
|
111
|
+
const frontier = [];
|
|
112
|
+
const messages = [];
|
|
113
|
+
let created = 0;
|
|
114
|
+
const make = (source) => {
|
|
115
|
+
created++;
|
|
116
|
+
const text = typeof source === "string" ? source : readMember(source, "message");
|
|
117
|
+
if (typeof text === "string" && text !== "" && !messages.includes(text))
|
|
118
|
+
messages.push(text);
|
|
119
|
+
const rawCode = readMember(source, "code");
|
|
120
|
+
const code = typeof rawCode !== "string" ? undefined : HOP_CODE_SHAPE.test(rawCode) ? rawCode : UNREADABLE_HOP_CODE;
|
|
121
|
+
const message = code ?? (source instanceof ProxyHopNote ? source.message : UNCLASSIFIED_HOP_FAILURE);
|
|
122
|
+
const wire = source instanceof AggregateError ? new AggregateError([], message) : new Error(message);
|
|
123
|
+
if (code !== undefined)
|
|
124
|
+
Object.assign(wire, { code });
|
|
125
|
+
if (typeof source === "object" && source !== null) {
|
|
126
|
+
copies.set(source, wire);
|
|
127
|
+
frontier.push([source, wire]);
|
|
128
|
+
}
|
|
129
|
+
return wire;
|
|
130
|
+
};
|
|
131
|
+
const copy = (source) => {
|
|
132
|
+
const known = typeof source === "object" && source !== null ? copies.get(source) : undefined;
|
|
133
|
+
return known ?? (created < CONNECT_ERROR_GRAPH_BUDGET ? make(source) : undefined);
|
|
134
|
+
};
|
|
135
|
+
const place = (target, key, value) => void Object.defineProperty(target, key, { value, writable: true, configurable: true, enumerable: false });
|
|
136
|
+
const wire = make(raw);
|
|
137
|
+
for (let next = frontier.shift(); next !== undefined; next = frontier.shift()) {
|
|
138
|
+
const [source, target] = next;
|
|
139
|
+
if (!isErrorLike(source))
|
|
140
|
+
continue;
|
|
141
|
+
const cause = readMember(source, "cause");
|
|
142
|
+
const below = cause === undefined || cause === null ? undefined : copy(cause);
|
|
143
|
+
if (below !== undefined)
|
|
144
|
+
place(target, "cause", below);
|
|
145
|
+
const members = readMember(source, "errors");
|
|
146
|
+
if (members === undefined)
|
|
147
|
+
continue;
|
|
148
|
+
const length = readMember(members, "length");
|
|
149
|
+
if (typeof length !== "number" || !Number.isInteger(length) || length <= 0) {
|
|
150
|
+
place(target, "errors", null);
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
const legs = Array.from({ length: Math.min(length, CONNECT_ERROR_GRAPH_BUDGET) }, (_, i) => copy(readMember(members, String(i)))).filter((leg) => leg !== undefined);
|
|
154
|
+
place(target, "errors", Array.isArray(members) ? legs : Object.assign({ length: legs.length }, Object.fromEntries(legs.map((leg, i) => [String(i), leg]))));
|
|
155
|
+
}
|
|
156
|
+
return { wire, detail: redactSecrets(messages.join(" ← ")).slice(0, HOP_DETAIL_CAP) };
|
|
157
|
+
}
|
|
83
158
|
export class OutboundProxyError extends Error {
|
|
84
|
-
proxy;
|
|
85
159
|
from;
|
|
86
160
|
name = "OutboundProxyError";
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
161
|
+
#hopDetail;
|
|
162
|
+
constructor(proxy, from, cause, refusedWith) {
|
|
163
|
+
const { wire, detail } = projectHopFailure(cause);
|
|
164
|
+
super(`via outbound proxy (from ${from})`, { cause: wire });
|
|
90
165
|
this.from = from;
|
|
166
|
+
Object.defineProperty(this, "proxy", { value: proxy, enumerable: false });
|
|
167
|
+
if (refusedWith !== undefined)
|
|
168
|
+
Object.assign(this, { code: PROXY_TUNNEL_REFUSED, statusCode: refusedWith });
|
|
169
|
+
this.#hopDetail = detail;
|
|
170
|
+
}
|
|
171
|
+
get hopDetail() {
|
|
172
|
+
return this.#hopDetail;
|
|
91
173
|
}
|
|
92
174
|
toString() {
|
|
93
175
|
let root = this.cause;
|
|
@@ -132,7 +214,7 @@ class ProxyDialPool extends Pool {
|
|
|
132
214
|
}
|
|
133
215
|
}
|
|
134
216
|
function proxyRefusal(shown, from, statusCode, leg) {
|
|
135
|
-
return new OutboundProxyError(shown, from, new
|
|
217
|
+
return new OutboundProxyError(shown, from, new ProxyHopNote(`proxy answered ${leg} with HTTP ${String(statusCode)}`), statusCode);
|
|
136
218
|
}
|
|
137
219
|
function withDialDeadline(options) {
|
|
138
220
|
const inner = Reflect.get(options, "connect");
|
|
@@ -144,13 +226,13 @@ function withDialDeadline(options) {
|
|
|
144
226
|
opts,
|
|
145
227
|
(err, socket) => {
|
|
146
228
|
if (err === null && socket !== null) {
|
|
147
|
-
const expire = () => void socket.destroy(new
|
|
229
|
+
const expire = () => void socket.destroy(new ProxyHopNote(`proxy did not answer CONNECT within ${String(PROXY_CONNECT_DEADLINE_MS)} ms`));
|
|
148
230
|
dialDeadlines.set(socket, setTimeout(expire, Math.max(0, expiresAt - performance.now())));
|
|
149
231
|
socket.once("close", () => clearDialDeadline(socket));
|
|
150
232
|
callback(null, socket);
|
|
151
233
|
}
|
|
152
234
|
else
|
|
153
|
-
callback(err ?? new
|
|
235
|
+
callback(err ?? new ProxyHopNote("proxy connector returned neither a socket nor an error"), null);
|
|
154
236
|
},
|
|
155
237
|
]);
|
|
156
238
|
};
|
|
@@ -231,7 +313,7 @@ function withProxyAttribution(options, shown, from) {
|
|
|
231
313
|
else if (socket !== null)
|
|
232
314
|
callback(null, socket);
|
|
233
315
|
else
|
|
234
|
-
callback(new OutboundProxyError(shown, from, new
|
|
316
|
+
callback(new OutboundProxyError(shown, from, new ProxyHopNote("proxy connector returned neither a socket nor an error")), null);
|
|
235
317
|
},
|
|
236
318
|
]);
|
|
237
319
|
};
|
|
@@ -271,19 +353,21 @@ export const OUTBOUND_CONNECT_FAILURE_CODES = {
|
|
|
271
353
|
ENETUNREACH: "the network is unreachable",
|
|
272
354
|
};
|
|
273
355
|
const isConnectFailureCode = (c) => typeof c === "string" && Object.hasOwn(OUTBOUND_CONNECT_FAILURE_CODES, c);
|
|
274
|
-
|
|
275
|
-
export function classifyOutboundConnectFailure(error) {
|
|
356
|
+
function scanConnectFailure(error) {
|
|
276
357
|
const queue = [error];
|
|
277
358
|
let proxyInUse = null;
|
|
359
|
+
let proxyHop = null;
|
|
278
360
|
for (let seen = 0; queue.length > 0 && seen < CONNECT_ERROR_GRAPH_BUDGET; seen++) {
|
|
279
361
|
const node = queue.shift();
|
|
280
362
|
if (typeof node !== "object" || node === null)
|
|
281
363
|
continue;
|
|
282
|
-
if (node instanceof OutboundProxyError)
|
|
364
|
+
if (node instanceof OutboundProxyError) {
|
|
283
365
|
proxyInUse = node.from;
|
|
366
|
+
proxyHop = node.hopDetail;
|
|
367
|
+
}
|
|
284
368
|
const code = Object.getOwnPropertyDescriptor(node, "code")?.value;
|
|
285
369
|
if (isConnectFailureCode(code))
|
|
286
|
-
return { code, proxyInUse };
|
|
370
|
+
return { code, proxyInUse, proxyHop };
|
|
287
371
|
const cause = Object.getOwnPropertyDescriptor(node, "cause")?.value;
|
|
288
372
|
if (cause !== undefined)
|
|
289
373
|
queue.push(cause);
|
|
@@ -293,6 +377,10 @@ export function classifyOutboundConnectFailure(error) {
|
|
|
293
377
|
}
|
|
294
378
|
return undefined;
|
|
295
379
|
}
|
|
380
|
+
export function classifyOutboundConnectFailure(error) {
|
|
381
|
+
const found = scanConnectFailure(error);
|
|
382
|
+
return found === undefined ? undefined : { code: found.code, proxyInUse: found.proxyInUse };
|
|
383
|
+
}
|
|
296
384
|
function targetHostOf(request) {
|
|
297
385
|
const path = Reflect.get(request, "path");
|
|
298
386
|
const origin = Reflect.get(request, "origin");
|
|
@@ -343,16 +431,17 @@ function startConnectObserver(logger, nowMs) {
|
|
|
343
431
|
if (request === undefined || Reflect.get(request, "method") === "CONNECT")
|
|
344
432
|
return;
|
|
345
433
|
const call = requestCalls.get(request);
|
|
346
|
-
const
|
|
347
|
-
if (
|
|
434
|
+
const found = answeredRequests.has(request) ? undefined : scanConnectFailure(Reflect.get(message, "error"));
|
|
435
|
+
if (found === undefined) {
|
|
348
436
|
if (call !== undefined)
|
|
349
437
|
call.last = { kind: "other" };
|
|
350
438
|
return;
|
|
351
439
|
}
|
|
352
|
-
const failure = { targetHost: targetHostOf(request),
|
|
440
|
+
const failure = { targetHost: targetHostOf(request), code: found.code, proxyInUse: found.proxyInUse };
|
|
353
441
|
current.last = { ...failure, atMs: Date.now() };
|
|
354
442
|
logger.warn("outbound_connect_failed", {
|
|
355
443
|
...failure,
|
|
444
|
+
...(found.proxyHop !== null ? { proxyHop: found.proxyHop } : {}),
|
|
356
445
|
proxyReadAtMs: current.proxyReadAtMs,
|
|
357
446
|
sinceBootMs: Math.round(process.uptime() * 1000),
|
|
358
447
|
hint: `${OUTBOUND_CONNECT_FAILURE_CODES[failure.code]} — ${failure.proxyInUse !== null ? `this hop went through the proxy from ${failure.proxyInUse} (read once at boot)` : "this hop was a direct connection"}; if the VPN / proxy changed since boot, restart the server so it re-reads HTTP(S)_PROXY / NO_PROXY`,
|