oasis_test_v2 2.2.72 → 2.2.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1234 -49
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -845,6 +845,30 @@ var init_runtime = __esm({
|
|
|
845
845
|
}
|
|
846
846
|
});
|
|
847
847
|
|
|
848
|
+
// ../contract/src/runtime-allocation.ts
|
|
849
|
+
var RUNTIME_FAULT_RECOVERY, RUNTIME_FAULT_PROBE_INITIAL_MS, RUNTIME_FAULT_PROBE_MAX_INTERVAL_MS, RUNTIME_FAULT_PROBE_MAX_TOTAL_MS, RUNTIME_QUOTA_DERATE_THRESHOLD_PERCENT, RUNTIME_ALLOCATION_MAX_ATTEMPTS;
|
|
850
|
+
var init_runtime_allocation = __esm({
|
|
851
|
+
"../contract/src/runtime-allocation.ts"() {
|
|
852
|
+
"use strict";
|
|
853
|
+
RUNTIME_FAULT_RECOVERY = {
|
|
854
|
+
"quota-exhausted": "readonly-probe",
|
|
855
|
+
// 登录失效直接走人工:恢复需要人在那台机器上重新登录,放一条真会话过去试只是又坏一件活。
|
|
856
|
+
unauthenticated: "manual",
|
|
857
|
+
"node-offline": "connection",
|
|
858
|
+
"runtime-missing": "connection",
|
|
859
|
+
"startup-failed": "half-open",
|
|
860
|
+
crashed: "half-open",
|
|
861
|
+
timeout: "half-open",
|
|
862
|
+
"node-error": "half-open"
|
|
863
|
+
};
|
|
864
|
+
RUNTIME_FAULT_PROBE_INITIAL_MS = 6e4;
|
|
865
|
+
RUNTIME_FAULT_PROBE_MAX_INTERVAL_MS = 30 * 6e4;
|
|
866
|
+
RUNTIME_FAULT_PROBE_MAX_TOTAL_MS = 6 * 60 * 6e4;
|
|
867
|
+
RUNTIME_QUOTA_DERATE_THRESHOLD_PERCENT = 60;
|
|
868
|
+
RUNTIME_ALLOCATION_MAX_ATTEMPTS = 3;
|
|
869
|
+
}
|
|
870
|
+
});
|
|
871
|
+
|
|
848
872
|
// ../contract/src/handle-passthrough.ts
|
|
849
873
|
function makePassthroughProxy(spec) {
|
|
850
874
|
const overrides = spec.overrides ?? {};
|
|
@@ -2260,6 +2284,7 @@ var init_src = __esm({
|
|
|
2260
2284
|
init_storage();
|
|
2261
2285
|
init_avatar();
|
|
2262
2286
|
init_runtime();
|
|
2287
|
+
init_runtime_allocation();
|
|
2263
2288
|
init_handle_passthrough();
|
|
2264
2289
|
init_parse_extra_args();
|
|
2265
2290
|
init_daemon_package();
|
|
@@ -136770,7 +136795,7 @@ var init_bus = __esm({
|
|
|
136770
136795
|
notify: (ids2, payload2) => this.io.notify(ids2, payload2)
|
|
136771
136796
|
};
|
|
136772
136797
|
let ok3 = true;
|
|
136773
|
-
let
|
|
136798
|
+
let failureReason2 = null;
|
|
136774
136799
|
for (const h of handlers) {
|
|
136775
136800
|
const effectKey = `${h.name}#effect`;
|
|
136776
136801
|
if (record9.appliedAt && await this.store.transaction((tx) => tx.isHandlerDone(record9.seq, effectKey))) {
|
|
@@ -136793,7 +136818,7 @@ var init_bus = __esm({
|
|
|
136793
136818
|
await this.store.transaction((tx) => tx.deferEvent(record9.seq, "applied", lastErr, this.retryAt()));
|
|
136794
136819
|
return false;
|
|
136795
136820
|
}
|
|
136796
|
-
|
|
136821
|
+
failureReason2 = lastErr;
|
|
136797
136822
|
} else {
|
|
136798
136823
|
await this.store.transaction(async (tx) => {
|
|
136799
136824
|
if (buffered.length) await tx.appendEvents(buffered.map((b2) => ({
|
|
@@ -136812,7 +136837,7 @@ var init_bus = __esm({
|
|
|
136812
136837
|
});
|
|
136813
136838
|
}
|
|
136814
136839
|
}
|
|
136815
|
-
await this.store.transaction((tx) => tx.markEvent(record9.seq, "done",
|
|
136840
|
+
await this.store.transaction((tx) => tx.markEvent(record9.seq, "done", failureReason2, this.clock.now()));
|
|
136816
136841
|
return ok3;
|
|
136817
136842
|
}
|
|
136818
136843
|
/* ═══════════════════════ scan:唯一的调度入口 ═══════════════════════ */
|
|
@@ -142340,7 +142365,10 @@ var init_dispatcher = __esm({
|
|
|
142340
142365
|
* 画板只服务于「产界面」,评审/回复/收口回到能真正读写、能投票的通用运行时。
|
|
142341
142366
|
*/
|
|
142342
142367
|
async resolveBindingFor(spec) {
|
|
142343
|
-
const binding = await this.opts.resolveBinding?.(spec.actor
|
|
142368
|
+
const binding = await this.opts.resolveBinding?.(spec.actor, {
|
|
142369
|
+
...spec.artifactId ? { unitOfWork: spec.artifactId } : {},
|
|
142370
|
+
action: spec.action
|
|
142371
|
+
}) ?? void 0;
|
|
142344
142372
|
if (binding && spec.action !== "produce" && binding.runtimeKind === CANVAS_RUNTIME_KIND) {
|
|
142345
142373
|
return { ...binding, runtimeKind: NON_PRODUCE_FALLBACK_RUNTIME_KIND };
|
|
142346
142374
|
}
|
|
@@ -165172,9 +165200,24 @@ var init_live_chat = __esm({
|
|
|
165172
165200
|
}
|
|
165173
165201
|
if (turn.status === "running") {
|
|
165174
165202
|
turn.cancelCause = "user";
|
|
165175
|
-
|
|
165176
|
-
|
|
165177
|
-
|
|
165203
|
+
const sendKill = turn.handle.sendKill;
|
|
165204
|
+
if (typeof sendKill === "function") {
|
|
165205
|
+
let sent = false;
|
|
165206
|
+
try {
|
|
165207
|
+
sent = sendKill.call(turn.handle);
|
|
165208
|
+
} catch {
|
|
165209
|
+
}
|
|
165210
|
+
if (!sent) {
|
|
165211
|
+
try {
|
|
165212
|
+
turn.handle.cancelLocally?.();
|
|
165213
|
+
} catch {
|
|
165214
|
+
}
|
|
165215
|
+
}
|
|
165216
|
+
} else {
|
|
165217
|
+
try {
|
|
165218
|
+
turn.handle.kill();
|
|
165219
|
+
} catch {
|
|
165220
|
+
}
|
|
165178
165221
|
}
|
|
165179
165222
|
}
|
|
165180
165223
|
return { stopped: true };
|
|
@@ -202018,11 +202061,18 @@ var init_service3 = __esm({
|
|
|
202018
202061
|
const actors = await this.options.resolveActors(companyId);
|
|
202019
202062
|
const bindings = await actors.listBindings();
|
|
202020
202063
|
const reachability = async (actorId) => {
|
|
202021
|
-
const
|
|
202022
|
-
|
|
202023
|
-
|
|
202024
|
-
|
|
202025
|
-
|
|
202064
|
+
const primaryReason = await (async () => {
|
|
202065
|
+
const binding = bindings.find((b2) => b2.actorId === actorId && b2.status === "active");
|
|
202066
|
+
if (!binding) return "\u6CA1\u6709\u63A5\u8FD0\u884C\u65F6";
|
|
202067
|
+
if (!this.options.probeRuntime) return null;
|
|
202068
|
+
const probed = await this.options.probeRuntime({ actorId, nodeId: binding.nodeId, runtimeKind: binding.runtimeKind }).catch(() => ({ reachable: true }));
|
|
202069
|
+
return probed.reachable ? null : probed.reason ?? "\u8FD0\u884C\u65F6\u4E0D\u53EF\u8FBE";
|
|
202070
|
+
})();
|
|
202071
|
+
if (primaryReason === null) return { ok: true };
|
|
202072
|
+
if (!this.options.probeAllocation) return { ok: false, reason: primaryReason };
|
|
202073
|
+
const alt = await this.options.probeAllocation({ actorId, ...companyId ? { companyId } : {} }).catch(() => ({ reachable: true }));
|
|
202074
|
+
if (alt.reachable) return { ok: true };
|
|
202075
|
+
return { ok: false, reason: alt.reason ? `${alt.reason}\uFF08\u4E3B\u8FD0\u884C\u65F6\uFF1A${primaryReason}\uFF09` : primaryReason };
|
|
202026
202076
|
};
|
|
202027
202077
|
if (target.startsWith("actor:")) {
|
|
202028
202078
|
const actor = await actors.getActor(target);
|
|
@@ -203985,6 +204035,7 @@ function assembleDelegationsDomain(deps) {
|
|
|
203985
204035
|
...input.binaryFiles ? { binaryFiles: input.binaryFiles } : {}
|
|
203986
204036
|
}),
|
|
203987
204037
|
...deps.probeRuntime ? { probeRuntime: deps.probeRuntime } : {},
|
|
204038
|
+
...deps.probeAllocation ? { probeAllocation: deps.probeAllocation } : {},
|
|
203988
204039
|
// 轮次账本 = 「这一次派发到底有没有发生过」的权威答案(见 service 的 resolveChatTurns 注释)。
|
|
203989
204040
|
...deps.resolveChatTurns ? { resolveChatTurns: deps.resolveChatTurns } : {},
|
|
203990
204041
|
...deps.enabled ? { enabled: deps.enabled } : {},
|
|
@@ -210344,6 +210395,7 @@ async function startOasisServer(opts) {
|
|
|
210344
210395
|
// 按公司解析;单租户装配退回那唯一一个。两者都没有 → 域层自动关掉租约回收。
|
|
210345
210396
|
resolveChatTurns: async (companyId) => (opts.resolveChatTurns ? await opts.resolveChatTurns(companyId) : void 0) ?? opts.chatTurns,
|
|
210346
210397
|
...opts.delegationProbeRuntime ? { probeRuntime: opts.delegationProbeRuntime } : {},
|
|
210398
|
+
...opts.delegationProbeAllocation ? { probeAllocation: opts.delegationProbeAllocation } : {},
|
|
210347
210399
|
...opts.chatDelegationEnabled ? { enabled: opts.chatDelegationEnabled } : {},
|
|
210348
210400
|
...opts.delegationWorkdir ? { workdir: opts.delegationWorkdir } : {},
|
|
210349
210401
|
...opts.delegationMetrics ? { metrics: opts.delegationMetrics } : {}
|
|
@@ -213451,6 +213503,9 @@ ${composed}`;
|
|
|
213451
213503
|
kill: () => {
|
|
213452
213504
|
void session.kill?.();
|
|
213453
213505
|
},
|
|
213506
|
+
// Bug 0232:把「帧送没送出去 / 就地收尾」这对出口透到注册表,`/stop` 才判得了。
|
|
213507
|
+
// 条件透传不是省事:上层按「方法在不在」分流,本地 adapter 没有这条就该没有。
|
|
213508
|
+
...typeof session.sendKill === "function" ? { sendKill: () => session.sendKill(), cancelLocally: () => session.cancelLocally?.() ?? false } : {},
|
|
213454
213509
|
// ADR-0093 chat 道:把插入能力透到注册表,POST …/append 才有得调。
|
|
213455
213510
|
...typeof session.appendInput === "function" ? { appendInput: (input) => session.appendInput(input) } : {},
|
|
213456
213511
|
get canAppendInput() {
|
|
@@ -215381,6 +215436,7 @@ var init_memory_registry_store = __esm({
|
|
|
215381
215436
|
agentPrefs = /* @__PURE__ */ new Map();
|
|
215382
215437
|
teams = /* @__PURE__ */ new Map();
|
|
215383
215438
|
bindings = /* @__PURE__ */ new Map();
|
|
215439
|
+
runtimeCandidates = /* @__PURE__ */ new Map();
|
|
215384
215440
|
runtimeConfigs = /* @__PURE__ */ new Map();
|
|
215385
215441
|
runtimeModelOverrides = /* @__PURE__ */ new Map();
|
|
215386
215442
|
// key: ${runtimeId}::${modelId}
|
|
@@ -215482,6 +215538,28 @@ var init_memory_registry_store = __esm({
|
|
|
215482
215538
|
async listBindings() {
|
|
215483
215539
|
return [...this.bindings.values()].map((b2) => ({ ...b2 }));
|
|
215484
215540
|
}
|
|
215541
|
+
/* 候选 runtime(ADR 0616):与 PG 实现同一套语义——按 ord 升序读、整份替换、同 (node,kind) 去重。 */
|
|
215542
|
+
async listRuntimeCandidates(actorId) {
|
|
215543
|
+
return (this.runtimeCandidates.get(actorId) ?? []).map((c) => ({ ...c }));
|
|
215544
|
+
}
|
|
215545
|
+
async replaceRuntimeCandidates(actorId, candidates) {
|
|
215546
|
+
const seen = /* @__PURE__ */ new Set();
|
|
215547
|
+
const rows = [];
|
|
215548
|
+
for (const c of candidates) {
|
|
215549
|
+
const key2 = `${c.nodeId}::${c.runtimeKind}`;
|
|
215550
|
+
const model = typeof c.model === "string" && c.model.trim() ? c.model.trim() : null;
|
|
215551
|
+
const existing = rows.findIndex((r) => `${r.nodeId}::${r.runtimeKind}` === key2);
|
|
215552
|
+
if (existing >= 0) {
|
|
215553
|
+
rows[existing] = { ...rows[existing], model };
|
|
215554
|
+
continue;
|
|
215555
|
+
}
|
|
215556
|
+
seen.add(key2);
|
|
215557
|
+
rows.push({ actorId, nodeId: c.nodeId, runtimeKind: c.runtimeKind, model, ord: rows.length });
|
|
215558
|
+
}
|
|
215559
|
+
if (rows.length === 0) this.runtimeCandidates.delete(actorId);
|
|
215560
|
+
else this.runtimeCandidates.set(actorId, rows);
|
|
215561
|
+
return rows.map((c) => ({ ...c }));
|
|
215562
|
+
}
|
|
215485
215563
|
async upsertRuntimeConfig(c) {
|
|
215486
215564
|
this.runtimeConfigs.set(c.runtimeId, { ...c });
|
|
215487
215565
|
}
|
|
@@ -222640,6 +222718,20 @@ var init_company_scoped_registry = __esm({
|
|
|
222640
222718
|
async getBinding(actorId) {
|
|
222641
222719
|
return await this.assertScoped(actorId, "getBinding") ? this.base.getBinding(actorId) : null;
|
|
222642
222720
|
}
|
|
222721
|
+
/* 候选 runtime(ADR 0616):与 binding 同一道租户闸——越界读回空、越界写抛。
|
|
222722
|
+
|
|
222723
|
+
刻意**不**按 `this.base` 有没有这两个方法来决定自己有没有(那样得写成类字段,而类字段的初始化
|
|
222724
|
+
早于构造参数属性赋值,`this.base` 在那一刻还是 undefined)。底座没实现时:读回空、写抛一句
|
|
222725
|
+
指名道姓的错——比静默丢掉一份候选清单强。 */
|
|
222726
|
+
async listRuntimeCandidates(actorId) {
|
|
222727
|
+
if (!this.base.listRuntimeCandidates) return [];
|
|
222728
|
+
return await this.assertScoped(actorId, "listRuntimeCandidates") ? this.base.listRuntimeCandidates(actorId) : [];
|
|
222729
|
+
}
|
|
222730
|
+
async replaceRuntimeCandidates(actorId, candidates) {
|
|
222731
|
+
await this.requireScoped(actorId, "replaceRuntimeCandidates");
|
|
222732
|
+
if (!this.base.replaceRuntimeCandidates) throw new Error("\u5F53\u524D\u767B\u8BB0\u5B58\u50A8\u5B9E\u73B0\u4E0D\u652F\u6301\u5019\u9009\u8FD0\u884C\u65F6\uFF08replaceRuntimeCandidates \u672A\u5B9E\u73B0\uFF09");
|
|
222733
|
+
return this.base.replaceRuntimeCandidates(actorId, candidates);
|
|
222734
|
+
}
|
|
222643
222735
|
async listBindings() {
|
|
222644
222736
|
const [all2, ids2] = await Promise.all([this.base.listBindings(), this.scopeIds()]);
|
|
222645
222737
|
const actors = [];
|
|
@@ -225804,6 +225896,8 @@ function wireRecoveredChatTurn(deps) {
|
|
|
225804
225896
|
kill: () => {
|
|
225805
225897
|
void handle.kill();
|
|
225806
225898
|
},
|
|
225899
|
+
// Bug 0232:重挂的轮一样会被人点停止、一样可能撞上节点够不着——同一对出口原样透传。
|
|
225900
|
+
...typeof handle.sendKill === "function" ? { sendKill: () => handle.sendKill(), cancelLocally: () => handle.cancelLocally?.() ?? false } : {},
|
|
225807
225901
|
// ADR-0093 chat 道:重挂的轮同样能被插入——否则「serve 重启过」会变成一条人看不见的能力差异。
|
|
225808
225902
|
// **整份透传**(2026-09-17):此前只转 `text`,插话链路要的消费回执(`requireConsumptionReceipt`)、
|
|
225809
225903
|
// 幂等键(`clientMessageKey`)与附件(`files` / `binaryFiles`)全被剥掉——runtime 照样读到了正文,
|
|
@@ -228400,6 +228494,32 @@ var init_service5 = __esm({
|
|
|
228400
228494
|
listBindings() {
|
|
228401
228495
|
return this.opts.store.listBindings();
|
|
228402
228496
|
}
|
|
228497
|
+
/* ---------- 候选 runtime(ADR 0616) ---------- */
|
|
228498
|
+
/** 按 `ord` 升序。底座没实现(旧存储实现)→ 空清单,等于「只有主绑定」。 */
|
|
228499
|
+
listRuntimeCandidates(actorId) {
|
|
228500
|
+
return this.opts.store.listRuntimeCandidates?.(actorId) ?? Promise.resolve([]);
|
|
228501
|
+
}
|
|
228502
|
+
/**
|
|
228503
|
+
* 整份替换候选清单(加 / 删 / 排序在界面上是同一个动作)。返回写后的整份。
|
|
228504
|
+
*
|
|
228505
|
+
* 校验放在这里而不是路由:候选是「派到哪」的事实源,写进去的 `(nodeId, runtimeKind)`
|
|
228506
|
+
* 必须真的存在于这家公司的名册上——否则分配那一侧只会在几个月后的某次主机故障里,
|
|
228507
|
+
* 报一句「节点不存在」然后把活挂住,而错是在写入那一刻犯的。
|
|
228508
|
+
*/
|
|
228509
|
+
async replaceRuntimeCandidates(actorId, candidates, by) {
|
|
228510
|
+
if (!this.opts.store.replaceRuntimeCandidates) {
|
|
228511
|
+
throw new Error("\u5F53\u524D\u767B\u8BB0\u5B58\u50A8\u5B9E\u73B0\u4E0D\u652F\u6301\u5019\u9009\u8FD0\u884C\u65F6");
|
|
228512
|
+
}
|
|
228513
|
+
const saved = await this.opts.store.replaceRuntimeCandidates(actorId, candidates);
|
|
228514
|
+
await this.audit({
|
|
228515
|
+
kind: "binding_change",
|
|
228516
|
+
actorId,
|
|
228517
|
+
by,
|
|
228518
|
+
at: this.now(),
|
|
228519
|
+
detail: { candidates: saved.map((c) => ({ nodeId: c.nodeId, runtimeKind: c.runtimeKind, model: c.model ?? null })) }
|
|
228520
|
+
});
|
|
228521
|
+
return saved;
|
|
228522
|
+
}
|
|
228403
228523
|
/** 派单接缝的真实现:替换第零步假实现,第三部分代码零改动(决策 0001) */
|
|
228404
228524
|
resolveBinding() {
|
|
228405
228525
|
return async (actorId) => {
|
|
@@ -230849,6 +230969,42 @@ function actorsDomain(opts) {
|
|
|
230849
230969
|
);
|
|
230850
230970
|
return { status: 201, body: cfg };
|
|
230851
230971
|
});
|
|
230972
|
+
router.get("/api/actors/:id/runtime-candidates", async (req) => {
|
|
230973
|
+
const { service } = await resolveCtx(req.auth.companyId);
|
|
230974
|
+
return { status: 200, body: { items: await service.listRuntimeCandidates(req.params.id) } };
|
|
230975
|
+
});
|
|
230976
|
+
router.put("/api/actors/:id/runtime-candidates", async (req) => {
|
|
230977
|
+
const { service } = await resolveCtx(req.auth.companyId);
|
|
230978
|
+
const actorId = req.params.id;
|
|
230979
|
+
if (!await service.getActor(actorId)) throw new ApiError(404, "NOT_FOUND", "\u6CA1\u6709\u8FD9\u4E2A\u5458\u5DE5");
|
|
230980
|
+
const raw = req.body?.candidates;
|
|
230981
|
+
if (!Array.isArray(raw)) throw new ApiError(400, "BAD_REQUEST", "candidates \u5FC5\u987B\u662F\u6570\u7EC4");
|
|
230982
|
+
if (raw.length > 20) throw new ApiError(400, "BAD_REQUEST", "\u5019\u9009\u8FD0\u884C\u65F6\u6700\u591A 20 \u6761");
|
|
230983
|
+
const candidates = [];
|
|
230984
|
+
for (const item of raw) {
|
|
230985
|
+
const c = item;
|
|
230986
|
+
if (typeof c.nodeId !== "string" || !c.nodeId.trim()) throw new ApiError(400, "BAD_REQUEST", "\u6BCF\u6761\u5019\u9009\u90FD\u8981\u6709 nodeId");
|
|
230987
|
+
if (typeof c.runtimeKind !== "string" || !c.runtimeKind.trim()) throw new ApiError(400, "BAD_REQUEST", "\u6BCF\u6761\u5019\u9009\u90FD\u8981\u6709 runtimeKind");
|
|
230988
|
+
if (c.model !== void 0 && c.model !== null && typeof c.model !== "string") {
|
|
230989
|
+
throw new ApiError(400, "BAD_REQUEST", "model \u5FC5\u987B\u662F\u5B57\u7B26\u4E32\u6216 null\uFF08null = \u8DDF\u968F\u8BE5 runtime \u9ED8\u8BA4\uFF09");
|
|
230990
|
+
}
|
|
230991
|
+
candidates.push({
|
|
230992
|
+
nodeId: c.nodeId.trim(),
|
|
230993
|
+
runtimeKind: c.runtimeKind.trim(),
|
|
230994
|
+
...c.model === void 0 ? {} : { model: typeof c.model === "string" ? c.model : null }
|
|
230995
|
+
});
|
|
230996
|
+
}
|
|
230997
|
+
if (opts.resolveNodeTenancy) {
|
|
230998
|
+
for (const c of candidates) {
|
|
230999
|
+
const facts = await opts.resolveNodeTenancy(c.nodeId).catch(() => null) ?? null;
|
|
231000
|
+
const foreign = foreignNodeBlock(facts, req.auth.companyId);
|
|
231001
|
+
if (foreign) throw new ApiError(400, "NODE_NOT_IN_COMPANY", foreign);
|
|
231002
|
+
if (!facts) throw new ApiError(400, "NODE_NOT_FOUND", `\u627E\u4E0D\u5230\u8282\u70B9 ${c.nodeId}`);
|
|
231003
|
+
}
|
|
231004
|
+
}
|
|
231005
|
+
const items = await service.replaceRuntimeCandidates(actorId, candidates, req.auth.actor);
|
|
231006
|
+
return { status: 200, body: { items } };
|
|
231007
|
+
});
|
|
230852
231008
|
router.get("/api/actors/:id/config/versions", async (req) => {
|
|
230853
231009
|
const { service } = await resolveCtx(req.auth.companyId);
|
|
230854
231010
|
return { status: 200, body: { items: await service.listConfigVersions(req.params.id) } };
|
|
@@ -242335,6 +242491,59 @@ var init_sender = __esm({
|
|
|
242335
242491
|
}
|
|
242336
242492
|
});
|
|
242337
242493
|
|
|
242494
|
+
// ../server/src/domains/runtime-allocation/faults.ts
|
|
242495
|
+
function faultReasonOfExit(reason, errorMessage2) {
|
|
242496
|
+
if (reason === "server-unreachable") return "node-error";
|
|
242497
|
+
if (reason === "auth-error") return "node-error";
|
|
242498
|
+
if (reason === "rate-limit") return "quota-exhausted";
|
|
242499
|
+
if (reason === "error" && errorMessage2?.startsWith("\u8FD9\u4E00\u8F6E\u6CA1\u80FD\u5728\u8282\u70B9\u4E0A\u542F\u52A8\uFF1A")) return "startup-failed";
|
|
242500
|
+
return null;
|
|
242501
|
+
}
|
|
242502
|
+
function probeIntervalMs(attempt) {
|
|
242503
|
+
const raw = RUNTIME_FAULT_PROBE_INITIAL_MS * 2 ** Math.max(0, attempt - 1);
|
|
242504
|
+
return Math.min(raw, RUNTIME_FAULT_PROBE_MAX_INTERVAL_MS);
|
|
242505
|
+
}
|
|
242506
|
+
function faultAfterFailure(previous3, input, now) {
|
|
242507
|
+
const nowIso = new Date(now).toISOString();
|
|
242508
|
+
const markedAt = previous3?.markedAt ?? nowIso;
|
|
242509
|
+
const attempts = (previous3?.attempts ?? 0) + 1;
|
|
242510
|
+
const base = {
|
|
242511
|
+
nodeId: input.nodeId,
|
|
242512
|
+
runtimeKind: input.runtimeKind,
|
|
242513
|
+
reason: input.reason,
|
|
242514
|
+
...input.detail ? { detail: input.detail.slice(0, 500) } : {},
|
|
242515
|
+
markedAt,
|
|
242516
|
+
attempts,
|
|
242517
|
+
awaitingManual: false
|
|
242518
|
+
};
|
|
242519
|
+
if (RUNTIME_FAULT_RECOVERY[input.reason] === "manual") {
|
|
242520
|
+
return { ...base, awaitingManual: true };
|
|
242521
|
+
}
|
|
242522
|
+
const elapsed = now - Date.parse(markedAt);
|
|
242523
|
+
if (Number.isFinite(elapsed) && elapsed >= RUNTIME_FAULT_PROBE_MAX_TOTAL_MS) {
|
|
242524
|
+
return { ...base, awaitingManual: true };
|
|
242525
|
+
}
|
|
242526
|
+
return { ...base, nextProbeAt: new Date(now + probeIntervalMs(attempts)).toISOString() };
|
|
242527
|
+
}
|
|
242528
|
+
function quotaClearsFault(fault, quotaStatus) {
|
|
242529
|
+
if (RUNTIME_FAULT_RECOVERY[fault.reason] !== "readonly-probe" && fault.reason !== "unauthenticated") return false;
|
|
242530
|
+
return quotaStatus === "ok";
|
|
242531
|
+
}
|
|
242532
|
+
function describeFault(fault, now) {
|
|
242533
|
+
const head = `${fault.nodeId} \xB7 ${fault.runtimeKind} \u6545\u969C\uFF08${fault.reason}${fault.detail ? `\uFF1A${fault.detail}` : ""}\uFF09`;
|
|
242534
|
+
if (fault.awaitingManual) return `${head}\uFF0C\u5DF2\u8BD5 ${fault.attempts} \u6B21\u4ECD\u672A\u6062\u590D\uFF0C\u7B49\u4EBA\u5DE5\u6062\u590D`;
|
|
242535
|
+
if (!fault.nextProbeAt) return `${head}\uFF0C\u7B49\u8FDE\u63A5\u6062\u590D`;
|
|
242536
|
+
const waitMs = Date.parse(fault.nextProbeAt) - now;
|
|
242537
|
+
const mins = Math.max(0, Math.round(waitMs / 6e4));
|
|
242538
|
+
return `${head}\uFF0C\u5DF2\u8BD5 ${fault.attempts} \u6B21\uFF0C${mins} \u5206\u949F\u540E\u653E\u4E00\u6761\u65B0\u4F1A\u8BDD\u8FC7\u53BB\u8BD5`;
|
|
242539
|
+
}
|
|
242540
|
+
var init_faults = __esm({
|
|
242541
|
+
"../server/src/domains/runtime-allocation/faults.ts"() {
|
|
242542
|
+
"use strict";
|
|
242543
|
+
init_src();
|
|
242544
|
+
}
|
|
242545
|
+
});
|
|
242546
|
+
|
|
242338
242547
|
// ../server/src/domains/nodes/runtime-status.ts
|
|
242339
242548
|
function runtimeConnectionStatus(nodeId, connected) {
|
|
242340
242549
|
return connected.has(nodeId) ? "online" : "offline";
|
|
@@ -242898,7 +243107,7 @@ var init_runtime_activity = __esm({
|
|
|
242898
243107
|
|
|
242899
243108
|
// ../server/src/node-health.ts
|
|
242900
243109
|
function deriveNodeHealth(input) {
|
|
242901
|
-
const staleMs = input.staleMs ??
|
|
243110
|
+
const staleMs = input.staleMs ?? NODE_HEARTBEAT_STALE_MS;
|
|
242902
243111
|
const maxFail = input.maxConsecutiveFailures ?? 3;
|
|
242903
243112
|
const cf = input.snapshot?.consecutiveFailures ?? 0;
|
|
242904
243113
|
const heartbeatFresh = input.lastHeartbeatMs !== void 0 && input.now - input.lastHeartbeatMs < staleMs;
|
|
@@ -242957,7 +243166,7 @@ function describeNodeFault(f2, now = Date.now()) {
|
|
|
242957
243166
|
`\u5728\u90A3\u4E4B\u524D\uFF0C\u91CD\u8BD5\u3001\u6362\u4EBA\u3001\u6539\u5185\u5BB9\u90FD\u4E0D\u4F1A\u8BA9\u5B83\u53D8\u597D\u3002`
|
|
242958
243167
|
].join("");
|
|
242959
243168
|
}
|
|
242960
|
-
var NodeHealthTracker;
|
|
243169
|
+
var NodeHealthTracker, NODE_HEARTBEAT_STALE_MS;
|
|
242961
243170
|
var init_node_health = __esm({
|
|
242962
243171
|
"../server/src/node-health.ts"() {
|
|
242963
243172
|
"use strict";
|
|
@@ -242999,6 +243208,7 @@ var init_node_health = __esm({
|
|
|
242999
243208
|
this.onChange?.();
|
|
243000
243209
|
}
|
|
243001
243210
|
};
|
|
243211
|
+
NODE_HEARTBEAT_STALE_MS = 9e4;
|
|
243002
243212
|
}
|
|
243003
243213
|
});
|
|
243004
243214
|
|
|
@@ -243034,14 +243244,40 @@ async function resolveRuntimeModels(deps, runtimeId) {
|
|
|
243034
243244
|
}
|
|
243035
243245
|
function nodesDomain(deps) {
|
|
243036
243246
|
const quotaByNode = /* @__PURE__ */ new Map();
|
|
243247
|
+
const faultByTarget = /* @__PURE__ */ new Map();
|
|
243248
|
+
const faultKey = (nodeId, runtimeKind) => `${nodeId}::${runtimeKind}`;
|
|
243249
|
+
const refreshFaults = async () => {
|
|
243250
|
+
const rows = await deps.nodeStore.listRuntimeFaults();
|
|
243251
|
+
faultByTarget.clear();
|
|
243252
|
+
for (const f2 of rows) faultByTarget.set(faultKey(f2.nodeId, f2.runtimeKind), f2);
|
|
243253
|
+
};
|
|
243254
|
+
void refreshFaults().catch((err) => console.warn(`[nodes] \u6545\u969C\u6807\u8BB0\u6C34\u5408\u5931\u8D25\uFF08\u9875\u9762\u4E0A\u6682\u65F6\u770B\u4E0D\u5230\u6545\u969C\u6761\uFF09\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243255
|
+
let faultsReadAtMs = 0;
|
|
243256
|
+
const FAULT_CACHE_TTL_MS = 1e4;
|
|
243257
|
+
const refreshFaultsIfStale = async () => {
|
|
243258
|
+
if (Date.now() - faultsReadAtMs < FAULT_CACHE_TTL_MS) return;
|
|
243259
|
+
faultsReadAtMs = Date.now();
|
|
243260
|
+
await refreshFaults().catch((err) => console.warn(`[nodes] \u6545\u969C\u6807\u8BB0\u91CD\u8BFB\u5931\u8D25\uFF08\u672C\u8F6E\u6CBF\u7528\u4E0A\u6B21\u7ED3\u679C\uFF09\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243261
|
+
};
|
|
243262
|
+
void deps.nodeStore.listRuntimeQuotas().then((rows) => {
|
|
243263
|
+
for (const row of rows) {
|
|
243264
|
+
if (row.runtimeKind !== "codex" && row.runtimeKind !== "claude-code") continue;
|
|
243265
|
+
const list2 = quotaByNode.get(row.nodeId) ?? [];
|
|
243266
|
+
list2.push({ kind: row.runtimeKind, status: row.status, checkedAt: row.checkedAt, windows: row.windows, ...row.retryAt ? { retryAt: row.retryAt } : {} });
|
|
243267
|
+
quotaByNode.set(row.nodeId, list2);
|
|
243268
|
+
}
|
|
243269
|
+
if (rows.length > 0) console.log(`[nodes] \u989D\u5EA6\u6C34\u5408\uFF1A\u4ECE\u5E93\u91CC\u8BFB\u56DE ${rows.length} \u6761\u8FD0\u884C\u65F6\u989D\u5EA6`);
|
|
243270
|
+
}).catch((err) => console.warn(`[nodes] \u989D\u5EA6\u6C34\u5408\u5931\u8D25\uFF08\u7167\u65E7\u7B49\u4E0B\u4E00\u6B21\u91C7\u96C6\uFF09\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243037
243271
|
const UPDATE_TIMEOUT_MS = 15e4;
|
|
243038
243272
|
const nodeUpdates = /* @__PURE__ */ new Map();
|
|
243039
243273
|
function runtimeInfoOf(r) {
|
|
243040
243274
|
const quota = quotaByNode.get(r.nodeId)?.find((q2) => q2.kind === r.kind);
|
|
243275
|
+
const fault = faultByTarget.get(faultKey(r.nodeId, r.kind));
|
|
243041
243276
|
const activeRunCount = deps.activeRunCountOf?.(r.nodeId, r.kind) ?? 0;
|
|
243042
243277
|
const activeRuns = deps.activeRunsOf?.(r.nodeId, r.kind);
|
|
243043
243278
|
return {
|
|
243044
243279
|
...quota ? { quota: { status: quota.status, checkedAt: quota.checkedAt, windows: quota.windows, ...quota.retryAt ? { retryAt: quota.retryAt } : {} } } : {},
|
|
243280
|
+
...fault ? { fault } : {},
|
|
243045
243281
|
id: r.id,
|
|
243046
243282
|
nodeId: r.nodeId,
|
|
243047
243283
|
hostname: r.hostname,
|
|
@@ -243134,7 +243370,18 @@ function nodesDomain(deps) {
|
|
|
243134
243370
|
if (msg.type !== "runtime_quotas") return;
|
|
243135
243371
|
const daemon = hub.connectedDaemons().find((d) => d.daemonId === daemonId);
|
|
243136
243372
|
if (!daemon) return;
|
|
243137
|
-
|
|
243373
|
+
const quotas = sanitizeRuntimeQuotas(msg.quotas).filter((q2) => daemon.meta.adapters.includes(q2.kind));
|
|
243374
|
+
quotaByNode.set(daemonId, quotas);
|
|
243375
|
+
void nodeStore.saveRuntimeQuotas(daemonId, quotas.map((q2) => ({ ...q2, nodeId: daemonId, runtimeKind: q2.kind }))).catch((err) => console.warn(`[nodes] \u989D\u5EA6\u843D\u5E93\u5931\u8D25(node=${daemonId})\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243376
|
+
void (async () => {
|
|
243377
|
+
for (const q2 of quotas) {
|
|
243378
|
+
const existing = faultByTarget.get(faultKey(daemonId, q2.kind));
|
|
243379
|
+
if (!existing || !quotaClearsFault(existing, q2.status)) continue;
|
|
243380
|
+
await nodeStore.clearRuntimeFault(daemonId, q2.kind);
|
|
243381
|
+
faultByTarget.delete(faultKey(daemonId, q2.kind));
|
|
243382
|
+
console.log(`[nodes] \u53EA\u8BFB\u91C7\u96C6\u56DE ok\uFF0C\u81EA\u52A8\u89E3\u9664\u6545\u969C\uFF1A${daemonId} \xB7 ${q2.kind}\uFF08\u539F\u56E0 ${existing.reason}\uFF09`);
|
|
243383
|
+
}
|
|
243384
|
+
})().catch((err) => console.warn(`[nodes] \u6545\u969C\u81EA\u52A8\u89E3\u9664\u5931\u8D25(node=${daemonId})\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243138
243385
|
});
|
|
243139
243386
|
hub.addMessageListener(async (daemonId, msg) => {
|
|
243140
243387
|
if (msg.type !== "hello") return;
|
|
@@ -243238,6 +243485,7 @@ function nodesDomain(deps) {
|
|
|
243238
243485
|
await deps.nodeStore.setNodeRuntimesOnline(daemonId, true);
|
|
243239
243486
|
}
|
|
243240
243487
|
router.get("/api/nodes", async (req) => {
|
|
243488
|
+
await refreshFaultsIfStale();
|
|
243241
243489
|
const hub = deps.getHub();
|
|
243242
243490
|
if (hub) wireHub(hub);
|
|
243243
243491
|
const daemonVersionById = new Map((hub?.connectedDaemons() ?? []).map((d) => [d.daemonId, d.meta.daemonVersion]));
|
|
@@ -243286,6 +243534,10 @@ function nodesDomain(deps) {
|
|
|
243286
243534
|
...sourceIpById.get(n.id) ? { sourceIp: sourceIpById.get(n.id) } : {},
|
|
243287
243535
|
...n.name ? { name: n.name } : {},
|
|
243288
243536
|
...updateStatus ? { updateStatus } : {},
|
|
243537
|
+
/* 并发上限与当前在跑数(ADR 0616 D2):`maxConcurrent` 只在真设过时带这一格——
|
|
243538
|
+
缺省就是「未设置 = 不限制」,带一个 null 进去会让前端不得不再判一次。 */
|
|
243539
|
+
...n.maxConcurrent != null ? { maxConcurrent: n.maxConcurrent } : {},
|
|
243540
|
+
activeRunCount,
|
|
243289
243541
|
...lastDisconnectedAt || lastReconnectedAt ? {
|
|
243290
243542
|
connection: {
|
|
243291
243543
|
...lastDisconnectedAt ? { lastDisconnectedAt } : {},
|
|
@@ -243485,6 +243737,38 @@ function nodesDomain(deps) {
|
|
|
243485
243737
|
deps.onStatusChange?.();
|
|
243486
243738
|
return { status: 200, body: { ok: true, name } };
|
|
243487
243739
|
});
|
|
243740
|
+
router.put("/api/nodes/:nodeId/max-concurrent", async (req) => {
|
|
243741
|
+
const nodeId = req.params["nodeId"];
|
|
243742
|
+
if (!nodeId) return { status: 400, body: { error: "missing nodeId" } };
|
|
243743
|
+
const forbidden = await forbidUnlessOrgManager(req, "\u8BBE\u7F6E\u8282\u70B9\u5E76\u53D1\u4E0A\u9650");
|
|
243744
|
+
if (forbidden) return forbidden;
|
|
243745
|
+
const scope2 = req.auth.companyId !== void 0 ? { companyId: req.auth.companyId } : void 0;
|
|
243746
|
+
const node2 = (await deps.nodeStore.listNodes(scope2)).find((n) => n.id === nodeId);
|
|
243747
|
+
if (!node2) return { status: 404, body: { error: "NODE_NOT_FOUND", message: "\u8282\u70B9\u4E0D\u5B58\u5728\u6216\u4E0D\u5C5E\u4E8E\u5F53\u524D\u7EC4\u7EC7" } };
|
|
243748
|
+
const raw = req.body ?? {};
|
|
243749
|
+
const value2 = raw.maxConcurrent;
|
|
243750
|
+
if (value2 !== null && value2 !== void 0 && !(Number.isSafeInteger(value2) && value2 >= 0 && value2 <= 1e3)) {
|
|
243751
|
+
return { status: 400, body: { error: "BAD_REQUEST", message: "maxConcurrent \u5FC5\u987B\u662F 0\u20131000 \u7684\u6574\u6570\uFF0C\u6216 null\uFF08\u4E0D\u9650\u5236\uFF09" } };
|
|
243752
|
+
}
|
|
243753
|
+
const limit = value2 === void 0 || value2 === null ? null : value2;
|
|
243754
|
+
await deps.nodeStore.setNodeMaxConcurrent(nodeId, limit);
|
|
243755
|
+
deps.onStatusChange?.();
|
|
243756
|
+
return { status: 200, body: { ok: true, maxConcurrent: limit } };
|
|
243757
|
+
});
|
|
243758
|
+
router.post("/api/nodes/:nodeId/runtimes/:runtimeKind/recover", async (req) => {
|
|
243759
|
+
const nodeId = req.params["nodeId"];
|
|
243760
|
+
const runtimeKind = req.params["runtimeKind"];
|
|
243761
|
+
if (!nodeId || !runtimeKind) return { status: 400, body: { error: "missing nodeId/runtimeKind" } };
|
|
243762
|
+
const forbidden = await forbidUnlessOrgManager(req, "\u624B\u52A8\u6062\u590D\u8FD0\u884C\u65F6");
|
|
243763
|
+
if (forbidden) return forbidden;
|
|
243764
|
+
const scope2 = req.auth.companyId !== void 0 ? { companyId: req.auth.companyId } : void 0;
|
|
243765
|
+
const node2 = (await deps.nodeStore.listNodes(scope2)).find((n) => n.id === nodeId);
|
|
243766
|
+
if (!node2) return { status: 404, body: { error: "NODE_NOT_FOUND", message: "\u8282\u70B9\u4E0D\u5B58\u5728\u6216\u4E0D\u5C5E\u4E8E\u5F53\u524D\u7EC4\u7EC7" } };
|
|
243767
|
+
const cleared = await deps.nodeStore.clearRuntimeFault(nodeId, runtimeKind);
|
|
243768
|
+
faultByTarget.delete(faultKey(nodeId, runtimeKind));
|
|
243769
|
+
deps.onStatusChange?.();
|
|
243770
|
+
return { status: 200, body: { ok: true, cleared } };
|
|
243771
|
+
});
|
|
243488
243772
|
router.delete("/api/nodes/:nodeId", async (req) => {
|
|
243489
243773
|
const nodeId = req.params["nodeId"];
|
|
243490
243774
|
if (!nodeId) return { status: 400, body: { error: "missing nodeId" } };
|
|
@@ -243663,6 +243947,7 @@ var init_routes7 = __esm({
|
|
|
243663
243947
|
"../server/src/domains/nodes/routes.ts"() {
|
|
243664
243948
|
"use strict";
|
|
243665
243949
|
init_src();
|
|
243950
|
+
init_faults();
|
|
243666
243951
|
init_runtime_status();
|
|
243667
243952
|
import_node_crypto61 = require("node:crypto");
|
|
243668
243953
|
import_node_fs17 = require("node:fs");
|
|
@@ -243706,6 +243991,400 @@ var init_runtime_drift = __esm({
|
|
|
243706
243991
|
}
|
|
243707
243992
|
});
|
|
243708
243993
|
|
|
243994
|
+
// ../server/src/domains/runtime-allocation/allocate.ts
|
|
243995
|
+
function runtimeTargetKey(target) {
|
|
243996
|
+
return `${target.nodeId}::${target.runtimeKind}`;
|
|
243997
|
+
}
|
|
243998
|
+
function idlenessOf(node2) {
|
|
243999
|
+
const limit = node2.maxConcurrent;
|
|
244000
|
+
if (limit === null || limit === void 0) return 1;
|
|
244001
|
+
if (limit <= 0) return 0;
|
|
244002
|
+
return Math.max(0, 1 - node2.activeRunCount / limit);
|
|
244003
|
+
}
|
|
244004
|
+
function atCapacity(node2) {
|
|
244005
|
+
const limit = node2.maxConcurrent;
|
|
244006
|
+
if (limit === null || limit === void 0) return false;
|
|
244007
|
+
return node2.activeRunCount >= limit;
|
|
244008
|
+
}
|
|
244009
|
+
function quotaDerateFactor(quota) {
|
|
244010
|
+
if (!quota || quota.status !== "ok" || quota.windows.length === 0) return { factor: 1 };
|
|
244011
|
+
const worst = Math.max(...quota.windows.map((w2) => w2.usedPercent));
|
|
244012
|
+
if (worst <= RUNTIME_QUOTA_DERATE_THRESHOLD_PERCENT) return { factor: 1, worstUsedPercent: worst };
|
|
244013
|
+
const remaining = Math.max(0, 100 - worst);
|
|
244014
|
+
return { factor: remaining / (100 - RUNTIME_QUOTA_DERATE_THRESHOLD_PERCENT), worstUsedPercent: worst };
|
|
244015
|
+
}
|
|
244016
|
+
function faultFromQuota(target, quota, now) {
|
|
244017
|
+
if (!quota) return null;
|
|
244018
|
+
const base = { nodeId: target.nodeId, runtimeKind: target.runtimeKind, markedAt: quota.checkedAt, attempts: 0 };
|
|
244019
|
+
if (quota.status === "unauthenticated") {
|
|
244020
|
+
return { ...base, reason: "unauthenticated", awaitingManual: true, detail: "\u8FD0\u884C\u65F6\u767B\u5F55\u5DF2\u5931\u6548\uFF0C\u9700\u8981\u5728\u8BE5\u673A\u5668\u4E0A\u91CD\u65B0\u767B\u5F55" };
|
|
244021
|
+
}
|
|
244022
|
+
if (quota.status === "rate_limited") {
|
|
244023
|
+
return {
|
|
244024
|
+
...base,
|
|
244025
|
+
reason: "quota-exhausted",
|
|
244026
|
+
awaitingManual: false,
|
|
244027
|
+
...quota.retryAt ? { nextProbeAt: quota.retryAt } : {},
|
|
244028
|
+
detail: "provider \u62A5\u9650\u6D41/\u989D\u5EA6\u8017\u5C3D"
|
|
244029
|
+
};
|
|
244030
|
+
}
|
|
244031
|
+
if (quota.status !== "ok") return null;
|
|
244032
|
+
const exhausted = quota.windows.filter((w2) => w2.usedPercent >= 100);
|
|
244033
|
+
if (exhausted.length === 0) return null;
|
|
244034
|
+
const resets = exhausted.map((w2) => w2.resetsAt).filter((v2) => Boolean(v2)).sort();
|
|
244035
|
+
const latest = resets.at(-1);
|
|
244036
|
+
return {
|
|
244037
|
+
...base,
|
|
244038
|
+
reason: "quota-exhausted",
|
|
244039
|
+
awaitingManual: false,
|
|
244040
|
+
...latest && Date.parse(latest) > now ? { nextProbeAt: latest } : {},
|
|
244041
|
+
detail: `\u989D\u5EA6\u5DF2\u7528\u6EE1\uFF08${exhausted.map((w2) => `${w2.bucket} ${Math.round(w2.usedPercent)}%`).join("\u3001")}\uFF09`
|
|
244042
|
+
};
|
|
244043
|
+
}
|
|
244044
|
+
function faultIsBlocking(fault, now) {
|
|
244045
|
+
if (fault.awaitingManual) return true;
|
|
244046
|
+
if (!fault.nextProbeAt) return true;
|
|
244047
|
+
return Date.parse(fault.nextProbeAt) > now;
|
|
244048
|
+
}
|
|
244049
|
+
function failureReason(rejections) {
|
|
244050
|
+
if (rejections.length === 0) return "no-candidates";
|
|
244051
|
+
const causes = new Set(rejections.map((r) => r.cause));
|
|
244052
|
+
if (causes.size === 1) {
|
|
244053
|
+
const only = [...causes][0];
|
|
244054
|
+
if (only === "at-capacity") return "all-at-capacity";
|
|
244055
|
+
if (only === "already-tried") return "all-tried";
|
|
244056
|
+
return "all-faulted";
|
|
244057
|
+
}
|
|
244058
|
+
const faultLike = /* @__PURE__ */ new Set(["node-unknown", "node-offline", "runtime-missing", "faulted"]);
|
|
244059
|
+
if ([...causes].every((c) => faultLike.has(c))) return "all-faulted";
|
|
244060
|
+
return "mixed";
|
|
244061
|
+
}
|
|
244062
|
+
function allocateRuntime(input, now = Date.now()) {
|
|
244063
|
+
const rejections = [];
|
|
244064
|
+
const eligible = [];
|
|
244065
|
+
for (const candidate of input.candidates) {
|
|
244066
|
+
const key2 = runtimeTargetKey(candidate);
|
|
244067
|
+
const node2 = input.nodes.get(candidate.nodeId);
|
|
244068
|
+
const nodeName = node2?.displayName?.trim() || candidate.nodeId;
|
|
244069
|
+
const reject = (cause, detail) => rejections.push({ nodeId: candidate.nodeId, runtimeKind: candidate.runtimeKind, nodeName, cause, ...detail ? { detail } : {} });
|
|
244070
|
+
if (input.alreadyTried?.has(key2)) {
|
|
244071
|
+
reject("already-tried", input.triedDetail?.get(key2) ?? "\u672C\u6B21\u5DF2\u8BD5\u8FC7\u5E76\u5931\u8D25");
|
|
244072
|
+
continue;
|
|
244073
|
+
}
|
|
244074
|
+
if (!node2) {
|
|
244075
|
+
reject("node-unknown", "\u8282\u70B9\u4E0D\u5B58\u5728\u3001\u5DF2\u5220\u9664\uFF0C\u6216\u4E0D\u5C5E\u4E8E\u5F53\u524D\u516C\u53F8");
|
|
244076
|
+
continue;
|
|
244077
|
+
}
|
|
244078
|
+
if (!node2.online) {
|
|
244079
|
+
reject("node-offline", "\u8282\u70B9\u5F53\u524D\u79BB\u7EBF");
|
|
244080
|
+
continue;
|
|
244081
|
+
}
|
|
244082
|
+
if (!node2.runtimeKinds.includes(candidate.runtimeKind)) {
|
|
244083
|
+
reject("runtime-missing", "\u8BE5\u673A\u5668\u4E0A\u6CA1\u6709\u8FD9\u4E2A\u8FD0\u884C\u65F6\uFF08\u672A\u88C5\u597D\u3001\u5DF2\u5378\u8F7D\u6216\u672A\u4E0A\u62A5\uFF09");
|
|
244084
|
+
continue;
|
|
244085
|
+
}
|
|
244086
|
+
const fault = input.faults?.get(key2) ?? faultFromQuota(candidate, input.quotas?.get(key2), now);
|
|
244087
|
+
if (fault && faultIsBlocking(fault, now)) {
|
|
244088
|
+
reject("faulted", `${fault.reason}${fault.detail ? `\uFF1A${fault.detail}` : ""}${fault.awaitingManual ? "\uFF08\u7B49\u4EBA\u5DE5\u6062\u590D\uFF09" : ""}`);
|
|
244089
|
+
continue;
|
|
244090
|
+
}
|
|
244091
|
+
if (atCapacity(node2)) {
|
|
244092
|
+
reject("at-capacity", `\u5728\u8DD1 ${node2.activeRunCount} / \u4E0A\u9650 ${node2.maxConcurrent}`);
|
|
244093
|
+
continue;
|
|
244094
|
+
}
|
|
244095
|
+
eligible.push(candidate);
|
|
244096
|
+
}
|
|
244097
|
+
if (eligible.length === 0) return { ok: false, reason: failureReason(rejections), rejections };
|
|
244098
|
+
const primary = eligible.find((c) => c.primary);
|
|
244099
|
+
const rank = (candidate) => {
|
|
244100
|
+
const node2 = input.nodes.get(candidate.nodeId);
|
|
244101
|
+
const idleness = idlenessOf(node2);
|
|
244102
|
+
const { factor, worstUsedPercent } = quotaDerateFactor(input.quotas?.get(runtimeTargetKey(candidate)));
|
|
244103
|
+
return {
|
|
244104
|
+
candidate,
|
|
244105
|
+
nodeName: node2.displayName?.trim() || candidate.nodeId,
|
|
244106
|
+
idleness,
|
|
244107
|
+
quotaFactor: factor,
|
|
244108
|
+
weight: idleness * factor,
|
|
244109
|
+
...worstUsedPercent !== void 0 ? { worstQuotaUsedPercent: worstUsedPercent } : {}
|
|
244110
|
+
};
|
|
244111
|
+
};
|
|
244112
|
+
if (primary) {
|
|
244113
|
+
const ranked2 = [rank(primary)];
|
|
244114
|
+
return { ok: true, chosen: primary, nodeName: ranked2[0].nodeName, source: "primary", ranked: ranked2, rejections };
|
|
244115
|
+
}
|
|
244116
|
+
const ranked = eligible.map(rank).sort((a, b2) => b2.weight - a.weight || a.candidate.ord - b2.candidate.ord || a.candidate.nodeId.localeCompare(b2.candidate.nodeId) || a.candidate.runtimeKind.localeCompare(b2.candidate.runtimeKind));
|
|
244117
|
+
const top = ranked[0];
|
|
244118
|
+
return { ok: true, chosen: top.candidate, nodeName: top.nodeName, source: "candidate", ranked, rejections };
|
|
244119
|
+
}
|
|
244120
|
+
function canRetryAllocation(attempt) {
|
|
244121
|
+
return attempt < RUNTIME_ALLOCATION_MAX_ATTEMPTS;
|
|
244122
|
+
}
|
|
244123
|
+
function describeAllocationChoice(result) {
|
|
244124
|
+
const head = result.source === "primary" ? `\u9009\u4E2D\u4E3B\u8FD0\u884C\u65F6 ${result.nodeName} \xB7 ${result.chosen.runtimeKind}` : `\u4E3B\u8FD0\u884C\u65F6\u4E0D\u53EF\u7528\uFF0C\u6309\u6743\u91CD\u9009\u4E2D ${result.nodeName} \xB7 ${result.chosen.runtimeKind}`;
|
|
244125
|
+
const detail = result.ranked.map((r) => {
|
|
244126
|
+
const quota = r.worstQuotaUsedPercent === void 0 ? "\u65E0\u989D\u5EA6\u6570\u636E" : `\u989D\u5EA6\u5DF2\u7528 ${Math.round(r.worstQuotaUsedPercent)}%`;
|
|
244127
|
+
return `${r.nodeName}\xB7${r.candidate.runtimeKind} \u6743\u91CD ${r.weight.toFixed(2)}\uFF08\u7A7A\u95F2\u5EA6 ${r.idleness.toFixed(2)} \xD7 \u989D\u5EA6\u7CFB\u6570 ${r.quotaFactor.toFixed(2)}\uFF0C${quota}\uFF09`;
|
|
244128
|
+
}).join("\uFF1B");
|
|
244129
|
+
const skipped = result.rejections.length > 0 ? `\uFF1B\u6392\u9664 ${result.rejections.map((r) => `${r.nodeName}\xB7${r.runtimeKind}(${r.cause})`).join("\u3001")}` : "";
|
|
244130
|
+
return `${head}\u3002${detail}${skipped}`;
|
|
244131
|
+
}
|
|
244132
|
+
var init_allocate = __esm({
|
|
244133
|
+
"../server/src/domains/runtime-allocation/allocate.ts"() {
|
|
244134
|
+
"use strict";
|
|
244135
|
+
init_src();
|
|
244136
|
+
}
|
|
244137
|
+
});
|
|
244138
|
+
|
|
244139
|
+
// ../server/src/domains/runtime-allocation/errors.ts
|
|
244140
|
+
function describeRejections(rejections) {
|
|
244141
|
+
return rejections.map((r) => `${r.nodeName} \xB7 ${r.runtimeKind}\uFF08${CAUSE_TEXT[r.cause]}${r.detail ? `\uFF1A${r.detail}` : ""}\uFF09`).join("\uFF1B");
|
|
244142
|
+
}
|
|
244143
|
+
function requireAllocation(result, who) {
|
|
244144
|
+
if (result.ok) return result;
|
|
244145
|
+
const subject = who?.actorName?.trim() ? `\u300C${who.actorName.trim()}\u300D` : "\u8FD9\u4F4D Agent";
|
|
244146
|
+
const what = who?.what?.trim() ? `${who.what.trim()}\uFF1A` : "";
|
|
244147
|
+
const detail = describeRejections(result.rejections);
|
|
244148
|
+
if (result.reason === "no-candidates") {
|
|
244149
|
+
throw new RuntimeAllocationError(
|
|
244150
|
+
"RUNTIME_ALLOCATION_NO_CANDIDATES",
|
|
244151
|
+
`${what}${subject}\u6CA1\u6709\u53EF\u7528\u7684\u8FD0\u884C\u65F6\u76EE\u6807\uFF1A\u65E2\u6CA1\u6709\u4E3B\u8FD0\u884C\u65F6\u7ED1\u5B9A\uFF0C\u4E5F\u6CA1\u6709\u914D\u7F6E\u5019\u9009\u8FD0\u884C\u65F6\u3002\u8BF7\u5728\u5458\u5DE5\u8BE6\u60C5\u9875\u7684\u300C\u9ED8\u8BA4\u8FD0\u884C\u914D\u7F6E\u300D\u91CC\u7ED1\u5B9A\u4E00\u4E2A\u8FD0\u884C\u65F6\uFF0C\u6216\u52A0\u4E00\u6761\u5019\u9009\u8FD0\u884C\u65F6\u3002`,
|
|
244152
|
+
result.rejections
|
|
244153
|
+
);
|
|
244154
|
+
}
|
|
244155
|
+
if (result.reason === "all-at-capacity") {
|
|
244156
|
+
throw new RuntimeAllocationError(
|
|
244157
|
+
"RUNTIME_ALLOCATION_ALL_AT_CAPACITY",
|
|
244158
|
+
`${what}${subject}\u7684\u8FD0\u884C\u65F6\u5168\u90E8\u5DF2\u5230\u5E76\u53D1\u4E0A\u9650\uFF0C\u672C\u6B21\u6CA1\u6709\u6D3E\u51FA\uFF1A${detail}\u3002\u8BF7\u7B49\u5F53\u524D\u4F1A\u8BDD\u8DD1\u5B8C\uFF0C\u6216\u5728\u8FD0\u884C\u65F6\u7BA1\u7406\u9875\u628A\u8FD9\u4E9B\u8282\u70B9\u7684\u5E76\u53D1\u4E0A\u9650\u8C03\u9AD8\u3002`,
|
|
244159
|
+
result.rejections,
|
|
244160
|
+
503
|
|
244161
|
+
);
|
|
244162
|
+
}
|
|
244163
|
+
throw new RuntimeAllocationError(
|
|
244164
|
+
"RUNTIME_ALLOCATION_ALL_UNAVAILABLE",
|
|
244165
|
+
`${what}${subject}\u7684\u8FD0\u884C\u65F6\u76EE\u524D\u90FD\u4E0D\u53EF\u7528\uFF0C\u672C\u6B21\u6CA1\u6709\u6D3E\u51FA\uFF1A${detail}\u3002\u8BF7\u786E\u8BA4\u90A3\u4E9B\u673A\u5668\u5F00\u7740\u3001\u8282\u70B9\u7A0B\u5E8F\u5728\u8FD0\u884C\u3001\u8FD0\u884C\u65F6\u5DF2\u767B\u5F55\u4E14\u989D\u5EA6\u672A\u8017\u5C3D\uFF1B\u6216\u5728\u5458\u5DE5\u8BE6\u60C5\u9875\u52A0\u4E00\u6761\u522B\u7684\u5019\u9009\u8FD0\u884C\u65F6\u3002`,
|
|
244166
|
+
result.rejections,
|
|
244167
|
+
result.reason === "all-tried" ? 409 : 503
|
|
244168
|
+
);
|
|
244169
|
+
}
|
|
244170
|
+
var RuntimeAllocationError, CAUSE_TEXT;
|
|
244171
|
+
var init_errors6 = __esm({
|
|
244172
|
+
"../server/src/domains/runtime-allocation/errors.ts"() {
|
|
244173
|
+
"use strict";
|
|
244174
|
+
RuntimeAllocationError = class extends Error {
|
|
244175
|
+
constructor(code2, message, rejections, status = 409) {
|
|
244176
|
+
super(message);
|
|
244177
|
+
this.code = code2;
|
|
244178
|
+
this.rejections = rejections;
|
|
244179
|
+
this.status = status;
|
|
244180
|
+
this.name = "RuntimeAllocationError";
|
|
244181
|
+
}
|
|
244182
|
+
};
|
|
244183
|
+
CAUSE_TEXT = {
|
|
244184
|
+
"node-unknown": "\u8282\u70B9\u4E0D\u53EF\u7528",
|
|
244185
|
+
"node-offline": "\u8282\u70B9\u79BB\u7EBF",
|
|
244186
|
+
"runtime-missing": "\u673A\u5668\u4E0A\u6CA1\u6709\u8FD9\u4E2A\u8FD0\u884C\u65F6",
|
|
244187
|
+
faulted: "\u6807\u8BB0\u4E3A\u6545\u969C",
|
|
244188
|
+
"at-capacity": "\u5DF2\u5230\u5E76\u53D1\u4E0A\u9650",
|
|
244189
|
+
"already-tried": "\u672C\u8F6E\u5DF2\u8BD5\u8FC7\u5E76\u5931\u8D25"
|
|
244190
|
+
};
|
|
244191
|
+
}
|
|
244192
|
+
});
|
|
244193
|
+
|
|
244194
|
+
// ../server/src/domains/runtime-allocation/service.ts
|
|
244195
|
+
function isRuntimeClassFailure(reason, errorMessage2) {
|
|
244196
|
+
if (reason === "server-unreachable" || reason === "rate-limit" || reason === "auth-error") return true;
|
|
244197
|
+
if (reason === "error" && errorMessage2?.startsWith("\u8FD9\u4E00\u8F6E\u6CA1\u80FD\u5728\u8282\u70B9\u4E0A\u542F\u52A8\uFF1A")) return true;
|
|
244198
|
+
return false;
|
|
244199
|
+
}
|
|
244200
|
+
function createRuntimeAllocator(ports) {
|
|
244201
|
+
const units = /* @__PURE__ */ new Map();
|
|
244202
|
+
const log3 = ports.log ?? ((line2) => console.log(line2));
|
|
244203
|
+
const sweep = (nowMs) => {
|
|
244204
|
+
for (const [key2, state] of units) if (nowMs - state.updatedAtMs > UNIT_STATE_TTL_MS) units.delete(key2);
|
|
244205
|
+
};
|
|
244206
|
+
const stateOf = (unitOfWork, nowMs) => {
|
|
244207
|
+
sweep(nowMs);
|
|
244208
|
+
const existing = units.get(unitOfWork);
|
|
244209
|
+
if (existing) return existing;
|
|
244210
|
+
const fresh = { tried: /* @__PURE__ */ new Map(), updatedAtMs: nowMs };
|
|
244211
|
+
units.set(unitOfWork, fresh);
|
|
244212
|
+
return fresh;
|
|
244213
|
+
};
|
|
244214
|
+
const flatten = (binding, candidates, primary) => {
|
|
244215
|
+
const out = [];
|
|
244216
|
+
const seen = /* @__PURE__ */ new Set();
|
|
244217
|
+
const head = primary ?? (binding?.nodeId && binding.runtimeKind && binding.status === "active" ? { nodeId: binding.nodeId, runtimeKind: binding.runtimeKind, model: null } : null);
|
|
244218
|
+
if (head) {
|
|
244219
|
+
out.push({ nodeId: head.nodeId, runtimeKind: head.runtimeKind, primary: true, ord: -1, model: head.model ?? null });
|
|
244220
|
+
seen.add(runtimeTargetKey(head));
|
|
244221
|
+
}
|
|
244222
|
+
for (const c of candidates) {
|
|
244223
|
+
const key2 = runtimeTargetKey(c);
|
|
244224
|
+
if (seen.has(key2)) continue;
|
|
244225
|
+
seen.add(key2);
|
|
244226
|
+
out.push({ nodeId: c.nodeId, runtimeKind: c.runtimeKind, primary: false, ord: c.ord, model: c.model ?? null });
|
|
244227
|
+
}
|
|
244228
|
+
return out;
|
|
244229
|
+
};
|
|
244230
|
+
const gather = async (actorId, primary) => {
|
|
244231
|
+
const [binding, candidateRows, nodeStates, faultRows, quotaRows] = await Promise.all([
|
|
244232
|
+
ports.primaryBinding(actorId).catch(() => null),
|
|
244233
|
+
ports.candidates(actorId).catch(() => []),
|
|
244234
|
+
ports.nodeStates(),
|
|
244235
|
+
ports.faults().catch(() => []),
|
|
244236
|
+
ports.quotas().catch(() => [])
|
|
244237
|
+
]);
|
|
244238
|
+
return {
|
|
244239
|
+
candidates: flatten(binding, candidateRows, primary),
|
|
244240
|
+
nodes: new Map(nodeStates.map((n) => [n.nodeId, n])),
|
|
244241
|
+
faults: new Map(faultRows.map((f2) => [runtimeTargetKey(f2), f2])),
|
|
244242
|
+
quotas: new Map(quotaRows.map((q2) => [runtimeTargetKey(q2), q2]))
|
|
244243
|
+
};
|
|
244244
|
+
};
|
|
244245
|
+
return {
|
|
244246
|
+
/** 跑一次分配但**不抛错**(给需要自己决定怎么呈现失败的调用方,比如要落一条失败卡的那条路)。 */
|
|
244247
|
+
async tryAllocate(input) {
|
|
244248
|
+
const now = input.now ?? Date.now();
|
|
244249
|
+
const state = stateOf(input.unitOfWork, now);
|
|
244250
|
+
const gathered = await gather(input.actorId, input.primary);
|
|
244251
|
+
return allocateRuntime({
|
|
244252
|
+
candidates: gathered.candidates,
|
|
244253
|
+
nodes: gathered.nodes,
|
|
244254
|
+
faults: gathered.faults,
|
|
244255
|
+
quotas: gathered.quotas,
|
|
244256
|
+
alreadyTried: new Set(state.tried.keys()),
|
|
244257
|
+
triedDetail: state.tried
|
|
244258
|
+
}, now);
|
|
244259
|
+
},
|
|
244260
|
+
/**
|
|
244261
|
+
* 分配,选不出就抛 {@link RuntimeAllocationError}(经 `requireAllocation` 那一处——日后接排队改那里)。
|
|
244262
|
+
* 成功时记下「这件活这次派到哪」,供 {@link noteFailure} 回报。
|
|
244263
|
+
*/
|
|
244264
|
+
async allocateFor(input) {
|
|
244265
|
+
const now = input.now ?? Date.now();
|
|
244266
|
+
const state = stateOf(input.unitOfWork, now);
|
|
244267
|
+
const actorName = await ports.actorName?.(input.actorId).catch(() => null) ?? null;
|
|
244268
|
+
if (!canRetryAllocation(state.tried.size)) {
|
|
244269
|
+
const tried = [...state.tried.entries()].map(([key2, detail]) => {
|
|
244270
|
+
const [nodeId, runtimeKind] = key2.split("::");
|
|
244271
|
+
return { nodeId, runtimeKind, nodeName: nodeId, cause: "already-tried", detail };
|
|
244272
|
+
});
|
|
244273
|
+
throw new RuntimeAllocationError(
|
|
244274
|
+
"RUNTIME_ALLOCATION_ALL_UNAVAILABLE",
|
|
244275
|
+
`${input.what ? `${input.what}\uFF1A` : ""}${actorName ? `\u300C${actorName}\u300D` : "\u8FD9\u4F4D Agent"}\u7684\u8FD0\u884C\u65F6\u5DF2\u8FDE\u7EED\u6362\u4E86 ${RUNTIME_ALLOCATION_MAX_ATTEMPTS} \u6B21\u90FD\u6CA1\u8DD1\u8D77\u6765\uFF0C\u4E0D\u518D\u81EA\u52A8\u91CD\u8BD5\uFF1A${tried.map((t) => `${t.nodeName} \xB7 ${t.runtimeKind}\uFF08${t.detail}\uFF09`).join("\uFF1B")}\u3002\u8BF7\u5148\u4FEE\u597D\u5176\u4E2D\u4E00\u53F0\u673A\u5668\uFF0C\u6216\u5728\u5458\u5DE5\u8BE6\u60C5\u9875\u52A0\u4E00\u6761\u522B\u7684\u5019\u9009\u8FD0\u884C\u65F6\u540E\u91CD\u65B0\u53D1\u8D77\u3002`,
|
|
244276
|
+
tried
|
|
244277
|
+
);
|
|
244278
|
+
}
|
|
244279
|
+
const result = requireAllocation(
|
|
244280
|
+
await this.tryAllocate({ actorId: input.actorId, unitOfWork: input.unitOfWork, ...input.primary !== void 0 ? { primary: input.primary } : {}, now }),
|
|
244281
|
+
{ actorName, ...input.what ? { what: input.what } : {} }
|
|
244282
|
+
);
|
|
244283
|
+
const attempt = state.tried.size + 1;
|
|
244284
|
+
state.lastAssigned = { nodeId: result.chosen.nodeId, runtimeKind: result.chosen.runtimeKind };
|
|
244285
|
+
state.updatedAtMs = now;
|
|
244286
|
+
const explanation = describeAllocationChoice(result);
|
|
244287
|
+
log3(`[runtime-alloc] ${input.unitOfWork} \u7B2C ${attempt} \u6B21\uFF1A${explanation}`);
|
|
244288
|
+
return {
|
|
244289
|
+
binding: { actorId: input.actorId, nodeId: result.chosen.nodeId, runtimeKind: result.chosen.runtimeKind, status: "active" },
|
|
244290
|
+
model: result.chosen.model ?? null,
|
|
244291
|
+
source: result.source,
|
|
244292
|
+
attempt,
|
|
244293
|
+
explanation
|
|
244294
|
+
};
|
|
244295
|
+
},
|
|
244296
|
+
/**
|
|
244297
|
+
* 这件活在上次分给它的那个目标上**因 runtime/节点原因**失败了。下一次 `allocateFor` 会排除它。
|
|
244298
|
+
*
|
|
244299
|
+
* 返回 false = 这条失败不算 runtime/节点的错(agent 自己干砸了),什么都没记。
|
|
244300
|
+
*/
|
|
244301
|
+
noteFailure(input) {
|
|
244302
|
+
if (!isRuntimeClassFailure(input.reason, input.errorMessage)) return false;
|
|
244303
|
+
const now = input.now ?? Date.now();
|
|
244304
|
+
const state = units.get(input.unitOfWork);
|
|
244305
|
+
const target = input.target ?? state?.lastAssigned;
|
|
244306
|
+
if (!state || !target) return false;
|
|
244307
|
+
const detail = input.errorMessage?.trim() || String(input.reason ?? "runtime \u5931\u8D25");
|
|
244308
|
+
state.tried.set(runtimeTargetKey(target), detail.slice(0, 200));
|
|
244309
|
+
state.updatedAtMs = now;
|
|
244310
|
+
log3(`[runtime-alloc] ${input.unitOfWork} \u5728 ${target.nodeId} \xB7 ${target.runtimeKind} \u4E0A\u5931\u8D25\uFF08${detail.slice(0, 80)}\uFF09\u2192 \u4E0B\u4E00\u8F6E\u6362\u5019\u9009`);
|
|
244311
|
+
return true;
|
|
244312
|
+
},
|
|
244313
|
+
/**
|
|
244314
|
+
* 这件活这一轮**不是因为 runtime/节点**结束的(跑成了、agent 自己干砸了、被停了)→ 丢掉整条失败链。
|
|
244315
|
+
*
|
|
244316
|
+
* 这是重试上限只对「连着换机器都没跑起来」生效的机制:正常派发一次就清一次,
|
|
244317
|
+
* 所以一个长命的工单节点不会因为派过很多次而撞上上限。
|
|
244318
|
+
*/
|
|
244319
|
+
noteSettled(unitOfWork) {
|
|
244320
|
+
units.delete(unitOfWork);
|
|
244321
|
+
},
|
|
244322
|
+
/**
|
|
244323
|
+
* 一次失败的**完整处置**(ADR 0616 D8):记进这件活的失败链(下一次换候选)**并**给这个目标
|
|
244324
|
+
* 落一条故障行(跨会话生效——别的活也别再往这台机器上派)。
|
|
244325
|
+
*
|
|
244326
|
+
* 两件事必须同一处做:只记失败链的话,这台机器的坏只对「这一件活」成立,下一件活照样一头撞上去;
|
|
244327
|
+
* 只落故障行的话,这一件活的重试计数没人管,撞满上限也不会停。
|
|
244328
|
+
*
|
|
244329
|
+
* 故障落库失败只告警、不抛:失败链已经记上了,换候选照常。
|
|
244330
|
+
*/
|
|
244331
|
+
async recordFailure(input) {
|
|
244332
|
+
const now = input.now ?? Date.now();
|
|
244333
|
+
const chained = this.noteFailure(input);
|
|
244334
|
+
const target = input.target ?? units.get(input.unitOfWork)?.lastAssigned;
|
|
244335
|
+
const reason = faultReasonOfExit(input.reason, input.errorMessage);
|
|
244336
|
+
if (!target || !reason || !ports.saveFault) return { chained, fault: null };
|
|
244337
|
+
try {
|
|
244338
|
+
const previous3 = await ports.getFault?.(target) ?? null;
|
|
244339
|
+
const fault = faultAfterFailure(previous3, {
|
|
244340
|
+
...target,
|
|
244341
|
+
reason,
|
|
244342
|
+
...input.errorMessage?.trim() ? { detail: input.errorMessage.trim() } : {}
|
|
244343
|
+
}, now);
|
|
244344
|
+
await ports.saveFault(fault);
|
|
244345
|
+
log3(`[runtime-alloc] ${describeFault(fault, now)}`);
|
|
244346
|
+
return { chained, fault };
|
|
244347
|
+
} catch (err) {
|
|
244348
|
+
log3(`[runtime-alloc] \u6545\u969C\u843D\u5E93\u5931\u8D25\uFF08\u6362\u5019\u9009\u7167\u5E38\uFF09${target.nodeId}\xB7${target.runtimeKind}\uFF1A${String(err)}`);
|
|
244349
|
+
return { chained, fault: null };
|
|
244350
|
+
}
|
|
244351
|
+
},
|
|
244352
|
+
/**
|
|
244353
|
+
* 这件活这一轮**不是因为 runtime/节点**结束的 → 丢掉失败链,并把上次那个目标的故障行解除掉。
|
|
244354
|
+
*
|
|
244355
|
+
* 「跑成了」就是半开试探要的那个答案:到点放一条新会话过去试、它成了 → 解除。这条路与只读采集
|
|
244356
|
+
* 那条(额度/登录回 ok 自动解除)并列,覆盖的正是**采集看不出来**的那一类。
|
|
244357
|
+
*/
|
|
244358
|
+
async recordSettled(unitOfWork, target) {
|
|
244359
|
+
const resolved = target ?? units.get(unitOfWork)?.lastAssigned;
|
|
244360
|
+
this.noteSettled(unitOfWork);
|
|
244361
|
+
if (!resolved || !ports.clearFault) return;
|
|
244362
|
+
try {
|
|
244363
|
+
await ports.clearFault(resolved);
|
|
244364
|
+
} catch (err) {
|
|
244365
|
+
log3(`[runtime-alloc] \u6545\u969C\u89E3\u9664\u5931\u8D25 ${resolved.nodeId}\xB7${resolved.runtimeKind}\uFF1A${String(err)}`);
|
|
244366
|
+
}
|
|
244367
|
+
},
|
|
244368
|
+
/** 只给测试与诊断用:这件活当前失败链的账。 */
|
|
244369
|
+
inspect(unitOfWork) {
|
|
244370
|
+
const s2 = units.get(unitOfWork);
|
|
244371
|
+
if (!s2) return null;
|
|
244372
|
+
return { failures: s2.tried.size, tried: [...s2.tried.keys()], ...s2.lastAssigned ? { lastAssigned: s2.lastAssigned } : {} };
|
|
244373
|
+
}
|
|
244374
|
+
};
|
|
244375
|
+
}
|
|
244376
|
+
var UNIT_STATE_TTL_MS;
|
|
244377
|
+
var init_service7 = __esm({
|
|
244378
|
+
"../server/src/domains/runtime-allocation/service.ts"() {
|
|
244379
|
+
"use strict";
|
|
244380
|
+
init_src();
|
|
244381
|
+
init_allocate();
|
|
244382
|
+
init_faults();
|
|
244383
|
+
init_errors6();
|
|
244384
|
+
UNIT_STATE_TTL_MS = 2 * 60 * 6e4;
|
|
244385
|
+
}
|
|
244386
|
+
});
|
|
244387
|
+
|
|
243709
244388
|
// ../server/src/domains/types/index.ts
|
|
243710
244389
|
function typesDomain(deps) {
|
|
243711
244390
|
const adminOf = async (companyId) => {
|
|
@@ -243829,6 +244508,10 @@ var init_node_store = __esm({
|
|
|
243829
244508
|
MemoryNodeStore = class {
|
|
243830
244509
|
nodes = /* @__PURE__ */ new Map();
|
|
243831
244510
|
runtimes = /* @__PURE__ */ new Map();
|
|
244511
|
+
/** 故障标记,键 = `${nodeId}::${runtimeKind}`(与 `runtimeTargetKey` 同一个键形状)。 */
|
|
244512
|
+
faults = /* @__PURE__ */ new Map();
|
|
244513
|
+
/** 落库额度,键同上。 */
|
|
244514
|
+
quotas = /* @__PURE__ */ new Map();
|
|
243832
244515
|
async upsertNode(node2) {
|
|
243833
244516
|
this.nodes.set(node2.id, { ...this.nodes.get(node2.id), ...node2 });
|
|
243834
244517
|
}
|
|
@@ -243941,6 +244624,27 @@ var init_node_store = __esm({
|
|
|
243941
244624
|
}
|
|
243942
244625
|
return { nodeDeleted: false };
|
|
243943
244626
|
}
|
|
244627
|
+
/* ---------- 运行时分配(ADR 0616) ---------- */
|
|
244628
|
+
async setNodeMaxConcurrent(nodeId, limit) {
|
|
244629
|
+
const n = this.nodes.get(nodeId);
|
|
244630
|
+
if (n) n.maxConcurrent = limit;
|
|
244631
|
+
}
|
|
244632
|
+
async upsertRuntimeFault(fault) {
|
|
244633
|
+
this.faults.set(`${fault.nodeId}::${fault.runtimeKind}`, { ...fault });
|
|
244634
|
+
}
|
|
244635
|
+
async clearRuntimeFault(nodeId, runtimeKind) {
|
|
244636
|
+
return this.faults.delete(`${nodeId}::${runtimeKind}`);
|
|
244637
|
+
}
|
|
244638
|
+
async listRuntimeFaults(filter) {
|
|
244639
|
+
return [...this.faults.values()].filter((f2) => filter?.nodeId ? f2.nodeId === filter.nodeId : true).filter((f2) => filter?.companyId ? this.nodes.get(f2.nodeId)?.companyId === filter.companyId : true).map((f2) => ({ ...f2 }));
|
|
244640
|
+
}
|
|
244641
|
+
async saveRuntimeQuotas(nodeId, quotas) {
|
|
244642
|
+
for (const key2 of [...this.quotas.keys()]) if (key2.startsWith(`${nodeId}::`)) this.quotas.delete(key2);
|
|
244643
|
+
for (const q2 of quotas) this.quotas.set(`${q2.nodeId}::${q2.runtimeKind}`, { ...q2 });
|
|
244644
|
+
}
|
|
244645
|
+
async listRuntimeQuotas(filter) {
|
|
244646
|
+
return [...this.quotas.values()].filter((q2) => filter?.nodeId ? q2.nodeId === filter.nodeId : true).filter((q2) => filter?.companyId ? this.nodes.get(q2.nodeId)?.companyId === filter.companyId : true).map((q2) => ({ ...q2 }));
|
|
244647
|
+
}
|
|
243944
244648
|
};
|
|
243945
244649
|
FileNodeStore = class _FileNodeStore extends MemoryNodeStore {
|
|
243946
244650
|
constructor(file) {
|
|
@@ -243953,15 +244657,19 @@ var init_node_store = __esm({
|
|
|
243953
244657
|
const snap = JSON.parse(fs26.readFileSync(file, "utf8"));
|
|
243954
244658
|
for (const n of snap.nodes ?? []) s2.nodes.set(n.id, n);
|
|
243955
244659
|
for (const r of snap.runtimes ?? []) s2.runtimes.set(r.id, r);
|
|
244660
|
+
for (const f2 of snap.faults ?? []) s2.faults.set(`${f2.nodeId}::${f2.runtimeKind}`, f2);
|
|
244661
|
+
for (const q2 of snap.quotas ?? []) s2.quotas.set(`${q2.nodeId}::${q2.runtimeKind}`, q2);
|
|
243956
244662
|
} catch {
|
|
243957
|
-
fs26.writeFileSync(file, JSON.stringify({ nodes: [], runtimes: [] }, null, 2));
|
|
244663
|
+
fs26.writeFileSync(file, JSON.stringify({ nodes: [], runtimes: [], faults: [], quotas: [] }, null, 2));
|
|
243958
244664
|
}
|
|
243959
244665
|
return s2;
|
|
243960
244666
|
}
|
|
243961
244667
|
flush() {
|
|
243962
244668
|
fs26.writeFileSync(this.file, JSON.stringify({
|
|
243963
244669
|
nodes: [...this.nodes.values()],
|
|
243964
|
-
runtimes: [...this.runtimes.values()]
|
|
244670
|
+
runtimes: [...this.runtimes.values()],
|
|
244671
|
+
faults: [...this.faults.values()],
|
|
244672
|
+
quotas: [...this.quotas.values()]
|
|
243965
244673
|
}, null, 2));
|
|
243966
244674
|
}
|
|
243967
244675
|
async upsertNode(node2) {
|
|
@@ -244025,6 +244733,25 @@ var init_node_store = __esm({
|
|
|
244025
244733
|
this.flush();
|
|
244026
244734
|
return r;
|
|
244027
244735
|
}
|
|
244736
|
+
// ⚠ 同上:并发上限、故障标记、额度三类写入漏了 override 就只活在内存,重启回到「没上限、没故障、
|
|
244737
|
+
// 没额度数据」——而那三件事恰恰都是「重启不能丢」才有意义。
|
|
244738
|
+
async setNodeMaxConcurrent(nodeId, limit) {
|
|
244739
|
+
await super.setNodeMaxConcurrent(nodeId, limit);
|
|
244740
|
+
this.flush();
|
|
244741
|
+
}
|
|
244742
|
+
async upsertRuntimeFault(fault) {
|
|
244743
|
+
await super.upsertRuntimeFault(fault);
|
|
244744
|
+
this.flush();
|
|
244745
|
+
}
|
|
244746
|
+
async clearRuntimeFault(nodeId, runtimeKind) {
|
|
244747
|
+
const r = await super.clearRuntimeFault(nodeId, runtimeKind);
|
|
244748
|
+
if (r) this.flush();
|
|
244749
|
+
return r;
|
|
244750
|
+
}
|
|
244751
|
+
async saveRuntimeQuotas(nodeId, quotas) {
|
|
244752
|
+
await super.saveRuntimeQuotas(nodeId, quotas);
|
|
244753
|
+
this.flush();
|
|
244754
|
+
}
|
|
244028
244755
|
};
|
|
244029
244756
|
}
|
|
244030
244757
|
});
|
|
@@ -245298,7 +246025,7 @@ function compactModelChunks(rows) {
|
|
|
245298
246025
|
return out;
|
|
245299
246026
|
}
|
|
245300
246027
|
var EVENT_PAGE_LIMIT, MODEL_CHUNK_COMPACT_WINDOW_MS, MODEL_CHUNK_RAW_MULTIPLIER, TraceService;
|
|
245301
|
-
var
|
|
246028
|
+
var init_service8 = __esm({
|
|
245302
246029
|
"../server/src/domains/trace/service.ts"() {
|
|
245303
246030
|
"use strict";
|
|
245304
246031
|
init_sink();
|
|
@@ -246074,9 +246801,9 @@ function createTraceDomain(opts) {
|
|
|
246074
246801
|
var init_trace2 = __esm({
|
|
246075
246802
|
"../server/src/domains/trace/index.ts"() {
|
|
246076
246803
|
"use strict";
|
|
246077
|
-
|
|
246804
|
+
init_service8();
|
|
246078
246805
|
init_routes9();
|
|
246079
|
-
|
|
246806
|
+
init_service8();
|
|
246080
246807
|
init_routes9();
|
|
246081
246808
|
init_read_scope();
|
|
246082
246809
|
init_sink();
|
|
@@ -246609,7 +247336,7 @@ function stateFromAttempt(attempt, handoff, checkpoint, now, stalledAfterMs, ope
|
|
|
246609
247336
|
return "failed";
|
|
246610
247337
|
}
|
|
246611
247338
|
var import_node_crypto64, BLOCKING_STATES, HANDOFF_OUTCOMES2, RESUME_EXTERNAL_EFFECT_LIMIT, RESUME_SNAPSHOT_PREFIX, RESUME_SNAPSHOT_SUFFIX, ExecutionContinuityService;
|
|
246612
|
-
var
|
|
247339
|
+
var init_service9 = __esm({
|
|
246613
247340
|
"../server/src/domains/execution-continuity/service.ts"() {
|
|
246614
247341
|
"use strict";
|
|
246615
247342
|
init_src();
|
|
@@ -247429,8 +248156,8 @@ var init_execution_continuity2 = __esm({
|
|
|
247429
248156
|
"../server/src/domains/execution-continuity/index.ts"() {
|
|
247430
248157
|
"use strict";
|
|
247431
248158
|
init_routes10();
|
|
247432
|
-
|
|
247433
|
-
|
|
248159
|
+
init_service9();
|
|
248160
|
+
init_service9();
|
|
247434
248161
|
init_routes10();
|
|
247435
248162
|
init_handoff_work_bridge();
|
|
247436
248163
|
init_upstream_handoff_input();
|
|
@@ -250517,7 +251244,7 @@ function appendSourceNote(brief, automation, source, atIso) {
|
|
|
250517
251244
|
\u672C\u5DE5\u5355\u7531\u81EA\u52A8\u5316\u300C${automation.name}\u300D\u4E8E ${atIso} \u89E6\u53D1\uFF08\u6765\u6E90\uFF1A${source}\uFF09\u3002\u5F00\u5DE5\u540E\u8BF7\u628A\u5DE5\u5355\u6807\u9898\u6539\u6210\u51C6\u786E\u53CD\u6620\u5B9E\u9645\u5DE5\u4F5C\u7684\u63CF\u8FF0\u3002`;
|
|
250518
251245
|
}
|
|
250519
251246
|
var import_node_crypto66, DATE_TOKEN, AutomationsService, EVENT_RETRY_BASE_MS, EVENT_RETRY_MAX_MS, EVENT_RETRY_MAX_ATTEMPTS, eventAttempts, eventLastAttemptAt, eventLastError, CHAT_REPLY_MAX;
|
|
250520
|
-
var
|
|
251247
|
+
var init_service10 = __esm({
|
|
250521
251248
|
"../server/src/domains/automations/service.ts"() {
|
|
250522
251249
|
"use strict";
|
|
250523
251250
|
import_node_crypto66 = require("node:crypto");
|
|
@@ -252252,7 +252979,7 @@ var init_routes11 = __esm({
|
|
|
252252
252979
|
init_src();
|
|
252253
252980
|
init_templates();
|
|
252254
252981
|
init_router();
|
|
252255
|
-
|
|
252982
|
+
init_service10();
|
|
252256
252983
|
init_projection2();
|
|
252257
252984
|
init_scheduler();
|
|
252258
252985
|
init_registry3();
|
|
@@ -252367,10 +253094,10 @@ function createAutomationsDomain(opts) {
|
|
|
252367
253094
|
var init_automations = __esm({
|
|
252368
253095
|
"../server/src/domains/automations/index.ts"() {
|
|
252369
253096
|
"use strict";
|
|
252370
|
-
|
|
253097
|
+
init_service10();
|
|
252371
253098
|
init_webhook();
|
|
252372
253099
|
init_routes11();
|
|
252373
|
-
|
|
253100
|
+
init_service10();
|
|
252374
253101
|
init_routes11();
|
|
252375
253102
|
init_projection2();
|
|
252376
253103
|
init_webhook();
|
|
@@ -253186,6 +253913,32 @@ var init_daemon_adapter = __esm({
|
|
|
253186
253913
|
}
|
|
253187
253914
|
}
|
|
253188
253915
|
}
|
|
253916
|
+
/**
|
|
253917
|
+
* Bug 0232:**就地**收尾一次派发——用户点了停止、而 kill 帧送不出去(节点 socket 非 OPEN)时用。
|
|
253918
|
+
*
|
|
253919
|
+
* 走的是和「服务端自判死」完全同一个 {@link settle},所以三件必须做的事一次做齐:
|
|
253920
|
+
* 兑现在途的插话等待者、触发 exitCbs(本轮收口、释放 `chat_session_turns` 的槽、输入框解冻)、
|
|
253921
|
+
* 把 dispatchId 记进 `settledDispatchIds`。
|
|
253922
|
+
*
|
|
253923
|
+
* 为什么这样不会造出两个 agent 一起跑(防双跑的三道防线都是现成的):
|
|
253924
|
+
* 1. 节点上的会话进程自己有看门狗——连不上服务端超过 `SESSION_LIMITS.suicideAfterUnreachableMs`
|
|
253925
|
+
* (5 分钟)就自尽(`packages/cli/src/daemon/session-process.ts`);
|
|
253926
|
+
* 2. 契约上钉着不变式 `suicideAfterUnreachableMs ≤ reapGraceMs`,服务端判死必然发生在它自尽之后;
|
|
253927
|
+
* 3. 它要是在这之前连回来了,hub 的三级认领(`daemon-hub.ts` 的 `judgeSession`)第一级就命中
|
|
253928
|
+
* `settledDispatchIds` → 判 `settled` → 下发 `evict` → 会话 `selfTerminate("evicted")`。
|
|
253929
|
+
* 所以僵尸的存活时间是 `min(网络恢复时间, 5 分钟)`,这个窗口自己会关。
|
|
253930
|
+
*
|
|
253931
|
+
* 归因不在这里写:`cancelCause` 在点停止那一刻就由 `LiveChatRegistry.stop` 写成了 `user`,
|
|
253932
|
+
* 收尾时读出来,页面照旧显示「用户停止了输出」。
|
|
253933
|
+
*
|
|
253934
|
+
* @returns true = 确由这次调用收的尾;false = 这次派发不在在途表里(已收尾 / 不归本 adapter 管)。
|
|
253935
|
+
*/
|
|
253936
|
+
cancelLocally(dispatchId, errorMessage2 = "\u7528\u6237\u505C\u6B62\uFF08\u8282\u70B9\u79BB\u7EBF\uFF0C\u672A\u7B49\u5230\u9000\u51FA\u5E27\uFF09") {
|
|
253937
|
+
const entry = this.pending.get(dispatchId);
|
|
253938
|
+
if (!entry || entry.exited) return false;
|
|
253939
|
+
this.settle(dispatchId, { code: null, reason: "cancelled", errorMessage: errorMessage2 });
|
|
253940
|
+
return true;
|
|
253941
|
+
}
|
|
253189
253942
|
/** 当前在途(未退出)会话数;传 runtimeKind 时收窄到单个 runtime 实例。 */
|
|
253190
253943
|
activeRunCount(nodeId, runtimeKind) {
|
|
253191
253944
|
let n = 0;
|
|
@@ -253435,14 +254188,18 @@ var init_daemon_adapter = __esm({
|
|
|
253435
254188
|
const appendTimeoutMs = this.appendTimeoutMs;
|
|
253436
254189
|
const sessionReadyWaitMs = this.sessionReadyWaitMs;
|
|
253437
254190
|
const confirm = (key2) => this.confirmInputReceipt(nodeId, dispatchId, key2);
|
|
254191
|
+
const sendKill = () => routeToSession(hub, nodeId, dispatchId, { type: "kill", dispatchId });
|
|
254192
|
+
const cancelLocally = () => this.cancelLocally(dispatchId);
|
|
253438
254193
|
return {
|
|
253439
254194
|
id: dispatchId,
|
|
253440
254195
|
get supportsLiveProtocol() {
|
|
253441
254196
|
return entry.supportsLiveProtocol;
|
|
253442
254197
|
},
|
|
253443
254198
|
async kill() {
|
|
253444
|
-
|
|
254199
|
+
sendKill();
|
|
253445
254200
|
},
|
|
254201
|
+
sendKill,
|
|
254202
|
+
cancelLocally,
|
|
253446
254203
|
onExit(cb) {
|
|
253447
254204
|
if (entry.exited) cb(entry.exited);
|
|
253448
254205
|
else entry.exitCbs.push(cb);
|
|
@@ -253527,6 +254284,8 @@ var init_daemon_adapter = __esm({
|
|
|
253527
254284
|
const appendTimeoutMs = this.appendTimeoutMs;
|
|
253528
254285
|
const sessionReadyWaitMs = this.sessionReadyWaitMs;
|
|
253529
254286
|
const confirm = (key2) => this.confirmInputReceipt(nodeId, dispatchId, key2);
|
|
254287
|
+
const sendKill = () => routeToSession(hub, nodeId, dispatchId, { type: "kill", dispatchId });
|
|
254288
|
+
const cancelLocally = () => this.cancelLocally(dispatchId);
|
|
253530
254289
|
return {
|
|
253531
254290
|
id: dispatchId,
|
|
253532
254291
|
stashDroppedFrames: stashed.droppedFrames ?? 0,
|
|
@@ -253534,8 +254293,10 @@ var init_daemon_adapter = __esm({
|
|
|
253534
254293
|
return entry.supportsLiveProtocol;
|
|
253535
254294
|
},
|
|
253536
254295
|
async kill() {
|
|
253537
|
-
|
|
254296
|
+
sendKill();
|
|
253538
254297
|
},
|
|
254298
|
+
sendKill,
|
|
254299
|
+
cancelLocally,
|
|
253539
254300
|
onExit(cb) {
|
|
253540
254301
|
if (entry.exited) cb(entry.exited);
|
|
253541
254302
|
else entry.exitCbs.push(cb);
|
|
@@ -255705,6 +256466,18 @@ var init_postgres_registry = __esm({
|
|
|
255705
256466
|
status text NOT NULL -- active | paused
|
|
255706
256467
|
)`);
|
|
255707
256468
|
await pool.query(`
|
|
256469
|
+
CREATE TABLE IF NOT EXISTS "${s2}".actor_runtime_candidates (
|
|
256470
|
+
actor_id text NOT NULL,
|
|
256471
|
+
node_id text NOT NULL,
|
|
256472
|
+
runtime_kind text NOT NULL,
|
|
256473
|
+
model text,
|
|
256474
|
+
ord integer NOT NULL,
|
|
256475
|
+
updated_at timestamptz NOT NULL,
|
|
256476
|
+
updated_by text NOT NULL,
|
|
256477
|
+
PRIMARY KEY (actor_id, node_id, runtime_kind)
|
|
256478
|
+
)`);
|
|
256479
|
+
await pool.query(`CREATE INDEX IF NOT EXISTS actor_runtime_candidates_actor_ord_idx ON "${s2}".actor_runtime_candidates (actor_id, ord)`);
|
|
256480
|
+
await pool.query(`
|
|
255708
256481
|
CREATE TABLE IF NOT EXISTS "${s2}".connectors (
|
|
255709
256482
|
id text PRIMARY KEY,
|
|
255710
256483
|
name text NOT NULL,
|
|
@@ -256062,6 +256835,49 @@ var init_postgres_registry = __esm({
|
|
|
256062
256835
|
const r = await this.pool.query(`SELECT * FROM ${this.s}.actor_bindings ORDER BY actor_id`);
|
|
256063
256836
|
return r.rows.map((row) => ({ actorId: row.actor_id, nodeId: row.node_id, runtimeKind: row.runtime_kind, status: row.status }));
|
|
256064
256837
|
}
|
|
256838
|
+
/* ---------- 候选 runtime(ADR 0616) ---------- */
|
|
256839
|
+
async listRuntimeCandidates(actorId) {
|
|
256840
|
+
const r = await this.pool.query(
|
|
256841
|
+
// 第二、三排序键让同 ord 的存量脏数据也有**确定**次序——分配算法的定序是全序的,
|
|
256842
|
+
// 读侧若随机返回,那句「为什么选了它」下一次就解释不了同一件事。
|
|
256843
|
+
`SELECT actor_id, node_id, runtime_kind, model, ord FROM ${this.s}.actor_runtime_candidates
|
|
256844
|
+
WHERE actor_id = $1 ORDER BY ord, node_id, runtime_kind`,
|
|
256845
|
+
[actorId]
|
|
256846
|
+
);
|
|
256847
|
+
return r.rows.map((row) => ({
|
|
256848
|
+
actorId: row.actor_id,
|
|
256849
|
+
nodeId: row.node_id,
|
|
256850
|
+
runtimeKind: row.runtime_kind,
|
|
256851
|
+
model: row.model ?? null,
|
|
256852
|
+
ord: Number(row.ord)
|
|
256853
|
+
}));
|
|
256854
|
+
}
|
|
256855
|
+
async replaceRuntimeCandidates(actorId, candidates, by) {
|
|
256856
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
256857
|
+
const updatedBy = by ?? "actor:system:registry";
|
|
256858
|
+
const client = await this.pool.connect();
|
|
256859
|
+
try {
|
|
256860
|
+
await client.query("BEGIN");
|
|
256861
|
+
await client.query(`DELETE FROM ${this.s}.actor_runtime_candidates WHERE actor_id = $1`, [actorId]);
|
|
256862
|
+
for (const [i, c] of candidates.entries()) {
|
|
256863
|
+
const model = typeof c.model === "string" && c.model.trim() ? c.model.trim() : null;
|
|
256864
|
+
await client.query(
|
|
256865
|
+
`INSERT INTO ${this.s}.actor_runtime_candidates (actor_id, node_id, runtime_kind, model, ord, updated_at, updated_by)
|
|
256866
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
|
256867
|
+
ON CONFLICT (actor_id, node_id, runtime_kind) DO UPDATE SET model=$4, ord=$5, updated_at=$6, updated_by=$7`,
|
|
256868
|
+
[actorId, c.nodeId, c.runtimeKind, model, i, now, updatedBy]
|
|
256869
|
+
);
|
|
256870
|
+
}
|
|
256871
|
+
await client.query("COMMIT");
|
|
256872
|
+
} catch (err) {
|
|
256873
|
+
await client.query("ROLLBACK").catch(() => {
|
|
256874
|
+
});
|
|
256875
|
+
throw err;
|
|
256876
|
+
} finally {
|
|
256877
|
+
client.release();
|
|
256878
|
+
}
|
|
256879
|
+
return this.listRuntimeCandidates(actorId);
|
|
256880
|
+
}
|
|
256065
256881
|
/* ---------- Runtime 默认配置 ---------- */
|
|
256066
256882
|
async upsertRuntimeConfig(c) {
|
|
256067
256883
|
await this.pool.query(
|
|
@@ -265205,7 +266021,7 @@ var init_postgres_delegations = __esm({
|
|
|
265205
266021
|
});
|
|
265206
266022
|
|
|
265207
266023
|
// ../storage/src/postgres-nodes.ts
|
|
265208
|
-
var import_node_crypto75, ident13, PostgresNodeStore, PostgresNodeTokenStore, rowToNode, rowToRuntime;
|
|
266024
|
+
var import_node_crypto75, ident13, PostgresNodeStore, PostgresNodeTokenStore, rowToNode, rowToFault, rowToQuota, rowToRuntime;
|
|
265209
266025
|
var init_postgres_nodes = __esm({
|
|
265210
266026
|
"../storage/src/postgres-nodes.ts"() {
|
|
265211
266027
|
"use strict";
|
|
@@ -265255,6 +266071,33 @@ var init_postgres_nodes = __esm({
|
|
|
265255
266071
|
await pool.query(`CREATE UNIQUE INDEX IF NOT EXISTS node_runtimes_company_id_uq ON "${s2}".node_runtimes (company_id, id)`);
|
|
265256
266072
|
await pool.query(`CREATE INDEX IF NOT EXISTS node_runtimes_company_node_idx ON "${s2}".node_runtimes (company_id, node_id)`);
|
|
265257
266073
|
await pool.query(`CREATE INDEX IF NOT EXISTS node_runtimes_company_status_idx ON "${s2}".node_runtimes (company_id, status)`);
|
|
266074
|
+
await pool.query(`ALTER TABLE "${s2}".nodes ADD COLUMN IF NOT EXISTS max_concurrent integer`);
|
|
266075
|
+
await pool.query(`
|
|
266076
|
+
CREATE TABLE IF NOT EXISTS "${s2}".node_runtime_faults (
|
|
266077
|
+
node_id text NOT NULL REFERENCES "${s2}".nodes(id) ON DELETE CASCADE,
|
|
266078
|
+
runtime_kind text NOT NULL,
|
|
266079
|
+
company_id text,
|
|
266080
|
+
reason text NOT NULL,
|
|
266081
|
+
detail text,
|
|
266082
|
+
marked_at timestamptz NOT NULL,
|
|
266083
|
+
attempts integer NOT NULL DEFAULT 0,
|
|
266084
|
+
next_probe_at timestamptz,
|
|
266085
|
+
awaiting_manual boolean NOT NULL DEFAULT false,
|
|
266086
|
+
PRIMARY KEY (node_id, runtime_kind)
|
|
266087
|
+
)`);
|
|
266088
|
+
await pool.query(`CREATE INDEX IF NOT EXISTS node_runtime_faults_company_idx ON "${s2}".node_runtime_faults (company_id)`);
|
|
266089
|
+
await pool.query(`
|
|
266090
|
+
CREATE TABLE IF NOT EXISTS "${s2}".node_runtime_quotas (
|
|
266091
|
+
node_id text NOT NULL REFERENCES "${s2}".nodes(id) ON DELETE CASCADE,
|
|
266092
|
+
runtime_kind text NOT NULL,
|
|
266093
|
+
company_id text,
|
|
266094
|
+
status text NOT NULL,
|
|
266095
|
+
checked_at timestamptz NOT NULL,
|
|
266096
|
+
retry_at timestamptz,
|
|
266097
|
+
windows jsonb NOT NULL DEFAULT '[]'::jsonb,
|
|
266098
|
+
PRIMARY KEY (node_id, runtime_kind)
|
|
266099
|
+
)`);
|
|
266100
|
+
await pool.query(`CREATE INDEX IF NOT EXISTS node_runtime_quotas_company_idx ON "${s2}".node_runtime_quotas (company_id)`);
|
|
265258
266101
|
return new _PostgresNodeStore(pool, schema);
|
|
265259
266102
|
}
|
|
265260
266103
|
async upsertNode(n) {
|
|
@@ -265408,6 +266251,95 @@ var init_postgres_nodes = __esm({
|
|
|
265408
266251
|
}
|
|
265409
266252
|
return { nodeDeleted: false };
|
|
265410
266253
|
}
|
|
266254
|
+
/* ---------- 运行时分配(ADR 0616):并发上限 / 故障标记 / 落库额度 ---------- */
|
|
266255
|
+
async setNodeMaxConcurrent(nodeId, limit) {
|
|
266256
|
+
await this.pool.query(`UPDATE ${this.s}.nodes SET max_concurrent=$2 WHERE id=$1`, [nodeId, limit]);
|
|
266257
|
+
}
|
|
266258
|
+
async upsertRuntimeFault(fault) {
|
|
266259
|
+
await this.pool.query(
|
|
266260
|
+
// company_id 从 node 行继承(node 是唯一事实源,同 upsertRuntime 的兜底),任何调用方都绕不过去。
|
|
266261
|
+
`INSERT INTO ${this.s}.node_runtime_faults
|
|
266262
|
+
(node_id, runtime_kind, company_id, reason, detail, marked_at, attempts, next_probe_at, awaiting_manual)
|
|
266263
|
+
VALUES ($1,$2,(SELECT company_id FROM ${this.s}.nodes WHERE id=$1),$3,$4,$5,$6,$7,$8)
|
|
266264
|
+
ON CONFLICT (node_id, runtime_kind) DO UPDATE SET
|
|
266265
|
+
company_id = COALESCE(EXCLUDED.company_id, ${this.s}.node_runtime_faults.company_id),
|
|
266266
|
+
reason=EXCLUDED.reason, detail=EXCLUDED.detail, marked_at=EXCLUDED.marked_at,
|
|
266267
|
+
attempts=EXCLUDED.attempts, next_probe_at=EXCLUDED.next_probe_at, awaiting_manual=EXCLUDED.awaiting_manual`,
|
|
266268
|
+
[
|
|
266269
|
+
fault.nodeId,
|
|
266270
|
+
fault.runtimeKind,
|
|
266271
|
+
fault.reason,
|
|
266272
|
+
fault.detail ?? null,
|
|
266273
|
+
fault.markedAt,
|
|
266274
|
+
fault.attempts,
|
|
266275
|
+
fault.nextProbeAt ?? null,
|
|
266276
|
+
fault.awaitingManual
|
|
266277
|
+
]
|
|
266278
|
+
);
|
|
266279
|
+
}
|
|
266280
|
+
async clearRuntimeFault(nodeId, runtimeKind) {
|
|
266281
|
+
const r = await this.pool.query(
|
|
266282
|
+
`DELETE FROM ${this.s}.node_runtime_faults WHERE node_id=$1 AND runtime_kind=$2`,
|
|
266283
|
+
[nodeId, runtimeKind]
|
|
266284
|
+
);
|
|
266285
|
+
return (r.rowCount ?? 0) > 0;
|
|
266286
|
+
}
|
|
266287
|
+
async listRuntimeFaults(filter) {
|
|
266288
|
+
const where = [];
|
|
266289
|
+
const args = [];
|
|
266290
|
+
if (filter?.nodeId) {
|
|
266291
|
+
args.push(filter.nodeId);
|
|
266292
|
+
where.push(`node_id = $${args.length}`);
|
|
266293
|
+
}
|
|
266294
|
+
if (filter?.companyId) {
|
|
266295
|
+
args.push(filter.companyId);
|
|
266296
|
+
where.push(`company_id = $${args.length}`);
|
|
266297
|
+
}
|
|
266298
|
+
const r = await this.pool.query(
|
|
266299
|
+
`SELECT * FROM ${this.s}.node_runtime_faults${where.length ? ` WHERE ${where.join(" AND ")}` : ""} ORDER BY node_id, runtime_kind`,
|
|
266300
|
+
args
|
|
266301
|
+
);
|
|
266302
|
+
return r.rows.map(rowToFault);
|
|
266303
|
+
}
|
|
266304
|
+
async saveRuntimeQuotas(nodeId, quotas) {
|
|
266305
|
+
const client = await this.pool.connect();
|
|
266306
|
+
try {
|
|
266307
|
+
await client.query("BEGIN");
|
|
266308
|
+
await client.query(`DELETE FROM ${this.s}.node_runtime_quotas WHERE node_id=$1`, [nodeId]);
|
|
266309
|
+
for (const q2 of quotas) {
|
|
266310
|
+
await client.query(
|
|
266311
|
+
`INSERT INTO ${this.s}.node_runtime_quotas
|
|
266312
|
+
(node_id, runtime_kind, company_id, status, checked_at, retry_at, windows)
|
|
266313
|
+
VALUES ($1,$2,(SELECT company_id FROM ${this.s}.nodes WHERE id=$1),$3,$4,$5,$6::jsonb)`,
|
|
266314
|
+
[nodeId, q2.runtimeKind, q2.status, q2.checkedAt, q2.retryAt ?? null, JSON.stringify(q2.windows ?? [])]
|
|
266315
|
+
);
|
|
266316
|
+
}
|
|
266317
|
+
await client.query("COMMIT");
|
|
266318
|
+
} catch (err) {
|
|
266319
|
+
await client.query("ROLLBACK").catch(() => {
|
|
266320
|
+
});
|
|
266321
|
+
throw err;
|
|
266322
|
+
} finally {
|
|
266323
|
+
client.release();
|
|
266324
|
+
}
|
|
266325
|
+
}
|
|
266326
|
+
async listRuntimeQuotas(filter) {
|
|
266327
|
+
const where = [];
|
|
266328
|
+
const args = [];
|
|
266329
|
+
if (filter?.nodeId) {
|
|
266330
|
+
args.push(filter.nodeId);
|
|
266331
|
+
where.push(`node_id = $${args.length}`);
|
|
266332
|
+
}
|
|
266333
|
+
if (filter?.companyId) {
|
|
266334
|
+
args.push(filter.companyId);
|
|
266335
|
+
where.push(`company_id = $${args.length}`);
|
|
266336
|
+
}
|
|
266337
|
+
const r = await this.pool.query(
|
|
266338
|
+
`SELECT * FROM ${this.s}.node_runtime_quotas${where.length ? ` WHERE ${where.join(" AND ")}` : ""} ORDER BY node_id, runtime_kind`,
|
|
266339
|
+
args
|
|
266340
|
+
);
|
|
266341
|
+
return r.rows.map(rowToQuota);
|
|
266342
|
+
}
|
|
265411
266343
|
};
|
|
265412
266344
|
PostgresNodeTokenStore = class _PostgresNodeTokenStore {
|
|
265413
266345
|
constructor(pool, schema) {
|
|
@@ -265450,7 +266382,28 @@ var init_postgres_nodes = __esm({
|
|
|
265450
266382
|
nodeVersion: row.node_version,
|
|
265451
266383
|
status: row.status,
|
|
265452
266384
|
lastSeenAt: new Date(row.last_seen_at).toISOString(),
|
|
265453
|
-
enrolledAt: new Date(row.enrolled_at).toISOString()
|
|
266385
|
+
enrolledAt: new Date(row.enrolled_at).toISOString(),
|
|
266386
|
+
// 只在库里真有值时带上这一格:`undefined` 与 `null` 在 `exactOptionalPropertyTypes` 下不等价,
|
|
266387
|
+
// 而判据只认「有没有上限」——带一个 null 进去会让下游不得不再判一次。
|
|
266388
|
+
...row.max_concurrent != null ? { maxConcurrent: Number(row.max_concurrent) } : {}
|
|
266389
|
+
});
|
|
266390
|
+
rowToFault = (row) => ({
|
|
266391
|
+
nodeId: row.node_id,
|
|
266392
|
+
runtimeKind: row.runtime_kind,
|
|
266393
|
+
reason: row.reason,
|
|
266394
|
+
...row.detail ? { detail: row.detail } : {},
|
|
266395
|
+
markedAt: new Date(row.marked_at).toISOString(),
|
|
266396
|
+
attempts: Number(row.attempts ?? 0),
|
|
266397
|
+
...row.next_probe_at ? { nextProbeAt: new Date(row.next_probe_at).toISOString() } : {},
|
|
266398
|
+
awaitingManual: Boolean(row.awaiting_manual)
|
|
266399
|
+
});
|
|
266400
|
+
rowToQuota = (row) => ({
|
|
266401
|
+
nodeId: row.node_id,
|
|
266402
|
+
runtimeKind: row.runtime_kind,
|
|
266403
|
+
status: row.status,
|
|
266404
|
+
checkedAt: new Date(row.checked_at).toISOString(),
|
|
266405
|
+
...row.retry_at ? { retryAt: new Date(row.retry_at).toISOString() } : {},
|
|
266406
|
+
windows: Array.isArray(row.windows) ? row.windows : []
|
|
265454
266407
|
});
|
|
265455
266408
|
rowToRuntime = (row) => ({
|
|
265456
266409
|
id: row.id,
|
|
@@ -268652,6 +269605,16 @@ var init_cutover_aware_registry = __esm({
|
|
|
268652
269605
|
async listBindings() {
|
|
268653
269606
|
return (await this.orgStore()).listBindings();
|
|
268654
269607
|
}
|
|
269608
|
+
// 候选 runtime(ADR 0616):与 binding 同一侧(组织面);底座没实现就退化成「没有候选」。
|
|
269609
|
+
async listRuntimeCandidates(actorId) {
|
|
269610
|
+
const store = await this.orgStore();
|
|
269611
|
+
return store.listRuntimeCandidates ? store.listRuntimeCandidates(actorId) : [];
|
|
269612
|
+
}
|
|
269613
|
+
async replaceRuntimeCandidates(actorId, candidates) {
|
|
269614
|
+
const store = await this.orgStoreForWrite();
|
|
269615
|
+
if (!store.replaceRuntimeCandidates) throw new Error("\u5F53\u524D\u767B\u8BB0\u5B58\u50A8\u5B9E\u73B0\u4E0D\u652F\u6301\u5019\u9009\u8FD0\u884C\u65F6\uFF08replaceRuntimeCandidates \u672A\u5B9E\u73B0\uFF09");
|
|
269616
|
+
return store.replaceRuntimeCandidates(actorId, candidates);
|
|
269617
|
+
}
|
|
268655
269618
|
async upsertRuntimeConfig(...args) {
|
|
268656
269619
|
return (await this.orgStoreForWrite()).upsertRuntimeConfig(...args);
|
|
268657
269620
|
}
|
|
@@ -274375,6 +275338,7 @@ __export(src_exports2, {
|
|
|
274375
275338
|
MemoryWriteConflictError: () => MemoryWriteConflictError,
|
|
274376
275339
|
MirrorEngine: () => MirrorEngine,
|
|
274377
275340
|
MissingCompanyContextError: () => MissingCompanyContextError,
|
|
275341
|
+
NODE_HEARTBEAT_STALE_MS: () => NODE_HEARTBEAT_STALE_MS,
|
|
274378
275342
|
NPM_PROBE_TIMEOUT_MS: () => NPM_PROBE_TIMEOUT_MS,
|
|
274379
275343
|
NdjsonOplogStore: () => NdjsonOplogStore,
|
|
274380
275344
|
NodeHealthTracker: () => NodeHealthTracker,
|
|
@@ -274404,6 +275368,7 @@ __export(src_exports2, {
|
|
|
274404
275368
|
RolesAdminService: () => RolesAdminService,
|
|
274405
275369
|
Router: () => Router,
|
|
274406
275370
|
RunSettlementFunnel: () => RunSettlementFunnel,
|
|
275371
|
+
RuntimeAllocationError: () => RuntimeAllocationError,
|
|
274407
275372
|
RuntimeStartMarker: () => RuntimeStartMarker,
|
|
274408
275373
|
SERVER_PING: () => SERVER_PING,
|
|
274409
275374
|
SHARED_GRAPH_BODY: () => SHARED_GRAPH_BODY,
|
|
@@ -274427,6 +275392,7 @@ __export(src_exports2, {
|
|
|
274427
275392
|
actorStats: () => actorStats,
|
|
274428
275393
|
actorsDomain: () => actorsDomain,
|
|
274429
275394
|
agentDefaultRuntimeForNewSession: () => agentDefaultRuntimeForNewSession,
|
|
275395
|
+
allocateRuntime: () => allocateRuntime,
|
|
274430
275396
|
answerFallbackReason: () => answerFallbackReason,
|
|
274431
275397
|
appendChatMessageOnce: () => appendChatMessageOnce,
|
|
274432
275398
|
applyBuiltinSkillVariants: () => applyBuiltinSkillVariants,
|
|
@@ -274437,6 +275403,7 @@ __export(src_exports2, {
|
|
|
274437
275403
|
assertCreatableVariable: () => assertCreatableVariable,
|
|
274438
275404
|
assistantRowRecovery: () => assistantRowRecovery,
|
|
274439
275405
|
assistantsDomain: () => assistantsDomain,
|
|
275406
|
+
atCapacity: () => atCapacity,
|
|
274440
275407
|
automationsDomain: () => automationsDomain,
|
|
274441
275408
|
backfillOrgModel: () => backfillOrgModel,
|
|
274442
275409
|
backfillOrgModelOnStartup: () => backfillOrgModelOnStartup,
|
|
@@ -274469,6 +275436,7 @@ __export(src_exports2, {
|
|
|
274469
275436
|
buildWorkorderSummaries: () => buildWorkorderSummaries,
|
|
274470
275437
|
builtinSkillToFiles: () => builtinSkillToFiles,
|
|
274471
275438
|
canAnnounceRuntimeSwitch: () => canAnnounceRuntimeSwitch,
|
|
275439
|
+
canRetryAllocation: () => canRetryAllocation,
|
|
274472
275440
|
chatBroadcastEnabledFromEnv: () => chatBroadcastEnabledFromEnv,
|
|
274473
275441
|
chatDelegationEnabledFromEnv: () => chatDelegationEnabledFromEnv,
|
|
274474
275442
|
chatDispatchScope: () => chatDispatchScope,
|
|
@@ -274536,6 +275504,7 @@ __export(src_exports2, {
|
|
|
274536
275504
|
createRegistryAuditSink: () => createRegistryAuditSink,
|
|
274537
275505
|
createReworkHandoffInputResolver: () => createReworkHandoffInputResolver,
|
|
274538
275506
|
createRunSettlementWiring: () => createRunSettlementWiring,
|
|
275507
|
+
createRuntimeAllocator: () => createRuntimeAllocator,
|
|
274539
275508
|
createSkillMarketClientFromEnv: () => createSkillMarketClientFromEnv,
|
|
274540
275509
|
createSpawnMutex: () => createSpawnMutex,
|
|
274541
275510
|
createTokenIssuer: () => createTokenIssuer,
|
|
@@ -274560,8 +275529,10 @@ __export(src_exports2, {
|
|
|
274560
275529
|
deliverDelegationReturn: () => deliverDelegationReturn,
|
|
274561
275530
|
deliverWorkorderBroadcast: () => deliverWorkorderBroadcast,
|
|
274562
275531
|
deriveNodeHealth: () => deriveNodeHealth,
|
|
275532
|
+
describeAllocationChoice: () => describeAllocationChoice,
|
|
274563
275533
|
describeAttachmentsForAppend: () => describeAttachmentsForAppend,
|
|
274564
275534
|
describeNodeFault: () => describeNodeFault,
|
|
275535
|
+
describeRejections: () => describeRejections,
|
|
274565
275536
|
deterministicKnowledgePlan: () => deterministicKnowledgePlan,
|
|
274566
275537
|
deterministicUuid: () => deterministicUuid,
|
|
274567
275538
|
digestOf: () => digestOf2,
|
|
@@ -274588,6 +275559,8 @@ __export(src_exports2, {
|
|
|
274588
275559
|
fallbackTurnId: () => fallbackTurnId,
|
|
274589
275560
|
fastPathFallbackReason: () => fastPathFallbackReason,
|
|
274590
275561
|
fastPathQualityReason: () => fastPathQualityReason,
|
|
275562
|
+
faultFromQuota: () => faultFromQuota,
|
|
275563
|
+
faultIsBlocking: () => faultIsBlocking,
|
|
274591
275564
|
fenceDispatch: () => fenceDispatch,
|
|
274592
275565
|
fetchLatestNpmVersion: () => fetchLatestNpmVersion,
|
|
274593
275566
|
fetchSkillFromSource: () => fetchSkillFromSource,
|
|
@@ -274603,6 +275576,7 @@ __export(src_exports2, {
|
|
|
274603
275576
|
hashPassword: () => hashPassword,
|
|
274604
275577
|
hideDeleted: () => hideDeleted,
|
|
274605
275578
|
humanSubmitSourceRunId: () => humanSubmitSourceRunId,
|
|
275579
|
+
idlenessOf: () => idlenessOf,
|
|
274606
275580
|
isActiveCoordinatorAgent: () => isActiveCoordinatorAgent,
|
|
274607
275581
|
isCompanyCrossTenantPath: () => isCompanyCrossTenantPath,
|
|
274608
275582
|
isCoordinatorAgentName: () => isCoordinatorAgentName,
|
|
@@ -274621,6 +275595,7 @@ __export(src_exports2, {
|
|
|
274621
275595
|
isNodeHealthy: () => isNodeHealthy,
|
|
274622
275596
|
isOperatorRequest: () => isOperatorRequest,
|
|
274623
275597
|
isRunTerminalStatus: () => isRunTerminalStatus,
|
|
275598
|
+
isRuntimeClassFailure: () => isRuntimeClassFailure,
|
|
274624
275599
|
isRuntimeDrifted: () => isRuntimeDrifted,
|
|
274625
275600
|
isRuntimeRunReader: () => isRuntimeRunReader,
|
|
274626
275601
|
isSealedKnowledgeSnapshotContent: () => isSealedKnowledgeSnapshotContent,
|
|
@@ -274717,6 +275692,7 @@ __export(src_exports2, {
|
|
|
274717
275692
|
providerKnowledgeHash: () => providerKnowledgeHash,
|
|
274718
275693
|
providerRoundTripKnowledgeHash: () => providerRoundTripKnowledgeHash,
|
|
274719
275694
|
publishMarker: () => publishMarker,
|
|
275695
|
+
quotaDerateFactor: () => quotaDerateFactor,
|
|
274720
275696
|
readCredentialAuditLog: () => readCredentialAuditLog,
|
|
274721
275697
|
readLocalPack: () => readLocalPack,
|
|
274722
275698
|
readRegisteredToolVersions: () => readRegisteredToolVersions,
|
|
@@ -274754,6 +275730,7 @@ __export(src_exports2, {
|
|
|
274754
275730
|
repairSubmissionDelivery: () => repairSubmissionDelivery,
|
|
274755
275731
|
repairSubmissionDeliveryAcrossCompanies: () => repairSubmissionDeliveryAcrossCompanies,
|
|
274756
275732
|
replaceModelPriceCacheForTest: () => replaceModelPriceCacheForTest,
|
|
275733
|
+
requireAllocation: () => requireAllocation,
|
|
274757
275734
|
requiresKnowledgePlanner: () => requiresKnowledgePlanner,
|
|
274758
275735
|
resendSender: () => resendSender,
|
|
274759
275736
|
resetBuildInfoCacheForTest: () => resetBuildInfoCacheForTest,
|
|
@@ -274777,6 +275754,7 @@ __export(src_exports2, {
|
|
|
274777
275754
|
runtimeConnectionStatus: () => runtimeConnectionStatus,
|
|
274778
275755
|
runtimeDriftBlock: () => runtimeDriftBlock,
|
|
274779
275756
|
runtimeSwitchedNotice: () => runtimeSwitchedNotice,
|
|
275757
|
+
runtimeTargetKey: () => runtimeTargetKey,
|
|
274780
275758
|
safeAttachmentName: () => safeAttachmentName,
|
|
274781
275759
|
sanitizeSkillDir: () => sanitizeSkillDir,
|
|
274782
275760
|
scanFrontmatterEntries: () => scanFrontmatterEntries,
|
|
@@ -274912,6 +275890,9 @@ var init_src11 = __esm({
|
|
|
274912
275890
|
init_coordinator_runs();
|
|
274913
275891
|
init_usage_read();
|
|
274914
275892
|
init_runtime_drift();
|
|
275893
|
+
init_allocate();
|
|
275894
|
+
init_errors6();
|
|
275895
|
+
init_service7();
|
|
274915
275896
|
init_runtime_status();
|
|
274916
275897
|
init_runtime_tenancy();
|
|
274917
275898
|
init_types4();
|
|
@@ -275287,6 +276268,7 @@ var import_node_crypto91 = require("node:crypto");
|
|
|
275287
276268
|
|
|
275288
276269
|
// ../cli/src/serve.ts
|
|
275289
276270
|
init_src11();
|
|
276271
|
+
init_src11();
|
|
275290
276272
|
init_chat_cancel_attribution();
|
|
275291
276273
|
init_src();
|
|
275292
276274
|
init_src11();
|
|
@@ -277919,7 +278901,13 @@ function refreshConnectorSkillsInBackground(dataDir, apply) {
|
|
|
277919
278901
|
}
|
|
277920
278902
|
var CONNECTOR_SKILL_REFRESH_MS = 24 * 60 * 6e4;
|
|
277921
278903
|
var SESSION_TOKEN_GRACE_MS = 15 * 6e4;
|
|
277922
|
-
function
|
|
278904
|
+
function allocatedTurnOf(req) {
|
|
278905
|
+
return Boolean(req.delegatedChildTurn) || req.executionProfile === "knowledge-single-turn" || req.executionProfile === "discussion-background";
|
|
278906
|
+
}
|
|
278907
|
+
function allocationUnitOfWorkOf(req) {
|
|
278908
|
+
return `chat-session:${req.chatSessionId ?? req.dispatchId ?? req.actorId}`;
|
|
278909
|
+
}
|
|
278910
|
+
function makeResumeRetryProxy(first, spawnFallback, runtimeRetry) {
|
|
277923
278911
|
const relay = () => {
|
|
277924
278912
|
const cbs = [];
|
|
277925
278913
|
const buf = [];
|
|
@@ -277968,8 +278956,11 @@ function makeResumeRetryProxy(first, spawnFallback) {
|
|
|
277968
278956
|
return chatExit?.reason === "session-not-found";
|
|
277969
278957
|
};
|
|
277970
278958
|
const done = first.done.catch(async (err) => {
|
|
277971
|
-
if (sawOutput
|
|
277972
|
-
const
|
|
278959
|
+
if (sawOutput) throw err;
|
|
278960
|
+
const exit3 = err instanceof Error ? err.chatExit ?? {} : {};
|
|
278961
|
+
const spawnSecond = isSessionNotFound2(err) && spawnFallback ? spawnFallback : runtimeRetry?.shouldRetry({ ...exit3.reason ? { reason: exit3.reason } : {}, ...exit3.detail ?? exit3.summary ? { detail: exit3.detail ?? exit3.summary } : {} }) ? () => runtimeRetry.spawn({ ...exit3.reason ? { reason: exit3.reason } : {}, ...exit3.detail ?? exit3.summary ? { detail: exit3.detail ?? exit3.summary } : {} }) : null;
|
|
278962
|
+
if (!spawnSecond) throw err;
|
|
278963
|
+
const second = await spawnSecond();
|
|
277973
278964
|
active = second;
|
|
277974
278965
|
wire(second);
|
|
277975
278966
|
await second.done;
|
|
@@ -280271,6 +281262,66 @@ async function startServe(opts) {
|
|
|
280271
281262
|
return (await ctx.service.latestConfig(actorId))?.model ?? null;
|
|
280272
281263
|
};
|
|
280273
281264
|
agentDefaultRuntimeResolvers = { resolveBinding: resolveCompanyBinding, resolveActorModel: resolveAgentConfiguredModel };
|
|
281265
|
+
const runtimeAllocators = /* @__PURE__ */ new Map();
|
|
281266
|
+
const runtimeAllocatorFor = (companyId) => {
|
|
281267
|
+
const hit = runtimeAllocators.get(companyId);
|
|
281268
|
+
if (hit) return hit;
|
|
281269
|
+
const allocator = createRuntimeAllocator({
|
|
281270
|
+
// 主绑定与候选清单都**按公司解析**:全局 registry 对已切 `actor-config` 的公司是一份冻结旧镜像
|
|
281271
|
+
//(理由与 `resolveCompanyBinding` 文件头同源)。
|
|
281272
|
+
primaryBinding: (actorId) => resolveCompanyBinding(companyId, actorId),
|
|
281273
|
+
candidates: async (actorId) => {
|
|
281274
|
+
const registry2 = (await getEngine(companyId)).registry;
|
|
281275
|
+
return registry2.listRuntimeCandidates ? registry2.listRuntimeCandidates(actorId) : [];
|
|
281276
|
+
},
|
|
281277
|
+
nodeStates: async () => {
|
|
281278
|
+
const [nodes, runtimes] = await Promise.all([
|
|
281279
|
+
nodeStore.listNodes({ companyId }),
|
|
281280
|
+
nodeStore.listRuntimes(void 0, { companyId })
|
|
281281
|
+
]);
|
|
281282
|
+
return nodes.filter((n) => n.status !== "deleted").map((n) => ({
|
|
281283
|
+
nodeId: n.id,
|
|
281284
|
+
// 报名称不报 id(ADR 0612):节点名 → 主机名 → 万不得已才是 id。
|
|
281285
|
+
displayName: n.name?.trim() || n.hostname || n.id,
|
|
281286
|
+
/**
|
|
281287
|
+
* 「在线」只取**可达性**:长连接在 + 心跳新鲜。刻意**不用** `healthOfNode(...).healthy`
|
|
281288
|
+
* ——那个判据还叠了「连败 ≥ 3 次」这一档软不健康。
|
|
281289
|
+
*
|
|
281290
|
+
* 为什么不叠:委派子会话此前走的是 `bindingRuntimeState`(在线 + 上报了这个 adapter),
|
|
281291
|
+
* 连败次数根本不参与。把它叠进来,一台**还连着、还能跑**但最近连败过三次的机器会被分配
|
|
281292
|
+
* 排除掉,而这个员工要是没配候选,那一轮就从「照旧派出去」变成当场 503 —— 拿一个新功能
|
|
281293
|
+
* 去掐掉一条本来能走的路。
|
|
281294
|
+
*
|
|
281295
|
+
* 连败这类「机器看着在、但活总是挂」的判断归**故障标记**(ADR 0616 D8,第 2 批):
|
|
281296
|
+
* 它会写进 `node_runtime_faults`,那时分配读的是故障行,粒度是 (节点, runtime) 而不是整台机器,
|
|
281297
|
+
* 也带得上「下次什么时候再试」。工单派发那条路的 `checkDispatchable` 仍按原样拦连败节点,
|
|
281298
|
+
* 与改动前逐字相同。
|
|
281299
|
+
*/
|
|
281300
|
+
online: (hub?.connectedDaemons() ?? []).some((d) => d.daemonId === n.id) && (() => {
|
|
281301
|
+
const seen = hub?.lastSeenAt(n.id);
|
|
281302
|
+
return seen !== void 0 && Date.now() - seen < NODE_HEARTBEAT_STALE_MS;
|
|
281303
|
+
})(),
|
|
281304
|
+
/* 机器上有什么 runtime:漂移行(登记表有、机器探测不到)不算有——判据同 `runtimeDriftBlock`。
|
|
281305
|
+
`presentOnMachine` 缺省(还没对账过的旧行)一律当作正常,与那道闸逐字一致。 */
|
|
281306
|
+
runtimeKinds: runtimes.filter((r) => r.nodeId === n.id && r.presentOnMachine !== false).map((r) => r.kind),
|
|
281307
|
+
maxConcurrent: n.maxConcurrent ?? null,
|
|
281308
|
+
activeRunCount: activeRunCountOf(n.id)
|
|
281309
|
+
}));
|
|
281310
|
+
},
|
|
281311
|
+
faults: () => nodeStore.listRuntimeFaults({ companyId }),
|
|
281312
|
+
quotas: () => nodeStore.listRuntimeQuotas({ companyId }),
|
|
281313
|
+
// 故障的读 / 写 / 解除(ADR 0616 D8):三个一起接,缺一个就会出现「标了但下次算不出间隔」
|
|
281314
|
+
// 或者「跑成了也不解除」这种半拉子状态。
|
|
281315
|
+
getFault: async (t) => (await nodeStore.listRuntimeFaults({ nodeId: t.nodeId })).find((f2) => f2.runtimeKind === t.runtimeKind) ?? null,
|
|
281316
|
+
saveFault: (fault) => nodeStore.upsertRuntimeFault(fault),
|
|
281317
|
+
clearFault: async (t) => {
|
|
281318
|
+
await nodeStore.clearRuntimeFault(t.nodeId, t.runtimeKind);
|
|
281319
|
+
},
|
|
281320
|
+
actorName: async (actorId) => (await (await actors.resolveCtx(companyId)).service.getActor(actorId).catch(() => null))?.name ?? null
|
|
281321
|
+
});
|
|
281322
|
+
runtimeAllocators.set(companyId, allocator);
|
|
281323
|
+
return allocator;
|
|
281324
|
+
};
|
|
280274
281325
|
if (pgPool) {
|
|
280275
281326
|
const pinPool = pgPool;
|
|
280276
281327
|
void (async () => {
|
|
@@ -280979,6 +282030,25 @@ async function startServe(opts) {
|
|
|
280979
282030
|
}
|
|
280980
282031
|
return { reachable: true };
|
|
280981
282032
|
},
|
|
282033
|
+
/**
|
|
282034
|
+
* ★ ADR 0616:上面那道只问主绑定那台机器。候选 runtime 上线之后「主的机器离线」不再等于
|
|
282035
|
+
* 「这位专家派不出去」——这一问把**运行时分配**接进委派的第一道闸。
|
|
282036
|
+
*
|
|
282037
|
+
* 不接的后果 2026-09-23 在 v2 didi 上实测过:主绑定改到离线节点、候选配在在线节点,
|
|
282038
|
+
* `oasis delegate` 仍当场回 `EXPERT_UNREACHABLE`「绑定节点 … 当前不在线」,分配那一段
|
|
282039
|
+
* **一行都没跑到**。
|
|
282040
|
+
*
|
|
282041
|
+
* 用 `tryAllocate`(不抛、不消耗重试次数):这里只是问「挑不挑得出来」,真正的分配在
|
|
282042
|
+
* 那一轮派发时再跑一次——两次之间机器状态可能变,以派发那一次为准。
|
|
282043
|
+
*/
|
|
282044
|
+
delegationProbeAllocation: async ({ actorId, companyId }) => {
|
|
282045
|
+
const result = await runtimeAllocatorFor(companyId ?? defaultCompanyId).tryAllocate({
|
|
282046
|
+
actorId,
|
|
282047
|
+
unitOfWork: `delegation-preflight:${actorId}`
|
|
282048
|
+
});
|
|
282049
|
+
if (result.ok) return { reachable: true };
|
|
282050
|
+
return { reachable: false, reason: describeRejections(result.rejections) || "\u6CA1\u6709\u53EF\u7528\u7684\u8FD0\u884C\u65F6\u76EE\u6807" };
|
|
282051
|
+
},
|
|
280982
282052
|
// ADR 会话投递 D4:/api/chat 与渠道入站据它占同一个会话槽(自动化侧经 automations 域接同一实例)。
|
|
280983
282053
|
chatTurns: chatTurnStore,
|
|
280984
282054
|
// ADR 0176 第二步(缺陷 B):按当前公司解析轮次账本——turn 必须与它的会话同 schema,
|
|
@@ -281491,6 +282561,16 @@ async function startServe(opts) {
|
|
|
281491
282561
|
const isolatedKnowledgeTurn = executionProfile === "knowledge-single-turn";
|
|
281492
282562
|
const discussionBackground = executionProfile === "discussion-background";
|
|
281493
282563
|
const isolatedBackground = isolatedKnowledgeTurn || discussionBackground;
|
|
282564
|
+
const allocatedTurn = allocatedTurnOf({
|
|
282565
|
+
...delegatedChildTurn ? { delegatedChildTurn } : {},
|
|
282566
|
+
...executionProfile ? { executionProfile } : {}
|
|
282567
|
+
});
|
|
282568
|
+
const allocationWhat = delegatedChildTurn ? "\u59D4\u6D3E\u5B50\u4F1A\u8BDD" : isolatedKnowledgeTurn ? "\u77E5\u8BC6\u5E93\u5355\u8F6E" : "\u540E\u53F0\u6574\u7406";
|
|
282569
|
+
const allocationUnitOfWork = allocationUnitOfWorkOf({
|
|
282570
|
+
...chatSessionId ? { chatSessionId } : {},
|
|
282571
|
+
...requestDispatchId ? { dispatchId: requestDispatchId } : {},
|
|
282572
|
+
actorId
|
|
282573
|
+
});
|
|
281494
282574
|
const runCompanyId = companyId ?? defaultCompanyId;
|
|
281495
282575
|
const actorCtxForCompany = await actors.resolveCtx(runCompanyId);
|
|
281496
282576
|
const actorService = actorCtxForCompany.service;
|
|
@@ -281520,6 +282600,7 @@ async function startServe(opts) {
|
|
|
281520
282600
|
const priorChatSession = chatSessionId && turnChatStore ? await turnChatStore.getSession(chatSessionId) : null;
|
|
281521
282601
|
let binding;
|
|
281522
282602
|
let runtimeTargetSource = "actor_binding";
|
|
282603
|
+
let allocatedModelOverride;
|
|
281523
282604
|
if (stableDispatch) {
|
|
281524
282605
|
binding = { actorId, ...stableDispatch.target, status: "active" };
|
|
281525
282606
|
} else {
|
|
@@ -281546,14 +282627,42 @@ async function startServe(opts) {
|
|
|
281546
282627
|
}
|
|
281547
282628
|
});
|
|
281548
282629
|
} catch (e) {
|
|
281549
|
-
if (e instanceof SessionRuntimeOverrideError)
|
|
281550
|
-
|
|
282630
|
+
if (e instanceof SessionRuntimeOverrideError) {
|
|
282631
|
+
if (!allocatedTurn) throw new ApiError(e.status, e.code, e.message);
|
|
282632
|
+
console.log(`[runtime-alloc] ${allocationWhat}\u9489\u4F4F\u7684\u76EE\u6807\u4E0D\u53EF\u7528\uFF08${e.code}\uFF09\uFF0C\u4EA4\u7ED9\u5206\u914D\u6311\u5019\u9009`);
|
|
282633
|
+
resolved = null;
|
|
282634
|
+
} else throw e;
|
|
281551
282635
|
}
|
|
281552
|
-
if (!resolved) {
|
|
282636
|
+
if (!resolved && !allocatedTurn) {
|
|
281553
282637
|
throw new ApiError(409, "ACTOR_BINDING_MISSING", `\u8FD9\u4F4D Agent \u8FD8\u6CA1\u6709\u7ED1\u5B9A\u8FD0\u884C\u65F6\uFF0C\u8BF7${chatRebindHint("\u7ED1\u5B9A\u4E00\u4E2A\u53EF\u7528\u7684\u8FD0\u884C\u65F6")}\u540E\u518D\u53D1`);
|
|
281554
282638
|
}
|
|
281555
|
-
binding = resolved
|
|
281556
|
-
runtimeTargetSource = resolved
|
|
282639
|
+
binding = resolved?.binding ?? { actorId, nodeId: "", runtimeKind: "", status: "active" };
|
|
282640
|
+
runtimeTargetSource = resolved?.source ?? "actor_binding";
|
|
282641
|
+
if (allocatedTurn) {
|
|
282642
|
+
const pinned = sessionRuntimeOverride(priorChatSession);
|
|
282643
|
+
const primary = resolved ? { nodeId: resolved.binding.nodeId, runtimeKind: resolved.binding.runtimeKind } : pinned ? { nodeId: pinned.nodeId, runtimeKind: pinned.runtimeKind } : null;
|
|
282644
|
+
try {
|
|
282645
|
+
const allocated = await runtimeAllocatorFor(runCompanyId).allocateFor({
|
|
282646
|
+
actorId,
|
|
282647
|
+
// 「同一件活」= 这条子会话。重试上限算在它头上,不与同一位专家的别条委派互相扣次数。
|
|
282648
|
+
unitOfWork: allocationUnitOfWork,
|
|
282649
|
+
what: allocationWhat,
|
|
282650
|
+
primary
|
|
282651
|
+
});
|
|
282652
|
+
if (allocated.binding.nodeId !== binding.nodeId || allocated.binding.runtimeKind !== binding.runtimeKind) {
|
|
282653
|
+
binding = allocated.binding;
|
|
282654
|
+
runtimeTargetSource = "session_override";
|
|
282655
|
+
allocatedModelOverride = allocated.model ?? void 0;
|
|
282656
|
+
}
|
|
282657
|
+
} catch (e) {
|
|
282658
|
+
if (e instanceof RuntimeAllocationError) throw new ApiError(e.status === 503 ? 503 : 409, e.code, e.message);
|
|
282659
|
+
console.warn(`[runtime-alloc] ${allocationWhat}\u5206\u914D\u5931\u8D25\uFF1A${String(e)}`);
|
|
282660
|
+
if (!resolved) throw e;
|
|
282661
|
+
}
|
|
282662
|
+
}
|
|
282663
|
+
if (!binding.nodeId || !binding.runtimeKind) {
|
|
282664
|
+
throw new ApiError(409, "ACTOR_BINDING_MISSING", `\u8FD9\u4F4D Agent \u8FD8\u6CA1\u6709\u53EF\u7528\u7684\u8FD0\u884C\u65F6\uFF0C\u8BF7${chatRebindHint("\u7ED1\u5B9A\u4E00\u4E2A\u53EF\u7528\u7684\u8FD0\u884C\u65F6")}\u540E\u518D\u53D1`);
|
|
282665
|
+
}
|
|
281557
282666
|
}
|
|
281558
282667
|
const requireDispatchNode = async (target) => {
|
|
281559
282668
|
const targetNode = await nodeStore.getNode(target.nodeId);
|
|
@@ -281777,7 +282886,7 @@ async function startServe(opts) {
|
|
|
281777
282886
|
};
|
|
281778
282887
|
const provision = isolatedKnowledgeTurn ? { env: {}, wrapperPaths: [], requiredTools: [], requiredToolVersions: {}, connectorCreds: [], cleanup: async () => void 0 } : await buildActorProvision(actorService, actorId, chatDispatchScope(priorChatSession));
|
|
281779
282888
|
const actorCtx = await buildActorContext(actorService, actorId).catch(() => null);
|
|
281780
|
-
const sessionModel = configuredModel(sessionModelOverride);
|
|
282889
|
+
const sessionModel = configuredModel(allocatedModelOverride !== void 0 ? allocatedModelOverride : sessionModelOverride);
|
|
281781
282890
|
const actorConfiguredModel = isolatedBackground ? configuredModel(actorCtx?.config?.model) : void 0;
|
|
281782
282891
|
const chatResolvedModel = sessionModel ?? actorConfiguredModel;
|
|
281783
282892
|
const chatModel = chatResolvedModel ?? await registryStore.getRuntimeConfig(`runtime:${binding.nodeId}:${binding.runtimeKind}`).then((c) => configuredModel(c?.model)).catch(() => void 0);
|
|
@@ -281982,6 +283091,17 @@ async function startServe(opts) {
|
|
|
281982
283091
|
const done = new Promise((resolve11, reject) => {
|
|
281983
283092
|
handle.onExit((info) => {
|
|
281984
283093
|
chatLiveSessions.delete(chatJobKey);
|
|
283094
|
+
if (allocatedTurn) {
|
|
283095
|
+
const unitOfWork = allocationUnitOfWork;
|
|
283096
|
+
const allocator = runtimeAllocatorFor(runCompanyId);
|
|
283097
|
+
const target = { nodeId: binding.nodeId, runtimeKind: binding.runtimeKind };
|
|
283098
|
+
void (isRuntimeClassFailure(info.reason, info.errorMessage) ? allocator.recordFailure({
|
|
283099
|
+
unitOfWork,
|
|
283100
|
+
target,
|
|
283101
|
+
...info.reason ? { reason: info.reason } : {},
|
|
283102
|
+
...info.errorMessage ? { errorMessage: info.errorMessage } : {}
|
|
283103
|
+
}) : allocator.recordSettled(unitOfWork, target)).catch((err) => console.warn(`[runtime-alloc] \u4F1A\u8BDD\u5931\u8D25\u5904\u7F6E\u51FA\u9519\uFF08\u4E0D\u5F71\u54CD\u6536\u5C3E\uFF09\uFF1A${String(err)}`));
|
|
283104
|
+
}
|
|
281985
283105
|
setTimeout(() => {
|
|
281986
283106
|
chatRunByDispatch.delete(handle.id);
|
|
281987
283107
|
}, 6e4).unref?.();
|
|
@@ -282052,6 +283172,9 @@ async function startServe(opts) {
|
|
|
282052
283172
|
},
|
|
282053
283173
|
done,
|
|
282054
283174
|
kill: () => handle.kill(),
|
|
283175
|
+
// Bug 0232:`kill()` 报不了帧的去向(契约钉死 `Promise<void>`),这一对能。
|
|
283176
|
+
// 条件透传:本地 adapter 的句柄没有这条,上层按「方法在不在」分流回老行为。
|
|
283177
|
+
...typeof handle.sendKill === "function" ? { sendKill: () => handle.sendKill(), cancelLocally: () => handle.cancelLocally?.() ?? false } : {},
|
|
282055
283178
|
// ADR-0093 对话追加(chat 道):透传底层句柄的 appendInput。runtime 不原生支持时 handle 上
|
|
282056
283179
|
// 压根没这个方法,这里也就不给——server 据此让插入入口置灰,而不是让人按下去才发现不行。
|
|
282057
283180
|
// 不带 interrupt:人插一句话走回合边界排队,不作废在飞回合(那是 produce 道 ADR-0094 的语义)。
|
|
@@ -282088,8 +283211,37 @@ async function startServe(opts) {
|
|
|
282088
283211
|
firstPromise = dispatchChatImpl(chatReq);
|
|
282089
283212
|
}
|
|
282090
283213
|
const first = await firstPromise;
|
|
282091
|
-
|
|
282092
|
-
|
|
283214
|
+
const runtimeRetry = allocatedTurnOf({
|
|
283215
|
+
...chatReq.delegatedChildTurn ? { delegatedChildTurn: chatReq.delegatedChildTurn } : {},
|
|
283216
|
+
...chatReq.executionProfile ? { executionProfile: chatReq.executionProfile } : {}
|
|
283217
|
+
}) ? (() => {
|
|
283218
|
+
const unitOfWork = allocationUnitOfWorkOf({
|
|
283219
|
+
...chatReq.chatSessionId ? { chatSessionId: chatReq.chatSessionId } : {},
|
|
283220
|
+
...chatReq.dispatchId ? { dispatchId: chatReq.dispatchId } : {},
|
|
283221
|
+
actorId: chatReq.actorId
|
|
283222
|
+
});
|
|
283223
|
+
const allocator = runtimeAllocatorFor(chatReq.companyId ?? defaultCompanyId);
|
|
283224
|
+
return {
|
|
283225
|
+
shouldRetry: (exit3) => isRuntimeClassFailure(exit3.reason, exit3.detail) && canRetryAllocation(allocator.inspect(unitOfWork)?.failures ?? 0),
|
|
283226
|
+
spawn: async (exit3) => {
|
|
283227
|
+
const last = allocator.inspect(unitOfWork)?.lastAssigned;
|
|
283228
|
+
if (last) {
|
|
283229
|
+
allocator.noteFailure({
|
|
283230
|
+
unitOfWork,
|
|
283231
|
+
target: last,
|
|
283232
|
+
...exit3.reason ? { reason: exit3.reason } : {},
|
|
283233
|
+
...exit3.detail ? { errorMessage: exit3.detail } : {}
|
|
283234
|
+
});
|
|
283235
|
+
}
|
|
283236
|
+
console.log(`[runtime-alloc] ${unitOfWork} \u56E0 ${exit3.reason ?? "runtime \u6545\u969C"} \u5931\u8D25\u4E14\u672A\u4EA7\u51FA\u4EFB\u4F55\u8F93\u51FA \u2192 \u6362\u5019\u9009\u91CD\u8D77\u8FD9\u4E00\u8F6E`);
|
|
283237
|
+
return dispatchChatImpl(chatReq);
|
|
283238
|
+
}
|
|
283239
|
+
};
|
|
283240
|
+
})() : void 0;
|
|
283241
|
+
if (!chatReq.sessionId) return runtimeRetry ? makeResumeRetryProxy(first, null, runtimeRetry) : first;
|
|
283242
|
+
if (chatReq.executionProfile === "knowledge-single-turn" || chatReq.executionProfile === "discussion-background") {
|
|
283243
|
+
return runtimeRetry ? makeResumeRetryProxy(first, null, runtimeRetry) : first;
|
|
283244
|
+
}
|
|
282093
283245
|
return makeResumeRetryProxy(first, async () => {
|
|
282094
283246
|
const message = chatReq.fallbackMessage ?? await buildResumeFallbackMessage(chatReq) ?? chatReq.message;
|
|
282095
283247
|
const { admitDispatch: _alreadyAdmitted, ...fallbackReq } = chatReq;
|
|
@@ -282100,7 +283252,7 @@ async function startServe(opts) {
|
|
|
282100
283252
|
message,
|
|
282101
283253
|
...chatReq.dispatchId && isHumanSubmitDispatchId(chatReq.dispatchId) ? { dispatchId: resumeFallbackDispatchId(chatReq.dispatchId) } : {}
|
|
282102
283254
|
});
|
|
282103
|
-
});
|
|
283255
|
+
}, runtimeRetry);
|
|
282104
283256
|
},
|
|
282105
283257
|
trajectoryFeed: {
|
|
282106
283258
|
subscribe(sessionId, listener) {
|
|
@@ -282425,6 +283577,15 @@ async function startServe(opts) {
|
|
|
282425
283577
|
const mapping = dispatchToWork.get(msg.dispatchId);
|
|
282426
283578
|
const exitBus = mapping ? busForCompany(mapping.companyId) ?? busForWorkorder(mapping.workorderId) : null;
|
|
282427
283579
|
if (mapping && exitBus) {
|
|
283580
|
+
if (mapping.artifactId && mapping.companyId) {
|
|
283581
|
+
const allocator = runtimeAllocatorFor(mapping.companyId);
|
|
283582
|
+
const unitOfWork = mapping.artifactId;
|
|
283583
|
+
void (isRuntimeClassFailure(msg.info.reason, msg.info.errorMessage) ? allocator.recordFailure({
|
|
283584
|
+
unitOfWork,
|
|
283585
|
+
...msg.info.reason ? { reason: msg.info.reason } : {},
|
|
283586
|
+
...msg.info.errorMessage ? { errorMessage: msg.info.errorMessage } : {}
|
|
283587
|
+
}) : allocator.recordSettled(unitOfWork)).catch((err) => console.warn(`[runtime-alloc] \u5DE5\u5355\u6D3E\u53D1\u5931\u8D25\u5904\u7F6E\u51FA\u9519\uFF08\u4E0D\u5F71\u54CD\u6536\u53E3\uFF09\uFF1A${String(err)}`));
|
|
283588
|
+
}
|
|
282428
283589
|
await exitBus.submit({
|
|
282429
283590
|
companyId: "",
|
|
282430
283591
|
workorderId: mapping.workorderId,
|
|
@@ -282884,7 +284045,8 @@ async function startServe(opts) {
|
|
|
282884
284045
|
try {
|
|
282885
284046
|
result = replyIssueId ? await bridge.replyWork(rev.artifactId, rev.author, replyIssueId, ledgerId) : await bridge.produce(rev.artifactId, rev.author, workId, ledgerId, opts2?.resumeEligible, opts2?.protocolRecovery?.round);
|
|
282886
284047
|
} catch (err) {
|
|
282887
|
-
|
|
284048
|
+
const reason = err instanceof RuntimeAllocationError ? err.message : `dispatch error: ${err instanceof Error ? err.message : String(err)}`;
|
|
284049
|
+
void dispatchLedgerFor(companyId).then((l) => l.close(ledgerId, { outcome: "failed", reason })).catch(ledgerWarn("close(error)"));
|
|
282888
284050
|
console.log(`[new-engine] dispatchWork ${workId} \u6D3E\u53D1\u5F02\u5E38\uFF1A${String(err)} \u2192 \u7F6E work failed`);
|
|
282889
284051
|
const failBus = busForCompany(companyId);
|
|
282890
284052
|
if (failBus) {
|
|
@@ -282899,7 +284061,7 @@ async function startServe(opts) {
|
|
|
282899
284061
|
semanticsVersion: 3,
|
|
282900
284062
|
dispatchId: ledgerId,
|
|
282901
284063
|
outcome: "failed",
|
|
282902
|
-
detail: { reason
|
|
284064
|
+
detail: { reason }
|
|
282903
284065
|
}
|
|
282904
284066
|
});
|
|
282905
284067
|
}
|
|
@@ -282910,6 +284072,9 @@ async function startServe(opts) {
|
|
|
282910
284072
|
dispatchToWork.set(result.dispatchId, {
|
|
282911
284073
|
workId,
|
|
282912
284074
|
workorderId: wid,
|
|
284075
|
+
// ★ ADR 0616:运行时分配按「同一件活」记失败链,工单这条路的单位就是这个节点(artifactId)。
|
|
284076
|
+
// 退出帧上只有 dispatchId,所以必须在这里记下来——否则失败时不知道该把哪条链标上。
|
|
284077
|
+
artifactId: rev.artifactId,
|
|
282913
284078
|
// ★ 记下归属公司:session_started / session_exited / 收口漏斗都据它把事件投回**这一家**的 bus。
|
|
282914
284079
|
companyId,
|
|
282915
284080
|
...delivery ? { nodeId: delivery.nodeId } : {}
|
|
@@ -283183,7 +284348,27 @@ async function startServe(opts) {
|
|
|
283183
284348
|
}),
|
|
283184
284349
|
serverUrl: localUrl,
|
|
283185
284350
|
serverUrlFor: (ctx) => serverUrlForBinding(ctx.binding),
|
|
283186
|
-
|
|
284351
|
+
/**
|
|
284352
|
+
* ★ ADR 0616:工单节点派活(produce / review / integrate / reply)的目标由**运行时分配**决定,
|
|
284353
|
+
* 不再直接读那条主绑定。两个触发时机都落在这里:
|
|
284354
|
+
*
|
|
284355
|
+
* ① 开一个新会话时 —— 每次 spawn 前跑一次分配(排除故障与满并发、主可用就选主)。
|
|
284356
|
+
* ② 上一轮因 runtime/节点原因失败时 —— `session_exited` 那边已经把失败目标记进这件活的
|
|
284357
|
+
* 失败链,这一次分配会自动换到另一个候选;连着换满 `RUNTIME_ALLOCATION_MAX_ATTEMPTS` 次
|
|
284358
|
+
* 仍起不来就抛错(由 `dispatchWork` 的 catch 落成 work failed,原文带机器名与原因)。
|
|
284359
|
+
*
|
|
284360
|
+
* 没带 `ctx.unitOfWork` 的调用(`executorSignature` 只要一个指纹)退回旧路:那条路不该
|
|
284361
|
+
* 消耗一次分配尝试,更不该因为「所有候选都试过了」而抛错——指纹算不出来只会退化,不该阻断。
|
|
284362
|
+
*/
|
|
284363
|
+
resolveBinding: async (actorId, ctx) => {
|
|
284364
|
+
if (!ctx?.unitOfWork) return (await runActors()).service.resolveBinding()(actorId);
|
|
284365
|
+
const resolved = await runtimeAllocatorFor(companyId).allocateFor({
|
|
284366
|
+
actorId,
|
|
284367
|
+
unitOfWork: ctx.unitOfWork,
|
|
284368
|
+
what: "\u5DE5\u5355\u8282\u70B9\u6D3E\u6D3B"
|
|
284369
|
+
});
|
|
284370
|
+
return resolved.binding;
|
|
284371
|
+
},
|
|
283187
284372
|
prepareReadonlyInputs: async ({ actor, artifactType, binding, codeRepo, pinnedRevisions }) => {
|
|
283188
284373
|
let prepared = null;
|
|
283189
284374
|
try {
|
|
@@ -284589,7 +285774,7 @@ ${nodeFault}` : "");
|
|
|
284589
285774
|
const zbEnabled = opts.zeroOutputBreaker?.enabled ?? !zbDisabledByEnv;
|
|
284590
285775
|
const zbIntervalMs = opts.zeroOutputBreaker?.intervalMs ?? 3e4;
|
|
284591
285776
|
const zbWindowMs = opts.zeroOutputBreaker?.windowMs ?? (envInt("OASIS_ZERO_OUTPUT_BREAKER_WINDOW_MIN") ?? 30) * 6e4;
|
|
284592
|
-
const zbThreshold = opts.zeroOutputBreaker?.threshold ?? envInt("OASIS_ZERO_OUTPUT_BREAKER_N") ??
|
|
285777
|
+
const zbThreshold = opts.zeroOutputBreaker?.threshold ?? envInt("OASIS_ZERO_OUTPUT_BREAKER_N") ?? 20;
|
|
284593
285778
|
const zbCostThresholdMicros = opts.zeroOutputBreaker?.costThresholdMicros ?? (envInt("OASIS_ZERO_OUTPUT_BREAKER_COST_USD") ?? 0) * 1e6;
|
|
284594
285779
|
if (zbEnabled && zbIntervalMs > 0 && (zbThreshold > 0 || zbCostThresholdMicros > 0)) {
|
|
284595
285780
|
const zbActions = /* @__PURE__ */ new Set(["produce", "review"]);
|
|
@@ -289795,7 +290980,7 @@ async function waitForCommandAuthorization(base, headers, draftId, fallback) {
|
|
|
289795
290980
|
}
|
|
289796
290981
|
}
|
|
289797
290982
|
}
|
|
289798
|
-
var CLI_VERSION = "2.2.
|
|
290983
|
+
var CLI_VERSION = "2.2.74" ? "2.2.74" : "dev";
|
|
289799
290984
|
function apiHeaders(token, company) {
|
|
289800
290985
|
const headers = {
|
|
289801
290986
|
authorization: `Bearer ${token}`,
|
|
@@ -293601,7 +294786,7 @@ function shimScript() {
|
|
|
293601
294786
|
}
|
|
293602
294787
|
|
|
293603
294788
|
// src/index.ts
|
|
293604
|
-
var PKG_VERSION = true ? "2.2.
|
|
294789
|
+
var PKG_VERSION = true ? "2.2.74" : "dev";
|
|
293605
294790
|
var LOCAL_BIN = localBin();
|
|
293606
294791
|
var NPM_PREFIX = npmPrefix();
|
|
293607
294792
|
var INSTANCE = DEFAULT_INSTANCE;
|