oasis_test_v2 2.2.73 → 2.2.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +274 -47
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -846,10 +846,22 @@ var init_runtime = __esm({
|
|
|
846
846
|
});
|
|
847
847
|
|
|
848
848
|
// ../contract/src/runtime-allocation.ts
|
|
849
|
-
var RUNTIME_FAULT_PROBE_MAX_INTERVAL_MS, RUNTIME_FAULT_PROBE_MAX_TOTAL_MS, RUNTIME_QUOTA_DERATE_THRESHOLD_PERCENT, RUNTIME_ALLOCATION_MAX_ATTEMPTS;
|
|
849
|
+
var RUNTIME_FAULT_RECOVERY, RUNTIME_FAULT_PROBE_INITIAL_MS, RUNTIME_FAULT_PROBE_MAX_INTERVAL_MS, RUNTIME_FAULT_PROBE_MAX_TOTAL_MS, RUNTIME_QUOTA_DERATE_THRESHOLD_PERCENT, RUNTIME_ALLOCATION_MAX_ATTEMPTS;
|
|
850
850
|
var init_runtime_allocation = __esm({
|
|
851
851
|
"../contract/src/runtime-allocation.ts"() {
|
|
852
852
|
"use strict";
|
|
853
|
+
RUNTIME_FAULT_RECOVERY = {
|
|
854
|
+
"quota-exhausted": "readonly-probe",
|
|
855
|
+
// 登录失效直接走人工:恢复需要人在那台机器上重新登录,放一条真会话过去试只是又坏一件活。
|
|
856
|
+
unauthenticated: "manual",
|
|
857
|
+
"node-offline": "connection",
|
|
858
|
+
"runtime-missing": "connection",
|
|
859
|
+
"startup-failed": "half-open",
|
|
860
|
+
crashed: "half-open",
|
|
861
|
+
timeout: "half-open",
|
|
862
|
+
"node-error": "half-open"
|
|
863
|
+
};
|
|
864
|
+
RUNTIME_FAULT_PROBE_INITIAL_MS = 6e4;
|
|
853
865
|
RUNTIME_FAULT_PROBE_MAX_INTERVAL_MS = 30 * 6e4;
|
|
854
866
|
RUNTIME_FAULT_PROBE_MAX_TOTAL_MS = 6 * 60 * 6e4;
|
|
855
867
|
RUNTIME_QUOTA_DERATE_THRESHOLD_PERCENT = 60;
|
|
@@ -202049,11 +202061,18 @@ var init_service3 = __esm({
|
|
|
202049
202061
|
const actors = await this.options.resolveActors(companyId);
|
|
202050
202062
|
const bindings = await actors.listBindings();
|
|
202051
202063
|
const reachability = async (actorId) => {
|
|
202052
|
-
const
|
|
202053
|
-
|
|
202054
|
-
|
|
202055
|
-
|
|
202056
|
-
|
|
202064
|
+
const primaryReason = await (async () => {
|
|
202065
|
+
const binding = bindings.find((b2) => b2.actorId === actorId && b2.status === "active");
|
|
202066
|
+
if (!binding) return "\u6CA1\u6709\u63A5\u8FD0\u884C\u65F6";
|
|
202067
|
+
if (!this.options.probeRuntime) return null;
|
|
202068
|
+
const probed = await this.options.probeRuntime({ actorId, nodeId: binding.nodeId, runtimeKind: binding.runtimeKind }).catch(() => ({ reachable: true }));
|
|
202069
|
+
return probed.reachable ? null : probed.reason ?? "\u8FD0\u884C\u65F6\u4E0D\u53EF\u8FBE";
|
|
202070
|
+
})();
|
|
202071
|
+
if (primaryReason === null) return { ok: true };
|
|
202072
|
+
if (!this.options.probeAllocation) return { ok: false, reason: primaryReason };
|
|
202073
|
+
const alt = await this.options.probeAllocation({ actorId, ...companyId ? { companyId } : {} }).catch(() => ({ reachable: true }));
|
|
202074
|
+
if (alt.reachable) return { ok: true };
|
|
202075
|
+
return { ok: false, reason: alt.reason ? `${alt.reason}\uFF08\u4E3B\u8FD0\u884C\u65F6\uFF1A${primaryReason}\uFF09` : primaryReason };
|
|
202057
202076
|
};
|
|
202058
202077
|
if (target.startsWith("actor:")) {
|
|
202059
202078
|
const actor = await actors.getActor(target);
|
|
@@ -204016,6 +204035,7 @@ function assembleDelegationsDomain(deps) {
|
|
|
204016
204035
|
...input.binaryFiles ? { binaryFiles: input.binaryFiles } : {}
|
|
204017
204036
|
}),
|
|
204018
204037
|
...deps.probeRuntime ? { probeRuntime: deps.probeRuntime } : {},
|
|
204038
|
+
...deps.probeAllocation ? { probeAllocation: deps.probeAllocation } : {},
|
|
204019
204039
|
// 轮次账本 = 「这一次派发到底有没有发生过」的权威答案(见 service 的 resolveChatTurns 注释)。
|
|
204020
204040
|
...deps.resolveChatTurns ? { resolveChatTurns: deps.resolveChatTurns } : {},
|
|
204021
204041
|
...deps.enabled ? { enabled: deps.enabled } : {},
|
|
@@ -210375,6 +210395,7 @@ async function startOasisServer(opts) {
|
|
|
210375
210395
|
// 按公司解析;单租户装配退回那唯一一个。两者都没有 → 域层自动关掉租约回收。
|
|
210376
210396
|
resolveChatTurns: async (companyId) => (opts.resolveChatTurns ? await opts.resolveChatTurns(companyId) : void 0) ?? opts.chatTurns,
|
|
210377
210397
|
...opts.delegationProbeRuntime ? { probeRuntime: opts.delegationProbeRuntime } : {},
|
|
210398
|
+
...opts.delegationProbeAllocation ? { probeAllocation: opts.delegationProbeAllocation } : {},
|
|
210378
210399
|
...opts.chatDelegationEnabled ? { enabled: opts.chatDelegationEnabled } : {},
|
|
210379
210400
|
...opts.delegationWorkdir ? { workdir: opts.delegationWorkdir } : {},
|
|
210380
210401
|
...opts.delegationMetrics ? { metrics: opts.delegationMetrics } : {}
|
|
@@ -242470,6 +242491,59 @@ var init_sender = __esm({
|
|
|
242470
242491
|
}
|
|
242471
242492
|
});
|
|
242472
242493
|
|
|
242494
|
+
// ../server/src/domains/runtime-allocation/faults.ts
|
|
242495
|
+
function faultReasonOfExit(reason, errorMessage2) {
|
|
242496
|
+
if (reason === "server-unreachable") return "node-error";
|
|
242497
|
+
if (reason === "auth-error") return "node-error";
|
|
242498
|
+
if (reason === "rate-limit") return "quota-exhausted";
|
|
242499
|
+
if (reason === "error" && errorMessage2?.startsWith("\u8FD9\u4E00\u8F6E\u6CA1\u80FD\u5728\u8282\u70B9\u4E0A\u542F\u52A8\uFF1A")) return "startup-failed";
|
|
242500
|
+
return null;
|
|
242501
|
+
}
|
|
242502
|
+
function probeIntervalMs(attempt) {
|
|
242503
|
+
const raw = RUNTIME_FAULT_PROBE_INITIAL_MS * 2 ** Math.max(0, attempt - 1);
|
|
242504
|
+
return Math.min(raw, RUNTIME_FAULT_PROBE_MAX_INTERVAL_MS);
|
|
242505
|
+
}
|
|
242506
|
+
function faultAfterFailure(previous3, input, now) {
|
|
242507
|
+
const nowIso = new Date(now).toISOString();
|
|
242508
|
+
const markedAt = previous3?.markedAt ?? nowIso;
|
|
242509
|
+
const attempts = (previous3?.attempts ?? 0) + 1;
|
|
242510
|
+
const base = {
|
|
242511
|
+
nodeId: input.nodeId,
|
|
242512
|
+
runtimeKind: input.runtimeKind,
|
|
242513
|
+
reason: input.reason,
|
|
242514
|
+
...input.detail ? { detail: input.detail.slice(0, 500) } : {},
|
|
242515
|
+
markedAt,
|
|
242516
|
+
attempts,
|
|
242517
|
+
awaitingManual: false
|
|
242518
|
+
};
|
|
242519
|
+
if (RUNTIME_FAULT_RECOVERY[input.reason] === "manual") {
|
|
242520
|
+
return { ...base, awaitingManual: true };
|
|
242521
|
+
}
|
|
242522
|
+
const elapsed = now - Date.parse(markedAt);
|
|
242523
|
+
if (Number.isFinite(elapsed) && elapsed >= RUNTIME_FAULT_PROBE_MAX_TOTAL_MS) {
|
|
242524
|
+
return { ...base, awaitingManual: true };
|
|
242525
|
+
}
|
|
242526
|
+
return { ...base, nextProbeAt: new Date(now + probeIntervalMs(attempts)).toISOString() };
|
|
242527
|
+
}
|
|
242528
|
+
function quotaClearsFault(fault, quotaStatus) {
|
|
242529
|
+
if (RUNTIME_FAULT_RECOVERY[fault.reason] !== "readonly-probe" && fault.reason !== "unauthenticated") return false;
|
|
242530
|
+
return quotaStatus === "ok";
|
|
242531
|
+
}
|
|
242532
|
+
function describeFault(fault, now) {
|
|
242533
|
+
const head = `${fault.nodeId} \xB7 ${fault.runtimeKind} \u6545\u969C\uFF08${fault.reason}${fault.detail ? `\uFF1A${fault.detail}` : ""}\uFF09`;
|
|
242534
|
+
if (fault.awaitingManual) return `${head}\uFF0C\u5DF2\u8BD5 ${fault.attempts} \u6B21\u4ECD\u672A\u6062\u590D\uFF0C\u7B49\u4EBA\u5DE5\u6062\u590D`;
|
|
242535
|
+
if (!fault.nextProbeAt) return `${head}\uFF0C\u7B49\u8FDE\u63A5\u6062\u590D`;
|
|
242536
|
+
const waitMs = Date.parse(fault.nextProbeAt) - now;
|
|
242537
|
+
const mins = Math.max(0, Math.round(waitMs / 6e4));
|
|
242538
|
+
return `${head}\uFF0C\u5DF2\u8BD5 ${fault.attempts} \u6B21\uFF0C${mins} \u5206\u949F\u540E\u653E\u4E00\u6761\u65B0\u4F1A\u8BDD\u8FC7\u53BB\u8BD5`;
|
|
242539
|
+
}
|
|
242540
|
+
var init_faults = __esm({
|
|
242541
|
+
"../server/src/domains/runtime-allocation/faults.ts"() {
|
|
242542
|
+
"use strict";
|
|
242543
|
+
init_src();
|
|
242544
|
+
}
|
|
242545
|
+
});
|
|
242546
|
+
|
|
242473
242547
|
// ../server/src/domains/nodes/runtime-status.ts
|
|
242474
242548
|
function runtimeConnectionStatus(nodeId, connected) {
|
|
242475
242549
|
return connected.has(nodeId) ? "online" : "offline";
|
|
@@ -243170,6 +243244,21 @@ async function resolveRuntimeModels(deps, runtimeId) {
|
|
|
243170
243244
|
}
|
|
243171
243245
|
function nodesDomain(deps) {
|
|
243172
243246
|
const quotaByNode = /* @__PURE__ */ new Map();
|
|
243247
|
+
const faultByTarget = /* @__PURE__ */ new Map();
|
|
243248
|
+
const faultKey = (nodeId, runtimeKind) => `${nodeId}::${runtimeKind}`;
|
|
243249
|
+
const refreshFaults = async () => {
|
|
243250
|
+
const rows = await deps.nodeStore.listRuntimeFaults();
|
|
243251
|
+
faultByTarget.clear();
|
|
243252
|
+
for (const f2 of rows) faultByTarget.set(faultKey(f2.nodeId, f2.runtimeKind), f2);
|
|
243253
|
+
};
|
|
243254
|
+
void refreshFaults().catch((err) => console.warn(`[nodes] \u6545\u969C\u6807\u8BB0\u6C34\u5408\u5931\u8D25\uFF08\u9875\u9762\u4E0A\u6682\u65F6\u770B\u4E0D\u5230\u6545\u969C\u6761\uFF09\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243255
|
+
let faultsReadAtMs = 0;
|
|
243256
|
+
const FAULT_CACHE_TTL_MS = 1e4;
|
|
243257
|
+
const refreshFaultsIfStale = async () => {
|
|
243258
|
+
if (Date.now() - faultsReadAtMs < FAULT_CACHE_TTL_MS) return;
|
|
243259
|
+
faultsReadAtMs = Date.now();
|
|
243260
|
+
await refreshFaults().catch((err) => console.warn(`[nodes] \u6545\u969C\u6807\u8BB0\u91CD\u8BFB\u5931\u8D25\uFF08\u672C\u8F6E\u6CBF\u7528\u4E0A\u6B21\u7ED3\u679C\uFF09\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243261
|
+
};
|
|
243173
243262
|
void deps.nodeStore.listRuntimeQuotas().then((rows) => {
|
|
243174
243263
|
for (const row of rows) {
|
|
243175
243264
|
if (row.runtimeKind !== "codex" && row.runtimeKind !== "claude-code") continue;
|
|
@@ -243183,10 +243272,12 @@ function nodesDomain(deps) {
|
|
|
243183
243272
|
const nodeUpdates = /* @__PURE__ */ new Map();
|
|
243184
243273
|
function runtimeInfoOf(r) {
|
|
243185
243274
|
const quota = quotaByNode.get(r.nodeId)?.find((q2) => q2.kind === r.kind);
|
|
243275
|
+
const fault = faultByTarget.get(faultKey(r.nodeId, r.kind));
|
|
243186
243276
|
const activeRunCount = deps.activeRunCountOf?.(r.nodeId, r.kind) ?? 0;
|
|
243187
243277
|
const activeRuns = deps.activeRunsOf?.(r.nodeId, r.kind);
|
|
243188
243278
|
return {
|
|
243189
243279
|
...quota ? { quota: { status: quota.status, checkedAt: quota.checkedAt, windows: quota.windows, ...quota.retryAt ? { retryAt: quota.retryAt } : {} } } : {},
|
|
243280
|
+
...fault ? { fault } : {},
|
|
243190
243281
|
id: r.id,
|
|
243191
243282
|
nodeId: r.nodeId,
|
|
243192
243283
|
hostname: r.hostname,
|
|
@@ -243282,6 +243373,15 @@ function nodesDomain(deps) {
|
|
|
243282
243373
|
const quotas = sanitizeRuntimeQuotas(msg.quotas).filter((q2) => daemon.meta.adapters.includes(q2.kind));
|
|
243283
243374
|
quotaByNode.set(daemonId, quotas);
|
|
243284
243375
|
void nodeStore.saveRuntimeQuotas(daemonId, quotas.map((q2) => ({ ...q2, nodeId: daemonId, runtimeKind: q2.kind }))).catch((err) => console.warn(`[nodes] \u989D\u5EA6\u843D\u5E93\u5931\u8D25(node=${daemonId})\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243376
|
+
void (async () => {
|
|
243377
|
+
for (const q2 of quotas) {
|
|
243378
|
+
const existing = faultByTarget.get(faultKey(daemonId, q2.kind));
|
|
243379
|
+
if (!existing || !quotaClearsFault(existing, q2.status)) continue;
|
|
243380
|
+
await nodeStore.clearRuntimeFault(daemonId, q2.kind);
|
|
243381
|
+
faultByTarget.delete(faultKey(daemonId, q2.kind));
|
|
243382
|
+
console.log(`[nodes] \u53EA\u8BFB\u91C7\u96C6\u56DE ok\uFF0C\u81EA\u52A8\u89E3\u9664\u6545\u969C\uFF1A${daemonId} \xB7 ${q2.kind}\uFF08\u539F\u56E0 ${existing.reason}\uFF09`);
|
|
243383
|
+
}
|
|
243384
|
+
})().catch((err) => console.warn(`[nodes] \u6545\u969C\u81EA\u52A8\u89E3\u9664\u5931\u8D25(node=${daemonId})\uFF1A${err instanceof Error ? err.message : String(err)}`));
|
|
243285
243385
|
});
|
|
243286
243386
|
hub.addMessageListener(async (daemonId, msg) => {
|
|
243287
243387
|
if (msg.type !== "hello") return;
|
|
@@ -243385,6 +243485,7 @@ function nodesDomain(deps) {
|
|
|
243385
243485
|
await deps.nodeStore.setNodeRuntimesOnline(daemonId, true);
|
|
243386
243486
|
}
|
|
243387
243487
|
router.get("/api/nodes", async (req) => {
|
|
243488
|
+
await refreshFaultsIfStale();
|
|
243388
243489
|
const hub = deps.getHub();
|
|
243389
243490
|
if (hub) wireHub(hub);
|
|
243390
243491
|
const daemonVersionById = new Map((hub?.connectedDaemons() ?? []).map((d) => [d.daemonId, d.meta.daemonVersion]));
|
|
@@ -243654,6 +243755,20 @@ function nodesDomain(deps) {
|
|
|
243654
243755
|
deps.onStatusChange?.();
|
|
243655
243756
|
return { status: 200, body: { ok: true, maxConcurrent: limit } };
|
|
243656
243757
|
});
|
|
243758
|
+
router.post("/api/nodes/:nodeId/runtimes/:runtimeKind/recover", async (req) => {
|
|
243759
|
+
const nodeId = req.params["nodeId"];
|
|
243760
|
+
const runtimeKind = req.params["runtimeKind"];
|
|
243761
|
+
if (!nodeId || !runtimeKind) return { status: 400, body: { error: "missing nodeId/runtimeKind" } };
|
|
243762
|
+
const forbidden = await forbidUnlessOrgManager(req, "\u624B\u52A8\u6062\u590D\u8FD0\u884C\u65F6");
|
|
243763
|
+
if (forbidden) return forbidden;
|
|
243764
|
+
const scope2 = req.auth.companyId !== void 0 ? { companyId: req.auth.companyId } : void 0;
|
|
243765
|
+
const node2 = (await deps.nodeStore.listNodes(scope2)).find((n) => n.id === nodeId);
|
|
243766
|
+
if (!node2) return { status: 404, body: { error: "NODE_NOT_FOUND", message: "\u8282\u70B9\u4E0D\u5B58\u5728\u6216\u4E0D\u5C5E\u4E8E\u5F53\u524D\u7EC4\u7EC7" } };
|
|
243767
|
+
const cleared = await deps.nodeStore.clearRuntimeFault(nodeId, runtimeKind);
|
|
243768
|
+
faultByTarget.delete(faultKey(nodeId, runtimeKind));
|
|
243769
|
+
deps.onStatusChange?.();
|
|
243770
|
+
return { status: 200, body: { ok: true, cleared } };
|
|
243771
|
+
});
|
|
243657
243772
|
router.delete("/api/nodes/:nodeId", async (req) => {
|
|
243658
243773
|
const nodeId = req.params["nodeId"];
|
|
243659
243774
|
if (!nodeId) return { status: 400, body: { error: "missing nodeId" } };
|
|
@@ -243832,6 +243947,7 @@ var init_routes7 = __esm({
|
|
|
243832
243947
|
"../server/src/domains/nodes/routes.ts"() {
|
|
243833
243948
|
"use strict";
|
|
243834
243949
|
init_src();
|
|
243950
|
+
init_faults();
|
|
243835
243951
|
init_runtime_status();
|
|
243836
243952
|
import_node_crypto61 = require("node:crypto");
|
|
243837
243953
|
import_node_fs17 = require("node:fs");
|
|
@@ -244203,6 +244319,52 @@ function createRuntimeAllocator(ports) {
|
|
|
244203
244319
|
noteSettled(unitOfWork) {
|
|
244204
244320
|
units.delete(unitOfWork);
|
|
244205
244321
|
},
|
|
244322
|
+
/**
|
|
244323
|
+
* 一次失败的**完整处置**(ADR 0616 D8):记进这件活的失败链(下一次换候选)**并**给这个目标
|
|
244324
|
+
* 落一条故障行(跨会话生效——别的活也别再往这台机器上派)。
|
|
244325
|
+
*
|
|
244326
|
+
* 两件事必须同一处做:只记失败链的话,这台机器的坏只对「这一件活」成立,下一件活照样一头撞上去;
|
|
244327
|
+
* 只落故障行的话,这一件活的重试计数没人管,撞满上限也不会停。
|
|
244328
|
+
*
|
|
244329
|
+
* 故障落库失败只告警、不抛:失败链已经记上了,换候选照常。
|
|
244330
|
+
*/
|
|
244331
|
+
async recordFailure(input) {
|
|
244332
|
+
const now = input.now ?? Date.now();
|
|
244333
|
+
const chained = this.noteFailure(input);
|
|
244334
|
+
const target = input.target ?? units.get(input.unitOfWork)?.lastAssigned;
|
|
244335
|
+
const reason = faultReasonOfExit(input.reason, input.errorMessage);
|
|
244336
|
+
if (!target || !reason || !ports.saveFault) return { chained, fault: null };
|
|
244337
|
+
try {
|
|
244338
|
+
const previous3 = await ports.getFault?.(target) ?? null;
|
|
244339
|
+
const fault = faultAfterFailure(previous3, {
|
|
244340
|
+
...target,
|
|
244341
|
+
reason,
|
|
244342
|
+
...input.errorMessage?.trim() ? { detail: input.errorMessage.trim() } : {}
|
|
244343
|
+
}, now);
|
|
244344
|
+
await ports.saveFault(fault);
|
|
244345
|
+
log3(`[runtime-alloc] ${describeFault(fault, now)}`);
|
|
244346
|
+
return { chained, fault };
|
|
244347
|
+
} catch (err) {
|
|
244348
|
+
log3(`[runtime-alloc] \u6545\u969C\u843D\u5E93\u5931\u8D25\uFF08\u6362\u5019\u9009\u7167\u5E38\uFF09${target.nodeId}\xB7${target.runtimeKind}\uFF1A${String(err)}`);
|
|
244349
|
+
return { chained, fault: null };
|
|
244350
|
+
}
|
|
244351
|
+
},
|
|
244352
|
+
/**
|
|
244353
|
+
* 这件活这一轮**不是因为 runtime/节点**结束的 → 丢掉失败链,并把上次那个目标的故障行解除掉。
|
|
244354
|
+
*
|
|
244355
|
+
* 「跑成了」就是半开试探要的那个答案:到点放一条新会话过去试、它成了 → 解除。这条路与只读采集
|
|
244356
|
+
* 那条(额度/登录回 ok 自动解除)并列,覆盖的正是**采集看不出来**的那一类。
|
|
244357
|
+
*/
|
|
244358
|
+
async recordSettled(unitOfWork, target) {
|
|
244359
|
+
const resolved = target ?? units.get(unitOfWork)?.lastAssigned;
|
|
244360
|
+
this.noteSettled(unitOfWork);
|
|
244361
|
+
if (!resolved || !ports.clearFault) return;
|
|
244362
|
+
try {
|
|
244363
|
+
await ports.clearFault(resolved);
|
|
244364
|
+
} catch (err) {
|
|
244365
|
+
log3(`[runtime-alloc] \u6545\u969C\u89E3\u9664\u5931\u8D25 ${resolved.nodeId}\xB7${resolved.runtimeKind}\uFF1A${String(err)}`);
|
|
244366
|
+
}
|
|
244367
|
+
},
|
|
244206
244368
|
/** 只给测试与诊断用:这件活当前失败链的账。 */
|
|
244207
244369
|
inspect(unitOfWork) {
|
|
244208
244370
|
const s2 = units.get(unitOfWork);
|
|
@@ -244217,6 +244379,7 @@ var init_service7 = __esm({
|
|
|
244217
244379
|
"use strict";
|
|
244218
244380
|
init_src();
|
|
244219
244381
|
init_allocate();
|
|
244382
|
+
init_faults();
|
|
244220
244383
|
init_errors6();
|
|
244221
244384
|
UNIT_STATE_TTL_MS = 2 * 60 * 6e4;
|
|
244222
244385
|
}
|
|
@@ -278738,7 +278901,13 @@ function refreshConnectorSkillsInBackground(dataDir, apply) {
|
|
|
278738
278901
|
}
|
|
278739
278902
|
var CONNECTOR_SKILL_REFRESH_MS = 24 * 60 * 6e4;
|
|
278740
278903
|
var SESSION_TOKEN_GRACE_MS = 15 * 6e4;
|
|
278741
|
-
function
|
|
278904
|
+
function allocatedTurnOf(req) {
|
|
278905
|
+
return Boolean(req.delegatedChildTurn) || req.executionProfile === "knowledge-single-turn" || req.executionProfile === "discussion-background";
|
|
278906
|
+
}
|
|
278907
|
+
function allocationUnitOfWorkOf(req) {
|
|
278908
|
+
return `chat-session:${req.chatSessionId ?? req.dispatchId ?? req.actorId}`;
|
|
278909
|
+
}
|
|
278910
|
+
function makeResumeRetryProxy(first, spawnFallback, runtimeRetry) {
|
|
278742
278911
|
const relay = () => {
|
|
278743
278912
|
const cbs = [];
|
|
278744
278913
|
const buf = [];
|
|
@@ -278787,8 +278956,11 @@ function makeResumeRetryProxy(first, spawnFallback) {
|
|
|
278787
278956
|
return chatExit?.reason === "session-not-found";
|
|
278788
278957
|
};
|
|
278789
278958
|
const done = first.done.catch(async (err) => {
|
|
278790
|
-
if (sawOutput
|
|
278791
|
-
const
|
|
278959
|
+
if (sawOutput) throw err;
|
|
278960
|
+
const exit3 = err instanceof Error ? err.chatExit ?? {} : {};
|
|
278961
|
+
const spawnSecond = isSessionNotFound2(err) && spawnFallback ? spawnFallback : runtimeRetry?.shouldRetry({ ...exit3.reason ? { reason: exit3.reason } : {}, ...exit3.detail ?? exit3.summary ? { detail: exit3.detail ?? exit3.summary } : {} }) ? () => runtimeRetry.spawn({ ...exit3.reason ? { reason: exit3.reason } : {}, ...exit3.detail ?? exit3.summary ? { detail: exit3.detail ?? exit3.summary } : {} }) : null;
|
|
278962
|
+
if (!spawnSecond) throw err;
|
|
278963
|
+
const second = await spawnSecond();
|
|
278792
278964
|
active = second;
|
|
278793
278965
|
wire(second);
|
|
278794
278966
|
await second.done;
|
|
@@ -281138,6 +281310,13 @@ async function startServe(opts) {
|
|
|
281138
281310
|
},
|
|
281139
281311
|
faults: () => nodeStore.listRuntimeFaults({ companyId }),
|
|
281140
281312
|
quotas: () => nodeStore.listRuntimeQuotas({ companyId }),
|
|
281313
|
+
// 故障的读 / 写 / 解除(ADR 0616 D8):三个一起接,缺一个就会出现「标了但下次算不出间隔」
|
|
281314
|
+
// 或者「跑成了也不解除」这种半拉子状态。
|
|
281315
|
+
getFault: async (t) => (await nodeStore.listRuntimeFaults({ nodeId: t.nodeId })).find((f2) => f2.runtimeKind === t.runtimeKind) ?? null,
|
|
281316
|
+
saveFault: (fault) => nodeStore.upsertRuntimeFault(fault),
|
|
281317
|
+
clearFault: async (t) => {
|
|
281318
|
+
await nodeStore.clearRuntimeFault(t.nodeId, t.runtimeKind);
|
|
281319
|
+
},
|
|
281141
281320
|
actorName: async (actorId) => (await (await actors.resolveCtx(companyId)).service.getActor(actorId).catch(() => null))?.name ?? null
|
|
281142
281321
|
});
|
|
281143
281322
|
runtimeAllocators.set(companyId, allocator);
|
|
@@ -281851,6 +282030,25 @@ async function startServe(opts) {
|
|
|
281851
282030
|
}
|
|
281852
282031
|
return { reachable: true };
|
|
281853
282032
|
},
|
|
282033
|
+
/**
|
|
282034
|
+
* ★ ADR 0616:上面那道只问主绑定那台机器。候选 runtime 上线之后「主的机器离线」不再等于
|
|
282035
|
+
* 「这位专家派不出去」——这一问把**运行时分配**接进委派的第一道闸。
|
|
282036
|
+
*
|
|
282037
|
+
* 不接的后果 2026-09-23 在 v2 didi 上实测过:主绑定改到离线节点、候选配在在线节点,
|
|
282038
|
+
* `oasis delegate` 仍当场回 `EXPERT_UNREACHABLE`「绑定节点 … 当前不在线」,分配那一段
|
|
282039
|
+
* **一行都没跑到**。
|
|
282040
|
+
*
|
|
282041
|
+
* 用 `tryAllocate`(不抛、不消耗重试次数):这里只是问「挑不挑得出来」,真正的分配在
|
|
282042
|
+
* 那一轮派发时再跑一次——两次之间机器状态可能变,以派发那一次为准。
|
|
282043
|
+
*/
|
|
282044
|
+
delegationProbeAllocation: async ({ actorId, companyId }) => {
|
|
282045
|
+
const result = await runtimeAllocatorFor(companyId ?? defaultCompanyId).tryAllocate({
|
|
282046
|
+
actorId,
|
|
282047
|
+
unitOfWork: `delegation-preflight:${actorId}`
|
|
282048
|
+
});
|
|
282049
|
+
if (result.ok) return { reachable: true };
|
|
282050
|
+
return { reachable: false, reason: describeRejections(result.rejections) || "\u6CA1\u6709\u53EF\u7528\u7684\u8FD0\u884C\u65F6\u76EE\u6807" };
|
|
282051
|
+
},
|
|
281854
282052
|
// ADR 会话投递 D4:/api/chat 与渠道入站据它占同一个会话槽(自动化侧经 automations 域接同一实例)。
|
|
281855
282053
|
chatTurns: chatTurnStore,
|
|
281856
282054
|
// ADR 0176 第二步(缺陷 B):按当前公司解析轮次账本——turn 必须与它的会话同 schema,
|
|
@@ -282363,6 +282561,16 @@ async function startServe(opts) {
|
|
|
282363
282561
|
const isolatedKnowledgeTurn = executionProfile === "knowledge-single-turn";
|
|
282364
282562
|
const discussionBackground = executionProfile === "discussion-background";
|
|
282365
282563
|
const isolatedBackground = isolatedKnowledgeTurn || discussionBackground;
|
|
282564
|
+
const allocatedTurn = allocatedTurnOf({
|
|
282565
|
+
...delegatedChildTurn ? { delegatedChildTurn } : {},
|
|
282566
|
+
...executionProfile ? { executionProfile } : {}
|
|
282567
|
+
});
|
|
282568
|
+
const allocationWhat = delegatedChildTurn ? "\u59D4\u6D3E\u5B50\u4F1A\u8BDD" : isolatedKnowledgeTurn ? "\u77E5\u8BC6\u5E93\u5355\u8F6E" : "\u540E\u53F0\u6574\u7406";
|
|
282569
|
+
const allocationUnitOfWork = allocationUnitOfWorkOf({
|
|
282570
|
+
...chatSessionId ? { chatSessionId } : {},
|
|
282571
|
+
...requestDispatchId ? { dispatchId: requestDispatchId } : {},
|
|
282572
|
+
actorId
|
|
282573
|
+
});
|
|
282366
282574
|
const runCompanyId = companyId ?? defaultCompanyId;
|
|
282367
282575
|
const actorCtxForCompany = await actors.resolveCtx(runCompanyId);
|
|
282368
282576
|
const actorService = actorCtxForCompany.service;
|
|
@@ -282420,25 +282628,25 @@ async function startServe(opts) {
|
|
|
282420
282628
|
});
|
|
282421
282629
|
} catch (e) {
|
|
282422
282630
|
if (e instanceof SessionRuntimeOverrideError) {
|
|
282423
|
-
if (!
|
|
282424
|
-
console.log(`[runtime-alloc] \
|
|
282631
|
+
if (!allocatedTurn) throw new ApiError(e.status, e.code, e.message);
|
|
282632
|
+
console.log(`[runtime-alloc] ${allocationWhat}\u9489\u4F4F\u7684\u76EE\u6807\u4E0D\u53EF\u7528\uFF08${e.code}\uFF09\uFF0C\u4EA4\u7ED9\u5206\u914D\u6311\u5019\u9009`);
|
|
282425
282633
|
resolved = null;
|
|
282426
282634
|
} else throw e;
|
|
282427
282635
|
}
|
|
282428
|
-
if (!resolved && !
|
|
282636
|
+
if (!resolved && !allocatedTurn) {
|
|
282429
282637
|
throw new ApiError(409, "ACTOR_BINDING_MISSING", `\u8FD9\u4F4D Agent \u8FD8\u6CA1\u6709\u7ED1\u5B9A\u8FD0\u884C\u65F6\uFF0C\u8BF7${chatRebindHint("\u7ED1\u5B9A\u4E00\u4E2A\u53EF\u7528\u7684\u8FD0\u884C\u65F6")}\u540E\u518D\u53D1`);
|
|
282430
282638
|
}
|
|
282431
282639
|
binding = resolved?.binding ?? { actorId, nodeId: "", runtimeKind: "", status: "active" };
|
|
282432
282640
|
runtimeTargetSource = resolved?.source ?? "actor_binding";
|
|
282433
|
-
if (
|
|
282641
|
+
if (allocatedTurn) {
|
|
282434
282642
|
const pinned = sessionRuntimeOverride(priorChatSession);
|
|
282435
282643
|
const primary = resolved ? { nodeId: resolved.binding.nodeId, runtimeKind: resolved.binding.runtimeKind } : pinned ? { nodeId: pinned.nodeId, runtimeKind: pinned.runtimeKind } : null;
|
|
282436
282644
|
try {
|
|
282437
282645
|
const allocated = await runtimeAllocatorFor(runCompanyId).allocateFor({
|
|
282438
282646
|
actorId,
|
|
282439
282647
|
// 「同一件活」= 这条子会话。重试上限算在它头上,不与同一位专家的别条委派互相扣次数。
|
|
282440
|
-
unitOfWork:
|
|
282441
|
-
what:
|
|
282648
|
+
unitOfWork: allocationUnitOfWork,
|
|
282649
|
+
what: allocationWhat,
|
|
282442
282650
|
primary
|
|
282443
282651
|
});
|
|
282444
282652
|
if (allocated.binding.nodeId !== binding.nodeId || allocated.binding.runtimeKind !== binding.runtimeKind) {
|
|
@@ -282448,7 +282656,7 @@ async function startServe(opts) {
|
|
|
282448
282656
|
}
|
|
282449
282657
|
} catch (e) {
|
|
282450
282658
|
if (e instanceof RuntimeAllocationError) throw new ApiError(e.status === 503 ? 503 : 409, e.code, e.message);
|
|
282451
|
-
console.warn(`[runtime-alloc] \
|
|
282659
|
+
console.warn(`[runtime-alloc] ${allocationWhat}\u5206\u914D\u5931\u8D25\uFF1A${String(e)}`);
|
|
282452
282660
|
if (!resolved) throw e;
|
|
282453
282661
|
}
|
|
282454
282662
|
}
|
|
@@ -282883,23 +283091,16 @@ async function startServe(opts) {
|
|
|
282883
283091
|
const done = new Promise((resolve11, reject) => {
|
|
282884
283092
|
handle.onExit((info) => {
|
|
282885
283093
|
chatLiveSessions.delete(chatJobKey);
|
|
282886
|
-
if (
|
|
282887
|
-
|
|
282888
|
-
|
|
282889
|
-
|
|
282890
|
-
|
|
282891
|
-
|
|
282892
|
-
|
|
282893
|
-
|
|
282894
|
-
|
|
282895
|
-
|
|
282896
|
-
});
|
|
282897
|
-
} else {
|
|
282898
|
-
allocator.noteSettled(unitOfWork);
|
|
282899
|
-
}
|
|
282900
|
-
} catch (err) {
|
|
282901
|
-
console.warn(`[runtime-alloc] \u59D4\u6D3E\u5B50\u4F1A\u8BDD\u5931\u8D25\u56DE\u62A5\u51FA\u9519\uFF08\u4E0D\u5F71\u54CD\u6536\u5C3E\uFF09\uFF1A${String(err)}`);
|
|
282902
|
-
}
|
|
283094
|
+
if (allocatedTurn) {
|
|
283095
|
+
const unitOfWork = allocationUnitOfWork;
|
|
283096
|
+
const allocator = runtimeAllocatorFor(runCompanyId);
|
|
283097
|
+
const target = { nodeId: binding.nodeId, runtimeKind: binding.runtimeKind };
|
|
283098
|
+
void (isRuntimeClassFailure(info.reason, info.errorMessage) ? allocator.recordFailure({
|
|
283099
|
+
unitOfWork,
|
|
283100
|
+
target,
|
|
283101
|
+
...info.reason ? { reason: info.reason } : {},
|
|
283102
|
+
...info.errorMessage ? { errorMessage: info.errorMessage } : {}
|
|
283103
|
+
}) : allocator.recordSettled(unitOfWork, target)).catch((err) => console.warn(`[runtime-alloc] \u4F1A\u8BDD\u5931\u8D25\u5904\u7F6E\u51FA\u9519\uFF08\u4E0D\u5F71\u54CD\u6536\u5C3E\uFF09\uFF1A${String(err)}`));
|
|
282903
283104
|
}
|
|
282904
283105
|
setTimeout(() => {
|
|
282905
283106
|
chatRunByDispatch.delete(handle.id);
|
|
@@ -283010,8 +283211,37 @@ async function startServe(opts) {
|
|
|
283010
283211
|
firstPromise = dispatchChatImpl(chatReq);
|
|
283011
283212
|
}
|
|
283012
283213
|
const first = await firstPromise;
|
|
283013
|
-
|
|
283014
|
-
|
|
283214
|
+
const runtimeRetry = allocatedTurnOf({
|
|
283215
|
+
...chatReq.delegatedChildTurn ? { delegatedChildTurn: chatReq.delegatedChildTurn } : {},
|
|
283216
|
+
...chatReq.executionProfile ? { executionProfile: chatReq.executionProfile } : {}
|
|
283217
|
+
}) ? (() => {
|
|
283218
|
+
const unitOfWork = allocationUnitOfWorkOf({
|
|
283219
|
+
...chatReq.chatSessionId ? { chatSessionId: chatReq.chatSessionId } : {},
|
|
283220
|
+
...chatReq.dispatchId ? { dispatchId: chatReq.dispatchId } : {},
|
|
283221
|
+
actorId: chatReq.actorId
|
|
283222
|
+
});
|
|
283223
|
+
const allocator = runtimeAllocatorFor(chatReq.companyId ?? defaultCompanyId);
|
|
283224
|
+
return {
|
|
283225
|
+
shouldRetry: (exit3) => isRuntimeClassFailure(exit3.reason, exit3.detail) && canRetryAllocation(allocator.inspect(unitOfWork)?.failures ?? 0),
|
|
283226
|
+
spawn: async (exit3) => {
|
|
283227
|
+
const last = allocator.inspect(unitOfWork)?.lastAssigned;
|
|
283228
|
+
if (last) {
|
|
283229
|
+
allocator.noteFailure({
|
|
283230
|
+
unitOfWork,
|
|
283231
|
+
target: last,
|
|
283232
|
+
...exit3.reason ? { reason: exit3.reason } : {},
|
|
283233
|
+
...exit3.detail ? { errorMessage: exit3.detail } : {}
|
|
283234
|
+
});
|
|
283235
|
+
}
|
|
283236
|
+
console.log(`[runtime-alloc] ${unitOfWork} \u56E0 ${exit3.reason ?? "runtime \u6545\u969C"} \u5931\u8D25\u4E14\u672A\u4EA7\u51FA\u4EFB\u4F55\u8F93\u51FA \u2192 \u6362\u5019\u9009\u91CD\u8D77\u8FD9\u4E00\u8F6E`);
|
|
283237
|
+
return dispatchChatImpl(chatReq);
|
|
283238
|
+
}
|
|
283239
|
+
};
|
|
283240
|
+
})() : void 0;
|
|
283241
|
+
if (!chatReq.sessionId) return runtimeRetry ? makeResumeRetryProxy(first, null, runtimeRetry) : first;
|
|
283242
|
+
if (chatReq.executionProfile === "knowledge-single-turn" || chatReq.executionProfile === "discussion-background") {
|
|
283243
|
+
return runtimeRetry ? makeResumeRetryProxy(first, null, runtimeRetry) : first;
|
|
283244
|
+
}
|
|
283015
283245
|
return makeResumeRetryProxy(first, async () => {
|
|
283016
283246
|
const message = chatReq.fallbackMessage ?? await buildResumeFallbackMessage(chatReq) ?? chatReq.message;
|
|
283017
283247
|
const { admitDispatch: _alreadyAdmitted, ...fallbackReq } = chatReq;
|
|
@@ -283022,7 +283252,7 @@ async function startServe(opts) {
|
|
|
283022
283252
|
message,
|
|
283023
283253
|
...chatReq.dispatchId && isHumanSubmitDispatchId(chatReq.dispatchId) ? { dispatchId: resumeFallbackDispatchId(chatReq.dispatchId) } : {}
|
|
283024
283254
|
});
|
|
283025
|
-
});
|
|
283255
|
+
}, runtimeRetry);
|
|
283026
283256
|
},
|
|
283027
283257
|
trajectoryFeed: {
|
|
283028
283258
|
subscribe(sessionId, listener) {
|
|
@@ -283349,15 +283579,12 @@ async function startServe(opts) {
|
|
|
283349
283579
|
if (mapping && exitBus) {
|
|
283350
283580
|
if (mapping.artifactId && mapping.companyId) {
|
|
283351
283581
|
const allocator = runtimeAllocatorFor(mapping.companyId);
|
|
283352
|
-
|
|
283353
|
-
|
|
283354
|
-
|
|
283355
|
-
|
|
283356
|
-
|
|
283357
|
-
|
|
283358
|
-
} else {
|
|
283359
|
-
allocator.noteSettled(mapping.artifactId);
|
|
283360
|
-
}
|
|
283582
|
+
const unitOfWork = mapping.artifactId;
|
|
283583
|
+
void (isRuntimeClassFailure(msg.info.reason, msg.info.errorMessage) ? allocator.recordFailure({
|
|
283584
|
+
unitOfWork,
|
|
283585
|
+
...msg.info.reason ? { reason: msg.info.reason } : {},
|
|
283586
|
+
...msg.info.errorMessage ? { errorMessage: msg.info.errorMessage } : {}
|
|
283587
|
+
}) : allocator.recordSettled(unitOfWork)).catch((err) => console.warn(`[runtime-alloc] \u5DE5\u5355\u6D3E\u53D1\u5931\u8D25\u5904\u7F6E\u51FA\u9519\uFF08\u4E0D\u5F71\u54CD\u6536\u53E3\uFF09\uFF1A${String(err)}`));
|
|
283361
283588
|
}
|
|
283362
283589
|
await exitBus.submit({
|
|
283363
283590
|
companyId: "",
|
|
@@ -290753,7 +290980,7 @@ async function waitForCommandAuthorization(base, headers, draftId, fallback) {
|
|
|
290753
290980
|
}
|
|
290754
290981
|
}
|
|
290755
290982
|
}
|
|
290756
|
-
var CLI_VERSION = "2.2.
|
|
290983
|
+
var CLI_VERSION = "2.2.74" ? "2.2.74" : "dev";
|
|
290757
290984
|
function apiHeaders(token, company) {
|
|
290758
290985
|
const headers = {
|
|
290759
290986
|
authorization: `Bearer ${token}`,
|
|
@@ -294559,7 +294786,7 @@ function shimScript() {
|
|
|
294559
294786
|
}
|
|
294560
294787
|
|
|
294561
294788
|
// src/index.ts
|
|
294562
|
-
var PKG_VERSION = true ? "2.2.
|
|
294789
|
+
var PKG_VERSION = true ? "2.2.74" : "dev";
|
|
294563
294790
|
var LOCAL_BIN = localBin();
|
|
294564
294791
|
var NPM_PREFIX = npmPrefix();
|
|
294565
294792
|
var INSTANCE = DEFAULT_INSTANCE;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "oasis_test_v2",
|
|
3
|
-
"version": "2.2.
|
|
3
|
+
"version": "2.2.74",
|
|
4
4
|
"description": "Oasis node daemon + CLI — background daemon, auto-start, full server CLI",
|
|
5
5
|
"bin": {
|
|
6
6
|
"oasis": "./dist/index.js"
|
|
@@ -26,6 +26,6 @@
|
|
|
26
26
|
"node": ">=20"
|
|
27
27
|
},
|
|
28
28
|
"oasisRelease": {
|
|
29
|
-
"sourceHead": "
|
|
29
|
+
"sourceHead": "c5a1a29b8950671d6033bab4a5c1bf9f3b2ff51a"
|
|
30
30
|
}
|
|
31
31
|
}
|