niceeval 0.10.3-canary.6 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/niceeval.js +9 -4
- package/dist/agents/types.d.ts +5 -4
- package/dist/context/turn-errors.d.ts +27 -23
- package/dist/i18n/en.d.ts +15 -0
- package/dist/i18n/en.js +16 -1
- package/dist/i18n/zh-CN.d.ts +16 -1
- package/dist/i18n/zh-CN.js +16 -1
- package/dist/o11y/derive.js +28 -24
- package/dist/o11y/types.d.ts +8 -5
- package/dist/report/components/attempt-detail/UsageTable.js +4 -7
- package/dist/report/components/attempt-detail/compute.d.ts +2 -2
- package/dist/report/components/attempt-detail/compute.js +2 -6
- package/dist/report/components/attempt-detail/faces.js +7 -7
- package/dist/report/components/attempt-detail/index.js +0 -3
- package/dist/report/components/entity-lists/EvalList.js +0 -0
- package/dist/report/components/metric-views/compute.js +1 -1
- package/dist/report/model/types.d.ts +3 -4
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +6 -0
- package/dist/results/select.js +8 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +105 -7
- package/dist/sandbox/errors.d.ts +29 -0
- package/dist/sandbox/resolve.d.ts +9 -0
- package/dist/shared/failure-class.d.ts +91 -0
- package/dist/types.d.ts +1 -0
- package/dist/util.d.ts +3 -2
- package/dist/util.js +31 -5
- package/docs-site/zh/explanation/runner.mdx +35 -0
- package/docs-site/zh/reference/cli.mdx +10 -1
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/troubleshooting/debugging.mdx +11 -0
- package/docs-site/zh/tutorials/viewing-results.mdx +35 -3
- package/package.json +28 -24
- package/src/agents/ai-sdk.test.ts +26 -0
- package/src/agents/ai-sdk.ts +7 -4
- package/src/agents/index.ts +5 -3
- package/src/agents/langgraph.test.ts +30 -0
- package/src/agents/langgraph.ts +5 -2
- package/src/agents/openai-compat.test.ts +35 -0
- package/src/agents/openai-compat.ts +16 -4
- package/src/agents/sdk-streams.test.ts +66 -0
- package/src/agents/sdk-streams.ts +3 -1
- package/src/agents/types.ts +5 -4
- package/src/cli.ts +81 -16
- package/src/context/context.test.ts +34 -0
- package/src/context/context.ts +14 -5
- package/src/context/send-retry.test.ts +86 -0
- package/src/context/send-retry.ts +37 -12
- package/src/context/session.ts +24 -0
- package/src/context/turn-errors.test.ts +124 -17
- package/src/context/turn-errors.ts +60 -50
- package/src/define.ts +5 -0
- package/src/i18n/en.ts +22 -1
- package/src/i18n/zh-CN.ts +22 -1
- package/src/index.ts +8 -0
- package/src/o11y/cost.ts +4 -2
- package/src/o11y/derive.test.ts +40 -0
- package/src/o11y/derive.ts +28 -22
- package/src/o11y/otlp/sandbox-receiver.test.ts +201 -0
- package/src/o11y/otlp/sandbox-receiver.ts +73 -27
- package/src/o11y/parsers/bub.test.ts +30 -0
- package/src/o11y/parsers/bub.ts +5 -2
- package/src/o11y/parsers/codex.test.ts +19 -0
- package/src/o11y/parsers/codex.ts +5 -2
- package/src/o11y/types.ts +8 -5
- package/src/report/components/attempt-detail/UsageTable.tsx +4 -6
- package/src/report/components/attempt-detail/attempt-components.test.tsx +5 -8
- package/src/report/components/attempt-detail/compute.ts +2 -7
- package/src/report/components/attempt-detail/faces.ts +7 -7
- package/src/report/components/attempt-detail/index.tsx +0 -3
- package/src/report/components/entity-lists/EvalList.tsx +0 -0
- package/src/report/components/metric-views/compute.ts +1 -1
- package/src/report/model/types.ts +3 -4
- package/src/results/format.ts +9 -2
- package/src/results/index.ts +2 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +132 -21
- package/src/results/select.ts +12 -0
- package/src/results/skipped-notice.ts +0 -0
- package/src/runner/attempt.test.ts +116 -0
- package/src/runner/attempt.ts +89 -8
- package/src/runner/discover.ts +21 -3
- package/src/runner/feedback/coordinator.ts +24 -2
- package/src/runner/feedback/eval-conclusions.ts +6 -3
- package/src/runner/feedback/human.test.ts +300 -6
- package/src/runner/feedback/human.ts +132 -30
- package/src/runner/feedback/json.test.ts +127 -2
- package/src/runner/feedback/json.ts +51 -3
- package/src/runner/feedback/reducer.test.ts +328 -29
- package/src/runner/feedback/reducer.ts +84 -9
- package/src/runner/feedback/sink.ts +39 -1
- package/src/runner/fingerprint.ts +49 -19
- package/src/runner/gate-lease.test.ts +510 -0
- package/src/runner/gate-lease.ts +350 -0
- package/src/runner/lock.test.ts +454 -0
- package/src/runner/lock.ts +288 -0
- package/src/runner/report.test.ts +1 -0
- package/src/runner/run.test.ts +2044 -9
- package/src/runner/run.ts +825 -61
- package/src/runner/teardown-registry.ts +20 -78
- package/src/runner/types.ts +103 -7
- package/src/sandbox/errors.test.ts +72 -0
- package/src/sandbox/errors.ts +83 -0
- package/src/sandbox/keep-registry.ts +22 -46
- package/src/sandbox/resolve.test.ts +100 -0
- package/src/sandbox/resolve.ts +84 -27
- package/src/shared/entry-file-store.test.ts +149 -0
- package/src/shared/entry-file-store.ts +117 -0
- package/src/shared/failure-class.test.ts +137 -0
- package/src/shared/failure-class.ts +175 -0
- package/src/show/index.ts +5 -6
- package/src/show/render.test.ts +116 -15
- package/src/show/render.ts +124 -35
- package/src/types.ts +9 -0
- package/src/util.ts +31 -4
- package/src/view/app/App.tsx +5 -1
- package/src/view/client-dist/app.js +1 -1
- package/src/view/data.ts +5 -9
- package/src/view/shared/types.ts +7 -1
- package/src/view/view-report.test.ts +54 -0
package/src/runner/run.test.ts
CHANGED
|
@@ -1,8 +1,33 @@
|
|
|
1
1
|
// cases: docs/engineering/testing/unit/experiments-runner.md
|
|
2
2
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
3
|
-
|
|
3
|
+
|
|
4
|
+
// 「取锁后重查携带」的读取面计数探针:两个 wrapper 都原样委派给真实实现,只在中间记一笔,
|
|
5
|
+
// 因此对其余用例完全透明。工厂是惰性的——没有任何模块导入被 mock 的模块时它根本不执行,
|
|
6
|
+
// 于是「派发路径上一次全树扫描都不做」这条断言的成本恒为零。
|
|
7
|
+
const readSurfaceCalls = vi.hoisted(() => ({ forCase: 0, perEval: 0 }));
|
|
8
|
+
vi.mock("../results/open.ts", async (importOriginal) => {
|
|
9
|
+
const actual = await importOriginal<typeof import("../results/open.ts")>();
|
|
10
|
+
return {
|
|
11
|
+
...actual,
|
|
12
|
+
loadLatestResultsForCase: (...args: Parameters<typeof actual.loadLatestResultsForCase>) => {
|
|
13
|
+
readSurfaceCalls.forCase += 1;
|
|
14
|
+
return actual.loadLatestResultsForCase(...args);
|
|
15
|
+
},
|
|
16
|
+
};
|
|
17
|
+
});
|
|
18
|
+
vi.mock("../view/data.ts", async (importOriginal) => {
|
|
19
|
+
const actual = await importOriginal<typeof import("../view/data.ts")>();
|
|
20
|
+
return {
|
|
21
|
+
...actual,
|
|
22
|
+
loadLatestResultsPerEval: (...args: Parameters<typeof actual.loadLatestResultsPerEval>) => {
|
|
23
|
+
readSurfaceCalls.perEval += 1;
|
|
24
|
+
return actual.loadLatestResultsPerEval(...args);
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
});
|
|
28
|
+
import { mkdtemp, mkdir, readdir, rm, writeFile } from "node:fs/promises";
|
|
4
29
|
import { tmpdir } from "node:os";
|
|
5
|
-
import { join } from "node:path";
|
|
30
|
+
import { basename, join } from "node:path";
|
|
6
31
|
import { fileURLToPath } from "node:url";
|
|
7
32
|
import { judgeProbeTargets, runEvals } from "./run.ts";
|
|
8
33
|
import { defineSandbox, defineSandboxAgent } from "../define.ts";
|
|
@@ -19,9 +44,14 @@ import {
|
|
|
19
44
|
type ExperimentProgressInput,
|
|
20
45
|
} from "./feedback/sink.ts";
|
|
21
46
|
import { drainExperimentTeardowns, pendingExperimentTeardownCount } from "./experiment-cleanup-registry.ts";
|
|
47
|
+
import { computeFingerprint } from "./fingerprint.ts";
|
|
48
|
+
import { locksDirOf, pendingHeldCaseLockCount, type CaseLockRecord } from "./lock.ts";
|
|
49
|
+
import { pendingHeldGateLeaseCount } from "./gate-lease.ts";
|
|
50
|
+
import { slugHashEntryId } from "../shared/entry-file-store.ts";
|
|
22
51
|
import type { CapturedEvalSource } from "./eval-source.ts";
|
|
23
52
|
import type { CarryPlan } from "./fingerprint.ts";
|
|
24
|
-
import
|
|
53
|
+
import { ExperimentFatalError, EvalFatalError } from "../shared/failure-class.ts";
|
|
54
|
+
import type { AgentRun, DiagnosticRecord, RunFeedbackPlan, RunFeedbackState, RunOptions } from "./types.ts";
|
|
25
55
|
import type {
|
|
26
56
|
Agent,
|
|
27
57
|
CommandResult,
|
|
@@ -640,9 +670,10 @@ describe("runEvals · budget-exhausted 永久事件按每个被跳过的 attempt
|
|
|
640
670
|
expect(diag?.count).toBe(3); // 三个 attempt 各发一次,去重折成同一个 key、count 累加到 3
|
|
641
671
|
expect(diag?.data).toMatchObject({ experimentId, spent: 0, unstarted: 3 });
|
|
642
672
|
|
|
643
|
-
// reducer 不变量:每条 budget-exhausted 把一个 attempt 从 queued 挪进
|
|
644
|
-
// (与 assembleInvocationCompletion() 读取 count 折算
|
|
645
|
-
|
|
673
|
+
// reducer 不变量:每条 budget-exhausted 把一个 attempt 从 queued 挪进 skipped —— 没派发就
|
|
674
|
+
// 没有 verdict,不冒充 passed/failed(与 assembleInvocationCompletion() 读取 count 折算
|
|
675
|
+
// InvocationCompletion.unstarted 的口径一致)。
|
|
676
|
+
expect(coordinator.state).toMatchObject({ total: 3, reused: 0, running: 0, queued: 0, passed: 0, failed: 0, errored: 0, skipped: 3 });
|
|
646
677
|
});
|
|
647
678
|
});
|
|
648
679
|
});
|
|
@@ -792,8 +823,8 @@ describe("runEvals · 携入数量少于本次请求的 runs 时,差额必须真
|
|
|
792
823
|
expect(matches[0]!.locator).toBe(staleLocator); // 携入结果原样透传
|
|
793
824
|
|
|
794
825
|
// 不变量:携入 1 + early-exit 回填 2 == 本次请求的 runs:3,不留没有解释的差额
|
|
795
|
-
// (queued 必须真正归零,不能停在「还差 2 个不知道去哪」)。
|
|
796
|
-
expect(coordinator.state).toMatchObject({ total: 3, reused: 1, running: 0, queued: 0,
|
|
826
|
+
// (queued 必须真正归零,不能停在「还差 2 个不知道去哪」)。回填的两轮没真跑,进 skipped。
|
|
827
|
+
expect(coordinator.state).toMatchObject({ total: 3, reused: 1, running: 0, queued: 0, passed: 0, failed: 0, errored: 0, skipped: 2 });
|
|
797
828
|
});
|
|
798
829
|
});
|
|
799
830
|
|
|
@@ -862,7 +893,8 @@ describe("runEvals · 携入数量少于本次请求的 runs 时,差额必须真
|
|
|
862
893
|
expect(matches.map((r) => r.attempt).sort()).toEqual([0, 1, 2]);
|
|
863
894
|
expect(matches.every((r) => r.verdict === "failed")).toBe(true);
|
|
864
895
|
|
|
865
|
-
|
|
896
|
+
// 差额两次真的跑了,各自落进 failed —— 携入那条的 verdict 留在 reused,不摊进结局项。
|
|
897
|
+
expect(coordinator.state).toMatchObject({ total: 3, reused: 1, running: 0, queued: 0, passed: 0, failed: 2, errored: 0, skipped: 0 });
|
|
866
898
|
// InvocationSummary 的三条 failed(1 carry + 2 fresh)与终局 handoff 的 FailureNotice 清单同口径。
|
|
867
899
|
// carry 不能只进 summary 计数而从 FAILURES / agent handoff 消失。
|
|
868
900
|
expect(coordinator.state.failures).toHaveLength(3);
|
|
@@ -1168,6 +1200,7 @@ describe("runEvals · 实验级 setup/teardown", () => {
|
|
|
1168
1200
|
progressEvents.push(input);
|
|
1169
1201
|
},
|
|
1170
1202
|
precheck() {},
|
|
1203
|
+
lockWait() {},
|
|
1171
1204
|
lifecycle() {},
|
|
1172
1205
|
});
|
|
1173
1206
|
try {
|
|
@@ -1414,6 +1447,7 @@ describe("runEvals · 强杀后的启动自愈(收尾登记的补执行)", () =>
|
|
|
1414
1447
|
},
|
|
1415
1448
|
experimentProgress() {},
|
|
1416
1449
|
precheck() {},
|
|
1450
|
+
lockWait() {},
|
|
1417
1451
|
lifecycle() {},
|
|
1418
1452
|
});
|
|
1419
1453
|
|
|
@@ -1538,6 +1572,7 @@ describe("runEvals · 强杀后的启动自愈(收尾登记的补执行)", () =>
|
|
|
1538
1572
|
},
|
|
1539
1573
|
experimentProgress() {},
|
|
1540
1574
|
precheck() {},
|
|
1575
|
+
lockWait() {},
|
|
1541
1576
|
lifecycle() {},
|
|
1542
1577
|
});
|
|
1543
1578
|
|
|
@@ -1599,6 +1634,7 @@ describe("runEvals · 强杀后的启动自愈(收尾登记的补执行)", () =>
|
|
|
1599
1634
|
},
|
|
1600
1635
|
experimentProgress() {},
|
|
1601
1636
|
precheck() {},
|
|
1637
|
+
lockWait() {},
|
|
1602
1638
|
lifecycle() {},
|
|
1603
1639
|
});
|
|
1604
1640
|
|
|
@@ -1978,3 +2014,2002 @@ describe("runEvals · 全局并发位在退避期间确实让给别的实验", (
|
|
|
1978
2014
|
}
|
|
1979
2015
|
});
|
|
1980
2016
|
});
|
|
2017
|
+
|
|
2018
|
+
// ═══════════════════════ 用例锁与并发 Invocation ═══════════════════════
|
|
2019
|
+
// 契约:docs/feature/experiments/architecture.md「并发 Invocation:用例锁」
|
|
2020
|
+
// 裁决出处:memory/case-lock-wait-not-skip-ruling.md(撞锁等待而非跳过、粒度是单条用例)
|
|
2021
|
+
// 覆盖规范:docs/engineering/testing/unit/experiments-runner.md「用例锁与并发 Invocation」
|
|
2022
|
+
//
|
|
2023
|
+
// 这里只覆盖 run.ts 对 lock.ts 的调度层接线:取锁时机(携带规划之后、preflight/实验级
|
|
2024
|
+
// setup 之前)、等待语义(不占位、elsewhere 计数)、释放后重查携带(carried/dispatched/
|
|
2025
|
+
// 部分携入)、执行模式组合(--force)、释放路径(即使合成 errored 结果也要放锁)。心跳续租 /
|
|
2026
|
+
// 过期判据 / 接管 rename 互斥等锁原语自身的机制由 lock.test.ts 覆盖,不在这里重复。
|
|
2027
|
+
|
|
2028
|
+
function caseLockPath(root: string, experimentId: string, evalId: string): string {
|
|
2029
|
+
// 必须与 lock.ts 私有的 caseLockEntryId 用完全相同的方式构造,否则读写的不是同一个文件。
|
|
2030
|
+
const id = slugHashEntryId(`${experimentId}-${evalId}`, [experimentId, evalId]);
|
|
2031
|
+
return join(locksDirOf(root), `${id}.json`);
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
/** 直接写一条锁记录,绕开 acquireCaseLock —— 模拟"另一个进程持有/曾经持有这把锁";心跳
|
|
2035
|
+
* 完全由测试摆布,不会被本进程续租,陈旧与否只取决于种下的 heartbeatAt,不依赖真实时间流逝。 */
|
|
2036
|
+
async function seedCaseLock(root: string, record: CaseLockRecord): Promise<void> {
|
|
2037
|
+
const dir = locksDirOf(root);
|
|
2038
|
+
await mkdir(dir, { recursive: true });
|
|
2039
|
+
await writeFile(caseLockPath(root, record.experimentId, record.evalId), JSON.stringify(record, null, 2), "utf-8");
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
function freshLockRecord(experimentId: string, evalId: string, overrides: Partial<CaseLockRecord> = {}): CaseLockRecord {
|
|
2043
|
+
const now = new Date().toISOString();
|
|
2044
|
+
return { experimentId, evalId, pid: 999_111, host: "other-host", startedAt: now, heartbeatAt: now, ...overrides };
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
function staleLockRecord(experimentId: string, evalId: string, overrides: Partial<CaseLockRecord> = {}): CaseLockRecord {
|
|
2048
|
+
// 落后 CASE_LOCK_STALE_MS(30_000ms)以上 —— 稳稳越过判死边界(严格 `>`,不是 `>=`)。
|
|
2049
|
+
const staleHeartbeat = new Date(Date.now() - 40_000).toISOString();
|
|
2050
|
+
return { experimentId, evalId, pid: 999_222, host: "dead-host", startedAt: staleHeartbeat, heartbeatAt: staleHeartbeat, ...overrides };
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
/** 共享的 run() helper 不透传 priorResults;用例锁"释放后重查携带"分支专门按
|
|
2054
|
+
* RunOptions.priorResults 是否为 undefined 分支(force 模式整段跳过重查,见 cli.ts 的
|
|
2055
|
+
* `flags.force ? undefined : ...`),需要直接控场,故另建一个不影响其它测试的 helper。 */
|
|
2056
|
+
async function runWithPriorResults(
|
|
2057
|
+
evals: DiscoveredEval[],
|
|
2058
|
+
agentRuns: AgentRun[],
|
|
2059
|
+
opts: { priorResults?: EvalResult[]; root?: string; signal?: AbortSignal; maxConcurrency?: number } = {},
|
|
2060
|
+
): Promise<{ summary: InvocationSummary; root: string }> {
|
|
2061
|
+
const root = opts.root ?? (await makeRoot());
|
|
2062
|
+
const config: Config = {};
|
|
2063
|
+
const runOpts: RunOptions = {
|
|
2064
|
+
config,
|
|
2065
|
+
evals,
|
|
2066
|
+
agentRuns,
|
|
2067
|
+
reporters: [{ reporter: Artifacts(root), name: "artifacts", required: false }],
|
|
2068
|
+
maxConcurrency: opts.maxConcurrency ?? 3,
|
|
2069
|
+
niceevalRoot: root,
|
|
2070
|
+
...(opts.priorResults !== undefined ? { priorResults: opts.priorResults } : {}),
|
|
2071
|
+
...(opts.signal ? { signal: opts.signal } : {}),
|
|
2072
|
+
};
|
|
2073
|
+
const summary = await runEvals(runOpts);
|
|
2074
|
+
return { summary, root };
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
async function lockFilesRemaining(root: string): Promise<string[]> {
|
|
2078
|
+
try {
|
|
2079
|
+
return await readdir(locksDirOf(root));
|
|
2080
|
+
} catch {
|
|
2081
|
+
return [];
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
|
|
2085
|
+
/** 锁目录里此刻被哪几条用例持有(条目文件名映回 evalId,按 `candidates` 的给定顺序返回)。
|
|
2086
|
+
* 多开场景的分段点是「对方已经放手」,而这件事只在磁盘上可观测:对方的 attempt 跑完(探针
|
|
2087
|
+
* 记到 started / inFlight 归零)只说明 test() 返回了,锁要到该用例全部 attempt 收尾之后才删。 */
|
|
2088
|
+
async function lockedEvalIds(root: string, experimentId: string, candidates: string[]): Promise<string[]> {
|
|
2089
|
+
const files = new Set(await lockFilesRemaining(root));
|
|
2090
|
+
return candidates.filter((id) => files.has(basename(caseLockPath(root, experimentId, id))));
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
/** 等一个要读磁盘的判据在假时钟上成立。推法与 `advanceOnFakeClock` 的收尾段相同(只喂真实
|
|
2094
|
+
* 轮次、每轮顺手推 1ms 假时钟接住收尾链上的短定时器),但把虚拟时间的消耗压到最低:等待
|
|
2095
|
+
* 期间本进程自己也持着锁,推过 30s 判死线会被对方当成过期锁接管,场景就变了。 */
|
|
2096
|
+
async function awaitDiskOnFakeClock(isDone: () => Promise<boolean>, budgetRealMs = 20_000): Promise<void> {
|
|
2097
|
+
const until = realDateNow() + budgetRealMs;
|
|
2098
|
+
while (realDateNow() < until) {
|
|
2099
|
+
if (await isDone()) return;
|
|
2100
|
+
await vi.advanceTimersByTimeAsync(1);
|
|
2101
|
+
await new Promise<void>((resolve) => realSetTimeout(resolve, 5));
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
/** 模块装载期抓住的真实 setTimeout / Date.now:`vi.useFakeTimers()` 换掉的是全局绑定,
|
|
2106
|
+
* 这两个函数引用仍指向原生实现,假时钟场景里用它们换真实的宏任务轮次与真实墙钟。 */
|
|
2107
|
+
const realSetTimeout = globalThis.setTimeout;
|
|
2108
|
+
const realDateNow = Date.now;
|
|
2109
|
+
|
|
2110
|
+
/** 把真实事件循环喂到 `isDone()` 成立或真实墙钟预算用完为止。 */
|
|
2111
|
+
async function realTicksFor(isDone: () => boolean, budgetMs: number): Promise<void> {
|
|
2112
|
+
const until = realDateNow() + budgetMs;
|
|
2113
|
+
while (!isDone() && realDateNow() < until) {
|
|
2114
|
+
await new Promise<void>((resolve) => realSetTimeout(resolve, 0));
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
/**
|
|
2119
|
+
* 假时钟上跨过一段等待窗口(用例锁的 30s 判死线、锁与租约的 10s 轮询周期)的唯一可靠姿势。
|
|
2120
|
+
*
|
|
2121
|
+
* 两条都要满足,少一条就 flaky:
|
|
2122
|
+
* - **分步推,不一步推到底**:轮询链路每一轮都在"定时器回调恢复"和"注册下一轮定时器"之间
|
|
2123
|
+
* 插了真实磁盘 I/O(`readEntryFile` / `claimEntryFile` / `createLockFileExclusive`)。
|
|
2124
|
+
* `advanceTimersByTimeAsync` 只推进*当前已挂起*的定时器,一次推过整个 30s 窗口会在"看不到
|
|
2125
|
+
* 待处理定时器"的那一刻直接返回,把稍后才补挂上的定时器永远晾着(测试挂到超时,不是变慢)。
|
|
2126
|
+
* - **每步之间按真实墙钟让出宏任务轮次**:`advanceTimersByTimeAsync` 只喂微任务,真实磁盘
|
|
2127
|
+
* I/O 拿不到轮次就会被饿死。让出的量必须按真实墙钟算而不是固定圈数——并行跑整个测试套件
|
|
2128
|
+
* 时 I/O 慢一个量级,固定圈数在本机够、在负载下必然不够(本条就是这么被抓出来的)。
|
|
2129
|
+
*
|
|
2130
|
+
* 踩坑同源:memory/case-lock-gate-reorders-global-semaphore-queue.md「旁支」。
|
|
2131
|
+
*/
|
|
2132
|
+
async function advanceOnFakeClock(
|
|
2133
|
+
isDone: () => boolean,
|
|
2134
|
+
stepMs = 10_000,
|
|
2135
|
+
maxSteps = 8,
|
|
2136
|
+
perStepRealMs = 300,
|
|
2137
|
+
tailRealMs = 15_000,
|
|
2138
|
+
): Promise<void> {
|
|
2139
|
+
for (let i = 0; i < maxSteps && !isDone(); i++) {
|
|
2140
|
+
await vi.advanceTimersByTimeAsync(stepMs);
|
|
2141
|
+
await realTicksFor(isDone, perStepRealMs);
|
|
2142
|
+
}
|
|
2143
|
+
// 要跨的虚拟窗口到这里都跨过了,剩下的是纯真实 I/O 的收尾进度(取锁、跑 attempt、落盘、
|
|
2144
|
+
// 放锁)。只喂真实轮次,每轮顺手推 1ms 假时钟接住收尾链上可能挂的短定时器——步长必须小,
|
|
2145
|
+
// 否则慢机器上轮数一多会把 attempt 的外层超时也推过去。
|
|
2146
|
+
const until = realDateNow() + tailRealMs;
|
|
2147
|
+
while (!isDone() && realDateNow() < until) {
|
|
2148
|
+
await vi.advanceTimersByTimeAsync(1);
|
|
2149
|
+
await realTicksFor(isDone, 20);
|
|
2150
|
+
}
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
describe("runEvals · 用例锁: 取锁时机", () => {
|
|
2154
|
+
afterEach(() => {
|
|
2155
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2156
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2157
|
+
});
|
|
2158
|
+
|
|
2159
|
+
it("全部 attempt 都可携带的用例不取锁,不真的派发", async () => {
|
|
2160
|
+
const experimentId = "lock-timing-full-carry-exp";
|
|
2161
|
+
const evalId = "carried-eval";
|
|
2162
|
+
const evalDef = makeEval(evalId, () => {
|
|
2163
|
+
throw new Error("carried attempt must not be dispatched");
|
|
2164
|
+
});
|
|
2165
|
+
const agentRun: AgentRun = {
|
|
2166
|
+
agent: makeAgent("agent-lock-timing-carry"),
|
|
2167
|
+
flags: {},
|
|
2168
|
+
runs: 1,
|
|
2169
|
+
earlyExit: true,
|
|
2170
|
+
sandbox: fakeSandboxSpec(),
|
|
2171
|
+
timeoutMs: 5_000,
|
|
2172
|
+
selectedEvalIds: [evalId],
|
|
2173
|
+
experimentId,
|
|
2174
|
+
};
|
|
2175
|
+
// 指纹依赖 evalDef.sourcePath 的文件内容(makeEval 统一指向本测试文件)与 run 的配置字段,
|
|
2176
|
+
// 不依赖 test() 闭包本身 —— 用真实的 computeFingerprint 算,而不是随便编一个字符串,
|
|
2177
|
+
// 才能真的驱动到"指纹匹配"这条携带路径。
|
|
2178
|
+
const fingerprint = await computeFingerprint(evalDef, agentRun);
|
|
2179
|
+
const prior: EvalResult = {
|
|
2180
|
+
id: evalId,
|
|
2181
|
+
experimentId,
|
|
2182
|
+
agent: agentRun.agent.name,
|
|
2183
|
+
verdict: "passed",
|
|
2184
|
+
attempt: 0,
|
|
2185
|
+
fingerprint,
|
|
2186
|
+
startedAt: new Date().toISOString(),
|
|
2187
|
+
durationMs: 1,
|
|
2188
|
+
assertions: [],
|
|
2189
|
+
};
|
|
2190
|
+
|
|
2191
|
+
const { summary, root } = await runWithPriorResults([evalDef], [agentRun], { priorResults: [prior] });
|
|
2192
|
+
|
|
2193
|
+
expect(summary.results).toHaveLength(1);
|
|
2194
|
+
expect(summary.results[0]!.verdict).toBe("passed");
|
|
2195
|
+
// 被静态携带规划筛掉的 (experimentId, evalId) 组合从不出现在 attempts[] 里,天然不会
|
|
2196
|
+
// 走到取锁那一步 —— 磁盘上不该留下任何锁文件。
|
|
2197
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2198
|
+
});
|
|
2199
|
+
|
|
2200
|
+
it("等锁用例不触发实验级 setup:等待期间 setup 计数保持 0,接管后才恰好执行一次", async () => {
|
|
2201
|
+
vi.useFakeTimers();
|
|
2202
|
+
try {
|
|
2203
|
+
const root = await makeRoot();
|
|
2204
|
+
const experimentId = "lock-timing-setup-exp";
|
|
2205
|
+
const evalId = "setup-gated-eval";
|
|
2206
|
+
await seedCaseLock(root, freshLockRecord(experimentId, evalId));
|
|
2207
|
+
|
|
2208
|
+
let setupCalls = 0;
|
|
2209
|
+
let testCalls = 0;
|
|
2210
|
+
const evalDef = makeEval(evalId, () => {
|
|
2211
|
+
testCalls += 1;
|
|
2212
|
+
});
|
|
2213
|
+
const agentRun: AgentRun = {
|
|
2214
|
+
agent: makeAgent("agent-lock-timing-setup"),
|
|
2215
|
+
flags: {},
|
|
2216
|
+
runs: 1,
|
|
2217
|
+
earlyExit: false,
|
|
2218
|
+
sandbox: fakeSandboxSpec(),
|
|
2219
|
+
timeoutMs: 30_000,
|
|
2220
|
+
selectedEvalIds: [evalId],
|
|
2221
|
+
experimentId,
|
|
2222
|
+
setup: () => {
|
|
2223
|
+
setupCalls += 1;
|
|
2224
|
+
},
|
|
2225
|
+
};
|
|
2226
|
+
|
|
2227
|
+
const runPromise = runWithPriorResults([evalDef], [agentRun], { priorResults: [], root });
|
|
2228
|
+
|
|
2229
|
+
// 等到轮询真的挂起下一次心跳定时器(真实磁盘 I/O 已经跑过一轮),此刻还远没到 30s
|
|
2230
|
+
// 判死线:setup 不该被触发,eval 也不该被派发。
|
|
2231
|
+
await vi.waitFor(() => expect(vi.getTimerCount()).toBeGreaterThanOrEqual(1));
|
|
2232
|
+
expect(setupCalls).toBe(0);
|
|
2233
|
+
expect(testCalls).toBe(0);
|
|
2234
|
+
await vi.advanceTimersByTimeAsync(10_000); // 再推一个心跳周期,仍然远短于判死线
|
|
2235
|
+
await vi.waitFor(() => expect(vi.getTimerCount()).toBeGreaterThanOrEqual(1));
|
|
2236
|
+
expect(setupCalls).toBe(0);
|
|
2237
|
+
|
|
2238
|
+
// 继续推过判死线,接管发生 —— setup 此刻才第一次执行。
|
|
2239
|
+
await advanceOnFakeClock(() => setupCalls === 1);
|
|
2240
|
+
expect(setupCalls).toBe(1);
|
|
2241
|
+
|
|
2242
|
+
const { summary } = await runPromise;
|
|
2243
|
+
expect(summary.results).toHaveLength(1);
|
|
2244
|
+
expect(summary.results[0]!.verdict).toBe("passed");
|
|
2245
|
+
expect(setupCalls).toBe(1);
|
|
2246
|
+
expect(testCalls).toBe(1);
|
|
2247
|
+
} finally {
|
|
2248
|
+
vi.useRealTimers();
|
|
2249
|
+
}
|
|
2250
|
+
});
|
|
2251
|
+
});
|
|
2252
|
+
|
|
2253
|
+
describe("runEvals · 用例锁: 等待语义", () => {
|
|
2254
|
+
afterEach(() => {
|
|
2255
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2256
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2257
|
+
});
|
|
2258
|
+
|
|
2259
|
+
it("撞新鲜锁的用例不派发、不占全局并发位;elsewhere/queued 五项恒等式成立;过期后接管并真实派发(无匹配可携带)", async () => {
|
|
2260
|
+
vi.useFakeTimers();
|
|
2261
|
+
try {
|
|
2262
|
+
const root = await makeRoot();
|
|
2263
|
+
const experimentId = "lock-wait-exp";
|
|
2264
|
+
const evalIdLocked = "locked-eval";
|
|
2265
|
+
const evalIdFree = "free-eval";
|
|
2266
|
+
await seedCaseLock(root, freshLockRecord(experimentId, evalIdLocked));
|
|
2267
|
+
|
|
2268
|
+
let lockedCalls = 0;
|
|
2269
|
+
let freeCalls = 0;
|
|
2270
|
+
const evalLocked = makeEval(evalIdLocked, () => {
|
|
2271
|
+
lockedCalls += 1;
|
|
2272
|
+
});
|
|
2273
|
+
const evalFree = makeEval(evalIdFree, () => {
|
|
2274
|
+
freeCalls += 1;
|
|
2275
|
+
});
|
|
2276
|
+
const agentRun: AgentRun = {
|
|
2277
|
+
agent: makeAgent("agent-lock-wait"),
|
|
2278
|
+
flags: {},
|
|
2279
|
+
runs: 1,
|
|
2280
|
+
earlyExit: false,
|
|
2281
|
+
sandbox: fakeSandboxSpec(),
|
|
2282
|
+
timeoutMs: 30_000,
|
|
2283
|
+
selectedEvalIds: [evalIdLocked, evalIdFree],
|
|
2284
|
+
experimentId,
|
|
2285
|
+
};
|
|
2286
|
+
const plan: RunFeedbackPlan = {
|
|
2287
|
+
shape: { evals: 2, configs: 1, totalAttempts: 2, maxConcurrency: 1 },
|
|
2288
|
+
reused: 0,
|
|
2289
|
+
reusedFailures: [],
|
|
2290
|
+
};
|
|
2291
|
+
|
|
2292
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
2293
|
+
// maxConcurrency: 1(全局唯一名额)——只有在名额紧张到只有一个的情况下,free-eval
|
|
2294
|
+
// 仍能跑完,才证明 locked-eval 的等待确实没有占着这个唯一的全局位;宽松并发会掩盖这一点。
|
|
2295
|
+
const runPromise = runWithPriorResults([evalLocked, evalFree], [agentRun], {
|
|
2296
|
+
priorResults: [],
|
|
2297
|
+
root,
|
|
2298
|
+
maxConcurrency: 1,
|
|
2299
|
+
});
|
|
2300
|
+
|
|
2301
|
+
await vi.waitFor(() => expect(freeCalls).toBe(1));
|
|
2302
|
+
expect(lockedCalls).toBe(0); // 撞锁的用例仍未派发
|
|
2303
|
+
expect(coordinator.state.lockWaits.get(experimentId)?.waiting.has(evalIdLocked)).toBe(true);
|
|
2304
|
+
expect(coordinator.state.elsewhere).toBeGreaterThanOrEqual(1);
|
|
2305
|
+
const mid = coordinator.state;
|
|
2306
|
+
expectCountIdentity(mid);
|
|
2307
|
+
|
|
2308
|
+
// 推过 30s 判死线:种下的心跳没有任何进程真的在续租,过期后必须被接管。
|
|
2309
|
+
// 分步推 + 每步让出真实事件循环,理由见 advanceOnFakeClock 注释。
|
|
2310
|
+
await advanceOnFakeClock(() => lockedCalls === 1);
|
|
2311
|
+
expect(lockedCalls).toBe(1); // 确认真的走到了"接管后派发",不是轮询步数耗尽仍未接管
|
|
2312
|
+
|
|
2313
|
+
const { summary } = await runPromise;
|
|
2314
|
+
expect(summary.results).toHaveLength(2);
|
|
2315
|
+
expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
2316
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
2317
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2318
|
+
expect(
|
|
2319
|
+
coordinator.state.diagnostics.some((d) => d.key === `lock-taken-over:${experimentId}|${evalIdLocked}`),
|
|
2320
|
+
).toBe(true);
|
|
2321
|
+
});
|
|
2322
|
+
} finally {
|
|
2323
|
+
vi.useRealTimers();
|
|
2324
|
+
}
|
|
2325
|
+
});
|
|
2326
|
+
});
|
|
2327
|
+
|
|
2328
|
+
describe("runEvals · 用例锁: 释放后续接", () => {
|
|
2329
|
+
afterEach(() => {
|
|
2330
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2331
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2332
|
+
});
|
|
2333
|
+
|
|
2334
|
+
it("指纹匹配携入(carried):产出方落盘终态后来不及释放锁就死掉,续接方接管后直接携入、不重新派发", async () => {
|
|
2335
|
+
const root = await makeRoot();
|
|
2336
|
+
const experimentId = "lock-release-carry-exp";
|
|
2337
|
+
const evalId = "carry-release-eval";
|
|
2338
|
+
const producerRun: AgentRun = {
|
|
2339
|
+
agent: makeAgent("agent-lock-release-carry"),
|
|
2340
|
+
flags: {},
|
|
2341
|
+
runs: 1,
|
|
2342
|
+
earlyExit: true,
|
|
2343
|
+
sandbox: fakeSandboxSpec(),
|
|
2344
|
+
timeoutMs: 5_000,
|
|
2345
|
+
selectedEvalIds: [evalId],
|
|
2346
|
+
experimentId,
|
|
2347
|
+
};
|
|
2348
|
+
const producerEval = makeEval(evalId, () => {});
|
|
2349
|
+
// 用真实 run() 走一遍完整调度,把一条正确 fingerprint 的终态结果落到 root 上 ——
|
|
2350
|
+
// 比手工拼 EvalResult 更省事也更可信(fingerprint / artifactBase 这些字段很容易拼错)。
|
|
2351
|
+
const { summary: producerSummary } = await run([producerEval], [producerRun], { root });
|
|
2352
|
+
expect(producerSummary.results[0]!.verdict).toBe("passed");
|
|
2353
|
+
|
|
2354
|
+
// 模拟"产出方刚写完结果、还没来得及释放锁就被强杀":种一把过期锁,而不是等它自然释放。
|
|
2355
|
+
await seedCaseLock(root, staleLockRecord(experimentId, evalId));
|
|
2356
|
+
|
|
2357
|
+
const subjectEval = makeEval(evalId, () => {
|
|
2358
|
+
throw new Error("carried attempt must not be redispatched");
|
|
2359
|
+
});
|
|
2360
|
+
const plan: RunFeedbackPlan = {
|
|
2361
|
+
shape: { evals: 1, configs: 1, totalAttempts: 1, maxConcurrency: 3 },
|
|
2362
|
+
reused: 0,
|
|
2363
|
+
reusedFailures: [],
|
|
2364
|
+
};
|
|
2365
|
+
|
|
2366
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
2367
|
+
const { summary } = await runWithPriorResults([subjectEval], [producerRun], {
|
|
2368
|
+
priorResults: [],
|
|
2369
|
+
root,
|
|
2370
|
+
});
|
|
2371
|
+
|
|
2372
|
+
expect(summary.results).toHaveLength(1);
|
|
2373
|
+
expect(summary.results[0]!.verdict).toBe("passed");
|
|
2374
|
+
expect(coordinator.state.reused).toBe(1);
|
|
2375
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
2376
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2377
|
+
});
|
|
2378
|
+
});
|
|
2379
|
+
|
|
2380
|
+
it("不匹配转自跑(dispatched):磁盘上没有任何终态结果时,接管过期锁后真实派发", async () => {
|
|
2381
|
+
const root = await makeRoot();
|
|
2382
|
+
const experimentId = "lock-release-dispatch-exp";
|
|
2383
|
+
const evalId = "dispatch-release-eval";
|
|
2384
|
+
await seedCaseLock(root, staleLockRecord(experimentId, evalId));
|
|
2385
|
+
|
|
2386
|
+
let calls = 0;
|
|
2387
|
+
const evalDef = makeEval(evalId, () => {
|
|
2388
|
+
calls += 1;
|
|
2389
|
+
});
|
|
2390
|
+
const agentRun: AgentRun = {
|
|
2391
|
+
agent: makeAgent("agent-lock-release-dispatch"),
|
|
2392
|
+
flags: {},
|
|
2393
|
+
runs: 1,
|
|
2394
|
+
earlyExit: true,
|
|
2395
|
+
sandbox: fakeSandboxSpec(),
|
|
2396
|
+
timeoutMs: 5_000,
|
|
2397
|
+
selectedEvalIds: [evalId],
|
|
2398
|
+
experimentId,
|
|
2399
|
+
};
|
|
2400
|
+
|
|
2401
|
+
const { summary } = await runWithPriorResults([evalDef], [agentRun], { priorResults: [], root });
|
|
2402
|
+
|
|
2403
|
+
expect(calls).toBe(1);
|
|
2404
|
+
expect(summary.results).toHaveLength(1);
|
|
2405
|
+
expect(summary.results[0]!.verdict).toBe("passed");
|
|
2406
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2407
|
+
});
|
|
2408
|
+
|
|
2409
|
+
it("runs 部分携入部分补跑:已有 1 条终态时,续接方 runs:2 只补差额序号,不重跑已携入的序号", async () => {
|
|
2410
|
+
const root = await makeRoot();
|
|
2411
|
+
const experimentId = "lock-release-partial-exp";
|
|
2412
|
+
const evalId = "partial-release-eval";
|
|
2413
|
+
const producerRun: AgentRun = {
|
|
2414
|
+
agent: makeAgent("agent-lock-release-partial"),
|
|
2415
|
+
flags: {},
|
|
2416
|
+
runs: 1,
|
|
2417
|
+
earlyExit: false,
|
|
2418
|
+
sandbox: fakeSandboxSpec(),
|
|
2419
|
+
timeoutMs: 5_000,
|
|
2420
|
+
selectedEvalIds: [evalId],
|
|
2421
|
+
experimentId,
|
|
2422
|
+
};
|
|
2423
|
+
const producerEval = makeEval(evalId, () => {});
|
|
2424
|
+
const { summary: producerSummary } = await run([producerEval], [producerRun], { root });
|
|
2425
|
+
expect(producerSummary.results[0]!.verdict).toBe("passed");
|
|
2426
|
+
|
|
2427
|
+
await seedCaseLock(root, staleLockRecord(experimentId, evalId));
|
|
2428
|
+
|
|
2429
|
+
let calls = 0;
|
|
2430
|
+
const subjectEval = makeEval(evalId, () => {
|
|
2431
|
+
calls += 1;
|
|
2432
|
+
});
|
|
2433
|
+
// earlyExit: false —— 携入的 passed 会预置进 passedKeys(见 run.ts 对 lateCarriedResults
|
|
2434
|
+
// 的处理),开着 earlyExit 会让差额序号也被当成"已知会通过"提前省略,测不出"差额真的被
|
|
2435
|
+
// 重新派发"这件事本身。
|
|
2436
|
+
const subjectRun: AgentRun = { ...producerRun, runs: 2, earlyExit: false };
|
|
2437
|
+
|
|
2438
|
+
const { summary } = await runWithPriorResults([subjectEval], [subjectRun], { priorResults: [], root });
|
|
2439
|
+
|
|
2440
|
+
expect(calls).toBe(1); // 只有差额(序号 1)真的跑了一次,序号 0 被携入、没有重跑
|
|
2441
|
+
const results = summary.results.filter((r) => r.id === evalId);
|
|
2442
|
+
expect(results).toHaveLength(2);
|
|
2443
|
+
expect(results.every((r) => r.verdict === "passed")).toBe(true);
|
|
2444
|
+
expect(results.map((r) => r.attempt).sort()).toEqual([0, 1]);
|
|
2445
|
+
});
|
|
2446
|
+
});
|
|
2447
|
+
|
|
2448
|
+
describe("runEvals · 用例锁: 执行模式组合", () => {
|
|
2449
|
+
afterEach(() => {
|
|
2450
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2451
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2452
|
+
});
|
|
2453
|
+
|
|
2454
|
+
it("--force(RunOptions.priorResults 为 undefined)下,等待/接管后不消费携带 —— 即使指纹匹配的终态结果确实存在,也全部自跑", async () => {
|
|
2455
|
+
const root = await makeRoot();
|
|
2456
|
+
const experimentId = "lock-force-exp";
|
|
2457
|
+
const evalId = "force-eval";
|
|
2458
|
+
const producerRun: AgentRun = {
|
|
2459
|
+
agent: makeAgent("agent-lock-force"),
|
|
2460
|
+
flags: {},
|
|
2461
|
+
runs: 1,
|
|
2462
|
+
earlyExit: true,
|
|
2463
|
+
sandbox: fakeSandboxSpec(),
|
|
2464
|
+
timeoutMs: 5_000,
|
|
2465
|
+
selectedEvalIds: [evalId],
|
|
2466
|
+
experimentId,
|
|
2467
|
+
};
|
|
2468
|
+
const producerEval = makeEval(evalId, () => {});
|
|
2469
|
+
const { summary: producerSummary } = await run([producerEval], [producerRun], { root });
|
|
2470
|
+
expect(producerSummary.results[0]!.verdict).toBe("passed");
|
|
2471
|
+
|
|
2472
|
+
await seedCaseLock(root, staleLockRecord(experimentId, evalId));
|
|
2473
|
+
|
|
2474
|
+
let calls = 0;
|
|
2475
|
+
const subjectEval = makeEval(evalId, () => {
|
|
2476
|
+
calls += 1;
|
|
2477
|
+
});
|
|
2478
|
+
|
|
2479
|
+
// force 模式:cli.ts 在 --force 时整段不传 priorResults(不是传空数组) —— 这里同样
|
|
2480
|
+
// 省略 priorResults 字段,而不是传 []。
|
|
2481
|
+
const { summary } = await runWithPriorResults([subjectEval], [producerRun], { root });
|
|
2482
|
+
|
|
2483
|
+
expect(calls).toBe(1); // 真正重新派发了一次,不是悄悄吞成携入的旧结果
|
|
2484
|
+
expect(summary.results).toHaveLength(1);
|
|
2485
|
+
expect(summary.results[0]!.verdict).toBe("passed");
|
|
2486
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2487
|
+
});
|
|
2488
|
+
});
|
|
2489
|
+
|
|
2490
|
+
describe("runEvals · 用例锁: 释放路径", () => {
|
|
2491
|
+
afterEach(() => {
|
|
2492
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2493
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2494
|
+
});
|
|
2495
|
+
|
|
2496
|
+
it("实验级 setup 抛错、全部 attempt 合成 errored 结果时,锁仍必须被释放", async () => {
|
|
2497
|
+
const experimentId = "lock-setup-fail-exp";
|
|
2498
|
+
const evalId = "setup-fail-eval";
|
|
2499
|
+
const evalDef = makeEval(evalId, () => {});
|
|
2500
|
+
const agentRun: AgentRun = {
|
|
2501
|
+
agent: makeAgent("agent-lock-setup-fail"),
|
|
2502
|
+
flags: {},
|
|
2503
|
+
runs: 1,
|
|
2504
|
+
earlyExit: true,
|
|
2505
|
+
sandbox: fakeSandboxSpec(),
|
|
2506
|
+
timeoutMs: 5_000,
|
|
2507
|
+
selectedEvalIds: [evalId],
|
|
2508
|
+
experimentId,
|
|
2509
|
+
setup: () => {
|
|
2510
|
+
throw new Error("tunnel refused to start");
|
|
2511
|
+
},
|
|
2512
|
+
};
|
|
2513
|
+
|
|
2514
|
+
// 无竞争的全新取锁(没有种任何锁),证明即便本实验一个 attempt 都没有真正派发过 agent
|
|
2515
|
+
// (body 走的是合成 errored 的分支),外层 Effect.ensuring 挂的用例锁释放仍然会触发。
|
|
2516
|
+
const { summary, root } = await runWithPriorResults([evalDef], [agentRun], {});
|
|
2517
|
+
|
|
2518
|
+
expect(summary.results).toHaveLength(1);
|
|
2519
|
+
expect(summary.results[0]!.verdict).toBe("errored");
|
|
2520
|
+
expect(summary.results[0]!.error?.code).toBe("experiment-setup-failed");
|
|
2521
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2522
|
+
});
|
|
2523
|
+
});
|
|
2524
|
+
|
|
2525
|
+
// ─────────────────── 派发探针:「在飞峰值」与「启动集合」的统一观察面 ───────────────────
|
|
2526
|
+
// 覆盖规范对用例锁调度的断言面是可观察的调度事实(锁目录条目数、在飞峰值、启动集合),
|
|
2527
|
+
// 不是内部信号量的调用次数。下面这组用例统一用「eval 的 test() 里登记自己被真实派发、
|
|
2528
|
+
// 然后挂在 barrier 上」取样:barrier 没释放前,「此刻在飞几条」「谁被派发过」在任何时刻
|
|
2529
|
+
// 都可读,且被派发两次的用例会在 started 里出现两次(双跑当场可见)。
|
|
2530
|
+
|
|
2531
|
+
interface DispatchProbe {
|
|
2532
|
+
/** 真实执行过 test() 的 evalId,按进入顺序;同一个 id 出现两次 = 这条用例被双跑了。 */
|
|
2533
|
+
started: string[];
|
|
2534
|
+
inFlight: number;
|
|
2535
|
+
peak: number;
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
function newDispatchProbe(): DispatchProbe {
|
|
2539
|
+
return { started: [], inFlight: 0, peak: 0 };
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
/** 一条挂在 `barrier` 上的 eval:进入 test() 即计入 `probes` 里的每个探针(多开场景要同时
|
|
2543
|
+
* 记进「本侧」与「全局」两个探针),barrier 释放后立刻返回。 */
|
|
2544
|
+
function gatedEval(id: string, barrier: Promise<void>, ...probes: DispatchProbe[]): DiscoveredEval {
|
|
2545
|
+
return makeEval(id, async () => {
|
|
2546
|
+
for (const p of probes) {
|
|
2547
|
+
p.started.push(id);
|
|
2548
|
+
p.inFlight += 1;
|
|
2549
|
+
p.peak = Math.max(p.peak, p.inFlight);
|
|
2550
|
+
}
|
|
2551
|
+
await barrier;
|
|
2552
|
+
for (const p of probes) p.inFlight -= 1;
|
|
2553
|
+
});
|
|
2554
|
+
}
|
|
2555
|
+
|
|
2556
|
+
function makeBarrier(): { barrier: Promise<void>; release: () => void } {
|
|
2557
|
+
let release!: () => void;
|
|
2558
|
+
const barrier = new Promise<void>((resolve) => {
|
|
2559
|
+
release = resolve;
|
|
2560
|
+
});
|
|
2561
|
+
return { barrier, release };
|
|
2562
|
+
}
|
|
2563
|
+
|
|
2564
|
+
/** 派发探针场景统一用的 AgentRun 骨架:timeoutMs 开得远大于测试里推进的假时钟总量,
|
|
2565
|
+
* 免得「推过锁的 30s 判死线」顺带把在飞 attempt 推成外层超时。 */
|
|
2566
|
+
function probeRun(
|
|
2567
|
+
agent: Agent,
|
|
2568
|
+
experimentId: string,
|
|
2569
|
+
selectedEvalIds: string[],
|
|
2570
|
+
extra: Partial<AgentRun> = {},
|
|
2571
|
+
): AgentRun {
|
|
2572
|
+
return {
|
|
2573
|
+
agent,
|
|
2574
|
+
flags: {},
|
|
2575
|
+
runs: 1,
|
|
2576
|
+
earlyExit: false,
|
|
2577
|
+
sandbox: fakeSandboxSpec(),
|
|
2578
|
+
timeoutMs: 600_000,
|
|
2579
|
+
selectedEvalIds,
|
|
2580
|
+
experimentId,
|
|
2581
|
+
...extra,
|
|
2582
|
+
};
|
|
2583
|
+
}
|
|
2584
|
+
|
|
2585
|
+
/** 等一个只靠真实事件循环就会成立的条件(不需要推假时钟:撞锁挂起、attempt 起跑这些都发生
|
|
2586
|
+
* 在当下)。`vi.waitFor` 只让出真实轮次、**不推假时钟** —— 需要跨虚拟等待窗口的地方一律用
|
|
2587
|
+
* `advanceOnFakeClock`。真实超时放宽到 30s:并行跑整个测试套件时默认 1s 不够。 */
|
|
2588
|
+
async function waitForRealProgress(check: () => void): Promise<void> {
|
|
2589
|
+
await vi.waitFor(check, { timeout: 30_000, interval: 20 });
|
|
2590
|
+
}
|
|
2591
|
+
|
|
2592
|
+
/** 这组调度用例的 vitest 超时:要么在假时钟上跨 30s 锁判死线 / 10s 租约轮询周期(靠真实
|
|
2593
|
+
* 轮次驱动),要么等两条 runEvals 真的并行推进;并行跑整个测试套件时真实墙钟开销远大于默认 5s。 */
|
|
2594
|
+
const SCHEDULING_TEST_TIMEOUT_MS = 60_000;
|
|
2595
|
+
|
|
2596
|
+
describe("runEvals · 用例锁: 排队用例不持锁", () => {
|
|
2597
|
+
afterEach(() => {
|
|
2598
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2599
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2600
|
+
});
|
|
2601
|
+
|
|
2602
|
+
it("四条用例、全局并发 2:锁目录条目数等于在跑用例数(排队的两条不持锁),收尾后清空", async () => {
|
|
2603
|
+
const root = await makeRoot();
|
|
2604
|
+
const experimentId = "lock-queue-exp";
|
|
2605
|
+
const ids = ["q-a", "q-b", "q-c", "q-d"];
|
|
2606
|
+
const { barrier, release } = makeBarrier();
|
|
2607
|
+
const probe = newDispatchProbe();
|
|
2608
|
+
const evals = ids.map((id) => gatedEval(id, barrier, probe));
|
|
2609
|
+
const agentRun = probeRun(makeAgent("agent-lock-queue"), experimentId, ids);
|
|
2610
|
+
|
|
2611
|
+
const runPromise = runWithPriorResults(evals, [agentRun], { priorResults: [], root, maxConcurrency: 2 });
|
|
2612
|
+
try {
|
|
2613
|
+
await waitForRealProgress(() => expect(probe.inFlight).toBe(2));
|
|
2614
|
+
|
|
2615
|
+
// 关键断言:计划里有 4 条,此刻只有 2 条在跑 —— 锁目录也只有 2 条。取锁发生在派发时刻,
|
|
2616
|
+
// 排队中的两条还没摸过锁目录(旧的「计划期一次性全量取锁」在这里会是 4)。
|
|
2617
|
+
expect(await lockFilesRemaining(root)).toHaveLength(2);
|
|
2618
|
+
expect(probe.started).toHaveLength(2);
|
|
2619
|
+
} finally {
|
|
2620
|
+
release();
|
|
2621
|
+
}
|
|
2622
|
+
|
|
2623
|
+
const { summary } = await runPromise;
|
|
2624
|
+
expect(summary.results).toHaveLength(4);
|
|
2625
|
+
expect([...probe.started].sort()).toEqual([...ids].sort());
|
|
2626
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2627
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
2628
|
+
});
|
|
2629
|
+
|
|
2630
|
+
describe("runEvals · 用例锁: 撞锁转派", () => {
|
|
2631
|
+
afterEach(() => {
|
|
2632
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2633
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2634
|
+
});
|
|
2635
|
+
|
|
2636
|
+
it("撞新鲜锁的用例让出全局位,位子转派给下一条没被锁的用例:在飞峰值仍等于全局上限,启动集合是未被锁的那些", async () => {
|
|
2637
|
+
vi.useFakeTimers();
|
|
2638
|
+
try {
|
|
2639
|
+
const root = await makeRoot();
|
|
2640
|
+
const experimentId = "lock-handoff-exp";
|
|
2641
|
+
const lockedId = "h-locked";
|
|
2642
|
+
const freeIds = ["h-free-1", "h-free-2"];
|
|
2643
|
+
// 被锁的那条排在数组第一位:它一定会先摸到全局位,证明「转派」不是靠运气排在后面。
|
|
2644
|
+
await seedCaseLock(root, freshLockRecord(experimentId, lockedId));
|
|
2645
|
+
|
|
2646
|
+
const { barrier, release } = makeBarrier();
|
|
2647
|
+
const probe = newDispatchProbe();
|
|
2648
|
+
const evals = [lockedId, ...freeIds].map((id) => gatedEval(id, barrier, probe));
|
|
2649
|
+
const agentRun = probeRun(makeAgent("agent-lock-handoff"), experimentId, [lockedId, ...freeIds]);
|
|
2650
|
+
|
|
2651
|
+
let done = false;
|
|
2652
|
+
const runPromise = runWithPriorResults(evals, [agentRun], {
|
|
2653
|
+
priorResults: [],
|
|
2654
|
+
root,
|
|
2655
|
+
maxConcurrency: 2,
|
|
2656
|
+
}).then((r) => {
|
|
2657
|
+
done = true;
|
|
2658
|
+
return r;
|
|
2659
|
+
});
|
|
2660
|
+
try {
|
|
2661
|
+
await waitForRealProgress(() => expect(probe.inFlight).toBe(2));
|
|
2662
|
+
|
|
2663
|
+
// 峰值没有因为一条撞锁就塌成 1:让出来的位子当场被下一条没被锁的用例接手。
|
|
2664
|
+
expect(probe.peak).toBe(2);
|
|
2665
|
+
expect([...probe.started].sort()).toEqual([...freeIds].sort());
|
|
2666
|
+
expect(probe.started).not.toContain(lockedId);
|
|
2667
|
+
} finally {
|
|
2668
|
+
release();
|
|
2669
|
+
}
|
|
2670
|
+
|
|
2671
|
+
// 种下的心跳没人续租,推过 30s 判死线后被接管,这条用例照常补跑。
|
|
2672
|
+
await advanceOnFakeClock(() => done, 10_000, 12);
|
|
2673
|
+
expect(probe.started).toContain(lockedId);
|
|
2674
|
+
|
|
2675
|
+
const { summary } = await runPromise;
|
|
2676
|
+
expect(summary.results).toHaveLength(3);
|
|
2677
|
+
expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
2678
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2679
|
+
} finally {
|
|
2680
|
+
vi.useRealTimers();
|
|
2681
|
+
}
|
|
2682
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
2683
|
+
});
|
|
2684
|
+
|
|
2685
|
+
describe("runEvals · 用例锁: 多开分工", () => {
|
|
2686
|
+
afterEach(() => {
|
|
2687
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2688
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2689
|
+
});
|
|
2690
|
+
|
|
2691
|
+
// 两条 runEvals 共用同一个 niceevalRoot(不是各自建一个临时根 —— 那测的是"零竞争各自跑"),
|
|
2692
|
+
// 选择集完全相同:四条用例两边都选,各自全局上限 2,于是两边都必然剩下"第一波没抢到位"的
|
|
2693
|
+
// 第二波。谁跑哪些完全由锁自然分工,不靠测试摆布。
|
|
2694
|
+
//
|
|
2695
|
+
// 两段 barrier 是这条用例的关键:B 的第一波必须先真正收尾、把锁删掉,A 的第二波才会撞上
|
|
2696
|
+
// 一把**空**锁 —— 那正是要钉的路径(取到锁没等过、没接管过,重查携带仍必须发生)。锁的删除
|
|
2697
|
+
// 发生在该用例全部 attempt 收尾之后,所以分段点只能取"锁目录掉回只剩 A 手上的那两条",不能
|
|
2698
|
+
// 取"B 的 test() 跑过了":放早了 A 的第二波撞的是新鲜锁、走挂起窗口,而挂起窗口本来就重查,
|
|
2699
|
+
// 这条用例会绿得毫无意义(见 memory/multi-open-residual-window-closed-by-narrow-read.md)。
|
|
2700
|
+
//
|
|
2701
|
+
// 顺带钉住「携带来源不要求快照收尾」:A 的第二波携入 B 的结果时,B 整批还没跑完(它的两条
|
|
2702
|
+
// 正挂在 A 的锁上等),快照 snapshot.json 尚无 completedAt。
|
|
2703
|
+
it("两条 runEvals 同 root、选择集完全相同:真实派发的用例集不相交、并集覆盖选择集、全局在飞峰值达到两边上限之和,每条用例全局只被真实派发一次", async () => {
|
|
2704
|
+
vi.useFakeTimers();
|
|
2705
|
+
try {
|
|
2706
|
+
const root = await makeRoot();
|
|
2707
|
+
const experimentId = "multi-open-exp";
|
|
2708
|
+
const firstWave = ["m-1", "m-2"];
|
|
2709
|
+
const secondWave = ["m-3", "m-4"];
|
|
2710
|
+
const ids = [...firstWave, ...secondWave];
|
|
2711
|
+
const agent = makeAgent("agent-multi-open");
|
|
2712
|
+
const sandbox = fakeSandboxSpec();
|
|
2713
|
+
|
|
2714
|
+
const gateA = makeBarrier();
|
|
2715
|
+
const gateB = makeBarrier();
|
|
2716
|
+
const all = newDispatchProbe();
|
|
2717
|
+
const sideA = newDispatchProbe();
|
|
2718
|
+
const sideB = newDispatchProbe();
|
|
2719
|
+
const evalsA = ids.map((id) => gatedEval(id, gateA.barrier, sideA, all));
|
|
2720
|
+
const evalsB = ids.map((id) => gatedEval(id, gateB.barrier, sideB, all));
|
|
2721
|
+
// 指纹只吃 (eval 源码 + experimentId/agent/model/flags/sandbox/strict),不吃 selectedEvalIds
|
|
2722
|
+
// 与 runs —— 两侧指纹相同,携入对方跑出来的那半边才可能发生。
|
|
2723
|
+
const runA = probeRun(agent, experimentId, ids, { sandbox });
|
|
2724
|
+
const runB = probeRun(agent, experimentId, ids, { sandbox });
|
|
2725
|
+
|
|
2726
|
+
let aDone = false;
|
|
2727
|
+
let bDone = false;
|
|
2728
|
+
const pa = runWithPriorResults(evalsA, [runA], { priorResults: [], root, maxConcurrency: 2 }).then((r) => {
|
|
2729
|
+
aDone = true;
|
|
2730
|
+
return r;
|
|
2731
|
+
});
|
|
2732
|
+
await waitForRealProgress(() => expect(sideA.inFlight).toBe(2));
|
|
2733
|
+
|
|
2734
|
+
const pb = runWithPriorResults(evalsB, [runB], { priorResults: [], root, maxConcurrency: 2 }).then((r) => {
|
|
2735
|
+
bDone = true;
|
|
2736
|
+
return r;
|
|
2737
|
+
});
|
|
2738
|
+
await waitForRealProgress(() => expect(sideB.inFlight).toBe(2));
|
|
2739
|
+
|
|
2740
|
+
// ① 两边真实派发的用例集不相交;② 并集覆盖选择集;③ 全局在飞峰值 = 2 + 2。
|
|
2741
|
+
expect([...sideA.started].sort()).toEqual([...firstWave]);
|
|
2742
|
+
expect([...sideB.started].sort()).toEqual([...secondWave]);
|
|
2743
|
+
expect(sideA.started.filter((id) => sideB.started.includes(id))).toEqual([]);
|
|
2744
|
+
expect(all.peak).toBe(4);
|
|
2745
|
+
expect(await lockedEvalIds(root, experimentId, ids)).toEqual(ids);
|
|
2746
|
+
|
|
2747
|
+
// B 的第一波收尾:m-3 / m-4 落盘、锁被删掉,B 剩下的两条仍挂在 A 的锁上等。
|
|
2748
|
+
gateB.release();
|
|
2749
|
+
await awaitDiskOnFakeClock(async () => (await lockedEvalIds(root, experimentId, ids)).length === 2);
|
|
2750
|
+
expect(await lockedEvalIds(root, experimentId, ids)).toEqual(firstWave);
|
|
2751
|
+
|
|
2752
|
+
// A 的第一波收尾 → A 的第二波拿到全局位 → 干干净净地取到 m-3 / m-4 的锁。
|
|
2753
|
+
gateA.release();
|
|
2754
|
+
await advanceOnFakeClock(() => aDone && bDone, 10_000, 12);
|
|
2755
|
+
const [ra, rb] = await Promise.all([pa, pb]);
|
|
2756
|
+
|
|
2757
|
+
// 双跑当场可见:started 不去重(去重恰好会把双跑抹掉),长度必须正好是用例数。
|
|
2758
|
+
expect(all.started).toHaveLength(ids.length);
|
|
2759
|
+
expect([...all.started].sort()).toEqual([...ids]);
|
|
2760
|
+
// 两边到收尾为止都没有再多派发一条:第二波全是携入(排序只为消掉进入 test() 的先后,
|
|
2761
|
+
// 那是调度自由度;条数不排序,双跑会当场把长度顶上去)。
|
|
2762
|
+
expect([...sideA.started].sort()).toEqual([...firstWave]);
|
|
2763
|
+
expect([...sideB.started].sort()).toEqual([...secondWave]);
|
|
2764
|
+
// 两边各自结束时都拿到完整结果集,交集部分只花一份成本。
|
|
2765
|
+
expect(ra.summary.results).toHaveLength(4);
|
|
2766
|
+
expect(rb.summary.results).toHaveLength(4);
|
|
2767
|
+
expect(ra.summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
2768
|
+
expect(rb.summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
2769
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2770
|
+
} finally {
|
|
2771
|
+
vi.useRealTimers();
|
|
2772
|
+
}
|
|
2773
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
2774
|
+
|
|
2775
|
+
// 零竞争的最小形态:全程没有任何"碰上别人"的信号——A 起跑时锁目录还不存在,它从没撞过锁、
|
|
2776
|
+
// 没接管过、没等过实验闸名额。对方那条 runEvals 整体 await 到返回(跑完 c-2、落盘、删锁)
|
|
2777
|
+
// 之后才放 barrier,所以 A 的第二波是干干净净地取到一把空锁。任何「有没有碰上别人」的启发式
|
|
2778
|
+
// 在这条路径上都恒假,只有无条件重查才能让 c-2 不被重跑。
|
|
2779
|
+
it("对方跑完并释放锁后干净取到锁:仍必须重查携带 —— c-2 全局只被真实派发一次,携入的那条经一对瞬时 lock_wait 迁进 reused", async () => {
|
|
2780
|
+
const root = await makeRoot();
|
|
2781
|
+
const experimentId = "multi-open-clean-exp";
|
|
2782
|
+
const gatedId = "c-1";
|
|
2783
|
+
const sharedId = "c-2";
|
|
2784
|
+
const agent = makeAgent("agent-multi-open-clean");
|
|
2785
|
+
const sandbox = fakeSandboxSpec();
|
|
2786
|
+
const { barrier, release } = makeBarrier();
|
|
2787
|
+
const all = newDispatchProbe();
|
|
2788
|
+
|
|
2789
|
+
// A:c-1 挂在 barrier 上占住唯一的全局位,c-2 排在队列里(还没摸过锁目录)。
|
|
2790
|
+
const evalsA = [gatedEval(gatedId, barrier, all), gatedEval(sharedId, barrier, all)];
|
|
2791
|
+
const runA = probeRun(agent, experimentId, [gatedId, sharedId], { sandbox });
|
|
2792
|
+
// 对方:只选 c-2,不挂 barrier。
|
|
2793
|
+
const evalsB = [gatedEval(sharedId, Promise.resolve(), all)];
|
|
2794
|
+
const runB = probeRun(agent, experimentId, [sharedId], { sandbox });
|
|
2795
|
+
|
|
2796
|
+
// 反馈计数是进程级的,两条 runEvals 都报进同一个 coordinator:total 取两边之和(A 的 2 条
|
|
2797
|
+
// + 对方的 1 条),五项恒等式必须在这个合计口径上成立。
|
|
2798
|
+
const plan: RunFeedbackPlan = {
|
|
2799
|
+
shape: { evals: 3, configs: 1, totalAttempts: 3, maxConcurrency: 1 },
|
|
2800
|
+
reused: 0,
|
|
2801
|
+
reusedFailures: [],
|
|
2802
|
+
};
|
|
2803
|
+
|
|
2804
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
2805
|
+
const pa = runWithPriorResults(evalsA, [runA], { priorResults: [], root, maxConcurrency: 1 });
|
|
2806
|
+
let peerLocator: string | undefined;
|
|
2807
|
+
try {
|
|
2808
|
+
await waitForRealProgress(() => expect(all.inFlight).toBe(1));
|
|
2809
|
+
expect(all.started).toEqual([gatedId]); // c-2 还排着队,没取过锁
|
|
2810
|
+
|
|
2811
|
+
const { summary: peer } = await runWithPriorResults(evalsB, [runB], {
|
|
2812
|
+
priorResults: [],
|
|
2813
|
+
root,
|
|
2814
|
+
maxConcurrency: 1,
|
|
2815
|
+
});
|
|
2816
|
+
expect(peer.results).toHaveLength(1);
|
|
2817
|
+
expect(peer.results[0]!.verdict).toBe("passed");
|
|
2818
|
+
peerLocator = peer.results[0]!.locator;
|
|
2819
|
+
expect(await lockFilesRemaining(root)).toHaveLength(1); // 只剩 A 手上的 c-1
|
|
2820
|
+
} finally {
|
|
2821
|
+
release();
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
const { summary } = await pa;
|
|
2825
|
+
|
|
2826
|
+
expect(all.started.filter((id) => id === sharedId)).toHaveLength(1);
|
|
2827
|
+
expect(all.started).toEqual([gatedId, sharedId]);
|
|
2828
|
+
expect(summary.results).toHaveLength(2);
|
|
2829
|
+
expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
2830
|
+
// 携入的是对方那条结果本身(locator 原样透传),不是自己重跑出来的新条目。
|
|
2831
|
+
expect(summary.results.find((r) => r.id === sharedId)!.locator).toBe(peerLocator);
|
|
2832
|
+
|
|
2833
|
+
// 没有等待窗口要关,携入的那条仍必须补一对瞬时的 started/resolved 才能从 queued 迁进
|
|
2834
|
+
// reused —— 少发这一对,五项恒等式当场破。
|
|
2835
|
+
expect(coordinator.state.reused).toBe(1);
|
|
2836
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
2837
|
+
expectCountIdentity(coordinator.state);
|
|
2838
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2839
|
+
});
|
|
2840
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
2841
|
+
});
|
|
2842
|
+
|
|
2843
|
+
describe("runEvals · 用例锁: 干净取锁下的执行模式组合", () => {
|
|
2844
|
+
afterEach(() => {
|
|
2845
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2846
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2847
|
+
});
|
|
2848
|
+
|
|
2849
|
+
// 与「执行模式组合」那条(等待 / 接管后自跑)的区别只有取锁路径:这里是无条件重查刚接管的
|
|
2850
|
+
// 那条干净取锁路径。重查变成无条件之后,`--force` 这道门就是携带与否的唯一判据 —— 漏掉它
|
|
2851
|
+
// 就会在多开下把 force 明确要重跑的用例悄悄吞成携入。
|
|
2852
|
+
it("--force(RunOptions.priorResults 为 undefined)下干净取到锁:对方刚跑完的那条照常自跑,窗口计数不留悬挂", async () => {
|
|
2853
|
+
const root = await makeRoot();
|
|
2854
|
+
const experimentId = "force-clean-exp";
|
|
2855
|
+
const gatedId = "fc-1";
|
|
2856
|
+
const sharedId = "fc-2";
|
|
2857
|
+
const agent = makeAgent("agent-force-clean");
|
|
2858
|
+
const sandbox = fakeSandboxSpec();
|
|
2859
|
+
const { barrier, release } = makeBarrier();
|
|
2860
|
+
const all = newDispatchProbe();
|
|
2861
|
+
|
|
2862
|
+
const evalsA = [gatedEval(gatedId, barrier, all), gatedEval(sharedId, barrier, all)];
|
|
2863
|
+
const runA = probeRun(agent, experimentId, [gatedId, sharedId], { sandbox });
|
|
2864
|
+
const evalsPeer = [gatedEval(sharedId, Promise.resolve(), all)];
|
|
2865
|
+
const runPeer = probeRun(agent, experimentId, [sharedId], { sandbox });
|
|
2866
|
+
|
|
2867
|
+
const plan: RunFeedbackPlan = {
|
|
2868
|
+
shape: { evals: 3, configs: 1, totalAttempts: 3, maxConcurrency: 1 },
|
|
2869
|
+
reused: 0,
|
|
2870
|
+
reusedFailures: [],
|
|
2871
|
+
};
|
|
2872
|
+
|
|
2873
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
2874
|
+
// force 模式:cli.ts 在 --force 时整段不传 priorResults(不是传空数组)——这里同样省略。
|
|
2875
|
+
const pa = runWithPriorResults(evalsA, [runA], { root, maxConcurrency: 1 });
|
|
2876
|
+
try {
|
|
2877
|
+
await waitForRealProgress(() => expect(all.inFlight).toBe(1));
|
|
2878
|
+
const { summary: peer } = await runWithPriorResults(evalsPeer, [runPeer], {
|
|
2879
|
+
priorResults: [],
|
|
2880
|
+
root,
|
|
2881
|
+
maxConcurrency: 1,
|
|
2882
|
+
});
|
|
2883
|
+
expect(peer.results[0]!.verdict).toBe("passed");
|
|
2884
|
+
} finally {
|
|
2885
|
+
release();
|
|
2886
|
+
}
|
|
2887
|
+
|
|
2888
|
+
const { summary } = await pa;
|
|
2889
|
+
|
|
2890
|
+
// 对方跑过一次、A 又跑了一次:这里的两次是**预期**,force 关掉的就是缓存。
|
|
2891
|
+
expect(all.started.filter((id) => id === sharedId)).toHaveLength(2);
|
|
2892
|
+
expect(summary.results).toHaveLength(2);
|
|
2893
|
+
expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
2894
|
+
expect(coordinator.state.reused).toBe(0);
|
|
2895
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
2896
|
+
expectCountIdentity(coordinator.state);
|
|
2897
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2898
|
+
});
|
|
2899
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
2900
|
+
});
|
|
2901
|
+
|
|
2902
|
+
describe("runEvals · 实验闸租约跨 runEvals 共享名额", () => {
|
|
2903
|
+
afterEach(() => {
|
|
2904
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
2905
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
2906
|
+
expect(pendingHeldGateLeaseCount()).toBe(0);
|
|
2907
|
+
});
|
|
2908
|
+
|
|
2909
|
+
// 两条 runEvals 各自有独立的进程内信号量,唯一共享的东西是同一个 niceevalRoot 下的租约条目
|
|
2910
|
+
// ——所以「峰值恒为 1」只可能来自跨 runEvals 的名额域。两边故意选不相交的 eval 子集:用例锁
|
|
2911
|
+
// 零交集,限流的只可能是实验闸(这正是 memory/case-lock-dispatch-time-acquire-ruling 里说的
|
|
2912
|
+
// 「双终端选不相交子集跑同一个 maxConcurrency: 1 实验,锁零交集,状态照踩」那个洞)。
|
|
2913
|
+
it("maxConcurrency: 1 的实验:两条 runEvals 同 root 跑不相交用例,该实验全局在飞峰值恒为 1", async () => {
|
|
2914
|
+
vi.useFakeTimers();
|
|
2915
|
+
try {
|
|
2916
|
+
const root = await makeRoot();
|
|
2917
|
+
const experimentId = "gate-cross-exp";
|
|
2918
|
+
const agent = makeAgent("agent-gate-cross");
|
|
2919
|
+
const sandbox = fakeSandboxSpec();
|
|
2920
|
+
const idsA = ["g-a1", "g-a2"];
|
|
2921
|
+
const idsB = ["g-b1", "g-b2"];
|
|
2922
|
+
|
|
2923
|
+
const all = newDispatchProbe();
|
|
2924
|
+
// 不用 barrier:名额是串行的,挂住任何一条都会把整条链堵死。改用一个假时钟上的
|
|
2925
|
+
// 让步点——两条 attempt 若真的同时持有名额,它们的让步窗口会重叠,峰值当场记成 2。
|
|
2926
|
+
const tickingEval = (id: string): DiscoveredEval =>
|
|
2927
|
+
makeEval(id, async () => {
|
|
2928
|
+
all.started.push(id);
|
|
2929
|
+
all.inFlight += 1;
|
|
2930
|
+
all.peak = Math.max(all.peak, all.inFlight);
|
|
2931
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 1));
|
|
2932
|
+
all.inFlight -= 1;
|
|
2933
|
+
});
|
|
2934
|
+
|
|
2935
|
+
const runA = probeRun(agent, experimentId, idsA, { sandbox, maxConcurrency: 1 });
|
|
2936
|
+
const runB = probeRun(agent, experimentId, idsB, { sandbox, maxConcurrency: 1 });
|
|
2937
|
+
|
|
2938
|
+
let aDone = false;
|
|
2939
|
+
let bDone = false;
|
|
2940
|
+
const pa = runWithPriorResults(idsA.map(tickingEval), [runA], {
|
|
2941
|
+
priorResults: [],
|
|
2942
|
+
root,
|
|
2943
|
+
maxConcurrency: 4,
|
|
2944
|
+
}).then((r) => {
|
|
2945
|
+
aDone = true;
|
|
2946
|
+
return r;
|
|
2947
|
+
});
|
|
2948
|
+
const pb = runWithPriorResults(idsB.map(tickingEval), [runB], {
|
|
2949
|
+
priorResults: [],
|
|
2950
|
+
root,
|
|
2951
|
+
maxConcurrency: 4,
|
|
2952
|
+
}).then((r) => {
|
|
2953
|
+
bDone = true;
|
|
2954
|
+
return r;
|
|
2955
|
+
});
|
|
2956
|
+
|
|
2957
|
+
// 名额交接跨 runEvals 走租约轮询(周期 = 心跳周期),必须分步推假时钟。
|
|
2958
|
+
await advanceOnFakeClock(() => aDone && bDone, 5_000, 40);
|
|
2959
|
+
const [ra, rb] = await Promise.all([pa, pb]);
|
|
2960
|
+
|
|
2961
|
+
expect(all.peak).toBe(1);
|
|
2962
|
+
expect([...all.started].sort()).toEqual([...idsA, ...idsB].sort());
|
|
2963
|
+
expect(ra.summary.results).toHaveLength(2);
|
|
2964
|
+
expect(rb.summary.results).toHaveLength(2);
|
|
2965
|
+
expect([...ra.summary.results, ...rb.summary.results].every((r) => r.verdict === "passed")).toBe(true);
|
|
2966
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
2967
|
+
} finally {
|
|
2968
|
+
vi.useRealTimers();
|
|
2969
|
+
}
|
|
2970
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
2971
|
+
|
|
2972
|
+
// 撞满名额只可能是别的 Invocation 占着——本进程自己的并发早被进程内信号量挡住了。不报的话
|
|
2973
|
+
// 面板只剩 `0 running · N queued` 干等,看不出在等谁,更看不出生效名额被对方更小的声明夹低。
|
|
2974
|
+
// 这里让 B 声明 3 却撞上 A 的 declaredN: 1,断言诊断把「生效 1 / 本次声明 3」两个数都说出来。
|
|
2975
|
+
it("撞满名额报 gate-lease-waiting:带上生效名额与本次声明,min-N 夹低时两个数都在", async () => {
|
|
2976
|
+
vi.useFakeTimers();
|
|
2977
|
+
try {
|
|
2978
|
+
const root = await makeRoot();
|
|
2979
|
+
const experimentId = "gate-wait-notice-exp";
|
|
2980
|
+
const agent = makeAgent("agent-gate-wait");
|
|
2981
|
+
const { barrier, release } = makeBarrier();
|
|
2982
|
+
|
|
2983
|
+
// A 先占住唯一的名额(declaredN: 1)并卡在 barrier 上,B 随后带着 declaredN: 3 进来撞满。
|
|
2984
|
+
const runA = probeRun(agent, experimentId, ["w-a"], { maxConcurrency: 1 });
|
|
2985
|
+
const runB = probeRun(agent, experimentId, ["w-b"], { maxConcurrency: 3 });
|
|
2986
|
+
const probeA = newDispatchProbe();
|
|
2987
|
+
|
|
2988
|
+
const pa = runWithPriorResults([gatedEval("w-a", barrier, probeA)], [runA], {
|
|
2989
|
+
priorResults: [],
|
|
2990
|
+
root,
|
|
2991
|
+
maxConcurrency: 4,
|
|
2992
|
+
});
|
|
2993
|
+
await advanceOnFakeClock(() => probeA.started.length === 1, 5_000, 40);
|
|
2994
|
+
|
|
2995
|
+
const plan: RunFeedbackPlan = {
|
|
2996
|
+
shape: { evals: 1, configs: 1, totalAttempts: 1, maxConcurrency: 4 },
|
|
2997
|
+
reused: 0,
|
|
2998
|
+
reusedFailures: [],
|
|
2999
|
+
};
|
|
3000
|
+
const pb = withCoordinator(plan, async (coordinator) => {
|
|
3001
|
+
let bDone = false;
|
|
3002
|
+
const inner = runWithPriorResults([makeEval("w-b", async () => {})], [runB], {
|
|
3003
|
+
priorResults: [],
|
|
3004
|
+
root,
|
|
3005
|
+
maxConcurrency: 4,
|
|
3006
|
+
}).then((r) => {
|
|
3007
|
+
bDone = true;
|
|
3008
|
+
return r;
|
|
3009
|
+
});
|
|
3010
|
+
|
|
3011
|
+
const waited = (): (typeof coordinator.state.diagnostics)[number] | undefined =>
|
|
3012
|
+
coordinator.state.diagnostics.find((d) => d.code === "gate-lease-waiting");
|
|
3013
|
+
await advanceOnFakeClock(() => waited() !== undefined, 5_000, 40);
|
|
3014
|
+
|
|
3015
|
+
const notice = waited();
|
|
3016
|
+
expect(notice?.severity).toBe("warning");
|
|
3017
|
+
expect(notice?.data).toMatchObject({ experimentId, effectiveN: 1, declaredN: 3 });
|
|
3018
|
+
expect(notice?.message).toContain(experimentId);
|
|
3019
|
+
|
|
3020
|
+
release();
|
|
3021
|
+
await advanceOnFakeClock(() => bDone, 20_000, 80);
|
|
3022
|
+
return inner;
|
|
3023
|
+
});
|
|
3024
|
+
|
|
3025
|
+
await Promise.all([pa, pb]);
|
|
3026
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3027
|
+
} finally {
|
|
3028
|
+
vi.useRealTimers();
|
|
3029
|
+
}
|
|
3030
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3031
|
+
});
|
|
3032
|
+
|
|
3033
|
+
describe("runEvals · 用例锁: runs > 1 的兄弟 attempt 共享同一把锁", () => {
|
|
3034
|
+
afterEach(() => {
|
|
3035
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3036
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3037
|
+
});
|
|
3038
|
+
|
|
3039
|
+
it("自己已持有的直接放行:runs: 3 三条 attempt 同时在飞,锁目录仍只有一条(锁是逐用例的,不是逐 attempt)", async () => {
|
|
3040
|
+
const root = await makeRoot();
|
|
3041
|
+
const experimentId = "lock-sibling-hold-exp";
|
|
3042
|
+
const evalId = "sibling-hold-eval";
|
|
3043
|
+
const { barrier, release } = makeBarrier();
|
|
3044
|
+
const probe = newDispatchProbe();
|
|
3045
|
+
const agentRun = probeRun(makeAgent("agent-lock-sibling-hold"), experimentId, [evalId], { runs: 3 });
|
|
3046
|
+
|
|
3047
|
+
const runPromise = runWithPriorResults([gatedEval(evalId, barrier, probe)], [agentRun], {
|
|
3048
|
+
priorResults: [],
|
|
3049
|
+
root,
|
|
3050
|
+
maxConcurrency: 3,
|
|
3051
|
+
});
|
|
3052
|
+
try {
|
|
3053
|
+
await waitForRealProgress(() => expect(probe.inFlight).toBe(3));
|
|
3054
|
+
expect(await lockFilesRemaining(root)).toHaveLength(1);
|
|
3055
|
+
} finally {
|
|
3056
|
+
release();
|
|
3057
|
+
}
|
|
3058
|
+
|
|
3059
|
+
const { summary } = await runPromise;
|
|
3060
|
+
expect(summary.results.map((r) => r.attempt).sort()).toEqual([0, 1, 2]);
|
|
3061
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3062
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3063
|
+
|
|
3064
|
+
it("别人持有时整组挂在同一个等待窗口上:只开一条 lock_wait、elsewhere 计为 3 且与 queued 互斥,接管后三条一起派发", async () => {
|
|
3065
|
+
vi.useFakeTimers();
|
|
3066
|
+
try {
|
|
3067
|
+
const root = await makeRoot();
|
|
3068
|
+
const experimentId = "lock-sibling-wait-exp";
|
|
3069
|
+
const evalId = "sibling-wait-eval";
|
|
3070
|
+
await seedCaseLock(root, freshLockRecord(experimentId, evalId));
|
|
3071
|
+
|
|
3072
|
+
let calls = 0;
|
|
3073
|
+
const evalDef = makeEval(evalId, () => {
|
|
3074
|
+
calls += 1;
|
|
3075
|
+
});
|
|
3076
|
+
const agentRun = probeRun(makeAgent("agent-lock-sibling-wait"), experimentId, [evalId], { runs: 3 });
|
|
3077
|
+
const plan: RunFeedbackPlan = {
|
|
3078
|
+
shape: { evals: 1, configs: 1, totalAttempts: 3, maxConcurrency: 3 },
|
|
3079
|
+
reused: 0,
|
|
3080
|
+
reusedFailures: [],
|
|
3081
|
+
};
|
|
3082
|
+
|
|
3083
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3084
|
+
let done = false;
|
|
3085
|
+
const runPromise = runWithPriorResults([evalDef], [agentRun], {
|
|
3086
|
+
priorResults: [],
|
|
3087
|
+
root,
|
|
3088
|
+
maxConcurrency: 3,
|
|
3089
|
+
}).then((r) => {
|
|
3090
|
+
done = true;
|
|
3091
|
+
return r;
|
|
3092
|
+
});
|
|
3093
|
+
|
|
3094
|
+
await waitForRealProgress(() => expect(coordinator.state.elsewhere).toBe(3));
|
|
3095
|
+
// 三条兄弟共享一次试锁与一个等待窗口:等待条目是「一条用例」而不是「三个 attempt」,
|
|
3096
|
+
// 但 elsewhere 计的是 attempt 数(五项恒等式的口径)。
|
|
3097
|
+
expect(coordinator.state.lockWaits.get(experimentId)?.waiting.size).toBe(1);
|
|
3098
|
+
expect(coordinator.state.queued).toBe(0); // elsewhere 与 queued 互斥
|
|
3099
|
+
const mid = coordinator.state;
|
|
3100
|
+
expectCountIdentity(mid);
|
|
3101
|
+
expect(calls).toBe(0);
|
|
3102
|
+
|
|
3103
|
+
await advanceOnFakeClock(() => done, 10_000, 12);
|
|
3104
|
+
const { summary } = await runPromise;
|
|
3105
|
+
|
|
3106
|
+
expect(calls).toBe(3);
|
|
3107
|
+
expect(summary.results.map((r) => r.attempt).sort()).toEqual([0, 1, 2]);
|
|
3108
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
3109
|
+
const end = coordinator.state;
|
|
3110
|
+
expectCountIdentity(end);
|
|
3111
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3112
|
+
});
|
|
3113
|
+
} finally {
|
|
3114
|
+
vi.useRealTimers();
|
|
3115
|
+
}
|
|
3116
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3117
|
+
});
|
|
3118
|
+
|
|
3119
|
+
describe("runEvals · 用例锁: 释放后重查携带逐 attempt 判定", () => {
|
|
3120
|
+
afterEach(() => {
|
|
3121
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3122
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3123
|
+
});
|
|
3124
|
+
|
|
3125
|
+
// 与「用例锁: 释放后续接」那组的区别:那组走的是「撞上过期锁 → 接管」这条路径,重查携带
|
|
3126
|
+
// 顺带在取锁里发生;这里走的是真实的挂起窗口(撞新鲜锁 → elsewhere → 持有方正常释放),
|
|
3127
|
+
// 断言面是 elsewhere 两个方向的迁移本身:命中的序号迁 reused、没命中的序号迁 queued 自跑。
|
|
3128
|
+
// 判定必须逐 attempt(memory/carry-must-be-per-attempt-not-whole-eval-key:按整段 key 判会
|
|
3129
|
+
// 让同 eval 里一个序号的终态连带携入其它序号)。
|
|
3130
|
+
it("磁盘只有序号 0 的终态时:序号 0 从 elsewhere 迁 reused 不重跑,序号 1 迁 queued 自跑", async () => {
|
|
3131
|
+
const root = await makeRoot();
|
|
3132
|
+
const experimentId = "lock-recheck-exp";
|
|
3133
|
+
const evalId = "recheck-eval";
|
|
3134
|
+
const agent = makeAgent("agent-lock-recheck");
|
|
3135
|
+
const sandbox = fakeSandboxSpec();
|
|
3136
|
+
|
|
3137
|
+
// 先用一次真实运行落下序号 0 的 passed 终态(指纹由生产路径自己算,不手工拼)。
|
|
3138
|
+
const producerRun = probeRun(agent, experimentId, [evalId], { sandbox });
|
|
3139
|
+
const { summary: produced } = await run([makeEval(evalId, () => {})], [producerRun], { root });
|
|
3140
|
+
expect(produced.results[0]!.verdict).toBe("passed");
|
|
3141
|
+
|
|
3142
|
+
// 另一条 Invocation 此刻正持有这把锁(心跳新鲜:走等待窗口,不是过期接管)。
|
|
3143
|
+
await seedCaseLock(root, freshLockRecord(experimentId, evalId));
|
|
3144
|
+
|
|
3145
|
+
vi.useFakeTimers();
|
|
3146
|
+
try {
|
|
3147
|
+
let calls = 0;
|
|
3148
|
+
const subjectEval = makeEval(evalId, () => {
|
|
3149
|
+
calls += 1;
|
|
3150
|
+
});
|
|
3151
|
+
const subjectRun = probeRun(agent, experimentId, [evalId], { sandbox, runs: 2 });
|
|
3152
|
+
const plan: RunFeedbackPlan = {
|
|
3153
|
+
shape: { evals: 1, configs: 1, totalAttempts: 2, maxConcurrency: 2 },
|
|
3154
|
+
reused: 0,
|
|
3155
|
+
reusedFailures: [],
|
|
3156
|
+
};
|
|
3157
|
+
|
|
3158
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3159
|
+
let done = false;
|
|
3160
|
+
const runPromise = runWithPriorResults([subjectEval], [subjectRun], {
|
|
3161
|
+
priorResults: [],
|
|
3162
|
+
root,
|
|
3163
|
+
maxConcurrency: 2,
|
|
3164
|
+
}).then((r) => {
|
|
3165
|
+
done = true;
|
|
3166
|
+
return r;
|
|
3167
|
+
});
|
|
3168
|
+
|
|
3169
|
+
await waitForRealProgress(() => expect(coordinator.state.elsewhere).toBe(2));
|
|
3170
|
+
expect(coordinator.state.reused).toBe(0); // 静态携带规划(priorResults 为空)一条都没命中
|
|
3171
|
+
|
|
3172
|
+
// 持有方正常收尾:锁文件消失 —— 等待窗口下一轮轮询就结束,并重新做一次携带规划。
|
|
3173
|
+
await rm(caseLockPath(root, experimentId, evalId), { force: true });
|
|
3174
|
+
await advanceOnFakeClock(() => done, 10_000, 12);
|
|
3175
|
+
const { summary } = await runPromise;
|
|
3176
|
+
|
|
3177
|
+
expect(calls).toBe(1); // 只有缺的序号 1 自跑,序号 0 是携入
|
|
3178
|
+
expect(coordinator.state.reused).toBe(1); // elsewhere → reused
|
|
3179
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
3180
|
+
const end = coordinator.state;
|
|
3181
|
+
expectCountIdentity(end);
|
|
3182
|
+
expect(summary.results.map((r) => r.attempt).sort()).toEqual([0, 1]);
|
|
3183
|
+
expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
3184
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3185
|
+
});
|
|
3186
|
+
} finally {
|
|
3187
|
+
vi.useRealTimers();
|
|
3188
|
+
}
|
|
3189
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3190
|
+
|
|
3191
|
+
// 同一条逐 attempt 判定,换到干净取锁这条路径上再验一次:两条路径放走的错误类别不同。挂起
|
|
3192
|
+
// 窗口那条错在「迁移数与报进 elsewhere 的条数对不上」;这条错在「补发的那对瞬时 lock_wait
|
|
3193
|
+
// 报了整组而不是只报真正携入的那几条」——没携入的兄弟从没离开过 queued,连它们一起报会在
|
|
3194
|
+
// 两条事件之间把 queued 扣穿(极端时序下兄弟已进 running,那一扣就是负数)。
|
|
3195
|
+
it("干净取锁后只携入对方跑出来的序号 0:序号 1 照常自跑,补发的一对瞬时 lock_wait 只报携入的那一条", async () => {
|
|
3196
|
+
const root = await makeRoot();
|
|
3197
|
+
const experimentId = "clean-recheck-partial-exp";
|
|
3198
|
+
const gatedId = "p-1";
|
|
3199
|
+
const sharedId = "p-2";
|
|
3200
|
+
const agent = makeAgent("agent-clean-recheck-partial");
|
|
3201
|
+
const sandbox = fakeSandboxSpec();
|
|
3202
|
+
const { barrier, release } = makeBarrier();
|
|
3203
|
+
const all = newDispatchProbe();
|
|
3204
|
+
const sideA = newDispatchProbe();
|
|
3205
|
+
|
|
3206
|
+
// A 要两轮;对方只跑了 p-2 的序号 0。
|
|
3207
|
+
const evalsA = [gatedEval(gatedId, barrier, sideA, all), gatedEval(sharedId, barrier, sideA, all)];
|
|
3208
|
+
const runA = probeRun(agent, experimentId, [gatedId, sharedId], { sandbox, runs: 2 });
|
|
3209
|
+
const evalsPeer = [gatedEval(sharedId, Promise.resolve(), all)];
|
|
3210
|
+
const runPeer = probeRun(agent, experimentId, [sharedId], { sandbox });
|
|
3211
|
+
|
|
3212
|
+
const plan: RunFeedbackPlan = {
|
|
3213
|
+
shape: { evals: 3, configs: 1, totalAttempts: 5, maxConcurrency: 1 },
|
|
3214
|
+
reused: 0,
|
|
3215
|
+
reusedFailures: [],
|
|
3216
|
+
};
|
|
3217
|
+
|
|
3218
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3219
|
+
const pa = runWithPriorResults(evalsA, [runA], { priorResults: [], root, maxConcurrency: 1 });
|
|
3220
|
+
try {
|
|
3221
|
+
await waitForRealProgress(() => expect(all.inFlight).toBe(1));
|
|
3222
|
+
expect(sideA.started).toEqual([gatedId]);
|
|
3223
|
+
const { summary: peer } = await runWithPriorResults(evalsPeer, [runPeer], {
|
|
3224
|
+
priorResults: [],
|
|
3225
|
+
root,
|
|
3226
|
+
maxConcurrency: 1,
|
|
3227
|
+
});
|
|
3228
|
+
expect(peer.results.map((r) => r.attempt)).toEqual([0]);
|
|
3229
|
+
} finally {
|
|
3230
|
+
release();
|
|
3231
|
+
}
|
|
3232
|
+
|
|
3233
|
+
const { summary } = await pa;
|
|
3234
|
+
|
|
3235
|
+
// p-1 两轮都自跑;p-2 只补跑缺的那一轮 —— 判定逐 attempt,不按整条用例一刀切。
|
|
3236
|
+
expect(sideA.started.filter((id) => id === sharedId)).toHaveLength(1);
|
|
3237
|
+
expect(sideA.started.filter((id) => id === gatedId)).toHaveLength(2);
|
|
3238
|
+
const shared = summary.results.filter((r) => r.id === sharedId);
|
|
3239
|
+
expect(shared.map((r) => r.attempt).sort()).toEqual([0, 1]);
|
|
3240
|
+
expect(summary.results).toHaveLength(4);
|
|
3241
|
+
expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
3242
|
+
expect(coordinator.state.reused).toBe(1); // 只有序号 0 迁进 reused
|
|
3243
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
3244
|
+
expectCountIdentity(coordinator.state);
|
|
3245
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3246
|
+
});
|
|
3247
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3248
|
+
});
|
|
3249
|
+
|
|
3250
|
+
describe("runEvals · 用例锁: 取锁后重查携带的读取面", () => {
|
|
3251
|
+
afterEach(() => {
|
|
3252
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3253
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3254
|
+
});
|
|
3255
|
+
|
|
3256
|
+
// 无条件重查的前提是这次读盘便宜:它压在「已经握着全局并发位 + 实验闸名额 + 用例锁」的关键
|
|
3257
|
+
// 路径上,回答的是一个 per-case 的问题(这条用例现在还缺哪些 attempt)。用全根扫描回答它会
|
|
3258
|
+
// 随 .niceeval 历史线性变慢且永不收敛,单开也照付——所以读取面的形状本身是契约的一部分:
|
|
3259
|
+
// 按用例数各读一次收窄读取面,一次全树扫描都不做(runs 的兄弟共享同一次,不是按 attempt 数)。
|
|
3260
|
+
it("三条用例 × runs 2:收窄读取面恰好被调用 3 次(按用例数,不按 attempt 数),全树扫描 0 次", async () => {
|
|
3261
|
+
const root = await makeRoot();
|
|
3262
|
+
const experimentId = "read-surface-exp";
|
|
3263
|
+
const ids = ["r-1", "r-2", "r-3"];
|
|
3264
|
+
const sandbox = fakeSandboxSpec();
|
|
3265
|
+
|
|
3266
|
+
// 先落一批历史结果(另一个 agent ⇒ 指纹不同,不会被携入):读取面要在"结果树非空"的前提
|
|
3267
|
+
// 下计数,否则测不出"读了什么"与"读了多大一片"的区别。
|
|
3268
|
+
const historyRun = probeRun(makeAgent("agent-read-surface-history"), experimentId, ids, { sandbox });
|
|
3269
|
+
await run(ids.map((id) => makeEval(id, () => {})), [historyRun], { root });
|
|
3270
|
+
|
|
3271
|
+
let calls = 0;
|
|
3272
|
+
const evals = ids.map((id) =>
|
|
3273
|
+
makeEval(id, () => {
|
|
3274
|
+
calls += 1;
|
|
3275
|
+
}),
|
|
3276
|
+
);
|
|
3277
|
+
const subjectRun = probeRun(makeAgent("agent-read-surface"), experimentId, ids, { sandbox, runs: 2 });
|
|
3278
|
+
|
|
3279
|
+
readSurfaceCalls.forCase = 0;
|
|
3280
|
+
readSurfaceCalls.perEval = 0;
|
|
3281
|
+
const { summary } = await runWithPriorResults(evals, [subjectRun], { priorResults: [], root });
|
|
3282
|
+
|
|
3283
|
+
expect(calls).toBe(6); // 3 条用例 × 2 轮,历史结果指纹不同、一条都没被携入
|
|
3284
|
+
expect(summary.results).toHaveLength(6);
|
|
3285
|
+
expect(readSurfaceCalls.forCase).toBe(ids.length);
|
|
3286
|
+
expect(readSurfaceCalls.perEval).toBe(0);
|
|
3287
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3288
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3289
|
+
});
|
|
3290
|
+
|
|
3291
|
+
// ═══════════════════════ 止损闸(空间轴消费) ═══════════════════════
|
|
3292
|
+
// 覆盖规范:docs/engineering/testing/unit/experiments-runner.md「止损闸(空间轴消费)」
|
|
3293
|
+
// 契约:docs/feature/error-classification/README.md「止损语义」、architecture.md「止损执行体」
|
|
3294
|
+
//
|
|
3295
|
+
// 断言面全部是可观察的调度事实:test() 被真实调用了几次(启动集合)、`summary.results` 有几条、
|
|
3296
|
+
// 反馈状态的五项计数与诊断、`snapshot.json` 的实验域诊断。闸的内部形态(latch / AbortController /
|
|
3297
|
+
// 检查点次数)不进断言 —— 覆盖规范「观察面与边界」明确不锁内部信号量与 Promise 图。
|
|
3298
|
+
|
|
3299
|
+
/** 两条通路共用的折叠键(run.ts 的 haltGateKey);反馈流按 key 取,持久化侧按 code 过滤。 */
|
|
3300
|
+
const experimentHaltKey = (experimentId: string): string => `dispatch-halted:experiment:${experimentId}`;
|
|
3301
|
+
const evalHaltKey = (experimentId: string, evalId: string): string =>
|
|
3302
|
+
`dispatch-halted:eval:${experimentId}|${evalId}`;
|
|
3303
|
+
|
|
3304
|
+
/** 持久化通路:snapshot.json 里该 Experiment 的 dispatch-halted 诊断(可能一条都没有)。 */
|
|
3305
|
+
async function snapshotHaltDiagnostics(root: string, experimentId: string): Promise<DiagnosticRecord[]> {
|
|
3306
|
+
const results = await openResults(root);
|
|
3307
|
+
const exp = results.experiments.find((e) => e.id === experimentId);
|
|
3308
|
+
return (exp?.latest.diagnostics ?? []).filter((d) => d.code === "dispatch-halted");
|
|
3309
|
+
}
|
|
3310
|
+
|
|
3311
|
+
/** 八项计数恒等式(docs/feature/experiments/cli.md):任何一帧都必须成立。 */
|
|
3312
|
+
function expectCountIdentity(state: RunFeedbackState): void {
|
|
3313
|
+
expect(state.total).toBe(
|
|
3314
|
+
state.reused +
|
|
3315
|
+
state.running +
|
|
3316
|
+
state.elsewhere +
|
|
3317
|
+
state.queued +
|
|
3318
|
+
state.passed +
|
|
3319
|
+
state.failed +
|
|
3320
|
+
state.errored +
|
|
3321
|
+
state.skipped,
|
|
3322
|
+
);
|
|
3323
|
+
}
|
|
3324
|
+
|
|
3325
|
+
describe("runEvals · 止损闸: 触发", () => {
|
|
3326
|
+
afterEach(() => {
|
|
3327
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3328
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3329
|
+
});
|
|
3330
|
+
|
|
3331
|
+
// 触发(experiment 档)+ 记账 + 不连坐三面一次跑完:全局并发 1 让派发严格串行,被 halt 的
|
|
3332
|
+
// 实验只可能跑掉第一条;同批另一个实验的用例照常跑完,证明闸是按实验隔离的。
|
|
3333
|
+
it("ExperimentFatalError:同实验剩余 attempt 全停、计 unstarted,同批其它实验不连坐;errored 的 error code 不被 scope 改写", async () => {
|
|
3334
|
+
const haltedExp = "halt-trigger-exp";
|
|
3335
|
+
const bystanderExp = "halt-bystander-exp";
|
|
3336
|
+
const startedHalted: string[] = [];
|
|
3337
|
+
const startedBystander: string[] = [];
|
|
3338
|
+
const haltedEvals = ["a-fatal", "a-2", "a-3"].map((id) =>
|
|
3339
|
+
makeEval(id, () => {
|
|
3340
|
+
startedHalted.push(id);
|
|
3341
|
+
if (id === "a-fatal") throw new ExperimentFatalError("shared tunnel is down; run `make tunnel` and retry");
|
|
3342
|
+
}),
|
|
3343
|
+
);
|
|
3344
|
+
// 同批的对照实验:一条抛普通 Error(证明 error code 与 scope 声明无关,两边同为
|
|
3345
|
+
// unexpected-error),另一条照常通过 —— 普通 Error 不落任何闸,这个实验一条都不少跑。
|
|
3346
|
+
const bystanderEvals = ["b-plain-throw", "b-ok"].map((id) =>
|
|
3347
|
+
makeEval(id, () => {
|
|
3348
|
+
startedBystander.push(id);
|
|
3349
|
+
if (id === "b-plain-throw") throw new Error("just a normal failure");
|
|
3350
|
+
}),
|
|
3351
|
+
);
|
|
3352
|
+
const haltedRun = probeRun(makeAgent("agent-halt-trigger"), haltedExp, ["a-fatal", "a-2", "a-3"]);
|
|
3353
|
+
const bystanderRun = probeRun(makeAgent("agent-halt-bystander"), bystanderExp, ["b-plain-throw", "b-ok"]);
|
|
3354
|
+
const plan: RunFeedbackPlan = {
|
|
3355
|
+
shape: { evals: 5, configs: 2, totalAttempts: 5, maxConcurrency: 1 },
|
|
3356
|
+
reused: 0,
|
|
3357
|
+
reusedFailures: [],
|
|
3358
|
+
};
|
|
3359
|
+
|
|
3360
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3361
|
+
const { summary, root } = await runWithPriorResults([...haltedEvals, ...bystanderEvals], [haltedRun, bystanderRun], {
|
|
3362
|
+
priorResults: [],
|
|
3363
|
+
maxConcurrency: 1,
|
|
3364
|
+
});
|
|
3365
|
+
|
|
3366
|
+
// 触发:撞死的那条照常跑完并落账,同实验剩下两条一次都没进过 test()。
|
|
3367
|
+
expect(startedHalted).toEqual(["a-fatal"]);
|
|
3368
|
+
// 不连坐:另一个实验的两条全跑了(其中一条自己也 errored,照样不影响同批第三方)。
|
|
3369
|
+
expect([...startedBystander].sort()).toEqual(["b-ok", "b-plain-throw"]);
|
|
3370
|
+
|
|
3371
|
+
const halted = summary.results.filter((r) => r.experimentId === haltedExp);
|
|
3372
|
+
expect(halted).toHaveLength(1); // 不为没跑过的 attempt 制造 errored 记录
|
|
3373
|
+
expect(halted[0]!.verdict).toBe("errored");
|
|
3374
|
+
const bystander = summary.results.filter((r) => r.experimentId === bystanderExp);
|
|
3375
|
+
expect(bystander).toHaveLength(2);
|
|
3376
|
+
// error code 保持所属阶段的原有值:scope 是路由标记,不改写 AttemptError 的公开形状 ——
|
|
3377
|
+
// 声明了 scope 的那条与只抛普通 Error 的那条 code/phase 完全一致。
|
|
3378
|
+
const plainThrow = bystander.find((r) => r.id === "b-plain-throw")!;
|
|
3379
|
+
expect(halted[0]!.error?.code).toBe(plainThrow.error?.code);
|
|
3380
|
+
expect(halted[0]!.error?.code).toBe("unexpected-error");
|
|
3381
|
+
expect(halted[0]!.error?.phase).toBe(plainThrow.error?.phase);
|
|
3382
|
+
|
|
3383
|
+
// 记账:两条未派发计 unstarted(cli.ts 的 assembleInvocationCompletion 读的就是这条诊断的
|
|
3384
|
+
// data.unstarted,unstarted > 0 即完成状态 incomplete),五项计数恒等式收束时仍成立。
|
|
3385
|
+
const notice = coordinator.state.diagnostics.find((d) => d.key === experimentHaltKey(haltedExp));
|
|
3386
|
+
expect(notice).toBeDefined();
|
|
3387
|
+
expect(notice!.severity).toBe("error");
|
|
3388
|
+
expect(notice!.data?.unstarted).toBe(2);
|
|
3389
|
+
expect(coordinator.state.diagnostics.some((d) => d.key.startsWith("dispatch-halted:eval:"))).toBe(false);
|
|
3390
|
+
expectCountIdentity(coordinator.state);
|
|
3391
|
+
expect(coordinator.state.queued).toBe(0);
|
|
3392
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
3393
|
+
|
|
3394
|
+
// 同批其它实验的 snapshot 上不留任何 dispatch-halted。
|
|
3395
|
+
expect(await snapshotHaltDiagnostics(root, bystanderExp)).toEqual([]);
|
|
3396
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3397
|
+
});
|
|
3398
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3399
|
+
|
|
3400
|
+
// 触发(eval 档)+ 诊断双通路:runs: 3 下只停本 eval 剩余的两个 attempt,同实验另一个 eval 的
|
|
3401
|
+
// 三个 attempt 一个不少;两条通路(反馈流通知 / snapshot.json)各自带齐 scope、evalId 与 phase。
|
|
3402
|
+
it("EvalFatalError:只停本 eval 剩余 attempt(同实验另一个 eval 的 3 个 attempt 照跑);双通路的 scope/evalId/phase 同源", async () => {
|
|
3403
|
+
const experimentId = "halt-eval-scope-exp";
|
|
3404
|
+
const message = "fixture corrupted: regenerate data/fixtures";
|
|
3405
|
+
let fatalCalls = 0;
|
|
3406
|
+
let okCalls = 0;
|
|
3407
|
+
const evalFatal = makeEval("f-fatal", () => {
|
|
3408
|
+
fatalCalls += 1;
|
|
3409
|
+
throw new EvalFatalError(message);
|
|
3410
|
+
});
|
|
3411
|
+
const evalOk = makeEval("f-ok", () => {
|
|
3412
|
+
okCalls += 1;
|
|
3413
|
+
});
|
|
3414
|
+
const agentRun = probeRun(makeAgent("agent-halt-eval-scope"), experimentId, ["f-fatal", "f-ok"], { runs: 3 });
|
|
3415
|
+
const plan: RunFeedbackPlan = {
|
|
3416
|
+
shape: { evals: 2, configs: 1, totalAttempts: 6, maxConcurrency: 1 },
|
|
3417
|
+
reused: 0,
|
|
3418
|
+
reusedFailures: [],
|
|
3419
|
+
};
|
|
3420
|
+
|
|
3421
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3422
|
+
const { summary, root } = await runWithPriorResults([evalFatal, evalOk], [agentRun], {
|
|
3423
|
+
priorResults: [],
|
|
3424
|
+
maxConcurrency: 1,
|
|
3425
|
+
});
|
|
3426
|
+
|
|
3427
|
+
expect(fatalCalls).toBe(1); // 序号 1、2 被闸拦下
|
|
3428
|
+
expect(okCalls).toBe(3); // 同实验另一个 eval 完全不受影响
|
|
3429
|
+
expect(summary.results.filter((r) => r.id === "f-fatal")).toHaveLength(1);
|
|
3430
|
+
expect(summary.results.filter((r) => r.id === "f-ok").map((r) => r.attempt).sort()).toEqual([0, 1, 2]);
|
|
3431
|
+
|
|
3432
|
+
// 通路一:运行期反馈流。
|
|
3433
|
+
const notice = coordinator.state.diagnostics.find((d) => d.key === evalHaltKey(experimentId, "f-fatal"));
|
|
3434
|
+
expect(notice).toBeDefined();
|
|
3435
|
+
expect(notice!.code).toBe("dispatch-halted");
|
|
3436
|
+
expect(notice!.message).toContain(message);
|
|
3437
|
+
expect(notice!.data).toMatchObject({ scope: "eval", evalId: "f-fatal", phase: "eval.run" });
|
|
3438
|
+
expect(notice!.data?.unstarted).toBe(2);
|
|
3439
|
+
// 实验闸没落下:同实验其它 eval 的派发不该被 eval 档声明连带停掉。
|
|
3440
|
+
expect(coordinator.state.diagnostics.some((d) => d.key === experimentHaltKey(experimentId))).toBe(false);
|
|
3441
|
+
expectCountIdentity(coordinator.state);
|
|
3442
|
+
|
|
3443
|
+
// 通路二:snapshot.json 的实验域诊断。同源(同一份 message/phase/scope),但各自累计 ——
|
|
3444
|
+
// 反馈流那条含未派发记账刷新的 count,持久化这条只按声明次数折叠(这里只声明过一次)。
|
|
3445
|
+
const persisted = await snapshotHaltDiagnostics(root, experimentId);
|
|
3446
|
+
expect(persisted).toHaveLength(1);
|
|
3447
|
+
expect(persisted[0]).toMatchObject({
|
|
3448
|
+
code: "dispatch-halted",
|
|
3449
|
+
level: "error",
|
|
3450
|
+
phase: "eval.run",
|
|
3451
|
+
data: { scope: "eval", evalId: "f-fatal" },
|
|
3452
|
+
});
|
|
3453
|
+
expect(persisted[0]!.message).toContain(message);
|
|
3454
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3455
|
+
});
|
|
3456
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3457
|
+
});
|
|
3458
|
+
|
|
3459
|
+
describe("runEvals · 止损闸: 组合(时间轴先走,空间轴只对终局失败生效)", () => {
|
|
3460
|
+
afterEach(() => {
|
|
3461
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3462
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3463
|
+
});
|
|
3464
|
+
|
|
3465
|
+
// 同一份 { retryable: true, scope: "experiment" } 声明,两个方向各一条区分力场景:被重试
|
|
3466
|
+
// 吸收就到不了闸(本例),重试耗尽的终局失败才读 scope(下一例)。
|
|
3467
|
+
it("可重试失败被重试吸收:同一份带 scope 的声明重试成功后不落闸,同实验后续用例照常派发", async () => {
|
|
3468
|
+
vi.useFakeTimers();
|
|
3469
|
+
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0.9);
|
|
3470
|
+
try {
|
|
3471
|
+
const experimentId = "halt-absorbed-exp";
|
|
3472
|
+
let sendCalls = 0;
|
|
3473
|
+
const agent = defineSandboxAgent({
|
|
3474
|
+
name: "agent-halt-absorbed",
|
|
3475
|
+
send: async () => {
|
|
3476
|
+
sendCalls += 1;
|
|
3477
|
+
// 只有第一次 send 撞限流:退避一次后成功,失败 Turn 从不外泄给 expectOk()。
|
|
3478
|
+
return sendCalls === 1 ? retryableFailureTurn("rate limited, please retry later") : okTurn();
|
|
3479
|
+
},
|
|
3480
|
+
});
|
|
3481
|
+
const started: string[] = [];
|
|
3482
|
+
const evals = ["c-1", "c-2"].map((id) =>
|
|
3483
|
+
makeEval(id, async (t) => {
|
|
3484
|
+
started.push(id);
|
|
3485
|
+
(await t.send("go")).expectOk();
|
|
3486
|
+
}),
|
|
3487
|
+
);
|
|
3488
|
+
const agentRun = probeRun(agent, experimentId, ["c-1", "c-2"], {
|
|
3489
|
+
maxConcurrency: 1,
|
|
3490
|
+
// 实验分类器排在 adapter / 兜底之前:同一段限流文本被认领成「可重试 + 实验级」。
|
|
3491
|
+
classifyFailure: (f) =>
|
|
3492
|
+
f.text.includes("rate limited") ? { retryable: true, reason: "rate_limit", scope: "experiment" } : undefined,
|
|
3493
|
+
});
|
|
3494
|
+
const plan: RunFeedbackPlan = {
|
|
3495
|
+
shape: { evals: 2, configs: 1, totalAttempts: 2, maxConcurrency: 1 },
|
|
3496
|
+
reused: 0,
|
|
3497
|
+
reusedFailures: [],
|
|
3498
|
+
};
|
|
3499
|
+
|
|
3500
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3501
|
+
let done = false;
|
|
3502
|
+
const runPromise = runWithPriorResults(evals, [agentRun], {
|
|
3503
|
+
priorResults: [],
|
|
3504
|
+
maxConcurrency: 1,
|
|
3505
|
+
}).then((r) => {
|
|
3506
|
+
done = true;
|
|
3507
|
+
return r;
|
|
3508
|
+
});
|
|
3509
|
+
|
|
3510
|
+
await advanceOnFakeClock(() => done, 10_000, 12);
|
|
3511
|
+
const { summary, root } = await runPromise;
|
|
3512
|
+
|
|
3513
|
+
expect([...started].sort()).toEqual(["c-1", "c-2"]); // 闸没落下:第二条照常派发
|
|
3514
|
+
expect(summary.results).toHaveLength(2);
|
|
3515
|
+
expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
3516
|
+
expect(sendCalls).toBe(3); // c-1 失败 1 次 + 重试成功 1 次,c-2 成功 1 次
|
|
3517
|
+
expect(coordinator.state.diagnostics.some((d) => d.key.startsWith("dispatch-halted:"))).toBe(false);
|
|
3518
|
+
expect(await snapshotHaltDiagnostics(root, experimentId)).toEqual([]);
|
|
3519
|
+
});
|
|
3520
|
+
} finally {
|
|
3521
|
+
vi.useRealTimers();
|
|
3522
|
+
randomSpy.mockRestore();
|
|
3523
|
+
}
|
|
3524
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3525
|
+
|
|
3526
|
+
it("重试耗尽后的终局失败才读 scope:同一份声明在耗尽路径上照常落闸,同实验剩余用例停派发", async () => {
|
|
3527
|
+
vi.useFakeTimers();
|
|
3528
|
+
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0.9);
|
|
3529
|
+
try {
|
|
3530
|
+
const experimentId = "halt-exhausted-exp";
|
|
3531
|
+
let sendCalls = 0;
|
|
3532
|
+
const agent = defineSandboxAgent({
|
|
3533
|
+
name: "agent-halt-exhausted",
|
|
3534
|
+
send: async () => {
|
|
3535
|
+
sendCalls += 1;
|
|
3536
|
+
return retryableFailureTurn("rate limited, please retry later"); // 永远撞限流,重试必耗尽
|
|
3537
|
+
},
|
|
3538
|
+
});
|
|
3539
|
+
const started: string[] = [];
|
|
3540
|
+
const evals = ["d-1", "d-2"].map((id) =>
|
|
3541
|
+
makeEval(id, async (t) => {
|
|
3542
|
+
started.push(id);
|
|
3543
|
+
(await t.send("go")).expectOk();
|
|
3544
|
+
}),
|
|
3545
|
+
);
|
|
3546
|
+
const agentRun = probeRun(agent, experimentId, ["d-1", "d-2"], {
|
|
3547
|
+
maxConcurrency: 1,
|
|
3548
|
+
classifyFailure: (f) =>
|
|
3549
|
+
f.text.includes("rate limited") ? { retryable: true, reason: "rate_limit", scope: "experiment" } : undefined,
|
|
3550
|
+
});
|
|
3551
|
+
const plan: RunFeedbackPlan = {
|
|
3552
|
+
shape: { evals: 2, configs: 1, totalAttempts: 2, maxConcurrency: 1 },
|
|
3553
|
+
reused: 0,
|
|
3554
|
+
reusedFailures: [],
|
|
3555
|
+
};
|
|
3556
|
+
|
|
3557
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3558
|
+
let done = false;
|
|
3559
|
+
const runPromise = runWithPriorResults(evals, [agentRun], {
|
|
3560
|
+
priorResults: [],
|
|
3561
|
+
maxConcurrency: 1,
|
|
3562
|
+
}).then((r) => {
|
|
3563
|
+
done = true;
|
|
3564
|
+
return r;
|
|
3565
|
+
});
|
|
3566
|
+
|
|
3567
|
+
// send 级预算封顶 4 次尝试,退避 4.5s / 9s / 18s(Math.random 定量成 0.9)。
|
|
3568
|
+
await advanceOnFakeClock(() => done, 10_000, 20);
|
|
3569
|
+
const { summary } = await runPromise;
|
|
3570
|
+
|
|
3571
|
+
expect(started).toEqual(["d-1"]); // 耗尽后的终局失败落闸:d-2 不再派发
|
|
3572
|
+
expect(sendCalls).toBe(4);
|
|
3573
|
+
expect(summary.results).toHaveLength(1);
|
|
3574
|
+
expect(summary.results[0]!.verdict).toBe("errored");
|
|
3575
|
+
const notice = coordinator.state.diagnostics.find((d) => d.key === experimentHaltKey(experimentId));
|
|
3576
|
+
expect(notice).toBeDefined();
|
|
3577
|
+
expect(notice!.data?.unstarted).toBe(1);
|
|
3578
|
+
});
|
|
3579
|
+
} finally {
|
|
3580
|
+
vi.useRealTimers();
|
|
3581
|
+
randomSpy.mockRestore();
|
|
3582
|
+
}
|
|
3583
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3584
|
+
});
|
|
3585
|
+
|
|
3586
|
+
describe("runEvals · 止损闸: 幂等与不可逆", () => {
|
|
3587
|
+
afterEach(() => {
|
|
3588
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3589
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3590
|
+
});
|
|
3591
|
+
|
|
3592
|
+
it("并发三条同时声明同一死因:两条通路各折叠成一条 dispatch-halted、count 累加到 3", async () => {
|
|
3593
|
+
const experimentId = "halt-idempotent-exp";
|
|
3594
|
+
const { barrier, release } = makeBarrier();
|
|
3595
|
+
const probe = newDispatchProbe();
|
|
3596
|
+
const ids = ["i-1", "i-2", "i-3"];
|
|
3597
|
+
// 三条一起挂在同一个 barrier 上:释放的瞬间三条并发抛出同一个实验级声明,这正是
|
|
3598
|
+
// 「并发 attempt 同时声明同一死因」的常态形状。
|
|
3599
|
+
const evals = ids.map((id) =>
|
|
3600
|
+
makeEval(id, async () => {
|
|
3601
|
+
probe.started.push(id);
|
|
3602
|
+
probe.inFlight += 1;
|
|
3603
|
+
probe.peak = Math.max(probe.peak, probe.inFlight);
|
|
3604
|
+
await barrier;
|
|
3605
|
+
probe.inFlight -= 1;
|
|
3606
|
+
throw new ExperimentFatalError("shared credentials expired; refresh .env");
|
|
3607
|
+
}),
|
|
3608
|
+
);
|
|
3609
|
+
const agentRun = probeRun(makeAgent("agent-halt-idempotent"), experimentId, ids);
|
|
3610
|
+
const plan: RunFeedbackPlan = {
|
|
3611
|
+
shape: { evals: 3, configs: 1, totalAttempts: 3, maxConcurrency: 3 },
|
|
3612
|
+
reused: 0,
|
|
3613
|
+
reusedFailures: [],
|
|
3614
|
+
};
|
|
3615
|
+
|
|
3616
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3617
|
+
const runPromise = runWithPriorResults(evals, [agentRun], { priorResults: [], maxConcurrency: 3 });
|
|
3618
|
+
try {
|
|
3619
|
+
await waitForRealProgress(() => expect(probe.inFlight).toBe(3));
|
|
3620
|
+
} finally {
|
|
3621
|
+
release();
|
|
3622
|
+
}
|
|
3623
|
+
const { summary, root } = await runPromise;
|
|
3624
|
+
|
|
3625
|
+
expect(summary.results).toHaveLength(3);
|
|
3626
|
+
expect(summary.results.every((r) => r.verdict === "errored")).toBe(true);
|
|
3627
|
+
|
|
3628
|
+
const notices = coordinator.state.diagnostics.filter((d) => d.key.startsWith("dispatch-halted:"));
|
|
3629
|
+
expect(notices).toHaveLength(1); // 三次声明折叠成一条(dedupeKey 相同)
|
|
3630
|
+
expect(notices[0]!.count).toBe(3);
|
|
3631
|
+
expect(notices[0]!.data?.unstarted).toBe(0); // 三条都在飞,一条都没被拦下
|
|
3632
|
+
|
|
3633
|
+
const persisted = await snapshotHaltDiagnostics(root, experimentId);
|
|
3634
|
+
expect(persisted).toHaveLength(1);
|
|
3635
|
+
expect(persisted[0]!.count).toBe(3);
|
|
3636
|
+
expect(persisted[0]!.data).toMatchObject({ scope: "experiment" });
|
|
3637
|
+
expect(persisted[0]!.data?.evalId).toBeUndefined(); // 实验闸没有 evalId
|
|
3638
|
+
});
|
|
3639
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3640
|
+
|
|
3641
|
+
it("落闸后在飞 attempt 成功也不重开派发:在飞的照常跑完落账,排队的仍然全停", async () => {
|
|
3642
|
+
const experimentId = "halt-irreversible-exp";
|
|
3643
|
+
const fatal = makeBarrier();
|
|
3644
|
+
const survivor = makeBarrier();
|
|
3645
|
+
const started: string[] = [];
|
|
3646
|
+
const ids = ["n-fatal", "n-survivor", "n-late-1", "n-late-2"];
|
|
3647
|
+
const evals = ids.map((id) =>
|
|
3648
|
+
makeEval(id, async () => {
|
|
3649
|
+
started.push(id);
|
|
3650
|
+
if (id === "n-fatal") {
|
|
3651
|
+
await fatal.barrier;
|
|
3652
|
+
throw new ExperimentFatalError("shared service died mid-run");
|
|
3653
|
+
}
|
|
3654
|
+
if (id === "n-survivor") await survivor.barrier;
|
|
3655
|
+
}),
|
|
3656
|
+
);
|
|
3657
|
+
const agentRun = probeRun(makeAgent("agent-halt-irreversible"), experimentId, ids);
|
|
3658
|
+
const plan: RunFeedbackPlan = {
|
|
3659
|
+
shape: { evals: 4, configs: 1, totalAttempts: 4, maxConcurrency: 2 },
|
|
3660
|
+
reused: 0,
|
|
3661
|
+
reusedFailures: [],
|
|
3662
|
+
};
|
|
3663
|
+
|
|
3664
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3665
|
+
const runPromise = runWithPriorResults(evals, [agentRun], { priorResults: [], maxConcurrency: 2 });
|
|
3666
|
+
try {
|
|
3667
|
+
// 全局并发 2:前两条在飞,后两条排队。
|
|
3668
|
+
await waitForRealProgress(() => expect(started).toHaveLength(2));
|
|
3669
|
+
fatal.release();
|
|
3670
|
+
// 等闸真的落下(诊断出现)再放行幸存者,确保它的成功发生在落闸之后。
|
|
3671
|
+
await waitForRealProgress(() =>
|
|
3672
|
+
expect(coordinator.state.diagnostics.some((d) => d.key === experimentHaltKey(experimentId))).toBe(true),
|
|
3673
|
+
);
|
|
3674
|
+
} finally {
|
|
3675
|
+
survivor.release();
|
|
3676
|
+
}
|
|
3677
|
+
const { summary } = await runPromise;
|
|
3678
|
+
|
|
3679
|
+
// 不抢占:在飞的 n-survivor 照常跑完并如实落账(passed);它的成功不重开派发。
|
|
3680
|
+
expect([...started].sort()).toEqual(["n-fatal", "n-survivor"]);
|
|
3681
|
+
expect(summary.results).toHaveLength(2);
|
|
3682
|
+
expect(summary.results.find((r) => r.id === "n-survivor")!.verdict).toBe("passed");
|
|
3683
|
+
expect(summary.results.find((r) => r.id === "n-fatal")!.verdict).toBe("errored");
|
|
3684
|
+
const notice = coordinator.state.diagnostics.find((d) => d.key === experimentHaltKey(experimentId));
|
|
3685
|
+
expect(notice!.data?.unstarted).toBe(2);
|
|
3686
|
+
expectCountIdentity(coordinator.state);
|
|
3687
|
+
expect(coordinator.state.queued).toBe(0);
|
|
3688
|
+
});
|
|
3689
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3690
|
+
});
|
|
3691
|
+
|
|
3692
|
+
describe("runEvals · 止损闸: 不抢占(等待集经 interruption 中止)", () => {
|
|
3693
|
+
afterEach(() => {
|
|
3694
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3695
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3696
|
+
});
|
|
3697
|
+
|
|
3698
|
+
// 挂在 elsewhere 上的用例等的是「别人的锁什么时候放」,轮询周期是 10s 心跳。闸落下时它必须
|
|
3699
|
+
// 经既有 interruption 通路当场退出等待集,而不是陪着等满一个周期 —— 所以这条用真实时钟跑,
|
|
3700
|
+
// 断言面是硬墙钟耗时。
|
|
3701
|
+
it("落闸时挂在 elsewhere 的用例立刻退出等待集(远早于 10s 轮询周期),计 unstarted 且五项恒等式不破", async () => {
|
|
3702
|
+
const root = await makeRoot();
|
|
3703
|
+
const experimentId = "halt-elsewhere-exp";
|
|
3704
|
+
const lockedId = "w-locked";
|
|
3705
|
+
await seedCaseLock(root, freshLockRecord(experimentId, lockedId));
|
|
3706
|
+
|
|
3707
|
+
let lockedCalls = 0;
|
|
3708
|
+
const evalLocked = makeEval(lockedId, () => {
|
|
3709
|
+
lockedCalls += 1;
|
|
3710
|
+
});
|
|
3711
|
+
const evalFatal = makeEval("w-fatal", () => {
|
|
3712
|
+
throw new ExperimentFatalError("shared svc down");
|
|
3713
|
+
});
|
|
3714
|
+
const agentRun = probeRun(makeAgent("agent-halt-elsewhere"), experimentId, [lockedId, "w-fatal"]);
|
|
3715
|
+
const plan: RunFeedbackPlan = {
|
|
3716
|
+
shape: { evals: 2, configs: 1, totalAttempts: 2, maxConcurrency: 2 },
|
|
3717
|
+
reused: 0,
|
|
3718
|
+
reusedFailures: [],
|
|
3719
|
+
};
|
|
3720
|
+
|
|
3721
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3722
|
+
const startedAt = Date.now();
|
|
3723
|
+
const { summary } = await runWithPriorResults([evalLocked, evalFatal], [agentRun], {
|
|
3724
|
+
priorResults: [],
|
|
3725
|
+
root,
|
|
3726
|
+
maxConcurrency: 2,
|
|
3727
|
+
});
|
|
3728
|
+
const elapsedMs = Date.now() - startedAt;
|
|
3729
|
+
|
|
3730
|
+
// 硬时间断言:锁等待轮询周期是 10s,不走中止通路的话整段至少要拖满一个周期。
|
|
3731
|
+
expect(elapsedMs).toBeLessThan(8_000);
|
|
3732
|
+
expect(lockedCalls).toBe(0); // 等待集里的那条从没被派发
|
|
3733
|
+
expect(summary.results).toHaveLength(1); // 不为没跑过的 attempt 制造 errored 记录
|
|
3734
|
+
expect(summary.results[0]!.id).toBe("w-fatal");
|
|
3735
|
+
expect(summary.results[0]!.verdict).toBe("errored");
|
|
3736
|
+
|
|
3737
|
+
const notice = coordinator.state.diagnostics.find((d) => d.key === experimentHaltKey(experimentId));
|
|
3738
|
+
expect(notice!.data?.unstarted).toBe(1);
|
|
3739
|
+
// elsewhere 收支平账:进过等待集的那条必须被原数报回来,恒等式不留悬空的差额。
|
|
3740
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
3741
|
+
expect(coordinator.state.queued).toBe(0);
|
|
3742
|
+
expectCountIdentity(coordinator.state);
|
|
3743
|
+
});
|
|
3744
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3745
|
+
});
|
|
3746
|
+
|
|
3747
|
+
describe("runEvals · 止损闸: teardown 边界", () => {
|
|
3748
|
+
afterEach(() => {
|
|
3749
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3750
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3751
|
+
});
|
|
3752
|
+
|
|
3753
|
+
it("实验级 teardown 抛声明:降级为普通 teardown 诊断,不落闸", async () => {
|
|
3754
|
+
const experimentId = "halt-exp-teardown";
|
|
3755
|
+
const started: string[] = [];
|
|
3756
|
+
const ids = ["t-1", "t-2"];
|
|
3757
|
+
const evals = ids.map((id) =>
|
|
3758
|
+
makeEval(id, () => {
|
|
3759
|
+
started.push(id);
|
|
3760
|
+
}),
|
|
3761
|
+
);
|
|
3762
|
+
const agentRun = probeRun(makeAgent("agent-halt-exp-teardown"), experimentId, ids, {
|
|
3763
|
+
maxConcurrency: 1,
|
|
3764
|
+
setup: () => {},
|
|
3765
|
+
teardown: () => {
|
|
3766
|
+
throw new ExperimentFatalError("shared db handle leaked; restart the stack");
|
|
3767
|
+
},
|
|
3768
|
+
});
|
|
3769
|
+
const plan: RunFeedbackPlan = {
|
|
3770
|
+
shape: { evals: 2, configs: 1, totalAttempts: 2, maxConcurrency: 1 },
|
|
3771
|
+
reused: 0,
|
|
3772
|
+
reusedFailures: [],
|
|
3773
|
+
};
|
|
3774
|
+
|
|
3775
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3776
|
+
const { summary, root } = await runWithPriorResults(evals, [agentRun], {
|
|
3777
|
+
priorResults: [],
|
|
3778
|
+
maxConcurrency: 1,
|
|
3779
|
+
});
|
|
3780
|
+
|
|
3781
|
+
expect([...started].sort()).toEqual(ids);
|
|
3782
|
+
expect(summary.results).toHaveLength(2);
|
|
3783
|
+
expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
|
|
3784
|
+
// 降级:只留既有的 teardown 失败诊断,一条 dispatch-halted 都不产生(两条通路都不产生)。
|
|
3785
|
+
const teardownDiag = coordinator.state.diagnostics.find(
|
|
3786
|
+
(d) => d.key === `experiment-teardown-failed:${experimentId}`,
|
|
3787
|
+
);
|
|
3788
|
+
expect(teardownDiag).toBeDefined();
|
|
3789
|
+
expect(teardownDiag!.message).toContain("shared db handle leaked");
|
|
3790
|
+
expect(coordinator.state.diagnostics.some((d) => d.key.startsWith("dispatch-halted:"))).toBe(false);
|
|
3791
|
+
expect(await snapshotHaltDiagnostics(root, experimentId)).toEqual([]);
|
|
3792
|
+
});
|
|
3793
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3794
|
+
|
|
3795
|
+
it("per-attempt teardown 抛声明:照常落闸(剩余两条计 unstarted),但 verdict 仍是 passed", async () => {
|
|
3796
|
+
const experimentId = "halt-attempt-teardown";
|
|
3797
|
+
const started: string[] = [];
|
|
3798
|
+
const ids = ["k-1", "k-2", "k-3"];
|
|
3799
|
+
const evals = ids.map((id) =>
|
|
3800
|
+
makeEval(id, () => {
|
|
3801
|
+
started.push(id);
|
|
3802
|
+
}),
|
|
3803
|
+
);
|
|
3804
|
+
// eval.teardown 是 attempt 收尾链的第一段,失败只记诊断、不改判定;空间轴声明照常落闸。
|
|
3805
|
+
evals[0]!.teardown = () => {
|
|
3806
|
+
throw new ExperimentFatalError("shared db handle leaked; restart the stack");
|
|
3807
|
+
};
|
|
3808
|
+
const agentRun = probeRun(makeAgent("agent-halt-attempt-teardown"), experimentId, ids);
|
|
3809
|
+
const plan: RunFeedbackPlan = {
|
|
3810
|
+
shape: { evals: 3, configs: 1, totalAttempts: 3, maxConcurrency: 1 },
|
|
3811
|
+
reused: 0,
|
|
3812
|
+
reusedFailures: [],
|
|
3813
|
+
};
|
|
3814
|
+
|
|
3815
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3816
|
+
const { summary, root } = await runWithPriorResults(evals, [agentRun], {
|
|
3817
|
+
priorResults: [],
|
|
3818
|
+
maxConcurrency: 1,
|
|
3819
|
+
});
|
|
3820
|
+
|
|
3821
|
+
expect(started).toEqual(["k-1"]);
|
|
3822
|
+
expect(summary.results).toHaveLength(1);
|
|
3823
|
+
expect(summary.results[0]!.verdict).toBe("passed"); // 落闸不改 verdict
|
|
3824
|
+
expect(summary.results[0]!.diagnostics?.some((d) => d.phase === "eval.teardown")).toBe(true);
|
|
3825
|
+
|
|
3826
|
+
const notice = coordinator.state.diagnostics.find((d) => d.key === experimentHaltKey(experimentId));
|
|
3827
|
+
expect(notice).toBeDefined();
|
|
3828
|
+
expect(notice!.data?.unstarted).toBe(2);
|
|
3829
|
+
expect(notice!.data?.phase).toBe("eval.teardown"); // 触发失败所在的生命周期阶段
|
|
3830
|
+
const persisted = await snapshotHaltDiagnostics(root, experimentId);
|
|
3831
|
+
expect(persisted).toHaveLength(1);
|
|
3832
|
+
expect(persisted[0]!.phase).toBe("eval.teardown");
|
|
3833
|
+
expectCountIdentity(coordinator.state);
|
|
3834
|
+
});
|
|
3835
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3836
|
+
});
|
|
3837
|
+
|
|
3838
|
+
// ─────────────── 用例锁:elsewhere 收支平账的两条回归(五项恒等式的守护) ───────────────
|
|
3839
|
+
// 覆盖规范:docs/engineering/testing/unit/experiments-runner.md「用例锁与并发 Invocation」的
|
|
3840
|
+
// 「挂起用例……计入独立的 `elsewhere` 计数且与 `queued` 互斥、五项计数恒等式成立」与
|
|
3841
|
+
// 「执行模式组合——`--force` 等待后全部自跑」两分句。
|
|
3842
|
+
//
|
|
3843
|
+
// 上面「执行模式组合」那组走的是过期锁接管路径,压根没进过挂起窗口;这里的三条专测窗口本身
|
|
3844
|
+
// 的收支:报进 elsewhere 多少条,收尾就要原数报回多少条 —— 差额挂住即恒等式当场破。
|
|
3845
|
+
|
|
3846
|
+
describe("runEvals · 用例锁: --force 撞新鲜锁的挂起窗口", () => {
|
|
3847
|
+
afterEach(() => {
|
|
3848
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3849
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3850
|
+
});
|
|
3851
|
+
|
|
3852
|
+
// bug: --force 下撞**新鲜**锁只发 started 不发 resolved,elsewhere 永久挂住、五项恒等式当场破。
|
|
3853
|
+
// 上面「执行模式组合」那条走的是 stale 锁接管(从没进过挂起窗口),盖不住这个窗口。
|
|
3854
|
+
it("--force + 新鲜锁 + 持有方释放:窗口照常收 resolved,elsewhere 归零、恒等式成立,用例全部自跑", async () => {
|
|
3855
|
+
const root = await makeRoot();
|
|
3856
|
+
const experimentId = "force-fresh-lock-exp";
|
|
3857
|
+
const evalId = "force-fresh-eval";
|
|
3858
|
+
const agent = makeAgent("agent-force-fresh");
|
|
3859
|
+
const sandbox = fakeSandboxSpec();
|
|
3860
|
+
|
|
3861
|
+
// 先落一条指纹匹配的 passed 终态:--force 下它不该被消费,等完窗口照样自跑。
|
|
3862
|
+
const producerRun = probeRun(agent, experimentId, [evalId], { sandbox });
|
|
3863
|
+
const { summary: produced } = await run([makeEval(evalId, () => {})], [producerRun], { root });
|
|
3864
|
+
expect(produced.results[0]!.verdict).toBe("passed");
|
|
3865
|
+
|
|
3866
|
+
// 另一条 Invocation 此刻正持有这把锁(心跳新鲜:走真实的挂起窗口,不是过期接管)。
|
|
3867
|
+
await seedCaseLock(root, freshLockRecord(experimentId, evalId));
|
|
3868
|
+
|
|
3869
|
+
vi.useFakeTimers();
|
|
3870
|
+
try {
|
|
3871
|
+
let calls = 0;
|
|
3872
|
+
const subjectEval = makeEval(evalId, () => {
|
|
3873
|
+
calls += 1;
|
|
3874
|
+
});
|
|
3875
|
+
const plan: RunFeedbackPlan = {
|
|
3876
|
+
shape: { evals: 1, configs: 1, totalAttempts: 1, maxConcurrency: 2 },
|
|
3877
|
+
reused: 0,
|
|
3878
|
+
reusedFailures: [],
|
|
3879
|
+
};
|
|
3880
|
+
|
|
3881
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3882
|
+
let done = false;
|
|
3883
|
+
// force 模式:cli.ts 在 --force 时整段不传 priorResults(不是传空数组)。
|
|
3884
|
+
const runPromise = runWithPriorResults([subjectEval], [producerRun], { root, maxConcurrency: 2 }).then((r) => {
|
|
3885
|
+
done = true;
|
|
3886
|
+
return r;
|
|
3887
|
+
});
|
|
3888
|
+
|
|
3889
|
+
await waitForRealProgress(() => expect(coordinator.state.elsewhere).toBe(1));
|
|
3890
|
+
expect(coordinator.state.queued).toBe(0); // elsewhere 与 queued 互斥
|
|
3891
|
+
expectCountIdentity(coordinator.state);
|
|
3892
|
+
|
|
3893
|
+
// 持有方正常收尾:锁文件消失,下一轮轮询即结束等待。
|
|
3894
|
+
await rm(caseLockPath(root, experimentId, evalId), { force: true });
|
|
3895
|
+
await advanceOnFakeClock(() => done, 10_000, 12);
|
|
3896
|
+
const { summary } = await runPromise;
|
|
3897
|
+
|
|
3898
|
+
expect(calls).toBe(1); // --force:等完窗口后自跑,不消费磁盘上那条匹配终态
|
|
3899
|
+
expect(coordinator.state.reused).toBe(0);
|
|
3900
|
+
expect(coordinator.state.elsewhere).toBe(0); // 报进去 1 条,原数报了回来
|
|
3901
|
+
expectCountIdentity(coordinator.state);
|
|
3902
|
+
expect(summary.results).toHaveLength(1);
|
|
3903
|
+
expect(summary.results[0]!.verdict).toBe("passed");
|
|
3904
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
3905
|
+
});
|
|
3906
|
+
} finally {
|
|
3907
|
+
vi.useRealTimers();
|
|
3908
|
+
}
|
|
3909
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3910
|
+
});
|
|
3911
|
+
|
|
3912
|
+
describe("runEvals · 用例锁: 等待窗口的 elsewhere 收支平账", () => {
|
|
3913
|
+
afterEach(() => {
|
|
3914
|
+
expect(activeFeedbackSinkCount()).toBe(0);
|
|
3915
|
+
expect(pendingHeldCaseLockCount()).toBe(0);
|
|
3916
|
+
});
|
|
3917
|
+
|
|
3918
|
+
// 报进 elsewhere 的条数必须原数报回:等待期间被中断而提前 settle 的 attempt 会让「本组还剩
|
|
3919
|
+
// 几条没收尾」缩水,收尾时拿当下的剩余条数当迁移数,差额就永远挂在 elsewhere 上。
|
|
3920
|
+
it("挂起期间用户中断:runs: 2 整组报进 elsewhere 的两条被原数报回,elsewhere 归零、恒等式成立", async () => {
|
|
3921
|
+
const root = await makeRoot();
|
|
3922
|
+
const experimentId = "elsewhere-interrupt-exp";
|
|
3923
|
+
const evalId = "elsewhere-interrupt-eval";
|
|
3924
|
+
await seedCaseLock(root, freshLockRecord(experimentId, evalId));
|
|
3925
|
+
|
|
3926
|
+
let calls = 0;
|
|
3927
|
+
const evalDef = makeEval(evalId, () => {
|
|
3928
|
+
calls += 1;
|
|
3929
|
+
});
|
|
3930
|
+
const agentRun = probeRun(makeAgent("agent-elsewhere-interrupt"), experimentId, [evalId], { runs: 2 });
|
|
3931
|
+
const plan: RunFeedbackPlan = {
|
|
3932
|
+
shape: { evals: 1, configs: 1, totalAttempts: 2, maxConcurrency: 2 },
|
|
3933
|
+
reused: 0,
|
|
3934
|
+
reusedFailures: [],
|
|
3935
|
+
};
|
|
3936
|
+
|
|
3937
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3938
|
+
const ac = new AbortController();
|
|
3939
|
+
const runPromise = runWithPriorResults([evalDef], [agentRun], {
|
|
3940
|
+
priorResults: [],
|
|
3941
|
+
root,
|
|
3942
|
+
maxConcurrency: 2,
|
|
3943
|
+
signal: ac.signal,
|
|
3944
|
+
});
|
|
3945
|
+
|
|
3946
|
+
await waitForRealProgress(() => expect(coordinator.state.elsewhere).toBe(2));
|
|
3947
|
+
ac.abort();
|
|
3948
|
+
await runPromise;
|
|
3949
|
+
|
|
3950
|
+
// 窗口的收尾事件是被中断唤醒后才发的,可能晚于 runEvals 结算一个事件循环轮次。
|
|
3951
|
+
await waitForRealProgress(() => expect(coordinator.state.elsewhere).toBe(0));
|
|
3952
|
+
expectCountIdentity(coordinator.state);
|
|
3953
|
+
expect(coordinator.state.queued).toBe(2); // 两条都退回 queued(中断路径不派发)
|
|
3954
|
+
expect(calls).toBe(0);
|
|
3955
|
+
});
|
|
3956
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
3957
|
+
|
|
3958
|
+
// 接管 / 多开下补的那对瞬时 started+resolved 只报**真正携入**的那几条:runs > 1 时没携入的
|
|
3959
|
+
// 兄弟从没离开 queued,把整组都报一遍会把 queued 扣穿成负数、reused 多记一条。
|
|
3960
|
+
it("接管后重查携带命中一条:只有那一条走 elsewhere → reused,兄弟照常自跑且 queued 不被扣穿", async () => {
|
|
3961
|
+
const root = await makeRoot();
|
|
3962
|
+
const experimentId = "recheck-partial-count-exp";
|
|
3963
|
+
const evalId = "recheck-partial-count-eval";
|
|
3964
|
+
const agent = makeAgent("agent-recheck-partial-count");
|
|
3965
|
+
const sandbox = fakeSandboxSpec();
|
|
3966
|
+
|
|
3967
|
+
// 先落一条序号 0 的 passed 终态(指纹由生产路径自己算)。
|
|
3968
|
+
const producerRun = probeRun(agent, experimentId, [evalId], { sandbox });
|
|
3969
|
+
const { summary: produced } = await run([makeEval(evalId, () => {})], [producerRun], { root });
|
|
3970
|
+
expect(produced.results[0]!.verdict).toBe("passed");
|
|
3971
|
+
|
|
3972
|
+
// 产出方写完结果还没释放锁就死了:留一把过期锁,续接方接管时重查携带。
|
|
3973
|
+
await seedCaseLock(root, staleLockRecord(experimentId, evalId));
|
|
3974
|
+
|
|
3975
|
+
const { barrier, release } = makeBarrier();
|
|
3976
|
+
const probe = newDispatchProbe();
|
|
3977
|
+
const subjectRun = probeRun(agent, experimentId, [evalId], { sandbox, runs: 2 });
|
|
3978
|
+
const plan: RunFeedbackPlan = {
|
|
3979
|
+
shape: { evals: 1, configs: 1, totalAttempts: 2, maxConcurrency: 2 },
|
|
3980
|
+
reused: 0,
|
|
3981
|
+
reusedFailures: [],
|
|
3982
|
+
};
|
|
3983
|
+
|
|
3984
|
+
await withCoordinator(plan, async (coordinator) => {
|
|
3985
|
+
const runPromise = runWithPriorResults([gatedEval(evalId, barrier, probe)], [subjectRun], {
|
|
3986
|
+
priorResults: [],
|
|
3987
|
+
root,
|
|
3988
|
+
maxConcurrency: 2,
|
|
3989
|
+
});
|
|
3990
|
+
try {
|
|
3991
|
+
// 冻结在「序号 0 已携入、序号 1 正在跑」这一帧上取样 —— 扣穿是瞬时中间态,跑完再看
|
|
3992
|
+
// 计数已经自愈,只有这一帧能证明。
|
|
3993
|
+
await waitForRealProgress(() => {
|
|
3994
|
+
expect(coordinator.state.reused).toBe(1);
|
|
3995
|
+
expect(probe.inFlight).toBe(1);
|
|
3996
|
+
});
|
|
3997
|
+
const mid = coordinator.state;
|
|
3998
|
+
expect(mid.reused).toBe(1); // 只有真正携入的那一条迁进 reused
|
|
3999
|
+
expect(mid.running).toBe(1);
|
|
4000
|
+
expect(mid.queued).toBe(0); // 没携入的兄弟从没离开 queued,不被多扣一次
|
|
4001
|
+
expect(mid.elsewhere).toBe(0);
|
|
4002
|
+
expectCountIdentity(mid);
|
|
4003
|
+
} finally {
|
|
4004
|
+
release();
|
|
4005
|
+
}
|
|
4006
|
+
|
|
4007
|
+
const { summary } = await runPromise;
|
|
4008
|
+
expect(probe.started).toEqual([evalId]); // 只有缺的序号 1 真的派发过
|
|
4009
|
+
expect(summary.results.map((r) => r.attempt).sort()).toEqual([0, 1]);
|
|
4010
|
+
expect(coordinator.state.elsewhere).toBe(0);
|
|
4011
|
+
expectCountIdentity(coordinator.state);
|
|
4012
|
+
expect(await lockFilesRemaining(root)).toEqual([]);
|
|
4013
|
+
});
|
|
4014
|
+
}, SCHEDULING_TEST_TIMEOUT_MS);
|
|
4015
|
+
});
|