dsh-flakefinder 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +50 -0
- package/README.md +101 -0
- package/cordis.patch.yml +19 -0
- package/lib/args.d.ts +21 -0
- package/lib/args.js +81 -0
- package/lib/classify.d.ts +48 -0
- package/lib/classify.js +103 -0
- package/lib/config.d.ts +44 -0
- package/lib/config.js +102 -0
- package/lib/index.d.ts +38 -0
- package/lib/index.js +56 -0
- package/lib/parse.d.ts +27 -0
- package/lib/parse.js +105 -0
- package/lib/runner.d.ts +66 -0
- package/lib/runner.js +98 -0
- package/lib/store.d.ts +58 -0
- package/lib/store.js +123 -0
- package/lib/tools.d.ts +31 -0
- package/lib/tools.js +332 -0
- package/package.json +60 -0
package/README.en.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# dsh-flakefinder
|
|
2
|
+
|
|
3
|
+
> Teach agents the difference between broken code and a flaky test.
|
|
4
|
+
|
|
5
|
+
A DeepSeek Harness plugin for test stability: run tests repeatedly, classify flaky cases, keep history, maintain a quarantine manifest, and gate writes with approvals. Zero runtime dependencies.
|
|
6
|
+
|
|
7
|
+
 
|
|
8
|
+
|
|
9
|
+
[](https://awesome-dsh-plugin.com)
|
|
10
|
+
|
|
11
|
+
## Tools
|
|
12
|
+
|
|
13
|
+
| Tool | Purpose | Write |
|
|
14
|
+
| :-- | :-- | :-- |
|
|
15
|
+
| `flaky_detect` | Repeat tests N times and classify stable-pass / stable-fail / flaky | history only |
|
|
16
|
+
| `flaky_history` | Query detection history, filter by target | no |
|
|
17
|
+
| `flaky_report` | Combine quarantine manifest and history into a report | no |
|
|
18
|
+
| `flaky_quarantine` | Write flaky cases to `.flakefinder.json` | yes (approval) |
|
|
19
|
+
| `flaky_clear` | Remove recovered cases from the manifest | yes (approval) |
|
|
20
|
+
|
|
21
|
+
Frameworks: vitest / jest / node:test, auto-detected in that order.
|
|
22
|
+
|
|
23
|
+
## Install
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
dsh plugin --profile web add dsh-flakefinder
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Example
|
|
30
|
+
|
|
31
|
+
```text
|
|
32
|
+
User: src/checkout.test.ts keeps failing recently, is it real?
|
|
33
|
+
Agent:
|
|
34
|
+
flaky_detect(target="src/checkout.test.ts", runs=5)
|
|
35
|
+
-> verdict: flaky (3/5 passed; failures cluster on fake timers)
|
|
36
|
+
-> flaky_quarantine(tests=["src/checkout.test.ts > timer restore"], reason="timer race")
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Config
|
|
40
|
+
|
|
41
|
+
See `cordis.patch.yml`; defaults: `defaultRuns=5`, `maxRuns=20`, `timeoutMs=120000`, `writeApproval=true`.
|
|
42
|
+
|
|
43
|
+
## Development
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pnpm test
|
|
47
|
+
pnpm typecheck
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
MIT
|
package/README.md
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# dsh-flakefinder
|
|
2
|
+
|
|
3
|
+
> 让 agent 分清「代码坏了」还是「测试在抽风」。
|
|
4
|
+
|
|
5
|
+
DeepSeek Harness 测试稳定性插件:重复运行测试并识别 flaky 用例,历史留档、隔离清单、写操作审批门。零运行时依赖。
|
|
6
|
+
|
|
7
|
+
 
|
|
8
|
+
|
|
9
|
+
[](https://awesome-dsh-plugin.com)
|
|
10
|
+
|
|
11
|
+
## 工具
|
|
12
|
+
|
|
13
|
+
| 工具 | 作用 | 写操作 |
|
|
14
|
+
| :-- | :-- | :-- |
|
|
15
|
+
| `flaky_detect` | 重复运行测试 N 次,判定 stable-pass / stable-fail / flaky | 否(写历史) |
|
|
16
|
+
| `flaky_history` | 查询检测历史,按目标过滤 | 否 |
|
|
17
|
+
| `flaky_report` | 汇总隔离清单 + 历史,输出稳定性报告 | 否 |
|
|
18
|
+
| `flaky_quarantine` | 把 flaky 用例写入 `.flakefinder.json` 隔离清单 | 是(审批门) |
|
|
19
|
+
| `flaky_clear` | 从隔离清单移除已恢复的用例 | 是(审批门) |
|
|
20
|
+
|
|
21
|
+
支持框架:
|
|
22
|
+
|
|
23
|
+
- vitest:读取 `node_modules/vitest/vitest.mjs`,使用 JSON reporter
|
|
24
|
+
- jest:读取 `node_modules/jest/bin/jest.js`,使用 `--json --outputFile`
|
|
25
|
+
- node:test:`node --test --test-reporter=tap`,解析 TAP
|
|
26
|
+
- `framework: auto` 按 vitest → jest → node:test 自动探测
|
|
27
|
+
|
|
28
|
+
## 安装
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
dsh plugin --profile web add dsh-flakefinder
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
或手动安装后在 profile 的 `cordis.patch.yml` 插入:
|
|
35
|
+
|
|
36
|
+
```yaml
|
|
37
|
+
- id: flakefinder
|
|
38
|
+
name: 'dsh-flakefinder'
|
|
39
|
+
config:
|
|
40
|
+
defaultRuns: 5
|
|
41
|
+
writeApproval: true
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## 使用示例
|
|
45
|
+
|
|
46
|
+
```text
|
|
47
|
+
用户:src/checkout.test.ts 最近老失败,帮我判断一下
|
|
48
|
+
Agent:
|
|
49
|
+
flaky_detect(target="src/checkout.test.ts", runs=5)
|
|
50
|
+
→ 判定:flaky;3/5 通过,失败集中在 useFakeTimers 用例
|
|
51
|
+
→ flaky_quarantine(tests=["src/checkout.test.ts > 定时器恢复"], reason="定时器竞态")
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## 配置
|
|
55
|
+
|
|
56
|
+
| 字段 | 默认 | 说明 |
|
|
57
|
+
| :-- | :-- | :-- |
|
|
58
|
+
| `defaultRuns` | `5` | `flaky_detect` 默认重复次数(3-20) |
|
|
59
|
+
| `maxRuns` | `20` | 单次允许的最大重复次数(3-50) |
|
|
60
|
+
| `timeoutMs` | `120000` | 单轮测试运行超时 |
|
|
61
|
+
| `graceMs` | `10000` | 超时后宽限 |
|
|
62
|
+
| `writeApproval` | `true` | 隔离清单写操作是否走审批门 |
|
|
63
|
+
| `dataDir` | `DSH_HOME/.dsh-flakefinder` | 历史存储目录 |
|
|
64
|
+
| `quarantineFile` | `<cwd>/.flakefinder.json` | 隔离清单路径 |
|
|
65
|
+
|
|
66
|
+
## 隔离清单格式
|
|
67
|
+
|
|
68
|
+
```json
|
|
69
|
+
{
|
|
70
|
+
"version": 1,
|
|
71
|
+
"quarantined": [
|
|
72
|
+
{
|
|
73
|
+
"file": "src/checkout.test.ts",
|
|
74
|
+
"name": "使用假定时器后恢复真实定时器",
|
|
75
|
+
"reason": "定时器竞态,见 issue #12",
|
|
76
|
+
"since": "2026-08-16T00:00:00.000Z"
|
|
77
|
+
}
|
|
78
|
+
]
|
|
79
|
+
}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
隔离清单只记录,不修改测试源码;agent 跑测试前应先查阅 `flaky_report`。
|
|
83
|
+
|
|
84
|
+
## 工程
|
|
85
|
+
|
|
86
|
+
- Node >= 22.13,TypeScript,零运行时依赖
|
|
87
|
+
- 测试进程走 DSH 官方 subprocess 服务,argv 数组、无 shell
|
|
88
|
+
- 全量单测 30+:解析、判定、存储、审批门、注册与 manifest
|
|
89
|
+
- `pnpm test`:构建 + `node --test`
|
|
90
|
+
|
|
91
|
+
## 发布门禁
|
|
92
|
+
|
|
93
|
+
1. `pnpm test` 全绿
|
|
94
|
+
2. `pnpm typecheck`
|
|
95
|
+
3. 危险模式扫描(eval / child_process / 密钥零容忍)
|
|
96
|
+
4. 真实 profile 冒烟:全新临时 profile 安装插件 → 真 boot → stderr 零报错
|
|
97
|
+
5. manifest 自检(bundle patch / exports / files 白名单)
|
|
98
|
+
|
|
99
|
+
## License
|
|
100
|
+
|
|
101
|
+
MIT
|
package/cordis.patch.yml
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# dsh-flakefinder 组合包补丁:把测试稳定性插件行插入 profile。
|
|
2
|
+
# 用户 profile 的 cordis.patch.yml 可按 id 覆盖本行整个 config。
|
|
3
|
+
#
|
|
4
|
+
# 示例:
|
|
5
|
+
# - id: flakefinder
|
|
6
|
+
# name: 'dsh-flakefinder'
|
|
7
|
+
# config:
|
|
8
|
+
# defaultRuns: 5 # flaky_detect 默认重复次数(3-20)
|
|
9
|
+
# timeoutMs: 120000 # 单次测试运行超时(毫秒)
|
|
10
|
+
# writeApproval: true # quarantine/clear 是否走审批门(默认 true)
|
|
11
|
+
# dataDir: '' # 历史存储目录(默认 DSH_HOME/.dsh-flakefinder)
|
|
12
|
+
# quarantineFile: '' # 隔离清单(默认项目根 .flakefinder.json)
|
|
13
|
+
#
|
|
14
|
+
- insert:
|
|
15
|
+
- id: flakefinder
|
|
16
|
+
name: 'dsh-flakefinder'
|
|
17
|
+
config:
|
|
18
|
+
defaultRuns: 5
|
|
19
|
+
writeApproval: true
|
package/lib/args.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 测试框架探测与 argv 构建:直接调用本地 node_modules 里的 CLI 入口,
|
|
3
|
+
* 全程 argv 数组、无 shell,避免 npx 下载与命令注入。
|
|
4
|
+
*
|
|
5
|
+
* @module dsh-flakefinder/args
|
|
6
|
+
*/
|
|
7
|
+
export type Framework = 'auto' | 'vitest' | 'jest' | 'node';
|
|
8
|
+
export type ReportKind = 'json' | 'tap';
|
|
9
|
+
export interface TestPlan {
|
|
10
|
+
framework: Exclude<Framework, 'auto'>;
|
|
11
|
+
reportKind: ReportKind;
|
|
12
|
+
reportPath: string;
|
|
13
|
+
argv: string[];
|
|
14
|
+
label: string;
|
|
15
|
+
}
|
|
16
|
+
/** 解析 framework 参数;非法值抛中文错误。 */
|
|
17
|
+
export declare function readFramework(raw: unknown): Framework;
|
|
18
|
+
/** 在 cwd 里探测本地框架 CLI 入口;显式指定时缺失会给出中文指引。 */
|
|
19
|
+
export declare function detectFramework(cwd: string, requested: Framework): Exclude<Framework, 'auto'>;
|
|
20
|
+
/** 构建一次测试运行的执行计划。runIndex 仅用于给临时报告文件命名。 */
|
|
21
|
+
export declare function buildPlan(cwd: string, target: string, requested: Framework, runIndex: number): TestPlan;
|
package/lib/args.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 测试框架探测与 argv 构建:直接调用本地 node_modules 里的 CLI 入口,
|
|
3
|
+
* 全程 argv 数组、无 shell,避免 npx 下载与命令注入。
|
|
4
|
+
*
|
|
5
|
+
* @module dsh-flakefinder/args
|
|
6
|
+
*/
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import os from 'node:os';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import crypto from 'node:crypto';
|
|
11
|
+
import { assertTarget } from './config.js';
|
|
12
|
+
/** 解析 framework 参数;非法值抛中文错误。 */
|
|
13
|
+
export function readFramework(raw) {
|
|
14
|
+
if (raw === undefined || raw === null)
|
|
15
|
+
return 'auto';
|
|
16
|
+
if (raw === 'auto' || raw === 'vitest' || raw === 'jest' || raw === 'node')
|
|
17
|
+
return raw;
|
|
18
|
+
throw new Error('framework 只支持 auto / vitest / jest / node,收到:' + String(raw));
|
|
19
|
+
}
|
|
20
|
+
function localEntry(cwd, ...parts) {
|
|
21
|
+
const full = path.join(cwd, ...parts);
|
|
22
|
+
return fs.existsSync(full) ? full : null;
|
|
23
|
+
}
|
|
24
|
+
/** 在 cwd 里探测本地框架 CLI 入口;显式指定时缺失会给出中文指引。 */
|
|
25
|
+
export function detectFramework(cwd, requested) {
|
|
26
|
+
const vitest = localEntry(cwd, 'node_modules', 'vitest', 'vitest.mjs');
|
|
27
|
+
const jest = localEntry(cwd, 'node_modules', 'jest', 'bin', 'jest.js');
|
|
28
|
+
if (requested !== 'auto') {
|
|
29
|
+
if (requested === 'vitest' && vitest === null) {
|
|
30
|
+
throw new Error('framework=vitest,但在 ' + cwd + ' 下找不到 node_modules/vitest/vitest.mjs。请先安装 vitest(例如 pnpm add -D vitest)。');
|
|
31
|
+
}
|
|
32
|
+
if (requested === 'jest' && jest === null) {
|
|
33
|
+
throw new Error('framework=jest,但在 ' + cwd + ' 下找不到 node_modules/jest/bin/jest.js。请先安装 jest。');
|
|
34
|
+
}
|
|
35
|
+
return requested;
|
|
36
|
+
}
|
|
37
|
+
if (vitest !== null)
|
|
38
|
+
return 'vitest';
|
|
39
|
+
if (jest !== null)
|
|
40
|
+
return 'jest';
|
|
41
|
+
return 'node';
|
|
42
|
+
}
|
|
43
|
+
/** 构建一次测试运行的执行计划。runIndex 仅用于给临时报告文件命名。 */
|
|
44
|
+
export function buildPlan(cwd, target, requested, runIndex) {
|
|
45
|
+
const safeTarget = assertTarget(target);
|
|
46
|
+
const framework = detectFramework(cwd, requested);
|
|
47
|
+
const token = crypto.randomBytes(6).toString('hex');
|
|
48
|
+
if (framework === 'vitest') {
|
|
49
|
+
const entry = localEntry(cwd, 'node_modules', 'vitest', 'vitest.mjs');
|
|
50
|
+
if (entry === null)
|
|
51
|
+
throw new Error('未找到本地 vitest 入口:node_modules/vitest/vitest.mjs');
|
|
52
|
+
const reportPath = path.join(os.tmpdir(), 'dsh-flakefinder-vitest-' + process.pid + '-' + String(runIndex) + '-' + token + '.json');
|
|
53
|
+
return {
|
|
54
|
+
framework,
|
|
55
|
+
reportKind: 'json',
|
|
56
|
+
reportPath,
|
|
57
|
+
argv: [process.execPath, entry, 'run', safeTarget, '--reporter=json', '--outputFile', reportPath],
|
|
58
|
+
label: 'vitest run ' + safeTarget,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
if (framework === 'jest') {
|
|
62
|
+
const entry = localEntry(cwd, 'node_modules', 'jest', 'bin', 'jest.js');
|
|
63
|
+
if (entry === null)
|
|
64
|
+
throw new Error('未找到本地 jest 入口:node_modules/jest/bin/jest.js');
|
|
65
|
+
const reportPath = path.join(os.tmpdir(), 'dsh-flakefinder-jest-' + process.pid + '-' + String(runIndex) + '-' + token + '.json');
|
|
66
|
+
return {
|
|
67
|
+
framework,
|
|
68
|
+
reportKind: 'json',
|
|
69
|
+
reportPath,
|
|
70
|
+
argv: [process.execPath, entry, '--runInBand', '--json', '--outputFile', reportPath, safeTarget],
|
|
71
|
+
label: 'jest run ' + safeTarget,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
framework: 'node',
|
|
76
|
+
reportKind: 'tap',
|
|
77
|
+
reportPath: '',
|
|
78
|
+
argv: [process.execPath, '--test', '--test-reporter=tap', safeTarget],
|
|
79
|
+
label: 'node --test ' + safeTarget,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 多轮结果聚合与 flaky 判定:纯函数,便于单测边界。
|
|
3
|
+
*
|
|
4
|
+
* @module dsh-flakefinder/classify
|
|
5
|
+
*/
|
|
6
|
+
import type { NormalizedRun } from './parse.js';
|
|
7
|
+
export type RunVerdict = 'stable-pass' | 'stable-fail' | 'flaky' | 'empty' | 'skipped';
|
|
8
|
+
export type TestVerdict = 'stable-pass' | 'stable-fail' | 'flaky' | 'skipped';
|
|
9
|
+
export interface RunRecord {
|
|
10
|
+
runIndex: number;
|
|
11
|
+
success: boolean;
|
|
12
|
+
passed: number;
|
|
13
|
+
failed: number;
|
|
14
|
+
skipped: number;
|
|
15
|
+
durationMs: number;
|
|
16
|
+
}
|
|
17
|
+
export interface TestRecord {
|
|
18
|
+
id: string;
|
|
19
|
+
file: string;
|
|
20
|
+
name: string;
|
|
21
|
+
passes: number;
|
|
22
|
+
failures: number;
|
|
23
|
+
skips: number;
|
|
24
|
+
failureRate: number;
|
|
25
|
+
verdict: TestVerdict;
|
|
26
|
+
}
|
|
27
|
+
export interface DetectionResult {
|
|
28
|
+
runs: RunRecord[];
|
|
29
|
+
verdict: RunVerdict;
|
|
30
|
+
stablePassCount: number;
|
|
31
|
+
stableFailCount: number;
|
|
32
|
+
flakyCount: number;
|
|
33
|
+
skippedCount: number;
|
|
34
|
+
tests: TestRecord[];
|
|
35
|
+
durationMs: number;
|
|
36
|
+
}
|
|
37
|
+
export declare function aggregate(runs: Array<{
|
|
38
|
+
index: number;
|
|
39
|
+
report: NormalizedRun | null;
|
|
40
|
+
durationMs: number;
|
|
41
|
+
error?: string;
|
|
42
|
+
}>): DetectionResult;
|
|
43
|
+
/** 结果里需要写入历史的 flaky 用例。 */
|
|
44
|
+
export declare function flakyTests(result: DetectionResult): Array<{
|
|
45
|
+
file: string;
|
|
46
|
+
name: string;
|
|
47
|
+
failureRate: number;
|
|
48
|
+
}>;
|
package/lib/classify.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 多轮结果聚合与 flaky 判定:纯函数,便于单测边界。
|
|
3
|
+
*
|
|
4
|
+
* @module dsh-flakefinder/classify
|
|
5
|
+
*/
|
|
6
|
+
export function aggregate(runs) {
|
|
7
|
+
const records = runs.map(run => ({
|
|
8
|
+
runIndex: run.index,
|
|
9
|
+
success: run.report?.success === true,
|
|
10
|
+
passed: run.report?.passed ?? 0,
|
|
11
|
+
failed: run.report?.failed ?? 0,
|
|
12
|
+
skipped: run.report?.skipped ?? 0,
|
|
13
|
+
durationMs: run.durationMs,
|
|
14
|
+
}));
|
|
15
|
+
const active = runs.filter(run => run.report !== null);
|
|
16
|
+
if (active.length === 0) {
|
|
17
|
+
const detail = runs.find(run => run.error !== undefined);
|
|
18
|
+
throw new Error('没有一次运行产出可解析的测试报告。' + (detail?.error !== undefined ? ' 首个错误:' + detail.error : ''));
|
|
19
|
+
}
|
|
20
|
+
const byId = new Map();
|
|
21
|
+
for (const run of active) {
|
|
22
|
+
const tests = run.report?.tests ?? [];
|
|
23
|
+
for (const test of tests) {
|
|
24
|
+
let entry = byId.get(test.id);
|
|
25
|
+
if (entry === undefined) {
|
|
26
|
+
entry = { file: test.file, name: test.name, passes: 0, failures: 0, skips: 0 };
|
|
27
|
+
byId.set(test.id, entry);
|
|
28
|
+
}
|
|
29
|
+
if (test.status === 'passed')
|
|
30
|
+
entry.passes += 1;
|
|
31
|
+
else if (test.status === 'failed')
|
|
32
|
+
entry.failures += 1;
|
|
33
|
+
else
|
|
34
|
+
entry.skips += 1;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
const tests = [];
|
|
38
|
+
let stablePassCount = 0;
|
|
39
|
+
let stableFailCount = 0;
|
|
40
|
+
let flakyCount = 0;
|
|
41
|
+
let skippedCount = 0;
|
|
42
|
+
for (const [id, entry] of byId) {
|
|
43
|
+
const seen = entry.passes + entry.failures;
|
|
44
|
+
let verdict;
|
|
45
|
+
if (seen === 0) {
|
|
46
|
+
verdict = 'skipped';
|
|
47
|
+
skippedCount += 1;
|
|
48
|
+
}
|
|
49
|
+
else if (entry.failures === 0) {
|
|
50
|
+
verdict = 'stable-pass';
|
|
51
|
+
stablePassCount += 1;
|
|
52
|
+
}
|
|
53
|
+
else if (entry.passes === 0) {
|
|
54
|
+
verdict = 'stable-fail';
|
|
55
|
+
stableFailCount += 1;
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
verdict = 'flaky';
|
|
59
|
+
flakyCount += 1;
|
|
60
|
+
}
|
|
61
|
+
tests.push({
|
|
62
|
+
id,
|
|
63
|
+
file: entry.file,
|
|
64
|
+
name: entry.name,
|
|
65
|
+
passes: entry.passes,
|
|
66
|
+
failures: entry.failures,
|
|
67
|
+
skips: entry.skips,
|
|
68
|
+
failureRate: seen === 0 ? 0 : Math.round((entry.failures / seen) * 1000) / 10,
|
|
69
|
+
verdict,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
let verdict;
|
|
73
|
+
const passRuns = records.filter(run => run.success).length;
|
|
74
|
+
if (tests.every(test => test.verdict === 'skipped')) {
|
|
75
|
+
verdict = 'skipped';
|
|
76
|
+
}
|
|
77
|
+
else if (passRuns === records.length) {
|
|
78
|
+
verdict = 'stable-pass';
|
|
79
|
+
}
|
|
80
|
+
else if (passRuns === 0) {
|
|
81
|
+
verdict = 'stable-fail';
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
verdict = 'flaky';
|
|
85
|
+
}
|
|
86
|
+
tests.sort((a, b) => a.id.localeCompare(b.id));
|
|
87
|
+
return {
|
|
88
|
+
runs: records,
|
|
89
|
+
verdict,
|
|
90
|
+
stablePassCount,
|
|
91
|
+
stableFailCount,
|
|
92
|
+
flakyCount,
|
|
93
|
+
skippedCount,
|
|
94
|
+
tests,
|
|
95
|
+
durationMs: records.reduce((sum, item) => sum + item.durationMs, 0),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
/** 结果里需要写入历史的 flaky 用例。 */
|
|
99
|
+
export function flakyTests(result) {
|
|
100
|
+
return result.tests
|
|
101
|
+
.filter(test => test.verdict === 'flaky')
|
|
102
|
+
.map(test => ({ file: test.file, name: test.name, failureRate: test.failureRate }));
|
|
103
|
+
}
|
package/lib/config.d.ts
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-flakefinder 配置解析:重复次数、超时、审批策略与存储路径。
|
|
3
|
+
*
|
|
4
|
+
* @module dsh-flakefinder/config
|
|
5
|
+
*/
|
|
6
|
+
/** 插件行配置(cordis.patch.yml 里的 config 段,可缺省)。 */
|
|
7
|
+
export interface FlakeConfig {
|
|
8
|
+
/** flaky_detect 默认重复次数(3-20)。 */
|
|
9
|
+
defaultRuns?: number;
|
|
10
|
+
/** 单次运行允许的最大重复次数(3-50,默认 20)。 */
|
|
11
|
+
maxRuns?: number;
|
|
12
|
+
/** 单次测试运行的超时(毫秒,默认 120000)。 */
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
/** 超时后给测试进程的宽限时间(毫秒,默认 10000)。 */
|
|
15
|
+
graceMs?: number;
|
|
16
|
+
/** quarantine/clear 两个写工具是否走宿主审批门(默认 true)。 */
|
|
17
|
+
writeApproval?: boolean;
|
|
18
|
+
/** 历史存储目录(默认 DSH_HOME/.dsh-flakefinder)。 */
|
|
19
|
+
dataDir?: string;
|
|
20
|
+
/** 项目隔离清单路径(默认 cwd/.flakefinder.json)。 */
|
|
21
|
+
quarantineFile?: string;
|
|
22
|
+
}
|
|
23
|
+
/** 解析后的配置。 */
|
|
24
|
+
export interface ResolvedFlakeConfig {
|
|
25
|
+
defaultRuns: number;
|
|
26
|
+
maxRuns: number;
|
|
27
|
+
timeoutMs: number;
|
|
28
|
+
graceMs: number;
|
|
29
|
+
writeApproval: boolean;
|
|
30
|
+
dataDir: string;
|
|
31
|
+
quarantineFile: string;
|
|
32
|
+
}
|
|
33
|
+
/** 解析并校验配置,非法值抛出中文错误。 */
|
|
34
|
+
export declare function resolveConfig(config: FlakeConfig | undefined | null, cwd?: string): ResolvedFlakeConfig;
|
|
35
|
+
/** 解析并钳制 flaky_detect 的 runs 参数。 */
|
|
36
|
+
export declare function resolveRuns(raw: unknown, cfg: ResolvedFlakeConfig): number;
|
|
37
|
+
/** 校验测试目标:非空且不能以 - 开头,避免被测试框架解析成选项。 */
|
|
38
|
+
export declare function assertTarget(value: string): string;
|
|
39
|
+
/** 从参数里取字符串;不识别类型时返回 undefined。 */
|
|
40
|
+
export declare function optionalString(args: Record<string, unknown>, key: string): string | undefined;
|
|
41
|
+
/** 从参数里取必填字符串;缺失时抛中文错误。 */
|
|
42
|
+
export declare function requiredString(args: Record<string, unknown>, key: string, label: string): string;
|
|
43
|
+
/** 从参数里取整数;缺失用默认值,非法抛中文错误。 */
|
|
44
|
+
export declare function optionalInteger(args: Record<string, unknown>, key: string, label: string, lo: number, hi: number, fallback: number): number;
|
package/lib/config.js
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-flakefinder 配置解析:重复次数、超时、审批策略与存储路径。
|
|
3
|
+
*
|
|
4
|
+
* @module dsh-flakefinder/config
|
|
5
|
+
*/
|
|
6
|
+
import { homedir } from 'node:os';
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
const MIN_RUNS = 3;
|
|
9
|
+
const MAX_RUNS = 50;
|
|
10
|
+
/** DSH_HOME 缺省时退回用户目录下的 .dsh。 */
|
|
11
|
+
function defaultDataDir() {
|
|
12
|
+
const home = process.env.DSH_HOME?.trim();
|
|
13
|
+
return path.join(home !== undefined && home !== '' ? home : path.join(homedir(), '.dsh'), '.dsh-flakefinder');
|
|
14
|
+
}
|
|
15
|
+
/** 解析并校验配置,非法值抛出中文错误。 */
|
|
16
|
+
export function resolveConfig(config, cwd = process.cwd()) {
|
|
17
|
+
const cfg = config ?? {};
|
|
18
|
+
let defaultRuns = 5;
|
|
19
|
+
if (cfg.defaultRuns !== undefined) {
|
|
20
|
+
defaultRuns = readRuns('defaultRuns', cfg.defaultRuns, 3, 20);
|
|
21
|
+
}
|
|
22
|
+
let maxRuns = 20;
|
|
23
|
+
if (cfg.maxRuns !== undefined) {
|
|
24
|
+
maxRuns = readRuns('maxRuns', cfg.maxRuns, MIN_RUNS, MAX_RUNS);
|
|
25
|
+
}
|
|
26
|
+
if (defaultRuns > maxRuns)
|
|
27
|
+
defaultRuns = maxRuns;
|
|
28
|
+
let timeoutMs = 120000;
|
|
29
|
+
if (cfg.timeoutMs !== undefined) {
|
|
30
|
+
if (typeof cfg.timeoutMs !== 'number' || !Number.isFinite(cfg.timeoutMs) || cfg.timeoutMs <= 0) {
|
|
31
|
+
throw new Error('timeoutMs 必须是大于 0 的数字(毫秒)。');
|
|
32
|
+
}
|
|
33
|
+
timeoutMs = Math.min(10 * 60 * 1000, Math.max(10000, Math.round(cfg.timeoutMs)));
|
|
34
|
+
}
|
|
35
|
+
let graceMs = 10000;
|
|
36
|
+
if (cfg.graceMs !== undefined) {
|
|
37
|
+
if (typeof cfg.graceMs !== 'number' || !Number.isFinite(cfg.graceMs) || cfg.graceMs <= 0) {
|
|
38
|
+
throw new Error('graceMs 必须是大于 0 的数字(毫秒)。');
|
|
39
|
+
}
|
|
40
|
+
graceMs = Math.min(120000, Math.max(1000, Math.round(cfg.graceMs)));
|
|
41
|
+
}
|
|
42
|
+
const writeApproval = cfg.writeApproval !== false;
|
|
43
|
+
const dataDir = typeof cfg.dataDir === 'string' && cfg.dataDir.trim() !== ''
|
|
44
|
+
? path.resolve(cwd, cfg.dataDir.trim())
|
|
45
|
+
: defaultDataDir();
|
|
46
|
+
const quarantineFile = typeof cfg.quarantineFile === 'string' && cfg.quarantineFile.trim() !== ''
|
|
47
|
+
? path.resolve(cwd, cfg.quarantineFile.trim())
|
|
48
|
+
: path.join(cwd, '.flakefinder.json');
|
|
49
|
+
return { defaultRuns, maxRuns, timeoutMs, graceMs, writeApproval, dataDir, quarantineFile };
|
|
50
|
+
}
|
|
51
|
+
function readRuns(label, value, lo, hi) {
|
|
52
|
+
if (typeof value !== 'number' || !Number.isFinite(value))
|
|
53
|
+
throw new Error(label + ' 必须是数字。');
|
|
54
|
+
const rounded = Math.round(value);
|
|
55
|
+
if (rounded < lo || rounded > hi)
|
|
56
|
+
throw new Error(label + ' 必须在 ' + String(lo) + '-' + String(hi) + ' 之间。');
|
|
57
|
+
return rounded;
|
|
58
|
+
}
|
|
59
|
+
/** 解析并钳制 flaky_detect 的 runs 参数。 */
|
|
60
|
+
export function resolveRuns(raw, cfg) {
|
|
61
|
+
if (raw === undefined || raw === null)
|
|
62
|
+
return cfg.defaultRuns;
|
|
63
|
+
if (typeof raw !== 'number' || !Number.isFinite(raw))
|
|
64
|
+
throw new Error('runs 必须是数字(3-' + String(cfg.maxRuns) + ')。');
|
|
65
|
+
const runs = Math.round(raw);
|
|
66
|
+
if (runs < 3 || runs > cfg.maxRuns)
|
|
67
|
+
throw new Error('runs 必须在 3-' + String(cfg.maxRuns) + ' 之间。');
|
|
68
|
+
return runs;
|
|
69
|
+
}
|
|
70
|
+
/** 校验测试目标:非空且不能以 - 开头,避免被测试框架解析成选项。 */
|
|
71
|
+
export function assertTarget(value) {
|
|
72
|
+
const trimmed = value.trim();
|
|
73
|
+
if (trimmed === '')
|
|
74
|
+
throw new Error('target(测试文件/目录/glob)为必填,请提供非空字符串。');
|
|
75
|
+
if (trimmed.startsWith('-'))
|
|
76
|
+
throw new Error('target 不能以 - 开头:' + value);
|
|
77
|
+
return trimmed;
|
|
78
|
+
}
|
|
79
|
+
/** 从参数里取字符串;不识别类型时返回 undefined。 */
|
|
80
|
+
export function optionalString(args, key) {
|
|
81
|
+
const value = args[key];
|
|
82
|
+
return typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined;
|
|
83
|
+
}
|
|
84
|
+
/** 从参数里取必填字符串;缺失时抛中文错误。 */
|
|
85
|
+
export function requiredString(args, key, label) {
|
|
86
|
+
const value = optionalString(args, key);
|
|
87
|
+
if (value === undefined)
|
|
88
|
+
throw new Error(label + '(参数 ' + key + ')为必填,请提供非空字符串。');
|
|
89
|
+
return value;
|
|
90
|
+
}
|
|
91
|
+
/** 从参数里取整数;缺失用默认值,非法抛中文错误。 */
|
|
92
|
+
export function optionalInteger(args, key, label, lo, hi, fallback) {
|
|
93
|
+
const value = args[key];
|
|
94
|
+
if (value === undefined || value === null)
|
|
95
|
+
return fallback;
|
|
96
|
+
if (typeof value !== 'number' || !Number.isFinite(value))
|
|
97
|
+
throw new Error(label + '(参数 ' + key + ')必须是数字。');
|
|
98
|
+
const rounded = Math.round(value);
|
|
99
|
+
if (rounded < lo || rounded > hi)
|
|
100
|
+
throw new Error(label + '(参数 ' + key + ')必须在 ' + String(lo) + '-' + String(hi) + ' 之间。');
|
|
101
|
+
return rounded;
|
|
102
|
+
}
|
package/lib/index.d.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-flakefinder —— 测试稳定性工具插件(node 半身,配置走 cordis.patch.yml)。
|
|
3
|
+
*
|
|
4
|
+
* 插件导出 apply(ctx, config):注册五个面向模型的工具(flaky_detect / flaky_history /
|
|
5
|
+
* flaky_quarantine / flaky_clear / flaky_report)。测试进程走 DSH 官方 subprocess 服务
|
|
6
|
+
* (argv 数组、无 shell),隔离清单写操作默认走宿主审批门。零运行时依赖。
|
|
7
|
+
*
|
|
8
|
+
* @module dsh-flakefinder
|
|
9
|
+
*/
|
|
10
|
+
import { type FlakeConfig } from './config.js';
|
|
11
|
+
import { type SubprocessSpawnLike } from './runner.js';
|
|
12
|
+
import { type FlakeToolDefinition } from './tools.js';
|
|
13
|
+
/** cordis 服务注入:apply 里要用 ctx.subprocess 与 ctx.tools。 */
|
|
14
|
+
export declare const inject: string[];
|
|
15
|
+
/** 插件所需的最小 ctx 面。 */
|
|
16
|
+
export interface FlakePluginContext {
|
|
17
|
+
subprocess: {
|
|
18
|
+
spawn: SubprocessSpawnLike;
|
|
19
|
+
};
|
|
20
|
+
tools: {
|
|
21
|
+
register(definition: FlakeToolDefinition, options?: {
|
|
22
|
+
prepend?: boolean;
|
|
23
|
+
}): () => void;
|
|
24
|
+
};
|
|
25
|
+
get?(name: 'approval'): unknown;
|
|
26
|
+
on?(event: string, listener: () => void): () => void;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* 插件入口:解析配置、封装执行器与存储、注册五个工具。
|
|
30
|
+
*/
|
|
31
|
+
export declare function apply(ctx: FlakePluginContext, config?: FlakeConfig | null): void;
|
|
32
|
+
export * from './args.js';
|
|
33
|
+
export * from './classify.js';
|
|
34
|
+
export * from './config.js';
|
|
35
|
+
export * from './parse.js';
|
|
36
|
+
export * from './runner.js';
|
|
37
|
+
export * from './store.js';
|
|
38
|
+
export * from './tools.js';
|
package/lib/index.js
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-flakefinder —— 测试稳定性工具插件(node 半身,配置走 cordis.patch.yml)。
|
|
3
|
+
*
|
|
4
|
+
* 插件导出 apply(ctx, config):注册五个面向模型的工具(flaky_detect / flaky_history /
|
|
5
|
+
* flaky_quarantine / flaky_clear / flaky_report)。测试进程走 DSH 官方 subprocess 服务
|
|
6
|
+
* (argv 数组、无 shell),隔离清单写操作默认走宿主审批门。零运行时依赖。
|
|
7
|
+
*
|
|
8
|
+
* @module dsh-flakefinder
|
|
9
|
+
*/
|
|
10
|
+
import { resolveConfig } from './config.js';
|
|
11
|
+
import { createSubprocessRunner } from './runner.js';
|
|
12
|
+
import { createStore } from './store.js';
|
|
13
|
+
import { buildFlakeTools } from './tools.js';
|
|
14
|
+
/** cordis 服务注入:apply 里要用 ctx.subprocess 与 ctx.tools。 */
|
|
15
|
+
export const inject = ['subprocess', 'tools'];
|
|
16
|
+
/**
|
|
17
|
+
* 插件入口:解析配置、封装执行器与存储、注册五个工具。
|
|
18
|
+
*/
|
|
19
|
+
export function apply(ctx, config) {
|
|
20
|
+
let cfg;
|
|
21
|
+
try {
|
|
22
|
+
cfg = resolveConfig(config);
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
console.warn('[dsh-flakefinder] ' + (error instanceof Error ? error.message : String(error)));
|
|
26
|
+
cfg = resolveConfig(null);
|
|
27
|
+
}
|
|
28
|
+
const runner = createSubprocessRunner(ctx.subprocess.spawn, cfg.graceMs, cfg.timeoutMs);
|
|
29
|
+
const store = createStore(cfg.dataDir, cfg.quarantineFile);
|
|
30
|
+
const tools = buildFlakeTools(cfg, runner, store);
|
|
31
|
+
const disposers = [];
|
|
32
|
+
for (const definition of tools) {
|
|
33
|
+
const wrapped = { ...definition };
|
|
34
|
+
if (wrapped.gate !== undefined) {
|
|
35
|
+
const original = wrapped.gate.bind(wrapped);
|
|
36
|
+
wrapped.gate = async (exec, next) => {
|
|
37
|
+
const record = (typeof exec === 'object' && exec !== null ? exec : {});
|
|
38
|
+
return original({ ...record, approval: ctx.get?.('approval') }, next);
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
disposers.push(ctx.tools.register(wrapped, { prepend: true }));
|
|
42
|
+
}
|
|
43
|
+
if (typeof ctx.on === 'function') {
|
|
44
|
+
ctx.on('dispose', () => {
|
|
45
|
+
for (const dispose of disposers)
|
|
46
|
+
dispose();
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export * from './args.js';
|
|
51
|
+
export * from './classify.js';
|
|
52
|
+
export * from './config.js';
|
|
53
|
+
export * from './parse.js';
|
|
54
|
+
export * from './runner.js';
|
|
55
|
+
export * from './store.js';
|
|
56
|
+
export * from './tools.js';
|