@diffci.com/diffci 0.1.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -0
- package/action.yml +155 -0
- package/dist-client/src/client/cli.js +281 -0
- package/dist-client/src/client/context.js +173 -0
- package/dist-client/src/client/observe.js +230 -0
- package/dist-client/src/client/report.js +63 -0
- package/dist-client/src/client/submit.js +89 -0
- package/dist-client/src/client/workflow-guard.js +300 -0
- package/dist-client/src/git/git-diff.js +379 -0
- package/dist-client/src/git/types.js +1 -0
- package/dist-client/src/planner/path-baseline.js +131 -0
- package/dist-client/src/planner/test-command.js +124 -0
- package/dist-client/src/planner/types.js +1 -0
- package/dist-client/src/repo/analyzer.js +454 -0
- package/dist-client/src/repo/graph.js +958 -0
- package/dist-client/src/repo/impact-types.js +1 -0
- package/dist-client/src/repo/impact.js +625 -0
- package/dist-client/src/repo/layout.js +75 -0
- package/dist-client/src/repo/repo-config.js +63 -0
- package/dist-client/src/repo/runner-universe.js +253 -0
- package/dist-client/src/repo/test-discovery.js +383 -0
- package/dist-client/src/repo/test-fixture-ownership.js +76 -0
- package/dist-client/src/repo/test-framework.js +117 -0
- package/dist-client/src/repo/types.js +1 -0
- package/docs/distribution.md +81 -0
- package/package.json +118 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test discovery from test-runner configuration (2026-08-23, deepseek-harness benchmark Phase 2).
|
|
3
|
+
*
|
|
4
|
+
* Before this module DiffCI recognised a test file purely by the `.test.` / `.spec.` filename
|
|
5
|
+
* convention, hardcoded in three places (analyzer discovery, graph node flag, impact classification).
|
|
6
|
+
* On deepseek-harness that silently left 22 `*.snapshot.ts` and 136 `*.e2e.ts` suites - each run by
|
|
7
|
+
* its own `vitest run --config vitest.<family>.config.ts` CI job - outside the modelled test universe.
|
|
8
|
+
*
|
|
9
|
+
* This module reads the repository's OWN declaration of what a test is: the `include` globs of
|
|
10
|
+
* Vitest/Jest configuration files at the repository root. It is deliberately STATIC - config files
|
|
11
|
+
* are never imported or executed (they are repo code); string literals are lifted out of
|
|
12
|
+
* `include: [ ... ]` arrays (and `testMatch` for Jest) by a tolerant scanner. Anything it cannot read
|
|
13
|
+
* is simply not added, so the worst case is the pre-existing `.test.`/`.spec.` behaviour.
|
|
14
|
+
*
|
|
15
|
+
* Families are derived from the filename token between the last two dots (`foo.e2e.ts` -> e2e) -
|
|
16
|
+
* a convention, not a deepseek-specific rule - and the config file name (`vitest.e2e.config.ts`).
|
|
17
|
+
* Families matter downstream: a snapshot or e2e file is a real test that can be SELECTED, but it is
|
|
18
|
+
* executed by a different command (and may need credentials/browsers), so execution validation must
|
|
19
|
+
* treat families separately. Nothing here changes selection policy; it only widens what counts as a
|
|
20
|
+
* test. Repositories with no such config, or whose configs only restate `.test.`/`.spec.`, are
|
|
21
|
+
* unaffected.
|
|
22
|
+
*/
|
|
23
|
+
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
24
|
+
import { compileIgnoreRegexes, extractTestPatterns, isIgnoredPath, isUnderRoots } from "./runner-universe.js";
|
|
25
|
+
import { join } from "node:path";
|
|
26
|
+
export const DEFAULT_TEST_PATTERNS = [
|
|
27
|
+
"**/*.test.{ts,tsx,js,jsx,mjs,cjs,mts,cts}",
|
|
28
|
+
"**/*.spec.{ts,tsx,js,jsx,mjs,cjs,mts,cts}",
|
|
29
|
+
];
|
|
30
|
+
const FAMILY_TOKENS = {
|
|
31
|
+
test: "unit", spec: "unit", unit: "unit",
|
|
32
|
+
snapshot: "snapshot", snap: "snapshot",
|
|
33
|
+
e2e: "e2e",
|
|
34
|
+
integration: "integration", int: "integration", it: "integration",
|
|
35
|
+
bench: "benchmark", benchmark: "benchmark", perf: "benchmark",
|
|
36
|
+
};
|
|
37
|
+
/** Family of a test file from its filename token: `name.<token>.<ext>`. Undefined when the file has
|
|
38
|
+
* no recognised token (then it is only a test if a config include says so; treated as "unit"). */
|
|
39
|
+
export function testFamilyOfPath(filePath) {
|
|
40
|
+
const base = filePath.slice(filePath.lastIndexOf("/") + 1);
|
|
41
|
+
const parts = base.split(".");
|
|
42
|
+
if (parts.length < 3)
|
|
43
|
+
return undefined;
|
|
44
|
+
const token = parts[parts.length - 2].toLowerCase();
|
|
45
|
+
return FAMILY_TOKENS[token];
|
|
46
|
+
}
|
|
47
|
+
function familyOfConfigName(file) {
|
|
48
|
+
// vitest.<token>.config.ts / jest.<token>.config.js -> token; plain vitest.config.ts -> undefined
|
|
49
|
+
const m = /^(?:vitest|jest)\.([a-z0-9-]+)\.config\./i.exec(file);
|
|
50
|
+
if (!m)
|
|
51
|
+
return undefined;
|
|
52
|
+
const token = m[1].toLowerCase();
|
|
53
|
+
if (token in FAMILY_TOKENS)
|
|
54
|
+
return FAMILY_TOKENS[token];
|
|
55
|
+
if (token.includes("e2e"))
|
|
56
|
+
return "e2e";
|
|
57
|
+
if (token.includes("snapshot"))
|
|
58
|
+
return "snapshot";
|
|
59
|
+
if (token.includes("integration"))
|
|
60
|
+
return "integration";
|
|
61
|
+
return undefined; // e.g. "web", "web-stress": family comes from each file's token instead
|
|
62
|
+
}
|
|
63
|
+
const CONFIG_NAME = /^(vitest|jest)(\.[a-z0-9-]+)?\.config\.(ts|mts|cts|js|mjs|cjs)$/i;
|
|
64
|
+
/**
|
|
65
|
+
* Lift string literals out of `include: [ ... ]` / `testMatch: [ ... ]` arrays. Tolerates spreads,
|
|
66
|
+
* comments and conditional entries inside the array (their literals are lifted too - over-inclusion
|
|
67
|
+
* only makes MORE files count as tests, never fewer). Also follows one level of indirection:
|
|
68
|
+
* `include: someIdent` where `const someIdent = [ ... ]` is declared in the same file.
|
|
69
|
+
*/
|
|
70
|
+
/** Removes JS comments while leaving string literals untouched - a naive regex would eat the `/**\/`
|
|
71
|
+
* inside a glob like `tests/**\/*.spec.ts`. */
|
|
72
|
+
export function stripComments(source) {
|
|
73
|
+
let out = "";
|
|
74
|
+
let i = 0;
|
|
75
|
+
let quote;
|
|
76
|
+
while (i < source.length) {
|
|
77
|
+
const ch = source[i];
|
|
78
|
+
const next = source[i + 1];
|
|
79
|
+
if (quote) {
|
|
80
|
+
out += ch;
|
|
81
|
+
if (ch === "\\" && next !== undefined) {
|
|
82
|
+
out += next;
|
|
83
|
+
i += 2;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
if (ch === quote)
|
|
87
|
+
quote = undefined;
|
|
88
|
+
i++;
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (ch === "'" || ch === '"' || ch === "`") {
|
|
92
|
+
quote = ch;
|
|
93
|
+
out += ch;
|
|
94
|
+
i++;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (ch === "/" && next === "/") {
|
|
98
|
+
while (i < source.length && source[i] !== "\n")
|
|
99
|
+
i++;
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (ch === "/" && next === "*") {
|
|
103
|
+
const end = source.indexOf("*/", i + 2);
|
|
104
|
+
i = end === -1 ? source.length : end + 2;
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
out += ch;
|
|
108
|
+
i++;
|
|
109
|
+
}
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
/** Blanks `coverage: { ... }` blocks: their `include` lists INSTRUMENTED SOURCE files, not tests, and
|
|
113
|
+
* lifting them would mark every source file as a test (observed on deepseek-harness: +1,382 files). */
|
|
114
|
+
export function blankCoverageBlocks(source) {
|
|
115
|
+
let out = source;
|
|
116
|
+
for (let guard = 0; guard < 32; guard++) {
|
|
117
|
+
const m = /\bcoverage\s*:\s*\{/.exec(out);
|
|
118
|
+
if (!m)
|
|
119
|
+
break;
|
|
120
|
+
const open = m.index + m[0].length - 1;
|
|
121
|
+
let depth = 0;
|
|
122
|
+
let close = -1;
|
|
123
|
+
for (let i = open; i < out.length; i++) {
|
|
124
|
+
if (out[i] === "{")
|
|
125
|
+
depth++;
|
|
126
|
+
else if (out[i] === "}") {
|
|
127
|
+
depth--;
|
|
128
|
+
if (depth === 0) {
|
|
129
|
+
close = i;
|
|
130
|
+
break;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (close === -1)
|
|
135
|
+
break;
|
|
136
|
+
out = out.slice(0, m.index) + " ".repeat(close + 1 - m.index) + out.slice(close + 1);
|
|
137
|
+
}
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
export function extractIncludeGlobs(source) {
|
|
141
|
+
const stripped = blankCoverageBlocks(stripComments(source));
|
|
142
|
+
// Bracket-depth aware: returns the body of the array literal opening at `open` (index of "[").
|
|
143
|
+
const arrayBodyAt = (open) => {
|
|
144
|
+
let depth = 0;
|
|
145
|
+
for (let i = open; i < stripped.length; i++) {
|
|
146
|
+
const ch = stripped[i];
|
|
147
|
+
if (ch === "[")
|
|
148
|
+
depth++;
|
|
149
|
+
else if (ch === "]") {
|
|
150
|
+
depth--;
|
|
151
|
+
if (depth === 0)
|
|
152
|
+
return stripped.slice(open + 1, i);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return undefined;
|
|
156
|
+
};
|
|
157
|
+
const arrays = new Map();
|
|
158
|
+
for (const m of stripped.matchAll(/(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*(?::[^=]+)?=\s*\[/g)) {
|
|
159
|
+
const body = arrayBodyAt(m.index + m[0].length - 1);
|
|
160
|
+
if (body !== undefined)
|
|
161
|
+
arrays.set(m[1], body);
|
|
162
|
+
}
|
|
163
|
+
const globs = [];
|
|
164
|
+
const lift = (body) => { for (const s of body.matchAll(/['"`]([^'"`\n]+)['"`]/g))
|
|
165
|
+
globs.push(s[1]); };
|
|
166
|
+
for (const m of stripped.matchAll(/\b(?:include|testMatch)\s*:\s*(\[|([A-Za-z_$][\w$]*))/g)) {
|
|
167
|
+
if (m[1] === "[") {
|
|
168
|
+
const body = arrayBodyAt(m.index + m[0].length - 1);
|
|
169
|
+
if (body !== undefined)
|
|
170
|
+
lift(body);
|
|
171
|
+
}
|
|
172
|
+
else if (m[2] && arrays.has(m[2]))
|
|
173
|
+
lift(arrays.get(m[2]));
|
|
174
|
+
}
|
|
175
|
+
// Only keep things that look like file globs (contain a slash or a glob char) - drops e.g. env names
|
|
176
|
+
return Array.from(new Set(globs.filter((g) => /[*/]/.test(g) && !g.startsWith("!"))));
|
|
177
|
+
}
|
|
178
|
+
function scriptsInvoking(scripts, runner, configFile, isDefault) {
|
|
179
|
+
const result = [];
|
|
180
|
+
for (const [name, cmd] of Object.entries(scripts)) {
|
|
181
|
+
if (!new RegExp(`\\b${runner}\\b`).test(cmd))
|
|
182
|
+
continue;
|
|
183
|
+
const cfg = /--config(?:=|\s+)(\S+)/.exec(cmd)?.[1];
|
|
184
|
+
if (cfg ? cfg === configFile || cfg.endsWith(`/${configFile}`) : isDefault)
|
|
185
|
+
result.push(name);
|
|
186
|
+
}
|
|
187
|
+
return result.sort();
|
|
188
|
+
}
|
|
189
|
+
/** Static discovery of test-runner configs at the repository root. Never throws; never executes. */
|
|
190
|
+
export function discoverTestRunnerConfigs(repoPath, scripts = {}) {
|
|
191
|
+
const configs = [];
|
|
192
|
+
let entries = [];
|
|
193
|
+
try {
|
|
194
|
+
entries = existsSync(repoPath) ? readdirSync(repoPath) : [];
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
entries = [];
|
|
198
|
+
}
|
|
199
|
+
for (const name of entries.sort()) {
|
|
200
|
+
const m = CONFIG_NAME.exec(name);
|
|
201
|
+
if (!m)
|
|
202
|
+
continue;
|
|
203
|
+
const full = join(repoPath, name);
|
|
204
|
+
try {
|
|
205
|
+
if (!statSync(full).isFile())
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
catch {
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
let source = "";
|
|
212
|
+
try {
|
|
213
|
+
source = readFileSync(full, "utf8");
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
const runner = m[1].toLowerCase();
|
|
219
|
+
const isDefault = m[2] === undefined;
|
|
220
|
+
const extracted = extractTestPatterns(source);
|
|
221
|
+
configs.push({
|
|
222
|
+
file: name,
|
|
223
|
+
runner,
|
|
224
|
+
includes: extracted.includes,
|
|
225
|
+
scripts: scriptsInvoking(scripts, runner, name, isDefault),
|
|
226
|
+
family: familyOfConfigName(name),
|
|
227
|
+
excludeGlobs: extracted.excludeGlobs,
|
|
228
|
+
ignoreRegexSources: extracted.ignoreRegexSources,
|
|
229
|
+
roots: extracted.roots,
|
|
230
|
+
declaresTests: extracted.declaresTests,
|
|
231
|
+
isDefault,
|
|
232
|
+
authoritative: extracted.declaresTests && extracted.complete && extracted.includes.length > 0,
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
// THE REPLACEMENT RULE, and it is deliberately narrow.
|
|
236
|
+
//
|
|
237
|
+
// Only the DEFAULT config (`jest.config.ts`, `vitest.config.ts`) describes what the bare runner
|
|
238
|
+
// executes, so only the default config may replace the runner built-in globs. A named variant
|
|
239
|
+
// (`vitest.e2e.config.ts`) is a SEPARATE job: it ADDS its includes and says nothing about what
|
|
240
|
+
// plain `vitest` runs. Treating a variant as authority over the default universe would drop every
|
|
241
|
+
// `*.spec.ts` in a repository whose only config file happens to be an e2e one - narrowing on
|
|
242
|
+
// evidence that does not bear on the question.
|
|
243
|
+
//
|
|
244
|
+
// And the default config must have been COMPLETELY understood. Wrong-wide costs compute;
|
|
245
|
+
// wrong-narrow can hide a test the runner executes, which is the shape of a false green. Every
|
|
246
|
+
// ambiguity resolves wide.
|
|
247
|
+
const defaults = configs.filter((c) => c.isDefault);
|
|
248
|
+
const replacedDefaults = defaults.length > 0 && defaults.every((c) => c.authoritative);
|
|
249
|
+
const patterns = new Set(replacedDefaults ? [] : DEFAULT_TEST_PATTERNS);
|
|
250
|
+
for (const c of configs)
|
|
251
|
+
for (const g of c.includes)
|
|
252
|
+
patterns.add(g);
|
|
253
|
+
// Excludes, ignores and roots are NARROWING, so they are honoured only from configs whose
|
|
254
|
+
// declaration was fully understood - and only when the declaration is actually in force.
|
|
255
|
+
// Narrowing metadata is honoured ONLY from an authoritative DEFAULT config. A variant roots or
|
|
256
|
+
// ignore list governs that variant own job, and applying it repository-wide would over-narrow.
|
|
257
|
+
const authoritative = replacedDefaults ? defaults : [];
|
|
258
|
+
return {
|
|
259
|
+
configs,
|
|
260
|
+
patterns: Array.from(patterns),
|
|
261
|
+
replacedDefaults,
|
|
262
|
+
excludeGlobs: Array.from(new Set(authoritative.flatMap((c) => c.excludeGlobs))),
|
|
263
|
+
ignoreRegexSources: Array.from(new Set(authoritative.flatMap((c) => c.ignoreRegexSources))),
|
|
264
|
+
roots: Array.from(new Set(authoritative.flatMap((c) => c.roots))),
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
// --- glob matching (shared by analyzer / graph / impact so "is this a test?" has ONE answer) ---
|
|
268
|
+
function expandBraces(pattern) {
|
|
269
|
+
const match = /\{([^{}]*)\}/.exec(pattern);
|
|
270
|
+
if (!match)
|
|
271
|
+
return [pattern];
|
|
272
|
+
const prefix = pattern.slice(0, match.index);
|
|
273
|
+
const suffix = pattern.slice(match.index + match[0].length);
|
|
274
|
+
const out = [];
|
|
275
|
+
for (const alt of match[1].split(","))
|
|
276
|
+
out.push(...expandBraces(`${prefix}${alt}${suffix}`));
|
|
277
|
+
return out;
|
|
278
|
+
}
|
|
279
|
+
function escapeRegexLiteral(text) {
|
|
280
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Extended-glob groups - `?(a|b)`, `*(a|b)`, `+(a|b)`, `@(a|b)` - are not exotic: they appear in the
|
|
284
|
+
* published default include globs of both vitest (`**\/*.{test,spec}.?(c|m)[jt]s?(x)`) and jest, and
|
|
285
|
+
* in every config that copies them. Passed through to `RegExp` unchanged, `?(x)` reads as "an
|
|
286
|
+
* optional preceding character, then a literal x", so `__tests__/base.js` matched nothing at all.
|
|
287
|
+
* Measured cost before this fix (Phase 01 baseline, 2026-08-26): immerjs/immer discovered ZERO test
|
|
288
|
+
* files from its own explicit `include: ["**\/__tests__\/**\/*.[jt]s?(x)"]`, while still classifying
|
|
289
|
+
* 5 of 5 commits SELECTIVE at COMPLETE confidence.
|
|
290
|
+
*
|
|
291
|
+
* `!(...)` (negation) is deliberately not modelled - it needs real parsing to be correct. It is
|
|
292
|
+
* widened to "any single path segment", which OVER-includes. Over-inclusion counts extra files as
|
|
293
|
+
* tests; under-inclusion silently empties the test universe. The former is the safe direction.
|
|
294
|
+
*
|
|
295
|
+
* A bare `?` outside a group is a single-character wildcard (`[^/]`), which it also was not: it
|
|
296
|
+
* previously reached the regex as a quantifier over whatever preceded it.
|
|
297
|
+
*/
|
|
298
|
+
/**
|
|
299
|
+
* Wildcards inside an extglob body, translated by the same rules as the rest of the pattern.
|
|
300
|
+
*
|
|
301
|
+
* `?(*.)` in jest's `**\/?(*.)+(spec|test).[jt]s?(x)` means "optionally: anything, then a dot". The
|
|
302
|
+
* body is a glob in its own right, so escaping it as a literal turns it into "optionally the two
|
|
303
|
+
* characters `*` and `.`" - which no real path contains, so `src/foo.test.js` matched NOTHING while
|
|
304
|
+
* the bare `test.js` still matched. Found 2026-08-30 while diagnosing Prettier, where it was one of
|
|
305
|
+
* two defects in the same area (see the duplicate matcher deleted from impact.ts).
|
|
306
|
+
*
|
|
307
|
+
* Bracket expressions such as `[jt]` are left alone deliberately: they are already valid regex
|
|
308
|
+
* character classes and mean the same thing in both syntaxes.
|
|
309
|
+
*/
|
|
310
|
+
function translateExtglobBody(body) {
|
|
311
|
+
return body.replace(/\\/g, "\\\\").replace(/\./g, "\\.").replace(/\*/g, "[^/]*").replace(/\?/g, "[^/]");
|
|
312
|
+
}
|
|
313
|
+
function globToRegex(pattern) {
|
|
314
|
+
// Extglob bodies contain `*`, `?` and `|` that must not be rewritten by the wildcard rules below,
|
|
315
|
+
// so they are lifted out behind placeholders first and restored last.
|
|
316
|
+
const groups = [];
|
|
317
|
+
let working = pattern.replace(/([?*+@!])\(([^()]*)\)/g, (_match, operator, body) => {
|
|
318
|
+
const alternatives = body.split("|").map(translateExtglobBody).join("|");
|
|
319
|
+
const source = operator === "!"
|
|
320
|
+
? "[^/]*"
|
|
321
|
+
: `(?:${alternatives})${operator === "?" ? "?" : operator === "*" ? "*" : operator === "+" ? "+" : ""}`;
|
|
322
|
+
groups.push(source);
|
|
323
|
+
return `\0X${groups.length - 1}\0`;
|
|
324
|
+
});
|
|
325
|
+
working = working.replace(/\\/g, "\\\\").replace(/\./g, "\\.");
|
|
326
|
+
working = working
|
|
327
|
+
.replace(/\*\*\//g, "\0GS\0")
|
|
328
|
+
.replace(/\/\*\*/g, "\0SG\0")
|
|
329
|
+
.replace(/\*/g, "[^/]*")
|
|
330
|
+
.replace(/\?/g, "[^/]")
|
|
331
|
+
.replace(/\0GS\0/g, "(?:.*/)?")
|
|
332
|
+
.replace(/\0SG\0/g, "(?:/.*)?");
|
|
333
|
+
working = working.replace(/\0X(\d+)\0/g, (_match, index) => groups[Number(index)]);
|
|
334
|
+
return new RegExp(`^${working}$`);
|
|
335
|
+
}
|
|
336
|
+
/** Matches a repo-relative posix path against one glob, with brace and extglob support. Exported so
|
|
337
|
+
* analyzer discovery, graph node flags and impact classification all share ONE definition of a
|
|
338
|
+
* match rather than the two near-identical copies that existed before Phase 01. */
|
|
339
|
+
export function matchesGlob(path, pattern) {
|
|
340
|
+
return expandBraces(pattern).some((p) => globToRegex(p).test(path));
|
|
341
|
+
}
|
|
342
|
+
function compile(patterns) {
|
|
343
|
+
return patterns.flatMap((p) => expandBraces(p.includes("/") ? p : `**/${p}`)).map(globToRegex);
|
|
344
|
+
}
|
|
345
|
+
/** Builds a matcher over repo-relative posix paths. Patterns without a slash (bare filename globs)
|
|
346
|
+
* are treated as `**\/<pattern>` so a config's `*.spec.ts` still means "anywhere". */
|
|
347
|
+
export function createTestFileMatcher(patterns, options = {}) {
|
|
348
|
+
const regexes = compile(patterns);
|
|
349
|
+
const excludes = compile(options.excludePatterns ?? []);
|
|
350
|
+
const authoritative = compile(options.authoritativePatterns ?? []);
|
|
351
|
+
const ignoreRegexes = options.ignoreRegexes ?? [];
|
|
352
|
+
const roots = options.roots ?? [];
|
|
353
|
+
const fn = ((path) => {
|
|
354
|
+
// Runner-universe vetoes come FIRST, ahead of even the authoritative patterns. A file the
|
|
355
|
+
// configured runner will not execute is not a test DiffCI can select, however strongly
|
|
356
|
+
// DiffCI conventions or the repository own include globs say it looks like one.
|
|
357
|
+
if (!isUnderRoots(path, roots))
|
|
358
|
+
return false;
|
|
359
|
+
if (isIgnoredPath(path, ignoreRegexes))
|
|
360
|
+
return false;
|
|
361
|
+
if (authoritative.some((r) => r.test(path)))
|
|
362
|
+
return true;
|
|
363
|
+
if (excludes.some((r) => r.test(path)))
|
|
364
|
+
return false;
|
|
365
|
+
return regexes.some((r) => r.test(path));
|
|
366
|
+
});
|
|
367
|
+
Object.defineProperty(fn, "patterns", { value: Object.freeze([...patterns]) });
|
|
368
|
+
return fn;
|
|
369
|
+
}
|
|
370
|
+
/** The one place that turns a profile into a matcher, so analyzer discovery, graph node flags and
|
|
371
|
+
* impact classification cannot disagree about what a test is. */
|
|
372
|
+
export function testFileMatcherForProfile(profile) {
|
|
373
|
+
if (!profile.testPatterns)
|
|
374
|
+
return DEFAULT_TEST_FILE_MATCHER;
|
|
375
|
+
return createTestFileMatcher(profile.testPatterns, {
|
|
376
|
+
excludePatterns: profile.testExcludePatterns,
|
|
377
|
+
authoritativePatterns: profile.testAuthoritativePatterns,
|
|
378
|
+
ignoreRegexes: compileIgnoreRegexes(profile.testIgnoreRegexSources ?? []),
|
|
379
|
+
roots: profile.testRoots,
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
/** The pre-2026-08-23 behaviour, kept as the fallback when no profile is available. */
|
|
383
|
+
export const DEFAULT_TEST_FILE_MATCHER = createTestFileMatcher(DEFAULT_TEST_PATTERNS);
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conservative test-fixture ownership (2026-08-23, deepseek-harness benchmark Phase 3).
|
|
3
|
+
*
|
|
4
|
+
* Recorded fixtures such as `examples/acp-agent/tests/snapshots/<case>/session.jsonl` are read at
|
|
5
|
+
* runtime by the suite beside them (`examples/acp-agent/tests/acp.snapshot.ts` builds the directory
|
|
6
|
+
* from `import.meta.url` + 'snapshots'). The import graph cannot see that edge, so before this module
|
|
7
|
+
* every such change was "Unknown changed file" -> full fallback (157 files on one deepseek merge).
|
|
8
|
+
*
|
|
9
|
+
* These files are TEST INPUTS, never documentation: a change must SELECT the tests that read it, or
|
|
10
|
+
* a conservative superset, or stay unknown. The relationship is expressed purely structurally:
|
|
11
|
+
*
|
|
12
|
+
* <scope>/<tests-dir>/<fixture-dir>/... is owned by the test files that live in <scope>/<tests-dir>
|
|
13
|
+
*
|
|
14
|
+
* with every step guarded:
|
|
15
|
+
* 1. the nearest ancestor directory named `tests`, `test` or `__tests__` is the tests dir T;
|
|
16
|
+
* 2. the segment directly under T must be a fixture directory by name (`snapshots`,
|
|
17
|
+
* `goal-snapshots`, `fixtures`, `__fixtures__`, `__snapshots__`, ...) - a `.jsonl` elsewhere
|
|
18
|
+
* under T (or outside any tests dir) is NOT claimed;
|
|
19
|
+
* 3. owners are the recognised test files (profile test patterns) that are DIRECT children of T.
|
|
20
|
+
* When the fixture dir is snapshot-named and T has snapshot-family tests, only those are owned
|
|
21
|
+
* (a snapshots dir is read by snapshot suites); otherwise all direct test children; if T has no
|
|
22
|
+
* direct test children, every recognised test under T/** (wider superset);
|
|
23
|
+
* 4. if that still yields nothing - or the caller has no HEAD inventory to look in - the result is
|
|
24
|
+
* undefined and the file stays "unknown" (fallback). Deleted fixtures resolve against HEAD: the
|
|
25
|
+
* tests dir still exists -> its tests are selected; tests dir gone -> unknown.
|
|
26
|
+
*
|
|
27
|
+
* Ownership is by directory convention only; nothing here inspects file contents or file extensions,
|
|
28
|
+
* so the same rule covers `.jsonl`, `.json`, `.txt`, `.expected.*` fixtures alike.
|
|
29
|
+
*/
|
|
30
|
+
import { testFamilyOfPath } from "./test-discovery.js";
|
|
31
|
+
const TESTS_DIR_NAMES = new Set(["tests", "test", "__tests__"]);
|
|
32
|
+
const FIXTURE_DIR = /(^|[-_.])(snapshots?|fixtures?)([-_.]|$)|^__(snapshots|fixtures)__$/i;
|
|
33
|
+
export function resolveTestFixtureOwners(changedPath, isTestFile, repositoryFiles) {
|
|
34
|
+
if (!repositoryFiles)
|
|
35
|
+
return undefined;
|
|
36
|
+
if (isTestFile(changedPath))
|
|
37
|
+
return undefined; // a test is a test, not a fixture
|
|
38
|
+
const segments = changedPath.split("/");
|
|
39
|
+
// nearest tests-dir ancestor (search from the deepest directory upward)
|
|
40
|
+
let t = -1;
|
|
41
|
+
for (let i = segments.length - 2; i >= 0; i--) {
|
|
42
|
+
if (TESTS_DIR_NAMES.has(segments[i])) {
|
|
43
|
+
t = i;
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (t === -1)
|
|
48
|
+
return undefined;
|
|
49
|
+
const fixtureSegment = segments[t + 1];
|
|
50
|
+
// must be a fixture DIRECTORY (at least one more segment = the file itself), and fixture-named
|
|
51
|
+
if (fixtureSegment === undefined || t + 1 >= segments.length - 1 || !FIXTURE_DIR.test(fixtureSegment))
|
|
52
|
+
return undefined;
|
|
53
|
+
const testsDir = segments.slice(0, t + 1).join("/");
|
|
54
|
+
const prefix = `${testsDir}/`;
|
|
55
|
+
const direct = [];
|
|
56
|
+
const recursive = [];
|
|
57
|
+
for (const f of repositoryFiles) {
|
|
58
|
+
if (!f.startsWith(prefix) || !isTestFile(f))
|
|
59
|
+
continue;
|
|
60
|
+
recursive.push(f);
|
|
61
|
+
if (!f.slice(prefix.length).includes("/"))
|
|
62
|
+
direct.push(f);
|
|
63
|
+
}
|
|
64
|
+
if (recursive.length === 0)
|
|
65
|
+
return undefined;
|
|
66
|
+
const fixtureDir = `${testsDir}/${fixtureSegment}`;
|
|
67
|
+
if (direct.length > 0) {
|
|
68
|
+
if (/snapshot/i.test(fixtureSegment)) {
|
|
69
|
+
const snapshotOwners = direct.filter((f) => testFamilyOfPath(f) === "snapshot");
|
|
70
|
+
if (snapshotOwners.length > 0)
|
|
71
|
+
return { testsDir, fixtureDir, scope: "direct-snapshot-family", owners: snapshotOwners.sort() };
|
|
72
|
+
}
|
|
73
|
+
return { testsDir, fixtureDir, scope: "direct", owners: direct.sort() };
|
|
74
|
+
}
|
|
75
|
+
return { testsDir, fixtureDir, scope: "recursive", owners: recursive.sort() };
|
|
76
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/** End-to-end runners. They are real test frameworks, but a unit-test file that no config claims
|
|
2
|
+
* should not be routed to one, so they are never chosen as a repository's primary runner. */
|
|
3
|
+
export const END_TO_END_FRAMEWORKS = new Set(["playwright", "cypress"]);
|
|
4
|
+
/** Each framework's own published default include globs, transcribed. */
|
|
5
|
+
export const FRAMEWORK_DEFAULT_INCLUDES = {
|
|
6
|
+
// vitest: `include` defaults to ['**\/*.{test,spec}.?(c|m)[jt]s?(x)']
|
|
7
|
+
vitest: ["**/*.{test,spec}.?(c|m)[jt]s?(x)"],
|
|
8
|
+
// jest: `testMatch` defaults to ["**\/__tests__\/**\/*.[jt]s?(x)", "**\/?(*.)+(spec|test).[jt]s?(x)"]
|
|
9
|
+
jest: ["**/__tests__/**/*.[jt]s?(x)", "**/?(*.)+(spec|test).[jt]s?(x)"],
|
|
10
|
+
// mocha: `spec` defaults to './test/*.{js,cjs,mjs}'; TypeScript repositories run the same layout
|
|
11
|
+
// through a loader, so the TS extensions are included alongside.
|
|
12
|
+
mocha: ["test/*.{js,cjs,mjs,ts,mts,cts}"],
|
|
13
|
+
// ava: files defaults to test.js, test-*.js, test/**, **\/__tests__\/**, **\/*.spec.js,
|
|
14
|
+
// **\/*.test.js (ava also accepts .cjs/.mjs/.ts under its own compilation step).
|
|
15
|
+
ava: [
|
|
16
|
+
"test.{js,cjs,mjs,ts}",
|
|
17
|
+
"test-*.{js,cjs,mjs,ts}",
|
|
18
|
+
"test/**/*.{js,cjs,mjs,ts}",
|
|
19
|
+
"**/__tests__/**/*.{js,cjs,mjs,ts}",
|
|
20
|
+
"**/*.spec.{js,cjs,mjs,ts}",
|
|
21
|
+
"**/*.test.{js,cjs,mjs,ts}",
|
|
22
|
+
],
|
|
23
|
+
// tap: defaults to test/ and tap-snapshots/, plus *.test.* anywhere.
|
|
24
|
+
tap: ["test/**/*.{js,cjs,mjs,ts,mts,cts}", "**/*.test.{js,cjs,mjs,ts,mts,cts}"],
|
|
25
|
+
// node:test: the runner's own default discovery is **\/*.test.?(c|m)[jt]s plus files under a
|
|
26
|
+
// test/ directory.
|
|
27
|
+
"node:test": ["**/*.test.?(c|m)[jt]s", "test/**/*.{js,cjs,mjs,ts,mts,cts}"],
|
|
28
|
+
// jasmine: `spec_files` defaults to "**\/*[sS]pec.?(m)js", resolved under `spec_dir` ("spec").
|
|
29
|
+
jasmine: ["spec/**/*[sS]pec.?(m)js", "**/*[sS]pec.{js,mjs,ts}"],
|
|
30
|
+
// bun test: discovers *.test.{js,jsx,ts,tsx}, *_test.*, *.spec.* and *_spec.*
|
|
31
|
+
"bun:test": ["**/*.{test,spec}.{js,jsx,ts,tsx}", "**/*_{test,spec}.{js,jsx,ts,tsx}"],
|
|
32
|
+
// playwright: `testMatch` defaults to **\/*.@(spec|test).?(c|m)[jt]s?(x)
|
|
33
|
+
playwright: ["**/*.@(spec|test).?(c|m)[jt]s?(x)"],
|
|
34
|
+
// cypress: `specPattern` defaults to cypress/e2e/**\/*.cy.{js,jsx,ts,tsx}
|
|
35
|
+
cypress: ["cypress/e2e/**/*.cy.{js,jsx,ts,tsx}", "**/*.cy.{js,jsx,ts,tsx}"],
|
|
36
|
+
};
|
|
37
|
+
const MARKERS = [
|
|
38
|
+
{ framework: "vitest", packages: ["vitest"], script: /(?:^|[\s;&|])vitest(?:$|[\s;&|])/ },
|
|
39
|
+
{ framework: "jest", packages: ["jest", "ts-jest", "@swc/jest", "jest-cli"], script: /(?:^|[\s;&|/])jest(?:$|[\s;&|])/ },
|
|
40
|
+
{ framework: "mocha", packages: ["mocha", "ts-mocha"], script: /(?:^|[\s;&|/])mocha(?:$|[\s;&|])/ },
|
|
41
|
+
{ framework: "ava", packages: ["ava"], script: /(?:^|[\s;&|/])ava(?:$|[\s;&|])/ },
|
|
42
|
+
{ framework: "tap", packages: ["tap", "libtap"], script: /(?:^|[\s;&|/])tap(?:$|[\s;&|])/ },
|
|
43
|
+
{ framework: "node:test", packages: [], script: /(?:node|tsx)\s[^;&|]*--test(?:$|[\s;&|=])/ },
|
|
44
|
+
{ framework: "jasmine", packages: ["jasmine", "jasmine-core"], script: /(?:^|[\s;&|/])jasmine(?:$|[\s;&|])/ },
|
|
45
|
+
{ framework: "bun:test", packages: [], script: /(?:^|[\s;&|])bun\s+test(?:$|[\s;&|])/ },
|
|
46
|
+
// End-to-end runners last, so that a repository with both gets a unit-test runner as its primary.
|
|
47
|
+
{ framework: "playwright", packages: ["@playwright/test", "playwright"], script: /(?:^|[\s;&|/])playwright(?:$|[\s;&|])/ },
|
|
48
|
+
{ framework: "cypress", packages: ["cypress"], script: /(?:^|[\s;&|/])cypress(?:$|[\s;&|])/ },
|
|
49
|
+
];
|
|
50
|
+
/**
|
|
51
|
+
* Frameworks the repository declares. A dependency is stronger evidence than a script mention, but
|
|
52
|
+
* either is sufficient - a monorepo root often runs a framework it does not itself depend on, and a
|
|
53
|
+
* repository can depend on a framework it invokes only through a wrapper.
|
|
54
|
+
*/
|
|
55
|
+
export function detectDeclaredFrameworks(packageJson) {
|
|
56
|
+
if (!packageJson)
|
|
57
|
+
return { frameworks: [], evidence: {} };
|
|
58
|
+
const deps = { ...(packageJson.dependencies ?? {}), ...(packageJson.devDependencies ?? {}) };
|
|
59
|
+
const scripts = packageJson.scripts ?? {};
|
|
60
|
+
const frameworks = [];
|
|
61
|
+
const evidence = {};
|
|
62
|
+
for (const marker of MARKERS) {
|
|
63
|
+
const declaredPackage = marker.packages.find((p) => deps[p] !== undefined);
|
|
64
|
+
if (declaredPackage !== undefined) {
|
|
65
|
+
frameworks.push(marker.framework);
|
|
66
|
+
evidence[marker.framework] = `dependency:${declaredPackage}`;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const scriptEntry = Object.entries(scripts).find(([, command]) => marker.script.test(command));
|
|
70
|
+
if (scriptEntry) {
|
|
71
|
+
frameworks.push(marker.framework);
|
|
72
|
+
evidence[marker.framework] = `script:${scriptEntry[0]}`;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return { frameworks, evidence };
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Each framework's own published default EXCLUDES, transcribed - the other half of the defaults, and
|
|
79
|
+
* not optional. ava's defaults exclude `**\/fixtures\/**` and `**\/helpers\/**`; without that,
|
|
80
|
+
* `sindresorhus/execa` reports 337 test files where 193 of them are process fixtures ava would never
|
|
81
|
+
* run (measured 2026-08-26). An inflated test universe is a wrong denominator in every downstream
|
|
82
|
+
* savings figure, so "over-include and move on" is not good enough here.
|
|
83
|
+
*
|
|
84
|
+
* These apply only to files a framework's DEFAULT includes pulled in. A file that matches DiffCI's
|
|
85
|
+
* conventional `.test.`/`.spec.` patterns, or a glob the repository declared explicitly in its own
|
|
86
|
+
* config, is a test regardless of where it sits - the repository said so.
|
|
87
|
+
*/
|
|
88
|
+
export const FRAMEWORK_DEFAULT_EXCLUDES = {
|
|
89
|
+
vitest: ["**/node_modules/**", "**/dist/**", "**/cypress/**"],
|
|
90
|
+
jest: ["**/node_modules/**"],
|
|
91
|
+
mocha: ["**/node_modules/**"],
|
|
92
|
+
ava: ["**/fixtures/**", "**/helpers/**", "**/__helper__/**", "**/node_modules/**"],
|
|
93
|
+
tap: ["**/fixtures/**", "**/node_modules/**"],
|
|
94
|
+
"node:test": ["**/node_modules/**"],
|
|
95
|
+
jasmine: ["**/node_modules/**"],
|
|
96
|
+
"bun:test": ["**/node_modules/**"],
|
|
97
|
+
playwright: ["**/node_modules/**"],
|
|
98
|
+
cypress: ["**/node_modules/**", "**/cypress/support/**", "**/cypress/fixtures/**"],
|
|
99
|
+
};
|
|
100
|
+
/** The union of every declared framework's default include globs. */
|
|
101
|
+
export function defaultIncludesFor(frameworks) {
|
|
102
|
+
const patterns = new Set();
|
|
103
|
+
for (const framework of frameworks) {
|
|
104
|
+
for (const pattern of FRAMEWORK_DEFAULT_INCLUDES[framework])
|
|
105
|
+
patterns.add(pattern);
|
|
106
|
+
}
|
|
107
|
+
return Array.from(patterns);
|
|
108
|
+
}
|
|
109
|
+
/** The union of every declared framework's default exclude globs. */
|
|
110
|
+
export function defaultExcludesFor(frameworks) {
|
|
111
|
+
const patterns = new Set();
|
|
112
|
+
for (const framework of frameworks) {
|
|
113
|
+
for (const pattern of FRAMEWORK_DEFAULT_EXCLUDES[framework])
|
|
114
|
+
patterns.add(pattern);
|
|
115
|
+
}
|
|
116
|
+
return Array.from(patterns);
|
|
117
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# DiffCI Distribution
|
|
2
|
+
|
|
3
|
+
DiffCI has three install surfaces with the same initial contract: observe CI, write a report, and do
|
|
4
|
+
not change what the host repository runs.
|
|
5
|
+
|
|
6
|
+
## GitHub App
|
|
7
|
+
|
|
8
|
+
The DiffCI Shadow GitHub App is the lowest-friction research and design-partner path. It receives
|
|
9
|
+
repository events, runs shadow analysis outside the repository's CI jobs, and reconciles predictions
|
|
10
|
+
against real CI outcomes. Use it when a maintainer wants observation without adding a workflow step.
|
|
11
|
+
|
|
12
|
+
## GitHub Action
|
|
13
|
+
|
|
14
|
+
The GitHub Action is the OSS dependency-graph path. A repository installs DiffCI as its own
|
|
15
|
+
continue-on-error job:
|
|
16
|
+
|
|
17
|
+
```yaml
|
|
18
|
+
jobs:
|
|
19
|
+
diffci:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
continue-on-error: true
|
|
22
|
+
permissions:
|
|
23
|
+
contents: read
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v4
|
|
26
|
+
with:
|
|
27
|
+
fetch-depth: 0
|
|
28
|
+
- uses: DiffCI/DiffCI.com@v1
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
For the strongest supply-chain posture, pin the Action to a full commit SHA. `npx @diffci.com/diffci verify-workflow`
|
|
32
|
+
checks that the job is dedicated, read-only, not required by other jobs, and unable to alter the rest
|
|
33
|
+
of CI.
|
|
34
|
+
|
|
35
|
+
## npm CLI
|
|
36
|
+
|
|
37
|
+
The CLI is the standalone npm package surface. The package name is `@diffci.com/diffci`:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
npx @diffci.com/diffci observe
|
|
41
|
+
npx @diffci.com/diffci verify-workflow
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`observe` writes a JSON report outside the checkout by default. It never runs, skips, cancels, or
|
|
45
|
+
reorders tests. A hosted endpoint is opt-in: reports are sent only when both `DIFFCI_API_URL` and
|
|
46
|
+
`DIFFCI_TOKEN` are set, or when equivalent CLI flags are passed.
|
|
47
|
+
|
|
48
|
+
## Report Shape
|
|
49
|
+
|
|
50
|
+
A shadow report should answer the adoption question before it asks for operational trust:
|
|
51
|
+
|
|
52
|
+
```text
|
|
53
|
+
DiffCI - last 30 days
|
|
54
|
+
|
|
55
|
+
CI runs observed 423
|
|
56
|
+
Compute time 18,240 min
|
|
57
|
+
Potentially avoidable 6,810 min
|
|
58
|
+
Potential reduction 37.3%
|
|
59
|
+
|
|
60
|
+
Estimated compute avoided xxx CPU-hours
|
|
61
|
+
Estimated electricity xxx kWh
|
|
62
|
+
Estimated CO2 xxx kg
|
|
63
|
+
Estimated water xxx L
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
That makes the open-source proposition explicit: install DiffCI Shadow, change nothing in CI, and learn
|
|
67
|
+
how much compute may be wasted.
|
|
68
|
+
|
|
69
|
+
## Future Package Managers
|
|
70
|
+
|
|
71
|
+
npm plus the GitHub Action are enough to establish the pattern. Later package surfaces can wrap the
|
|
72
|
+
same observer contract:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install diffci
|
|
76
|
+
cargo install diffci
|
|
77
|
+
brew install diffci
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Those should ship only after the npm CLI and Action have signed releases, provenance, pinned build
|
|
81
|
+
workflows, and repeatable package verification.
|