@hyperfixi/testing-framework 2.9.4 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/package.json +7 -6
- package/src/agent-bench/README.md +162 -0
- package/src/agent-bench/agent-bench.test.ts +112 -0
- package/src/agent-bench/cli.ts +315 -0
- package/src/agent-bench/harness.ts +331 -0
- package/src/agent-bench/tasks.ts +214 -0
- package/src/agent-bench/variants.ts +241 -0
- package/src/multilingual/fidelity.ts +22 -376
- package/src/multilingual/shipped-examples-execution.test.ts +10 -5
- package/src/multilingual/shipped-examples-execution.ts +32 -20
|
@@ -45,6 +45,7 @@
|
|
|
45
45
|
|
|
46
46
|
import fs from 'node:fs';
|
|
47
47
|
import path from 'node:path';
|
|
48
|
+
import { execFileSync } from 'node:child_process';
|
|
48
49
|
import { createHash } from 'node:crypto';
|
|
49
50
|
import { createRequire } from 'node:module';
|
|
50
51
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
@@ -168,22 +169,33 @@ export function triggerEventOf(source: string): string | null {
|
|
|
168
169
|
return m ? (m[1] ?? null) : null;
|
|
169
170
|
}
|
|
170
171
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
172
|
+
/**
|
|
173
|
+
* Git-TRACKED .html files under `root` (repo-relative), as absolute paths.
|
|
174
|
+
*
|
|
175
|
+
* Tracked, not on-disk, because the corpus must be the SHIPPED tree: four
|
|
176
|
+
* `examples/` dirs are gitignored (experiments/, playground/,
|
|
177
|
+
* vite-plugin-test/, vite-plugin-multilingual/), and the disk walk this
|
|
178
|
+
* replaced counted whatever happened to be on the machine. That is how the
|
|
179
|
+
* gate's sanity floors and allowlist got calibrated against handlers that
|
|
180
|
+
* were never shipped — and why it failed on every clean checkout the first
|
|
181
|
+
* time CI ran it (#862). `git ls-files` sees the same corpus everywhere.
|
|
182
|
+
*
|
|
183
|
+
* git being unavailable is a THROW, not a fallback to the disk walk — a
|
|
184
|
+
* silent fallback would resurrect the corpus drift this exists to kill.
|
|
185
|
+
* A tracked file missing from disk (an uncommitted local deletion) is
|
|
186
|
+
* skipped: the working tree is mid-edit, and in CI checkout == index.
|
|
187
|
+
*/
|
|
188
|
+
function trackedHtml(repoRoot: string, root: string): string[] {
|
|
189
|
+
const out = execFileSync('git', ['ls-files', '-z', '--', root], {
|
|
190
|
+
cwd: repoRoot,
|
|
191
|
+
encoding: 'utf8',
|
|
192
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
193
|
+
});
|
|
194
|
+
return out
|
|
195
|
+
.split('\0')
|
|
196
|
+
.filter(rel => rel.endsWith('.html'))
|
|
197
|
+
.map(rel => path.join(repoRoot, rel))
|
|
198
|
+
.filter(full => fs.existsSync(full));
|
|
187
199
|
}
|
|
188
200
|
|
|
189
201
|
/** Extract the page's `[_]` handlers in document order. */
|
|
@@ -398,9 +410,9 @@ async function runHandlerOnEngine(
|
|
|
398
410
|
}
|
|
399
411
|
|
|
400
412
|
/**
|
|
401
|
-
* The sweep: walk `examples
|
|
402
|
-
* filters (each skip reasoned), and execute every
|
|
403
|
-
* engines.
|
|
413
|
+
* The sweep: walk the git-tracked `examples/**` corpus, extract handlers,
|
|
414
|
+
* apply the fair-denominator filters (each skip reasoned), and execute every
|
|
415
|
+
* eligible handler on both engines.
|
|
404
416
|
*/
|
|
405
417
|
export async function runShippedExamplesExecution(opts?: {
|
|
406
418
|
roots?: string[];
|
|
@@ -417,7 +429,7 @@ export async function runShippedExamplesExecution(opts?: {
|
|
|
417
429
|
let handlers = 0;
|
|
418
430
|
|
|
419
431
|
for (const root of roots) {
|
|
420
|
-
for (const full of
|
|
432
|
+
for (const full of trackedHtml(repoRoot, root)) {
|
|
421
433
|
const rel = path.relative(repoRoot, full);
|
|
422
434
|
const html = fs.readFileSync(full, 'utf8');
|
|
423
435
|
const pageHandlers = extractHandlers(rel, html);
|