@hyperfixi/testing-framework 2.9.3 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/package.json +7 -7
- package/src/multilingual/cli.ts +40 -0
- package/src/multilingual/orchestrator.ts +9 -4
- package/src/multilingual/reporters/regression-reporter.test.ts +103 -0
- package/src/multilingual/reporters/regression-reporter.ts +40 -0
- package/src/multilingual/shipped-examples-execution.test.ts +10 -5
- package/src/multilingual/shipped-examples-execution.ts +32 -20
- package/src/multilingual/types.ts +22 -0
- package/src/multilingual/validators/execution-validator.test.ts +2 -1
- package/src/multilingual/validators/execution-validator.ts +12 -0
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [2.10.0] - 2026-08-01
|
|
11
|
+
|
|
12
|
+
Full notes: [GitHub Releases](https://github.com/codetalcott/hyperfixi/releases/tag/v2.10.0).
|
|
13
|
+
|
|
14
|
+
Highlights: `swap`/`process`/`morph … using view transition` across all 24
|
|
15
|
+
languages (#870, #873, #875), typed command registration (#869, #871),
|
|
16
|
+
`tell`/`process` parser fixes (#861, #872), the R1 role-fidelity burn-down
|
|
17
|
+
(#864, #867, #868, #874, #878), and guards for the remaining hand-maintained
|
|
18
|
+
CI package lists (#862, #865).
|
|
19
|
+
|
|
20
|
+
## [2.9.0] - 2026-07-25
|
|
21
|
+
|
|
22
|
+
> **Gap note:** 2.9.1 through 2.9.4 shipped without entries in this file; their
|
|
23
|
+
> details live in [GitHub Releases](https://github.com/codetalcott/hyperfixi/releases).
|
|
24
|
+
|
|
10
25
|
### ⚠ BREAKING (types)
|
|
11
26
|
|
|
12
27
|
- **Domain renderers return `string | null`.** `renderSQL`, `renderJSX`, `renderTodo`,
|
|
@@ -379,7 +394,12 @@ _Synchronized version release. See git history for details._
|
|
|
379
394
|
- npm access token stored in GitHub Secrets
|
|
380
395
|
- 2FA recommended for npm organization
|
|
381
396
|
|
|
382
|
-
[Unreleased]: https://github.com/codetalcott/hyperfixi/compare/v2.
|
|
397
|
+
[Unreleased]: https://github.com/codetalcott/hyperfixi/compare/v2.10.0...HEAD
|
|
398
|
+
[2.10.0]: https://github.com/codetalcott/hyperfixi/compare/v2.9.0...v2.10.0
|
|
399
|
+
[2.9.0]: https://github.com/codetalcott/hyperfixi/compare/v2.8.0...v2.9.0
|
|
400
|
+
[2.8.0]: https://github.com/codetalcott/hyperfixi/compare/v2.5.1...v2.8.0
|
|
401
|
+
[2.5.1]: https://github.com/codetalcott/hyperfixi/compare/v2.5.0...v2.5.1
|
|
402
|
+
[2.5.0]: https://github.com/codetalcott/hyperfixi/compare/v2.4.0...v2.5.0
|
|
383
403
|
[2.4.0]: https://github.com/codetalcott/hyperfixi/compare/v2.3.1...v2.4.0
|
|
384
404
|
[2.3.1]: https://github.com/codetalcott/hyperfixi/compare/v2.3.0...v2.3.1
|
|
385
405
|
[2.3.0]: https://github.com/codetalcott/hyperfixi/compare/v2.2.1...v2.3.0
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hyperfixi/testing-framework",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.10.0",
|
|
4
4
|
"description": "Cross-platform behavior testing suite for LokaScript applications",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -57,14 +57,14 @@
|
|
|
57
57
|
"author": "LokaScript Contributors",
|
|
58
58
|
"license": "MIT",
|
|
59
59
|
"dependencies": {
|
|
60
|
-
"@hyperfixi/core": "^2.
|
|
61
|
-
"@hyperfixi/patterns-reference": "^2.
|
|
62
|
-
"@lokascript/i18n": "^2.
|
|
63
|
-
"@lokascript/semantic": "^2.
|
|
60
|
+
"@hyperfixi/core": "^2.10.0",
|
|
61
|
+
"@hyperfixi/patterns-reference": "^2.10.0",
|
|
62
|
+
"@lokascript/i18n": "^2.10.0",
|
|
63
|
+
"@lokascript/semantic": "^2.10.0",
|
|
64
64
|
"diff": "^8.0.3",
|
|
65
65
|
"esbuild": "^0.28.0",
|
|
66
66
|
"happy-dom": "^20.10.6",
|
|
67
|
-
"jsdom": "^
|
|
67
|
+
"jsdom": "^30.0.0",
|
|
68
68
|
"playwright": "^1.40.0",
|
|
69
69
|
"puppeteer": "^24.26.1",
|
|
70
70
|
"tsx": "^4.23.1",
|
|
@@ -73,7 +73,7 @@
|
|
|
73
73
|
},
|
|
74
74
|
"devDependencies": {
|
|
75
75
|
"@types/diff": "^5.0.0",
|
|
76
|
-
"@types/node": "^
|
|
76
|
+
"@types/node": "^26.1.2",
|
|
77
77
|
"@vitest/coverage-v8": "^4.0.0",
|
|
78
78
|
"hyperscript.org": "0.9.93",
|
|
79
79
|
"tsup": "^8.0.0",
|
package/src/multilingual/cli.ts
CHANGED
|
@@ -478,6 +478,28 @@ async function main(): Promise<void> {
|
|
|
478
478
|
r => r.avgRoleFidelityDelta < -AVG_ROLE_FIDELITY_DROP_TOLERANCE
|
|
479
479
|
);
|
|
480
480
|
|
|
481
|
+
// R1 — role-set flip ratchet: a pattern that was ROLE-FAITHFUL in the
|
|
482
|
+
// baseline (its action.role:valueType set matched the en reference
|
|
483
|
+
// exactly) and is now role-incomplete. Tolerance 0 for the same reason
|
|
484
|
+
// the lossy flip and R5 sit at 0: a flip is binary and structural, with
|
|
485
|
+
// no float/collation noise to absorb — the averages' tolerances are
|
|
486
|
+
// cross-machine headroom, not signal cushions.
|
|
487
|
+
// Not redundant with the avg ratchet above: one pattern losing one role
|
|
488
|
+
// moves avgRoleFidelity by ~0.0013 in a ~154-pattern language, 15× under
|
|
489
|
+
// its 0.02 tolerance. This was MEASURED as the gate's blind spot, not
|
|
490
|
+
// hypothesized: a `required: false` duration role on toggleSchema cost
|
|
491
|
+
// es/pl/vi the second toggle's positional destination on
|
|
492
|
+
// `toggle-aria-expanded` (`toggle.destination:expression` →
|
|
493
|
+
// `:reference`, i.e. `next .panel` silently became `me`) and
|
|
494
|
+
// --regression stayed GREEN; tools/triage-r1.ts found it by hand.
|
|
495
|
+
// Guarded by the baseline carrying `roleLossyPatterns` (the reporter
|
|
496
|
+
// returns [] without it), so an un-regenerated baseline never
|
|
497
|
+
// retro-flags. Pre-existing role-lossy patterns are the R1 burn-down
|
|
498
|
+
// list, not regressions.
|
|
499
|
+
const roleSetRegressions = allResults.flatMap(r =>
|
|
500
|
+
r.newRoleLossyPatterns.map(id => `${r.language}/${id}`)
|
|
501
|
+
);
|
|
502
|
+
|
|
481
503
|
// R3 — role-VALUE ratchet: same semantics as the role-fidelity ratchet,
|
|
482
504
|
// on the invariant-value signal (`action.role=value` multiset over the
|
|
483
505
|
// code-shaped subset: selectors, sigil refs, time literals,
|
|
@@ -693,6 +715,24 @@ async function main(): Promise<void> {
|
|
|
693
715
|
failed = true;
|
|
694
716
|
}
|
|
695
717
|
|
|
718
|
+
if (roleSetRegressions.length > 0) {
|
|
719
|
+
console.error(
|
|
720
|
+
`\n✗ Role-set regression vs baseline (R1): ${roleSetRegressions.length} ` +
|
|
721
|
+
`role-faithful pattern(s) now have an incomplete role set:`
|
|
722
|
+
);
|
|
723
|
+
for (const p of roleSetRegressions.slice(0, 20)) console.error(` ${p}`);
|
|
724
|
+
if (roleSetRegressions.length > 20) {
|
|
725
|
+
console.error(` … and ${roleSetRegressions.length - 20} more`);
|
|
726
|
+
}
|
|
727
|
+
console.error(
|
|
728
|
+
` (still parses, but lost/mistyped a role vs the en reference — one flip ` +
|
|
729
|
+
`moves avgRoleFidelity ~0.0013, far under its tolerance, so this is the ` +
|
|
730
|
+
`only signal that sees it. Itemize with tools/triage-r1.ts; if ` +
|
|
731
|
+
`intentional, regenerate the baseline with --save-baseline)`
|
|
732
|
+
);
|
|
733
|
+
failed = true;
|
|
734
|
+
}
|
|
735
|
+
|
|
696
736
|
if (valueRecallDrops.length > 0) {
|
|
697
737
|
console.error(
|
|
698
738
|
`\n✗ avgValueRecall dropped > ${AVG_VALUE_RECALL_DROP_TOLERANCE} in ` +
|
|
@@ -155,8 +155,7 @@ export class TestOrchestrator {
|
|
|
155
155
|
// Report regression if enabled
|
|
156
156
|
if (this.config.regression) {
|
|
157
157
|
const regressionReporter = this.reporters.find(r => r instanceof RegressionReporter) as
|
|
158
|
-
|
|
|
159
|
-
| undefined;
|
|
158
|
+
RegressionReporter | undefined;
|
|
160
159
|
|
|
161
160
|
if (regressionReporter?.hasBaseline()) {
|
|
162
161
|
const regressionResults = regressionReporter.getRegressionResults();
|
|
@@ -286,6 +285,7 @@ export class TestOrchestrator {
|
|
|
286
285
|
|
|
287
286
|
const degenerate: string[] = [];
|
|
288
287
|
const lossy: string[] = [];
|
|
288
|
+
const roleLossy: string[] = [];
|
|
289
289
|
const scores: number[] = [];
|
|
290
290
|
const precisionScores: number[] = [];
|
|
291
291
|
const multisetRecallScores: number[] = [];
|
|
@@ -331,6 +331,11 @@ export class TestOrchestrator {
|
|
|
331
331
|
if (roleFidelity !== undefined) {
|
|
332
332
|
result.roleFidelity = roleFidelity;
|
|
333
333
|
roleScores.push(roleFidelity);
|
|
334
|
+
// Per-pattern R1: the average above cannot see a single pattern
|
|
335
|
+
// losing a role (~0.0013 in a ~154-pattern language, vs the 0.02
|
|
336
|
+
// gate tolerance). Recording WHICH patterns are role-incomplete
|
|
337
|
+
// makes the flip binary and ratchetable at tolerance 0.
|
|
338
|
+
if (roleFidelity < 1) roleLossy.push(result.pattern.codeExampleId);
|
|
334
339
|
}
|
|
335
340
|
}
|
|
336
341
|
|
|
@@ -372,6 +377,7 @@ export class TestOrchestrator {
|
|
|
372
377
|
: undefined;
|
|
373
378
|
lang.degeneratePasses = degenerate.sort();
|
|
374
379
|
lang.lossyPasses = lossy.sort();
|
|
380
|
+
lang.roleLossyPatterns = roleLossy.sort();
|
|
375
381
|
}
|
|
376
382
|
}
|
|
377
383
|
|
|
@@ -471,8 +477,7 @@ export class TestOrchestrator {
|
|
|
471
477
|
*/
|
|
472
478
|
getRegressionReporter(): RegressionReporter | undefined {
|
|
473
479
|
return this.reporters.find(r => r instanceof RegressionReporter) as
|
|
474
|
-
|
|
|
475
|
-
| undefined;
|
|
480
|
+
RegressionReporter | undefined;
|
|
476
481
|
}
|
|
477
482
|
}
|
|
478
483
|
|
|
@@ -482,3 +482,106 @@ describe('RegressionReporter per-pattern parse ratchet — R5', () => {
|
|
|
482
482
|
expect(r.newFailures).toHaveLength(25);
|
|
483
483
|
});
|
|
484
484
|
});
|
|
485
|
+
|
|
486
|
+
describe('RegressionReporter role-set flip ratchet — R1 per-pattern', () => {
|
|
487
|
+
// LOCK: the avgRoleFidelity delta cannot see a single pattern losing a role —
|
|
488
|
+
// ~0.0013 in a ~154-pattern language, 15× under its 0.02 tolerance. That is
|
|
489
|
+
// not hypothetical: a `required: false` duration role on toggleSchema cost
|
|
490
|
+
// es/pl/vi the second toggle's positional destination on `toggle-aria-expanded`
|
|
491
|
+
// (`next .panel` silently became `me`) and `--regression` stayed green. This
|
|
492
|
+
// ratchet is the binary per-pattern signal that sees the flip, at tolerance 0.
|
|
493
|
+
let dir: string;
|
|
494
|
+
afterEach(() => {
|
|
495
|
+
if (dir) rmSync(dir, { recursive: true, force: true });
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
function reporterWith(baseline: Baseline): RegressionReporter {
|
|
499
|
+
dir = mkdtempSync(join(tmpdir(), 'roleflip-'));
|
|
500
|
+
const path = join(dir, 'baseline.json');
|
|
501
|
+
writeFileSync(path, JSON.stringify(baseline));
|
|
502
|
+
return new RegressionReporter(path);
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
function langWithRoleLossy(roleLossyPatterns: string[]): LanguageResults {
|
|
506
|
+
return {
|
|
507
|
+
...lang([pass('was-role-faithful'), pass('was-role-lossy'), pass('stable')], []),
|
|
508
|
+
roleLossyPatterns,
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
const baseline: Baseline = {
|
|
513
|
+
timestamp: '',
|
|
514
|
+
commit: 'base',
|
|
515
|
+
languages: {
|
|
516
|
+
ja: {
|
|
517
|
+
parseSuccess: 3,
|
|
518
|
+
parseFailure: 0,
|
|
519
|
+
parseRate: 1,
|
|
520
|
+
avgConfidence: 1,
|
|
521
|
+
avgRoleFidelity: 0.99,
|
|
522
|
+
roleLossyPatterns: ['was-role-lossy'],
|
|
523
|
+
bundleSize: undefined,
|
|
524
|
+
patterns: {
|
|
525
|
+
'was-role-faithful': { success: true, confidence: 1 },
|
|
526
|
+
'was-role-lossy': { success: true, confidence: 1 },
|
|
527
|
+
'was-failing': { success: false, confidence: 0 },
|
|
528
|
+
stable: { success: true, confidence: 1 },
|
|
529
|
+
},
|
|
530
|
+
},
|
|
531
|
+
},
|
|
532
|
+
bundles: {},
|
|
533
|
+
};
|
|
534
|
+
|
|
535
|
+
it('flags a role-faithful baseline pattern that is now role-incomplete', () => {
|
|
536
|
+
const reporter = reporterWith(baseline);
|
|
537
|
+
reporter.reportComplete(results(langWithRoleLossy(['was-role-faithful', 'was-role-lossy'])));
|
|
538
|
+
const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
|
|
539
|
+
expect(r.newRoleLossyPatterns).toEqual(['was-role-faithful']);
|
|
540
|
+
expect(r.newRoleLossyPatterns).not.toContain('was-role-lossy'); // burn-down, not regression
|
|
541
|
+
expect(r.status).toBe('regressed');
|
|
542
|
+
});
|
|
543
|
+
|
|
544
|
+
it('does not flag when the role-lossy set is unchanged', () => {
|
|
545
|
+
const reporter = reporterWith(baseline);
|
|
546
|
+
reporter.reportComplete(results(langWithRoleLossy(['was-role-lossy'])));
|
|
547
|
+
const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
|
|
548
|
+
expect(r.newRoleLossyPatterns).toEqual([]);
|
|
549
|
+
expect(r.status).not.toBe('regressed');
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
it('treats a role-lossy pattern that became faithful as improvement-neutral', () => {
|
|
553
|
+
const reporter = reporterWith(baseline);
|
|
554
|
+
reporter.reportComplete(results(langWithRoleLossy([])));
|
|
555
|
+
const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
|
|
556
|
+
expect(r.newRoleLossyPatterns).toEqual([]);
|
|
557
|
+
});
|
|
558
|
+
|
|
559
|
+
it('does not flag a pattern whose PARSE failed in the baseline (improvement path)', () => {
|
|
560
|
+
// `was-failing` did not parse in the baseline; parsing role-incompletely now
|
|
561
|
+
// is strictly better, not a regression.
|
|
562
|
+
const reporter = reporterWith(baseline);
|
|
563
|
+
reporter.reportComplete(results(langWithRoleLossy(['was-failing'])));
|
|
564
|
+
const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
|
|
565
|
+
expect(r.newRoleLossyPatterns).toEqual([]);
|
|
566
|
+
});
|
|
567
|
+
|
|
568
|
+
it('never retro-flags when the baseline has no roleLossyPatterns data', () => {
|
|
569
|
+
const noRoleData: Baseline = structuredClone(baseline);
|
|
570
|
+
delete noRoleData.languages.ja!.roleLossyPatterns;
|
|
571
|
+
const reporter = reporterWith(noRoleData);
|
|
572
|
+
reporter.reportComplete(
|
|
573
|
+
results(langWithRoleLossy(['was-role-faithful', 'was-role-lossy', 'stable']))
|
|
574
|
+
);
|
|
575
|
+
const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
|
|
576
|
+
expect(r.newRoleLossyPatterns).toEqual([]);
|
|
577
|
+
});
|
|
578
|
+
|
|
579
|
+
it('persists roleLossyPatterns when saving a baseline', () => {
|
|
580
|
+
const reporter = reporterWith(baseline);
|
|
581
|
+
const res = results(langWithRoleLossy(['was-role-lossy', 'stable']));
|
|
582
|
+
reporter.reportComplete(res);
|
|
583
|
+
reporter.saveAsBaseline(res);
|
|
584
|
+
const saved = JSON.parse(readFileSync(join(dir, 'baseline.json'), 'utf-8')) as Baseline;
|
|
585
|
+
expect(saved.languages.ja!.roleLossyPatterns).toEqual(['was-role-lossy', 'stable']);
|
|
586
|
+
});
|
|
587
|
+
});
|
|
@@ -153,6 +153,8 @@ export class RegressionReporter implements Reporter {
|
|
|
153
153
|
const newDegeneratePasses = this.findNewDegeneratePasses(langResult, baselineLang);
|
|
154
154
|
// Correctness ratchet (R0): faithful pass in baseline → lossy pass now.
|
|
155
155
|
const newLossyPasses = this.findNewLossyPasses(langResult, baselineLang);
|
|
156
|
+
// Role-set ratchet (R1): role-faithful in baseline → role-incomplete now.
|
|
157
|
+
const newRoleLossyPatterns = this.findNewRoleLossyPatterns(langResult, baselineLang);
|
|
156
158
|
// Execution ratchet (R2): faithful execution in baseline → divergent now.
|
|
157
159
|
const newExecutionFailures = this.findNewExecutionFailures(langResult, baselineLang);
|
|
158
160
|
|
|
@@ -163,6 +165,7 @@ export class RegressionReporter implements Reporter {
|
|
|
163
165
|
newFailures.length > 0 ||
|
|
164
166
|
newDegeneratePasses.length > 0 ||
|
|
165
167
|
newLossyPasses.length > 0 ||
|
|
168
|
+
newRoleLossyPatterns.length > 0 ||
|
|
166
169
|
newExecutionFailures.length > 0
|
|
167
170
|
) {
|
|
168
171
|
status = 'regressed';
|
|
@@ -185,6 +188,7 @@ export class RegressionReporter implements Reporter {
|
|
|
185
188
|
newSuccesses,
|
|
186
189
|
newDegeneratePasses,
|
|
187
190
|
newLossyPasses,
|
|
191
|
+
newRoleLossyPatterns,
|
|
188
192
|
newExecutionFailures,
|
|
189
193
|
status,
|
|
190
194
|
});
|
|
@@ -321,6 +325,39 @@ export class RegressionReporter implements Reporter {
|
|
|
321
325
|
return regressed.sort();
|
|
322
326
|
}
|
|
323
327
|
|
|
328
|
+
/**
|
|
329
|
+
* Role-set ratchet (R1): patterns that were *role-faithful* (roleFidelity 1.0,
|
|
330
|
+
* i.e. absent from the baseline's `roleLossyPatterns`) but now have an
|
|
331
|
+
* incomplete role set vs the en reference. The binary counterpart of the
|
|
332
|
+
* avgRoleFidelity delta, which cannot see a single pattern losing a role
|
|
333
|
+
* (~0.0013 in a ~154-pattern language, vs the 0.02 tolerance) — measured
|
|
334
|
+
* doing exactly that (es/pl/vi silently lost `toggle-aria-expanded`'s
|
|
335
|
+
* positional destination, `next .panel` → `me`, and the gate stayed green).
|
|
336
|
+
*
|
|
337
|
+
* Returns [] when the baseline carries no `roleLossyPatterns` data yet, so
|
|
338
|
+
* adopting the signal never retro-flags — same guard as the degenerate/lossy
|
|
339
|
+
* ratchets. A pattern already role-lossy in the baseline is the burn-down
|
|
340
|
+
* list, not a regression; one whose PARSE failed in the baseline and now
|
|
341
|
+
* parses role-incompletely is an improvement and is excluded by `wasPass`.
|
|
342
|
+
*/
|
|
343
|
+
private findNewRoleLossyPatterns(
|
|
344
|
+
current: LanguageResults,
|
|
345
|
+
baseline: {
|
|
346
|
+
patterns: Record<string, { success: boolean; confidence: number | undefined }> | undefined;
|
|
347
|
+
roleLossyPatterns?: string[] | undefined;
|
|
348
|
+
}
|
|
349
|
+
): string[] {
|
|
350
|
+
if (!baseline.roleLossyPatterns || !baseline.patterns) return [];
|
|
351
|
+
const baselineRoleLossy = new Set(baseline.roleLossyPatterns);
|
|
352
|
+
|
|
353
|
+
const regressed: string[] = [];
|
|
354
|
+
for (const id of current.roleLossyPatterns ?? []) {
|
|
355
|
+
const wasPass = baseline.patterns[id]?.success === true;
|
|
356
|
+
if (wasPass && !baselineRoleLossy.has(id)) regressed.push(id);
|
|
357
|
+
}
|
|
358
|
+
return regressed.sort();
|
|
359
|
+
}
|
|
360
|
+
|
|
324
361
|
/**
|
|
325
362
|
* Execution ratchet (R2): curated-subset patterns that executed faithfully in
|
|
326
363
|
* the baseline (scored, and not in its `executionFailures`) but diverge from
|
|
@@ -400,6 +437,9 @@ export class RegressionReporter implements Reporter {
|
|
|
400
437
|
executionFailures: langResult.executionFailures ?? undefined,
|
|
401
438
|
degeneratePasses: langResult.degeneratePasses ?? undefined,
|
|
402
439
|
lossyPasses: langResult.lossyPasses ?? undefined,
|
|
440
|
+
// R1 per-pattern — which patterns are role-incomplete vs the en
|
|
441
|
+
// reference. The role-set flip ratchet's baseline side.
|
|
442
|
+
roleLossyPatterns: langResult.roleLossyPatterns ?? undefined,
|
|
403
443
|
bundleSize: langResult.bundleSize ?? undefined,
|
|
404
444
|
patterns,
|
|
405
445
|
};
|
|
@@ -87,16 +87,21 @@ describe('shipped-examples execution gate', () => {
|
|
|
87
87
|
}, 240_000);
|
|
88
88
|
|
|
89
89
|
it('walks pages and compares handlers (sanity: extraction and both engines working)', () => {
|
|
90
|
-
// Floors well below current values (
|
|
90
|
+
// Floors well below current values (48 / 261 / 122 / 52) but far above
|
|
91
91
|
// zero: a broken walk, extractor, or engine bootstrap fails loudly here
|
|
92
92
|
// instead of making assertions 2-3 vacuously pass.
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
93
|
+
//
|
|
94
|
+
// Calibrated against the git-TRACKED corpus — the sweep ignores untracked
|
|
95
|
+
// examples/ dirs, so these numbers are the same on every machine and in
|
|
96
|
+
// CI. (The original floors were measured on a working tree with four
|
|
97
|
+
// gitignored dirs present and failed every clean checkout — #862.)
|
|
98
|
+
expect(result.pages).toBeGreaterThan(35);
|
|
99
|
+
expect(result.handlers).toBeGreaterThan(200);
|
|
100
|
+
expect(result.compared.length).toBeGreaterThan(90);
|
|
96
101
|
// Vacuous (empty-vs-empty) pairs are NOT parity evidence — the floor is on
|
|
97
102
|
// real, non-empty signature matches.
|
|
98
103
|
const realMatches = result.compared.filter(c => c.match && !c.vacuous).length;
|
|
99
|
-
expect(realMatches).toBeGreaterThan(
|
|
104
|
+
expect(realMatches).toBeGreaterThan(40);
|
|
100
105
|
});
|
|
101
106
|
|
|
102
107
|
it('has no NEW divergence from upstream outside the allowlist', () => {
|
|
@@ -45,6 +45,7 @@
|
|
|
45
45
|
|
|
46
46
|
import fs from 'node:fs';
|
|
47
47
|
import path from 'node:path';
|
|
48
|
+
import { execFileSync } from 'node:child_process';
|
|
48
49
|
import { createHash } from 'node:crypto';
|
|
49
50
|
import { createRequire } from 'node:module';
|
|
50
51
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
@@ -168,22 +169,33 @@ export function triggerEventOf(source: string): string | null {
|
|
|
168
169
|
return m ? (m[1] ?? null) : null;
|
|
169
170
|
}
|
|
170
171
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
172
|
+
/**
|
|
173
|
+
* Git-TRACKED .html files under `root` (repo-relative), as absolute paths.
|
|
174
|
+
*
|
|
175
|
+
* Tracked, not on-disk, because the corpus must be the SHIPPED tree: four
|
|
176
|
+
* `examples/` dirs are gitignored (experiments/, playground/,
|
|
177
|
+
* vite-plugin-test/, vite-plugin-multilingual/), and the disk walk this
|
|
178
|
+
* replaced counted whatever happened to be on the machine. That is how the
|
|
179
|
+
* gate's sanity floors and allowlist got calibrated against handlers that
|
|
180
|
+
* were never shipped — and why it failed on every clean checkout the first
|
|
181
|
+
* time CI ran it (#862). `git ls-files` sees the same corpus everywhere.
|
|
182
|
+
*
|
|
183
|
+
* git being unavailable is a THROW, not a fallback to the disk walk — a
|
|
184
|
+
* silent fallback would resurrect the corpus drift this exists to kill.
|
|
185
|
+
* A tracked file missing from disk (an uncommitted local deletion) is
|
|
186
|
+
* skipped: the working tree is mid-edit, and in CI checkout == index.
|
|
187
|
+
*/
|
|
188
|
+
function trackedHtml(repoRoot: string, root: string): string[] {
|
|
189
|
+
const out = execFileSync('git', ['ls-files', '-z', '--', root], {
|
|
190
|
+
cwd: repoRoot,
|
|
191
|
+
encoding: 'utf8',
|
|
192
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
193
|
+
});
|
|
194
|
+
return out
|
|
195
|
+
.split('\0')
|
|
196
|
+
.filter(rel => rel.endsWith('.html'))
|
|
197
|
+
.map(rel => path.join(repoRoot, rel))
|
|
198
|
+
.filter(full => fs.existsSync(full));
|
|
187
199
|
}
|
|
188
200
|
|
|
189
201
|
/** Extract the page's `[_]` handlers in document order. */
|
|
@@ -398,9 +410,9 @@ async function runHandlerOnEngine(
|
|
|
398
410
|
}
|
|
399
411
|
|
|
400
412
|
/**
|
|
401
|
-
* The sweep: walk `examples
|
|
402
|
-
* filters (each skip reasoned), and execute every
|
|
403
|
-
* engines.
|
|
413
|
+
* The sweep: walk the git-tracked `examples/**` corpus, extract handlers,
|
|
414
|
+
* apply the fair-denominator filters (each skip reasoned), and execute every
|
|
415
|
+
* eligible handler on both engines.
|
|
404
416
|
*/
|
|
405
417
|
export async function runShippedExamplesExecution(opts?: {
|
|
406
418
|
roots?: string[];
|
|
@@ -417,7 +429,7 @@ export async function runShippedExamplesExecution(opts?: {
|
|
|
417
429
|
let handlers = 0;
|
|
418
430
|
|
|
419
431
|
for (const root of roots) {
|
|
420
|
-
for (const full of
|
|
432
|
+
for (const full of trackedHtml(repoRoot, root)) {
|
|
421
433
|
const rel = path.relative(repoRoot, full);
|
|
422
434
|
const html = fs.readFileSync(full, 'utf8');
|
|
423
435
|
const pageHandlers = extractHandlers(rel, html);
|
|
@@ -308,6 +308,18 @@ export interface LanguageResults {
|
|
|
308
308
|
*/
|
|
309
309
|
lossyPasses?: string[] | undefined;
|
|
310
310
|
|
|
311
|
+
/**
|
|
312
|
+
* R1 — pattern IDs that parse with roleFidelity < 1 (the `action.role:valueType`
|
|
313
|
+
* set vs the English reference is incomplete). The per-pattern counterpart of
|
|
314
|
+
* `avgRoleFidelity`: a single pattern losing a role moves the average by
|
|
315
|
+
* ~0.0013 in a ~154-pattern language, which the 0.02 avg tolerance swallows —
|
|
316
|
+
* measured doing exactly that (es/pl/vi lost `toggle-aria-expanded`'s
|
|
317
|
+
* positional destination to a schema change and the gate stayed green). The
|
|
318
|
+
* role-set flip ratchet compares this list against the baseline's at
|
|
319
|
+
* tolerance 0.
|
|
320
|
+
*/
|
|
321
|
+
roleLossyPatterns?: string[] | undefined;
|
|
322
|
+
|
|
311
323
|
/** Duration in ms */
|
|
312
324
|
duration: number;
|
|
313
325
|
|
|
@@ -378,6 +390,8 @@ export interface Baseline {
|
|
|
378
390
|
degeneratePasses?: string[] | undefined;
|
|
379
391
|
/** Pattern IDs that pass *lossily* (fidelity in [0.5, 1.0) — drop ≥1 command). */
|
|
380
392
|
lossyPasses?: string[] | undefined;
|
|
393
|
+
/** R1 — pattern IDs whose role set is incomplete vs the en reference (roleFidelity < 1). */
|
|
394
|
+
roleLossyPatterns?: string[] | undefined;
|
|
381
395
|
bundleSize: number | undefined;
|
|
382
396
|
/** Pattern-level results for detailed tracking */
|
|
383
397
|
patterns: Record<string, { success: boolean; confidence: number | undefined }> | undefined;
|
|
@@ -434,6 +448,14 @@ export interface RegressionResult {
|
|
|
434
448
|
* misses. Empty when the baseline has no lossy data yet (never retro-flags).
|
|
435
449
|
*/
|
|
436
450
|
newLossyPasses: string[];
|
|
451
|
+
/**
|
|
452
|
+
* R1 — patterns that were *role-faithful* (roleFidelity 1.0) in the baseline
|
|
453
|
+
* but now have an incomplete role set (roleFidelity < 1). The binary
|
|
454
|
+
* per-pattern signal the avgRoleFidelity tolerance cannot provide: one flip
|
|
455
|
+
* moves the average by ~0.0013, far under 0.02. Empty when the baseline has
|
|
456
|
+
* no `roleLossyPatterns` data yet (never retro-flags).
|
|
457
|
+
*/
|
|
458
|
+
newRoleLossyPatterns: string[];
|
|
437
459
|
status: 'improved' | 'regressed' | 'unchanged';
|
|
438
460
|
}
|
|
439
461
|
|
|
@@ -19,7 +19,7 @@ import { describe, it, expect, beforeAll } from 'vitest';
|
|
|
19
19
|
import { ExecutionValidator, EXECUTION_SUBSET, loadExecutionSubset } from './execution-validator';
|
|
20
20
|
|
|
21
21
|
describe('R2 execution subset (lock)', () => {
|
|
22
|
-
it('contains exactly the
|
|
22
|
+
it('contains exactly the 48 curated patterns', () => {
|
|
23
23
|
// Changing this list recalibrates avgExecutionFidelity for every language.
|
|
24
24
|
// If you expand the subset, regenerate the baseline (--save-baseline) in
|
|
25
25
|
// the SAME PR and update this lock.
|
|
@@ -119,6 +119,7 @@ describe('R2 execution subset (lock)', () => {
|
|
|
119
119
|
'chained-access-possessive-dot',
|
|
120
120
|
'hide-with-transition',
|
|
121
121
|
'show-with-transition',
|
|
122
|
+
'swap-content',
|
|
122
123
|
// Wave 10 (SOV literal-role-extraction arc, PRs #560/#561): the two R2
|
|
123
124
|
// blockers — the fronted append content literal (bogus `event` role in
|
|
124
125
|
// the body-clause marker lookup) and the trailing bare increment amount
|
|
@@ -203,6 +203,16 @@ export const EXECUTION_SUBSET: readonly string[] = [
|
|
|
203
203
|
// Fixture adds `<div id="score">0</div>` (appended last).
|
|
204
204
|
'append-content',
|
|
205
205
|
'increment-by-amount',
|
|
206
|
+
// Expansion wave 11: `swap-content` — `swap #a with #b`. R2 stayed at 1.0
|
|
207
|
+
// through the ENTIRE swap breakage (the descriptor emitted modifiers
|
|
208
|
+
// SwapCommand never reads, so every language including English threw
|
|
209
|
+
// `could not parse arguments`), and then through eight languages binding the
|
|
210
|
+
// operands BACKWARDS once #845 made it execute. Both were invisible here for
|
|
211
|
+
// one reason: swap-content was not in this list. It is the missing gate for
|
|
212
|
+
// R3 family 6, and it turns the operand fix from "trust me" into a row.
|
|
213
|
+
// Fixture adds `#a`/`#b` with distinguishable content (appended last, so
|
|
214
|
+
// existing snapshot indices are preserved).
|
|
215
|
+
'swap-content',
|
|
206
216
|
];
|
|
207
217
|
|
|
208
218
|
/**
|
|
@@ -229,6 +239,8 @@ const FIXTURE_HTML = `<!DOCTYPE html><html><body>
|
|
|
229
239
|
<div id="sr-announce"></div>
|
|
230
240
|
<ul id="list"></ul>
|
|
231
241
|
<div id="score">0</div>
|
|
242
|
+
<div id="a">AAA</div>
|
|
243
|
+
<div id="b">BBB</div>
|
|
232
244
|
</body></html>`;
|
|
233
245
|
|
|
234
246
|
/** Per-pattern fixture preconditions (applied identically for every language). */
|