@hyperfixi/testing-framework 3.2.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +203 -1
- package/package.json +9 -9
- package/src/agent-bench/README.md +48 -1
- package/src/agent-bench/agent-bench.test.ts +7 -0
- package/src/agent-bench/harness.ts +39 -50
- package/src/multilingual/README.md +30 -11
- package/src/multilingual/bare-render-fidelity.ts +1 -1
- package/src/multilingual/cli.ts +5 -3
- package/src/multilingual/direct-path-shapes-gate.ts +56 -0
- package/src/multilingual/direct-path-shapes.1.test.ts +12 -0
- package/src/multilingual/direct-path-shapes.2.test.ts +12 -0
- package/src/multilingual/direct-path-shapes.3.test.ts +12 -0
- package/src/multilingual/direct-path-shapes.cases.json +2621 -0
- package/src/multilingual/direct-path-shapes.ts +459 -0
- package/src/multilingual/engine-parser-parity.test.ts +67 -0
- package/src/multilingual/orchestrator.ts +2 -1
- package/src/multilingual/render-fidelity.ts +1 -1
- package/src/multilingual/shipped-examples-execution.test.ts +44 -33
- package/src/multilingual/shipped-examples-execution.ts +55 -75
- package/src/multilingual/shipped-sources-engine.test.ts +106 -0
- package/src/multilingual/shipped-sources-localized.test.ts +87 -0
- package/src/multilingual/shipped-sources-validity.ts +127 -74
- package/src/multilingual/validators/execution-validator.test.ts +81 -23
- package/src/multilingual/validators/execution-validator.ts +140 -94
- package/src/multilingual/validators/parse-validator.ts +7 -20
- package/src/multilingual/value-matrix.accepted.test.ts +8 -11
- package/src/multilingual/value-matrix.isolation.test.ts +7 -6
- package/src/multilingual/value-matrix.ts +72 -86
- package/src/multilingual/shipped-sources-validity.test.ts +0 -95
|
@@ -3,20 +3,26 @@
|
|
|
3
3
|
* why and the execution model).
|
|
4
4
|
*
|
|
5
5
|
* Executes every eligible `_="…"` handler shipped in `examples/**` on BOTH
|
|
6
|
-
* hyperfixi and the real `hyperscript.org` engine, in jsdom, and
|
|
7
|
-
* divergence of their DOM effect signatures. Upstream is the
|
|
8
|
-
* oracle — the same role R4 gives it for validity. This is the gate
|
|
9
|
-
* have caught the #785 defect (a conditional body running
|
|
10
|
-
* shipped page) on BEHAVIOR: every parse-level gate stayed
|
|
11
|
-
* shipped.
|
|
6
|
+
* `@hyperfixi/engine` and the real `hyperscript.org` engine, in jsdom, and
|
|
7
|
+
* ratchets on divergence of their DOM effect signatures. Upstream is the
|
|
8
|
+
* behavioral oracle — the same role R4 gives it for validity. This is the gate
|
|
9
|
+
* that would have caught the #785 defect (a conditional body running
|
|
10
|
+
* unconditionally on a shipped page) on BEHAVIOR: every parse-level gate stayed
|
|
11
|
+
* green while it shipped. It is also the gate for a handler that parses on both
|
|
12
|
+
* engines and runs differently, which the parse-level list in
|
|
13
|
+
* shipped-sources-engine.test.ts cannot see.
|
|
12
14
|
*
|
|
13
15
|
* Assertions, matching the shipped-sources gate:
|
|
14
16
|
* 1. sanity — pages walked, handlers extracted, comparisons actually ran
|
|
15
17
|
* (guards the silent-zero failure mode);
|
|
16
|
-
* 2. no NEW divergence appears outside the committed allowlist
|
|
18
|
+
* 2. no NEW divergence appears outside the committed allowlist
|
|
19
|
+
* (`allowedEngineDivergences`);
|
|
17
20
|
* 3. no allowlisted key has silently converged (stale entries must be
|
|
18
21
|
* removed so the list only ever ratchets down).
|
|
19
22
|
*
|
|
23
|
+
* (Until Phase C4 the same three also ran for a lane on `@hyperfixi/core`'s
|
|
24
|
+
* runtime, against its own allowlist. It left with core's engine.)
|
|
25
|
+
*
|
|
20
26
|
* To update after an intentional change: re-run and regenerate
|
|
21
27
|
* `baselines/shipped-examples-execution.json` (the allowlist key embeds a
|
|
22
28
|
* source hash, so FIXING a handler changes its key and assertion 3 forces the
|
|
@@ -31,8 +37,8 @@
|
|
|
31
37
|
* (happy-dom) the DOM constructors already exist on globalThis, the harness's
|
|
32
38
|
* globals bootstrap refuses to overwrite what it does not own, and both
|
|
33
39
|
* engines then bind happy-dom's constructors — every instanceof against a
|
|
34
|
-
* jsdom element fails and every
|
|
35
|
-
* (measured: 0 real matches under happy-dom vs 74 under node).
|
|
40
|
+
* jsdom element fails and every signature comes back empty
|
|
41
|
+
* (measured on core's lane: 0 real matches under happy-dom vs 74 under node).
|
|
36
42
|
*/
|
|
37
43
|
|
|
38
44
|
import { readFileSync } from 'node:fs';
|
|
@@ -47,7 +53,8 @@ import {
|
|
|
47
53
|
} from './shipped-examples-execution';
|
|
48
54
|
|
|
49
55
|
interface AllowlistDoc {
|
|
50
|
-
|
|
56
|
+
/** `@hyperfixi/engine` against upstream. */
|
|
57
|
+
allowedEngineDivergences: Array<{
|
|
51
58
|
key: string;
|
|
52
59
|
file: string;
|
|
53
60
|
event: string;
|
|
@@ -61,9 +68,9 @@ const baselinePath = path.resolve(
|
|
|
61
68
|
'../../baselines/shipped-examples-execution.json'
|
|
62
69
|
);
|
|
63
70
|
const allowlist = JSON.parse(readFileSync(baselinePath, 'utf8')) as AllowlistDoc;
|
|
64
|
-
const
|
|
71
|
+
const allowedOnEngine = new Set(allowlist.allowedEngineDivergences.map(e => e.key));
|
|
65
72
|
|
|
66
|
-
describe('shipped-examples execution gate', () => {
|
|
73
|
+
describe('shipped-examples execution gate: @hyperfixi/engine against upstream', () => {
|
|
67
74
|
let result: ExecutionParityResult;
|
|
68
75
|
|
|
69
76
|
beforeAll(async () => {
|
|
@@ -76,10 +83,11 @@ describe('shipped-examples execution gate', () => {
|
|
|
76
83
|
const r = s.reason.split(':')[0] ?? s.reason;
|
|
77
84
|
reasons.set(r, (reasons.get(r) ?? 0) + 1);
|
|
78
85
|
}
|
|
79
|
-
const
|
|
86
|
+
const onEngine = result.engineCompared;
|
|
80
87
|
console.log(
|
|
81
88
|
`[shipped-examples-execution] pages=${result.pages} handlers=${result.handlers} ` +
|
|
82
|
-
`compared=${
|
|
89
|
+
`compared=${onEngine.length} (vacuous=${onEngine.filter(c => c.vacuous).length}) ` +
|
|
90
|
+
`diverging=${onEngine.filter(c => !c.match).length} skipped=${result.skipped.length}`
|
|
83
91
|
);
|
|
84
92
|
for (const [r, n] of [...reasons].sort((a, b) => b[1] - a[1])) {
|
|
85
93
|
console.log(`[shipped-examples-execution] skip ×${n}: ${r}`);
|
|
@@ -87,9 +95,9 @@ describe('shipped-examples execution gate', () => {
|
|
|
87
95
|
}, 240_000);
|
|
88
96
|
|
|
89
97
|
it('walks pages and compares handlers (sanity: extraction and both engines working)', () => {
|
|
90
|
-
// Floors well below current values
|
|
91
|
-
//
|
|
92
|
-
//
|
|
98
|
+
// Floors well below current values but far above zero: a broken walk,
|
|
99
|
+
// extractor, or engine bootstrap fails loudly here instead of making
|
|
100
|
+
// assertions 2-3 vacuously pass.
|
|
93
101
|
//
|
|
94
102
|
// Calibrated against the git-TRACKED corpus — the sweep ignores untracked
|
|
95
103
|
// examples/ dirs, so these numbers are the same on every machine and in
|
|
@@ -97,46 +105,49 @@ describe('shipped-examples execution gate', () => {
|
|
|
97
105
|
// gitignored dirs present and failed every clean checkout — #862.)
|
|
98
106
|
expect(result.pages).toBeGreaterThan(35);
|
|
99
107
|
expect(result.handlers).toBeGreaterThan(200);
|
|
100
|
-
expect(result.
|
|
108
|
+
expect(result.engineCompared.length).toBeGreaterThan(90);
|
|
101
109
|
// Vacuous (empty-vs-empty) pairs are NOT parity evidence — the floor is on
|
|
102
110
|
// real, non-empty signature matches.
|
|
103
|
-
const realMatches = result.
|
|
104
|
-
expect(realMatches).toBeGreaterThan(
|
|
111
|
+
const realMatches = result.engineCompared.filter(c => c.match && !c.vacuous).length;
|
|
112
|
+
expect(realMatches).toBeGreaterThan(60);
|
|
105
113
|
});
|
|
106
114
|
|
|
107
115
|
it('has no NEW divergence from upstream outside the allowlist', () => {
|
|
108
|
-
const unexpected = result.
|
|
116
|
+
const unexpected = result.engineCompared.filter(c => !c.match && !allowedOnEngine.has(c.key));
|
|
109
117
|
expect(
|
|
110
118
|
unexpected,
|
|
111
119
|
unexpected.length
|
|
112
|
-
? `\nShipped handlers whose DOM effect DIVERGES from the hyperscript.org
|
|
113
|
-
`(fix the
|
|
120
|
+
? `\nShipped handlers whose DOM effect on @hyperfixi/engine DIVERGES from the hyperscript.org ` +
|
|
121
|
+
`engine\n(the engine follows upstream's source: fix the engine, or allowlist with a reason ` +
|
|
122
|
+
`under allowedEngineDivergences):\n` +
|
|
114
123
|
unexpected
|
|
115
124
|
.map(
|
|
116
125
|
f =>
|
|
117
126
|
` [${f.key}]\n` +
|
|
118
127
|
` "${f.excerpt}"\n` +
|
|
119
|
-
`
|
|
120
|
-
` upstream
|
|
128
|
+
` engine : ${JSON.stringify(f.engineEffects).slice(0, 300)}\n` +
|
|
129
|
+
` upstream: ${JSON.stringify(f.upstreamEffects).slice(0, 300)}`
|
|
121
130
|
)
|
|
122
131
|
.join('\n') +
|
|
123
|
-
`\n\nTriage guidance: an EMPTY
|
|
124
|
-
`means
|
|
125
|
-
`
|
|
126
|
-
`existing family reasons in baselines/shipped-examples-execution.json before adding a new one.`
|
|
132
|
+
`\n\nTriage guidance: an EMPTY engine signature with a non-empty upstream one usually\n` +
|
|
133
|
+
`means the engine silently dropped behavior (the #785 class).\n` +
|
|
134
|
+
`One source on both, on one page: npx tsx packages/engine/tools/probe.mts '<source>'`
|
|
127
135
|
: ''
|
|
128
136
|
).toEqual([]);
|
|
129
137
|
});
|
|
130
138
|
|
|
131
139
|
it('has no stale allowlist entries (a now-converged handler must be removed so the list ratchets down)', () => {
|
|
132
|
-
const stillDiverging = new Set(result.
|
|
133
|
-
const stale = allowlist.
|
|
140
|
+
const stillDiverging = new Set(result.engineCompared.filter(c => !c.match).map(c => c.key));
|
|
141
|
+
const stale = allowlist.allowedEngineDivergences
|
|
142
|
+
.map(e => e.key)
|
|
143
|
+
.filter(k => !stillDiverging.has(k));
|
|
134
144
|
expect(
|
|
135
145
|
stale,
|
|
136
146
|
stale.length
|
|
137
|
-
? `\nThese
|
|
147
|
+
? `\nThese handlers no longer diverge on the engine (fixed, or edited — the key embeds a\n` +
|
|
138
148
|
`source hash; or no longer eligible, in which case the coverage loss should be deliberate).\n` +
|
|
139
|
-
`Remove them from baselines/shipped-examples-execution.json:\n
|
|
149
|
+
`Remove them from allowedEngineDivergences in baselines/shipped-examples-execution.json:\n ` +
|
|
150
|
+
stale.join('\n ')
|
|
140
151
|
: ''
|
|
141
152
|
).toEqual([]);
|
|
142
153
|
});
|
|
@@ -9,24 +9,27 @@
|
|
|
9
9
|
*
|
|
10
10
|
* This gate executes the handlers we actually ship. For each `_="…"` attribute
|
|
11
11
|
* in `examples/**`, the full page is loaded into two jsdom instances — one
|
|
12
|
-
* processed by hyperfixi
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
12
|
+
* processed by `@hyperfixi/engine`, one by the real `hyperscript.org` engine
|
|
13
|
+
* (`processNode` on both) — the handler's trigger event is dispatched, and the
|
|
14
|
+
* resulting DOM effect signatures (../effect-signature.ts, shared with the R2
|
|
15
|
+
* execution ratchet) are diffed against each other. Upstream plays the role R4
|
|
16
|
+
* gives it for validity: the oracle. A divergence means the engine's runtime
|
|
17
|
+
* behavior differs from upstream's ON A SHIPPED PAGE — exactly the #785/#786
|
|
18
|
+
* failure mode, caught on behavior instead of by luck. The parse-level list of
|
|
19
|
+
* what the engine rejects (shipped-sources-engine.test.ts) cannot see a handler
|
|
20
|
+
* that parses on both and runs differently; this can. A handler upstream
|
|
21
|
+
* accepts and the engine rejects is compared too, and diverges.
|
|
22
|
+
*
|
|
23
|
+
* (Until Phase C4 a second lane ran each handler on `@hyperfixi/core`'s runtime
|
|
24
|
+
* against the same oracle. It left with core's engine; its ten allowlisted
|
|
25
|
+
* divergences were core's.)
|
|
21
26
|
*
|
|
22
27
|
* ## Fair denominator (mirrors R4)
|
|
23
|
-
* A handler is only compared when
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* timer / navigation constructs. Every exclusion is recorded with a reason —
|
|
29
|
-
* the skip list is part of the result, never silent (`no silent caps`).
|
|
28
|
+
* A handler is only compared when upstream parses it (an engine-only form, such
|
|
29
|
+
* as `new X()`, has no oracle). It must also be deterministically executable in
|
|
30
|
+
* jsdom: triggered by a dispatchable event, free of network / timer /
|
|
31
|
+
* navigation constructs. Every exclusion is recorded with a reason — the skip
|
|
32
|
+
* list is part of the result, never silent (`no silent caps`).
|
|
30
33
|
*
|
|
31
34
|
* ## Execution model
|
|
32
35
|
* Per handler and engine: a FRESH jsdom of the whole page, every eligible
|
|
@@ -126,12 +129,12 @@ export interface SkippedHandler extends ShippedHandler {
|
|
|
126
129
|
reason: string;
|
|
127
130
|
}
|
|
128
131
|
|
|
129
|
-
/** One
|
|
130
|
-
export interface
|
|
132
|
+
/** One handler run on upstream and on `@hyperfixi/engine`; signatures either match or not. */
|
|
133
|
+
export interface EngineComparedHandler extends ShippedHandler {
|
|
131
134
|
/** Stable key: file + source hash + event. Fixing a source changes its key. */
|
|
132
135
|
key: string;
|
|
133
136
|
event: string;
|
|
134
|
-
|
|
137
|
+
engineEffects: string[];
|
|
135
138
|
upstreamEffects: string[];
|
|
136
139
|
match: boolean;
|
|
137
140
|
/**
|
|
@@ -149,10 +152,10 @@ export interface ExecutionParityResult {
|
|
|
149
152
|
pages: number;
|
|
150
153
|
/** Handlers found (before eligibility). */
|
|
151
154
|
handlers: number;
|
|
152
|
-
/** Handlers executed on both engines. */
|
|
153
|
-
compared: ComparedHandler[];
|
|
154
155
|
/** Handlers excluded, each with its reason. */
|
|
155
156
|
skipped: SkippedHandler[];
|
|
157
|
+
/** Handlers upstream accepts, executed on upstream and on `@hyperfixi/engine`. */
|
|
158
|
+
engineCompared: EngineComparedHandler[];
|
|
156
159
|
}
|
|
157
160
|
|
|
158
161
|
/** Stable key for one handler execution. */
|
|
@@ -211,14 +214,14 @@ export function extractHandlers(file: string, html: string): ShippedHandler[] {
|
|
|
211
214
|
|
|
212
215
|
/** Minimal engine surfaces, injected by `initEngines`. */
|
|
213
216
|
export interface Engines {
|
|
214
|
-
/**
|
|
215
|
-
compileClean(source: string): boolean;
|
|
216
|
-
/** upstream parse check (fair-denominator side 2). Returns error strings; [] = valid. */
|
|
217
|
+
/** upstream parse check (the fair denominator). Returns error strings; [] = valid. */
|
|
217
218
|
upstreamErrors(source: string): string[];
|
|
218
|
-
/** Install a handler on an element via hyperfixi (the public eval surface). */
|
|
219
|
-
hyperfixiInstall(source: string, el: Element): Promise<void>;
|
|
220
219
|
/** Install a handler on an element via the upstream engine. */
|
|
221
220
|
upstreamInstall(el: Element): void;
|
|
221
|
+
/** `@hyperfixi/engine`'s parse errors for a source; [] = it parses. */
|
|
222
|
+
engineErrors(source: string): string[];
|
|
223
|
+
/** Install a handler on an element via `@hyperfixi/engine`. */
|
|
224
|
+
engineInstall(el: Element): void;
|
|
222
225
|
}
|
|
223
226
|
|
|
224
227
|
/**
|
|
@@ -288,22 +291,6 @@ export async function initEngines(): Promise<Engines> {
|
|
|
288
291
|
installRejectionTrap();
|
|
289
292
|
installGlobals(new JSDOM('<!doctype html><html><body></body></html>'));
|
|
290
293
|
|
|
291
|
-
// Install goes through parse + a FRESH Runtime per handler with the element
|
|
292
|
-
// as context — the same shape the browser attribute processor uses, and the
|
|
293
|
-
// same shape the R2 validator proved stable across per-page document swaps.
|
|
294
|
-
// (The api singleton `hyperscript.eval` binds document-dependent state at
|
|
295
|
-
// first use, which is correct in a browser — document identity never changes
|
|
296
|
-
// within a realm — but silently resolves later PAGES' selectors against the
|
|
297
|
-
// first page here. Measured: a two-page probe no-opped page 2's toggle.)
|
|
298
|
-
const core = (await import('@hyperfixi/core')) as unknown as {
|
|
299
|
-
hyperscript: {
|
|
300
|
-
compileSync(code: string): { ok: boolean; errors?: Array<{ message: string }> };
|
|
301
|
-
};
|
|
302
|
-
parse(code: string): { success: boolean; node?: unknown };
|
|
303
|
-
Runtime: new () => { execute(ast: unknown, ctx: unknown): Promise<unknown> };
|
|
304
|
-
createContext(el: HTMLElement): unknown;
|
|
305
|
-
};
|
|
306
|
-
|
|
307
294
|
const require = createRequire(import.meta.url);
|
|
308
295
|
const esm = path.join(path.dirname(require.resolve('hyperscript.org')), '_hyperscript.esm.js');
|
|
309
296
|
const hs = (await import(pathToFileURL(esm).href)).default as {
|
|
@@ -311,15 +298,20 @@ export async function initEngines(): Promise<Engines> {
|
|
|
311
298
|
processNode(el: Node): void;
|
|
312
299
|
};
|
|
313
300
|
|
|
301
|
+
const engine = await import('@hyperfixi/engine');
|
|
302
|
+
engine.register(...engine.everything);
|
|
303
|
+
|
|
314
304
|
return {
|
|
315
|
-
|
|
305
|
+
engineErrors(source) {
|
|
316
306
|
try {
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
return false;
|
|
307
|
+
return engine.api.parse(source).errors.map(e => e.message.split('\n')[0] ?? '');
|
|
308
|
+
} catch (e) {
|
|
309
|
+
return ['threw: ' + (e as Error).message.split('\n')[0]];
|
|
321
310
|
}
|
|
322
311
|
},
|
|
312
|
+
engineInstall(el) {
|
|
313
|
+
engine.api.processNode(el);
|
|
314
|
+
},
|
|
323
315
|
upstreamErrors(source) {
|
|
324
316
|
try {
|
|
325
317
|
return (hs.parse(source)?.errors ?? []).map(e => e.message);
|
|
@@ -327,14 +319,6 @@ export async function initEngines(): Promise<Engines> {
|
|
|
327
319
|
return ['threw: ' + (e as Error).message.split('\n')[0]];
|
|
328
320
|
}
|
|
329
321
|
},
|
|
330
|
-
async hyperfixiInstall(source, el) {
|
|
331
|
-
const parsed = core.parse(source);
|
|
332
|
-
if (!parsed.success || !parsed.node) {
|
|
333
|
-
throw new Error('parse failed at install (eligibility should have caught this)');
|
|
334
|
-
}
|
|
335
|
-
const runtime = new core.Runtime();
|
|
336
|
-
await runtime.execute(parsed.node, core.createContext(el as HTMLElement));
|
|
337
|
-
},
|
|
338
322
|
upstreamInstall(el) {
|
|
339
323
|
hs.processNode(el);
|
|
340
324
|
},
|
|
@@ -412,7 +396,7 @@ async function runHandlerOnEngine(
|
|
|
412
396
|
/**
|
|
413
397
|
* The sweep: walk the git-tracked `examples/**` corpus, extract handlers,
|
|
414
398
|
* apply the fair-denominator filters (each skip reasoned), and execute every
|
|
415
|
-
* eligible handler on
|
|
399
|
+
* eligible handler on upstream and on the engine.
|
|
416
400
|
*/
|
|
417
401
|
export async function runShippedExamplesExecution(opts?: {
|
|
418
402
|
roots?: string[];
|
|
@@ -423,8 +407,8 @@ export async function runShippedExamplesExecution(opts?: {
|
|
|
423
407
|
const roots = opts?.roots ?? DEFAULT_ROOTS;
|
|
424
408
|
const engines = opts?.engines ?? (await initEngines());
|
|
425
409
|
|
|
426
|
-
const compared: ComparedHandler[] = [];
|
|
427
410
|
const skipped: SkippedHandler[] = [];
|
|
411
|
+
const engineCompared: EngineComparedHandler[] = [];
|
|
428
412
|
let pages = 0;
|
|
429
413
|
let handlers = 0;
|
|
430
414
|
|
|
@@ -437,7 +421,8 @@ export async function runShippedExamplesExecution(opts?: {
|
|
|
437
421
|
pages++;
|
|
438
422
|
handlers += pageHandlers.length;
|
|
439
423
|
|
|
440
|
-
|
|
424
|
+
// The denominator: dispatchable, deterministic, and upstream accepts it.
|
|
425
|
+
const oracle: Array<ShippedHandler & { event: string }> = [];
|
|
441
426
|
for (const h of pageHandlers) {
|
|
442
427
|
if (!h.event) {
|
|
443
428
|
skipped.push({ ...h, reason: 'not an `on <event>` handler' });
|
|
@@ -452,21 +437,14 @@ export async function runShippedExamplesExecution(opts?: {
|
|
|
452
437
|
skipped.push({ ...h, reason: disq.reason });
|
|
453
438
|
continue;
|
|
454
439
|
}
|
|
455
|
-
if (!engines.compileClean(h.source)) {
|
|
456
|
-
skipped.push({
|
|
457
|
-
...h,
|
|
458
|
-
reason: 'hyperfixi does not compile it clean (shipped-sources gate territory)',
|
|
459
|
-
});
|
|
460
|
-
continue;
|
|
461
|
-
}
|
|
462
440
|
const upstreamErrs = engines.upstreamErrors(h.source);
|
|
463
441
|
if (upstreamErrs.length > 0) {
|
|
464
442
|
skipped.push({ ...h, reason: `upstream rejects it (no oracle): ${upstreamErrs[0]}` });
|
|
465
443
|
continue;
|
|
466
444
|
}
|
|
467
|
-
|
|
445
|
+
oracle.push(h as ShippedHandler & { event: string });
|
|
468
446
|
}
|
|
469
|
-
if (
|
|
447
|
+
if (oracle.length === 0) continue;
|
|
470
448
|
|
|
471
449
|
// Both engines log runtime errors to the console during dispatch
|
|
472
450
|
// (COMMAND FAILED etc.). That is expected data here — the effect
|
|
@@ -481,17 +459,19 @@ export async function runShippedExamplesExecution(opts?: {
|
|
|
481
459
|
const noop = () => {};
|
|
482
460
|
console.log = console.warn = console.error = console.debug = noop;
|
|
483
461
|
try {
|
|
484
|
-
for (const h of
|
|
485
|
-
const
|
|
486
|
-
|
|
487
|
-
);
|
|
488
|
-
const theirs = await runHandlerOnEngine(html, eligible, h, (_hh, el) =>
|
|
462
|
+
for (const h of oracle) {
|
|
463
|
+
const rejected = engines.engineErrors(h.source)[0];
|
|
464
|
+
const theirs = await runHandlerOnEngine(html, oracle, h, (_hh, el) =>
|
|
489
465
|
engines.upstreamInstall(el)
|
|
490
466
|
);
|
|
491
|
-
|
|
467
|
+
const ours =
|
|
468
|
+
rejected !== undefined
|
|
469
|
+
? [`<engine rejects the source: ${rejected}>`]
|
|
470
|
+
: await runHandlerOnEngine(html, oracle, h, (_hh, el) => engines.engineInstall(el));
|
|
471
|
+
engineCompared.push({
|
|
492
472
|
...h,
|
|
493
473
|
key: keyFor(h),
|
|
494
|
-
|
|
474
|
+
engineEffects: ours,
|
|
495
475
|
upstreamEffects: theirs,
|
|
496
476
|
match: JSON.stringify(ours) === JSON.stringify(theirs),
|
|
497
477
|
vacuous: ours.length === 0 && theirs.length === 0,
|
|
@@ -507,5 +487,5 @@ export async function runShippedExamplesExecution(opts?: {
|
|
|
507
487
|
}
|
|
508
488
|
}
|
|
509
489
|
|
|
510
|
-
return { pages, handlers,
|
|
490
|
+
return { pages, handlers, skipped, engineCompared };
|
|
511
491
|
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shipped sources on `@hyperfixi/engine`: every English source must parse.
|
|
3
|
+
*
|
|
4
|
+
* Every hyperscript source this repository ships in English (`examples/` and
|
|
5
|
+
* the doc trees; a source under a non-English `lang` is the localized gate's)
|
|
6
|
+
* is put to the engine's parser. The ones it rejects are listed in
|
|
7
|
+
* `baselines/shipped-sources-engine.json`, each with the reason it is still
|
|
8
|
+
* there. Three assertions, as the sibling gates:
|
|
9
|
+
* 1. sanity: sources were found and the engine reads most of them;
|
|
10
|
+
* 2. no NEW rejected source outside the list. A page or a doc example
|
|
11
|
+
* written in syntax the engine lacks (core 3.x's own forms among it)
|
|
12
|
+
* does not run; write upstream's spelling (`packages/engine/README.md`
|
|
13
|
+
* lists the forms that were considered and not kept), or list it with a
|
|
14
|
+
* reason;
|
|
15
|
+
* 3. no stale entry: a listed source the engine now reads, or one that was
|
|
16
|
+
* edited (the key embeds a hash of the source), must be removed, so the
|
|
17
|
+
* list only shrinks. It has been empty since the history pages moved to
|
|
18
|
+
* `call history.pushState` (Phase B1).
|
|
19
|
+
*
|
|
20
|
+
* Until Phase C4 the denominator was the sources `@hyperfixi/core` compiled
|
|
21
|
+
* clean (see `checkShippedSourcesOnEngine`).
|
|
22
|
+
*
|
|
23
|
+
* This is a PARSE-level gate. A source can parse on both engines and run
|
|
24
|
+
* differently; the engine lane of `shipped-examples-execution.test.ts` is the
|
|
25
|
+
* gate for that.
|
|
26
|
+
*
|
|
27
|
+
* To see the list by form, and beside upstream's verdict:
|
|
28
|
+
* `npx tsx packages/engine/tools/shipped-sources.mts`.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { readFileSync } from 'node:fs';
|
|
32
|
+
import { fileURLToPath } from 'node:url';
|
|
33
|
+
import path from 'node:path';
|
|
34
|
+
import { JSDOM } from 'jsdom';
|
|
35
|
+
import { describe, it, expect, beforeAll } from 'vitest';
|
|
36
|
+
import {
|
|
37
|
+
checkShippedSourcesOnEngine,
|
|
38
|
+
type ShippedSourcesOnEngineResult,
|
|
39
|
+
} from './shipped-sources-validity';
|
|
40
|
+
|
|
41
|
+
interface RejectedDoc {
|
|
42
|
+
rejected: Array<{ key: string; file: string; error: string; excerpt: string; reason: string }>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const baselinePath = path.resolve(
|
|
46
|
+
path.dirname(fileURLToPath(import.meta.url)),
|
|
47
|
+
'../../baselines/shipped-sources-engine.json'
|
|
48
|
+
);
|
|
49
|
+
const listed = JSON.parse(readFileSync(baselinePath, 'utf8')) as RejectedDoc;
|
|
50
|
+
const allowed = new Set(listed.rejected.map(e => e.key));
|
|
51
|
+
|
|
52
|
+
describe('shipped sources on @hyperfixi/engine', () => {
|
|
53
|
+
let result: ShippedSourcesOnEngineResult;
|
|
54
|
+
|
|
55
|
+
beforeAll(async () => {
|
|
56
|
+
const { api, everything, register } = await import('@hyperfixi/engine');
|
|
57
|
+
register(...everything);
|
|
58
|
+
const dom = new JSDOM('<!doctype html><html><body></body></html>');
|
|
59
|
+
result = checkShippedSourcesOnEngine(
|
|
60
|
+
code => {
|
|
61
|
+
try {
|
|
62
|
+
return api.parse(code).errors.map(error => error.message);
|
|
63
|
+
} catch (e) {
|
|
64
|
+
return ['threw: ' + (e instanceof Error ? e.message : String(e))];
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
dom.window.document as unknown as Parameters<typeof checkShippedSourcesOnEngine>[1]
|
|
68
|
+
);
|
|
69
|
+
}, 120_000);
|
|
70
|
+
|
|
71
|
+
it('finds the shipped sources and the engine reads most of them (sanity)', () => {
|
|
72
|
+
expect(result.checked).toBeGreaterThan(300);
|
|
73
|
+
expect(result.engineAccepts).toBeGreaterThan(300);
|
|
74
|
+
// Written in another language: the localized gate's, not this one's.
|
|
75
|
+
expect(result.localized).toBeGreaterThan(10);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('has no NEW shipped source the engine rejects outside the list', () => {
|
|
79
|
+
const unexpected = result.rejections.filter(r => !allowed.has(r.key));
|
|
80
|
+
expect(
|
|
81
|
+
unexpected,
|
|
82
|
+
unexpected.length
|
|
83
|
+
? `\nShipped sources @hyperfixi/engine rejects (write upstream's spelling,\n` +
|
|
84
|
+
`or list it in baselines/shipped-sources-engine.json with a reason):\n` +
|
|
85
|
+
unexpected
|
|
86
|
+
.map(r => ` [${r.key}]\n "${r.excerpt}"\n -> ${r.error}`)
|
|
87
|
+
.join('\n') +
|
|
88
|
+
`\n\nOne source on the engine and on upstream, side by side:\n` +
|
|
89
|
+
` npx tsx packages/engine/tools/probe.mts '<source>'`
|
|
90
|
+
: ''
|
|
91
|
+
).toEqual([]);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it('has no stale entries (a source the engine now reads must be removed, so the list only shrinks)', () => {
|
|
95
|
+
const stillRejected = new Set(result.rejections.map(r => r.key));
|
|
96
|
+
const stale = listed.rejected.map(e => e.key).filter(key => !stillRejected.has(key));
|
|
97
|
+
expect(
|
|
98
|
+
stale,
|
|
99
|
+
stale.length
|
|
100
|
+
? `\nThese listed sources are no longer rejected (read by the engine now, or edited: the key\n` +
|
|
101
|
+
`embeds a hash of the source). Remove them from baselines/shipped-sources-engine.json:\n ` +
|
|
102
|
+
stale.join('\n ')
|
|
103
|
+
: ''
|
|
104
|
+
).toEqual([]);
|
|
105
|
+
});
|
|
106
|
+
});
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shipped sources written in another language, through the multilingual path.
|
|
3
|
+
*
|
|
4
|
+
* Both shipped-sources gates hand every `_` to an English parser, which
|
|
5
|
+
* rejects a script written in Spanish, so they skip it — and nothing else
|
|
6
|
+
* looked. On a page such a script runs through `@lokascript/hyperscript-
|
|
7
|
+
* adapter`, which translates it under its element's `lang`. This gate does
|
|
8
|
+
* the same and requires English that upstream `_hyperscript` and
|
|
9
|
+
* `@hyperfixi/engine` both parse (Phase C2c; fifteen attributes today: the
|
|
10
|
+
* `live` blocks of examples/hx-v4-i18n/live-multilang.html, the example in
|
|
11
|
+
* docs/BROWSER_BUNDLES.md's "Hyperscript in another language", and the
|
|
12
|
+
* multilingual section of packages/core/docs/EXAMPLES.md).
|
|
13
|
+
*
|
|
14
|
+
* @vitest-environment node
|
|
15
|
+
* Required: the hosts and the collector need real jsdom documents.
|
|
16
|
+
*/
|
|
17
|
+
import { createRequire } from 'node:module';
|
|
18
|
+
import { pathToFileURL } from 'node:url';
|
|
19
|
+
import { describe, it, expect, beforeAll } from 'vitest';
|
|
20
|
+
import { JSDOM } from 'jsdom';
|
|
21
|
+
import { preprocess } from '@lokascript/hyperscript-adapter';
|
|
22
|
+
import { collectLocalizedSources, type LocalizedSource } from './shipped-sources-validity';
|
|
23
|
+
import { installGlobals } from './shipped-examples-execution';
|
|
24
|
+
|
|
25
|
+
interface ParseHost {
|
|
26
|
+
parse(source: string): { errors?: Array<{ message: string }> } | undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
describe('shipped sources written in another language', () => {
|
|
30
|
+
const sources: LocalizedSource[] = collectLocalizedSources();
|
|
31
|
+
let upstream: ParseHost;
|
|
32
|
+
let engine: ParseHost;
|
|
33
|
+
|
|
34
|
+
beforeAll(async () => {
|
|
35
|
+
installGlobals(new JSDOM('<!doctype html><html><body></body></html>'));
|
|
36
|
+
const require = createRequire(import.meta.url);
|
|
37
|
+
const esm = require.resolve('hyperscript.org').replace(/[^/\\]+$/, '_hyperscript.esm.js');
|
|
38
|
+
upstream = (await import(pathToFileURL(esm).href)).default as ParseHost;
|
|
39
|
+
const engineModule = await import('@hyperfixi/engine');
|
|
40
|
+
engineModule.register(...engineModule.everything);
|
|
41
|
+
engine = engineModule.api;
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('finds them (a page that moves them changes this count on purpose)', () => {
|
|
45
|
+
expect(sources.map(s => `${s.file} [${s.lang}]`)).toEqual([
|
|
46
|
+
'examples/hx-v4-i18n/live-multilang.html [es]',
|
|
47
|
+
'examples/hx-v4-i18n/live-multilang.html [ja]',
|
|
48
|
+
'examples/hx-v4-i18n/live-multilang.html [ar]',
|
|
49
|
+
// The engine + adapter example in "Hyperscript in another language" (C-R3).
|
|
50
|
+
'docs/BROWSER_BUNDLES.md [ja]',
|
|
51
|
+
// "Multilingual Examples". Marked with `lang` in C4b: core's English parser had
|
|
52
|
+
// kept them out of the engine gate; the engine gate now reads everything else.
|
|
53
|
+
'packages/core/docs/EXAMPLES.md [es]',
|
|
54
|
+
'packages/core/docs/EXAMPLES.md [ja]',
|
|
55
|
+
'packages/core/docs/EXAMPLES.md [ar]',
|
|
56
|
+
'packages/core/docs/EXAMPLES.md [es]',
|
|
57
|
+
'packages/core/docs/EXAMPLES.md [ja]',
|
|
58
|
+
'packages/core/docs/EXAMPLES.md [ko]',
|
|
59
|
+
'packages/core/docs/EXAMPLES.md [es]',
|
|
60
|
+
'packages/core/docs/EXAMPLES.md [fr]',
|
|
61
|
+
'packages/core/docs/EXAMPLES.md [de]',
|
|
62
|
+
'packages/core/docs/EXAMPLES.md [es]',
|
|
63
|
+
'packages/core/docs/EXAMPLES.md [pt]',
|
|
64
|
+
]);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('each translates to English upstream and the engine both parse', () => {
|
|
68
|
+
const failures: string[] = [];
|
|
69
|
+
for (const s of sources) {
|
|
70
|
+
let english: string;
|
|
71
|
+
try {
|
|
72
|
+
english = preprocess(s.source, s.lang);
|
|
73
|
+
} catch (e) {
|
|
74
|
+
failures.push(`${s.file} [${s.lang}] threw: ${(e as Error).message}`);
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
const first = (host: ParseHost) => host.parse(english)?.errors?.[0]?.message.split('\n')[0];
|
|
78
|
+
const problem =
|
|
79
|
+
english === s.source
|
|
80
|
+
? 'not translated'
|
|
81
|
+
: (first(upstream) && `upstream: ${first(upstream)}`) ||
|
|
82
|
+
(first(engine) && `engine: ${first(engine)}`);
|
|
83
|
+
if (problem) failures.push(`${s.file} [${s.lang}] ${problem}\n ${s.source}\n → ${english}`);
|
|
84
|
+
}
|
|
85
|
+
expect(failures).toEqual([]);
|
|
86
|
+
});
|
|
87
|
+
});
|