@zsailer/jupyter-server-side-documents-conformance 0.0.0-stage → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COVERAGE.md +56 -0
- package/README.md +139 -2
- package/adapters/generic.ts +17 -0
- package/adapters/http.ts +84 -0
- package/bin/args.mjs +202 -0
- package/bin/conformance.mjs +303 -0
- package/bin/picker.mjs +256 -0
- package/bin/welcome.mjs +95 -0
- package/known-gaps/README.md +12 -0
- package/known-gaps/jsd.json +1 -0
- package/package.json +25 -4
- package/playwright.config.ts +32 -0
- package/runtime/adapter.ts +60 -0
- package/runtime/cells.ts +99 -0
- package/runtime/network.ts +84 -0
- package/runtime/recording.ts +159 -0
- package/runtime/report.ts +135 -0
- package/runtime/reporter.ts +60 -0
- package/runtime/story.ts +482 -0
- package/runtime/tab.ts +364 -0
- package/runtime/types.ts +63 -0
- package/scripts/summarize.py +100 -0
- package/server/conformance_config.py +48 -0
- package/server/jsd_control.py +91 -0
- package/stories.json +981 -0
- package/tests/generated/ai.spec.ts +450 -0
- package/tests/generated/crash.spec.ts +267 -0
- package/tests/generated/edges.spec.ts +549 -0
- package/tests/generated/files.spec.ts +237 -0
- package/tests/generated/handshake.spec.ts +429 -0
- package/tests/generated/lineage.spec.ts +241 -0
- package/tests/generated/outputs.spec.ts +258 -0
- package/tests/generated/rebuild.spec.ts +446 -0
- package/tests/generated/residual.spec.ts +356 -0
- package/tests/generated/stop.spec.ts +168 -0
- package/tsconfig.json +14 -0
package/package.json
CHANGED
|
@@ -1,6 +1,27 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zsailer/jupyter-server-side-documents-conformance",
|
|
3
|
-
"version": "0.0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.0.3",
|
|
4
|
+
"description": "Compliance tests for real-time collaboration backends in JupyterLab, generated from the storyboards.",
|
|
5
|
+
"bin": {
|
|
6
|
+
"rtc-conformance": "bin/conformance.mjs"
|
|
7
|
+
},
|
|
8
|
+
"license": "BSD-3-Clause",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "git+https://github.com/Zsailer/jupyter-server-side-documents-conformance.git"
|
|
12
|
+
},
|
|
13
|
+
"homepage": "https://github.com/Zsailer/jupyter-server-side-documents-conformance",
|
|
14
|
+
"publishConfig": {
|
|
15
|
+
"access": "public"
|
|
16
|
+
},
|
|
17
|
+
"scripts": {
|
|
18
|
+
"test": "playwright test",
|
|
19
|
+
"typecheck": "tsc --noEmit"
|
|
20
|
+
},
|
|
21
|
+
"dependencies": {
|
|
22
|
+
"@jupyterlab/galata": "^5.6.0",
|
|
23
|
+
"@playwright/test": "^1.60.0",
|
|
24
|
+
"@types/node": "^22.0.0",
|
|
25
|
+
"typescript": "^5.4.0"
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The compliance suite runs against a JupyterLab you already have running.
|
|
3
|
+
*
|
|
4
|
+
* COMPLIANCE_BASE_URL where it is (default http://localhost:8899)
|
|
5
|
+
* COMPLIANCE_ADAPTER generic (default) | jsd | the path of your own adapter module
|
|
6
|
+
* COMPLIANCE_ROOT_DIR the folder the server serves; lets the tests change the file from outside Jupyter
|
|
7
|
+
* COMPLIANCE_SERVER_CMD optional: a command that starts the server, which Playwright starts and stops for you
|
|
8
|
+
*
|
|
9
|
+
* The server must be started the way JupyterLab's own UI tests start it (no login, and the app exposed to the page):
|
|
10
|
+
* `from jupyterlab.galata import configure_jupyter_server; configure_jupyter_server(c)` in its config.
|
|
11
|
+
*/
|
|
12
|
+
import { defineConfig } from '@playwright/test';
|
|
13
|
+
|
|
14
|
+
// galata's own Playwright configuration (CommonJS, untyped)
|
|
15
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
16
|
+
const baseConfig: any = require('@jupyterlab/galata/lib/playwright-config');
|
|
17
|
+
|
|
18
|
+
const baseURL = process.env.COMPLIANCE_BASE_URL ?? 'http://localhost:8899';
|
|
19
|
+
|
|
20
|
+
export default defineConfig({
|
|
21
|
+
...baseConfig,
|
|
22
|
+
testDir: './tests',
|
|
23
|
+
// a story opens several browsers, goes offline, waits for saves: give it room
|
|
24
|
+
timeout: Number(process.env.COMPLIANCE_STORY_TIMEOUT_MS ?? 240000),
|
|
25
|
+
retries: 0,
|
|
26
|
+
workers: Number(process.env.COMPLIANCE_WORKERS ?? 1),
|
|
27
|
+
// a click that cannot be made (something is in the way) fails with a message and a picture, not after the whole story's time
|
|
28
|
+
use: { ...baseConfig.use, baseURL, actionTimeout: Number(process.env.COMPLIANCE_ACTION_TIMEOUT_MS ?? 30000) },
|
|
29
|
+
webServer: process.env.COMPLIANCE_SERVER_CMD
|
|
30
|
+
? { command: process.env.COMPLIANCE_SERVER_CMD, url: `${baseURL}/lab`, timeout: 120000, reuseExistingServer: true }
|
|
31
|
+
: undefined
|
|
32
|
+
});
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { IJupyterLabPageFixture } from '@jupyterlab/galata';
|
|
2
|
+
import type { Capability } from './types';
|
|
3
|
+
|
|
4
|
+
/** What an adapter gets to work with. */
|
|
5
|
+
export interface AdapterContext {
|
|
6
|
+
/** A page of the running JupyterLab, for requests that need its server settings (token, xsrf). */
|
|
7
|
+
admin: IJupyterLabPageFixture;
|
|
8
|
+
baseURL: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The part of the backend the tests cannot reach through JupyterLab and the browser.
|
|
13
|
+
*
|
|
14
|
+
* The tests verify what a person sees: what is in their tab, in a fresh tab, and in the file. They need a backend's
|
|
15
|
+
* help only to cause the events a story describes, and an adapter says which of these it can do. An adapter that does
|
|
16
|
+
* nothing but declare `contents` is a valid adapter: it runs every story that needs nothing more.
|
|
17
|
+
*/
|
|
18
|
+
export interface BackendAdapter {
|
|
19
|
+
readonly name: string;
|
|
20
|
+
readonly capabilities: ReadonlySet<Capability>;
|
|
21
|
+
|
|
22
|
+
/** Ask the running server what it offers, before the tests decide which stories to run. */
|
|
23
|
+
discover?(baseURL: string): Promise<void>;
|
|
24
|
+
|
|
25
|
+
/** The cells (sources) of the server's own live copy, or null if the server holds no copy of `path`. */
|
|
26
|
+
serverCells?(ctx: AdapterContext, path: string): Promise<string[] | null>;
|
|
27
|
+
|
|
28
|
+
/** The server lets go of the notebook, saving it; the next client to connect makes it build the notebook again. */
|
|
29
|
+
releaseRoom?(ctx: AdapterContext, path: string): Promise<void>;
|
|
30
|
+
|
|
31
|
+
/** Kill the server without a save. Every tab loses its connection. */
|
|
32
|
+
crashServer?(ctx: AdapterContext): Promise<void>;
|
|
33
|
+
/** Start the server again after a crash, on the same address, against the same files. */
|
|
34
|
+
restartServer?(ctx: AdapterContext): Promise<void>;
|
|
35
|
+
|
|
36
|
+
/** Lose or delay a message between a tab and the server. */
|
|
37
|
+
wireFault?(ctx: AdapterContext, fault: 'lose-reply' | 'late-reply', tab: import('./tab').Tab): Promise<void>;
|
|
38
|
+
|
|
39
|
+
/** Let an agent edit the notebook: through the server while it holds the notebook, else through the file. */
|
|
40
|
+
agentEdit?(ctx: AdapterContext, path: string, cells: string[]): Promise<void>;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Which adapter to use: `COMPLIANCE_ADAPTER` is `http` (default: the backend implements the control contract, or offers
|
|
45
|
+
* nothing), `generic` (never asks the server for anything), or the path of a module whose default export is a
|
|
46
|
+
* BackendAdapter.
|
|
47
|
+
*/
|
|
48
|
+
export async function loadAdapter(): Promise<BackendAdapter> {
|
|
49
|
+
const which = process.env.COMPLIANCE_ADAPTER || 'http';
|
|
50
|
+
const path = which === 'generic' || which === 'http' ? `../adapters/${which}` : which;
|
|
51
|
+
let adapter: any = await import(path);
|
|
52
|
+
// however the module was loaded (ES module, CommonJS, transpiled), the adapter is the object with `capabilities`
|
|
53
|
+
while (adapter && !adapter.capabilities && adapter.default) {
|
|
54
|
+
adapter = adapter.default;
|
|
55
|
+
}
|
|
56
|
+
if (!adapter || !adapter.capabilities) {
|
|
57
|
+
throw new Error(`COMPLIANCE_ADAPTER=${which}: ${path} does not export a BackendAdapter as its default`);
|
|
58
|
+
}
|
|
59
|
+
return adapter as BackendAdapter;
|
|
60
|
+
}
|
package/runtime/cells.ts
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import type { Expect } from './types';
|
|
2
|
+
|
|
3
|
+
/** A cell source as the Contents API and the shared model give it: a string, or a list of lines. */
|
|
4
|
+
export function sourceText(source: unknown): string {
|
|
5
|
+
return Array.isArray(source) ? source.join('') : String(source ?? '');
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
const norm = (s: string): string => s.replace(/\r\n/g, '\n').trim();
|
|
9
|
+
|
|
10
|
+
/** Whether one cell reads as expected. */
|
|
11
|
+
export function cellMatches(actual: string, expected: Expect): boolean {
|
|
12
|
+
const a = norm(actual);
|
|
13
|
+
if (typeof expected === 'string') {
|
|
14
|
+
return a === norm(expected);
|
|
15
|
+
}
|
|
16
|
+
if ('contains' in expected) {
|
|
17
|
+
return expected.contains.every(part => a.includes(part));
|
|
18
|
+
}
|
|
19
|
+
return a.startsWith(norm(expected.base)) && expected.parts.every(part => a.includes(part));
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function describeExpect(e: Expect): string {
|
|
23
|
+
if (typeof e === 'string') {
|
|
24
|
+
return JSON.stringify(e);
|
|
25
|
+
}
|
|
26
|
+
if ('contains' in e) {
|
|
27
|
+
return `a cell holding ${e.contains.map(p => JSON.stringify(p)).join(' and ')}`;
|
|
28
|
+
}
|
|
29
|
+
return `${JSON.stringify(e.base)} followed by ${e.parts.map(p => JSON.stringify(p)).join(' and ')} (any order)`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface IMatch {
|
|
33
|
+
ok: boolean;
|
|
34
|
+
/** Why not, in the words of a person comparing two notebooks. */
|
|
35
|
+
reason: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Whether the cells of a notebook are the expected ones. Ordered, unless `ordered` is false (then any assignment of
|
|
40
|
+
* the expected cells to the actual ones will do).
|
|
41
|
+
*/
|
|
42
|
+
export function matchCells(actual: string[], expected: Expect[], ordered = true): IMatch {
|
|
43
|
+
if (actual.length !== expected.length) {
|
|
44
|
+
return { ok: false, reason: `${actual.length} cells, expected ${expected.length}` };
|
|
45
|
+
}
|
|
46
|
+
if (ordered) {
|
|
47
|
+
for (let i = 0; i < expected.length; i++) {
|
|
48
|
+
if (!cellMatches(actual[i], expected[i])) {
|
|
49
|
+
return { ok: false, reason: `cell ${i + 1} reads ${JSON.stringify(norm(actual[i]))}, expected ${describeExpect(expected[i])}` };
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return { ok: true, reason: '' };
|
|
53
|
+
}
|
|
54
|
+
const left = actual.slice();
|
|
55
|
+
for (const e of expected) {
|
|
56
|
+
const at = left.findIndex(a => cellMatches(a, e));
|
|
57
|
+
if (at < 0) {
|
|
58
|
+
return { ok: false, reason: `no cell reads as ${describeExpect(e)}` };
|
|
59
|
+
}
|
|
60
|
+
left.splice(at, 1);
|
|
61
|
+
}
|
|
62
|
+
return { ok: true, reason: '' };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The expected cells, for a message: one per line. */
|
|
66
|
+
export function showExpected(expected: Expect[]): string {
|
|
67
|
+
return expected.map((e, i) => ` ${i + 1}. ${describeExpect(e)}`).join('\n');
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function showActual(actual: string[]): string {
|
|
71
|
+
return actual.map((a, i) => ` ${i + 1}. ${JSON.stringify(norm(a))}`).join('\n') || ' (no cells)';
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Helpers the generated tests use to say what a cell holds. */
|
|
75
|
+
export const E = {
|
|
76
|
+
/** Starts with `base`, then holds every part in any order. */
|
|
77
|
+
parts: (base: string, ...parts: string[]): Expect => ({ base, parts }),
|
|
78
|
+
/** Holds every one of these. */
|
|
79
|
+
contains: (...contains: string[]): Expect => ({ contains })
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The text a person types to add `piece` at the end of a cell. Typed text is separated from what is there by a space,
|
|
84
|
+
* so a typed piece never glues itself to the code before it.
|
|
85
|
+
*/
|
|
86
|
+
export const typed = (piece: string): string => ' ' + piece;
|
|
87
|
+
|
|
88
|
+
/** A notebook (nbformat 4) holding code cells with these sources. `ids: false` gives the older format without cell ids. */
|
|
89
|
+
export function notebookJSON(sources: string[], ids = true): string {
|
|
90
|
+
const cells = sources.map((source, i) => ({
|
|
91
|
+
cell_type: 'code',
|
|
92
|
+
...(ids ? { id: `cell-${i}-${Math.random().toString(36).slice(2, 8)}` } : {}),
|
|
93
|
+
metadata: {},
|
|
94
|
+
source,
|
|
95
|
+
outputs: [],
|
|
96
|
+
execution_count: null
|
|
97
|
+
}));
|
|
98
|
+
return JSON.stringify({ cells, metadata: {}, nbformat: 4, nbformat_minor: ids ? 5 : 4 }, null, 1);
|
|
99
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { BrowserContext, Page } from '@playwright/test';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Two ways to take one browser off the network.
|
|
5
|
+
*
|
|
6
|
+
* `emulate` is Playwright's own `context.setOffline()`. It is the simplest, but in Chromium it slows the start of every
|
|
7
|
+
* other browser opened while it is on: measured here, a second browser took 5.9 s to start JupyterLab instead of 0.8 s
|
|
8
|
+
* (tests/probe-offline.spec.ts), and a story with several tabs opens browsers at all times. It is kept for comparison:
|
|
9
|
+
* COMPLIANCE_OFFLINE=emulate.
|
|
10
|
+
*
|
|
11
|
+
* `cut`, the default, keeps the outage inside the one browser (a second browser then starts in 0.8 s, as with none offline): its HTTP requests are refused (as ERR_INTERNET_DISCONNECTED, the error
|
|
12
|
+
* a real outage gives) and its WebSockets are closed, and while it is cut every WebSocket it tries to open fails at once.
|
|
13
|
+
*/
|
|
14
|
+
export type OfflineMode = 'emulate' | 'cut';
|
|
15
|
+
|
|
16
|
+
export const OFFLINE_MODE: OfflineMode = process.env.COMPLIANCE_OFFLINE === 'emulate' ? 'emulate' : 'cut';
|
|
17
|
+
|
|
18
|
+
/** Installed in every page of the browser before the page's own scripts run: it keeps track of the WebSockets. */
|
|
19
|
+
export const NETWORK_SHIM = `(() => {
|
|
20
|
+
if (window.__nbs) return;
|
|
21
|
+
const Native = window.WebSocket;
|
|
22
|
+
const state = { offline: false, sockets: new Set() };
|
|
23
|
+
window.__nbs = state;
|
|
24
|
+
function Shim(url, protocols) {
|
|
25
|
+
const target = state.offline ? 'ws://127.0.0.1:9/offline' : url;
|
|
26
|
+
const ws = protocols === undefined ? new Native(target) : new Native(target, protocols);
|
|
27
|
+
state.sockets.add(ws);
|
|
28
|
+
ws.addEventListener('close', () => state.sockets.delete(ws));
|
|
29
|
+
return ws;
|
|
30
|
+
}
|
|
31
|
+
Shim.prototype = Native.prototype;
|
|
32
|
+
for (const k of ['CONNECTING', 'OPEN', 'CLOSING', 'CLOSED']) Shim[k] = Native[k];
|
|
33
|
+
window.WebSocket = Shim;
|
|
34
|
+
})();`;
|
|
35
|
+
|
|
36
|
+
const cut = new WeakMap<BrowserContext, { offline: boolean; routed: boolean }>();
|
|
37
|
+
|
|
38
|
+
/** Call once, when the browser is made, before any page loads. Does nothing in `emulate` mode. */
|
|
39
|
+
export async function prepareNetwork(context: BrowserContext, mode: OfflineMode = OFFLINE_MODE): Promise<void> {
|
|
40
|
+
if (mode !== 'cut') {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
cut.set(context, { offline: false, routed: false });
|
|
44
|
+
await context.addInitScript(NETWORK_SHIM);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Take the browser off the network, or put it back. */
|
|
48
|
+
export async function setNetwork(context: BrowserContext, offline: boolean, mode: OfflineMode = OFFLINE_MODE): Promise<void> {
|
|
49
|
+
if (mode !== 'cut') {
|
|
50
|
+
await context.setOffline(offline);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const state = cut.get(context);
|
|
54
|
+
if (!state) {
|
|
55
|
+
throw new Error('prepareNetwork() was not called when this browser was made');
|
|
56
|
+
}
|
|
57
|
+
state.offline = offline;
|
|
58
|
+
if (!state.routed) {
|
|
59
|
+
state.routed = true;
|
|
60
|
+
await context.route('**/*', route => (state.offline ? route.abort('internetdisconnected') : route.fallback()));
|
|
61
|
+
}
|
|
62
|
+
await Promise.all(context.pages().map(page => apply(page, offline)));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function apply(page: Page, offline: boolean): Promise<void> {
|
|
66
|
+
await page
|
|
67
|
+
.evaluate((off: boolean) => {
|
|
68
|
+
const nbs = (window as any).__nbs;
|
|
69
|
+
if (!nbs) {
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
nbs.offline = off;
|
|
73
|
+
if (off) {
|
|
74
|
+
for (const socket of Array.from(nbs.sockets as Set<WebSocket>)) {
|
|
75
|
+
try {
|
|
76
|
+
socket.close();
|
|
77
|
+
} catch {
|
|
78
|
+
/* already closed */
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}, offline)
|
|
83
|
+
.catch(() => undefined);
|
|
84
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
|
|
4
|
+
/** The video of one tab, and when it started. */
|
|
5
|
+
export interface TabRecording {
|
|
6
|
+
name: string;
|
|
7
|
+
/** Milliseconds since the epoch at which the tab's browser was opened (its video starts then). */
|
|
8
|
+
startedAt: number;
|
|
9
|
+
/** Where Playwright put the video, once the tab was closed. */
|
|
10
|
+
file: string | null;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Videos bigger than this in all are linked, not embedded, so the page stays small enough to open. */
|
|
14
|
+
const EMBED_LIMIT = 40 * 1024 * 1024;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Keeps what is needed to watch a story afterwards: a video of every tab, and when each step of the storyboard
|
|
18
|
+
* happened. `finish()` writes one page that plays them all side by side, in step with each other.
|
|
19
|
+
*
|
|
20
|
+
* Playwright only records the page it creates itself, which is not where the story happens: each tab is a browser of
|
|
21
|
+
* its own, opened by the test.
|
|
22
|
+
*/
|
|
23
|
+
export class Recorder {
|
|
24
|
+
readonly t0 = Date.now();
|
|
25
|
+
readonly tabs: TabRecording[] = [];
|
|
26
|
+
readonly steps: { at: number; text: string }[] = [];
|
|
27
|
+
|
|
28
|
+
readonly dir: string;
|
|
29
|
+
private readonly title: string;
|
|
30
|
+
|
|
31
|
+
constructor(dir: string, title = '') {
|
|
32
|
+
this.dir = dir;
|
|
33
|
+
this.title = title;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Something a person would have had to do or would have seen, worth saying in the summary. */
|
|
37
|
+
readonly notes: string[] = [];
|
|
38
|
+
|
|
39
|
+
note(text: string): void {
|
|
40
|
+
this.notes.push(text);
|
|
41
|
+
this.step(text);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
step(text: string): void {
|
|
45
|
+
this.steps.push({ at: Date.now(), text });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Write `watch.html` into the folder. Returns its path, or null if there is nothing to watch. */
|
|
49
|
+
finish(): string | null {
|
|
50
|
+
const tabs = this.tabs.filter(t => t.file && fs.existsSync(t.file)).sort((a, b) => a.startedAt - b.startedAt);
|
|
51
|
+
if (!tabs.length) {
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
const total = tabs.reduce((n, t) => n + fs.statSync(t.file!).size, 0);
|
|
55
|
+
const embed = total <= EMBED_LIMIT;
|
|
56
|
+
const videos = tabs.map(t => ({
|
|
57
|
+
name: t.name,
|
|
58
|
+
offset: (t.startedAt - this.t0) / 1000,
|
|
59
|
+
src: embed ? `data:video/webm;base64,${fs.readFileSync(t.file!).toString('base64')}` : path.relative(this.dir, t.file!)
|
|
60
|
+
}));
|
|
61
|
+
const steps = this.steps.map(s => ({ at: (s.at - this.t0) / 1000, text: s.text }));
|
|
62
|
+
const out = path.join(this.dir, 'watch.html');
|
|
63
|
+
fs.mkdirSync(this.dir, { recursive: true });
|
|
64
|
+
fs.writeFileSync(out, watchPage(this.title, videos, steps));
|
|
65
|
+
return out;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Keep what a tab looked like as files in the story's results folder: a picture, and the page as text. Files stay where
|
|
71
|
+
* they are put, whichever reporter ran; an attachment made from bytes lives only in an HTML report, which a run with the
|
|
72
|
+
* list reporter never makes.
|
|
73
|
+
*/
|
|
74
|
+
export async function keepLook(
|
|
75
|
+
dir: string,
|
|
76
|
+
name: string,
|
|
77
|
+
when: string,
|
|
78
|
+
page: { screenshot(options?: { timeout?: number }): Promise<Buffer>; locator(selector: string): { ariaSnapshot(options?: { timeout?: number }): Promise<string> } },
|
|
79
|
+
/** Anything else worth keeping with it: what the browser reported, say. */
|
|
80
|
+
extra = ''
|
|
81
|
+
): Promise<string[]> {
|
|
82
|
+
const stem = path.join(dir, 'looks', `${name.replace(/[^\w.-]+/g, '_')} ${when.replace(/[^\w.-]+/g, '_')}`);
|
|
83
|
+
fs.mkdirSync(path.dirname(stem), { recursive: true });
|
|
84
|
+
const written: string[] = [];
|
|
85
|
+
try {
|
|
86
|
+
fs.writeFileSync(`${stem}.png`, await page.screenshot({ timeout: 5000 }));
|
|
87
|
+
written.push(`${stem}.png`);
|
|
88
|
+
} catch {
|
|
89
|
+
/* the page is gone */
|
|
90
|
+
}
|
|
91
|
+
try {
|
|
92
|
+
fs.writeFileSync(`${stem}.txt`, await page.locator('body').ariaSnapshot({ timeout: 5000 }));
|
|
93
|
+
written.push(`${stem}.txt`);
|
|
94
|
+
} catch {
|
|
95
|
+
/* the page is gone */
|
|
96
|
+
}
|
|
97
|
+
if (extra) {
|
|
98
|
+
fs.writeFileSync(`${stem}.events.txt`, extra);
|
|
99
|
+
written.push(`${stem}.events.txt`);
|
|
100
|
+
}
|
|
101
|
+
return written;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const esc = (s: string): string => s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
|
105
|
+
|
|
106
|
+
/** The page that plays the videos of a story side by side, one clock for all of them. */
|
|
107
|
+
export function watchPage(title: string, videos: { name: string; offset: number; src: string }[], steps: { at: number; text: string }[]): string {
|
|
108
|
+
const cells = videos
|
|
109
|
+
.map(
|
|
110
|
+
(v, i) =>
|
|
111
|
+
`<figure><figcaption>${esc(v.name)}</figcaption><video id="v${i}" src="${v.src}" muted playsinline preload="auto"></video></figure>`
|
|
112
|
+
)
|
|
113
|
+
.join('');
|
|
114
|
+
const data = JSON.stringify({ offsets: videos.map(v => v.offset), steps }).replace(/</g, '\\u003c');
|
|
115
|
+
return `<!doctype html>
|
|
116
|
+
<html lang="en"><head><meta charset="utf-8"><title>${esc(title)}</title>
|
|
117
|
+
<style>
|
|
118
|
+
:root{color-scheme:light dark;font:15px/1.5 -apple-system,Segoe UI,Helvetica,Arial,sans-serif}
|
|
119
|
+
body{margin:0;padding:16px 20px 40px}
|
|
120
|
+
h1{font-size:1.05rem;margin:0 0 12px}
|
|
121
|
+
.row{display:grid;grid-template-columns:repeat(${Math.min(videos.length, 3)},minmax(0,1fr));gap:12px}
|
|
122
|
+
figure{margin:0}figcaption{font-weight:600;margin-bottom:4px}
|
|
123
|
+
video{width:100%;background:#000;border-radius:6px}
|
|
124
|
+
.bar{display:flex;gap:10px;align-items:center;margin:14px 0 8px}
|
|
125
|
+
button,select{font:inherit;padding:4px 12px}
|
|
126
|
+
input[type=range]{flex:1}
|
|
127
|
+
#step{padding:10px 14px;border-left:4px solid #1976d2;background:rgba(25,118,210,.1);border-radius:4px;min-height:1.5em}
|
|
128
|
+
ol{color:gray;font-size:13.5px}ol li.on{color:inherit;font-weight:600}
|
|
129
|
+
</style></head><body>
|
|
130
|
+
<h1>${esc(title)}</h1>
|
|
131
|
+
<div class="row">${cells}</div>
|
|
132
|
+
<div class="bar"><button id="play">Play</button><input id="seek" type="range" min="0" step="0.05"><span id="clock"></span>
|
|
133
|
+
<select id="speed"><option>0.5</option><option selected>1</option><option>2</option><option>4</option></select>×</div>
|
|
134
|
+
<div id="step"></div>
|
|
135
|
+
<ol id="steps"></ol>
|
|
136
|
+
<script>
|
|
137
|
+
const D=${data};
|
|
138
|
+
const vids=D.offsets.map((_,i)=>document.getElementById('v'+i));
|
|
139
|
+
const seek=document.getElementById('seek'),play=document.getElementById('play'),clock=document.getElementById('clock'),stepEl=document.getElementById('step'),list=document.getElementById('steps'),speed=document.getElementById('speed');
|
|
140
|
+
let end=Math.max(...vids.map((v,i)=>D.offsets[i]+(isFinite(v.duration)?v.duration:0)),D.steps.length?D.steps[D.steps.length-1].at+3:0);
|
|
141
|
+
let t=0,playing=false,last=0;
|
|
142
|
+
D.steps.forEach(s=>{const li=document.createElement('li');li.textContent=s.text;list.appendChild(li)});
|
|
143
|
+
function show(){
|
|
144
|
+
vids.forEach((v,i)=>{const local=t-D.offsets[i];
|
|
145
|
+
if(local<0){v.pause();v.currentTime=0;}
|
|
146
|
+
else if(isFinite(v.duration)&&local>=v.duration){v.pause();v.currentTime=v.duration;}
|
|
147
|
+
else{if(Math.abs(v.currentTime-local)>0.25)v.currentTime=local;if(playing&&v.paused)v.play();if(!playing&&!v.paused)v.pause();v.playbackRate=+speed.value;}});
|
|
148
|
+
let k=-1;D.steps.forEach((s,i)=>{if(t>=s.at)k=i});
|
|
149
|
+
stepEl.textContent=k>=0?D.steps[k].text:'';[...list.children].forEach((li,i)=>li.classList.toggle('on',i===k));
|
|
150
|
+
seek.max=end;seek.value=t;clock.textContent=t.toFixed(1)+' / '+end.toFixed(1)+' s';play.textContent=playing?'Pause':'Play';
|
|
151
|
+
}
|
|
152
|
+
function tick(now){if(playing){t=Math.min(end,t+(now-last)/1000*+speed.value);if(t>=end)playing=false;}last=now;show();requestAnimationFrame(tick)}
|
|
153
|
+
play.onclick=()=>{if(t>=end)t=0;playing=!playing;if(!playing)vids.forEach(v=>v.pause())};
|
|
154
|
+
seek.oninput=()=>{t=+seek.value;show()};
|
|
155
|
+
vids.forEach(v=>v.addEventListener('loadedmetadata',()=>{end=Math.max(end,...vids.map((w,i)=>D.offsets[i]+(isFinite(w.duration)?w.duration:0)));show()}));
|
|
156
|
+
requestAnimationFrame(tick);show();
|
|
157
|
+
</script></body></html>
|
|
158
|
+
`;
|
|
159
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a run looks like to a person: one line per story as it finishes, then a short summary. Pure functions, so they can
|
|
3
|
+
* be tested without a browser; reporter.ts is the thin Playwright part.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export type Kind = 'pass' | 'fail' | 'gap' | 'miss' | 'skip';
|
|
7
|
+
|
|
8
|
+
export interface Row {
|
|
9
|
+
id: string;
|
|
10
|
+
title: string;
|
|
11
|
+
kind: Kind;
|
|
12
|
+
/** Why: the first line of a failure, the reason for a skip, what a known gap or a miss says. */
|
|
13
|
+
detail: string;
|
|
14
|
+
ms: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** The parts of Playwright's TestCase and TestResult that a row is made from. */
|
|
18
|
+
export interface TestLike {
|
|
19
|
+
title: string;
|
|
20
|
+
annotations: { type: string; description?: string }[];
|
|
21
|
+
expectedStatus: string;
|
|
22
|
+
}
|
|
23
|
+
export interface ResultLike {
|
|
24
|
+
status: string;
|
|
25
|
+
duration: number;
|
|
26
|
+
error?: { message?: string };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const ANSI = /\u001b\[[0-9;]*m/g;
|
|
30
|
+
|
|
31
|
+
/** The first line of an error that tells a person what went wrong: for a step, which step and then why. */
|
|
32
|
+
export function reason(message: string | undefined): string {
|
|
33
|
+
const lines = (message ?? '')
|
|
34
|
+
.replace(ANSI, '')
|
|
35
|
+
.split('\n')
|
|
36
|
+
.map(l => l.trim())
|
|
37
|
+
.filter(Boolean)
|
|
38
|
+
.map(l => l.replace(/^(\w*Error: )/, ''));
|
|
39
|
+
if (!lines.length) {
|
|
40
|
+
return 'failed';
|
|
41
|
+
}
|
|
42
|
+
const first = lines[0].startsWith('step ') && lines[1] ? `${lines[0]} — ${lines[1]}` : lines[0];
|
|
43
|
+
return first.length > 150 ? `${first.slice(0, 149)}…` : first;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function rowOf(test: TestLike, result: ResultLike): Row {
|
|
47
|
+
const m = /^(\S+) · (.*)$/.exec(test.title);
|
|
48
|
+
const id = m ? m[1] : test.title;
|
|
49
|
+
const title = m ? m[2] : '';
|
|
50
|
+
const note = (type: string): string | undefined => test.annotations.find(a => a.type === type)?.description;
|
|
51
|
+
const base = { id, title, ms: result.duration };
|
|
52
|
+
|
|
53
|
+
if (result.status === 'skipped') {
|
|
54
|
+
return { ...base, kind: 'skip', detail: note('skip') ?? 'skipped' };
|
|
55
|
+
}
|
|
56
|
+
const missed = note('proposed-not-met');
|
|
57
|
+
if (missed !== undefined) {
|
|
58
|
+
return { ...base, kind: 'miss', detail: missed };
|
|
59
|
+
}
|
|
60
|
+
const failed = result.status !== 'passed';
|
|
61
|
+
if (test.expectedStatus === 'failed') {
|
|
62
|
+
// a known gap is expected to fail: it failing is fine, and it passing means the entry is stale
|
|
63
|
+
return failed
|
|
64
|
+
? { ...base, kind: 'gap', detail: note('known-gap') ?? 'known gap' }
|
|
65
|
+
: { ...base, kind: 'fail', detail: 'now passes: remove it from the known gaps' };
|
|
66
|
+
}
|
|
67
|
+
return failed ? { ...base, kind: 'fail', detail: reason(result.error?.message) } : { ...base, kind: 'pass', detail: '' };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface Style {
|
|
71
|
+
color: boolean;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const paint = (style: Style, code: string, text: string): string => (style.color ? `\u001b[${code}m${text}\u001b[0m` : text);
|
|
75
|
+
export const dim = (s: Style, t: string) => paint(s, '2', t);
|
|
76
|
+
export const bold = (s: Style, t: string) => paint(s, '1', t);
|
|
77
|
+
const green = (s: Style, t: string) => paint(s, '32', t);
|
|
78
|
+
const red = (s: Style, t: string) => paint(s, '31', t);
|
|
79
|
+
const yellow = (s: Style, t: string) => paint(s, '33', t);
|
|
80
|
+
|
|
81
|
+
export function seconds(ms: number): string {
|
|
82
|
+
if (ms < 10000) {
|
|
83
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
84
|
+
}
|
|
85
|
+
const s = Math.round(ms / 1000);
|
|
86
|
+
return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${String(s % 60).padStart(2, '0')}s`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const MARK: Record<Kind, (s: Style) => string> = {
|
|
90
|
+
pass: s => green(s, '✓'),
|
|
91
|
+
fail: s => red(s, '✗'),
|
|
92
|
+
gap: s => yellow(s, '◐'),
|
|
93
|
+
miss: s => yellow(s, '◇'),
|
|
94
|
+
skip: s => dim(s, '○')
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/** The line (and, for anything but a pass, the reason under it) printed when a story finishes. */
|
|
98
|
+
export function rowLines(row: Row, style: Style, width = 100): string[] {
|
|
99
|
+
const time = dim(style, seconds(row.ms));
|
|
100
|
+
const room = Math.max(20, width - 8 - row.id.length - 10);
|
|
101
|
+
const title = row.title.length > room ? `${row.title.slice(0, room - 1)}…` : row.title;
|
|
102
|
+
const head = ` ${MARK[row.kind](style)} ${bold(style, row.id.padEnd(5))} ${title} ${time}`;
|
|
103
|
+
if (row.kind === 'pass') {
|
|
104
|
+
return [head];
|
|
105
|
+
}
|
|
106
|
+
const why = row.kind === 'fail' ? red(style, row.detail) : dim(style, row.kind === 'gap' ? `known gap: ${row.detail}` : `proposed, not met: ${row.detail}`);
|
|
107
|
+
return [head, ` ${why}`];
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The end of a run: the counts, why stories were skipped, and what to do next. */
|
|
111
|
+
export function summaryLines(rows: Row[], ms: number, style: Style): string[] {
|
|
112
|
+
const count = (k: Kind) => rows.filter(r => r.kind === k).length;
|
|
113
|
+
const parts = [
|
|
114
|
+
green(style, `${count('pass')} passed`),
|
|
115
|
+
count('fail') ? red(style, `${count('fail')} failed`) : undefined,
|
|
116
|
+
count('gap') ? yellow(style, `${count('gap')} known ${count('gap') === 1 ? 'gap' : 'gaps'}`) : undefined,
|
|
117
|
+
count('miss') ? yellow(style, `${count('miss')} proposed, not met`) : undefined,
|
|
118
|
+
count('skip') ? dim(style, `${count('skip')} skipped`) : undefined
|
|
119
|
+
].filter(Boolean) as string[];
|
|
120
|
+
|
|
121
|
+
const lines = ['', ` ${parts.join(dim(style, ' · '))} ${dim(style, seconds(ms))}`];
|
|
122
|
+
|
|
123
|
+
const skipped = new Map<string, string[]>();
|
|
124
|
+
for (const r of rows.filter(r => r.kind === 'skip')) {
|
|
125
|
+
skipped.set(r.detail, [...(skipped.get(r.detail) ?? []), r.id]);
|
|
126
|
+
}
|
|
127
|
+
if (skipped.size) {
|
|
128
|
+
lines.push('', ` ${dim(style, 'Skipped')}`);
|
|
129
|
+
for (const [why, ids] of [...skipped].sort((a, b) => b[1].length - a[1].length)) {
|
|
130
|
+
const shown = ids.length > 8 ? `${ids.slice(0, 8).join(', ')}, …` : ids.join(', ');
|
|
131
|
+
lines.push(` ${dim(style, `${String(ids.length).padStart(3)} ${why} (${shown})`)}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return lines;
|
|
135
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { FullResult, Reporter, TestCase, TestResult } from '@playwright/test/reporter';
|
|
2
|
+
import { rowLines, rowOf, Row, Style, summaryLines } from './report';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The reporter of the `rtc-conformance` command: a line per story as it finishes (skipped ones are only counted), a spinner
|
|
6
|
+
* line for the story running now, and a summary. It is not for CI logs that want the full detail: use Playwright's own
|
|
7
|
+
* reporters for that.
|
|
8
|
+
*/
|
|
9
|
+
export default class ConformanceReporter implements Reporter {
|
|
10
|
+
private rows: Row[] = [];
|
|
11
|
+
private started = Date.now();
|
|
12
|
+
private style: Style = { color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR };
|
|
13
|
+
private live = Boolean(process.stdout.isTTY);
|
|
14
|
+
private timer?: NodeJS.Timeout;
|
|
15
|
+
private frame = 0;
|
|
16
|
+
private running = '';
|
|
17
|
+
|
|
18
|
+
printsToStdio(): boolean {
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
onTestBegin(test: TestCase): void {
|
|
23
|
+
const id = test.title.split(' · ')[0];
|
|
24
|
+
this.running = `${id} ${test.title.slice(id.length + 3)}`;
|
|
25
|
+
if (this.live) {
|
|
26
|
+
this.timer = setInterval(() => this.spin(), 120);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
private spin(): void {
|
|
31
|
+
const frames = '⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏';
|
|
32
|
+
const width = process.stdout.columns ?? 100;
|
|
33
|
+
const text = ` ${frames[this.frame++ % frames.length]} ${this.running}`.slice(0, width - 1);
|
|
34
|
+
process.stdout.write(`\r\u001b[2K\u001b[2m${text}\u001b[0m`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
private clear(): void {
|
|
38
|
+
if (this.timer) {
|
|
39
|
+
clearInterval(this.timer);
|
|
40
|
+
this.timer = undefined;
|
|
41
|
+
}
|
|
42
|
+
if (this.live) {
|
|
43
|
+
process.stdout.write('\r\u001b[2K');
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
onTestEnd(test: TestCase, result: TestResult): void {
|
|
48
|
+
this.clear();
|
|
49
|
+
const row = rowOf(test, result);
|
|
50
|
+
this.rows.push(row);
|
|
51
|
+
if (row.kind !== 'skip') {
|
|
52
|
+
process.stdout.write(rowLines(row, this.style, process.stdout.columns ?? 100).join('\n') + '\n');
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
onEnd(_result: FullResult): void {
|
|
57
|
+
this.clear();
|
|
58
|
+
process.stdout.write(summaryLines(this.rows, Date.now() - this.started, this.style).join('\n') + '\n');
|
|
59
|
+
}
|
|
60
|
+
}
|