@zsailer/jupyter-server-side-documents-conformance 0.0.0-stage → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COVERAGE.md +56 -0
- package/README.md +139 -2
- package/adapters/generic.ts +17 -0
- package/adapters/http.ts +84 -0
- package/bin/args.mjs +202 -0
- package/bin/conformance.mjs +303 -0
- package/bin/picker.mjs +256 -0
- package/bin/welcome.mjs +95 -0
- package/known-gaps/README.md +12 -0
- package/known-gaps/jsd.json +1 -0
- package/package.json +25 -4
- package/playwright.config.ts +32 -0
- package/runtime/adapter.ts +60 -0
- package/runtime/cells.ts +99 -0
- package/runtime/network.ts +84 -0
- package/runtime/recording.ts +159 -0
- package/runtime/report.ts +135 -0
- package/runtime/reporter.ts +60 -0
- package/runtime/story.ts +482 -0
- package/runtime/tab.ts +364 -0
- package/runtime/types.ts +63 -0
- package/scripts/summarize.py +100 -0
- package/server/conformance_config.py +48 -0
- package/server/jsd_control.py +91 -0
- package/stories.json +981 -0
- package/tests/generated/ai.spec.ts +450 -0
- package/tests/generated/crash.spec.ts +267 -0
- package/tests/generated/edges.spec.ts +549 -0
- package/tests/generated/files.spec.ts +237 -0
- package/tests/generated/handshake.spec.ts +429 -0
- package/tests/generated/lineage.spec.ts +241 -0
- package/tests/generated/outputs.spec.ts +258 -0
- package/tests/generated/rebuild.spec.ts +446 -0
- package/tests/generated/residual.spec.ts +356 -0
- package/tests/generated/stop.spec.ts +168 -0
- package/tsconfig.json +14 -0
package/COVERAGE.md
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# What the compliance tests need from a backend
|
|
2
|
+
|
|
3
|
+
<!-- GENERATED by compliance/generate.py. Do not edit. -->
|
|
4
|
+
|
|
5
|
+
69 stories. A story runs when the backend's adapter can do everything in its **needs**; otherwise it is skipped, and the report says what was missing.
|
|
6
|
+
|
|
7
|
+
| needs | stories | ids |
|
|
8
|
+
|---|---:|---|
|
|
9
|
+
| _nothing beyond a plain JupyterLab_ | 10 | C3, C2, C1, C, E, M1, Z1, Z2, Z3, Z4 |
|
|
10
|
+
| agent | 3 | AI1, AI5, AI6 |
|
|
11
|
+
| crashServer | 4 | S2, S3, Q1, Z6 |
|
|
12
|
+
| fileSystem | 1 | Z5 |
|
|
13
|
+
| releaseRoom | 19 | A, H2, H3, H4, K, K2, N1, M2, R1, W1, W2, W3, S1, S5, S7, X1, X2, X4, Z7 |
|
|
14
|
+
| wireFaults | 4 | D1, D2, F, G3 |
|
|
15
|
+
| agent, crashServer | 1 | AI7 |
|
|
16
|
+
| agent, fileSystem | 1 | AI9 |
|
|
17
|
+
| crashServer, fileSystem | 1 | S9 |
|
|
18
|
+
| crashServer, historyRecord | 1 | S6 |
|
|
19
|
+
| fileSystem, historyRecord | 2 | S10, S11 |
|
|
20
|
+
| fileSystem, releaseRoom | 3 | P1, S4, S8 |
|
|
21
|
+
| historyRecord, releaseRoom | 11 | S12, S13, S14, T1, X5, O1, O2, O3, O4, O5, O6 |
|
|
22
|
+
| releaseRoom, wireFaults | 4 | B, G, G2, X3 |
|
|
23
|
+
| agent, fileSystem, releaseRoom | 3 | AI2, AI3, AI4 |
|
|
24
|
+
| agent, fileSystem, releaseRoom, wireFaults | 1 | AI8 |
|
|
25
|
+
|
|
26
|
+
## Each capability
|
|
27
|
+
|
|
28
|
+
| capability | stories that need it |
|
|
29
|
+
|---|---:|
|
|
30
|
+
| agent | 9 |
|
|
31
|
+
| crashServer | 7 |
|
|
32
|
+
| fileSystem | 12 |
|
|
33
|
+
| historyRecord | 14 |
|
|
34
|
+
| releaseRoom | 41 |
|
|
35
|
+
| wireFaults | 9 |
|
|
36
|
+
|
|
37
|
+
## Checked loosely
|
|
38
|
+
|
|
39
|
+
Stories where the storyboard describes a result in words that a check cannot be made from. Give `oracle.yaml` an entry to make them exact.
|
|
40
|
+
|
|
41
|
+
_none_
|
|
42
|
+
|
|
43
|
+
## Not checkable from the user's side
|
|
44
|
+
|
|
45
|
+
These describe something inside the server that a person cannot cause or see. They are generated, and skipped with the reason.
|
|
46
|
+
|
|
47
|
+
- **X1**: the server's internal queue when it shuts a notebook down: no one can put an update there and shut down at that moment
|
|
48
|
+
- **X2**: the server's internal queue when it shuts a notebook down: no one can send a damaged message from the browser
|
|
49
|
+
- **X3**: work the server registers for its own shutdown: nothing a person does or sees
|
|
50
|
+
- **X4**: the server's internal queue when it shuts a notebook down: whether an update is still waiting when a tab closes is a race
|
|
51
|
+
|
|
52
|
+
## Stories that are still open questions
|
|
53
|
+
|
|
54
|
+
The proposed answer is held to, unless `COMPLIANCE_DECISIONS` picks another.
|
|
55
|
+
|
|
56
|
+
P1, Z1, Z2, Z3, Z4, Z5, Z6, Z7, AI5, AI6, AI7
|
package/README.md
CHANGED
|
@@ -1,3 +1,140 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Compliance tests
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Tests that check **what a person experiences** in JupyterLab when documents are shared in real time, whatever server-side implementation sits behind it. They are generated from the [storyboards](../examples/storyboards/), so the page people read and the tests a backend must pass are the same stories.
|
|
4
|
+
|
|
5
|
+
> **Status:** the generator, the runtime and the adapter are tested without a browser. Runs against jupyter-server-documents have found real differences and harness bugs; see [Running it](#running-it). For how to run the tests against a codebase, start with [`docs/running-against-a-backend.md`](../docs/running-against-a-backend.md); to add another backend, [`docs/backend-contract.md`](../docs/backend-contract.md).
|
|
6
|
+
|
|
7
|
+
## The idea
|
|
8
|
+
|
|
9
|
+
A storyboard says what happens and how it must end:
|
|
10
|
+
|
|
11
|
+
> Two tabs have the notebook open. The second loses its connection and the user types into c1. Meanwhile the first tab deletes c1. Then the second tab reconnects. — *the deletion wins, as it does between two people who are online.*
|
|
12
|
+
|
|
13
|
+
`generate.py` turns each story into a [Galata](https://github.com/jupyterlab/jupyterlab/tree/main/galata) test that does exactly that in real browsers — one browser per tab, so one can go offline on its own — with real keystrokes, and at the end checks what **every copy** holds: each open tab, a fresh tab (what the server holds), and the file.
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
await s.step(4, "Second tab", "types “+B-TYPED” at the end of c1", async () => {
|
|
17
|
+
await tab2.type(1, " +B-TYPED");
|
|
18
|
+
await s.expectTab(tab2, ["c0", "c1 +B-TYPED", "c2"]);
|
|
19
|
+
});
|
|
20
|
+
...
|
|
21
|
+
await s.ends([
|
|
22
|
+
{ tab: ["c0", "c2"], tab2: ["c0", "c2"], server: ["c0", "c2"], file: ["c0", "c2"] }, // proposed
|
|
23
|
+
{ tab: ["c0", E.contains("c1", "+B-TYPED"), "c2"], ... }, // the alternative
|
|
24
|
+
]);
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Nothing in a test names an implementation. The tests see the browser, the Contents API and, where a story needs it, an **adapter**.
|
|
28
|
+
|
|
29
|
+
## What a backend must be able to do
|
|
30
|
+
|
|
31
|
+
A story needs the backend's help only to *cause* an event: the server letting go of a notebook, dying, a file changed from outside. A backend's **adapter** declares which of these it can do. A story whose needs the adapter does not meet is **skipped, with the reason**, so a report says what was verified and what was not — not "all green" because nothing ran.
|
|
32
|
+
|
|
33
|
+
| capability | what it is |
|
|
34
|
+
|---|---|
|
|
35
|
+
| *(none)* | people typing, adding, moving, deleting; tabs going offline, reconnecting, opening, closing |
|
|
36
|
+
| `releaseRoom` | the server lets go of a notebook (saving it); the next client makes it build it again from the file |
|
|
37
|
+
| `fileSystem` | the file is changed, removed or restored from outside Jupyter |
|
|
38
|
+
| `crashServer` | the server dies without saving, and starts again |
|
|
39
|
+
| `wireFaults` | a message between a tab and the server is lost or late |
|
|
40
|
+
| `agent` | an AI agent edits the notebook, through the server or the file |
|
|
41
|
+
| `historyRecord` | the server keeps a history record next to the file (optional) |
|
|
42
|
+
| `serverState` | *(optional)* the server can say what its own copy holds; otherwise a fresh tab is opened to see |
|
|
43
|
+
|
|
44
|
+
[`COVERAGE.md`](COVERAGE.md) is generated with the tests (`npm run generate`; none of the generated files are committed): it lists what every story needs and how many stories each capability unlocks. Today `releaseRoom` is by far the most needed, because most stories turn on the server rebuilding a notebook from its file. Making that portable (for instance, by restarting the server) is the most useful thing a backend can offer.
|
|
45
|
+
|
|
46
|
+
## Adapters
|
|
47
|
+
|
|
48
|
+
| adapter | does | use |
|
|
49
|
+
|---|---|---|
|
|
50
|
+
| `http` (default) | asks the server what it can do (`GET /conformance/capabilities`) and uses the [control contract](../docs/backend-contract.md) for it; with no answer it does nothing beyond the browser and the Contents API | any backend, with or without the contract |
|
|
51
|
+
| `generic` | never asks the server for anything | a server you know has no contract |
|
|
52
|
+
| your own | `--adapter ./path/to/adapter.ts` (`COMPLIANCE_ADAPTER`): a module whose default export is a [`BackendAdapter`](runtime/adapter.ts) | a backend that needs more than HTTP, such as one that can crash and restart its server |
|
|
53
|
+
|
|
54
|
+
The suite takes nothing from a backend's repo. For jupyter-server-documents it brings its own server config
|
|
55
|
+
([`server/conformance_config.py`](server/conformance_config.py)) and its own side of the contract
|
|
56
|
+
([`server/jsd_control.py`](server/jsd_control.py)).
|
|
57
|
+
|
|
58
|
+
## Running it
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
rtc-conformance run # start jupyter-server-documents, run every story, stop it
|
|
62
|
+
rtc-conformance run Z3 C2 # just these
|
|
63
|
+
rtc-conformance run --url http://localhost:8888 # a server that is already running
|
|
64
|
+
rtc-conformance run --start "my-server --port {port}" # your own command that starts one
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The command is `bin/conformance.mjs` (`node compliance/bin/conformance.mjs` in a checkout); see
|
|
68
|
+
[`docs/running-against-a-backend.md`](../docs/running-against-a-backend.md). Behind it is Playwright, which you can
|
|
69
|
+
also run yourself against a server that is already up:
|
|
70
|
+
|
|
71
|
+
```sh
|
|
72
|
+
cd compliance
|
|
73
|
+
npm install && npx playwright install chromium
|
|
74
|
+
npm run generate
|
|
75
|
+
COMPLIANCE_BASE_URL=http://localhost:8899 COMPLIANCE_ROOT_DIR=/tmp/compliance-root npx playwright test
|
|
76
|
+
npx playwright test -g "Z3" # one story
|
|
77
|
+
npx playwright test --grep @group-residual # one group
|
|
78
|
+
npx playwright test --grep-invert @needs-alignment # only the settled stories
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
The server must be set up the way JupyterLab's own UI tests set one up (no login, the app exposed to the page):
|
|
82
|
+
`server/conformance_config.py` does it for a Jupyter Server based backend, with `CONFORMANCE_CONTROL=jsd_control` to load
|
|
83
|
+
the backend's side of the contract.
|
|
84
|
+
|
|
85
|
+
| variable | |
|
|
86
|
+
|---|---|
|
|
87
|
+
| `COMPLIANCE_BASE_URL` | where the server is (default `http://localhost:8899`) |
|
|
88
|
+
| `COMPLIANCE_ADAPTER` | `http` (default), `generic`, or a path |
|
|
89
|
+
| `COMPLIANCE_KNOWN_GAPS` | a JSON file of accepted stories this backend is known not to meet yet |
|
|
90
|
+
| `COMPLIANCE_ROOT_DIR` | the folder the server serves; lets tests change the file from outside Jupyter |
|
|
91
|
+
| `COMPLIANCE_DECISIONS` | `'{"Z3":1}'` holds Z3 to its second option instead of the proposed one |
|
|
92
|
+
| `COMPLIANCE_TIMEOUT_MS`, `COMPLIANCE_SAVE_TIMEOUT_MS`, `COMPLIANCE_STABLE_MS`, `COMPLIANCE_SETTLE_MS` | waits (default 20 s, 40 s, 1.5 s, 0.75 s) |
|
|
93
|
+
|
|
94
|
+
## Reading a run
|
|
95
|
+
|
|
96
|
+
`rtc-conformance run` ends with a summary; the same is available after any run with `python3 -I compliance/scripts/summarize.py`: one line per story (passed, or failed with the first line of why), what was skipped and what each skip lacked, and, for every story that is still an open question, **which of the storyboard's options the backend does**. A failing step names the storyboard step it was in, and every open tab is photographed at that moment.
|
|
97
|
+
|
|
98
|
+
## Watching a story
|
|
99
|
+
|
|
100
|
+
A story happens in several browsers at once, and Playwright films only the page it makes itself. So the tests film every tab (`runtime/recording.ts`) and write, next to the results of each story, **`watch.html`**: the videos of all the tabs side by side, on one clock, with the storyboard step shown underneath and a list of the steps. It is also attached to the report. `rtc-conformance run` prints where the pages of failed stories are, and `--open` opens them.
|
|
101
|
+
|
|
102
|
+
## Going offline
|
|
103
|
+
|
|
104
|
+
A tab goes offline by having **its own browser** cut off, not with Playwright's `context.setOffline()`: in Chromium that makes every *other* browser opened while it is on start about seven times slower (measured: 5.9 s against 0.8 s, `tests/probe-offline.spec.ts`), which made any story that opens a tab after another has gone offline time out. `runtime/network.ts` refuses the browser's requests the way a real outage does (`ERR_INTERNET_DISCONNECTED`), closes its WebSockets, and makes any it opens fail at once, all inside that one browser. `COMPLIANCE_OFFLINE=emulate` brings back Playwright's own, for comparison. When a tab is back online and JupyterLab asks the person to click "Reconnect", the test clicks it, and the run says so.
|
|
105
|
+
|
|
106
|
+
## Open questions in the storyboards
|
|
107
|
+
|
|
108
|
+
A story marked *Needs alignment* has a proposed answer and alternatives. The test holds the backend to the **proposed** answer, tags itself `@needs-alignment`, and says so in its annotations. When the question is settled, the storyboard changes, the tests are generated again, and the answer is enforced everywhere at once.
|
|
109
|
+
|
|
110
|
+
## How a story becomes a check
|
|
111
|
+
|
|
112
|
+
For each step the generator knows what to do (`kind`) and, by comparing the cells of a lane before and after, **which keys to press**: add a cell *there*, type at the end of *that* cell, delete *this* one, move *that* one (the cell the story says moved, not the ones around it). It then writes what each lane must show after the steps a person does, and at the end.
|
|
113
|
+
|
|
114
|
+
- **Names to sources.** A cell `c1` is a code cell whose source is `c1`. Typed text `+B-TYPED` is typed after a space, so the cell reads `c1 +B-TYPED`. A note that rewrites a cell (`EXTERNAL EDIT`, `total = a - b`) is its new source.
|
|
115
|
+
- **Exact where the story is exact.** A cell is matched exactly (whitespace around it ignored). Typing from two people may land in either order. A recovered copy is found by what it holds, whatever it is wrapped in. Order is ignored where the story says a cell "stands at one of the two places".
|
|
116
|
+
- **What must not happen is enforced by counting.** The list at the end is exact, so a duplicate (`!c0=twice`) fails it; the check then waits to see that the notebook **stays** as it is, which catches a duplicate that arrives late.
|
|
117
|
+
- **What the storyboard leaves to words** goes in [`oracle.yaml`](oracle.yaml). Until a story has what it needs there, `COVERAGE.md` lists it under *checked loosely*.
|
|
118
|
+
|
|
119
|
+
## What is not here yet
|
|
120
|
+
|
|
121
|
+
- `restartServer` / a portable `releaseRoom`: today only an adapter that has an endpoint can release a room.
|
|
122
|
+
- Wire faults (`wireFaults`): lose or delay one message of the handshake. `page.routeWebSocket` can do it from the browser, but the message to lose is protocol-specific, so it belongs in an adapter.
|
|
123
|
+
- Agents, and history records: adapter-specific by nature.
|
|
124
|
+
- Outputs: the stories that mention outputs are checked for their cells, not their outputs.
|
|
125
|
+
|
|
126
|
+
## Layout
|
|
127
|
+
|
|
128
|
+
```
|
|
129
|
+
compliance/
|
|
130
|
+
generate.py the storyboards → tests/generated/*.spec.ts, stories.json, COVERAGE.md (--check for CI)
|
|
131
|
+
oracle.yaml what the storyboards leave to words
|
|
132
|
+
runtime/ Tab, Story, the matcher, the adapter interface
|
|
133
|
+
adapters/ http (the default: speaks the control contract), generic
|
|
134
|
+
server/ what the tests bring to a Jupyter Server backend: its config, and the jsd side of the contract
|
|
135
|
+
bin/ the `rtc-conformance` command
|
|
136
|
+
known-gaps/ accepted stories a backend is known not to meet yet
|
|
137
|
+
tests/generated/ GENERATED: one spec per group of stories. Do not edit.
|
|
138
|
+
unit/ tests of the matcher (no browser)
|
|
139
|
+
scripts/ package.py (the npm package), summarize.py
|
|
140
|
+
```
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { BackendAdapter } from '../runtime/adapter';
|
|
2
|
+
import type { Capability } from '../runtime/types';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Any JupyterLab with real-time collaboration, and nothing else known about it.
|
|
6
|
+
*
|
|
7
|
+
* It can do what the browser and the Contents API can do, so it runs every story whose steps are people typing,
|
|
8
|
+
* adding, moving and deleting cells, tabs going offline and coming back, and tabs opening and closing. With
|
|
9
|
+
* `COMPLIANCE_ROOT_DIR` (the folder the server serves) it can also change the file from outside.
|
|
10
|
+
*/
|
|
11
|
+
const capabilities = new Set<Capability>(['contents']);
|
|
12
|
+
if (process.env.COMPLIANCE_ROOT_DIR) {
|
|
13
|
+
capabilities.add('fileSystem');
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const adapter: BackendAdapter = { name: 'generic', capabilities };
|
|
17
|
+
export default adapter;
|
package/adapters/http.ts
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { AdapterContext, BackendAdapter } from '../runtime/adapter';
|
|
2
|
+
import type { Capability } from '../runtime/types';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Any backend that implements the conformance control contract (docs/backend-contract.md): three endpoints under
|
|
6
|
+
* `/conformance/` that let a test ask what the server holds for a notebook and make it let go of one. What the backend
|
|
7
|
+
* can do is not written here: the server says so (`GET /conformance/capabilities`), and a server without the
|
|
8
|
+
* endpoints is a backend that can do nothing beyond what a browser and the Contents API can.
|
|
9
|
+
*
|
|
10
|
+
* GET /conformance/capabilities { backend, capabilities: [...] }
|
|
11
|
+
* GET /conformance/room?path=… { cells: [source, ...] }, or 404 if the server holds no copy
|
|
12
|
+
* POST /conformance/release-room?path=… the server saves the notebook and lets go of it
|
|
13
|
+
*/
|
|
14
|
+
const KNOWN: ReadonlySet<string> = new Set(['serverState', 'releaseRoom']);
|
|
15
|
+
|
|
16
|
+
const capabilities = new Set<Capability>(['contents']);
|
|
17
|
+
if (process.env.COMPLIANCE_ROOT_DIR) {
|
|
18
|
+
capabilities.add('fileSystem');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Ask the server what it offers. A server that does not answer offers nothing. */
|
|
22
|
+
export async function discover(baseURL: string, fetchImpl: typeof fetch = fetch): Promise<{ backend?: string; offered: Capability[] }> {
|
|
23
|
+
try {
|
|
24
|
+
const res = await fetchImpl(`${baseURL.replace(/\/$/, '')}/conformance/capabilities`, { signal: AbortSignal.timeout(5000) });
|
|
25
|
+
if (!res.ok) {
|
|
26
|
+
return { offered: [] };
|
|
27
|
+
}
|
|
28
|
+
const body: any = await res.json();
|
|
29
|
+
const offered = (Array.isArray(body.capabilities) ? body.capabilities : []).filter((c: unknown) => typeof c === 'string' && KNOWN.has(c));
|
|
30
|
+
return { backend: typeof body.backend === 'string' ? body.backend : undefined, offered: offered as Capability[] };
|
|
31
|
+
} catch {
|
|
32
|
+
return { offered: [] };
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function call(ctx: AdapterContext, subpath: string, method: 'GET' | 'POST'): Promise<{ status: number; body: any }> {
|
|
37
|
+
return ctx.admin.evaluate(
|
|
38
|
+
async ({ subpath, method }) => {
|
|
39
|
+
const settings = (window as any).jupyterapp.serviceManager.serverSettings;
|
|
40
|
+
const xsrf = document.cookie.match(/\b_xsrf=([^;]+)/)?.[1] ?? '';
|
|
41
|
+
const headers: Record<string, string> = { 'X-XSRFToken': xsrf };
|
|
42
|
+
if (settings.token) {
|
|
43
|
+
headers['Authorization'] = `token ${settings.token}`;
|
|
44
|
+
}
|
|
45
|
+
const res = await fetch(settings.baseUrl + subpath, { method, headers });
|
|
46
|
+
return { status: res.status, body: res.ok ? await res.json() : null };
|
|
47
|
+
},
|
|
48
|
+
{ subpath, method }
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const adapter: BackendAdapter = {
|
|
53
|
+
// the backend's own name, once the server has said it: known-gaps/<name>.json is looked up by it
|
|
54
|
+
name: 'unknown',
|
|
55
|
+
capabilities,
|
|
56
|
+
|
|
57
|
+
async discover(baseURL) {
|
|
58
|
+
const { backend, offered } = await discover(baseURL);
|
|
59
|
+
if (backend) {
|
|
60
|
+
(adapter as { name: string }).name = backend;
|
|
61
|
+
}
|
|
62
|
+
offered.forEach(c => capabilities.add(c));
|
|
63
|
+
},
|
|
64
|
+
|
|
65
|
+
async serverCells(ctx, path) {
|
|
66
|
+
const { status, body } = await call(ctx, `conformance/room?path=${encodeURIComponent(path)}`, 'GET');
|
|
67
|
+
if (status === 404) {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
if (status !== 200) {
|
|
71
|
+
throw new Error(`conformance/room -> HTTP ${status}`);
|
|
72
|
+
}
|
|
73
|
+
return body.cells as string[];
|
|
74
|
+
},
|
|
75
|
+
|
|
76
|
+
async releaseRoom(ctx, path) {
|
|
77
|
+
const { status } = await call(ctx, `conformance/release-room?path=${encodeURIComponent(path)}`, 'POST');
|
|
78
|
+
if (status !== 200 && status !== 404) {
|
|
79
|
+
throw new Error(`conformance/release-room -> HTTP ${status}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
export default adapter;
|
package/bin/args.mjs
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
// Argument parsing and help for the `rtc-conformance` command. No dependencies, and nothing here touches the machine.
|
|
2
|
+
|
|
3
|
+
// Help is written to fit an ordinary terminal: no line is longer than 78 characters (a test keeps it so).
|
|
4
|
+
const MAIN = `rtc-conformance: hold a collaboration backend to what people should see
|
|
5
|
+
|
|
6
|
+
What it does
|
|
7
|
+
Starts your server (or uses one that is running), opens real browsers and
|
|
8
|
+
plays each story the way a person would: a tab goes offline and keeps
|
|
9
|
+
typing, the server lets go of a notebook, a file changes underneath. Then it
|
|
10
|
+
tells you which stories your backend meets, and shows you the ones it
|
|
11
|
+
doesn't.
|
|
12
|
+
|
|
13
|
+
Start here
|
|
14
|
+
rtc-conformance pick choose stories from a list, then run them
|
|
15
|
+
rtc-conformance run Z3 run one story
|
|
16
|
+
rtc-conformance run run every story (takes a while)
|
|
17
|
+
|
|
18
|
+
Commands
|
|
19
|
+
run [story ...] run the stories named, or all of them
|
|
20
|
+
pick choose stories from a list (space selects, enter runs)
|
|
21
|
+
list see every story, and what each needs from a backend
|
|
22
|
+
help [command] more about a command, with examples
|
|
23
|
+
|
|
24
|
+
Learn more
|
|
25
|
+
rtc-conformance help run every option for running, and examples
|
|
26
|
+
docs/running-against-a-backend.md a longer guide
|
|
27
|
+
`;
|
|
28
|
+
|
|
29
|
+
const RUN = `rtc-conformance run: run the stories against a backend
|
|
30
|
+
|
|
31
|
+
Usage
|
|
32
|
+
rtc-conformance run [story ...] [options]
|
|
33
|
+
|
|
34
|
+
Which stories
|
|
35
|
+
rtc-conformance run every story
|
|
36
|
+
rtc-conformance run Z3 C2 these two
|
|
37
|
+
rtc-conformance run 'AI*' every story whose id starts with AI
|
|
38
|
+
rtc-conformance run --group edges one group of the page (see \`list\`)
|
|
39
|
+
rtc-conformance run -i pick them from a list first
|
|
40
|
+
|
|
41
|
+
Which server
|
|
42
|
+
--backend <name> how to start the server: jsd (default), the built-in
|
|
43
|
+
way to start jupyter-server-documents
|
|
44
|
+
--env <dir> the Python environment with jupyter_server_documents
|
|
45
|
+
(default: the one on your PATH)
|
|
46
|
+
--start <command> your own command that starts a JupyterLab on {port}.
|
|
47
|
+
The backend must implement the control contract
|
|
48
|
+
(docs/backend-contract.md) for the stories that need
|
|
49
|
+
its help
|
|
50
|
+
--url <url> use a server that is already running
|
|
51
|
+
--root-dir <dir> the folder that server serves (lets stories change the
|
|
52
|
+
file from outside)
|
|
53
|
+
--adapter <path> your own adapter module (default: ask the server what
|
|
54
|
+
it can do)
|
|
55
|
+
|
|
56
|
+
What you see
|
|
57
|
+
--headed show the browsers
|
|
58
|
+
--open open the side-by-side video page of each failed story
|
|
59
|
+
--out <dir> where videos and results go
|
|
60
|
+
(default ./conformance-results)
|
|
61
|
+
--known-gaps <file> JSON of accepted stories this backend does not meet yet
|
|
62
|
+
(default ./conformance-known-gaps.json)
|
|
63
|
+
|
|
64
|
+
Examples
|
|
65
|
+
rtc-conformance run Z3 --env ~/envs/jsd
|
|
66
|
+
rtc-conformance run --group handshake --headed
|
|
67
|
+
rtc-conformance run --url http://localhost:8888
|
|
68
|
+
rtc-conformance run --start "my-server --port {port}"
|
|
69
|
+
|
|
70
|
+
A run ends with a line per story and a summary. The exit code is 0 when no
|
|
71
|
+
story failed, so it works as a CI step.
|
|
72
|
+
✓ passes
|
|
73
|
+
✗ fails
|
|
74
|
+
◐ known gap: fails as expected
|
|
75
|
+
◇ proposed case, not met: reported, never fails the run
|
|
76
|
+
skipped: the backend cannot do something the story needs; the summary says
|
|
77
|
+
what
|
|
78
|
+
`;
|
|
79
|
+
|
|
80
|
+
const PICK = `rtc-conformance pick: choose stories from a list, then run them
|
|
81
|
+
|
|
82
|
+
Usage
|
|
83
|
+
rtc-conformance pick [options]
|
|
84
|
+
It takes the same options as \`rtc-conformance run\`.
|
|
85
|
+
|
|
86
|
+
In the list
|
|
87
|
+
↑ ↓ (or j k) move
|
|
88
|
+
space select or unselect the story under the arrow
|
|
89
|
+
a select everything shown (again: unselect it)
|
|
90
|
+
g select the whole group the arrow is in
|
|
91
|
+
/ filter by what you type: an id, a word in a title, a group,
|
|
92
|
+
a need like releaseRoom (escape clears it)
|
|
93
|
+
c clear the selection
|
|
94
|
+
enter run what is selected (or the story under the arrow if
|
|
95
|
+
nothing is)
|
|
96
|
+
q or escape leave without running
|
|
97
|
+
|
|
98
|
+
Needs a terminal. In a script, name the stories:
|
|
99
|
+
rtc-conformance run Z3 C2
|
|
100
|
+
`;
|
|
101
|
+
|
|
102
|
+
const LIST = `rtc-conformance list: see every story and what each needs from a backend
|
|
103
|
+
|
|
104
|
+
Usage
|
|
105
|
+
rtc-conformance list [--group <name>] [--filter <text>]
|
|
106
|
+
|
|
107
|
+
--group <name> only the stories of one group
|
|
108
|
+
--filter <text> only the stories whose id, title or needs contain the text
|
|
109
|
+
(for example: releaseRoom, offline, AI)
|
|
110
|
+
|
|
111
|
+
A story that "needs releaseRoom" is skipped on a backend that cannot make the
|
|
112
|
+
server let go of a notebook.
|
|
113
|
+
`;
|
|
114
|
+
|
|
115
|
+
const TOPICS = { main: MAIN, run: RUN, pick: PICK, list: LIST };
|
|
116
|
+
export const COMMANDS = ['run', 'pick', 'list', 'help'];
|
|
117
|
+
|
|
118
|
+
export function helpFor(topic = 'main') {
|
|
119
|
+
return TOPICS[topic] ?? MAIN;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const VALUE = new Set(['backend', 'url', 'start', 'adapter', 'root-dir', 'group', 'known-gaps', 'env', 'out', 'filter']);
|
|
123
|
+
const SWITCH = new Set(['headed', 'open', 'help', 'version', 'interactive']);
|
|
124
|
+
const SHORT = { h: 'help', v: 'version', i: 'interactive' };
|
|
125
|
+
|
|
126
|
+
/** The closest candidate to a mistyped word, if one is close enough to be what was meant. */
|
|
127
|
+
export function suggest(word, candidates) {
|
|
128
|
+
const distance = (a, b) => {
|
|
129
|
+
const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]);
|
|
130
|
+
for (let j = 1; j <= b.length; j++) {
|
|
131
|
+
d[0][j] = j;
|
|
132
|
+
}
|
|
133
|
+
for (let i = 1; i <= a.length; i++) {
|
|
134
|
+
for (let j = 1; j <= b.length; j++) {
|
|
135
|
+
d[i][j] = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return d[a.length][b.length];
|
|
139
|
+
};
|
|
140
|
+
const ranked = [...candidates].map(c => [c, distance(word, c)]).sort((x, y) => x[1] - y[1]);
|
|
141
|
+
return ranked.length && ranked[0][1] <= Math.max(1, Math.floor(word.length / 3)) ? ranked[0][0] : undefined;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** @returns {{command: string, stories: string[], options: Record<string, string | boolean>, topic: string}} */
|
|
145
|
+
export function parse(argv) {
|
|
146
|
+
const options = {};
|
|
147
|
+
const positional = [];
|
|
148
|
+
for (let i = 0; i < argv.length; i++) {
|
|
149
|
+
const arg = argv[i];
|
|
150
|
+
if (arg.startsWith('--')) {
|
|
151
|
+
const [name, inline] = arg.slice(2).split(/=(.*)/s, 2);
|
|
152
|
+
if (SWITCH.has(name)) {
|
|
153
|
+
options[name] = true;
|
|
154
|
+
} else if (VALUE.has(name)) {
|
|
155
|
+
const value = inline ?? argv[++i];
|
|
156
|
+
if (value === undefined || value.startsWith('--')) {
|
|
157
|
+
throw new Error(`--${name} needs a value`);
|
|
158
|
+
}
|
|
159
|
+
options[name] = value;
|
|
160
|
+
} else {
|
|
161
|
+
const close = suggest(name, [...SWITCH, ...VALUE]);
|
|
162
|
+
throw new Error(`unknown option --${name}${close ? `. Did you mean --${close}?` : ''}`);
|
|
163
|
+
}
|
|
164
|
+
} else if (/^-[a-z]$/.test(arg) && SHORT[arg[1]]) {
|
|
165
|
+
options[SHORT[arg[1]]] = true;
|
|
166
|
+
} else if (arg.startsWith('-')) {
|
|
167
|
+
throw new Error(`unknown option ${arg}`);
|
|
168
|
+
} else {
|
|
169
|
+
positional.push(arg);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
if (positional.length === 0 && !options.help && !options.version) {
|
|
173
|
+
return { command: 'welcome', stories: [], options, topic: 'main' };
|
|
174
|
+
}
|
|
175
|
+
const [first = 'help', ...rest] = positional;
|
|
176
|
+
if (options.version && !options.help) {
|
|
177
|
+
return { command: 'version', stories: [], options, topic: 'main' };
|
|
178
|
+
}
|
|
179
|
+
if (first === 'help') {
|
|
180
|
+
return { command: 'help', stories: [], options, topic: rest[0] ?? 'main' };
|
|
181
|
+
}
|
|
182
|
+
if (!COMMANDS.includes(first)) {
|
|
183
|
+
const close = suggest(first, COMMANDS);
|
|
184
|
+
throw new Error(`unknown command "${first}"${close ? `. Did you mean \`${close}\`?` : ''}`);
|
|
185
|
+
}
|
|
186
|
+
if (options.help) {
|
|
187
|
+
return { command: 'help', stories: [], options, topic: first };
|
|
188
|
+
}
|
|
189
|
+
return { command: first, stories: rest, options, topic: first };
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** The Playwright --grep for the stories named (ids, or a prefix with *) in a group. Undefined means all of them. */
|
|
193
|
+
export function grepFor(stories, group) {
|
|
194
|
+
const ids = stories.length
|
|
195
|
+
? `(?:^| )(?:${stories.map(s => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^ ]*')).join('|')}) · `
|
|
196
|
+
: '';
|
|
197
|
+
const tag = group ? `^(?=.*@group-${group.replace(/[^a-z0-9-]/gi, '')})` : '';
|
|
198
|
+
if (!ids && !tag) {
|
|
199
|
+
return undefined;
|
|
200
|
+
}
|
|
201
|
+
return tag ? tag + '.*' + ids : ids;
|
|
202
|
+
}
|