@zsailer/jupyter-server-side-documents-conformance 0.0.0-stage → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,482 @@
1
+ import { expect, IJupyterLabPageFixture, test } from '@jupyterlab/galata';
2
+ import type { Browser, TestInfo } from '@playwright/test';
3
+ import { ALWAYS, Capability, Ending, Expect, StoryMeta } from './types';
4
+ import { AdapterContext, BackendAdapter, loadAdapter } from './adapter';
5
+ import { matchCells, notebookJSON, showActual, showExpected } from './cells';
6
+ import { Tab, TabEnv, WAIT } from './tab';
7
+ import { keepLook, Recorder } from './recording';
8
+ import * as fs from 'node:fs';
9
+ import * as nodePath from 'node:path';
10
+
11
+ export { E, typed } from './cells';
12
+
13
+ /**
14
+ * Which option of a decision to hold the backend to, per story: `COMPLIANCE_DECISIONS='{"Z3":1}'` picks the second
15
+ * option of Z3. The default is the proposed one (the first).
16
+ */
17
+ function chosenOption(id: string, count: number): number {
18
+ const chosen = JSON.parse(process.env.COMPLIANCE_DECISIONS || '{}')[id];
19
+ return typeof chosen === 'number' && chosen >= 0 && chosen < count ? chosen : 0;
20
+ }
21
+
22
+ /** What a story's body works with. */
23
+ export class Story {
24
+ /** The path of the notebook on the server. */
25
+ path = '';
26
+ private readonly tabs = new Map<string, Tab>();
27
+
28
+ constructor(
29
+ readonly meta: StoryMeta,
30
+ readonly adapter: BackendAdapter,
31
+ private readonly ctx: AdapterContext,
32
+ private readonly env: TabEnv,
33
+ private readonly dir: string
34
+ ) {
35
+ this.recorder = env.recorder;
36
+ }
37
+
38
+ private readonly recorder: Recorder;
39
+
40
+ /** Create the notebook file the story starts from. `ids: false` makes an older notebook without cell ids. */
41
+ async notebook(sources: string[], opts: { ids?: boolean } = {}): Promise<void> {
42
+ this.path = `${this.dir}/${this.meta.id.toLowerCase()}.ipynb`;
43
+ await this.ctx.admin.contents.uploadContent(notebookJSON(sources, opts.ids !== false), 'text', this.path);
44
+ }
45
+
46
+ /** A tab of the story. It exists from the start, but is not in a browser until it opens the notebook. */
47
+ tab(name: string): Tab {
48
+ let tab = this.tabs.get(name);
49
+ if (!tab) {
50
+ tab = new Tab(name, this.env, this.path);
51
+ this.tabs.set(name, tab);
52
+ }
53
+ return tab;
54
+ }
55
+
56
+ /** One step of the storyboard: it shows in the report under the words the storyboard uses. */
57
+ async step(n: number, who: string, text: string, body: () => Promise<void>): Promise<void> {
58
+ const name = `${n}. ${who} ${text}`;
59
+ this.recorder.step(name);
60
+ try {
61
+ await test.step(name, body);
62
+ } catch (e) {
63
+ await this.showTabs(`step ${n}`);
64
+ const error = e as Error;
65
+ error.message = `step ${name}\n\n${error.message}`;
66
+ throw error;
67
+ }
68
+ }
69
+
70
+ // ---- the server and the file ----
71
+
72
+ /** The server lets go of the notebook, saving it, and builds it again from the file when it is next needed. */
73
+ async releaseRoom(): Promise<void> {
74
+ await this.adapter.releaseRoom!(this.ctx, this.path);
75
+ }
76
+
77
+ async crashServer(): Promise<void> {
78
+ await this.adapter.crashServer!(this.ctx);
79
+ }
80
+
81
+ async restartServer(): Promise<void> {
82
+ await this.adapter.restartServer!(this.ctx);
83
+ }
84
+
85
+ /** The file is changed from outside Jupyter. */
86
+ async writeFileOutside(sources: string[], opts: { ids?: boolean } = {}): Promise<void> {
87
+ const root = process.env.COMPLIANCE_ROOT_DIR!;
88
+ fs.writeFileSync(nodePath.join(root, this.path), notebookJSON(sources, opts.ids !== false));
89
+ }
90
+
91
+ async removeFileOutside(): Promise<void> {
92
+ fs.rmSync(nodePath.join(process.env.COMPLIANCE_ROOT_DIR!, this.path), { force: true });
93
+ }
94
+
95
+ /** What the file holds now. */
96
+ async fileCells(): Promise<string[] | null> {
97
+ return this.ctx.admin.evaluate(async (p: string) => {
98
+ const settings = (window as any).jupyterapp.serviceManager.serverSettings;
99
+ const res = await fetch(settings.baseUrl + 'api/contents/' + encodeURI(p) + '?content=1&type=notebook&_=' + Date.now(), {
100
+ headers: settings.token ? { Authorization: `token ${settings.token}` } : {}
101
+ });
102
+ if (!res.ok) {
103
+ return null;
104
+ }
105
+ const model = await res.json();
106
+ return model.content.cells.map((c: any) => (Array.isArray(c.source) ? c.source.join('') : c.source));
107
+ }, this.path);
108
+ }
109
+
110
+ /** What the server's own copy holds: from the backend if it can say, else what a fresh tab opens with. */
111
+ async serverCells(): Promise<string[] | null> {
112
+ if (this.adapter.capabilities.has('serverState') && this.adapter.serverCells) {
113
+ return this.adapter.serverCells(this.ctx, this.path);
114
+ }
115
+ const probe = new Tab('probe', this.env, this.path, false);
116
+ try {
117
+ await probe.open();
118
+ return await probe.cells();
119
+ } finally {
120
+ await probe.close();
121
+ }
122
+ }
123
+
124
+ /** A message between a tab and the server is lost or late. */
125
+ async wireFault(fault: 'lose-reply' | 'late-reply', tab: Tab): Promise<void> {
126
+ await this.adapter.wireFault!(this.ctx, fault, tab);
127
+ }
128
+
129
+ /** An AI agent edits the notebook so that it reads as `sources`. */
130
+ async agent(_what: string, sources: string[]): Promise<void> {
131
+ await this.adapter.agentEdit!(this.ctx, this.path, sources);
132
+ }
133
+
134
+ // ---- waiting for things to settle ----
135
+
136
+ /**
137
+ * Wait until what a tab did online has reached the server, so that what happens next (the connection dropping, the
138
+ * server letting go) finds it there. With a backend that can say what the server holds this waits for that; without
139
+ * one it gives the edit a moment.
140
+ */
141
+ async settle(tab: Tab): Promise<void> {
142
+ if (this.adapter.capabilities.has('serverState') && this.adapter.serverCells) {
143
+ const mine = await tab.cells();
144
+ await expect
145
+ .poll(async () => JSON.stringify(await this.adapter.serverCells!(this.ctx, this.path)), {
146
+ timeout: WAIT.eventually,
147
+ message: `${tab.name}: what it did never reached the server`
148
+ })
149
+ .toBe(JSON.stringify(mine));
150
+ } else {
151
+ await tab.page!.waitForTimeout(Number(process.env.COMPLIANCE_SETTLE_MS ?? 750));
152
+ }
153
+ }
154
+
155
+ /**
156
+ * Check what a place holds, waiting for it to get there and then for it to stay: a duplicate that arrives late is
157
+ * a failure too.
158
+ */
159
+ async expectCells(
160
+ where: string,
161
+ read: () => Promise<string[] | null>,
162
+ expected: Expect[],
163
+ opts: { ordered?: boolean; timeout?: number; stable?: number } = {}
164
+ ): Promise<void> {
165
+ const ordered = opts.ordered ?? this.meta.ordered ?? true;
166
+ const timeout = opts.timeout ?? WAIT.eventually;
167
+ const stable = opts.stable ?? WAIT.stable;
168
+ const started = Date.now();
169
+ let actual: string[] | null = null;
170
+ let verdict = { ok: false, reason: 'nothing to read' };
171
+ while (Date.now() - started < timeout) {
172
+ actual = await read();
173
+ verdict = actual ? matchCells(actual, expected, ordered) : { ok: false, reason: 'does not hold the notebook' };
174
+ if (verdict.ok) {
175
+ break;
176
+ }
177
+ await new Promise(r => setTimeout(r, 250));
178
+ }
179
+ const fail = (why: string) => {
180
+ throw new Error(`${where}: ${why}\n\nexpected:\n${showExpected(expected)}\n\nactual:\n${showActual(actual ?? [])}`);
181
+ };
182
+ if (!verdict.ok) {
183
+ fail(verdict.reason);
184
+ }
185
+ const settledAt = Date.now();
186
+ while (Date.now() - settledAt < stable) {
187
+ await new Promise(r => setTimeout(r, 250));
188
+ actual = await read();
189
+ const again = actual ? matchCells(actual, expected, ordered) : { ok: false, reason: 'no longer holds the notebook' };
190
+ if (!again.ok) {
191
+ fail(`${again.reason} (it changed after it had settled)`);
192
+ }
193
+ }
194
+ }
195
+
196
+ /** What a tab shows, now. */
197
+ async expectTab(tab: Tab, expected: Expect[], opts: { ordered?: boolean; timeout?: number; stable?: number } = {}): Promise<void> {
198
+ await this.expectCells(`tab "${tab.name}"`, () => tab.cells(), expected, { timeout: 5000, stable: 0, ...opts });
199
+ }
200
+
201
+ /** What a tab shows once everything has arrived. */
202
+ async expectTabEventually(tab: Tab, expected: Expect[], opts: { ordered?: boolean; timeout?: number; stable?: number } = {}): Promise<void> {
203
+ await this.expectCells(`tab "${tab.name}"`, () => tab.cells(), expected, opts);
204
+ }
205
+
206
+ /** What the file holds once it has been saved. */
207
+ async expectFile(expected: Expect[], opts: { ordered?: boolean; timeout?: number; stable?: number } = {}): Promise<void> {
208
+ await this.expectCells('the file', () => this.fileCells(), expected, { timeout: WAIT.save, ...opts });
209
+ }
210
+
211
+ /**
212
+ * Wait until every copy a person could look at (each open tab, and the server if it can be read cheaply) shows the
213
+ * same cells, and keeps showing them for a moment. A tab that was offline has not caught up until it agrees with the
214
+ * others, and a check made before then could pass on the strength of a copy that has not yet been told.
215
+ *
216
+ * Returns what they agree on, or why they never did; `compared` says whether there was more than one copy to compare.
217
+ */
218
+ private async converge(): Promise<{ compared: boolean; agreed: string[] | null; why: string | null }> {
219
+ const readers: { name: string; read: () => Promise<string[] | null> }[] = [];
220
+ for (const tab of this.tabs.values()) {
221
+ if (tab.isOpen) {
222
+ readers.push({ name: `tab "${tab.name}"`, read: () => tab.cells() });
223
+ }
224
+ }
225
+ if (this.adapter.capabilities.has('serverState') && this.adapter.serverCells) {
226
+ readers.push({ name: 'the server', read: () => this.serverCells() });
227
+ }
228
+ if (readers.length < 2) {
229
+ return { compared: false, agreed: null, why: null };
230
+ }
231
+ const started = Date.now();
232
+ let seen: (string[] | null)[] = [];
233
+ let since = 0;
234
+ while (Date.now() - started < WAIT.eventually + WAIT.stable) {
235
+ seen = await Promise.all(readers.map(r => r.read()));
236
+ const same = seen.every(s => s !== null && JSON.stringify(s) === JSON.stringify(seen[0]));
237
+ if (!same) {
238
+ since = 0;
239
+ } else if (since === 0) {
240
+ since = Date.now();
241
+ } else if (Date.now() - since >= WAIT.stable) {
242
+ return { compared: true, agreed: seen[0], why: null };
243
+ }
244
+ await new Promise(r => setTimeout(r, 250));
245
+ }
246
+ return {
247
+ compared: true,
248
+ agreed: null,
249
+ why: 'the copies never agreed with each other:\n' + readers.map((r, i) => `\n${r.name}:\n${showActual(seen[i] ?? [])}`).join('\n')
250
+ };
251
+ }
252
+
253
+ /** What every place a person could look holds now, without waiting for it to change. */
254
+ private async observe(): Promise<Record<string, string[] | null>> {
255
+ const seen: Record<string, string[] | null> = {};
256
+ for (const [name, tab] of this.tabs) {
257
+ if (tab.isOpen) {
258
+ seen[name] = await tab.cells();
259
+ }
260
+ }
261
+ if (this.adapter.capabilities.has('serverState') && this.adapter.serverCells) {
262
+ seen.server = await this.serverCells();
263
+ }
264
+ seen.file = await this.fileCells();
265
+ return seen;
266
+ }
267
+
268
+ /**
269
+ * Which of the endings the storyboard names the backend actually does, lane by lane: for an open question the useful
270
+ * answer is not pass or fail but "this is the option it implements".
271
+ */
272
+ private implemented(endings: Ending[], seen: Record<string, string[] | null>): string[] {
273
+ const ordered = this.meta.ordered ?? true;
274
+ const lines: string[] = [];
275
+ endings.forEach((ending, i) => {
276
+ const matched: string[] = [];
277
+ const missed: string[] = [];
278
+ for (const [lane, expected] of Object.entries(ending.ends)) {
279
+ const actual = seen[lane];
280
+ if (!expected || actual === undefined) {
281
+ continue; // a place that cannot be looked at
282
+ }
283
+ (actual !== null && matchCells(actual, expected, ordered).ok ? matched : missed).push(lane);
284
+ }
285
+ const what = `option ${i + 1}${i === 0 ? ' (proposed)' : ''}: ${ending.label}`;
286
+ if (matched.length && !missed.length) {
287
+ lines.push(`does ${what}`);
288
+ } else if (matched.length) {
289
+ lines.push(`partly ${what} (${matched.join(', ')}; not ${missed.join(', ')})`);
290
+ }
291
+ });
292
+ return lines.length ? lines : ['matches none of the options the storyboard names'];
293
+ }
294
+
295
+ /**
296
+ * How the story must end: what every copy holds. `endings[0]` is the proposed answer; if the story is an open
297
+ * question the others are its alternatives, and `COMPLIANCE_DECISIONS` picks which one is held to.
298
+ *
299
+ * It waits for the copies to agree, then reports every copy that is wrong, not only the first. For an open question it
300
+ * also says which of the storyboard's options the backend does.
301
+ */
302
+ async ends(endings: Ending[]): Promise<void> {
303
+ const ends = endings[chosenOption(this.meta.id, endings.length)].ends;
304
+ this.recorder.step('The end: every copy is compared with what the storyboard says');
305
+ await test.step('the end: every copy holds what the storyboard says', async () => {
306
+ const failures: string[] = [];
307
+ const copies = await this.converge();
308
+ if (copies.why) {
309
+ failures.push(copies.why);
310
+ }
311
+ // Once the copies have agreed and stayed agreed there is nothing left to wait for: look once. (If they have not,
312
+ // say what each holds and move on.) The file is the exception: it is saved a little after the others.
313
+ const look = copies.compared ? { timeout: 1500, stable: 0 } : {};
314
+ for (const [lane, expected] of Object.entries(ends)) {
315
+ if (!expected) {
316
+ continue;
317
+ }
318
+ try {
319
+ if (lane === 'file') {
320
+ const now = await this.fileCells();
321
+ const caughtUp = copies.agreed !== null && now !== null && JSON.stringify(now) === JSON.stringify(copies.agreed);
322
+ await this.expectFile(expected, caughtUp ? { timeout: 1500, stable: 0 } : {});
323
+ } else if (lane === 'server') {
324
+ await this.expectCells('the server', () => this.serverCells(), expected, look);
325
+ } else {
326
+ const tab = this.tabs.get(lane);
327
+ if (tab?.isOpen) {
328
+ await this.expectTabEventually(tab, expected, look);
329
+ }
330
+ }
331
+ } catch (e) {
332
+ failures.push((e as Error).message);
333
+ }
334
+ }
335
+ let verdict: string[] = [];
336
+ if (endings.length > 1) {
337
+ verdict = this.implemented(endings, await this.observe());
338
+ const info = test.info();
339
+ info.annotations.push({ type: 'does', description: verdict.join(' | ') });
340
+ fs.mkdirSync(info.project.outputDir, { recursive: true });
341
+ fs.appendFileSync(nodePath.join(info.project.outputDir, 'implements.jsonl'), JSON.stringify({ id: this.meta.id, title: this.meta.title, verdict }) + '\n');
342
+ }
343
+ this.recorder.step(failures.length ? 'The end: a copy is not what the storyboard says' : 'The end: every copy holds what the storyboard says');
344
+ if (failures.length) {
345
+ await this.showTabs('the end');
346
+ const which = verdict.length ? `\n\nOf the options the storyboard names, this backend:\n ${verdict.join('\n ')}` : '';
347
+ throw new Error(failures.join('\n\n----\n\n') + which);
348
+ }
349
+ });
350
+ }
351
+
352
+ /** Keep a picture of every open tab: what each looked like when something went wrong. */
353
+ private async showTabs(when: string): Promise<void> {
354
+ for (const tab of this.tabs.values()) {
355
+ if (tab.page) {
356
+ await keepLook(this.recorder.dir, tab.name, when, tab.page as any);
357
+ }
358
+ }
359
+ }
360
+
361
+ /** What a person would have had to do or seen, worth saying in the summary of the run. */
362
+ private keepNotes(): void {
363
+ const info = test.info();
364
+ for (const note of this.recorder.notes) {
365
+ info.annotations.push({ type: 'a person would have noticed', description: note });
366
+ }
367
+ if (this.recorder.notes.length) {
368
+ fs.mkdirSync(info.project.outputDir, { recursive: true });
369
+ for (const note of this.recorder.notes) {
370
+ fs.appendFileSync(nodePath.join(info.project.outputDir, 'notes.jsonl'), JSON.stringify({ id: this.meta.id, note }) + '\n');
371
+ }
372
+ }
373
+ }
374
+
375
+ /** Close every tab, which completes their videos, and put the videos in the report. */
376
+ async closeAll(): Promise<void> {
377
+ for (const tab of this.tabs.values()) {
378
+ await tab.close().catch(() => undefined);
379
+ }
380
+ const info = test.info();
381
+ for (const rec of this.recorder.tabs) {
382
+ if (rec.file) {
383
+ await info.attach(`video: ${rec.name}`, { path: rec.file, contentType: 'video/webm' }).catch(() => undefined);
384
+ }
385
+ }
386
+ this.keepNotes();
387
+ const watch = this.recorder.finish();
388
+ if (watch) {
389
+ await info.attach('watch every tab side by side (open this file)', { path: watch, contentType: 'text/html' }).catch(() => undefined);
390
+ console.log(`watch every tab side by side: ${watch}`);
391
+ }
392
+ }
393
+ }
394
+
395
+ interface Fixtures {
396
+ browser: Browser;
397
+ baseURL?: string;
398
+ page: IJupyterLabPageFixture;
399
+ tmpPath: string;
400
+ waitForApplication: TabEnv['waitForApplication'];
401
+ }
402
+
403
+ /**
404
+ * The arguments of `test()` for one storyboard: `test(...story(meta, async s => { ... }))`. The generated file calls
405
+ * `test()` itself, so that Playwright reports each story in the file it is in.
406
+ *
407
+ * The test is skipped, with the reason, if the backend adapter lacks something the story needs.
408
+ */
409
+ /**
410
+ * The accepted stories a backend is known not to meet yet, with why: the file named by COMPLIANCE_KNOWN_GAPS (so the
411
+ * backend's own repo can keep it), else known-gaps/<adapter>.json.
412
+ */
413
+ function knownGaps(adapterName: string): Record<string, string> {
414
+ const file = process.env.COMPLIANCE_KNOWN_GAPS || nodePath.join(__dirname, '..', 'known-gaps', `${adapterName}.json`);
415
+ try {
416
+ return JSON.parse(fs.readFileSync(file, 'utf-8'));
417
+ } catch {
418
+ return {};
419
+ }
420
+ }
421
+
422
+ export function story(
423
+ meta: StoryMeta,
424
+ body: (s: Story) => Promise<void>
425
+ ): [string, { tag: string[] }, (fixtures: Fixtures, testInfo: TestInfo) => Promise<void>] {
426
+ const tag = ['@compliance', `@group-${meta.group}`].concat(meta.needsAlignment ? ['@needs-alignment'] : []);
427
+ return [
428
+ `${meta.id} · ${meta.title}`,
429
+ { tag },
430
+ async ({ browser, baseURL, page, tmpPath, waitForApplication }, testInfo) => {
431
+ const adapter = await loadAdapter();
432
+ await adapter.discover?.(baseURL!);
433
+ const have = new Set<string>([...adapter.capabilities, ...ALWAYS]);
434
+ const missing = meta.requires.filter((c: Capability) => !have.has(c));
435
+ testInfo.annotations.push({ type: 'needs', description: meta.requires.join(', ') || 'nothing beyond a plain JupyterLab' });
436
+ if (meta.needsAlignment) {
437
+ testInfo.annotations.push({ type: 'needs-alignment', description: 'the storyboard still asks a question; the proposed answer is held to' });
438
+ }
439
+ if (meta.loose?.length) {
440
+ testInfo.annotations.push({ type: 'checked-loosely', description: meta.loose.join('; ') });
441
+ }
442
+ if (meta.proposed) {
443
+ testInfo.annotations.push({ type: 'proposed', description: 'not agreed yet: a miss is reported, not failed' });
444
+ }
445
+ const gap = knownGaps(adapter.name)[meta.id];
446
+ if (gap) {
447
+ testInfo.annotations.push({ type: 'known-gap', description: gap });
448
+ test.fail(true, `known gap in "${adapter.name}": ${gap}`);
449
+ }
450
+ test.skip(Boolean(meta.internal), `not checkable from the user's side: ${meta.internal}`);
451
+ test.skip(missing.length > 0, `adapter "${adapter.name}" cannot: ${missing.join(', ')}`);
452
+
453
+ try {
454
+ fs.mkdirSync(testInfo.project.outputDir, { recursive: true });
455
+ fs.writeFileSync(nodePath.join(testInfo.project.outputDir, 'adapter.json'), JSON.stringify({ name: adapter.name, capabilities: [...adapter.capabilities, ...ALWAYS] }));
456
+ } catch {
457
+ /* the summary will not know which adapter ran */
458
+ }
459
+ const ctx: AdapterContext = { admin: page, baseURL: baseURL! };
460
+ const env: TabEnv = {
461
+ browser,
462
+ baseURL: baseURL!,
463
+ waitForApplication,
464
+ dir: tmpPath,
465
+ recorder: new Recorder(testInfo.outputDir, `${meta.id} · ${meta.title}`)
466
+ };
467
+ const s = new Story(meta, adapter, ctx, env, tmpPath);
468
+ try {
469
+ await body(s);
470
+ } catch (error) {
471
+ if (!meta.proposed) {
472
+ throw error;
473
+ }
474
+ const message = error instanceof Error ? error.message : String(error);
475
+ testInfo.annotations.push({ type: 'proposed-not-met', description: message.split('\n')[0] });
476
+ fs.writeFileSync(nodePath.join(testInfo.outputDir, 'not-met.txt'), message);
477
+ } finally {
478
+ await s.closeAll();
479
+ }
480
+ }
481
+ ];
482
+ }