@mostra/conformance 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +69 -2
- package/baselines/reference-host-rust-broken.json +8 -0
- package/baselines/reference-host-ts-broken.json +8 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +106 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +403 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +4 -0
- package/dist/protocol-RQ9lbehE.js +207 -0
- package/dist/protocol-RQ9lbehE.js.map +1 -0
- package/dist/reference-host-BJ-A_Qd_.d.ts +257 -0
- package/dist/reference-host-BJ-A_Qd_.d.ts.map +1 -0
- package/dist/reference-host-d6gs3Tal.js +409 -0
- package/dist/reference-host-d6gs3Tal.js.map +1 -0
- package/dist/reference-host.d.ts +2 -0
- package/dist/reference-host.js +3 -0
- package/dist/runner-CvoSJlfk.js +1940 -0
- package/dist/runner-CvoSJlfk.js.map +1 -0
- package/package.json +67 -3
- package/src/baseline.ts +51 -0
- package/src/cli.ts +117 -0
- package/src/fixtures.ts +87 -0
- package/src/harness/client.ts +224 -0
- package/src/harness/protocol.ts +381 -0
- package/src/harness/serve.ts +146 -0
- package/src/index.ts +65 -0
- package/src/reference-host.ts +359 -0
- package/src/runner.ts +189 -0
- package/src/scenario.ts +376 -0
- package/src/scenarios/delivery.ts +267 -0
- package/src/scenarios/index.ts +20 -0
- package/src/scenarios/interaction.ts +453 -0
- package/src/scenarios/ports.ts +278 -0
- package/src/scenarios/session.ts +183 -0
- package/src/scenarios/viewing.ts +184 -0
- package/src/version.ts +29 -0
- package/src/viewer.ts +380 -0
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* `mostra-conformance-reference-host`: the TypeScript reference host (HO-05). A host built the way
|
|
4
|
+
* the docs recommend, on `@mostra/sdk`, with a store that keeps each submission once by its
|
|
5
|
+
* `deliveryId` (H-03), every port the SDK serves, and the harness shim the kit drives. It is the
|
|
6
|
+
* model a TypeScript host copies, and the kit's own proof that its scenarios pass.
|
|
7
|
+
*
|
|
8
|
+
* mostra-conformance-reference-host every port
|
|
9
|
+
* mostra-conformance-reference-host --standalone no port: Mostra's built-in adapters
|
|
10
|
+
* mostra-conformance-reference-host --unscripted every port, without the `faults` feature:
|
|
11
|
+
* its policy allows both channels and it names people by its own IDs (`user-1` …), so the
|
|
12
|
+
* kit judges it by the answers it reports, as it must any host that can't be scripted
|
|
13
|
+
* mostra-conformance-reference-host --break <fault,…> a deliberately broken host:
|
|
14
|
+
* store-duplicates stores every delivery call, so a redelivery is stored twice
|
|
15
|
+
* drafts-as-submissions takes draft mirrors for submissions
|
|
16
|
+
* ignore-policy allows both channels whatever the kit scripted
|
|
17
|
+
*
|
|
18
|
+
* Its own runtime is found like any SDK host's (`MOSTRA_BUNDLE`, `MOSTRA_RUNTIME_BIN`, the pinned
|
|
19
|
+
* bundle, a development checkout).
|
|
20
|
+
*/
|
|
21
|
+
import { parseArgs } from "node:util";
|
|
22
|
+
import {
|
|
23
|
+
type ArtifactSummary,
|
|
24
|
+
type ControlsPolicy,
|
|
25
|
+
createDiagnostic,
|
|
26
|
+
type DraftMirror,
|
|
27
|
+
hostMethods,
|
|
28
|
+
type PortIdentityResolveParams,
|
|
29
|
+
type PortSecretsGetParams,
|
|
30
|
+
type PortTrustedControlsActivatedParams,
|
|
31
|
+
RpcError,
|
|
32
|
+
type SubmissionEnvelope,
|
|
33
|
+
type SubmissionsDeliverResult,
|
|
34
|
+
} from "@mostra/protocol";
|
|
35
|
+
import { createMostra, type Mostra, type Ports } from "@mostra/sdk";
|
|
36
|
+
import type {
|
|
37
|
+
EventsParams,
|
|
38
|
+
EventsResult,
|
|
39
|
+
ScriptParams,
|
|
40
|
+
StartParams,
|
|
41
|
+
StartResult,
|
|
42
|
+
StoredDelivery,
|
|
43
|
+
StoreResult,
|
|
44
|
+
} from "./harness/protocol.ts";
|
|
45
|
+
import { serveHarness } from "./harness/serve.ts";
|
|
46
|
+
import { kitVersion } from "./version.ts";
|
|
47
|
+
|
|
48
|
+
/** The faults a broken reference host can carry. */
|
|
49
|
+
export const REFERENCE_FAULTS: readonly string[] = [
|
|
50
|
+
"store-duplicates",
|
|
51
|
+
"drafts-as-submissions",
|
|
52
|
+
"ignore-policy",
|
|
53
|
+
];
|
|
54
|
+
|
|
55
|
+
/** The reference host's store and port answers: what a real host keeps in its own database. */
|
|
56
|
+
export class ReferenceStore {
|
|
57
|
+
readonly #deliveries = new Map<string, StoredDelivery & { attempts: number[] }>();
|
|
58
|
+
readonly #drafts: StoreResult["drafts"][number][] = [];
|
|
59
|
+
readonly #activations: StoreResult["activations"][number][] = [];
|
|
60
|
+
readonly #identity: StoreResult["identity"][number][] = [];
|
|
61
|
+
readonly #policies: StoreResult["policies"][number][] = [];
|
|
62
|
+
readonly #secrets: StoreResult["secrets"][number][] = [];
|
|
63
|
+
readonly #faults: ReadonlySet<string>;
|
|
64
|
+
#order = 0;
|
|
65
|
+
/** How the scripted ports answer (feature `faults`). */
|
|
66
|
+
#deliver: Extract<ScriptParams, { port: "submissions" }>["answer"] = "accept";
|
|
67
|
+
#policy: Extract<ScriptParams, { port: "trustedControls" }>["answer"] = {
|
|
68
|
+
submitChannels: ["shell"],
|
|
69
|
+
controls: ["submit", "choose", "approve"],
|
|
70
|
+
};
|
|
71
|
+
#person: Extract<ScriptParams, { port: "identity" }>["answer"] = "headers";
|
|
72
|
+
#secretValues: Readonly<Record<string, string>> = {};
|
|
73
|
+
|
|
74
|
+
/** People by the header that named them, for an unscripted host's own IDs. */
|
|
75
|
+
readonly #people = new Map<string, string>();
|
|
76
|
+
readonly #unscripted: boolean;
|
|
77
|
+
|
|
78
|
+
constructor(faults: ReadonlySet<string> = new Set(), unscripted = false) {
|
|
79
|
+
this.#faults = faults;
|
|
80
|
+
this.#unscripted = unscripted;
|
|
81
|
+
if (unscripted) this.#policy = { submitChannels: ["artifact", "shell"] };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Sets how a port answers from now on (feature `faults`). */
|
|
85
|
+
script(params: ScriptParams): void {
|
|
86
|
+
if (params.port === "submissions") this.#deliver = params.answer;
|
|
87
|
+
else if (params.port === "trustedControls") this.#policy = params.answer;
|
|
88
|
+
else if (params.port === "identity") this.#person = params.answer;
|
|
89
|
+
else this.#secretValues = params.answer;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Records one delivery the way a host's database would: once per `deliveryId`. */
|
|
93
|
+
#record(envelope: SubmissionEnvelope | (DraftMirror & { kind: string })): void {
|
|
94
|
+
const id =
|
|
95
|
+
"deliveryId" in envelope
|
|
96
|
+
? envelope.deliveryId
|
|
97
|
+
: `draft-${envelope.viewerId}-${envelope.revision}`;
|
|
98
|
+
const existing = this.#deliveries.get(id);
|
|
99
|
+
const attempt = "attempt" in envelope ? envelope.attempt : 1;
|
|
100
|
+
if (existing) {
|
|
101
|
+
existing.attempts.push(attempt);
|
|
102
|
+
(existing as { calls: number }).calls += 1;
|
|
103
|
+
if (this.#faults.has("store-duplicates")) (existing as { stored: number }).stored += 1;
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const results = "results" in envelope ? envelope.results : undefined;
|
|
107
|
+
this.#order += 1;
|
|
108
|
+
this.#deliveries.set(id, {
|
|
109
|
+
deliveryId: id,
|
|
110
|
+
submissionId: results?.submission.id ?? "",
|
|
111
|
+
artifactId: envelope.artifactId,
|
|
112
|
+
calls: 1,
|
|
113
|
+
stored: 1,
|
|
114
|
+
attempts: [attempt],
|
|
115
|
+
kind: envelope.kind,
|
|
116
|
+
channel: results?.submission.channel ?? "",
|
|
117
|
+
viewerKind: results?.submission.viewer.kind ?? "",
|
|
118
|
+
...(results?.submission.viewer.ref !== undefined
|
|
119
|
+
? { viewerRef: results.submission.viewer.ref }
|
|
120
|
+
: {}),
|
|
121
|
+
order: this.#order,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** The ports, as the SDK takes them. */
|
|
126
|
+
ports(): Ports {
|
|
127
|
+
return {
|
|
128
|
+
submissions: {
|
|
129
|
+
deliver: async (envelope: SubmissionEnvelope): Promise<SubmissionsDeliverResult> => {
|
|
130
|
+
const answer = this.#deliver;
|
|
131
|
+
if (answer === "fail") throw new Error("the host's store is unavailable");
|
|
132
|
+
if (answer === "hang") return new Promise(() => undefined);
|
|
133
|
+
if (answer === "retry") return { accepted: false, reason: "busy", retry: true };
|
|
134
|
+
if (answer === "undeliverable")
|
|
135
|
+
return { accepted: false, reason: "undeliverable", retry: false };
|
|
136
|
+
if (answer === "reject") return { accepted: false, reason: "invalid", retry: false };
|
|
137
|
+
// A redelivery of a delivery the store holds is accepted again, never stored twice.
|
|
138
|
+
this.#record(envelope);
|
|
139
|
+
return { accepted: true };
|
|
140
|
+
},
|
|
141
|
+
},
|
|
142
|
+
drafts: {
|
|
143
|
+
mirror: (draft: DraftMirror) => {
|
|
144
|
+
this.#drafts.push({
|
|
145
|
+
artifactId: draft.artifactId,
|
|
146
|
+
viewerId: draft.viewerId,
|
|
147
|
+
revision: draft.revision,
|
|
148
|
+
kind: draft.kind,
|
|
149
|
+
});
|
|
150
|
+
if (this.#faults.has("drafts-as-submissions")) this.#record(draft);
|
|
151
|
+
},
|
|
152
|
+
},
|
|
153
|
+
identity: {
|
|
154
|
+
resolve: async (request: PortIdentityResolveParams) => {
|
|
155
|
+
const call = {
|
|
156
|
+
artifactId: request.artifactId,
|
|
157
|
+
via: request.via,
|
|
158
|
+
headers: Object.keys(request.headers).sort(),
|
|
159
|
+
};
|
|
160
|
+
const answer = this.#person;
|
|
161
|
+
if (answer === "fail" || answer === "hang") {
|
|
162
|
+
this.#identity.push(call);
|
|
163
|
+
if (answer === "fail") throw new Error("the host's directory is unavailable");
|
|
164
|
+
return new Promise<never>(() => undefined);
|
|
165
|
+
}
|
|
166
|
+
let person: { ref: string; name?: string } | null = answer === "headers" ? null : answer;
|
|
167
|
+
if (answer === "headers") {
|
|
168
|
+
const user = request.headers["x-forwarded-user"];
|
|
169
|
+
if (user && this.#unscripted && !this.#people.has(user)) {
|
|
170
|
+
this.#people.set(user, `user-${this.#people.size + 1}`);
|
|
171
|
+
}
|
|
172
|
+
const ref = user && this.#unscripted ? this.#people.get(user) : user;
|
|
173
|
+
person = user && ref ? { ref, name: user.split("@")[0] ?? user } : null;
|
|
174
|
+
}
|
|
175
|
+
this.#identity.push({ ...call, answer: person });
|
|
176
|
+
return person;
|
|
177
|
+
},
|
|
178
|
+
},
|
|
179
|
+
trustedControls: {
|
|
180
|
+
policy: async (artifact: ArtifactSummary): Promise<ControlsPolicy> => {
|
|
181
|
+
const answer = this.#policy;
|
|
182
|
+
if (this.#faults.has("ignore-policy")) {
|
|
183
|
+
const both: ControlsPolicy = { submitChannels: ["shell", "artifact"] };
|
|
184
|
+
this.#policies.push({ artifactId: artifact.id, answer: both });
|
|
185
|
+
return both;
|
|
186
|
+
}
|
|
187
|
+
if (answer === "fail" || answer === "hang") {
|
|
188
|
+
this.#policies.push({ artifactId: artifact.id });
|
|
189
|
+
if (answer === "fail") throw new Error("the host's policy service is unavailable");
|
|
190
|
+
return new Promise<never>(() => undefined);
|
|
191
|
+
}
|
|
192
|
+
this.#policies.push({ artifactId: artifact.id, answer: answer as ControlsPolicy });
|
|
193
|
+
return answer as ControlsPolicy;
|
|
194
|
+
},
|
|
195
|
+
activated: (event: PortTrustedControlsActivatedParams) => {
|
|
196
|
+
this.#activations.push({
|
|
197
|
+
artifactId: event.artifactId,
|
|
198
|
+
control: event.control,
|
|
199
|
+
viewId: event.viewId,
|
|
200
|
+
});
|
|
201
|
+
},
|
|
202
|
+
},
|
|
203
|
+
secrets: {
|
|
204
|
+
get: (request: PortSecretsGetParams) => {
|
|
205
|
+
this.#secrets.push({ artifactId: request.artifactId, name: request.name });
|
|
206
|
+
return this.#secretValues[request.name] ?? null;
|
|
207
|
+
},
|
|
208
|
+
},
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** What the store holds now. */
|
|
213
|
+
snapshot(): StoreResult {
|
|
214
|
+
return {
|
|
215
|
+
deliveries: [...this.#deliveries.values()].sort((a, b) => a.order - b.order),
|
|
216
|
+
drafts: [...this.#drafts],
|
|
217
|
+
activations: [...this.#activations],
|
|
218
|
+
identity: [...this.#identity],
|
|
219
|
+
policies: [...this.#policies],
|
|
220
|
+
secrets: [...this.#secrets],
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Runs the reference host's harness on stdio. */
|
|
226
|
+
export async function runReferenceHost(
|
|
227
|
+
options: { standalone?: boolean; unscripted?: boolean; faults?: readonly string[] } = {},
|
|
228
|
+
): Promise<void> {
|
|
229
|
+
const faults = new Set(options.faults ?? []);
|
|
230
|
+
const unscripted = options.unscripted ?? false;
|
|
231
|
+
const store = new ReferenceStore(faults, unscripted);
|
|
232
|
+
let mostra: Mostra | undefined;
|
|
233
|
+
const ports = options.standalone ? {} : store.ports();
|
|
234
|
+
const portNames = Object.keys(ports);
|
|
235
|
+
const name = options.standalone
|
|
236
|
+
? "reference-host-ts-standalone"
|
|
237
|
+
: unscripted
|
|
238
|
+
? "reference-host-ts-unscripted"
|
|
239
|
+
: faults.size > 0
|
|
240
|
+
? "reference-host-ts-broken"
|
|
241
|
+
: "reference-host-ts";
|
|
242
|
+
await serveHarness({
|
|
243
|
+
initialize: () => ({
|
|
244
|
+
host: { name, version: kitVersion() },
|
|
245
|
+
ports: portNames,
|
|
246
|
+
features: options.standalone || unscripted ? ["events"] : ["faults", "events"],
|
|
247
|
+
}),
|
|
248
|
+
async start(params: StartParams): Promise<StartResult> {
|
|
249
|
+
if (params.mode === "attach") {
|
|
250
|
+
throw new RpcError(
|
|
251
|
+
createDiagnostic("MOSTRA_HOST_INVALID_PARAMS", {
|
|
252
|
+
message: "The TypeScript reference host spawns its instance; it doesn't attach.",
|
|
253
|
+
}),
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
await mostra?.close();
|
|
257
|
+
mostra = await createMostra({
|
|
258
|
+
home: params.home,
|
|
259
|
+
client: { name, version: kitVersion() },
|
|
260
|
+
ports,
|
|
261
|
+
env: { ...process.env, ...params.env },
|
|
262
|
+
onLog: () => undefined,
|
|
263
|
+
});
|
|
264
|
+
return {
|
|
265
|
+
revision: mostra.revision,
|
|
266
|
+
instance: mostra.instance,
|
|
267
|
+
warnings: mostra.warnings,
|
|
268
|
+
};
|
|
269
|
+
},
|
|
270
|
+
async call(method: string, params: unknown, timeoutMs?: number) {
|
|
271
|
+
const definition = hostMethods.find((candidate) => candidate.method === method);
|
|
272
|
+
if (!mostra) {
|
|
273
|
+
throw new RpcError(
|
|
274
|
+
createDiagnostic("MOSTRA_HOST_CONNECTION_CLOSED", { message: "No instance runs." }),
|
|
275
|
+
);
|
|
276
|
+
}
|
|
277
|
+
if (!definition) {
|
|
278
|
+
throw new RpcError(
|
|
279
|
+
createDiagnostic("MOSTRA_HOST_METHOD_UNKNOWN", { message: `No method ${method}.` }),
|
|
280
|
+
-32601,
|
|
281
|
+
);
|
|
282
|
+
}
|
|
283
|
+
return mostra.call(definition, params, timeoutMs !== undefined ? { timeoutMs } : {});
|
|
284
|
+
},
|
|
285
|
+
async stop() {
|
|
286
|
+
const current = mostra;
|
|
287
|
+
mostra = undefined;
|
|
288
|
+
await current?.close();
|
|
289
|
+
},
|
|
290
|
+
store: () => store.snapshot(),
|
|
291
|
+
...(options.standalone || unscripted
|
|
292
|
+
? {}
|
|
293
|
+
: { script: (params: ScriptParams) => store.script(params) }),
|
|
294
|
+
async events(params: EventsParams): Promise<EventsResult> {
|
|
295
|
+
if (!mostra) throw new Error("No instance runs.");
|
|
296
|
+
const pages: EventsResult["pages"][number][] = [];
|
|
297
|
+
const stream = mostra
|
|
298
|
+
.events({
|
|
299
|
+
name: params.name,
|
|
300
|
+
filter: {
|
|
301
|
+
...(params.filter.artifacts ? { artifacts: [...params.filter.artifacts] } : {}),
|
|
302
|
+
...(params.filter.types ? { types: [...params.filter.types] } : {}),
|
|
303
|
+
},
|
|
304
|
+
})
|
|
305
|
+
[Symbol.asyncIterator]();
|
|
306
|
+
const deadline = Date.now() + params.timeoutMs;
|
|
307
|
+
try {
|
|
308
|
+
while (pages.length < params.pages) {
|
|
309
|
+
const left = deadline - Date.now();
|
|
310
|
+
if (left <= 0) break;
|
|
311
|
+
const next = await Promise.race([
|
|
312
|
+
stream.next(),
|
|
313
|
+
new Promise<"timeout">((done) => setTimeout(() => done("timeout"), left)),
|
|
314
|
+
]);
|
|
315
|
+
if (next === "timeout" || next.done) break;
|
|
316
|
+
pages.push({
|
|
317
|
+
cursor: next.value.cursor,
|
|
318
|
+
events: next.value.events.map((event) => ({ ...event })),
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
} finally {
|
|
322
|
+
// Asking for the next page is what tells the SDK the last one was handled: it acknowledges
|
|
323
|
+
// it at once, then waits for a page this stream won't read, so the wait is cut short.
|
|
324
|
+
await Promise.race([
|
|
325
|
+
stream.next().catch(() => undefined),
|
|
326
|
+
new Promise((done) => setTimeout(done, 1000)),
|
|
327
|
+
]);
|
|
328
|
+
await stream.return?.();
|
|
329
|
+
}
|
|
330
|
+
return { pages };
|
|
331
|
+
},
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if (
|
|
336
|
+
import.meta.url === `file://${process.argv[1]}` ||
|
|
337
|
+
process.argv[1]?.endsWith("reference-host.js")
|
|
338
|
+
) {
|
|
339
|
+
const { values } = parseArgs({
|
|
340
|
+
options: {
|
|
341
|
+
standalone: { type: "boolean" },
|
|
342
|
+
unscripted: { type: "boolean" },
|
|
343
|
+
break: { type: "string" },
|
|
344
|
+
},
|
|
345
|
+
strict: true,
|
|
346
|
+
});
|
|
347
|
+
const faults = values.break ? values.break.split(",").filter(Boolean) : [];
|
|
348
|
+
for (const fault of faults) {
|
|
349
|
+
if (!REFERENCE_FAULTS.includes(fault)) {
|
|
350
|
+
process.stderr.write(`Unknown fault ${fault}; known: ${REFERENCE_FAULTS.join(", ")}\n`);
|
|
351
|
+
process.exit(2);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
void runReferenceHost({
|
|
355
|
+
standalone: values.standalone ?? false,
|
|
356
|
+
unscripted: values.unscripted ?? false,
|
|
357
|
+
faults,
|
|
358
|
+
}).then(() => process.exit(0));
|
|
359
|
+
}
|
package/src/runner.ts
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs scenarios against a host (HO-05): each in a fresh directory with a fresh shim and home,
|
|
3
|
+
* one at a time, each bounded in time; then judges the run against the host's baseline.
|
|
4
|
+
*/
|
|
5
|
+
import { mkdtempSync, rmdirSync } from "node:fs";
|
|
6
|
+
import { tmpdir } from "node:os";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { type Baseline, NO_BASELINE } from "./baseline.ts";
|
|
9
|
+
import type { InitializeResult } from "./harness/protocol.ts";
|
|
10
|
+
import { NotApplicable, type RunSettings, type Scenario, ScenarioContext } from "./scenario.ts";
|
|
11
|
+
import { SCENARIOS } from "./scenarios/index.ts";
|
|
12
|
+
|
|
13
|
+
/** How one scenario went. */
|
|
14
|
+
export type ScenarioStatus =
|
|
15
|
+
/** The property held. */
|
|
16
|
+
| "passed"
|
|
17
|
+
/** It didn't, and the baseline doesn't expect that: the run fails. */
|
|
18
|
+
| "failed"
|
|
19
|
+
/** The host lacks a port or feature the scenario needs. */
|
|
20
|
+
| "skipped"
|
|
21
|
+
/** It didn't, as the baseline expects. */
|
|
22
|
+
| "expected-failure"
|
|
23
|
+
/** It held, though the baseline expects it to fail: the baseline is stale, the run fails. */
|
|
24
|
+
| "unexpected-pass";
|
|
25
|
+
|
|
26
|
+
/** One scenario's outcome. */
|
|
27
|
+
export interface ScenarioOutcome {
|
|
28
|
+
/** The scenario's ID. */
|
|
29
|
+
readonly id: string;
|
|
30
|
+
/** What the scenario proves. */
|
|
31
|
+
readonly title: string;
|
|
32
|
+
/** The requirements and host needs it exercises. */
|
|
33
|
+
readonly covers: readonly string[];
|
|
34
|
+
/** How it went. */
|
|
35
|
+
readonly status: ScenarioStatus;
|
|
36
|
+
/** Why it failed or was skipped, or what the baseline says. */
|
|
37
|
+
readonly reason?: string;
|
|
38
|
+
/** How long it took. */
|
|
39
|
+
readonly durationMs: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A run's report. */
|
|
43
|
+
export interface ConformanceReport {
|
|
44
|
+
/** What the host said about itself. */
|
|
45
|
+
readonly host: InitializeResult | undefined;
|
|
46
|
+
/** Each scenario's outcome, in the order they ran. */
|
|
47
|
+
readonly outcomes: readonly ScenarioOutcome[];
|
|
48
|
+
/** Whether the run passes: nothing `failed` and nothing `unexpected-pass`. */
|
|
49
|
+
readonly ok: boolean;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Options of {@link runConformance}. */
|
|
53
|
+
export interface RunOptions {
|
|
54
|
+
/** The host's harness shim, as a command line (run through the shell). */
|
|
55
|
+
readonly host: string;
|
|
56
|
+
/** Variables for the shim's process, on top of this process's. */
|
|
57
|
+
readonly hostEnv?: Readonly<Record<string, string>>;
|
|
58
|
+
/** Only the scenarios whose ID equals or starts with one of these (`delivery.`). */
|
|
59
|
+
readonly only?: readonly string[];
|
|
60
|
+
/** The host's expected failures. */
|
|
61
|
+
readonly baseline?: Baseline;
|
|
62
|
+
/** Publish through verify's gate (needs the verify browser). Default false. */
|
|
63
|
+
readonly verify?: boolean;
|
|
64
|
+
/** Each scenario's time limit. Default 240 s. */
|
|
65
|
+
readonly timeoutMs?: number;
|
|
66
|
+
/** Where scenario directories go. Default a fresh directory under `/tmp` (short socket paths). */
|
|
67
|
+
readonly workDir?: string;
|
|
68
|
+
/** Each outcome as it is known. */
|
|
69
|
+
readonly onOutcome?: (outcome: ScenarioOutcome) => void;
|
|
70
|
+
/** The scenarios to choose from. Default every scenario of the kit. */
|
|
71
|
+
readonly scenarios?: readonly Scenario[];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function selected(scenarios: readonly Scenario[], only: readonly string[] | undefined): Scenario[] {
|
|
75
|
+
if (!only || only.length === 0) return [...scenarios];
|
|
76
|
+
return scenarios.filter((scenario) =>
|
|
77
|
+
only.some((prefix) => scenario.id === prefix || scenario.id.startsWith(prefix)),
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Runs one scenario: `passed`, `failed` or `skipped`, before the baseline. */
|
|
82
|
+
async function runOne(
|
|
83
|
+
scenario: Scenario,
|
|
84
|
+
settings: RunSettings,
|
|
85
|
+
dir: string,
|
|
86
|
+
timeoutMs: number,
|
|
87
|
+
): Promise<{ status: "passed" | "failed" | "skipped"; reason?: string; host?: InitializeResult }> {
|
|
88
|
+
const context = new ScenarioContext(settings, dir);
|
|
89
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
90
|
+
try {
|
|
91
|
+
await context.open();
|
|
92
|
+
for (const feature of scenario.needs ?? []) context.requireFeature(feature);
|
|
93
|
+
await Promise.race([
|
|
94
|
+
scenario.run(context),
|
|
95
|
+
new Promise<never>((_, fail) => {
|
|
96
|
+
timer = setTimeout(
|
|
97
|
+
() => fail(new Error(`the scenario didn't finish within ${timeoutMs / 1000} s`)),
|
|
98
|
+
timeoutMs,
|
|
99
|
+
);
|
|
100
|
+
}),
|
|
101
|
+
]);
|
|
102
|
+
return { status: "passed", ...(context.host.info ? { host: context.host.info } : {}) };
|
|
103
|
+
} catch (error) {
|
|
104
|
+
const host = context.host?.info;
|
|
105
|
+
if (error instanceof NotApplicable)
|
|
106
|
+
return { status: "skipped", reason: error.message, ...(host ? { host } : {}) };
|
|
107
|
+
const stderr = context.host?.stderr.slice(-3).join(" | ");
|
|
108
|
+
const message = (error as Error).message ?? String(error);
|
|
109
|
+
return {
|
|
110
|
+
status: "failed",
|
|
111
|
+
reason:
|
|
112
|
+
stderr && !(error as Error).name?.startsWith("Scenario")
|
|
113
|
+
? `${message} (host: ${stderr})`
|
|
114
|
+
: message,
|
|
115
|
+
...(host ? { host } : {}),
|
|
116
|
+
};
|
|
117
|
+
} finally {
|
|
118
|
+
clearTimeout(timer);
|
|
119
|
+
await context.close().catch(() => undefined);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Runs the kit's scenarios against a host and judges them against its baseline. */
|
|
124
|
+
export async function runConformance(options: RunOptions): Promise<ConformanceReport> {
|
|
125
|
+
const baseline = options.baseline ?? NO_BASELINE;
|
|
126
|
+
// A baseline naming a scenario the run doesn't know would expect nothing, silently: refused here
|
|
127
|
+
// as `readBaseline` refuses it, whoever built the baseline (GPT review of WP-12, finding 9).
|
|
128
|
+
const catalog = (options.scenarios ?? SCENARIOS).map((scenario) => scenario.id);
|
|
129
|
+
const unknown = Object.keys(baseline.expectedFailures).filter((id) => !catalog.includes(id));
|
|
130
|
+
if (unknown.length > 0) {
|
|
131
|
+
throw new Error(`The baseline names scenarios the kit doesn't have: ${unknown.join(", ")}.`);
|
|
132
|
+
}
|
|
133
|
+
const settings: RunSettings = {
|
|
134
|
+
host: options.host,
|
|
135
|
+
hostEnv: options.hostEnv ?? {},
|
|
136
|
+
verify: options.verify ?? false,
|
|
137
|
+
};
|
|
138
|
+
const root =
|
|
139
|
+
options.workDir ?? mkdtempSync(join(process.platform === "darwin" ? "/tmp" : tmpdir(), "mc-"));
|
|
140
|
+
const outcomes: ScenarioOutcome[] = [];
|
|
141
|
+
let host: InitializeResult | undefined;
|
|
142
|
+
let index = 0;
|
|
143
|
+
for (const scenario of selected(options.scenarios ?? SCENARIOS, options.only)) {
|
|
144
|
+
index += 1;
|
|
145
|
+
const started = Date.now();
|
|
146
|
+
const result = await runOne(
|
|
147
|
+
scenario,
|
|
148
|
+
settings,
|
|
149
|
+
join(root, `s${index}`),
|
|
150
|
+
options.timeoutMs ?? 240_000,
|
|
151
|
+
);
|
|
152
|
+
host ??= result.host;
|
|
153
|
+
const expected = baseline.expectedFailures[scenario.id];
|
|
154
|
+
let status: ScenarioStatus = result.status;
|
|
155
|
+
let reason = result.reason;
|
|
156
|
+
if (expected !== undefined && result.status === "failed") {
|
|
157
|
+
status = "expected-failure";
|
|
158
|
+
reason = `${expected} (${result.reason ?? "failed"})`;
|
|
159
|
+
} else if (expected !== undefined && result.status === "passed") {
|
|
160
|
+
status = "unexpected-pass";
|
|
161
|
+
reason = `the baseline expects a failure (${expected}), and it passed: remove it from the baseline`;
|
|
162
|
+
}
|
|
163
|
+
const outcome: ScenarioOutcome = {
|
|
164
|
+
id: scenario.id,
|
|
165
|
+
title: scenario.title,
|
|
166
|
+
covers: scenario.covers,
|
|
167
|
+
status,
|
|
168
|
+
...(reason !== undefined ? { reason } : {}),
|
|
169
|
+
durationMs: Date.now() - started,
|
|
170
|
+
};
|
|
171
|
+
outcomes.push(outcome);
|
|
172
|
+
options.onOutcome?.(outcome);
|
|
173
|
+
}
|
|
174
|
+
// Each scenario removed its own directory; the run's own is removed when the kit made it.
|
|
175
|
+
if (options.workDir === undefined) {
|
|
176
|
+
try {
|
|
177
|
+
rmdirSync(root);
|
|
178
|
+
} catch {
|
|
179
|
+
// Not empty: something a scenario couldn't remove stays for a look.
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
host,
|
|
184
|
+
outcomes,
|
|
185
|
+
ok: outcomes.every(
|
|
186
|
+
(outcome) => outcome.status !== "failed" && outcome.status !== "unexpected-pass",
|
|
187
|
+
),
|
|
188
|
+
};
|
|
189
|
+
}
|