@intellectual-frontiers/test 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/dist/record.d.ts +222 -0
- package/dist/record.js +471 -0
- package/dist/test.d.ts +108 -0
- package/dist/test.js +588 -0
- package/package.json +46 -4
- package/src/record.ts +612 -0
- package/src/test.ts +713 -0
package/dist/test.js
ADDED
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
// Spec: 0001-test FR-001
|
|
2
|
+
/**
|
|
3
|
+
* Tests that run unchanged under `bun test` and `node --test` (commons 0005-typescript-runtime FR-013): registration on
|
|
4
|
+
* `node:test`, assertions on `node:assert/strict`, with short plain names (`assert`, `assertEquals`, `assertStringIncludes`, ...). `proves` names the
|
|
5
|
+
* requirements a test proves, as erga's kernel `test.ts` does, and keeps them for a report. Kept out of `./mod.ts`, so that importing the
|
|
6
|
+
* runtime never loads a test runner.
|
|
7
|
+
*
|
|
8
|
+
* Every test registered here is recorded by this module's own code, not by a runner's reporter (commons 0008-test-evidence FR-001 to
|
|
9
|
+
* FR-004): its id (the SHA-256 of its file and name), file, name, requirements, risk (the option `risk`, `not-high` by default), start and
|
|
10
|
+
* end, outcome (`pass`, `fail`, `skip`, `todo` or `cancelled`), a skip's or todo's reason, a failure's message, expected and actual values
|
|
11
|
+
* (redacted and cut to 4096 characters) and attempt (`IFC_TEST_ATTEMPT`, 1 by default), as a `test` span and a log record whose parent is
|
|
12
|
+
* the run's span from `TRACEPARENT`, on a resource that is the installation record. Where `IFC_TEST_RECORD` names a file, each test's
|
|
13
|
+
* records are appended to it as OTLP JSON lines as it starts and as it ends, and a test that has not ended when the process exits is
|
|
14
|
+
* appended as `cancelled`; otherwise they are kept in memory (`recorded`) and given to each listener (`onTestRecord`). The file of a test
|
|
15
|
+
* is the file that called `test` or `proves`, from the call's stack, relative to the working folder, or the option `file`. A test's
|
|
16
|
+
* outcome is never changed by its recording: a failure is still thrown to the runner.
|
|
17
|
+
*
|
|
18
|
+
* A test's body is given a context `t` whose `t.step(name, fn)` runs a step as a subtest of `node:test` (`TestContext.test`), awaited, and
|
|
19
|
+
* records it in the test's evidence (commons 0005-typescript-runtime FR-060): a `test step` span, a child of the test's span (or of the
|
|
20
|
+
* step it is in), as it ends, and the test's `steps`, which its `test` span names in `ifc.test.steps`. A step that fails fails its test
|
|
21
|
+
* with the step's error, on every runner, so that the test's record says why.
|
|
22
|
+
*
|
|
23
|
+
* `only` runs a test alone among the tests of its file (FR-060). Each runner is given `node:test`'s `only`, and this module also skips
|
|
24
|
+
* the other tests of the file itself, so that "the one marked only runs alone" holds on each runner, and each of the others is recorded
|
|
25
|
+
* as `skip` with the reason `ONLY_REASON`:
|
|
26
|
+
* - `node --test --test-only` runs only the tests marked `only`, of itself;
|
|
27
|
+
* - `node --test` without `--test-only` (Node 26 then says "'only' and 'runOnly' require the --test-only command-line option") and
|
|
28
|
+
* `bun test` (Bun's `node:test` ignores `only`) run every test: the others then skip themselves, and a runner shows them as passed
|
|
29
|
+
* (Bun) or skipped (Node);
|
|
30
|
+
* - `bun test --only` runs no test of `node:test` at all, `only` or not, as its `node:test` never marks one: give `bun test` no `--only`.
|
|
31
|
+
* The others of the file are those registered before the first test marked `only` that have not started, and every one registered after
|
|
32
|
+
* it; `bun test` loads every file in one process, so the mark is kept for each file, never for the process.
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* ```ts
|
|
36
|
+
* import { assertEquals, proves } from "./test.ts";
|
|
37
|
+
* proves(["0005-typescript-runtime FR-013"], "one and one", () => assertEquals(1 + 1, 2));
|
|
38
|
+
* proves(["0008-test-evidence FR-001"], "a high-risk test", () => assertEquals(2 * 2, 4), { risk: "high" });
|
|
39
|
+
* ```
|
|
40
|
+
*
|
|
41
|
+
* @module
|
|
42
|
+
*/
|
|
43
|
+
import nodeAssert from "node:assert/strict";
|
|
44
|
+
import { createHash } from "node:crypto";
|
|
45
|
+
import { appendFileSync, realpathSync } from "node:fs";
|
|
46
|
+
import { isAbsolute, relative, resolve } from "node:path";
|
|
47
|
+
import process from "node:process";
|
|
48
|
+
import { test as nodeTest } from "node:test";
|
|
49
|
+
import { fileURLToPath } from "node:url";
|
|
50
|
+
import { inspect } from "node:util";
|
|
51
|
+
import { extractContext, redactText, spanIdFrom, traceIdFrom, } from "@intellectual-frontiers/telemetry/otel";
|
|
52
|
+
import { cut, installationFromEnv, installationResource, recordLine, recordPath, testStepSpan, TEST_SCOPE, testCaseLog, testCaseSpan, testStartLog, reviewLog, warningLog, } from "./record.js";
|
|
53
|
+
import { installed } from "@intellectual-frontiers/telemetry/host/otel";
|
|
54
|
+
import { build, runtimeName, runtimeVersion } from "@intellectual-frontiers/system/proc";
|
|
55
|
+
/**
|
|
56
|
+
* How long a test may run when it names no `timeout`: five minutes. Bun's own default of 5 seconds ends a test that starts real programs on a slow runner (Windows above all) for the
|
|
57
|
+
* wrong reason; five minutes is above any limit a consumer's runner sets (its own tools pass up to a few minutes), so it never shortens one, and a test that hangs still ends.
|
|
58
|
+
*/
|
|
59
|
+
export const DEFAULT_TIMEOUT_MS = 300_000;
|
|
60
|
+
/** The reason a test is recorded as skipped where another test of its file is marked `only` (FR-060). */
|
|
61
|
+
export const ONLY_REASON = "another test of this file is marked only";
|
|
62
|
+
/** The requirements each test proves, in the order the tests were registered, for a report. */
|
|
63
|
+
export const proofs = [];
|
|
64
|
+
/** The requirements declared as proven by review rather than a test, with the reason. */
|
|
65
|
+
export const reviews = [];
|
|
66
|
+
/** Every test of this process as it was recorded, in the order they ended (skipped and todo tests as they were registered) (FR-004). */
|
|
67
|
+
export const recorded = [];
|
|
68
|
+
const listeners = [];
|
|
69
|
+
/**
|
|
70
|
+
* A listener given each test as it ends, beside `recorded` and `IFC_TEST_RECORD`: where an OpenTelemetry pipeline is installed, it hands
|
|
71
|
+
* `testCaseSpan` and `testCaseLog` of the event to its processors (0008-test-evidence FR-004). Returns the function that removes it.
|
|
72
|
+
*/
|
|
73
|
+
export function onTestRecord(listener) {
|
|
74
|
+
listeners.push(listener);
|
|
75
|
+
return () => {
|
|
76
|
+
const i = listeners.indexOf(listener);
|
|
77
|
+
if (i >= 0)
|
|
78
|
+
listeners.splice(i, 1);
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
let session;
|
|
82
|
+
function randomBytes(n) {
|
|
83
|
+
const b = new Uint8Array(n);
|
|
84
|
+
crypto.getRandomValues(b);
|
|
85
|
+
return b;
|
|
86
|
+
}
|
|
87
|
+
function newSpanId() {
|
|
88
|
+
for (;;) {
|
|
89
|
+
const s = spanIdFrom(randomBytes(8));
|
|
90
|
+
if (s.kind === "ok")
|
|
91
|
+
return s.value;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
function newTraceId() {
|
|
95
|
+
for (;;) {
|
|
96
|
+
const t = traceIdFrom(randomBytes(16));
|
|
97
|
+
if (t.kind === "ok")
|
|
98
|
+
return t.value;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
function nowNanos() {
|
|
102
|
+
return BigInt(Math.round((performance.timeOrigin + performance.now()) * 1000)) * 1000n;
|
|
103
|
+
}
|
|
104
|
+
function append(line) {
|
|
105
|
+
const file = current().file;
|
|
106
|
+
if (file === undefined)
|
|
107
|
+
return;
|
|
108
|
+
try {
|
|
109
|
+
appendFileSync(file, `${line}\n`);
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
// A record that cannot be written never changes the test's outcome; the run sees the test as missing.
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function current() {
|
|
116
|
+
if (session !== undefined)
|
|
117
|
+
return session;
|
|
118
|
+
const env = process.env;
|
|
119
|
+
const parent = extractContext(env["TRACEPARENT"], env["TRACESTATE"]);
|
|
120
|
+
const { installation, warnings } = installationFromEnv({ runtime: runtimeName(), runtimeVersion: runtimeVersion(), os: build.os, arch: build.arch }, env);
|
|
121
|
+
const attempt = Number(env["IFC_TEST_ATTEMPT"] ?? "1");
|
|
122
|
+
const file = env["IFC_TEST_RECORD"]?.trim();
|
|
123
|
+
session = {
|
|
124
|
+
traceId: parent?.traceId ?? newTraceId(),
|
|
125
|
+
...(parent === undefined ? {} : { parentSpanId: parent.spanId }),
|
|
126
|
+
resource: installationResource(installation),
|
|
127
|
+
file: file === undefined || file === "" ? undefined : file,
|
|
128
|
+
attempt: Number.isSafeInteger(attempt) && attempt >= 1 ? attempt : 1,
|
|
129
|
+
};
|
|
130
|
+
if (warnings.length > 0)
|
|
131
|
+
append(recordLine(session.resource, { records: warnings.map((w) => warningLog(w, nowNanos())) }));
|
|
132
|
+
return session;
|
|
133
|
+
}
|
|
134
|
+
function contextFor(spanId) {
|
|
135
|
+
const s = current();
|
|
136
|
+
return { traceId: s.traceId, spanId, ...(s.parentSpanId === undefined ? {} : { parentSpanId: s.parentSpanId }) };
|
|
137
|
+
}
|
|
138
|
+
function emit(test, spanId) {
|
|
139
|
+
const s = current();
|
|
140
|
+
const context = contextFor(spanId);
|
|
141
|
+
recorded.push(test);
|
|
142
|
+
if (s.file !== undefined) {
|
|
143
|
+
append(recordLine(s.resource, { spans: [testCaseSpan(test, context)] }));
|
|
144
|
+
append(recordLine(s.resource, { records: [testCaseLog(test, context)] }));
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
// Without IFC_TEST_RECORD, the test goes to the installed pipeline, if any (0008-test-evidence FR-004).
|
|
148
|
+
const pipeline = installed();
|
|
149
|
+
if (pipeline !== undefined) {
|
|
150
|
+
const source = { resource: s.resource, scope: TEST_SCOPE };
|
|
151
|
+
pipeline.emit({ kind: "span", span: testCaseSpan(test, context) }, source);
|
|
152
|
+
pipeline.emit({ kind: "log", record: testCaseLog(test, context) }, source);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
for (const l of listeners) {
|
|
156
|
+
try {
|
|
157
|
+
l({ test, resource: s.resource, context });
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
// A listener's failure never changes a test's outcome.
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
function emitStep(step, test, spanId, parentSpanId) {
|
|
165
|
+
const s = current();
|
|
166
|
+
const span = testStepSpan(step, test, { traceId: s.traceId, spanId, parentSpanId });
|
|
167
|
+
if (s.file !== undefined)
|
|
168
|
+
append(recordLine(s.resource, { spans: [span] }));
|
|
169
|
+
else
|
|
170
|
+
installed()?.emit({ kind: "span", span }, { resource: s.resource, scope: TEST_SCOPE });
|
|
171
|
+
}
|
|
172
|
+
const running = new Map();
|
|
173
|
+
let exitHooked = false;
|
|
174
|
+
function onExit() {
|
|
175
|
+
for (const [spanId, r] of [...running.entries()]) {
|
|
176
|
+
running.delete(spanId);
|
|
177
|
+
emit({ ...r.base, end: nowNanos(), outcome: "cancelled", reason: "the process exited before the test ended" }, spanId);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
function hookExit() {
|
|
181
|
+
if (exitHooked)
|
|
182
|
+
return;
|
|
183
|
+
exitHooked = true;
|
|
184
|
+
process.on("exit", onExit);
|
|
185
|
+
}
|
|
186
|
+
// ---------------------------------------------------------------------------------------------------------------------------- the test's file
|
|
187
|
+
const reals = new Map();
|
|
188
|
+
const SELF = normalise(import.meta.url);
|
|
189
|
+
function normalise(location) {
|
|
190
|
+
let path = location;
|
|
191
|
+
if (path.startsWith("file:")) {
|
|
192
|
+
try {
|
|
193
|
+
path = fileURLToPath(path);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return path;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return real(isAbsolute(path) ? path : resolve(path));
|
|
200
|
+
}
|
|
201
|
+
/** A path with its links resolved (macOS's `/var` is `/private/var`, and a stack may name either), or as given where it does not resolve. */
|
|
202
|
+
function real(path) {
|
|
203
|
+
const known = reals.get(path);
|
|
204
|
+
if (known !== undefined)
|
|
205
|
+
return known;
|
|
206
|
+
let r = path;
|
|
207
|
+
try {
|
|
208
|
+
r = realpathSync(path);
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
// A path that is not there (an eval's) is kept as it is.
|
|
212
|
+
}
|
|
213
|
+
reals.set(path, r);
|
|
214
|
+
return r;
|
|
215
|
+
}
|
|
216
|
+
const FRAME = /(?:\(|at\s+)((?:file:\/\/)?[^()\s]+?):\d+:\d+\)?\s*$/;
|
|
217
|
+
/** The file of the code that called into this module, relative to the working folder with `/`, from the call's stack. */
|
|
218
|
+
function callerFile() {
|
|
219
|
+
const stack = new Error("where").stack ?? "";
|
|
220
|
+
for (const line of stack.split("\n").slice(1)) {
|
|
221
|
+
const m = FRAME.exec(line.trim());
|
|
222
|
+
const location = m?.[1];
|
|
223
|
+
if (location === undefined || location.startsWith("node:") || location.startsWith("ext:"))
|
|
224
|
+
continue;
|
|
225
|
+
const path = normalise(location);
|
|
226
|
+
if (path === SELF)
|
|
227
|
+
continue;
|
|
228
|
+
return recordPath(relative(real(process.cwd()), path));
|
|
229
|
+
}
|
|
230
|
+
return "unknown";
|
|
231
|
+
}
|
|
232
|
+
// ---------------------------------------------------------------------------------------------------------------------------- recording
|
|
233
|
+
function testIdOf(file, name) {
|
|
234
|
+
return createHash("sha256").update(`${file}\0${name}`).digest("hex");
|
|
235
|
+
}
|
|
236
|
+
function shown(value) {
|
|
237
|
+
return cut(redactText(typeof value === "string" ? value : inspect(value, { depth: 6, sorted: true, breakLength: 100, maxStringLength: 8192 })));
|
|
238
|
+
}
|
|
239
|
+
function failure(e) {
|
|
240
|
+
const message = cut(redactText(e instanceof Error ? e.message : String(e)));
|
|
241
|
+
const assertion = e instanceof Error &&
|
|
242
|
+
(e.name === "AssertionError" || ("code" in e && e.code === "ERR_ASSERTION")) &&
|
|
243
|
+
"operator" in e &&
|
|
244
|
+
e.operator !== "fail" &&
|
|
245
|
+
e.operator !== undefined;
|
|
246
|
+
if (!assertion || !("expected" in e) || !("actual" in e))
|
|
247
|
+
return { message };
|
|
248
|
+
return { message, expected: shown(e.expected), actual: shown(e.actual) };
|
|
249
|
+
}
|
|
250
|
+
class TimedOut extends Error {
|
|
251
|
+
/** How long the test was given. */
|
|
252
|
+
ms;
|
|
253
|
+
constructor(ms) {
|
|
254
|
+
super(`the test did not end within ${ms} ms`);
|
|
255
|
+
this.name = "TimedOut";
|
|
256
|
+
this.ms = ms;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
async function within(fn, ms) {
|
|
260
|
+
if (ms === undefined) {
|
|
261
|
+
await fn();
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
let timer;
|
|
265
|
+
try {
|
|
266
|
+
await Promise.race([
|
|
267
|
+
Promise.resolve().then(fn),
|
|
268
|
+
new Promise((_, reject) => {
|
|
269
|
+
timer = setTimeout(() => reject(new TimedOut(ms)), ms);
|
|
270
|
+
}),
|
|
271
|
+
]);
|
|
272
|
+
}
|
|
273
|
+
finally {
|
|
274
|
+
if (timer !== undefined)
|
|
275
|
+
clearTimeout(timer);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* The context of a test or a step (FR-060): each step is a subtest of `node:test` on `node`, the runner's context, and is recorded in
|
|
280
|
+
* `into` and as a span whose parent is `parent` (the test's span or the enclosing step's), as it ends.
|
|
281
|
+
*/
|
|
282
|
+
function stepContext(name, node, test, testSpanId, parent, into, skipped) {
|
|
283
|
+
const step = async (stepName, fn) => {
|
|
284
|
+
if (!running.has(testSpanId))
|
|
285
|
+
throw new Error(`the step ${JSON.stringify(stepName)} ran after its test ${JSON.stringify(test.name)} ended`);
|
|
286
|
+
const spanId = newSpanId();
|
|
287
|
+
const start = nowNanos();
|
|
288
|
+
const inner = [];
|
|
289
|
+
let ran = false;
|
|
290
|
+
let skippedHere = false;
|
|
291
|
+
let thrown;
|
|
292
|
+
await node.test(stepName, async (sub) => {
|
|
293
|
+
ran = true;
|
|
294
|
+
try {
|
|
295
|
+
await fn(stepContext(stepName, sub, test, testSpanId, spanId, inner, () => {
|
|
296
|
+
skippedHere = true;
|
|
297
|
+
}));
|
|
298
|
+
}
|
|
299
|
+
catch (e) {
|
|
300
|
+
thrown = { error: e };
|
|
301
|
+
throw e;
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
const outcome = thrown !== undefined ? "fail" : ran && !skippedHere ? "pass" : "skip";
|
|
305
|
+
const recordedStep = {
|
|
306
|
+
name: stepName,
|
|
307
|
+
start,
|
|
308
|
+
end: nowNanos(),
|
|
309
|
+
outcome,
|
|
310
|
+
...(thrown === undefined ? {} : { message: failure(thrown.error).message }),
|
|
311
|
+
...(inner.length === 0 ? {} : { steps: inner }),
|
|
312
|
+
};
|
|
313
|
+
into.push(recordedStep);
|
|
314
|
+
emitStep(recordedStep, test, spanId, parent);
|
|
315
|
+
if (thrown !== undefined)
|
|
316
|
+
throw thrown.error;
|
|
317
|
+
};
|
|
318
|
+
const skip = (reason) => {
|
|
319
|
+
skipped(reason);
|
|
320
|
+
if (typeof node.skip === "function")
|
|
321
|
+
node.skip(reason);
|
|
322
|
+
};
|
|
323
|
+
return { name, step, skip };
|
|
324
|
+
}
|
|
325
|
+
async function runCase(base, fn, timeout, node) {
|
|
326
|
+
hookExit();
|
|
327
|
+
const spanId = newSpanId();
|
|
328
|
+
const started = { ...base, start: nowNanos(), attempt: current().attempt };
|
|
329
|
+
running.set(spanId, { base: started, spanId });
|
|
330
|
+
if (current().file !== undefined)
|
|
331
|
+
append(recordLine(current().resource, { records: [testStartLog(started, contextFor(spanId))] }));
|
|
332
|
+
const steps = [];
|
|
333
|
+
const withSteps = () => (steps.length === 0 ? {} : { steps: [...steps] });
|
|
334
|
+
let skipped;
|
|
335
|
+
const context = stepContext(base.name, node, base, spanId, spanId, steps, (reason) => {
|
|
336
|
+
skipped = { reason: reason === undefined ? undefined : reasonOf(reason) };
|
|
337
|
+
});
|
|
338
|
+
try {
|
|
339
|
+
await within(() => fn(context), timeout);
|
|
340
|
+
}
|
|
341
|
+
catch (e) {
|
|
342
|
+
if (running.delete(spanId)) {
|
|
343
|
+
emit(e instanceof TimedOut
|
|
344
|
+
? { ...started, end: nowNanos(), outcome: "cancelled", reason: e.message, ...withSteps() }
|
|
345
|
+
: { ...started, end: nowNanos(), outcome: "fail", ...failure(e), ...withSteps() }, spanId);
|
|
346
|
+
}
|
|
347
|
+
throw e;
|
|
348
|
+
}
|
|
349
|
+
if (!running.delete(spanId))
|
|
350
|
+
return;
|
|
351
|
+
const reason = skipped?.reason;
|
|
352
|
+
emit(skipped === undefined
|
|
353
|
+
? { ...started, end: nowNanos(), outcome: "pass", ...withSteps() }
|
|
354
|
+
: { ...started, end: nowNanos(), outcome: "skip", ...(reason === undefined ? {} : { reason }), ...withSteps() }, spanId);
|
|
355
|
+
}
|
|
356
|
+
function reasonOf(v) {
|
|
357
|
+
return typeof v === "string" && v.trim() !== "" ? cut(redactText(v)) : undefined;
|
|
358
|
+
}
|
|
359
|
+
/** For each file, whether a test of it is marked `only`, and its tests that have not started (FR-060). */
|
|
360
|
+
const onlyOf = new Map();
|
|
361
|
+
function onlyState(file) {
|
|
362
|
+
const known = onlyOf.get(file);
|
|
363
|
+
if (known !== undefined)
|
|
364
|
+
return known;
|
|
365
|
+
const state = { marked: false, waiting: new Set() };
|
|
366
|
+
onlyOf.set(file, state);
|
|
367
|
+
return state;
|
|
368
|
+
}
|
|
369
|
+
function register(name, requirements, fn, options, file) {
|
|
370
|
+
const base = {
|
|
371
|
+
id: testIdOf(file, name),
|
|
372
|
+
file,
|
|
373
|
+
name,
|
|
374
|
+
requirements: [...requirements],
|
|
375
|
+
risk: options.risk ?? "not-high",
|
|
376
|
+
...(options.characteristic === undefined ? {} : { characteristic: options.characteristic }),
|
|
377
|
+
};
|
|
378
|
+
const notRun = (outcome, v) => {
|
|
379
|
+
const at = nowNanos();
|
|
380
|
+
const reason = reasonOf(v);
|
|
381
|
+
emit({ ...base, start: at, end: at, outcome, attempt: current().attempt, ...(reason === undefined ? {} : { reason }) }, newSpanId());
|
|
382
|
+
};
|
|
383
|
+
if (options.todo !== undefined && options.todo !== false) {
|
|
384
|
+
notRun("todo", options.todo);
|
|
385
|
+
void nodeTest(name, { todo: options.todo }, () => undefined);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
if (options.skip !== undefined && options.skip !== false) {
|
|
389
|
+
notRun("skip", options.skip);
|
|
390
|
+
void nodeTest(name, { skip: options.skip }, () => undefined);
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
const state = onlyState(file);
|
|
394
|
+
const only = options.only === true;
|
|
395
|
+
if (only && !state.marked) {
|
|
396
|
+
state.marked = true;
|
|
397
|
+
for (const w of state.waiting) {
|
|
398
|
+
if (w.started)
|
|
399
|
+
continue;
|
|
400
|
+
w.skipped = true;
|
|
401
|
+
w.skip();
|
|
402
|
+
}
|
|
403
|
+
state.waiting.clear();
|
|
404
|
+
}
|
|
405
|
+
else if (!only && state.marked) {
|
|
406
|
+
notRun("skip", ONLY_REASON);
|
|
407
|
+
void nodeTest(name, { skip: ONLY_REASON }, () => undefined);
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
// a test that starts real programs (git, openssl) is slow on a CI runner, Windows above all, and Bun's runner ends a test at 5 seconds unless it is told otherwise
|
|
411
|
+
const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
|
|
412
|
+
const waiting = only ? undefined : { started: false, skipped: false, skip: () => notRun("skip", ONLY_REASON) };
|
|
413
|
+
if (waiting !== undefined)
|
|
414
|
+
state.waiting.add(waiting);
|
|
415
|
+
const body = (t) => {
|
|
416
|
+
if (waiting !== undefined) {
|
|
417
|
+
state.waiting.delete(waiting);
|
|
418
|
+
if (waiting.skipped) {
|
|
419
|
+
// Recorded as skipped when the test marked only was registered; the runner is told where it can be.
|
|
420
|
+
if (typeof t.skip === "function")
|
|
421
|
+
t.skip(ONLY_REASON);
|
|
422
|
+
return Promise.resolve();
|
|
423
|
+
}
|
|
424
|
+
waiting.started = true;
|
|
425
|
+
}
|
|
426
|
+
return runCase(base, fn, timeout, t);
|
|
427
|
+
};
|
|
428
|
+
void nodeTest(name, { timeout, ...(only ? { only: true } : {}) }, body);
|
|
429
|
+
}
|
|
430
|
+
/** A test registered with the runtime's runner, and recorded (0008-test-evidence FR-001). */
|
|
431
|
+
export function test(name, fn, options = {}) {
|
|
432
|
+
register(name, [], fn, options, options.file === undefined ? callerFile() : recordPath(options.file));
|
|
433
|
+
}
|
|
434
|
+
/** A test that proves one or more requirements, named for them: `0005-typescript-runtime FR-013: <name>`. */
|
|
435
|
+
export function proves(requirements, name, fn, options = {}) {
|
|
436
|
+
const full = `${requirements.join(", ")}: ${name}`;
|
|
437
|
+
proofs.push({ test: full, requirements });
|
|
438
|
+
register(full, requirements, fn, options, options.file === undefined ? callerFile() : recordPath(options.file));
|
|
439
|
+
}
|
|
440
|
+
/** A requirement proven by review rather than by a test, with the reason; recorded as an `ifc.test.review` log record (0008-test-evidence FR-006). */
|
|
441
|
+
export function reviewed(requirement, reason) {
|
|
442
|
+
reviews.push({ requirement, reason });
|
|
443
|
+
const r = { requirement, reason };
|
|
444
|
+
const s = current();
|
|
445
|
+
const record = reviewLog(r, nowNanos(), s.traceId);
|
|
446
|
+
if (s.file !== undefined)
|
|
447
|
+
append(recordLine(s.resource, { records: [record] }));
|
|
448
|
+
else
|
|
449
|
+
installed()?.emit({ kind: "log", record }, { resource: s.resource, scope: TEST_SCOPE });
|
|
450
|
+
}
|
|
451
|
+
/** Fails unless `value` is truthy; narrows its type. */
|
|
452
|
+
export function assert(value, message) {
|
|
453
|
+
if (message === undefined)
|
|
454
|
+
nodeAssert.ok(value);
|
|
455
|
+
else
|
|
456
|
+
nodeAssert.ok(value, message);
|
|
457
|
+
}
|
|
458
|
+
/** Fails unless the two are deeply and strictly equal (the same types, prototypes and values). */
|
|
459
|
+
export function assertEquals(actual, expected, message) {
|
|
460
|
+
if (message === undefined)
|
|
461
|
+
nodeAssert.deepStrictEqual(actual, expected);
|
|
462
|
+
else
|
|
463
|
+
nodeAssert.deepStrictEqual(actual, expected, message);
|
|
464
|
+
}
|
|
465
|
+
/** Fails where the two are deeply and strictly equal. */
|
|
466
|
+
export function assertNotEquals(actual, expected, message) {
|
|
467
|
+
if (message === undefined)
|
|
468
|
+
nodeAssert.notDeepStrictEqual(actual, expected);
|
|
469
|
+
else
|
|
470
|
+
nodeAssert.notDeepStrictEqual(actual, expected, message);
|
|
471
|
+
}
|
|
472
|
+
/** Fails unless the two are the same value (`Object.is`). */
|
|
473
|
+
export function assertStrictEquals(actual, expected, message) {
|
|
474
|
+
if (message === undefined)
|
|
475
|
+
nodeAssert.strictEqual(actual, expected);
|
|
476
|
+
else
|
|
477
|
+
nodeAssert.strictEqual(actual, expected, message);
|
|
478
|
+
}
|
|
479
|
+
/** Fails unless the value is neither `null` nor `undefined`; narrows its type. */
|
|
480
|
+
export function assertExists(value, message) {
|
|
481
|
+
nodeAssert.ok(value !== null && value !== undefined, message ?? "expected a value, found null or undefined");
|
|
482
|
+
}
|
|
483
|
+
/**
|
|
484
|
+
* The value, narrowed to `NonNullable`, or a failure as {@link assertExists} fails where it is `null` or `undefined` (FR-065), so that a
|
|
485
|
+
* test reads `the(map.get(key)).name` where an assertion would need a statement of its own.
|
|
486
|
+
*/
|
|
487
|
+
export function the(value, message) {
|
|
488
|
+
assertExists(value, message);
|
|
489
|
+
return value;
|
|
490
|
+
}
|
|
491
|
+
/** Fails unless the value is an instance of the class; narrows its type. */
|
|
492
|
+
export function assertInstanceOf(value, Class, message) {
|
|
493
|
+
nodeAssert.ok(value instanceof Class, message ?? `expected an instance of ${Class.name}`);
|
|
494
|
+
}
|
|
495
|
+
/** Fails unless the text includes the part. */
|
|
496
|
+
export function assertStringIncludes(actual, expected, message) {
|
|
497
|
+
nodeAssert.ok(actual.includes(expected), message ?? `expected ${JSON.stringify(actual)} to include ${JSON.stringify(expected)}`);
|
|
498
|
+
}
|
|
499
|
+
/** Fails unless the two are different values (`Object.is`), as `@std/assert`'s `assertNotStrictEquals` (FR-060). */
|
|
500
|
+
export function assertNotStrictEquals(actual, expected, message) {
|
|
501
|
+
if (message === undefined)
|
|
502
|
+
nodeAssert.notStrictEqual(actual, expected);
|
|
503
|
+
else
|
|
504
|
+
nodeAssert.notStrictEqual(actual, expected, message);
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* Fails unless the two numbers are within `tolerance` of each other (`1e-7` by default), as `@std/assert`'s `assertAlmostEquals`
|
|
508
|
+
* (FR-060): the same value (`Object.is`, so `NaN` is close to `NaN` and an infinity to itself) passes, else `|expected - actual|` must not
|
|
509
|
+
* exceed the tolerance.
|
|
510
|
+
*/
|
|
511
|
+
export function assertAlmostEquals(actual, expected, tolerance = 1e-7, message) {
|
|
512
|
+
if (Object.is(actual, expected))
|
|
513
|
+
return;
|
|
514
|
+
const delta = Math.abs(expected - actual);
|
|
515
|
+
if (delta <= tolerance)
|
|
516
|
+
return;
|
|
517
|
+
const f = (n) => (Number.isInteger(n) ? String(n) : n.toExponential());
|
|
518
|
+
throw new nodeAssert.AssertionError({
|
|
519
|
+
message: `Expected actual: "${f(actual)}" to be close to "${f(expected)}": delta "${f(delta)}" is greater than "${f(tolerance)}"${message === undefined ? "." : `: ${message}`}`,
|
|
520
|
+
actual,
|
|
521
|
+
expected,
|
|
522
|
+
operator: "assertAlmostEquals",
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
/** Fails where the text matches the pattern, as `@std/assert`'s `assertNotMatch` (FR-060). */
|
|
526
|
+
export function assertNotMatch(actual, pattern, message) {
|
|
527
|
+
if (message === undefined)
|
|
528
|
+
nodeAssert.doesNotMatch(actual, pattern);
|
|
529
|
+
else
|
|
530
|
+
nodeAssert.doesNotMatch(actual, pattern, message);
|
|
531
|
+
}
|
|
532
|
+
/** Fails unless the text matches the pattern. */
|
|
533
|
+
export function assertMatch(actual, pattern, message) {
|
|
534
|
+
if (message === undefined)
|
|
535
|
+
nodeAssert.match(actual, pattern);
|
|
536
|
+
else
|
|
537
|
+
nodeAssert.match(actual, pattern, message);
|
|
538
|
+
}
|
|
539
|
+
/** Fails unless every expected element is deeply equal to some element of the array. */
|
|
540
|
+
export function assertArrayIncludes(actual, expected, message) {
|
|
541
|
+
const missing = expected.filter((e) => !actual.some((a) => isDeepStrictEqual(a, e)));
|
|
542
|
+
nodeAssert.ok(missing.length === 0, message ?? `expected the array to include ${JSON.stringify(missing)}`);
|
|
543
|
+
}
|
|
544
|
+
function isDeepStrictEqual(a, b) {
|
|
545
|
+
try {
|
|
546
|
+
nodeAssert.deepStrictEqual(a, b);
|
|
547
|
+
return true;
|
|
548
|
+
}
|
|
549
|
+
catch {
|
|
550
|
+
return false;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
/** The error a function throws; fails where it throws none, or one not of `Class`, or one whose message does not include `includes`. */
|
|
554
|
+
export function assertThrows(fn, Class, includes) {
|
|
555
|
+
try {
|
|
556
|
+
fn();
|
|
557
|
+
}
|
|
558
|
+
catch (e) {
|
|
559
|
+
checkThrown(e, Class, includes);
|
|
560
|
+
return e;
|
|
561
|
+
}
|
|
562
|
+
throw new nodeAssert.AssertionError({ message: "expected the function to throw" });
|
|
563
|
+
}
|
|
564
|
+
/** The error a promise rejects with; fails where it fulfils, or rejects with one not of `Class`, or whose message does not include `includes`. */
|
|
565
|
+
export async function assertRejects(fn, Class, includes) {
|
|
566
|
+
try {
|
|
567
|
+
await fn();
|
|
568
|
+
}
|
|
569
|
+
catch (e) {
|
|
570
|
+
checkThrown(e, Class, includes);
|
|
571
|
+
return e;
|
|
572
|
+
}
|
|
573
|
+
throw new nodeAssert.AssertionError({ message: "expected the promise to reject" });
|
|
574
|
+
}
|
|
575
|
+
function checkThrown(e, Class, includes) {
|
|
576
|
+
if (Class !== undefined && !(e instanceof Class)) {
|
|
577
|
+
throw new nodeAssert.AssertionError({
|
|
578
|
+
message: `expected ${Class.name}, got ${e instanceof Error ? e.name : String(e)}: ${String(e)}`,
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
if (includes !== undefined && !(e instanceof Error && e.message.includes(includes))) {
|
|
582
|
+
throw new nodeAssert.AssertionError({ message: `expected the error to include ${JSON.stringify(includes)}, got ${String(e)}` });
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
/** Fails, with the message. */
|
|
586
|
+
export function fail(message) {
|
|
587
|
+
throw new nodeAssert.AssertionError({ message });
|
|
588
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,48 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@intellectual-frontiers/test",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Tests that run unchanged under bun test and node --test, and are recorded as evidence by their own code: registration on node:test, the assertions under short plain names, proves (which names the requirements a test proves), only, steps, and the record of every test as an OpenTelemetry span and log record on the installation record.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"engines": {
|
|
8
|
+
"node": ">=26"
|
|
9
|
+
},
|
|
10
|
+
"sideEffects": false,
|
|
11
|
+
"types": "./dist/test.d.ts",
|
|
12
|
+
"exports": {
|
|
13
|
+
".": {
|
|
14
|
+
"types": "./dist/test.d.ts",
|
|
15
|
+
"bun": "./src/test.ts",
|
|
16
|
+
"default": "./dist/test.js"
|
|
17
|
+
},
|
|
18
|
+
"./record": {
|
|
19
|
+
"types": "./dist/record.d.ts",
|
|
20
|
+
"bun": "./src/record.ts",
|
|
21
|
+
"default": "./dist/record.js"
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
"dependencies": {
|
|
25
|
+
"@intellectual-frontiers/core": "0.1.3",
|
|
26
|
+
"@intellectual-frontiers/crypto": "0.1.3",
|
|
27
|
+
"@intellectual-frontiers/system": "0.1.3",
|
|
28
|
+
"@intellectual-frontiers/telemetry": "0.1.3"
|
|
29
|
+
},
|
|
30
|
+
"files": [
|
|
31
|
+
"dist",
|
|
32
|
+
"src",
|
|
33
|
+
"README.md",
|
|
34
|
+
"LICENSE"
|
|
35
|
+
],
|
|
36
|
+
"repository": {
|
|
37
|
+
"type": "git",
|
|
38
|
+
"url": "git+https://github.com/intellectual-frontiers/.github.git",
|
|
39
|
+
"directory": "commons/test"
|
|
40
|
+
},
|
|
41
|
+
"homepage": "https://github.com/intellectual-frontiers/.github/tree/main/commons/test",
|
|
42
|
+
"bugs": {
|
|
43
|
+
"url": "https://github.com/intellectual-frontiers/.github/issues"
|
|
44
|
+
},
|
|
45
|
+
"publishConfig": {
|
|
46
|
+
"access": "public"
|
|
47
|
+
}
|
|
48
|
+
}
|