@intellectual-frontiers/test 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/dist/record.d.ts +222 -0
- package/dist/record.js +471 -0
- package/dist/test.d.ts +108 -0
- package/dist/test.js +588 -0
- package/package.json +46 -4
- package/src/record.ts +612 -0
- package/src/test.ts +713 -0
package/dist/record.js
ADDED
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
// Spec: 0001-test FR-004
|
|
2
|
+
/**
|
|
3
|
+
* The capture half of test evidence (commons 0008-test-evidence FR-001 to FR-004), kept apart from the reports so that every test process
|
|
4
|
+
* loads only this: the model of a test case and of an installation, a test case as its `test` span and log record with the `test.*`
|
|
5
|
+
* conventions and the `ifc.test.*` attributes (`testCaseSpan`, `testCaseLog`), the mark a test leaves as it starts (`testStartLog`), so
|
|
6
|
+
* that one that never ends is closed as `cancelled` (`closeOrphans`), the installation record as a resource (`installationFromEnv`,
|
|
7
|
+
* which refuses a role that names a person, and `isPersonAttribute`), and the OTLP JSON lines a test process appends to
|
|
8
|
+
* `IFC_TEST_RECORD` (`recordLine`, `decodeLines`). `./test-report.ts` reads a run's record and renders it. A test's steps (commons
|
|
9
|
+
* 0005-typescript-runtime FR-060) are its `steps`, named by its span's `ifc.test.steps`, each a `test step` span whose parent is the
|
|
10
|
+
* test's span or the step it is in (`testStepSpan`).
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```ts
|
|
14
|
+
* import { installationFromEnv, installationResource, recordLine, testCaseSpan } from "./record.ts";
|
|
15
|
+
* const { installation } = installationFromEnv({ runtime: "node", runtimeVersion: "26.0.0", os: "linux", arch: "x86_64" }, { CI: "true" });
|
|
16
|
+
* // recordLine(installationResource(installation), { spans: [testCaseSpan(test, context)] }) is one line of IFC_TEST_RECORD
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* @module
|
|
20
|
+
*/
|
|
21
|
+
import { absurd, formatDecodeError, err, ok } from "@intellectual-frontiers/core";
|
|
22
|
+
import { parseSpanId, SEVERITY, } from "@intellectual-frontiers/telemetry/otel";
|
|
23
|
+
import { decodeLogs, decodeTraces, encodeLogs, encodeTraces } from "@intellectual-frontiers/telemetry/otlp";
|
|
24
|
+
import { sha256 } from "@intellectual-frontiers/crypto/sha256";
|
|
25
|
+
/** A role (lower-case words joined by `-`, `_`, `.` or `/`), never a person: the rule of `grc`'s `roleOf`, held here so that the record depends on nothing above it (a test of the whole keeps the two the same). */
|
|
26
|
+
const ROLE = /^[a-z][a-z0-9]*(?:[-_./][a-z0-9]+)*$/;
|
|
27
|
+
const roleOf = (text) => (ROLE.test(text) ? text : undefined);
|
|
28
|
+
/** Every outcome, in the order reports list them. */
|
|
29
|
+
export const OUTCOMES = ["pass", "fail", "skip", "todo", "cancelled"];
|
|
30
|
+
/** Whether a text is an outcome. */
|
|
31
|
+
export function isOutcome(text) {
|
|
32
|
+
return OUTCOMES.includes(text);
|
|
33
|
+
}
|
|
34
|
+
/** Whether a text is a risk. */
|
|
35
|
+
export function isRisk(text) {
|
|
36
|
+
return text === "high" || text === "not-high";
|
|
37
|
+
}
|
|
38
|
+
/** Whether a text is a step's outcome. */
|
|
39
|
+
export function isStepOutcome(text) {
|
|
40
|
+
return text === "pass" || text === "fail" || text === "skip";
|
|
41
|
+
}
|
|
42
|
+
// ---------------------------------------------------------------------------------------------------------------------------- names
|
|
43
|
+
/** The instrumentation scope of test records. */
|
|
44
|
+
export const TEST_SCOPE = { name: "ifc-test", version: "1" };
|
|
45
|
+
/** The name of a test's span. */
|
|
46
|
+
export const TEST_SPAN = "test";
|
|
47
|
+
/** The name of a step's span, a child of its test's span or of the step it is in (0005-typescript-runtime FR-060). */
|
|
48
|
+
export const STEP_SPAN = "test step";
|
|
49
|
+
/** The name of a run's span. */
|
|
50
|
+
export const RUN_SPAN = "test run";
|
|
51
|
+
/** The event names of the log records of a test record. */
|
|
52
|
+
export const EVENTS = {
|
|
53
|
+
case: "ifc.test.case",
|
|
54
|
+
start: "ifc.test.start",
|
|
55
|
+
review: "ifc.test.review",
|
|
56
|
+
runner: "ifc.test.runner",
|
|
57
|
+
warning: "ifc.test.warning",
|
|
58
|
+
};
|
|
59
|
+
/** The most characters a message, an expected or an actual value keeps (FR-001). */
|
|
60
|
+
export const TEXT_LIMIT = 4096;
|
|
61
|
+
/** A text cut to `TEXT_LIMIT` characters (without splitting a surrogate pair). */
|
|
62
|
+
export function cut(text) {
|
|
63
|
+
if (text.length <= TEXT_LIMIT)
|
|
64
|
+
return text;
|
|
65
|
+
const end = /[\uD800-\uDBFF]/.test(text.charAt(TEXT_LIMIT - 1)) ? TEXT_LIMIT - 1 : TEXT_LIMIT;
|
|
66
|
+
return text.slice(0, end);
|
|
67
|
+
}
|
|
68
|
+
/** A test's id: the SHA-256 of its file, a NUL and its name (FR-001). */
|
|
69
|
+
export function testId(file, name) {
|
|
70
|
+
return sha256(`${file}\0${name}`);
|
|
71
|
+
}
|
|
72
|
+
/** A file path as a record names it: `\` as `/`, without a leading `./`. */
|
|
73
|
+
export function recordPath(path) {
|
|
74
|
+
return path.replaceAll("\\", "/").replace(/^(?:\.\/)+/, "");
|
|
75
|
+
}
|
|
76
|
+
// ---------------------------------------------------------------------------------------------------------------------------- installation (FR-003)
|
|
77
|
+
/** Whether a resource attribute names a person or a machine of one, and is refused (FR-003): `user.*`, `enduser.*`, `host.name`. */
|
|
78
|
+
export function isPersonAttribute(key) {
|
|
79
|
+
return key.startsWith("user.") || key.startsWith("enduser.") || key === "host.name" || key === "host.id";
|
|
80
|
+
}
|
|
81
|
+
const ARCH = { x86_64: "amd64", x64: "amd64", aarch64: "arm64", arm: "arm32", ia32: "x86", x86: "x86" };
|
|
82
|
+
/** A machine's architecture as OpenTelemetry's `host.arch` names it (`amd64`, `arm64`, ...). */
|
|
83
|
+
export function hostArch(arch) {
|
|
84
|
+
return ARCH[arch] ?? arch;
|
|
85
|
+
}
|
|
86
|
+
const SHA256_TEXT = /^[0-9a-f]{64}$/;
|
|
87
|
+
/**
|
|
88
|
+
* The installation record of a process, from what the runtime says of itself and the environment given as a record (the kernel reads no
|
|
89
|
+
* environment): `IFC_TEST_REVISION` or `GITHUB_SHA`, `IFC_TEST_LOCKFILE_SHA256`, `IFC_TEST_PINS_SHA256`, and `IFC_TEST_ROLE` (`ci` where
|
|
90
|
+
* `CI` is set, else `developer`). A role that is not a role (an e-mail address, a name with capitals or spaces) is refused with a
|
|
91
|
+
* warning and the default used, and so is a digest that is not a SHA-256.
|
|
92
|
+
*/
|
|
93
|
+
export function installationFromEnv(self, env) {
|
|
94
|
+
const warnings = [];
|
|
95
|
+
const get = (k) => {
|
|
96
|
+
const v = env[k]?.trim();
|
|
97
|
+
return v === undefined || v === "" ? undefined : v;
|
|
98
|
+
};
|
|
99
|
+
const fallback = get("CI") === undefined || get("CI") === "false" ? "developer" : "ci";
|
|
100
|
+
const given = get("IFC_TEST_ROLE");
|
|
101
|
+
let role = fallback;
|
|
102
|
+
if (given !== undefined) {
|
|
103
|
+
if (roleOf(given) === undefined)
|
|
104
|
+
warnings.push(`IFC_TEST_ROLE is not a role (lower-case words, never a person); ${fallback} is used`);
|
|
105
|
+
else
|
|
106
|
+
role = given;
|
|
107
|
+
}
|
|
108
|
+
const digest = (k) => {
|
|
109
|
+
const v = get(k)?.toLowerCase();
|
|
110
|
+
if (v === undefined)
|
|
111
|
+
return undefined;
|
|
112
|
+
if (SHA256_TEXT.test(v))
|
|
113
|
+
return v;
|
|
114
|
+
warnings.push(`${k} is not a SHA-256; it is left out`);
|
|
115
|
+
return undefined;
|
|
116
|
+
};
|
|
117
|
+
const revision = get("IFC_TEST_REVISION") ?? get("GITHUB_SHA");
|
|
118
|
+
const lockfileSha256 = digest("IFC_TEST_LOCKFILE_SHA256");
|
|
119
|
+
const pinsSha256 = digest("IFC_TEST_PINS_SHA256");
|
|
120
|
+
return {
|
|
121
|
+
installation: {
|
|
122
|
+
runtime: self.runtime,
|
|
123
|
+
runtimeVersion: self.runtimeVersion,
|
|
124
|
+
os: self.os,
|
|
125
|
+
arch: hostArch(self.arch),
|
|
126
|
+
...(revision === undefined ? {} : { revision }),
|
|
127
|
+
...(lockfileSha256 === undefined ? {} : { lockfileSha256 }),
|
|
128
|
+
...(pinsSha256 === undefined ? {} : { pinsSha256 }),
|
|
129
|
+
role,
|
|
130
|
+
},
|
|
131
|
+
warnings,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
/** The resource of an installation record (FR-003). */
|
|
135
|
+
export function installationResource(i) {
|
|
136
|
+
return {
|
|
137
|
+
attributes: {
|
|
138
|
+
"service.name": "ifc-test",
|
|
139
|
+
"process.runtime.name": i.runtime,
|
|
140
|
+
"process.runtime.version": i.runtimeVersion,
|
|
141
|
+
"os.type": i.os,
|
|
142
|
+
"host.arch": i.arch,
|
|
143
|
+
...(i.revision === undefined ? {} : { "vcs.ref.head.revision": i.revision }),
|
|
144
|
+
...(i.lockfileSha256 === undefined ? {} : { "ifc.test.lockfile.sha256": i.lockfileSha256 }),
|
|
145
|
+
...(i.pinsSha256 === undefined ? {} : { "ifc.test.pins.sha256": i.pinsSha256 }),
|
|
146
|
+
"ifc.test.role": i.role,
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/** An attribute's value where it is a string. */
|
|
151
|
+
export function attributeText(a, key) {
|
|
152
|
+
const v = a[key];
|
|
153
|
+
return typeof v === "string" ? v : undefined;
|
|
154
|
+
}
|
|
155
|
+
/** An attribute's value where it is an array of strings, else none. */
|
|
156
|
+
export function attributeTexts(a, key) {
|
|
157
|
+
const v = a[key];
|
|
158
|
+
return Array.isArray(v) && v.every((e) => typeof e === "string") ? v : [];
|
|
159
|
+
}
|
|
160
|
+
/** An attribute's value where it is an integer. */
|
|
161
|
+
export function attributeInteger(a, key) {
|
|
162
|
+
const v = a[key];
|
|
163
|
+
if (typeof v === "bigint")
|
|
164
|
+
return Number(v);
|
|
165
|
+
return typeof v === "number" && Number.isInteger(v) ? v : undefined;
|
|
166
|
+
}
|
|
167
|
+
/** The installation record a resource holds; unknown fields are `unknown`. */
|
|
168
|
+
export function installationOf(r) {
|
|
169
|
+
const a = r.attributes;
|
|
170
|
+
const revision = attributeText(a, "vcs.ref.head.revision");
|
|
171
|
+
const lockfileSha256 = attributeText(a, "ifc.test.lockfile.sha256");
|
|
172
|
+
const pinsSha256 = attributeText(a, "ifc.test.pins.sha256");
|
|
173
|
+
return {
|
|
174
|
+
runtime: attributeText(a, "process.runtime.name") ?? "unknown",
|
|
175
|
+
runtimeVersion: attributeText(a, "process.runtime.version") ?? "unknown",
|
|
176
|
+
os: attributeText(a, "os.type") ?? "unknown",
|
|
177
|
+
arch: attributeText(a, "host.arch") ?? "unknown",
|
|
178
|
+
...(revision === undefined ? {} : { revision }),
|
|
179
|
+
...(lockfileSha256 === undefined ? {} : { lockfileSha256 }),
|
|
180
|
+
...(pinsSha256 === undefined ? {} : { pinsSha256 }),
|
|
181
|
+
role: attributeText(a, "ifc.test.role") ?? "unknown",
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
function caseAttributes(c) {
|
|
185
|
+
const status = c.outcome === "pass" ? "pass" : c.outcome === "fail" || c.outcome === "cancelled" ? "fail" : undefined;
|
|
186
|
+
return {
|
|
187
|
+
"test.case.name": c.name,
|
|
188
|
+
...(status === undefined ? {} : { "test.case.result.status": status }),
|
|
189
|
+
"test.suite.name": c.file,
|
|
190
|
+
"code.file.path": c.file,
|
|
191
|
+
"ifc.test.id": c.id,
|
|
192
|
+
"ifc.test.requirements": [...c.requirements],
|
|
193
|
+
"ifc.test.risk": c.risk,
|
|
194
|
+
"ifc.test.outcome": c.outcome,
|
|
195
|
+
"ifc.test.attempt": BigInt(c.attempt),
|
|
196
|
+
...(c.characteristic === undefined ? {} : { "ifc.test.characteristic": c.characteristic }),
|
|
197
|
+
...(c.reason === undefined ? {} : { "ifc.test.reason": c.reason }),
|
|
198
|
+
...(c.message === undefined ? {} : { "ifc.test.message": c.message }),
|
|
199
|
+
...(c.expected === undefined ? {} : { "ifc.test.expected": c.expected }),
|
|
200
|
+
...(c.actual === undefined ? {} : { "ifc.test.actual": c.actual }),
|
|
201
|
+
...(c.steps === undefined || c.steps.length === 0 ? {} : { "ifc.test.steps": stepPaths(c.steps) }),
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
/** The steps of a test, each by its path (a step within another is `outer > inner`), in their order with each step's own steps first. */
|
|
205
|
+
export function stepPaths(steps, within = []) {
|
|
206
|
+
return steps.flatMap((s) => [...stepPaths(s.steps ?? [], [...within, s.name]), [...within, s.name].join(" > ")]);
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* A step as its `test step` span (0005-typescript-runtime FR-060): a child of `ctx.parentSpanId` (its test's span or the step it is in),
|
|
210
|
+
* with its test's id and file, its name, outcome and a failure's message, and an error status where it failed.
|
|
211
|
+
*/
|
|
212
|
+
export function testStepSpan(step, test, ctx) {
|
|
213
|
+
return {
|
|
214
|
+
name: STEP_SPAN,
|
|
215
|
+
context: { traceId: ctx.traceId, spanId: ctx.spanId, traceFlags: 1, traceState: [], isRemote: false },
|
|
216
|
+
...(ctx.parentSpanId === undefined ? {} : { parentSpanId: ctx.parentSpanId }),
|
|
217
|
+
kind: "internal",
|
|
218
|
+
startTime: step.start,
|
|
219
|
+
endTime: step.end,
|
|
220
|
+
attributes: {
|
|
221
|
+
"test.suite.name": test.file,
|
|
222
|
+
"code.file.path": test.file,
|
|
223
|
+
"ifc.test.id": test.id,
|
|
224
|
+
"ifc.test.step.name": step.name,
|
|
225
|
+
"ifc.test.outcome": step.outcome,
|
|
226
|
+
...(step.message === undefined ? {} : { "ifc.test.message": step.message }),
|
|
227
|
+
},
|
|
228
|
+
events: [],
|
|
229
|
+
links: [],
|
|
230
|
+
status: step.outcome === "fail"
|
|
231
|
+
? { kind: "error", message: step.message ?? "failed" }
|
|
232
|
+
: step.outcome === "pass"
|
|
233
|
+
? { kind: "ok" }
|
|
234
|
+
: { kind: "unset" },
|
|
235
|
+
droppedAttributesCount: 0,
|
|
236
|
+
droppedEventsCount: 0,
|
|
237
|
+
droppedLinksCount: 0,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
/** A test as its `test` span (FR-002): the conventions, the `ifc.test.*` attributes, and an error status where it failed or was cancelled. */
|
|
241
|
+
export function testCaseSpan(c, ctx) {
|
|
242
|
+
const failed = c.outcome === "fail" || c.outcome === "cancelled";
|
|
243
|
+
return {
|
|
244
|
+
name: TEST_SPAN,
|
|
245
|
+
context: { traceId: ctx.traceId, spanId: ctx.spanId, traceFlags: 1, traceState: [], isRemote: false },
|
|
246
|
+
...(ctx.parentSpanId === undefined ? {} : { parentSpanId: ctx.parentSpanId }),
|
|
247
|
+
kind: "internal",
|
|
248
|
+
startTime: c.start,
|
|
249
|
+
endTime: c.end,
|
|
250
|
+
attributes: caseAttributes(c),
|
|
251
|
+
events: [],
|
|
252
|
+
links: [],
|
|
253
|
+
status: failed
|
|
254
|
+
? { kind: "error", message: c.message ?? c.reason ?? c.outcome }
|
|
255
|
+
: c.outcome === "pass"
|
|
256
|
+
? { kind: "ok" }
|
|
257
|
+
: { kind: "unset" },
|
|
258
|
+
droppedAttributesCount: 0,
|
|
259
|
+
droppedEventsCount: 0,
|
|
260
|
+
droppedLinksCount: 0,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
/** A test as its log record (FR-002): the same attributes, the name as its body, at ERROR where it failed or was cancelled. */
|
|
264
|
+
export function testCaseLog(c, ctx) {
|
|
265
|
+
const failed = c.outcome === "fail" || c.outcome === "cancelled";
|
|
266
|
+
return {
|
|
267
|
+
time: c.end,
|
|
268
|
+
observedTime: c.end,
|
|
269
|
+
severityNumber: failed ? SEVERITY.ERROR : SEVERITY.INFO,
|
|
270
|
+
severityText: failed ? "ERROR" : "INFO",
|
|
271
|
+
body: `${c.outcome} ${c.name}`,
|
|
272
|
+
attributes: caseAttributes(c),
|
|
273
|
+
droppedAttributesCount: 0,
|
|
274
|
+
eventName: EVENTS.case,
|
|
275
|
+
traceId: ctx.traceId,
|
|
276
|
+
spanId: ctx.spanId,
|
|
277
|
+
traceFlags: 1,
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
/** The mark a test leaves as it starts (FR-004): its span id, so that a test whose span never comes is closed as cancelled. */
|
|
281
|
+
export function testStartLog(c, ctx) {
|
|
282
|
+
return {
|
|
283
|
+
time: c.start,
|
|
284
|
+
observedTime: c.start,
|
|
285
|
+
severityNumber: SEVERITY.DEBUG,
|
|
286
|
+
severityText: "DEBUG",
|
|
287
|
+
body: `start ${c.name}`,
|
|
288
|
+
attributes: {
|
|
289
|
+
"test.case.name": c.name,
|
|
290
|
+
"test.suite.name": c.file,
|
|
291
|
+
"ifc.test.id": c.id,
|
|
292
|
+
"ifc.test.requirements": [...c.requirements],
|
|
293
|
+
"ifc.test.risk": c.risk,
|
|
294
|
+
"ifc.test.attempt": BigInt(c.attempt),
|
|
295
|
+
...(c.characteristic === undefined ? {} : { "ifc.test.characteristic": c.characteristic }),
|
|
296
|
+
...(ctx.parentSpanId === undefined ? {} : { "ifc.test.parent": ctx.parentSpanId }),
|
|
297
|
+
},
|
|
298
|
+
droppedAttributesCount: 0,
|
|
299
|
+
eventName: EVENTS.start,
|
|
300
|
+
traceId: ctx.traceId,
|
|
301
|
+
spanId: ctx.spanId,
|
|
302
|
+
traceFlags: 1,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
function plainLog(eventName, time, body, attributes, warn = false) {
|
|
306
|
+
return {
|
|
307
|
+
time,
|
|
308
|
+
observedTime: time,
|
|
309
|
+
severityNumber: warn ? SEVERITY.WARN : SEVERITY.INFO,
|
|
310
|
+
severityText: warn ? "WARN" : "INFO",
|
|
311
|
+
body,
|
|
312
|
+
attributes,
|
|
313
|
+
droppedAttributesCount: 0,
|
|
314
|
+
eventName,
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
/** A review declared in a test file or a list (`ifc.test.review`). */
|
|
318
|
+
export function reviewLog(r, time, traceId) {
|
|
319
|
+
const base = plainLog(EVENTS.review, time, `reviewed ${r.requirement}`, {
|
|
320
|
+
"ifc.test.requirement": r.requirement,
|
|
321
|
+
"ifc.test.reason": r.reason,
|
|
322
|
+
});
|
|
323
|
+
return traceId === undefined ? base : { ...base, traceId };
|
|
324
|
+
}
|
|
325
|
+
/** A warning of a test process or a run (`ifc.test.warning`), such as a role refused. */
|
|
326
|
+
export function warningLog(message, time) {
|
|
327
|
+
return plainLog(EVENTS.warning, time, message, { "ifc.test.warning": message }, true);
|
|
328
|
+
}
|
|
329
|
+
/** What one runner did (`ifc.test.runner`). */
|
|
330
|
+
export function runnerLog(r, time) {
|
|
331
|
+
const attributes = r.kind === "exited"
|
|
332
|
+
? { "process.runtime.name": r.runtime, "ifc.test.runner.found": true, "process.exit.code": BigInt(r.code) }
|
|
333
|
+
: { "process.runtime.name": r.runtime, "ifc.test.runner.found": false };
|
|
334
|
+
return plainLog(EVENTS.runner, time, r.kind === "exited" ? `${r.runtime} exited with ${r.code}` : `${r.runtime} is not installed`, attributes, r.kind === "missing" || (r.kind === "exited" && r.code !== 0));
|
|
335
|
+
}
|
|
336
|
+
/** One line of `IFC_TEST_RECORD`: an OTLP JSON traces or logs export request of one resource, without a line feed (FR-004). */
|
|
337
|
+
export function recordLine(resource, signal) {
|
|
338
|
+
return JSON.stringify("spans" in signal
|
|
339
|
+
? encodeTraces([{ resource, scope: TEST_SCOPE, spans: signal.spans }])
|
|
340
|
+
: encodeLogs([{ resource, scope: TEST_SCOPE, records: signal.records }]));
|
|
341
|
+
}
|
|
342
|
+
/** A record error as one sentence. */
|
|
343
|
+
export function formatRecordError(e) {
|
|
344
|
+
switch (e.kind) {
|
|
345
|
+
case "json":
|
|
346
|
+
return `line ${e.line} is not JSON: ${e.reason}`;
|
|
347
|
+
case "otlp":
|
|
348
|
+
return `line ${e.line} is not OTLP JSON: ${formatDecodeError(e.error)}`;
|
|
349
|
+
case "person":
|
|
350
|
+
return `the record names a person or a machine of one (${e.key}), which a test record must not`;
|
|
351
|
+
case "test":
|
|
352
|
+
return `the test span ${e.spanId} is not a test record: ${e.reason}`;
|
|
353
|
+
case "no-run":
|
|
354
|
+
return `the record has no run: ${e.reason}`;
|
|
355
|
+
default:
|
|
356
|
+
return absurd(e);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
/** The traces and logs of one OTLP JSON document (an export request, or a run's record holding both), as line `line`. */
|
|
360
|
+
export function decodeRecordDocument(value, line) {
|
|
361
|
+
const t = decodeTraces(value);
|
|
362
|
+
if (t.kind === "err")
|
|
363
|
+
return err({ kind: "otlp", line, error: t.error });
|
|
364
|
+
const l = decodeLogs(value);
|
|
365
|
+
if (l.kind === "err")
|
|
366
|
+
return err({ kind: "otlp", line, error: l.error });
|
|
367
|
+
return ok({ traces: t.value, logs: l.value });
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* The signals of `IFC_TEST_RECORD`'s lines, each an OTLP JSON export request (FR-004). A last line cut short (a process killed while it
|
|
371
|
+
* wrote) is left out with a warning; any other line that is not one is an error.
|
|
372
|
+
*/
|
|
373
|
+
export function decodeLines(input) {
|
|
374
|
+
const traces = [];
|
|
375
|
+
const logs = [];
|
|
376
|
+
const warnings = [];
|
|
377
|
+
const lines = input.split("\n");
|
|
378
|
+
for (const [i, raw] of lines.entries()) {
|
|
379
|
+
const line = raw.trim();
|
|
380
|
+
if (line === "")
|
|
381
|
+
continue;
|
|
382
|
+
let value;
|
|
383
|
+
try {
|
|
384
|
+
value = JSON.parse(line);
|
|
385
|
+
}
|
|
386
|
+
catch (e) {
|
|
387
|
+
if (i === lines.length - 1) {
|
|
388
|
+
warnings.push(`the last line of a test record was cut short and is left out`);
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
return err({ kind: "json", line: i + 1, reason: e instanceof Error ? e.message : String(e) });
|
|
392
|
+
}
|
|
393
|
+
const d = decodeRecordDocument(value, i + 1);
|
|
394
|
+
if (d.kind === "err")
|
|
395
|
+
return d;
|
|
396
|
+
traces.push(...d.value.traces);
|
|
397
|
+
logs.push(...d.value.logs);
|
|
398
|
+
}
|
|
399
|
+
return ok({ traces, logs, warnings });
|
|
400
|
+
}
|
|
401
|
+
function resourceKey(r, s) {
|
|
402
|
+
const keys = Object.keys(r.attributes).sort();
|
|
403
|
+
const enc = (v) => typeof v === "bigint"
|
|
404
|
+
? `${v}n`
|
|
405
|
+
: v instanceof Uint8Array
|
|
406
|
+
? [...v]
|
|
407
|
+
: Array.isArray(v)
|
|
408
|
+
? v.map((x) => (typeof x === "bigint" ? `${x}n` : x)) // an array attribute is one of these
|
|
409
|
+
: v;
|
|
410
|
+
return JSON.stringify([s.name, s.version ?? "", keys.map((k) => [k, enc(r.attributes[k])])]);
|
|
411
|
+
}
|
|
412
|
+
/** Batches of one resource and scope joined into one, in the order they first came. */
|
|
413
|
+
export function mergeBatches(signals) {
|
|
414
|
+
const t = new Map();
|
|
415
|
+
for (const b of signals.traces) {
|
|
416
|
+
const k = resourceKey(b.resource, b.scope);
|
|
417
|
+
const at = t.get(k) ?? { resource: b.resource, scope: b.scope, spans: [] };
|
|
418
|
+
at.spans.push(...b.spans);
|
|
419
|
+
t.set(k, at);
|
|
420
|
+
}
|
|
421
|
+
const l = new Map();
|
|
422
|
+
for (const b of signals.logs) {
|
|
423
|
+
const k = resourceKey(b.resource, b.scope);
|
|
424
|
+
const at = l.get(k) ?? { resource: b.resource, scope: b.scope, records: [] };
|
|
425
|
+
at.records.push(...b.records);
|
|
426
|
+
l.set(k, at);
|
|
427
|
+
}
|
|
428
|
+
return { traces: [...t.values()], logs: [...l.values()] };
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* The traces with a `cancelled` test span for each test that started and has no span (FR-004): its process died, or exited, before the
|
|
432
|
+
* test ended. Its end is `at`, the time the run saw its process end.
|
|
433
|
+
*/
|
|
434
|
+
export function closeOrphans(signals, at) {
|
|
435
|
+
const ended = new Set(signals.traces.flatMap((b) => b.spans.filter((s) => s.name === TEST_SPAN).map((s) => s.context.spanId)));
|
|
436
|
+
const extra = [];
|
|
437
|
+
for (const b of signals.logs) {
|
|
438
|
+
const spans = [];
|
|
439
|
+
for (const r of b.records) {
|
|
440
|
+
if (r.eventName !== EVENTS.start || r.spanId === undefined || r.traceId === undefined || ended.has(r.spanId))
|
|
441
|
+
continue;
|
|
442
|
+
ended.add(r.spanId);
|
|
443
|
+
const a = r.attributes;
|
|
444
|
+
const risk = attributeText(a, "ifc.test.risk") ?? "not-high";
|
|
445
|
+
const parent = attributeText(a, "ifc.test.parent");
|
|
446
|
+
const characteristic = attributeText(a, "ifc.test.characteristic");
|
|
447
|
+
const parentSpanId = parent === undefined ? undefined : parseSpanId(parent);
|
|
448
|
+
const c = {
|
|
449
|
+
id: attributeText(a, "ifc.test.id") ?? "",
|
|
450
|
+
file: attributeText(a, "test.suite.name") ?? "",
|
|
451
|
+
name: attributeText(a, "test.case.name") ?? "",
|
|
452
|
+
requirements: attributeTexts(a, "ifc.test.requirements"),
|
|
453
|
+
risk: isRisk(risk) ? risk : "not-high",
|
|
454
|
+
...(characteristic === undefined ? {} : { characteristic }),
|
|
455
|
+
start: r.time,
|
|
456
|
+
end: at > r.time ? at : r.time,
|
|
457
|
+
outcome: "cancelled",
|
|
458
|
+
reason: "its process ended before the test did",
|
|
459
|
+
attempt: attributeInteger(a, "ifc.test.attempt") ?? 1,
|
|
460
|
+
};
|
|
461
|
+
spans.push(testCaseSpan(c, {
|
|
462
|
+
traceId: r.traceId,
|
|
463
|
+
spanId: r.spanId,
|
|
464
|
+
...(parentSpanId?.kind === "ok" ? { parentSpanId: parentSpanId.value } : {}),
|
|
465
|
+
}));
|
|
466
|
+
}
|
|
467
|
+
if (spans.length > 0)
|
|
468
|
+
extra.push({ resource: b.resource, scope: b.scope, spans });
|
|
469
|
+
}
|
|
470
|
+
return extra.length === 0 ? signals : mergeBatches({ traces: [...signals.traces, ...extra], logs: signals.logs });
|
|
471
|
+
}
|
package/dist/test.d.ts
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { type Resource } from "@intellectual-frontiers/telemetry/otel";
|
|
2
|
+
import { type Risk, type TestCase, type TestContext } from "./record.js";
|
|
3
|
+
/**
|
|
4
|
+
* What a test's body, or a step's, is given (FR-060): its `name`; `step`, which runs a step as a subtest of `node:test`, awaited, and
|
|
5
|
+
* records it in the test's evidence (a step that fails rejects with its error, so that the test fails with it); and `skip`, which marks
|
|
6
|
+
* the test or the step skipped, with the reason, when its body returns (the body itself returns: `skip` does not stop it).
|
|
7
|
+
*/
|
|
8
|
+
export type StepContext = {
|
|
9
|
+
readonly name: string;
|
|
10
|
+
readonly step: (name: string, fn: StepFn) => Promise<void>;
|
|
11
|
+
readonly skip: (reason?: string) => void;
|
|
12
|
+
};
|
|
13
|
+
/** A step's body, given its own context for the steps within it. */
|
|
14
|
+
export type StepFn = (t: StepContext) => void | Promise<void>;
|
|
15
|
+
/** A test's body; it may take the context `t` for its steps. */
|
|
16
|
+
export type TestFn = (t: StepContext) => void | Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* How long a test may run when it names no `timeout`: five minutes. Bun's own default of 5 seconds ends a test that starts real programs on a slow runner (Windows above all) for the
|
|
19
|
+
* wrong reason; five minutes is above any limit a consumer's runner sets (its own tools pass up to a few minutes), so it never shortens one, and a test that hangs still ends.
|
|
20
|
+
*/
|
|
21
|
+
export declare const DEFAULT_TIMEOUT_MS = 300000;
|
|
22
|
+
/**
|
|
23
|
+
* How to run a test: `skip` it or mark it `todo` (with the reason as a string; neither runs its body), run it alone among the tests of
|
|
24
|
+
* its file (`only`), give it a `timeout` in milliseconds (a test that does not end by then is `cancelled`), its `risk` (`not-high` by
|
|
25
|
+
* default), or the `file` it is recorded under.
|
|
26
|
+
*/
|
|
27
|
+
export type TestOptions = {
|
|
28
|
+
readonly skip?: boolean | string;
|
|
29
|
+
readonly todo?: boolean | string;
|
|
30
|
+
readonly only?: boolean;
|
|
31
|
+
readonly timeout?: number;
|
|
32
|
+
readonly risk?: Risk;
|
|
33
|
+
/** The quality characteristic the test proves, in the words of ISO/IEC 25010, recorded as `ifc.test.characteristic` (0021-ifqs FR-012). */
|
|
34
|
+
readonly characteristic?: string;
|
|
35
|
+
readonly file?: string;
|
|
36
|
+
};
|
|
37
|
+
/** The reason a test is recorded as skipped where another test of its file is marked `only` (FR-060). */
|
|
38
|
+
export declare const ONLY_REASON = "another test of this file is marked only";
|
|
39
|
+
/** A requirement cited by its specification and number: `0005-typescript-runtime FR-013`, or another repository's `erga 241-typescript-first FR-015`. */
|
|
40
|
+
export type Requirement = string;
|
|
41
|
+
/** The requirements each test proves, in the order the tests were registered, for a report. */
|
|
42
|
+
export declare const proofs: {
|
|
43
|
+
readonly test: string;
|
|
44
|
+
readonly requirements: readonly Requirement[];
|
|
45
|
+
}[];
|
|
46
|
+
/** The requirements declared as proven by review rather than a test, with the reason. */
|
|
47
|
+
export declare const reviews: {
|
|
48
|
+
readonly requirement: Requirement;
|
|
49
|
+
readonly reason: string;
|
|
50
|
+
}[];
|
|
51
|
+
/** Every test of this process as it was recorded, in the order they ended (skipped and todo tests as they were registered) (FR-004). */
|
|
52
|
+
export declare const recorded: TestCase[];
|
|
53
|
+
/** What a listener is given for each test that ends: the case, and the resource and context its signals carry. */
|
|
54
|
+
export type TestRecordEvent = {
|
|
55
|
+
readonly test: TestCase;
|
|
56
|
+
readonly resource: Resource;
|
|
57
|
+
readonly context: TestContext;
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* A listener given each test as it ends, beside `recorded` and `IFC_TEST_RECORD`: where an OpenTelemetry pipeline is installed, it hands
|
|
61
|
+
* `testCaseSpan` and `testCaseLog` of the event to its processors (0008-test-evidence FR-004). Returns the function that removes it.
|
|
62
|
+
*/
|
|
63
|
+
export declare function onTestRecord(listener: (e: TestRecordEvent) => void): () => void;
|
|
64
|
+
/** A test registered with the runtime's runner, and recorded (0008-test-evidence FR-001). */
|
|
65
|
+
export declare function test(name: string, fn: TestFn, options?: TestOptions): void;
|
|
66
|
+
/** A test that proves one or more requirements, named for them: `0005-typescript-runtime FR-013: <name>`. */
|
|
67
|
+
export declare function proves(requirements: readonly [Requirement, ...Requirement[]], name: string, fn: TestFn, options?: TestOptions): void;
|
|
68
|
+
/** A requirement proven by review rather than by a test, with the reason; recorded as an `ifc.test.review` log record (0008-test-evidence FR-006). */
|
|
69
|
+
export declare function reviewed(requirement: Requirement, reason: string): void;
|
|
70
|
+
/** Fails unless `value` is truthy; narrows its type. */
|
|
71
|
+
export declare function assert(value: unknown, message?: string): asserts value;
|
|
72
|
+
/** Fails unless the two are deeply and strictly equal (the same types, prototypes and values). */
|
|
73
|
+
export declare function assertEquals<T>(actual: T, expected: T, message?: string): void;
|
|
74
|
+
/** Fails where the two are deeply and strictly equal. */
|
|
75
|
+
export declare function assertNotEquals<T>(actual: T, expected: T, message?: string): void;
|
|
76
|
+
/** Fails unless the two are the same value (`Object.is`). */
|
|
77
|
+
export declare function assertStrictEquals<T>(actual: T, expected: T, message?: string): void;
|
|
78
|
+
/** Fails unless the value is neither `null` nor `undefined`; narrows its type. */
|
|
79
|
+
export declare function assertExists<T>(value: T, message?: string): asserts value is NonNullable<T>;
|
|
80
|
+
/**
|
|
81
|
+
* The value, narrowed to `NonNullable`, or a failure as {@link assertExists} fails where it is `null` or `undefined` (FR-065), so that a
|
|
82
|
+
* test reads `the(map.get(key)).name` where an assertion would need a statement of its own.
|
|
83
|
+
*/
|
|
84
|
+
export declare function the<T>(value: T, message?: string): NonNullable<T>;
|
|
85
|
+
/** Fails unless the value is an instance of the class; narrows its type. */
|
|
86
|
+
export declare function assertInstanceOf<T>(value: unknown, Class: abstract new (...args: never[]) => T, message?: string): asserts value is T;
|
|
87
|
+
/** Fails unless the text includes the part. */
|
|
88
|
+
export declare function assertStringIncludes(actual: string, expected: string, message?: string): void;
|
|
89
|
+
/** Fails unless the two are different values (`Object.is`), as `@std/assert`'s `assertNotStrictEquals` (FR-060). */
|
|
90
|
+
export declare function assertNotStrictEquals<T>(actual: T, expected: T, message?: string): void;
|
|
91
|
+
/**
|
|
92
|
+
* Fails unless the two numbers are within `tolerance` of each other (`1e-7` by default), as `@std/assert`'s `assertAlmostEquals`
|
|
93
|
+
* (FR-060): the same value (`Object.is`, so `NaN` is close to `NaN` and an infinity to itself) passes, else `|expected - actual|` must not
|
|
94
|
+
* exceed the tolerance.
|
|
95
|
+
*/
|
|
96
|
+
export declare function assertAlmostEquals(actual: number, expected: number, tolerance?: number, message?: string): void;
|
|
97
|
+
/** Fails where the text matches the pattern, as `@std/assert`'s `assertNotMatch` (FR-060). */
|
|
98
|
+
export declare function assertNotMatch(actual: string, pattern: RegExp, message?: string): void;
|
|
99
|
+
/** Fails unless the text matches the pattern. */
|
|
100
|
+
export declare function assertMatch(actual: string, pattern: RegExp, message?: string): void;
|
|
101
|
+
/** Fails unless every expected element is deeply equal to some element of the array. */
|
|
102
|
+
export declare function assertArrayIncludes<T>(actual: readonly T[], expected: readonly T[], message?: string): void;
|
|
103
|
+
/** The error a function throws; fails where it throws none, or one not of `Class`, or one whose message does not include `includes`. */
|
|
104
|
+
export declare function assertThrows<E extends Error = Error>(fn: () => unknown, Class?: abstract new (...args: never[]) => E, includes?: string): E;
|
|
105
|
+
/** The error a promise rejects with; fails where it fulfils, or rejects with one not of `Class`, or whose message does not include `includes`. */
|
|
106
|
+
export declare function assertRejects<E extends Error = Error>(fn: () => PromiseLike<unknown>, Class?: abstract new (...args: never[]) => E, includes?: string): Promise<E>;
|
|
107
|
+
/** Fails, with the message. */
|
|
108
|
+
export declare function fail(message: string): never;
|