@intellectual-frontiers/test 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/dist/record.d.ts +222 -0
- package/dist/record.js +471 -0
- package/dist/test.d.ts +108 -0
- package/dist/test.js +588 -0
- package/package.json +46 -4
- package/src/record.ts +612 -0
- package/src/test.ts +713 -0
package/src/record.ts
ADDED
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
// Spec: 0001-test FR-004
|
|
2
|
+
/**
|
|
3
|
+
* The capture half of test evidence (commons 0008-test-evidence FR-001 to FR-004), kept apart from the reports so that every test process
|
|
4
|
+
* loads only this: the model of a test case and of an installation, a test case as its `test` span and log record with the `test.*`
|
|
5
|
+
* conventions and the `ifc.test.*` attributes (`testCaseSpan`, `testCaseLog`), the mark a test leaves as it starts (`testStartLog`), so
|
|
6
|
+
* that one that never ends is closed as `cancelled` (`closeOrphans`), the installation record as a resource (`installationFromEnv`,
|
|
7
|
+
* which refuses a role that names a person, and `isPersonAttribute`), and the OTLP JSON lines a test process appends to
|
|
8
|
+
* `IFC_TEST_RECORD` (`recordLine`, `decodeLines`). `./test-report.ts` reads a run's record and renders it. A test's steps (commons
|
|
9
|
+
* 0005-typescript-runtime FR-060) are its `steps`, named by its span's `ifc.test.steps`, each a `test step` span whose parent is the
|
|
10
|
+
* test's span or the step it is in (`testStepSpan`).
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```ts
|
|
14
|
+
* import { installationFromEnv, installationResource, recordLine, testCaseSpan } from "./record.ts";
|
|
15
|
+
* const { installation } = installationFromEnv({ runtime: "node", runtimeVersion: "26.0.0", os: "linux", arch: "x86_64" }, { CI: "true" });
|
|
16
|
+
* // recordLine(installationResource(installation), { spans: [testCaseSpan(test, context)] }) is one line of IFC_TEST_RECORD
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* @module
|
|
20
|
+
*/
|
|
21
|
+
import { absurd, type DecodeError, formatDecodeError, err, ok, type Result } from "@intellectual-frontiers/core";
|
|
22
|
+
import {
|
|
23
|
+
type AttributeValue,
|
|
24
|
+
type Attributes,
|
|
25
|
+
type LogRecord,
|
|
26
|
+
parseSpanId,
|
|
27
|
+
type Resource,
|
|
28
|
+
type Scope,
|
|
29
|
+
SEVERITY,
|
|
30
|
+
type SpanData,
|
|
31
|
+
type SpanId,
|
|
32
|
+
type TraceId,
|
|
33
|
+
} from "@intellectual-frontiers/telemetry/otel";
|
|
34
|
+
import { decodeLogs, decodeTraces, encodeLogs, encodeTraces, type LogBatch, type TraceBatch } from "@intellectual-frontiers/telemetry/otlp";
|
|
35
|
+
import { type Sha256, sha256 } from "@intellectual-frontiers/crypto/sha256";
|
|
36
|
+
|
|
37
|
+
/** A role (lower-case words joined by `-`, `_`, `.` or `/`), never a person: the rule of `grc`'s `roleOf`, held here so that the record depends on nothing above it (a test of the whole keeps the two the same). */
|
|
38
|
+
const ROLE = /^[a-z][a-z0-9]*(?:[-_./][a-z0-9]+)*$/;
|
|
39
|
+
const roleOf = (text: string): string | undefined => (ROLE.test(text) ? text : undefined);
|
|
40
|
+
|
|
41
|
+
// ---------------------------------------------------------------------------------------------------------------------------- the model
|
|
42
|
+
|
|
43
|
+
/** What became of a test (FR-001). */
|
|
44
|
+
export type Outcome = "pass" | "fail" | "skip" | "todo" | "cancelled";
|
|
45
|
+
|
|
46
|
+
/** Every outcome, in the order reports list them. */
|
|
47
|
+
export const OUTCOMES: readonly Outcome[] = ["pass", "fail", "skip", "todo", "cancelled"];
|
|
48
|
+
|
|
49
|
+
/** A test's risk, after the Computer Software Assurance guidance's high process risk: `high` or `not-high` (FR-001). */
|
|
50
|
+
export type Risk = "high" | "not-high";
|
|
51
|
+
|
|
52
|
+
/** Whether a text is an outcome. */
|
|
53
|
+
export function isOutcome(text: string): text is Outcome {
|
|
54
|
+
return (OUTCOMES as readonly string[]).includes(text);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Whether a text is a risk. */
|
|
58
|
+
export function isRisk(text: string): text is Risk {
|
|
59
|
+
return text === "high" || text === "not-high";
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** What became of a step (0005-typescript-runtime FR-060): it passed, it failed, or its runner did not run it. */
|
|
63
|
+
export type StepOutcome = "pass" | "fail" | "skip";
|
|
64
|
+
|
|
65
|
+
/** Whether a text is a step's outcome. */
|
|
66
|
+
export function isStepOutcome(text: string): text is StepOutcome {
|
|
67
|
+
return text === "pass" || text === "fail" || text === "skip";
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** One step of a test as it ran (0005-typescript-runtime FR-060): its name, times, outcome, a failure's message, and its own steps. */
|
|
71
|
+
export type TestStep = {
|
|
72
|
+
readonly name: string;
|
|
73
|
+
readonly start: bigint;
|
|
74
|
+
readonly end: bigint;
|
|
75
|
+
readonly outcome: StepOutcome;
|
|
76
|
+
readonly message?: string;
|
|
77
|
+
readonly steps?: readonly TestStep[];
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/** One test as it ran once (FR-001): times in nanoseconds since the Unix epoch; text already redacted and cut. */
|
|
81
|
+
export type TestCase = {
|
|
82
|
+
/** The SHA-256 of its file, a NUL and its name. */
|
|
83
|
+
readonly id: string;
|
|
84
|
+
/** Its file, relative to the working folder, with `/`. */
|
|
85
|
+
readonly file: string;
|
|
86
|
+
readonly name: string;
|
|
87
|
+
readonly requirements: readonly string[];
|
|
88
|
+
readonly risk: Risk;
|
|
89
|
+
/** The quality characteristic it proves, in the words of ISO/IEC 25010 (`ifc.test.characteristic`, 0021-ifqs FR-012); absent where it names none. */
|
|
90
|
+
readonly characteristic?: string;
|
|
91
|
+
readonly start: bigint;
|
|
92
|
+
readonly end: bigint;
|
|
93
|
+
readonly outcome: Outcome;
|
|
94
|
+
/** A skip's or a todo's reason, or why it was cancelled. */
|
|
95
|
+
readonly reason?: string;
|
|
96
|
+
readonly message?: string;
|
|
97
|
+
readonly expected?: string;
|
|
98
|
+
readonly actual?: string;
|
|
99
|
+
/** From 1. */
|
|
100
|
+
readonly attempt: number;
|
|
101
|
+
/** Its steps, in the order they ended; absent where it has none (0005-typescript-runtime FR-060). */
|
|
102
|
+
readonly steps?: readonly TestStep[];
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/** The installation record: the resource of a test process or of the run (FR-003). */
|
|
106
|
+
export type Installation = {
|
|
107
|
+
readonly runtime: string;
|
|
108
|
+
readonly runtimeVersion: string;
|
|
109
|
+
readonly os: string;
|
|
110
|
+
readonly arch: string;
|
|
111
|
+
readonly revision?: string;
|
|
112
|
+
readonly lockfileSha256?: string;
|
|
113
|
+
readonly pinsSha256?: string;
|
|
114
|
+
readonly role: string;
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
/** A requirement declared proven by review rather than by a test, with the reason. */
|
|
118
|
+
export type Review = { readonly requirement: string; readonly reason: string };
|
|
119
|
+
|
|
120
|
+
/** What one runner did: the runtime, its exit status, or that it is not installed. */
|
|
121
|
+
export type RunnerOutcome =
|
|
122
|
+
{ readonly kind: "exited"; readonly runtime: string; readonly code: number } | { readonly kind: "missing"; readonly runtime: string };
|
|
123
|
+
|
|
124
|
+
// ---------------------------------------------------------------------------------------------------------------------------- names
|
|
125
|
+
|
|
126
|
+
/** The instrumentation scope of test records. */
|
|
127
|
+
export const TEST_SCOPE: Scope = { name: "ifc-test", version: "1" };
|
|
128
|
+
|
|
129
|
+
/** The name of a test's span. */
|
|
130
|
+
export const TEST_SPAN = "test";
|
|
131
|
+
|
|
132
|
+
/** The name of a step's span, a child of its test's span or of the step it is in (0005-typescript-runtime FR-060). */
|
|
133
|
+
export const STEP_SPAN = "test step";
|
|
134
|
+
|
|
135
|
+
/** The name of a run's span. */
|
|
136
|
+
export const RUN_SPAN = "test run";
|
|
137
|
+
|
|
138
|
+
/** The event names of the log records of a test record. */
|
|
139
|
+
export const EVENTS = {
|
|
140
|
+
case: "ifc.test.case",
|
|
141
|
+
start: "ifc.test.start",
|
|
142
|
+
review: "ifc.test.review",
|
|
143
|
+
runner: "ifc.test.runner",
|
|
144
|
+
warning: "ifc.test.warning",
|
|
145
|
+
} as const;
|
|
146
|
+
|
|
147
|
+
/** The most characters a message, an expected or an actual value keeps (FR-001). */
|
|
148
|
+
export const TEXT_LIMIT = 4096;
|
|
149
|
+
|
|
150
|
+
/** A text cut to `TEXT_LIMIT` characters (without splitting a surrogate pair). */
|
|
151
|
+
export function cut(text: string): string {
|
|
152
|
+
if (text.length <= TEXT_LIMIT) return text;
|
|
153
|
+
const end = /[\uD800-\uDBFF]/.test(text.charAt(TEXT_LIMIT - 1)) ? TEXT_LIMIT - 1 : TEXT_LIMIT;
|
|
154
|
+
return text.slice(0, end);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** A test's id: the SHA-256 of its file, a NUL and its name (FR-001). */
|
|
158
|
+
export function testId(file: string, name: string): Promise<Sha256> {
|
|
159
|
+
return sha256(`${file}\0${name}`);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** A file path as a record names it: `\` as `/`, without a leading `./`. */
|
|
163
|
+
export function recordPath(path: string): string {
|
|
164
|
+
return path.replaceAll("\\", "/").replace(/^(?:\.\/)+/, "");
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// ---------------------------------------------------------------------------------------------------------------------------- installation (FR-003)
|
|
168
|
+
|
|
169
|
+
/** Whether a resource attribute names a person or a machine of one, and is refused (FR-003): `user.*`, `enduser.*`, `host.name`. */
|
|
170
|
+
export function isPersonAttribute(key: string): boolean {
|
|
171
|
+
return key.startsWith("user.") || key.startsWith("enduser.") || key === "host.name" || key === "host.id";
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const ARCH: Readonly<Record<string, string>> = { x86_64: "amd64", x64: "amd64", aarch64: "arm64", arm: "arm32", ia32: "x86", x86: "x86" };
|
|
175
|
+
|
|
176
|
+
/** A machine's architecture as OpenTelemetry's `host.arch` names it (`amd64`, `arm64`, ...). */
|
|
177
|
+
export function hostArch(arch: string): string {
|
|
178
|
+
return ARCH[arch] ?? arch;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const SHA256_TEXT = /^[0-9a-f]{64}$/;
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* The installation record of a process, from what the runtime says of itself and the environment given as a record (the kernel reads no
|
|
185
|
+
* environment): `IFC_TEST_REVISION` or `GITHUB_SHA`, `IFC_TEST_LOCKFILE_SHA256`, `IFC_TEST_PINS_SHA256`, and `IFC_TEST_ROLE` (`ci` where
|
|
186
|
+
* `CI` is set, else `developer`). A role that is not a role (an e-mail address, a name with capitals or spaces) is refused with a
|
|
187
|
+
* warning and the default used, and so is a digest that is not a SHA-256.
|
|
188
|
+
*/
|
|
189
|
+
export function installationFromEnv(
|
|
190
|
+
self: { readonly runtime: string; readonly runtimeVersion: string; readonly os: string; readonly arch: string },
|
|
191
|
+
env: Readonly<Record<string, string | undefined>>,
|
|
192
|
+
): { readonly installation: Installation; readonly warnings: readonly string[] } {
|
|
193
|
+
const warnings: string[] = [];
|
|
194
|
+
const get = (k: string): string | undefined => {
|
|
195
|
+
const v = env[k]?.trim();
|
|
196
|
+
return v === undefined || v === "" ? undefined : v;
|
|
197
|
+
};
|
|
198
|
+
const fallback = get("CI") === undefined || get("CI") === "false" ? "developer" : "ci";
|
|
199
|
+
const given = get("IFC_TEST_ROLE");
|
|
200
|
+
let role = fallback;
|
|
201
|
+
if (given !== undefined) {
|
|
202
|
+
if (roleOf(given) === undefined) warnings.push(`IFC_TEST_ROLE is not a role (lower-case words, never a person); ${fallback} is used`);
|
|
203
|
+
else role = given;
|
|
204
|
+
}
|
|
205
|
+
const digest = (k: string): string | undefined => {
|
|
206
|
+
const v = get(k)?.toLowerCase();
|
|
207
|
+
if (v === undefined) return undefined;
|
|
208
|
+
if (SHA256_TEXT.test(v)) return v;
|
|
209
|
+
warnings.push(`${k} is not a SHA-256; it is left out`);
|
|
210
|
+
return undefined;
|
|
211
|
+
};
|
|
212
|
+
const revision = get("IFC_TEST_REVISION") ?? get("GITHUB_SHA");
|
|
213
|
+
const lockfileSha256 = digest("IFC_TEST_LOCKFILE_SHA256");
|
|
214
|
+
const pinsSha256 = digest("IFC_TEST_PINS_SHA256");
|
|
215
|
+
return {
|
|
216
|
+
installation: {
|
|
217
|
+
runtime: self.runtime,
|
|
218
|
+
runtimeVersion: self.runtimeVersion,
|
|
219
|
+
os: self.os,
|
|
220
|
+
arch: hostArch(self.arch),
|
|
221
|
+
...(revision === undefined ? {} : { revision }),
|
|
222
|
+
...(lockfileSha256 === undefined ? {} : { lockfileSha256 }),
|
|
223
|
+
...(pinsSha256 === undefined ? {} : { pinsSha256 }),
|
|
224
|
+
role,
|
|
225
|
+
},
|
|
226
|
+
warnings,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** The resource of an installation record (FR-003). */
|
|
231
|
+
export function installationResource(i: Installation): Resource {
|
|
232
|
+
return {
|
|
233
|
+
attributes: {
|
|
234
|
+
"service.name": "ifc-test",
|
|
235
|
+
"process.runtime.name": i.runtime,
|
|
236
|
+
"process.runtime.version": i.runtimeVersion,
|
|
237
|
+
"os.type": i.os,
|
|
238
|
+
"host.arch": i.arch,
|
|
239
|
+
...(i.revision === undefined ? {} : { "vcs.ref.head.revision": i.revision }),
|
|
240
|
+
...(i.lockfileSha256 === undefined ? {} : { "ifc.test.lockfile.sha256": i.lockfileSha256 }),
|
|
241
|
+
...(i.pinsSha256 === undefined ? {} : { "ifc.test.pins.sha256": i.pinsSha256 }),
|
|
242
|
+
"ifc.test.role": i.role,
|
|
243
|
+
},
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** An attribute's value where it is a string. */
|
|
248
|
+
export function attributeText(a: Attributes, key: string): string | undefined {
|
|
249
|
+
const v = a[key];
|
|
250
|
+
return typeof v === "string" ? v : undefined;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** An attribute's value where it is an array of strings, else none. */
|
|
254
|
+
export function attributeTexts(a: Attributes, key: string): readonly string[] {
|
|
255
|
+
const v = a[key];
|
|
256
|
+
return Array.isArray(v) && v.every((e): e is string => typeof e === "string") ? v : [];
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** An attribute's value where it is an integer. */
|
|
260
|
+
export function attributeInteger(a: Attributes, key: string): number | undefined {
|
|
261
|
+
const v = a[key];
|
|
262
|
+
if (typeof v === "bigint") return Number(v);
|
|
263
|
+
return typeof v === "number" && Number.isInteger(v) ? v : undefined;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** The installation record a resource holds; unknown fields are `unknown`. */
|
|
267
|
+
export function installationOf(r: Resource): Installation {
|
|
268
|
+
const a = r.attributes;
|
|
269
|
+
const revision = attributeText(a, "vcs.ref.head.revision");
|
|
270
|
+
const lockfileSha256 = attributeText(a, "ifc.test.lockfile.sha256");
|
|
271
|
+
const pinsSha256 = attributeText(a, "ifc.test.pins.sha256");
|
|
272
|
+
return {
|
|
273
|
+
runtime: attributeText(a, "process.runtime.name") ?? "unknown",
|
|
274
|
+
runtimeVersion: attributeText(a, "process.runtime.version") ?? "unknown",
|
|
275
|
+
os: attributeText(a, "os.type") ?? "unknown",
|
|
276
|
+
arch: attributeText(a, "host.arch") ?? "unknown",
|
|
277
|
+
...(revision === undefined ? {} : { revision }),
|
|
278
|
+
...(lockfileSha256 === undefined ? {} : { lockfileSha256 }),
|
|
279
|
+
...(pinsSha256 === undefined ? {} : { pinsSha256 }),
|
|
280
|
+
role: attributeText(a, "ifc.test.role") ?? "unknown",
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// ---------------------------------------------------------------------------------------------------------------------------- capture (FR-001, FR-002, FR-004)
|
|
285
|
+
|
|
286
|
+
/** Where a test's signals belong: the trace, the test's own span id, and the run's span as its parent where there is one. */
|
|
287
|
+
export type TestContext = { readonly traceId: TraceId; readonly spanId: SpanId; readonly parentSpanId?: SpanId };
|
|
288
|
+
|
|
289
|
+
function caseAttributes(c: TestCase): Attributes {
|
|
290
|
+
const status = c.outcome === "pass" ? "pass" : c.outcome === "fail" || c.outcome === "cancelled" ? "fail" : undefined;
|
|
291
|
+
return {
|
|
292
|
+
"test.case.name": c.name,
|
|
293
|
+
...(status === undefined ? {} : { "test.case.result.status": status }),
|
|
294
|
+
"test.suite.name": c.file,
|
|
295
|
+
"code.file.path": c.file,
|
|
296
|
+
"ifc.test.id": c.id,
|
|
297
|
+
"ifc.test.requirements": [...c.requirements],
|
|
298
|
+
"ifc.test.risk": c.risk,
|
|
299
|
+
"ifc.test.outcome": c.outcome,
|
|
300
|
+
"ifc.test.attempt": BigInt(c.attempt),
|
|
301
|
+
...(c.characteristic === undefined ? {} : { "ifc.test.characteristic": c.characteristic }),
|
|
302
|
+
...(c.reason === undefined ? {} : { "ifc.test.reason": c.reason }),
|
|
303
|
+
...(c.message === undefined ? {} : { "ifc.test.message": c.message }),
|
|
304
|
+
...(c.expected === undefined ? {} : { "ifc.test.expected": c.expected }),
|
|
305
|
+
...(c.actual === undefined ? {} : { "ifc.test.actual": c.actual }),
|
|
306
|
+
...(c.steps === undefined || c.steps.length === 0 ? {} : { "ifc.test.steps": stepPaths(c.steps) }),
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** The steps of a test, each by its path (a step within another is `outer > inner`), in their order with each step's own steps first. */
|
|
311
|
+
export function stepPaths(steps: readonly TestStep[], within: readonly string[] = []): string[] {
|
|
312
|
+
return steps.flatMap((s) => [...stepPaths(s.steps ?? [], [...within, s.name]), [...within, s.name].join(" > ")]);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* A step as its `test step` span (0005-typescript-runtime FR-060): a child of `ctx.parentSpanId` (its test's span or the step it is in),
|
|
317
|
+
* with its test's id and file, its name, outcome and a failure's message, and an error status where it failed.
|
|
318
|
+
*/
|
|
319
|
+
export function testStepSpan(step: TestStep, test: Pick<TestCase, "id" | "file" | "name">, ctx: TestContext): SpanData {
|
|
320
|
+
return {
|
|
321
|
+
name: STEP_SPAN,
|
|
322
|
+
context: { traceId: ctx.traceId, spanId: ctx.spanId, traceFlags: 1, traceState: [], isRemote: false },
|
|
323
|
+
...(ctx.parentSpanId === undefined ? {} : { parentSpanId: ctx.parentSpanId }),
|
|
324
|
+
kind: "internal",
|
|
325
|
+
startTime: step.start,
|
|
326
|
+
endTime: step.end,
|
|
327
|
+
attributes: {
|
|
328
|
+
"test.suite.name": test.file,
|
|
329
|
+
"code.file.path": test.file,
|
|
330
|
+
"ifc.test.id": test.id,
|
|
331
|
+
"ifc.test.step.name": step.name,
|
|
332
|
+
"ifc.test.outcome": step.outcome,
|
|
333
|
+
...(step.message === undefined ? {} : { "ifc.test.message": step.message }),
|
|
334
|
+
},
|
|
335
|
+
events: [],
|
|
336
|
+
links: [],
|
|
337
|
+
status:
|
|
338
|
+
step.outcome === "fail"
|
|
339
|
+
? { kind: "error", message: step.message ?? "failed" }
|
|
340
|
+
: step.outcome === "pass"
|
|
341
|
+
? { kind: "ok" }
|
|
342
|
+
: { kind: "unset" },
|
|
343
|
+
droppedAttributesCount: 0,
|
|
344
|
+
droppedEventsCount: 0,
|
|
345
|
+
droppedLinksCount: 0,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** A test as its `test` span (FR-002): the conventions, the `ifc.test.*` attributes, and an error status where it failed or was cancelled. */
|
|
350
|
+
export function testCaseSpan(c: TestCase, ctx: TestContext): SpanData {
|
|
351
|
+
const failed = c.outcome === "fail" || c.outcome === "cancelled";
|
|
352
|
+
return {
|
|
353
|
+
name: TEST_SPAN,
|
|
354
|
+
context: { traceId: ctx.traceId, spanId: ctx.spanId, traceFlags: 1, traceState: [], isRemote: false },
|
|
355
|
+
...(ctx.parentSpanId === undefined ? {} : { parentSpanId: ctx.parentSpanId }),
|
|
356
|
+
kind: "internal",
|
|
357
|
+
startTime: c.start,
|
|
358
|
+
endTime: c.end,
|
|
359
|
+
attributes: caseAttributes(c),
|
|
360
|
+
events: [],
|
|
361
|
+
links: [],
|
|
362
|
+
status: failed
|
|
363
|
+
? { kind: "error", message: c.message ?? c.reason ?? c.outcome }
|
|
364
|
+
: c.outcome === "pass"
|
|
365
|
+
? { kind: "ok" }
|
|
366
|
+
: { kind: "unset" },
|
|
367
|
+
droppedAttributesCount: 0,
|
|
368
|
+
droppedEventsCount: 0,
|
|
369
|
+
droppedLinksCount: 0,
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** A test as its log record (FR-002): the same attributes, the name as its body, at ERROR where it failed or was cancelled. */
|
|
374
|
+
export function testCaseLog(c: TestCase, ctx: TestContext): LogRecord {
|
|
375
|
+
const failed = c.outcome === "fail" || c.outcome === "cancelled";
|
|
376
|
+
return {
|
|
377
|
+
time: c.end,
|
|
378
|
+
observedTime: c.end,
|
|
379
|
+
severityNumber: failed ? SEVERITY.ERROR : SEVERITY.INFO,
|
|
380
|
+
severityText: failed ? "ERROR" : "INFO",
|
|
381
|
+
body: `${c.outcome} ${c.name}`,
|
|
382
|
+
attributes: caseAttributes(c),
|
|
383
|
+
droppedAttributesCount: 0,
|
|
384
|
+
eventName: EVENTS.case,
|
|
385
|
+
traceId: ctx.traceId,
|
|
386
|
+
spanId: ctx.spanId,
|
|
387
|
+
traceFlags: 1,
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** The mark a test leaves as it starts (FR-004): its span id, so that a test whose span never comes is closed as cancelled. */
|
|
392
|
+
export function testStartLog(c: Omit<TestCase, "end" | "outcome">, ctx: TestContext): LogRecord {
|
|
393
|
+
return {
|
|
394
|
+
time: c.start,
|
|
395
|
+
observedTime: c.start,
|
|
396
|
+
severityNumber: SEVERITY.DEBUG,
|
|
397
|
+
severityText: "DEBUG",
|
|
398
|
+
body: `start ${c.name}`,
|
|
399
|
+
attributes: {
|
|
400
|
+
"test.case.name": c.name,
|
|
401
|
+
"test.suite.name": c.file,
|
|
402
|
+
"ifc.test.id": c.id,
|
|
403
|
+
"ifc.test.requirements": [...c.requirements],
|
|
404
|
+
"ifc.test.risk": c.risk,
|
|
405
|
+
"ifc.test.attempt": BigInt(c.attempt),
|
|
406
|
+
...(c.characteristic === undefined ? {} : { "ifc.test.characteristic": c.characteristic }),
|
|
407
|
+
...(ctx.parentSpanId === undefined ? {} : { "ifc.test.parent": ctx.parentSpanId }),
|
|
408
|
+
},
|
|
409
|
+
droppedAttributesCount: 0,
|
|
410
|
+
eventName: EVENTS.start,
|
|
411
|
+
traceId: ctx.traceId,
|
|
412
|
+
spanId: ctx.spanId,
|
|
413
|
+
traceFlags: 1,
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
function plainLog(eventName: string, time: bigint, body: string, attributes: Attributes, warn = false): LogRecord {
|
|
418
|
+
return {
|
|
419
|
+
time,
|
|
420
|
+
observedTime: time,
|
|
421
|
+
severityNumber: warn ? SEVERITY.WARN : SEVERITY.INFO,
|
|
422
|
+
severityText: warn ? "WARN" : "INFO",
|
|
423
|
+
body,
|
|
424
|
+
attributes,
|
|
425
|
+
droppedAttributesCount: 0,
|
|
426
|
+
eventName,
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** A review declared in a test file or a list (`ifc.test.review`). */
|
|
431
|
+
export function reviewLog(r: Review, time: bigint, traceId?: TraceId): LogRecord {
|
|
432
|
+
const base = plainLog(EVENTS.review, time, `reviewed ${r.requirement}`, {
|
|
433
|
+
"ifc.test.requirement": r.requirement,
|
|
434
|
+
"ifc.test.reason": r.reason,
|
|
435
|
+
});
|
|
436
|
+
return traceId === undefined ? base : { ...base, traceId };
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/** A warning of a test process or a run (`ifc.test.warning`), such as a role refused. */
|
|
440
|
+
export function warningLog(message: string, time: bigint): LogRecord {
|
|
441
|
+
return plainLog(EVENTS.warning, time, message, { "ifc.test.warning": message }, true);
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/** What one runner did (`ifc.test.runner`). */
|
|
445
|
+
export function runnerLog(r: RunnerOutcome, time: bigint): LogRecord {
|
|
446
|
+
const attributes: Attributes =
|
|
447
|
+
r.kind === "exited"
|
|
448
|
+
? { "process.runtime.name": r.runtime, "ifc.test.runner.found": true, "process.exit.code": BigInt(r.code) }
|
|
449
|
+
: { "process.runtime.name": r.runtime, "ifc.test.runner.found": false };
|
|
450
|
+
return plainLog(
|
|
451
|
+
EVENTS.runner,
|
|
452
|
+
time,
|
|
453
|
+
r.kind === "exited" ? `${r.runtime} exited with ${r.code}` : `${r.runtime} is not installed`,
|
|
454
|
+
attributes,
|
|
455
|
+
r.kind === "missing" || (r.kind === "exited" && r.code !== 0),
|
|
456
|
+
);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** One line of `IFC_TEST_RECORD`: an OTLP JSON traces or logs export request of one resource, without a line feed (FR-004). */
|
|
460
|
+
export function recordLine(
|
|
461
|
+
resource: Resource,
|
|
462
|
+
signal: { readonly spans: readonly SpanData[] } | { readonly records: readonly LogRecord[] },
|
|
463
|
+
): string {
|
|
464
|
+
return JSON.stringify(
|
|
465
|
+
"spans" in signal
|
|
466
|
+
? encodeTraces([{ resource, scope: TEST_SCOPE, spans: signal.spans }])
|
|
467
|
+
: encodeLogs([{ resource, scope: TEST_SCOPE, records: signal.records }]),
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/** Why the lines of a test record or a run's record could not be read. */
|
|
472
|
+
export type RecordError =
|
|
473
|
+
| { readonly kind: "json"; readonly line: number; readonly reason: string }
|
|
474
|
+
| { readonly kind: "otlp"; readonly line: number; readonly error: DecodeError }
|
|
475
|
+
| { readonly kind: "person"; readonly key: string }
|
|
476
|
+
| { readonly kind: "test"; readonly spanId: string; readonly reason: string }
|
|
477
|
+
| { readonly kind: "no-run"; readonly reason: string };
|
|
478
|
+
|
|
479
|
+
/** A record error as one sentence. */
|
|
480
|
+
export function formatRecordError(e: RecordError): string {
|
|
481
|
+
switch (e.kind) {
|
|
482
|
+
case "json":
|
|
483
|
+
return `line ${e.line} is not JSON: ${e.reason}`;
|
|
484
|
+
case "otlp":
|
|
485
|
+
return `line ${e.line} is not OTLP JSON: ${formatDecodeError(e.error)}`;
|
|
486
|
+
case "person":
|
|
487
|
+
return `the record names a person or a machine of one (${e.key}), which a test record must not`;
|
|
488
|
+
case "test":
|
|
489
|
+
return `the test span ${e.spanId} is not a test record: ${e.reason}`;
|
|
490
|
+
case "no-run":
|
|
491
|
+
return `the record has no run: ${e.reason}`;
|
|
492
|
+
default:
|
|
493
|
+
return absurd(e);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/** The signals of one or more export requests. */
|
|
498
|
+
export type Signals = { readonly traces: readonly TraceBatch[]; readonly logs: readonly LogBatch[] };
|
|
499
|
+
|
|
500
|
+
/** The traces and logs of one OTLP JSON document (an export request, or a run's record holding both), as line `line`. */
|
|
501
|
+
export function decodeRecordDocument(value: unknown, line: number): Result<Signals, RecordError> {
|
|
502
|
+
const t = decodeTraces(value);
|
|
503
|
+
if (t.kind === "err") return err({ kind: "otlp", line, error: t.error });
|
|
504
|
+
const l = decodeLogs(value);
|
|
505
|
+
if (l.kind === "err") return err({ kind: "otlp", line, error: l.error });
|
|
506
|
+
return ok({ traces: t.value, logs: l.value });
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* The signals of `IFC_TEST_RECORD`'s lines, each an OTLP JSON export request (FR-004). A last line cut short (a process killed while it
|
|
511
|
+
* wrote) is left out with a warning; any other line that is not one is an error.
|
|
512
|
+
*/
|
|
513
|
+
export function decodeLines(input: string): Result<Signals & { readonly warnings: readonly string[] }, RecordError> {
|
|
514
|
+
const traces: TraceBatch[] = [];
|
|
515
|
+
const logs: LogBatch[] = [];
|
|
516
|
+
const warnings: string[] = [];
|
|
517
|
+
const lines = input.split("\n");
|
|
518
|
+
for (const [i, raw] of lines.entries()) {
|
|
519
|
+
const line = raw.trim();
|
|
520
|
+
if (line === "") continue;
|
|
521
|
+
let value: unknown;
|
|
522
|
+
try {
|
|
523
|
+
value = JSON.parse(line);
|
|
524
|
+
} catch (e) {
|
|
525
|
+
if (i === lines.length - 1) {
|
|
526
|
+
warnings.push(`the last line of a test record was cut short and is left out`);
|
|
527
|
+
continue;
|
|
528
|
+
}
|
|
529
|
+
return err({ kind: "json", line: i + 1, reason: e instanceof Error ? e.message : String(e) });
|
|
530
|
+
}
|
|
531
|
+
const d = decodeRecordDocument(value, i + 1);
|
|
532
|
+
if (d.kind === "err") return d;
|
|
533
|
+
traces.push(...d.value.traces);
|
|
534
|
+
logs.push(...d.value.logs);
|
|
535
|
+
}
|
|
536
|
+
return ok({ traces, logs, warnings });
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
function resourceKey(r: Resource, s: Scope): string {
|
|
540
|
+
const keys = Object.keys(r.attributes).sort();
|
|
541
|
+
const enc = (v: AttributeValue | undefined): unknown =>
|
|
542
|
+
typeof v === "bigint"
|
|
543
|
+
? `${v}n`
|
|
544
|
+
: v instanceof Uint8Array
|
|
545
|
+
? [...v]
|
|
546
|
+
: Array.isArray(v)
|
|
547
|
+
? (v as readonly (string | boolean | bigint | number)[]).map((x): unknown => (typeof x === "bigint" ? `${x}n` : x)) // an array attribute is one of these
|
|
548
|
+
: v;
|
|
549
|
+
return JSON.stringify([s.name, s.version ?? "", keys.map((k) => [k, enc(r.attributes[k])])]);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/** Batches of one resource and scope joined into one, in the order they first came. */
|
|
553
|
+
export function mergeBatches(signals: Signals): Signals {
|
|
554
|
+
const t = new Map<string, { resource: Resource; scope: Scope; spans: SpanData[] }>();
|
|
555
|
+
for (const b of signals.traces) {
|
|
556
|
+
const k = resourceKey(b.resource, b.scope);
|
|
557
|
+
const at = t.get(k) ?? { resource: b.resource, scope: b.scope, spans: [] };
|
|
558
|
+
at.spans.push(...b.spans);
|
|
559
|
+
t.set(k, at);
|
|
560
|
+
}
|
|
561
|
+
const l = new Map<string, { resource: Resource; scope: Scope; records: LogRecord[] }>();
|
|
562
|
+
for (const b of signals.logs) {
|
|
563
|
+
const k = resourceKey(b.resource, b.scope);
|
|
564
|
+
const at = l.get(k) ?? { resource: b.resource, scope: b.scope, records: [] };
|
|
565
|
+
at.records.push(...b.records);
|
|
566
|
+
l.set(k, at);
|
|
567
|
+
}
|
|
568
|
+
return { traces: [...t.values()], logs: [...l.values()] };
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* The traces with a `cancelled` test span for each test that started and has no span (FR-004): its process died, or exited, before the
|
|
573
|
+
* test ended. Its end is `at`, the time the run saw its process end.
|
|
574
|
+
*/
|
|
575
|
+
export function closeOrphans(signals: Signals, at: bigint): Signals {
|
|
576
|
+
const ended = new Set(signals.traces.flatMap((b) => b.spans.filter((s) => s.name === TEST_SPAN).map((s) => s.context.spanId)));
|
|
577
|
+
const extra: TraceBatch[] = [];
|
|
578
|
+
for (const b of signals.logs) {
|
|
579
|
+
const spans: SpanData[] = [];
|
|
580
|
+
for (const r of b.records) {
|
|
581
|
+
if (r.eventName !== EVENTS.start || r.spanId === undefined || r.traceId === undefined || ended.has(r.spanId)) continue;
|
|
582
|
+
ended.add(r.spanId);
|
|
583
|
+
const a = r.attributes;
|
|
584
|
+
const risk = attributeText(a, "ifc.test.risk") ?? "not-high";
|
|
585
|
+
const parent = attributeText(a, "ifc.test.parent");
|
|
586
|
+
const characteristic = attributeText(a, "ifc.test.characteristic");
|
|
587
|
+
const parentSpanId = parent === undefined ? undefined : parseSpanId(parent);
|
|
588
|
+
const c: TestCase = {
|
|
589
|
+
id: attributeText(a, "ifc.test.id") ?? "",
|
|
590
|
+
file: attributeText(a, "test.suite.name") ?? "",
|
|
591
|
+
name: attributeText(a, "test.case.name") ?? "",
|
|
592
|
+
requirements: attributeTexts(a, "ifc.test.requirements"),
|
|
593
|
+
risk: isRisk(risk) ? risk : "not-high",
|
|
594
|
+
...(characteristic === undefined ? {} : { characteristic }),
|
|
595
|
+
start: r.time,
|
|
596
|
+
end: at > r.time ? at : r.time,
|
|
597
|
+
outcome: "cancelled",
|
|
598
|
+
reason: "its process ended before the test did",
|
|
599
|
+
attempt: attributeInteger(a, "ifc.test.attempt") ?? 1,
|
|
600
|
+
};
|
|
601
|
+
spans.push(
|
|
602
|
+
testCaseSpan(c, {
|
|
603
|
+
traceId: r.traceId,
|
|
604
|
+
spanId: r.spanId,
|
|
605
|
+
...(parentSpanId?.kind === "ok" ? { parentSpanId: parentSpanId.value } : {}),
|
|
606
|
+
}),
|
|
607
|
+
);
|
|
608
|
+
}
|
|
609
|
+
if (spans.length > 0) extra.push({ resource: b.resource, scope: b.scope, spans });
|
|
610
|
+
}
|
|
611
|
+
return extra.length === 0 ? signals : mergeBatches({ traces: [...signals.traces, ...extra], logs: signals.logs });
|
|
612
|
+
}
|