@intellectual-frontiers/test 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/dist/record.d.ts +222 -0
- package/dist/record.js +471 -0
- package/dist/test.d.ts +108 -0
- package/dist/test.js +588 -0
- package/package.json +46 -4
- package/src/record.ts +612 -0
- package/src/test.ts +713 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Intellectual Frontiers
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @intellectual-frontiers/test
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Tests that run unchanged under bun test and node --test, and are recorded as evidence by their own code: registration on node:test, the assertions under short plain names, proves (which names the requirements a test proves), only, steps, and the record of every test as an OpenTelemetry span and log record on the installation record.
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
bun add @intellectual-frontiers/test # or: npm install @intellectual-frontiers/test
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Sources and specifications: https://github.com/intellectual-frontiers/.github/tree/main/commons/test
|
package/dist/record.d.ts
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The capture half of test evidence (commons 0008-test-evidence FR-001 to FR-004), kept apart from the reports so that every test process
|
|
3
|
+
* loads only this: the model of a test case and of an installation, a test case as its `test` span and log record with the `test.*`
|
|
4
|
+
* conventions and the `ifc.test.*` attributes (`testCaseSpan`, `testCaseLog`), the mark a test leaves as it starts (`testStartLog`), so
|
|
5
|
+
* that one that never ends is closed as `cancelled` (`closeOrphans`), the installation record as a resource (`installationFromEnv`,
|
|
6
|
+
* which refuses a role that names a person, and `isPersonAttribute`), and the OTLP JSON lines a test process appends to
|
|
7
|
+
* `IFC_TEST_RECORD` (`recordLine`, `decodeLines`). `./test-report.ts` reads a run's record and renders it. A test's steps (commons
|
|
8
|
+
* 0005-typescript-runtime FR-060) are its `steps`, named by its span's `ifc.test.steps`, each a `test step` span whose parent is the
|
|
9
|
+
* test's span or the step it is in (`testStepSpan`).
|
|
10
|
+
*
|
|
11
|
+
* @example
|
|
12
|
+
* ```ts
|
|
13
|
+
* import { installationFromEnv, installationResource, recordLine, testCaseSpan } from "./record.js";
|
|
14
|
+
* const { installation } = installationFromEnv({ runtime: "node", runtimeVersion: "26.0.0", os: "linux", arch: "x86_64" }, { CI: "true" });
|
|
15
|
+
* // recordLine(installationResource(installation), { spans: [testCaseSpan(test, context)] }) is one line of IFC_TEST_RECORD
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* @module
|
|
19
|
+
*/
|
|
20
|
+
import { type DecodeError, type Result } from "@intellectual-frontiers/core";
|
|
21
|
+
import { type Attributes, type LogRecord, type Resource, type Scope, type SpanData, type SpanId, type TraceId } from "@intellectual-frontiers/telemetry/otel";
|
|
22
|
+
import { type LogBatch, type TraceBatch } from "@intellectual-frontiers/telemetry/otlp";
|
|
23
|
+
import { type Sha256 } from "@intellectual-frontiers/crypto/sha256";
|
|
24
|
+
/** What became of a test (FR-001). */
|
|
25
|
+
export type Outcome = "pass" | "fail" | "skip" | "todo" | "cancelled";
|
|
26
|
+
/** Every outcome, in the order reports list them. */
|
|
27
|
+
export declare const OUTCOMES: readonly Outcome[];
|
|
28
|
+
/** A test's risk, after the Computer Software Assurance guidance's high process risk: `high` or `not-high` (FR-001). */
|
|
29
|
+
export type Risk = "high" | "not-high";
|
|
30
|
+
/** Whether a text is an outcome. */
|
|
31
|
+
export declare function isOutcome(text: string): text is Outcome;
|
|
32
|
+
/** Whether a text is a risk. */
|
|
33
|
+
export declare function isRisk(text: string): text is Risk;
|
|
34
|
+
/** What became of a step (0005-typescript-runtime FR-060): it passed, it failed, or its runner did not run it. */
|
|
35
|
+
export type StepOutcome = "pass" | "fail" | "skip";
|
|
36
|
+
/** Whether a text is a step's outcome. */
|
|
37
|
+
export declare function isStepOutcome(text: string): text is StepOutcome;
|
|
38
|
+
/** One step of a test as it ran (0005-typescript-runtime FR-060): its name, times, outcome, a failure's message, and its own steps. */
|
|
39
|
+
export type TestStep = {
|
|
40
|
+
readonly name: string;
|
|
41
|
+
readonly start: bigint;
|
|
42
|
+
readonly end: bigint;
|
|
43
|
+
readonly outcome: StepOutcome;
|
|
44
|
+
readonly message?: string;
|
|
45
|
+
readonly steps?: readonly TestStep[];
|
|
46
|
+
};
|
|
47
|
+
/** One test as it ran once (FR-001): times in nanoseconds since the Unix epoch; text already redacted and cut. */
|
|
48
|
+
export type TestCase = {
|
|
49
|
+
/** The SHA-256 of its file, a NUL and its name. */
|
|
50
|
+
readonly id: string;
|
|
51
|
+
/** Its file, relative to the working folder, with `/`. */
|
|
52
|
+
readonly file: string;
|
|
53
|
+
readonly name: string;
|
|
54
|
+
readonly requirements: readonly string[];
|
|
55
|
+
readonly risk: Risk;
|
|
56
|
+
/** The quality characteristic it proves, in the words of ISO/IEC 25010 (`ifc.test.characteristic`, 0021-ifqs FR-012); absent where it names none. */
|
|
57
|
+
readonly characteristic?: string;
|
|
58
|
+
readonly start: bigint;
|
|
59
|
+
readonly end: bigint;
|
|
60
|
+
readonly outcome: Outcome;
|
|
61
|
+
/** A skip's or a todo's reason, or why it was cancelled. */
|
|
62
|
+
readonly reason?: string;
|
|
63
|
+
readonly message?: string;
|
|
64
|
+
readonly expected?: string;
|
|
65
|
+
readonly actual?: string;
|
|
66
|
+
/** From 1. */
|
|
67
|
+
readonly attempt: number;
|
|
68
|
+
/** Its steps, in the order they ended; absent where it has none (0005-typescript-runtime FR-060). */
|
|
69
|
+
readonly steps?: readonly TestStep[];
|
|
70
|
+
};
|
|
71
|
+
/** The installation record: the resource of a test process or of the run (FR-003). */
|
|
72
|
+
export type Installation = {
|
|
73
|
+
readonly runtime: string;
|
|
74
|
+
readonly runtimeVersion: string;
|
|
75
|
+
readonly os: string;
|
|
76
|
+
readonly arch: string;
|
|
77
|
+
readonly revision?: string;
|
|
78
|
+
readonly lockfileSha256?: string;
|
|
79
|
+
readonly pinsSha256?: string;
|
|
80
|
+
readonly role: string;
|
|
81
|
+
};
|
|
82
|
+
/** A requirement declared proven by review rather than by a test, with the reason. */
|
|
83
|
+
export type Review = {
|
|
84
|
+
readonly requirement: string;
|
|
85
|
+
readonly reason: string;
|
|
86
|
+
};
|
|
87
|
+
/** What one runner did: the runtime, its exit status, or that it is not installed. */
|
|
88
|
+
export type RunnerOutcome = {
|
|
89
|
+
readonly kind: "exited";
|
|
90
|
+
readonly runtime: string;
|
|
91
|
+
readonly code: number;
|
|
92
|
+
} | {
|
|
93
|
+
readonly kind: "missing";
|
|
94
|
+
readonly runtime: string;
|
|
95
|
+
};
|
|
96
|
+
/** The instrumentation scope of test records. */
|
|
97
|
+
export declare const TEST_SCOPE: Scope;
|
|
98
|
+
/** The name of a test's span. */
|
|
99
|
+
export declare const TEST_SPAN = "test";
|
|
100
|
+
/** The name of a step's span, a child of its test's span or of the step it is in (0005-typescript-runtime FR-060). */
|
|
101
|
+
export declare const STEP_SPAN = "test step";
|
|
102
|
+
/** The name of a run's span. */
|
|
103
|
+
export declare const RUN_SPAN = "test run";
|
|
104
|
+
/** The event names of the log records of a test record. */
|
|
105
|
+
export declare const EVENTS: {
|
|
106
|
+
readonly case: "ifc.test.case";
|
|
107
|
+
readonly start: "ifc.test.start";
|
|
108
|
+
readonly review: "ifc.test.review";
|
|
109
|
+
readonly runner: "ifc.test.runner";
|
|
110
|
+
readonly warning: "ifc.test.warning";
|
|
111
|
+
};
|
|
112
|
+
/** The most characters a message, an expected or an actual value keeps (FR-001). */
|
|
113
|
+
export declare const TEXT_LIMIT = 4096;
|
|
114
|
+
/** A text cut to `TEXT_LIMIT` characters (without splitting a surrogate pair). */
|
|
115
|
+
export declare function cut(text: string): string;
|
|
116
|
+
/** A test's id: the SHA-256 of its file, a NUL and its name (FR-001). */
|
|
117
|
+
export declare function testId(file: string, name: string): Promise<Sha256>;
|
|
118
|
+
/** A file path as a record names it: `\` as `/`, without a leading `./`. */
|
|
119
|
+
export declare function recordPath(path: string): string;
|
|
120
|
+
/** Whether a resource attribute names a person or a machine of one, and is refused (FR-003): `user.*`, `enduser.*`, `host.name`. */
|
|
121
|
+
export declare function isPersonAttribute(key: string): boolean;
|
|
122
|
+
/** A machine's architecture as OpenTelemetry's `host.arch` names it (`amd64`, `arm64`, ...). */
|
|
123
|
+
export declare function hostArch(arch: string): string;
|
|
124
|
+
/**
|
|
125
|
+
* The installation record of a process, from what the runtime says of itself and the environment given as a record (the kernel reads no
|
|
126
|
+
* environment): `IFC_TEST_REVISION` or `GITHUB_SHA`, `IFC_TEST_LOCKFILE_SHA256`, `IFC_TEST_PINS_SHA256`, and `IFC_TEST_ROLE` (`ci` where
|
|
127
|
+
* `CI` is set, else `developer`). A role that is not a role (an e-mail address, a name with capitals or spaces) is refused with a
|
|
128
|
+
* warning and the default used, and so is a digest that is not a SHA-256.
|
|
129
|
+
*/
|
|
130
|
+
export declare function installationFromEnv(self: {
|
|
131
|
+
readonly runtime: string;
|
|
132
|
+
readonly runtimeVersion: string;
|
|
133
|
+
readonly os: string;
|
|
134
|
+
readonly arch: string;
|
|
135
|
+
}, env: Readonly<Record<string, string | undefined>>): {
|
|
136
|
+
readonly installation: Installation;
|
|
137
|
+
readonly warnings: readonly string[];
|
|
138
|
+
};
|
|
139
|
+
/** The resource of an installation record (FR-003). */
|
|
140
|
+
export declare function installationResource(i: Installation): Resource;
|
|
141
|
+
/** An attribute's value where it is a string. */
|
|
142
|
+
export declare function attributeText(a: Attributes, key: string): string | undefined;
|
|
143
|
+
/** An attribute's value where it is an array of strings, else none. */
|
|
144
|
+
export declare function attributeTexts(a: Attributes, key: string): readonly string[];
|
|
145
|
+
/** An attribute's value where it is an integer. */
|
|
146
|
+
export declare function attributeInteger(a: Attributes, key: string): number | undefined;
|
|
147
|
+
/** The installation record a resource holds; unknown fields are `unknown`. */
|
|
148
|
+
export declare function installationOf(r: Resource): Installation;
|
|
149
|
+
/** Where a test's signals belong: the trace, the test's own span id, and the run's span as its parent where there is one. */
|
|
150
|
+
export type TestContext = {
|
|
151
|
+
readonly traceId: TraceId;
|
|
152
|
+
readonly spanId: SpanId;
|
|
153
|
+
readonly parentSpanId?: SpanId;
|
|
154
|
+
};
|
|
155
|
+
/** The steps of a test, each by its path (a step within another is `outer > inner`), in their order with each step's own steps first. */
|
|
156
|
+
export declare function stepPaths(steps: readonly TestStep[], within?: readonly string[]): string[];
|
|
157
|
+
/**
|
|
158
|
+
* A step as its `test step` span (0005-typescript-runtime FR-060): a child of `ctx.parentSpanId` (its test's span or the step it is in),
|
|
159
|
+
* with its test's id and file, its name, outcome and a failure's message, and an error status where it failed.
|
|
160
|
+
*/
|
|
161
|
+
export declare function testStepSpan(step: TestStep, test: Pick<TestCase, "id" | "file" | "name">, ctx: TestContext): SpanData;
|
|
162
|
+
/** A test as its `test` span (FR-002): the conventions, the `ifc.test.*` attributes, and an error status where it failed or was cancelled. */
|
|
163
|
+
export declare function testCaseSpan(c: TestCase, ctx: TestContext): SpanData;
|
|
164
|
+
/** A test as its log record (FR-002): the same attributes, the name as its body, at ERROR where it failed or was cancelled. */
|
|
165
|
+
export declare function testCaseLog(c: TestCase, ctx: TestContext): LogRecord;
|
|
166
|
+
/** The mark a test leaves as it starts (FR-004): its span id, so that a test whose span never comes is closed as cancelled. */
|
|
167
|
+
export declare function testStartLog(c: Omit<TestCase, "end" | "outcome">, ctx: TestContext): LogRecord;
|
|
168
|
+
/** A review declared in a test file or a list (`ifc.test.review`). */
|
|
169
|
+
export declare function reviewLog(r: Review, time: bigint, traceId?: TraceId): LogRecord;
|
|
170
|
+
/** A warning of a test process or a run (`ifc.test.warning`), such as a role refused. */
|
|
171
|
+
export declare function warningLog(message: string, time: bigint): LogRecord;
|
|
172
|
+
/** What one runner did (`ifc.test.runner`). */
|
|
173
|
+
export declare function runnerLog(r: RunnerOutcome, time: bigint): LogRecord;
|
|
174
|
+
/** One line of `IFC_TEST_RECORD`: an OTLP JSON traces or logs export request of one resource, without a line feed (FR-004). */
|
|
175
|
+
export declare function recordLine(resource: Resource, signal: {
|
|
176
|
+
readonly spans: readonly SpanData[];
|
|
177
|
+
} | {
|
|
178
|
+
readonly records: readonly LogRecord[];
|
|
179
|
+
}): string;
|
|
180
|
+
/** Why the lines of a test record or a run's record could not be read. */
|
|
181
|
+
export type RecordError = {
|
|
182
|
+
readonly kind: "json";
|
|
183
|
+
readonly line: number;
|
|
184
|
+
readonly reason: string;
|
|
185
|
+
} | {
|
|
186
|
+
readonly kind: "otlp";
|
|
187
|
+
readonly line: number;
|
|
188
|
+
readonly error: DecodeError;
|
|
189
|
+
} | {
|
|
190
|
+
readonly kind: "person";
|
|
191
|
+
readonly key: string;
|
|
192
|
+
} | {
|
|
193
|
+
readonly kind: "test";
|
|
194
|
+
readonly spanId: string;
|
|
195
|
+
readonly reason: string;
|
|
196
|
+
} | {
|
|
197
|
+
readonly kind: "no-run";
|
|
198
|
+
readonly reason: string;
|
|
199
|
+
};
|
|
200
|
+
/** A record error as one sentence. */
|
|
201
|
+
export declare function formatRecordError(e: RecordError): string;
|
|
202
|
+
/** The signals of one or more export requests. */
|
|
203
|
+
export type Signals = {
|
|
204
|
+
readonly traces: readonly TraceBatch[];
|
|
205
|
+
readonly logs: readonly LogBatch[];
|
|
206
|
+
};
|
|
207
|
+
/** The traces and logs of one OTLP JSON document (an export request, or a run's record holding both), as line `line`. */
|
|
208
|
+
export declare function decodeRecordDocument(value: unknown, line: number): Result<Signals, RecordError>;
|
|
209
|
+
/**
|
|
210
|
+
* The signals of `IFC_TEST_RECORD`'s lines, each an OTLP JSON export request (FR-004). A last line cut short (a process killed while it
|
|
211
|
+
* wrote) is left out with a warning; any other line that is not one is an error.
|
|
212
|
+
*/
|
|
213
|
+
export declare function decodeLines(input: string): Result<Signals & {
|
|
214
|
+
readonly warnings: readonly string[];
|
|
215
|
+
}, RecordError>;
|
|
216
|
+
/** Batches of one resource and scope joined into one, in the order they first came. */
|
|
217
|
+
export declare function mergeBatches(signals: Signals): Signals;
|
|
218
|
+
/**
|
|
219
|
+
* The traces with a `cancelled` test span for each test that started and has no span (FR-004): its process died, or exited, before the
|
|
220
|
+
* test ended. Its end is `at`, the time the run saw its process end.
|
|
221
|
+
*/
|
|
222
|
+
export declare function closeOrphans(signals: Signals, at: bigint): Signals;
|