@sdxc/spec 0.0.0-pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +21 -0
- package/README.md +924 -0
- package/dist/ast.d.ts +193 -0
- package/dist/ast.js +9 -0
- package/dist/builtins.d.ts +29 -0
- package/dist/builtins.js +66 -0
- package/dist/cli.d.ts +21 -0
- package/dist/cli.js +297 -0
- package/dist/diagnostics.d.ts +47 -0
- package/dist/diagnostics.js +8 -0
- package/dist/errors.d.ts +131 -0
- package/dist/errors.js +159 -0
- package/dist/executor.d.ts +66 -0
- package/dist/executor.js +320 -0
- package/dist/expectation.d.ts +61 -0
- package/dist/expectation.js +222 -0
- package/dist/index.d.ts +51 -0
- package/dist/index.js +36 -0
- package/dist/lexer.d.ts +22 -0
- package/dist/lexer.js +284 -0
- package/dist/loader.d.ts +21 -0
- package/dist/loader.js +81 -0
- package/dist/parser.d.ts +24 -0
- package/dist/parser.js +502 -0
- package/dist/permissions.d.ts +139 -0
- package/dist/permissions.js +325 -0
- package/dist/plugin.d.ts +90 -0
- package/dist/plugin.js +9 -0
- package/dist/plugins/browser.d.ts +24 -0
- package/dist/plugins/browser.js +896 -0
- package/dist/plugins/cli.d.ts +17 -0
- package/dist/plugins/cli.js +134 -0
- package/dist/plugins/db-e2e-probe.d.ts +14 -0
- package/dist/plugins/db-e2e-probe.js +112 -0
- package/dist/plugins/db.d.ts +19 -0
- package/dist/plugins/db.js +199 -0
- package/dist/plugins/demo.d.ts +17 -0
- package/dist/plugins/demo.js +70 -0
- package/dist/plugins/env.d.ts +18 -0
- package/dist/plugins/env.js +87 -0
- package/dist/plugins/fs.d.ts +16 -0
- package/dist/plugins/fs.js +415 -0
- package/dist/plugins/http.d.ts +19 -0
- package/dist/plugins/http.js +505 -0
- package/dist/plugins/jwt.d.ts +17 -0
- package/dist/plugins/jwt.js +342 -0
- package/dist/plugins/sample.d.ts +27 -0
- package/dist/plugins/sample.js +400 -0
- package/dist/plugins/url.d.ts +18 -0
- package/dist/plugins/url.js +126 -0
- package/dist/project-config.d.ts +163 -0
- package/dist/project-config.js +497 -0
- package/dist/registry.d.ts +56 -0
- package/dist/registry.js +110 -0
- package/dist/reporter.d.ts +30 -0
- package/dist/reporter.js +237 -0
- package/dist/run.d.ts +74 -0
- package/dist/run.js +179 -0
- package/dist/runner.d.ts +52 -0
- package/dist/runner.js +38 -0
- package/dist/source.d.ts +37 -0
- package/dist/source.js +31 -0
- package/dist/sources.d.ts +45 -0
- package/dist/sources.js +54 -0
- package/dist/tokens.d.ts +34 -0
- package/dist/tokens.js +25 -0
- package/dist/transport-stdio.d.ts +34 -0
- package/dist/transport-stdio.js +400 -0
- package/dist/values.d.ts +48 -0
- package/dist/values.js +52 -0
- package/dist/workers.d.ts +40 -0
- package/dist/workers.js +26 -0
- package/dist/workspace-none.d.ts +23 -0
- package/dist/workspace-none.js +33 -0
- package/dist/workspace.d.ts +47 -0
- package/dist/workspace.js +116 -0
- package/package.json +28 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Name resolution for the whole suite: which callable a dotted target means,
|
|
3
|
+
* given the plugins that are connected, the definitions that are loaded, and
|
|
4
|
+
* the namespaces a file imported with `use`. Ambiguity is always an error —
|
|
5
|
+
* the runtime never guesses.
|
|
6
|
+
*
|
|
7
|
+
* @author [Sergio Xalambrí](https://sergiodxa.com)
|
|
8
|
+
* @copyright Sergio Xalambrí 2026
|
|
9
|
+
*/
|
|
10
|
+
import type { Result } from "@sdxc/result";
|
|
11
|
+
import type { CommandNode, FixtureNode } from "./ast.js";
|
|
12
|
+
import type { Plugin, ToolDescriptor } from "./plugin.js";
|
|
13
|
+
import type { LoadedSuite } from "./sources.js";
|
|
14
|
+
import { ResolutionError } from "./errors.js";
|
|
15
|
+
/** What a call target resolved to: a plugin tool or a suite command. */
|
|
16
|
+
export type ResolvedCallable = {
|
|
17
|
+
kind: "tool";
|
|
18
|
+
/** The plugin owning the tool. */
|
|
19
|
+
plugin: Plugin;
|
|
20
|
+
/** The resolved tool's descriptor. */
|
|
21
|
+
descriptor: ToolDescriptor;
|
|
22
|
+
/** The tool's namespace, for diagnostics (`fs.write`). */
|
|
23
|
+
namespace: string;
|
|
24
|
+
} | {
|
|
25
|
+
kind: "command";
|
|
26
|
+
/** The resolved command definition. */
|
|
27
|
+
command: CommandNode;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* The suite's resolution table, built once after loading and consulted for
|
|
31
|
+
* every call. A dotted target resolves inside its namespace; a bare target
|
|
32
|
+
* resolves among suite commands and imported namespaces, erring on ambiguity.
|
|
33
|
+
*/
|
|
34
|
+
export interface Registry {
|
|
35
|
+
/**
|
|
36
|
+
* Resolve a call target as written in some file.
|
|
37
|
+
*
|
|
38
|
+
* @param target - The dotted target text, e.g. `"login"` or `"http.post"`.
|
|
39
|
+
* @param uses - The namespaces the calling file imported, in order.
|
|
40
|
+
*/
|
|
41
|
+
resolveCallable(target: string, uses: readonly string[]): Result<ResolvedCallable, ResolutionError>;
|
|
42
|
+
/** Resolve `fixture NAME` to its definition. */
|
|
43
|
+
resolveFixture(name: string): Result<FixtureNode, ResolutionError>;
|
|
44
|
+
/** Whether a bare name could resolve to anything callable (for `expect`). */
|
|
45
|
+
isCallable(target: string, uses: readonly string[]): boolean;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Build the suite's resolution table from the connected plugins and loaded
|
|
49
|
+
* definitions. Each plugin's descriptors are read once, since a plugin's tool
|
|
50
|
+
* set is stable for its lifetime, making every later resolution a map lookup.
|
|
51
|
+
*
|
|
52
|
+
* @param plugins - The connected plugins, one namespace each.
|
|
53
|
+
* @param suite - The loaded suite whose commands and fixtures resolve here.
|
|
54
|
+
* @returns The registry the executor consults for every call.
|
|
55
|
+
*/
|
|
56
|
+
export declare function createRegistry(plugins: Plugin[], suite: LoadedSuite): Registry;
|
package/dist/registry.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Name resolution for the whole suite: which callable a dotted target means,
|
|
3
|
+
* given the plugins that are connected, the definitions that are loaded, and
|
|
4
|
+
* the namespaces a file imported with `use`. Ambiguity is always an error —
|
|
5
|
+
* the runtime never guesses.
|
|
6
|
+
*
|
|
7
|
+
* @author [Sergio Xalambrí](https://sergiodxa.com)
|
|
8
|
+
* @copyright Sergio Xalambrí 2026
|
|
9
|
+
*/
|
|
10
|
+
import { failure, isSuccess, success } from "@sdxc/result";
|
|
11
|
+
import { ResolutionError } from "./errors.js";
|
|
12
|
+
/**
|
|
13
|
+
* Build the suite's resolution table from the connected plugins and loaded
|
|
14
|
+
* definitions. Each plugin's descriptors are read once, since a plugin's tool
|
|
15
|
+
* set is stable for its lifetime, making every later resolution a map lookup.
|
|
16
|
+
*
|
|
17
|
+
* @param plugins - The connected plugins, one namespace each.
|
|
18
|
+
* @param suite - The loaded suite whose commands and fixtures resolve here.
|
|
19
|
+
* @returns The registry the executor consults for every call.
|
|
20
|
+
*/
|
|
21
|
+
export function createRegistry(plugins, suite) {
|
|
22
|
+
let namespaces = new Map();
|
|
23
|
+
for (let plugin of plugins) {
|
|
24
|
+
let tools = new Map();
|
|
25
|
+
for (let descriptor of plugin.describe())
|
|
26
|
+
tools.set(descriptor.name, descriptor);
|
|
27
|
+
namespaces.set(plugin.namespace, { plugin, tools });
|
|
28
|
+
}
|
|
29
|
+
/** Resolve a `ns.tool` target inside its namespace, never elsewhere. */
|
|
30
|
+
function resolveQualified(target, namespace, tool) {
|
|
31
|
+
let entry = namespaces.get(namespace);
|
|
32
|
+
if (!entry) {
|
|
33
|
+
return failure(new ResolutionError("unknown-name", `Unknown name "${target}": no plugin provides the namespace "${namespace}".`));
|
|
34
|
+
}
|
|
35
|
+
let descriptor = entry.tools.get(tool);
|
|
36
|
+
if (!descriptor) {
|
|
37
|
+
let available = [...entry.tools.keys()].map((name) => `${namespace}.${name}`);
|
|
38
|
+
let listing = available.length > 0 ? `Its tools are: ${available.join(", ")}.` : `It exposes no tools.`;
|
|
39
|
+
return failure(new ResolutionError("unknown-name", `Unknown tool "${tool}" in namespace "${namespace}". ${listing}`));
|
|
40
|
+
}
|
|
41
|
+
let resolved = {
|
|
42
|
+
kind: "tool",
|
|
43
|
+
plugin: entry.plugin,
|
|
44
|
+
descriptor,
|
|
45
|
+
namespace,
|
|
46
|
+
};
|
|
47
|
+
return success(resolved);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Resolve a bare target among suite commands (which never need `use`) and
|
|
51
|
+
* the tools of the namespaces the calling file imported. More than one
|
|
52
|
+
* candidate is an ambiguity error — the runtime never guesses.
|
|
53
|
+
*/
|
|
54
|
+
function resolveBare(target, uses) {
|
|
55
|
+
let candidates = [];
|
|
56
|
+
let command = suite.commands.get(target);
|
|
57
|
+
if (command)
|
|
58
|
+
candidates.push({ qualified: target, resolved: { kind: "command", command } });
|
|
59
|
+
let visited = new Set();
|
|
60
|
+
for (let namespace of uses) {
|
|
61
|
+
if (visited.has(namespace))
|
|
62
|
+
continue;
|
|
63
|
+
visited.add(namespace);
|
|
64
|
+
let entry = namespaces.get(namespace);
|
|
65
|
+
if (!entry)
|
|
66
|
+
continue;
|
|
67
|
+
let descriptor = entry.tools.get(target);
|
|
68
|
+
if (!descriptor)
|
|
69
|
+
continue;
|
|
70
|
+
candidates.push({
|
|
71
|
+
qualified: `${namespace}.${target}`,
|
|
72
|
+
resolved: { kind: "tool", plugin: entry.plugin, descriptor, namespace },
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
let [first] = candidates;
|
|
76
|
+
if (first && candidates.length === 1)
|
|
77
|
+
return success(first.resolved);
|
|
78
|
+
if (candidates.length === 0) {
|
|
79
|
+
return failure(new ResolutionError("unknown-name", `Unknown name "${target}": it is not a suite command, and no namespace imported with \`use\` provides it.`));
|
|
80
|
+
}
|
|
81
|
+
let qualified = candidates.map((candidate) => candidate.qualified);
|
|
82
|
+
let described = candidates.map((candidate) => candidate.resolved.kind === "command"
|
|
83
|
+
? `the command "${candidate.qualified}"`
|
|
84
|
+
: candidate.qualified);
|
|
85
|
+
return failure(new ResolutionError("ambiguous-name", `Ambiguous name "${target}": it matches ${described.join(" and ")}. Use the fully qualified name.`, qualified));
|
|
86
|
+
}
|
|
87
|
+
/** Dispatch on the target's shape: bare, `ns.tool`, or too many dots. */
|
|
88
|
+
function resolveCallable(target, uses) {
|
|
89
|
+
let segments = target.split(".");
|
|
90
|
+
if (segments.length > 2) {
|
|
91
|
+
return failure(new ResolutionError("unknown-name", `Unknown name "${target}": a call target has at most one dot (namespace.tool).`));
|
|
92
|
+
}
|
|
93
|
+
let [head, tail] = segments;
|
|
94
|
+
if (head !== undefined && tail !== undefined)
|
|
95
|
+
return resolveQualified(target, head, tail);
|
|
96
|
+
return resolveBare(target, uses);
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
resolveCallable,
|
|
100
|
+
resolveFixture(name) {
|
|
101
|
+
let fixture = suite.fixtures.get(name);
|
|
102
|
+
if (fixture)
|
|
103
|
+
return success(fixture);
|
|
104
|
+
return failure(new ResolutionError("unknown-name", `Unknown fixture "${name}".`));
|
|
105
|
+
},
|
|
106
|
+
isCallable(target, uses) {
|
|
107
|
+
return isSuccess(resolveCallable(target, uses));
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The human reporter: renders a suite's structured results, and fatal
|
|
3
|
+
* pre-run failures, onto a `Sink` as plain text. It branches on diagnostic
|
|
4
|
+
* codes and structured error fields, never on error message text.
|
|
5
|
+
*
|
|
6
|
+
* @author [Sergio Xalambrí](https://sergiodxa.com)
|
|
7
|
+
* @copyright Sergio Xalambrí 2026
|
|
8
|
+
*/
|
|
9
|
+
import type { Sink, SuiteResult } from "./diagnostics.js";
|
|
10
|
+
import type { SpecError } from "./errors.js";
|
|
11
|
+
import type { SourceFile } from "./source.js";
|
|
12
|
+
/**
|
|
13
|
+
* Render a finished suite: a status line per test, denials sharing a remedy
|
|
14
|
+
* collapsed into one block naming the grant and its affected tests, and a
|
|
15
|
+
* summary reporting the run's wall-clock duration, accurate under concurrency.
|
|
16
|
+
*
|
|
17
|
+
* @param suite - The suite roll-up the runner produced.
|
|
18
|
+
* @param sources - Loaded file texts by path, for turning spans into lines.
|
|
19
|
+
* @param sink - Where the report is written.
|
|
20
|
+
*/
|
|
21
|
+
export declare function reportSuite(suite: SuiteResult, sources: Map<string, SourceFile>, sink: Sink): void;
|
|
22
|
+
/**
|
|
23
|
+
* Render a failure that prevented any test from running — an unreadable
|
|
24
|
+
* suite directory, a duplicate definition, or a file that failed to parse —
|
|
25
|
+
* naming the file (with line:column when known), the code, message, and remedy.
|
|
26
|
+
*
|
|
27
|
+
* @param error - The load-time failure.
|
|
28
|
+
* @param sink - Where the report is written.
|
|
29
|
+
*/
|
|
30
|
+
export declare function reportFatal(error: SpecError, sink: Sink): void;
|
package/dist/reporter.js
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The human reporter: renders a suite's structured results, and fatal
|
|
3
|
+
* pre-run failures, onto a `Sink` as plain text. It branches on diagnostic
|
|
4
|
+
* codes and structured error fields, never on error message text.
|
|
5
|
+
*
|
|
6
|
+
* @author [Sergio Xalambrí](https://sergiodxa.com)
|
|
7
|
+
* @copyright Sergio Xalambrí 2026
|
|
8
|
+
*/
|
|
9
|
+
import { readFileSync } from "node:fs";
|
|
10
|
+
import { positionAt } from "./source.js";
|
|
11
|
+
import { formatValue } from "./values.js";
|
|
12
|
+
/** Indentation prefixing every non-empty line of a failure's detail block. */
|
|
13
|
+
const DETAIL_INDENT = " ";
|
|
14
|
+
/**
|
|
15
|
+
* Render a finished suite: a status line per test, denials sharing a remedy
|
|
16
|
+
* collapsed into one block naming the grant and its affected tests, and a
|
|
17
|
+
* summary reporting the run's wall-clock duration, accurate under concurrency.
|
|
18
|
+
*
|
|
19
|
+
* @param suite - The suite roll-up the runner produced.
|
|
20
|
+
* @param sources - Loaded file texts by path, for turning spans into lines.
|
|
21
|
+
* @param sink - Where the report is written.
|
|
22
|
+
*/
|
|
23
|
+
export function reportSuite(suite, sources, sink) {
|
|
24
|
+
let separated = true;
|
|
25
|
+
let groups = [];
|
|
26
|
+
let byRemedy = new Map();
|
|
27
|
+
for (let result of suite.results) {
|
|
28
|
+
if (result.status === "passed") {
|
|
29
|
+
sink.write(`✓ ${result.title}\n`);
|
|
30
|
+
separated = false;
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
let grouping = result.error === undefined ? undefined : groupableDenial(result.error);
|
|
34
|
+
if (grouping !== undefined) {
|
|
35
|
+
accumulateDenial(groups, byRemedy, grouping, {
|
|
36
|
+
title: result.title,
|
|
37
|
+
location: failureLocation(result, sources),
|
|
38
|
+
});
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
sink.write(`✗ ${result.title} (${failureLocation(result, sources)})\n`);
|
|
42
|
+
separated = false;
|
|
43
|
+
if (result.error !== undefined) {
|
|
44
|
+
for (let line of detailLines(result.error)) {
|
|
45
|
+
sink.write(line === "" ? "\n" : `${DETAIL_INDENT}${line}\n`);
|
|
46
|
+
}
|
|
47
|
+
sink.write("\n");
|
|
48
|
+
separated = true;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
for (let group of groups) {
|
|
52
|
+
if (!separated)
|
|
53
|
+
sink.write("\n");
|
|
54
|
+
sink.write(`✗ ${groupHeader(group)}\n`);
|
|
55
|
+
for (let line of groupDetailLines(group)) {
|
|
56
|
+
sink.write(line === "" ? "\n" : `${DETAIL_INDENT}${line}\n`);
|
|
57
|
+
}
|
|
58
|
+
sink.write("\n");
|
|
59
|
+
separated = true;
|
|
60
|
+
}
|
|
61
|
+
if (!separated)
|
|
62
|
+
sink.write("\n");
|
|
63
|
+
sink.write(`${suite.passed} passed, ${suite.failed} failed (${Math.round(suite.wallMs)}ms)\n`);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Render a failure that prevented any test from running — an unreadable
|
|
67
|
+
* suite directory, a duplicate definition, or a file that failed to parse —
|
|
68
|
+
* naming the file (with line:column when known), the code, message, and remedy.
|
|
69
|
+
*
|
|
70
|
+
* @param error - The load-time failure.
|
|
71
|
+
* @param sink - Where the report is written.
|
|
72
|
+
*/
|
|
73
|
+
export function reportFatal(error, sink) {
|
|
74
|
+
let location = fatalLocation(error);
|
|
75
|
+
let suffix = location === undefined ? "" : ` (${location})`;
|
|
76
|
+
sink.write(`✗ ${error.code}: ${error.message}${suffix}\n`);
|
|
77
|
+
if (error.remedy !== undefined) {
|
|
78
|
+
sink.write(`${DETAIL_INDENT}remedy: ${error.remedy}\n`);
|
|
79
|
+
}
|
|
80
|
+
if (error.hint !== undefined) {
|
|
81
|
+
sink.write(`${DETAIL_INDENT}${error.hint}\n`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Where a failing test points: `file:line` when the error's span falls in a
|
|
86
|
+
* loaded source, the bare file path otherwise.
|
|
87
|
+
*/
|
|
88
|
+
function failureLocation(result, sources) {
|
|
89
|
+
let file = result.error?.file ?? result.file;
|
|
90
|
+
let span = result.error?.span;
|
|
91
|
+
let source = sources.get(file);
|
|
92
|
+
if (span !== undefined && source !== undefined) {
|
|
93
|
+
let position = positionAt(source, span.start);
|
|
94
|
+
return `${file}:${position.line}`;
|
|
95
|
+
}
|
|
96
|
+
return file;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* The unindented lines of a failure's detail block: the ADR-007-shaped
|
|
100
|
+
* denial block for permission failures, otherwise the diagnostic code and
|
|
101
|
+
* message followed by `expected:`/`observed:` and the remedy when carried.
|
|
102
|
+
*/
|
|
103
|
+
function detailLines(error) {
|
|
104
|
+
let denial = denialBlock(error);
|
|
105
|
+
if (denial !== undefined) {
|
|
106
|
+
if (error.hint !== undefined)
|
|
107
|
+
return [...denial, "", error.hint];
|
|
108
|
+
return denial;
|
|
109
|
+
}
|
|
110
|
+
let lines = [`${error.code}: ${error.message}`];
|
|
111
|
+
let comparison = error;
|
|
112
|
+
if (comparison.expected !== undefined)
|
|
113
|
+
pushLabeledValue(lines, "expected", comparison.expected);
|
|
114
|
+
if (comparison.observed !== undefined)
|
|
115
|
+
pushLabeledValue(lines, "observed", comparison.observed);
|
|
116
|
+
if (error.remedy !== undefined)
|
|
117
|
+
lines.push(`remedy: ${error.remedy}`);
|
|
118
|
+
if (error.hint !== undefined)
|
|
119
|
+
lines.push(error.hint);
|
|
120
|
+
return lines;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* The grouping fields of a permission denial the reporter accumulates, or
|
|
124
|
+
* undefined when {@link denialBlock} could not render it — a denial missing
|
|
125
|
+
* `permission`, `resource`, or `remedy` falls back to the inline detail path.
|
|
126
|
+
*/
|
|
127
|
+
function groupableDenial(error) {
|
|
128
|
+
if (denialBlock(error) === undefined)
|
|
129
|
+
return undefined;
|
|
130
|
+
let denial = error;
|
|
131
|
+
let permission = denial.permission;
|
|
132
|
+
let resource = denial.resource;
|
|
133
|
+
let remedy = error.remedy;
|
|
134
|
+
if (permission === undefined || resource === undefined || remedy === undefined)
|
|
135
|
+
return undefined;
|
|
136
|
+
return { permission, resource, remedy, hint: error.hint };
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Fold one denial into the group its remedy keys — creating the group in
|
|
140
|
+
* first-seen order when new — recording the resource (deduplicated, first-seen
|
|
141
|
+
* order) and the affected test in execution order.
|
|
142
|
+
*/
|
|
143
|
+
function accumulateDenial(groups, byRemedy, grouping, test) {
|
|
144
|
+
let group = byRemedy.get(grouping.remedy);
|
|
145
|
+
if (group === undefined) {
|
|
146
|
+
group = {
|
|
147
|
+
permission: grouping.permission,
|
|
148
|
+
resources: [],
|
|
149
|
+
remedy: grouping.remedy,
|
|
150
|
+
hint: grouping.hint,
|
|
151
|
+
tests: [],
|
|
152
|
+
};
|
|
153
|
+
byRemedy.set(grouping.remedy, group);
|
|
154
|
+
groups.push(group);
|
|
155
|
+
}
|
|
156
|
+
if (!group.resources.includes(grouping.resource))
|
|
157
|
+
group.resources.push(grouping.resource);
|
|
158
|
+
group.tests.push(test);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* The header of a denial group's block: the permission and how many tests it
|
|
162
|
+
* accounts for, with `test`/`tests` agreeing in number.
|
|
163
|
+
*/
|
|
164
|
+
function groupHeader(group) {
|
|
165
|
+
let count = group.tests.length;
|
|
166
|
+
return `Permission denied: ${group.permission} (${count} ${count === 1 ? "test" : "tests"})`;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* The unindented body lines of a denial group's block: the attempted
|
|
170
|
+
* resources, the shared remedy, and the affected tests. The two label lines
|
|
171
|
+
* stay verbatim so the design suite's substring assertions keep matching.
|
|
172
|
+
*/
|
|
173
|
+
function groupDetailLines(group) {
|
|
174
|
+
let lines = ["", "The spec attempted to reach:"];
|
|
175
|
+
for (let resource of group.resources)
|
|
176
|
+
lines.push(`> ${resource}`);
|
|
177
|
+
lines.push("", "Re-run with an appropriate permission, for example:", `> ${group.remedy}`);
|
|
178
|
+
if (group.hint !== undefined)
|
|
179
|
+
lines.push("", group.hint);
|
|
180
|
+
lines.push("", "Affected tests:");
|
|
181
|
+
for (let test of group.tests)
|
|
182
|
+
lines.push(`- ${test.title} (${test.location})`);
|
|
183
|
+
return lines;
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* The denial block the design suite requires: the permission, the attempted
|
|
187
|
+
* resource, and the exact flag that would grant it. Returns undefined when
|
|
188
|
+
* the error is not a permission denial or lacks the structured fields.
|
|
189
|
+
*/
|
|
190
|
+
function denialBlock(error) {
|
|
191
|
+
if (error.code !== "permission-denied")
|
|
192
|
+
return undefined;
|
|
193
|
+
let denial = error;
|
|
194
|
+
if (denial.permission === undefined || denial.resource === undefined)
|
|
195
|
+
return undefined;
|
|
196
|
+
if (denial.remedy === undefined)
|
|
197
|
+
return undefined;
|
|
198
|
+
return [
|
|
199
|
+
`Permission denied: ${denial.permission}`,
|
|
200
|
+
"",
|
|
201
|
+
"The spec attempted to reach:",
|
|
202
|
+
`> ${denial.resource}`,
|
|
203
|
+
"",
|
|
204
|
+
"Re-run with an appropriate permission, for example:",
|
|
205
|
+
`> ${denial.remedy}`,
|
|
206
|
+
];
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Append `label: value` to the detail lines, keeping each line of a
|
|
210
|
+
* multi-line rendering its own detail line so indentation stays consistent.
|
|
211
|
+
*/
|
|
212
|
+
function pushLabeledValue(lines, label, value) {
|
|
213
|
+
let parts = formatValue(value).split("\n");
|
|
214
|
+
lines.push(`${label}: ${parts[0] ?? ""}`);
|
|
215
|
+
for (let part of parts.slice(1))
|
|
216
|
+
lines.push(part);
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Where a fatal error points: `file:line:column` when the span is known and
|
|
220
|
+
* the file is readable — re-read from disk since fatal errors predate any
|
|
221
|
+
* loaded-source map — the bare file path otherwise, undefined without a file.
|
|
222
|
+
*/
|
|
223
|
+
function fatalLocation(error) {
|
|
224
|
+
if (error.file === undefined)
|
|
225
|
+
return undefined;
|
|
226
|
+
if (error.span === undefined)
|
|
227
|
+
return error.file;
|
|
228
|
+
let text;
|
|
229
|
+
try {
|
|
230
|
+
text = readFileSync(error.file, "utf8");
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
return error.file;
|
|
234
|
+
}
|
|
235
|
+
let position = positionAt({ path: error.file, text }, error.span.start);
|
|
236
|
+
return `${error.file}:${position.line}:${position.column}`;
|
|
237
|
+
}
|
package/dist/run.d.ts
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Executing an already-loaded suite: give every test a fresh workspace, run it,
|
|
3
|
+
* and collect structured results in source order. Everything the run depends on
|
|
4
|
+
* arrives as an argument — the suite, the plugin set, the grants, the workspace
|
|
5
|
+
* factory — so nothing here reaches for a filesystem, a process, or a specific
|
|
6
|
+
* runtime. `runner.ts` is the host convenience that supplies the usual answers;
|
|
7
|
+
* an embedder on a runtime without those answers calls this directly.
|
|
8
|
+
*
|
|
9
|
+
* Language semantics live in the executor and rendering lives in the reporter;
|
|
10
|
+
* this module owns only the lifecycle glue.
|
|
11
|
+
*
|
|
12
|
+
* @author [Sergio Xalambrí](https://sergiodxa.com)
|
|
13
|
+
* @copyright Sergio Xalambrí 2026
|
|
14
|
+
*/
|
|
15
|
+
import type { Result } from "@sdxc/result";
|
|
16
|
+
import type { Seed } from "@sdxc/sample";
|
|
17
|
+
import type { SuiteResult } from "./diagnostics.js";
|
|
18
|
+
import type { SpecError } from "./errors.js";
|
|
19
|
+
import type { Grants, PermissionSet } from "./permissions.js";
|
|
20
|
+
import type { Plugin } from "./plugin.js";
|
|
21
|
+
import type { LoadedSuite } from "./sources.js";
|
|
22
|
+
import type { Workspace } from "./workspace.js";
|
|
23
|
+
/** Creates the isolated workspace one test runs in. */
|
|
24
|
+
export type WorkspaceFactory = (permissions: PermissionSet) => Promise<Result<Workspace, SpecError>>;
|
|
25
|
+
/** Everything executing a loaded suite depends on, all of it injected. */
|
|
26
|
+
export interface RunTestsOptions {
|
|
27
|
+
/** The parsed suite, from `loadSuite` (a directory) or `loadSources` (strings). */
|
|
28
|
+
suite: LoadedSuite;
|
|
29
|
+
/**
|
|
30
|
+
* Every plugin whose namespace this run understands — the complete set, not
|
|
31
|
+
* additions to a default one. Leaving a namespace out makes it nonexistent
|
|
32
|
+
* to a spec, not merely forbidden, so referencing it fails to resolve.
|
|
33
|
+
*
|
|
34
|
+
* @see createBuiltinPlugins
|
|
35
|
+
*/
|
|
36
|
+
plugins: Plugin[];
|
|
37
|
+
/** The caller's permission grants, scoping what the registered plugins may reach. */
|
|
38
|
+
grants: Grants;
|
|
39
|
+
/** How each test's workspace is created; called once per test. */
|
|
40
|
+
createWorkspace: WorkspaceFactory;
|
|
41
|
+
/**
|
|
42
|
+
* How many tests may execute at once, each in its own isolated workspace.
|
|
43
|
+
* Results stay source-ordered regardless of completion order, though a
|
|
44
|
+
* shared, mutable app under test may still require `concurrency: 1`.
|
|
45
|
+
*
|
|
46
|
+
* @default 1
|
|
47
|
+
*/
|
|
48
|
+
concurrency?: number;
|
|
49
|
+
/**
|
|
50
|
+
* The run's seed, which every test's generated data descends from. A fixed
|
|
51
|
+
* default makes two runs of a suite produce identical data; pass a drawn
|
|
52
|
+
* seed to shake a suite for hidden dependence on particular values.
|
|
53
|
+
*
|
|
54
|
+
* @default "spec"
|
|
55
|
+
*/
|
|
56
|
+
seed?: Seed;
|
|
57
|
+
/**
|
|
58
|
+
* The suite directory, which a test's seed measures its file against so the
|
|
59
|
+
* data a suite generates does not follow the suite's absolute location on
|
|
60
|
+
* disk. Omit when the sources came from strings rather than a directory.
|
|
61
|
+
*/
|
|
62
|
+
root?: string;
|
|
63
|
+
}
|
|
64
|
+
/** The run seed used when a caller names none, so a bare run repeats exactly. */
|
|
65
|
+
export declare const DEFAULT_SEED = "spec";
|
|
66
|
+
/**
|
|
67
|
+
* Execute every test in a loaded suite. Test failures land as outcomes in the
|
|
68
|
+
* returned result, not thrown errors; only a workspace-creation failure aborts
|
|
69
|
+
* the run, since a test that never got a place to run has no pass or fail.
|
|
70
|
+
*
|
|
71
|
+
* @param options - The suite, its plugin set, the grants, and the workspace factory.
|
|
72
|
+
* @returns Per-test outcomes in source order, or the error that aborted the run.
|
|
73
|
+
*/
|
|
74
|
+
export declare function runTests(options: RunTestsOptions): Promise<Result<SuiteResult, SpecError>>;
|
package/dist/run.js
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Executing an already-loaded suite: give every test a fresh workspace, run it,
|
|
3
|
+
* and collect structured results in source order. Everything the run depends on
|
|
4
|
+
* arrives as an argument — the suite, the plugin set, the grants, the workspace
|
|
5
|
+
* factory — so nothing here reaches for a filesystem, a process, or a specific
|
|
6
|
+
* runtime. `runner.ts` is the host convenience that supplies the usual answers;
|
|
7
|
+
* an embedder on a runtime without those answers calls this directly.
|
|
8
|
+
*
|
|
9
|
+
* Language semantics live in the executor and rendering lives in the reporter;
|
|
10
|
+
* this module owns only the lifecycle glue.
|
|
11
|
+
*
|
|
12
|
+
* @author [Sergio Xalambrí](https://sergiodxa.com)
|
|
13
|
+
* @copyright Sergio Xalambrí 2026
|
|
14
|
+
*/
|
|
15
|
+
import { relative, sep } from "node:path";
|
|
16
|
+
import { isFailure, success } from "@sdxc/result";
|
|
17
|
+
import { createRandom } from "@sdxc/sample";
|
|
18
|
+
import { executeTest } from "./executor.js";
|
|
19
|
+
import { createPermissionSet } from "./permissions.js";
|
|
20
|
+
import { createRegistry } from "./registry.js";
|
|
21
|
+
/** The run seed used when a caller names none, so a bare run repeats exactly. */
|
|
22
|
+
export const DEFAULT_SEED = "spec";
|
|
23
|
+
/**
|
|
24
|
+
* The stream one test draws from: the run's seed and the test's identity, and
|
|
25
|
+
* nothing about when or in what order it ran. Two tests that share a file and
|
|
26
|
+
* a title share a stream, which is the same data for what is already the same
|
|
27
|
+
* name.
|
|
28
|
+
*
|
|
29
|
+
* A test is identified by its file's path inside the suite, never the absolute
|
|
30
|
+
* one, so a suite generates the same data wherever it is checked out and
|
|
31
|
+
* however the runner was pointed at it.
|
|
32
|
+
*/
|
|
33
|
+
function streamFor(seed, file, title, root) {
|
|
34
|
+
let within = root === undefined ? file : relative(root, file);
|
|
35
|
+
return createRandom(`${seed} ${within.split(sep).join("/")}#${title}`);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Execute every test in a loaded suite. Test failures land as outcomes in the
|
|
39
|
+
* returned result, not thrown errors; only a workspace-creation failure aborts
|
|
40
|
+
* the run, since a test that never got a place to run has no pass or fail.
|
|
41
|
+
*
|
|
42
|
+
* @param options - The suite, its plugin set, the grants, and the workspace factory.
|
|
43
|
+
* @returns Per-test outcomes in source order, or the error that aborted the run.
|
|
44
|
+
*/
|
|
45
|
+
export async function runTests(options) {
|
|
46
|
+
let suite = options.suite;
|
|
47
|
+
let plugins = options.plugins;
|
|
48
|
+
let registry = createRegistry(plugins, suite);
|
|
49
|
+
let permissions = createPermissionSet(options.grants);
|
|
50
|
+
/**
|
|
51
|
+
* `use` is file-scoped: a definition's body resolves bare names against the
|
|
52
|
+
* imports of the file that defined it, so errors raised inside a shared
|
|
53
|
+
* command report that file's location, not the calling test's.
|
|
54
|
+
*/
|
|
55
|
+
let usesByDefinition = new Map();
|
|
56
|
+
let fileByDefinition = new Map();
|
|
57
|
+
for (let file of suite.files) {
|
|
58
|
+
let imported = file.uses.map((use) => use.namespace);
|
|
59
|
+
for (let definition of file.definitions) {
|
|
60
|
+
usesByDefinition.set(definition, imported);
|
|
61
|
+
fileByDefinition.set(definition, file.path);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Every test flattened into one source-ordered work list; each unit carries
|
|
66
|
+
* its own file path and `use` imports, so a worker needs nothing but the
|
|
67
|
+
* shared registry and permissions to run it.
|
|
68
|
+
*/
|
|
69
|
+
let pending = [];
|
|
70
|
+
for (let file of suite.files) {
|
|
71
|
+
let imported = file.uses.map((use) => use.namespace);
|
|
72
|
+
for (let test of file.tests)
|
|
73
|
+
pending.push({ test, filePath: file.path, imported });
|
|
74
|
+
}
|
|
75
|
+
let concurrency = Math.max(1, Math.trunc(options.concurrency ?? 1));
|
|
76
|
+
let results = Array.from({ length: pending.length });
|
|
77
|
+
let nextIndex = 0;
|
|
78
|
+
/**
|
|
79
|
+
* The first fatal workspace-creation failure, if any. In-flight tests still
|
|
80
|
+
* finish and no new work is pulled once this is set; the run then returns
|
|
81
|
+
* this failure.
|
|
82
|
+
*/
|
|
83
|
+
let fatal;
|
|
84
|
+
/**
|
|
85
|
+
* Claims the next source index and runs it in a fresh workspace, repeating
|
|
86
|
+
* until the list drains or a fatal failure appears. Claiming `nextIndex` is
|
|
87
|
+
* race-free: nothing awaits between reading and incrementing it.
|
|
88
|
+
*/
|
|
89
|
+
async function runWorker() {
|
|
90
|
+
while (fatal === undefined) {
|
|
91
|
+
let index = nextIndex;
|
|
92
|
+
nextIndex += 1;
|
|
93
|
+
if (index >= pending.length)
|
|
94
|
+
return;
|
|
95
|
+
let unit = pending[index];
|
|
96
|
+
if (unit === undefined)
|
|
97
|
+
return;
|
|
98
|
+
let workspace = await options.createWorkspace(permissions);
|
|
99
|
+
if (isFailure(workspace)) {
|
|
100
|
+
fatal ??= workspace;
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
let startedAt = performance.now();
|
|
104
|
+
let outcome = await executeTest(unit.test, {
|
|
105
|
+
registry,
|
|
106
|
+
workspace: workspace.data,
|
|
107
|
+
permissions,
|
|
108
|
+
random: streamFor(options.seed ?? DEFAULT_SEED, unit.filePath, unit.test.title, options.root),
|
|
109
|
+
now: new Date(),
|
|
110
|
+
uses: unit.imported,
|
|
111
|
+
usesFor: (definition) => usesByDefinition.get(definition) ?? unit.imported,
|
|
112
|
+
fileFor: (definition) => fileByDefinition.get(definition),
|
|
113
|
+
grants: options.grants,
|
|
114
|
+
});
|
|
115
|
+
let durationMs = performance.now() - startedAt;
|
|
116
|
+
await workspace.data.cleanup();
|
|
117
|
+
if (isFailure(outcome)) {
|
|
118
|
+
let error = outcome.error;
|
|
119
|
+
if (error.file === undefined)
|
|
120
|
+
error.file = unit.filePath;
|
|
121
|
+
results[index] = {
|
|
122
|
+
title: unit.test.title,
|
|
123
|
+
file: unit.filePath,
|
|
124
|
+
status: "failed",
|
|
125
|
+
error,
|
|
126
|
+
durationMs,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
results[index] = {
|
|
131
|
+
title: unit.test.title,
|
|
132
|
+
file: unit.filePath,
|
|
133
|
+
status: "passed",
|
|
134
|
+
durationMs,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Wall-clock spans only the test-execution phase: captured immediately
|
|
141
|
+
* before workers start and immediately after the last one finishes, giving
|
|
142
|
+
* real elapsed time whether the run was sequential or concurrent.
|
|
143
|
+
*/
|
|
144
|
+
let wallStart = performance.now();
|
|
145
|
+
let wallMs = 0;
|
|
146
|
+
try {
|
|
147
|
+
let workerCount = Math.min(concurrency, pending.length);
|
|
148
|
+
let workers = [];
|
|
149
|
+
for (let slot = 0; slot < workerCount; slot += 1)
|
|
150
|
+
workers.push(runWorker());
|
|
151
|
+
await Promise.all(workers);
|
|
152
|
+
wallMs = performance.now() - wallStart;
|
|
153
|
+
if (fatal !== undefined)
|
|
154
|
+
return fatal;
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
/**
|
|
158
|
+
* Plugins with external state (a browser session, a connection) release
|
|
159
|
+
* it here, once per test. A throwing dispose here still lets a completed
|
|
160
|
+
* run return its results.
|
|
161
|
+
*/
|
|
162
|
+
for (let plugin of plugins) {
|
|
163
|
+
if (plugin.dispose === undefined)
|
|
164
|
+
continue;
|
|
165
|
+
try {
|
|
166
|
+
await plugin.dispose();
|
|
167
|
+
}
|
|
168
|
+
catch { }
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Every slot is filled once the run completes without a fatal failure,
|
|
173
|
+
* since each claimed index writes exactly one result; filtering here only
|
|
174
|
+
* narrows the sparse `undefined` type while preserving source order.
|
|
175
|
+
*/
|
|
176
|
+
let ordered = results.filter((result) => result !== undefined);
|
|
177
|
+
let passed = ordered.filter((result) => result.status === "passed").length;
|
|
178
|
+
return success({ results: ordered, passed, failed: ordered.length - passed, wallMs });
|
|
179
|
+
}
|