@specific.dev/spectest 0.112.0 → 0.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/daemon.js +16 -0
- package/dist/harness/fake-functions.d.ts +18 -0
- package/dist/harness/fake-functions.js +82 -0
- package/dist/harness/function-parameters.d.ts +9 -0
- package/dist/harness/function-parameters.js +42 -0
- package/dist/harness/protocol.d.ts +1 -1
- package/dist/harness/protocol.js +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +13 -4
- package/dist/interactive.d.ts +18 -0
- package/dist/interactive.js +39 -0
- package/package.json +2 -1
- package/src/daemon.ts +14 -0
- package/src/fake-functions.test.ts +62 -0
- package/src/harness/fake-functions.test.ts +77 -0
- package/src/harness/fake-functions.ts +78 -0
- package/src/harness/function-parameters.test.ts +45 -0
- package/src/harness/function-parameters.ts +43 -0
- package/src/harness/protocol.ts +2 -0
- package/src/index.ts +25 -4
- package/src/interactive-setup.test.ts +7 -2
- package/src/interactive.test.ts +79 -0
- package/src/interactive.ts +41 -0
package/README.md
CHANGED
|
@@ -20,6 +20,34 @@ Pub/Sub integrations are project-owned as of SDK 0.98.0. The former
|
|
|
20
20
|
Use a local service definition or `serviceGroup()` wrapper; the repository’s
|
|
21
21
|
`examples/google-pubsub` demonstrates this using public SDK APIs.
|
|
22
22
|
|
|
23
|
+
## Interactive entrypoints (0.114.0)
|
|
24
|
+
|
|
25
|
+
Choose the links shown in the interactive dashboard and PR preview comment:
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
const env = defineEnvironment({
|
|
29
|
+
name: "my-app",
|
|
30
|
+
services: { web, api },
|
|
31
|
+
interactive: {
|
|
32
|
+
instructions: "Sign in as demo@example.com and use 000000 as the code",
|
|
33
|
+
entrypoints: [
|
|
34
|
+
{ name: "App", url: "https://app.test" },
|
|
35
|
+
{ name: "Admin", url: "https://app.test/admin" },
|
|
36
|
+
{ name: "API shell", terminal: "api" },
|
|
37
|
+
],
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
List order is preserved. URLs must match declared service hostnames and are
|
|
43
|
+
translated to public preview URLs, retaining paths, queries and fragments.
|
|
44
|
+
Terminal targets are typed service names and use dashboard member permissions.
|
|
45
|
+
Entrypoints are display metadata; they do not change exposure or test execution.
|
|
46
|
+
With no entries, the only preview link is **Open environment**. Optional
|
|
47
|
+
`interactive.instructions` is a concise, direct plain-text sign-in description,
|
|
48
|
+
shown in a collapsed **Instructions** section in the dashboard and PR
|
|
49
|
+
comment. Blank instructions are omitted.
|
|
50
|
+
|
|
23
51
|
|
|
24
52
|
## Microphone fixtures and WebSocket fakes (0.103.0)
|
|
25
53
|
|
package/dist/daemon.js
CHANGED
|
@@ -43,6 +43,7 @@ import { isDesktopApp } from "./desktop.js";
|
|
|
43
43
|
// each easy to restate subtly differently.
|
|
44
44
|
import { buildArgFlags, buildContentKey as computeBuildContentKey, buildTargetFlags, imageTag, isGeneratedDockerignore, pickProjectIgnore, serviceDockerignore as composeServiceDockerignore, unionDockerignore, } from "./harness/build-context.js";
|
|
45
45
|
import { startServices } from "./harness/service-graph.js";
|
|
46
|
+
import { fakeFunctions, fakeFunctionParameters, callFakeFunction } from "./harness/fake-functions.js";
|
|
46
47
|
import { dispatchFake } from "./harness/fake-websocket.js";
|
|
47
48
|
import { casesMetadata as catalogueCases, groupsMetadata as catalogueGroups, } from "./harness/catalogue.js";
|
|
48
49
|
import { summarizeBuildKit } from "./harness/buildkit-progress.js";
|
|
@@ -5286,6 +5287,21 @@ export function harnessMethods(state) {
|
|
|
5286
5287
|
}),
|
|
5287
5288
|
projectSetup: async () => exclusive(state, "inFlightProjectSetup", "project-setup already in progress", () => runProjectSetup()),
|
|
5288
5289
|
interactiveSetup: async () => exclusive(state, "inFlightProjectSetup", "project-setup already in progress", () => runProjectSetup("setupInteractive")),
|
|
5290
|
+
fakeFunctions: async (params) => exclusive(state, "inFlightTest", "a test or eval is already running", async () => {
|
|
5291
|
+
const name = requireString(params, "fake");
|
|
5292
|
+
const fake = FAKES.get(name);
|
|
5293
|
+
if (!fake)
|
|
5294
|
+
throw notFound(`Unknown fake: ${name}`);
|
|
5295
|
+
await ensureFakeHelpers(name);
|
|
5296
|
+
if (params.member === undefined)
|
|
5297
|
+
return { functions: fakeFunctions(fake.helpers), parameters: fakeFunctionParameters(fake.helpers) };
|
|
5298
|
+
const member = requireString(params, "member");
|
|
5299
|
+
if (params.expressions !== undefined)
|
|
5300
|
+
return callFakeFunction(fake.helpers, member, [], [], params.expressions);
|
|
5301
|
+
if (!Array.isArray(params.args))
|
|
5302
|
+
throw badRequest("args must be a JSON array");
|
|
5303
|
+
return callFakeFunction(fake.helpers, member, params.args, params.omitted_args);
|
|
5304
|
+
}),
|
|
5289
5305
|
eval: async (params) => {
|
|
5290
5306
|
const code = requireString(params, "code");
|
|
5291
5307
|
return exclusive(state, "inFlightTest", "a test or eval is already running", () => {
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** Only own data properties are callable. Discovery never invokes getters. */
|
|
2
|
+
export declare function fakeFunctions(helpers: Record<string, unknown>): string[];
|
|
3
|
+
export declare function fakeFunctionParameters(helpers: Record<string, unknown>): {
|
|
4
|
+
[k: string]: import("./function-parameters.js").FunctionParameter[] | null;
|
|
5
|
+
};
|
|
6
|
+
/** Use the raw helpers: the test wrappers intentionally discard annotations
|
|
7
|
+
* when there is no recorder. Preserve both the value and its presentation. */
|
|
8
|
+
export declare function callFakeFunction(helpers: Record<string, unknown>, member: string, args: unknown[], omittedArgs?: number[], expressions?: string[]): Promise<{
|
|
9
|
+
result: {} | null;
|
|
10
|
+
annotation: import("../annotate.js").RenderAnnotation | null;
|
|
11
|
+
error: null;
|
|
12
|
+
duration_ms: number;
|
|
13
|
+
} | {
|
|
14
|
+
result: null;
|
|
15
|
+
annotation: null;
|
|
16
|
+
error: string;
|
|
17
|
+
duration_ms: number;
|
|
18
|
+
}>;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { parse } from "acorn";
|
|
2
|
+
import { functionParameters } from "./function-parameters.js";
|
|
3
|
+
import { readAnnotation } from "../annotate.js";
|
|
4
|
+
import { deepUnwrap } from "../inspect.js";
|
|
5
|
+
import { safeSerialize } from "../recorder.js";
|
|
6
|
+
import { runUninstrumented } from "./instrumentation-scope.js";
|
|
7
|
+
/** Only own data properties are callable. Discovery never invokes getters. */
|
|
8
|
+
export function fakeFunctions(helpers) {
|
|
9
|
+
return Object.getOwnPropertyNames(helpers).filter((name) => typeof Object.getOwnPropertyDescriptor(helpers, name)?.value === "function").sort();
|
|
10
|
+
}
|
|
11
|
+
export function fakeFunctionParameters(helpers) {
|
|
12
|
+
return Object.fromEntries(fakeFunctions(helpers).map(name => [name,
|
|
13
|
+
functionParameters(Object.getOwnPropertyDescriptor(helpers, name).value),
|
|
14
|
+
]));
|
|
15
|
+
}
|
|
16
|
+
/** Use the raw helpers: the test wrappers intentionally discard annotations
|
|
17
|
+
* when there is no recorder. Preserve both the value and its presentation. */
|
|
18
|
+
export async function callFakeFunction(helpers, member, args, omittedArgs = [], expressions) {
|
|
19
|
+
const fn = Object.getOwnPropertyDescriptor(helpers, member)?.value;
|
|
20
|
+
if (typeof fn !== "function")
|
|
21
|
+
throw new Error(`Unknown fake function: ${member}`);
|
|
22
|
+
if (!Array.isArray(omittedArgs) || omittedArgs.some(i => !Number.isInteger(i) || i < 0 || i >= args.length || args[i] !== null))
|
|
23
|
+
throw new Error("omitted_args must contain indices of null argument placeholders");
|
|
24
|
+
const omitted = new Set(omittedArgs);
|
|
25
|
+
const actualArgs = args.map((value, i) => omitted.has(i) ? undefined : value);
|
|
26
|
+
const start = Date.now();
|
|
27
|
+
try {
|
|
28
|
+
const value = await runUninstrumented(() => Reflect.apply(fn, helpers, expressions === undefined ? actualArgs : evaluateArguments(fn, expressions)));
|
|
29
|
+
const annotated = readAnnotation(value);
|
|
30
|
+
return {
|
|
31
|
+
result: safeSerialize(deepUnwrap(annotated ? annotated.value : value)) ?? null,
|
|
32
|
+
annotation: annotated?.annotation ?? null,
|
|
33
|
+
error: null,
|
|
34
|
+
duration_ms: Date.now() - start,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
return { result: null, annotation: null,
|
|
39
|
+
error: error instanceof Error ? error.message : String(error), duration_ms: Date.now() - start };
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Expressions execute only in the environment, never in either dashboard.
|
|
43
|
+
* Compile every field before evaluating any, so a syntax error cannot partially
|
|
44
|
+
* execute the call. Keep actual JS values (undefined, Date, functions, etc.). */
|
|
45
|
+
function evaluateArguments(fn, expressions) {
|
|
46
|
+
if (!Array.isArray(expressions) || expressions.some(source => typeof source !== "string"))
|
|
47
|
+
throw new Error("expressions must be an array of JavaScript expressions");
|
|
48
|
+
const parameters = functionParameters(fn);
|
|
49
|
+
const label = (index) => parameters?.[index]?.name ?? `Argument ${index + 1}`;
|
|
50
|
+
const withLabel = (index, action) => {
|
|
51
|
+
try {
|
|
52
|
+
return action();
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
throw new Error(`${label(index)}: ${error instanceof Error ? error.message : String(error)}`);
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
const compiled = expressions.map((source, index) => withLabel(index, () => {
|
|
59
|
+
if (!source.trim())
|
|
60
|
+
return null;
|
|
61
|
+
const expression = `(${source}\n)`;
|
|
62
|
+
const program = parse(expression, { ecmaVersion: "latest" });
|
|
63
|
+
if (program.body.length !== 1 || program.body[0].type !== "ExpressionStatement")
|
|
64
|
+
throw new Error("Enter a single JavaScript expression.");
|
|
65
|
+
return new Function(`"use strict"; return ${expression};`);
|
|
66
|
+
}));
|
|
67
|
+
// Preserve interior omissions but avoid inventing trailing arguments.
|
|
68
|
+
while (compiled.length && compiled.at(-1) === null)
|
|
69
|
+
compiled.pop();
|
|
70
|
+
const args = [];
|
|
71
|
+
compiled.forEach((evaluate, index) => withLabel(index, () => {
|
|
72
|
+
const value = evaluate ? evaluate() : undefined;
|
|
73
|
+
if (parameters?.[index]?.rest) {
|
|
74
|
+
if (!Array.isArray(value))
|
|
75
|
+
throw new Error("Enter an array expression for the remaining arguments.");
|
|
76
|
+
args.push(...value);
|
|
77
|
+
}
|
|
78
|
+
else
|
|
79
|
+
args.push(value);
|
|
80
|
+
}));
|
|
81
|
+
return args;
|
|
82
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export interface FunctionParameter {
|
|
2
|
+
name: string;
|
|
3
|
+
has_default: boolean;
|
|
4
|
+
rest: boolean;
|
|
5
|
+
destructured: boolean;
|
|
6
|
+
}
|
|
7
|
+
/** Parse runtime JavaScript, never execute it (including default expressions).
|
|
8
|
+
* Null means opaque/unparseable, while [] means a known zero-argument helper. */
|
|
9
|
+
export declare function functionParameters(fn: Function): FunctionParameter[] | null;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { parse } from "acorn";
|
|
2
|
+
/** Parse runtime JavaScript, never execute it (including default expressions).
|
|
3
|
+
* Null means opaque/unparseable, while [] means a known zero-argument helper. */
|
|
4
|
+
export function functionParameters(fn) {
|
|
5
|
+
const source = Function.prototype.toString.call(fn);
|
|
6
|
+
if (source.length > 256_000)
|
|
7
|
+
return null;
|
|
8
|
+
// Function/arrow expressions and object methods have different grammar.
|
|
9
|
+
for (const method of [false, true]) {
|
|
10
|
+
const text = method ? `({${source}})` : `(${source})`;
|
|
11
|
+
try {
|
|
12
|
+
const program = parse(text, { ecmaVersion: "latest", sourceType: "module" });
|
|
13
|
+
const statement = program.body[0];
|
|
14
|
+
if (statement?.type !== "ExpressionStatement")
|
|
15
|
+
continue;
|
|
16
|
+
let expression = statement.expression;
|
|
17
|
+
if (method) {
|
|
18
|
+
if (expression.type !== "ObjectExpression")
|
|
19
|
+
continue;
|
|
20
|
+
const properties = expression.properties;
|
|
21
|
+
if (properties.length !== 1 || properties[0]?.type !== "Property")
|
|
22
|
+
continue;
|
|
23
|
+
expression = properties[0].value;
|
|
24
|
+
}
|
|
25
|
+
if (expression.type !== "FunctionExpression" && expression.type !== "ArrowFunctionExpression")
|
|
26
|
+
continue;
|
|
27
|
+
return expression.params.map((parameter) => {
|
|
28
|
+
const rest = parameter.type === "RestElement";
|
|
29
|
+
let binding = rest ? parameter.argument : parameter;
|
|
30
|
+
const has_default = binding.type === "AssignmentPattern";
|
|
31
|
+
if (binding.type === "AssignmentPattern")
|
|
32
|
+
binding = binding.left;
|
|
33
|
+
return {
|
|
34
|
+
name: binding.type === "Identifier" ? binding.name : text.slice(binding.start, binding.end),
|
|
35
|
+
has_default, rest, destructured: binding.type !== "Identifier",
|
|
36
|
+
};
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
catch { /* Native/bound functions and unsupported syntax stay opaque. */ }
|
|
40
|
+
}
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
/** Protocol version this build speaks. Must match `PROTOCOL_VERSION` in protocol.rs. */
|
|
24
24
|
export declare const PROTOCOL_VERSION = 1;
|
|
25
25
|
/** Methods either side can send. */
|
|
26
|
-
export type Method = "hello" | "load" | "loadTests" | "fingerprint" | "bootstrap" | "interactiveSetup" | "run" | "eval" | "setRecordSecrets" | "teardown" | "shutdown" | "createArtifact" | "completeArtifact" | "progress" | "step";
|
|
26
|
+
export type Method = "hello" | "load" | "loadTests" | "fingerprint" | "bootstrap" | "interactiveSetup" | "run" | "fakeFunctions" | "eval" | "setRecordSecrets" | "teardown" | "shutdown" | "createArtifact" | "completeArtifact" | "progress" | "step";
|
|
27
27
|
/** Methods this build knows how to dispatch. A method outside this set still
|
|
28
28
|
* *parses* — it is answered with an error, never dropped. */
|
|
29
29
|
export declare const KNOWN_METHODS: ReadonlySet<string>;
|
package/dist/harness/protocol.js
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
export type { Phone, PhoneOptions, CallOptions, SpeechOptions } from "./phone.js";
|
|
2
2
|
import type { Phone, PhoneOptions } from "./phone.js";
|
|
3
3
|
import type { Android, AndroidApp } from "./android.js";
|
|
4
|
+
import { type InteractiveEntrypoint, type InteractiveEntrypoints } from "./interactive.js";
|
|
5
|
+
export type { InteractiveEntrypoint, InteractiveEntrypoints } from "./interactive.js";
|
|
4
6
|
export type { Android, AndroidApp, AndroidElement, AndroidRecordingOptions } from "./android.js";
|
|
5
7
|
import { strict as nodeAssert } from "node:assert";
|
|
6
8
|
export type { Carrier, Wrapped, WrappedObject, WrappedArray, WrappedHeaders, WrappedBuffer, WrappedBinaryView, WrappedBytes, WrappedBlob, WrappedFile, WrappedFormData, WrappedStream, WrappedReader, WrappedBYOBReader, WrappedReadResult, WrappedResponse, Provenanced, SpectestFetch, SpectestRequestInit, Unwrap, } from "./inspect.js";
|
|
@@ -69,6 +71,11 @@ export interface EnvironmentConfig<S extends ServicesMap = ServicesMap> {
|
|
|
69
71
|
services: S;
|
|
70
72
|
/** Sandbox timeout in seconds (default 1h). */
|
|
71
73
|
timeoutSecs?: number;
|
|
74
|
+
/** Display metadata only; entrypoints retain their author-declared order. */
|
|
75
|
+
interactive?: {
|
|
76
|
+
entrypoints: InteractiveEntrypoint[];
|
|
77
|
+
instructions?: string;
|
|
78
|
+
};
|
|
72
79
|
}
|
|
73
80
|
/**
|
|
74
81
|
* The shape that ends up in the JSON config the control plane and Rust
|
|
@@ -1717,6 +1724,15 @@ export interface EnvironmentInput<SI extends InputServicesMap, F extends FakesMa
|
|
|
1717
1724
|
/** Services and/or service groups, keyed by name. See
|
|
1718
1725
|
* {@link EnvironmentConfig.services} and {@link ServiceGroup}. */
|
|
1719
1726
|
services: SI;
|
|
1727
|
+
/** Links highlighted in the dashboard and PR previews, in list order.
|
|
1728
|
+
* URL entries target declared service hostnames; terminal entries name services. */
|
|
1729
|
+
interactive?: {
|
|
1730
|
+
entrypoints?: InteractiveEntrypoints<keyof NoInfer<ExpandServices<SI>> & string>;
|
|
1731
|
+
/** Concise, direct plain-text sign-in instructions shown in the dashboard
|
|
1732
|
+
* and a collapsed PR comment section. Example: "Sign in as demo@example.com
|
|
1733
|
+
* and use 000000 as the code". */
|
|
1734
|
+
instructions?: string;
|
|
1735
|
+
};
|
|
1720
1736
|
/** Sandbox timeout in seconds (default 1h). */
|
|
1721
1737
|
timeoutSecs?: number;
|
|
1722
1738
|
/** Fake servers — see {@link defineFake}. Keyed by stable name; the key
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { resolveEntrypoints } from "./interactive.js";
|
|
1
2
|
// Spectest SDK. The user's single `spectest/index.ts` calls
|
|
2
3
|
// `defineEnvironment({ name, services })` once, defines test cases via
|
|
3
4
|
// the returned `env.test(...)`, and default-exports `env.project([...])`.
|
|
@@ -422,10 +423,9 @@ export function defineFake(opts) {
|
|
|
422
423
|
* produces the file's default export.
|
|
423
424
|
*/
|
|
424
425
|
export function defineEnvironment(input) {
|
|
425
|
-
// Split fakes off the wire config
|
|
426
|
-
//
|
|
427
|
-
|
|
428
|
-
const { fakes, services: rawServices, ...rest } = input;
|
|
426
|
+
// Split fakes off the wire config, expand service groups, and validate
|
|
427
|
+
// interactive destinations against the final service names.
|
|
428
|
+
const { fakes, interactive, services: rawServices, ...rest } = input;
|
|
429
429
|
const expanded = expandServiceGroups(rawServices);
|
|
430
430
|
for (const [key, svc] of Object.entries(expanded)) {
|
|
431
431
|
for (const build of svc.imageBuilds ?? [])
|
|
@@ -440,6 +440,15 @@ export function defineEnvironment(input) {
|
|
|
440
440
|
services: expanded,
|
|
441
441
|
};
|
|
442
442
|
validateEnvironmentConfig(config);
|
|
443
|
+
if (interactive?.instructions !== undefined && typeof interactive.instructions !== "string") {
|
|
444
|
+
throw new Error("interactive.instructions must be a string");
|
|
445
|
+
}
|
|
446
|
+
if (interactive?.entrypoints !== undefined || interactive?.instructions !== undefined) {
|
|
447
|
+
config.interactive = {
|
|
448
|
+
entrypoints: interactive.entrypoints === undefined ? [] : resolveEntrypoints(interactive.entrypoints, config.services),
|
|
449
|
+
...(interactive.instructions === undefined ? {} : { instructions: interactive.instructions }),
|
|
450
|
+
};
|
|
451
|
+
}
|
|
443
452
|
if (fakes)
|
|
444
453
|
validateFakes(config, fakes);
|
|
445
454
|
// Every test created via this env's `.test(...)` is registered here. A
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** One destination highlighted in interactive environments and PR previews. */
|
|
2
|
+
export type InteractiveEntrypoint<Service extends string = string> = {
|
|
3
|
+
name: string;
|
|
4
|
+
} & ({
|
|
5
|
+
url: string;
|
|
6
|
+
terminal?: never;
|
|
7
|
+
} | {
|
|
8
|
+
terminal: Service;
|
|
9
|
+
url?: never;
|
|
10
|
+
});
|
|
11
|
+
/** Display order is exactly the order of this list. */
|
|
12
|
+
export type InteractiveEntrypoints<Service extends string = string> = readonly InteractiveEntrypoint<Service>[];
|
|
13
|
+
export declare function resolveEntrypoints(entries: InteractiveEntrypoints, services: Record<string, {
|
|
14
|
+
hostnames?: readonly string[];
|
|
15
|
+
tls?: readonly {
|
|
16
|
+
hostname: string;
|
|
17
|
+
}[];
|
|
18
|
+
}>): InteractiveEntrypoint[];
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export function resolveEntrypoints(entries, services) {
|
|
2
|
+
if (!Array.isArray(entries))
|
|
3
|
+
throw new Error("interactive.entrypoints must be a list");
|
|
4
|
+
const names = new Set();
|
|
5
|
+
return entries.map((entry, index) => {
|
|
6
|
+
const fail = (reason) => { throw new Error(`interactive.entrypoints[${index}]: ${reason}`); };
|
|
7
|
+
if (!entry || typeof entry.name !== "string" || !entry.name.trim())
|
|
8
|
+
fail("name must be a non-empty string");
|
|
9
|
+
const { name } = entry;
|
|
10
|
+
if (names.has(name))
|
|
11
|
+
fail(`duplicate name ${JSON.stringify(name)}`);
|
|
12
|
+
names.add(name);
|
|
13
|
+
if ((entry.url !== undefined) === (entry.terminal !== undefined))
|
|
14
|
+
fail("specify exactly one of url or terminal");
|
|
15
|
+
if (entry.terminal !== undefined) {
|
|
16
|
+
if (typeof entry.terminal !== "string" || !Object.hasOwn(services, entry.terminal)) {
|
|
17
|
+
fail("terminal must name a declared service");
|
|
18
|
+
}
|
|
19
|
+
return { name, terminal: entry.terminal };
|
|
20
|
+
}
|
|
21
|
+
if (typeof entry.url !== "string")
|
|
22
|
+
return fail("expected an absolute HTTP or HTTPS URL");
|
|
23
|
+
let url;
|
|
24
|
+
try {
|
|
25
|
+
url = new URL(entry.url);
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return fail("expected an absolute HTTP or HTTPS URL");
|
|
29
|
+
}
|
|
30
|
+
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.port) {
|
|
31
|
+
fail("use an HTTP or HTTPS service URL without credentials or a non-default port");
|
|
32
|
+
}
|
|
33
|
+
const declared = Object.values(services).some((service) => [...(service.hostnames ?? []), ...(service.tls ?? []).map((tls) => tls.hostname)]
|
|
34
|
+
.some((host) => !host.startsWith("*.") && host.toLowerCase() === url.hostname));
|
|
35
|
+
if (!declared)
|
|
36
|
+
fail(`hostname ${JSON.stringify(url.hostname)} is not a declared service endpoint`);
|
|
37
|
+
return { name, url: url.href };
|
|
38
|
+
});
|
|
39
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@specific.dev/spectest",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.114.0",
|
|
4
4
|
"description": "Spectest SDK for defining test environments in TypeScript.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@kubernetes/client-node": "^1.0.0",
|
|
57
57
|
"@xterm/headless": "^5.5.0",
|
|
58
|
+
"acorn": "8.18.0",
|
|
58
59
|
"dns-packet": "^5.6.1",
|
|
59
60
|
"playwright-core": "^1.61.1",
|
|
60
61
|
"ws": "^8.20.1"
|
package/src/daemon.ts
CHANGED
|
@@ -93,6 +93,7 @@ import {
|
|
|
93
93
|
unionDockerignore,
|
|
94
94
|
} from "./harness/build-context.js";
|
|
95
95
|
import { startServices } from "./harness/service-graph.js";
|
|
96
|
+
import { fakeFunctions, fakeFunctionParameters, callFakeFunction } from "./harness/fake-functions.js";
|
|
96
97
|
import { dispatchFake } from "./harness/fake-websocket.js";
|
|
97
98
|
import {
|
|
98
99
|
casesMetadata as catalogueCases,
|
|
@@ -6332,6 +6333,19 @@ export function harnessMethods(state: RouteState): MethodTable {
|
|
|
6332
6333
|
() => runProjectSetup("setupInteractive"),
|
|
6333
6334
|
),
|
|
6334
6335
|
|
|
6336
|
+
fakeFunctions: async (params) => exclusive(state, "inFlightTest", "a test or eval is already running", async () => {
|
|
6337
|
+
const name = requireString(params, "fake");
|
|
6338
|
+
const fake = FAKES.get(name);
|
|
6339
|
+
if (!fake) throw notFound(`Unknown fake: ${name}`);
|
|
6340
|
+
await ensureFakeHelpers(name);
|
|
6341
|
+
if (params.member === undefined) return { functions: fakeFunctions(fake.helpers!), parameters: fakeFunctionParameters(fake.helpers!) };
|
|
6342
|
+
const member = requireString(params, "member");
|
|
6343
|
+
if (params.expressions !== undefined)
|
|
6344
|
+
return callFakeFunction(fake.helpers!, member, [], [], params.expressions as string[]);
|
|
6345
|
+
if (!Array.isArray(params.args)) throw badRequest("args must be a JSON array");
|
|
6346
|
+
return callFakeFunction(fake.helpers!, member, params.args, params.omitted_args as number[] | undefined);
|
|
6347
|
+
}),
|
|
6348
|
+
|
|
6335
6349
|
eval: async (params) => {
|
|
6336
6350
|
const code = requireString(params, "code");
|
|
6337
6351
|
return exclusive(state, "inFlightTest", "a test or eval is already running", () => {
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
|
|
6
|
+
test("daemon dispatch discovers cached helpers, preserves annotations and excludes concurrent evals", async () => {
|
|
7
|
+
const root = await mkdtemp(join(tmpdir(), "spectest-fake-functions-"));
|
|
8
|
+
try {
|
|
9
|
+
await mkdir(join(root, "spectest"));
|
|
10
|
+
await writeFile(join(root, "spectest/index.ts"), `
|
|
11
|
+
import { defineEnvironment, defineFake, annotate } from ${JSON.stringify(new URL("./index.ts", import.meta.url).href)};
|
|
12
|
+
const env = defineEnvironment({ name: "helpers", services: {}, fakes: {
|
|
13
|
+
mail: defineFake({ name: "mail", hostnames: ["mail.test"], handler: () => new Response("ok"), helpers: () => {
|
|
14
|
+
let count = 0;
|
|
15
|
+
return {
|
|
16
|
+
receipt(amount) { count += amount; return annotate({ subject: "Receipt " + count }, "email"); },
|
|
17
|
+
async wait() { await globalThis.gate; return count; },
|
|
18
|
+
};
|
|
19
|
+
} }),
|
|
20
|
+
}});
|
|
21
|
+
export default env.project({});
|
|
22
|
+
`);
|
|
23
|
+
const script = `
|
|
24
|
+
import { strict as assert } from "node:assert";
|
|
25
|
+
import { METHODS } from ${JSON.stringify(new URL("./daemon.ts", import.meta.url).href)};
|
|
26
|
+
await METHODS.load({});
|
|
27
|
+
{
|
|
28
|
+
const discovery = await METHODS.fakeFunctions({fake:"mail"});
|
|
29
|
+
assert.deepEqual(discovery.functions, ["receipt", "wait"]);
|
|
30
|
+
assert.deepEqual(discovery.parameters.receipt, [{ name: "amount", has_default: false, rest: false, destructured: false }]);
|
|
31
|
+
assert.deepEqual(discovery.parameters.wait, []);
|
|
32
|
+
}
|
|
33
|
+
await assert.rejects(METHODS.fakeFunctions({fake:"missing"}), /Unknown fake/);
|
|
34
|
+
await assert.rejects(METHODS.fakeFunctions({fake:"mail", member:"receipt", args:{}}), /JSON array/);
|
|
35
|
+
for (const count of [1, 2]) {
|
|
36
|
+
const result = await METHODS.fakeFunctions({fake:"mail", member:"receipt", expressions:["2 - 1"]});
|
|
37
|
+
assert.equal(result.result.subject, "Receipt " + count);
|
|
38
|
+
assert.equal(result.annotation.kind, "email");
|
|
39
|
+
assert.equal(result.annotation.message.subject, result.result.subject);
|
|
40
|
+
}
|
|
41
|
+
let release;
|
|
42
|
+
globalThis.gate = new Promise(r => release = r);
|
|
43
|
+
const pending = METHODS.fakeFunctions({fake:"mail", member:"wait", expressions:[]});
|
|
44
|
+
await assert.rejects(METHODS.eval({code:"export default 1"}), /already running/);
|
|
45
|
+
await assert.rejects(METHODS.fakeFunctions({fake:"mail"}), /already running/);
|
|
46
|
+
release();
|
|
47
|
+
assert.equal((await pending).result, 2);
|
|
48
|
+
{
|
|
49
|
+
const discovery = await METHODS.fakeFunctions({fake:"mail"});
|
|
50
|
+
assert.deepEqual(discovery.functions, ["receipt", "wait"]);
|
|
51
|
+
assert.deepEqual(discovery.parameters.receipt, [{ name: "amount", has_default: false, rest: false, destructured: false }]);
|
|
52
|
+
assert.deepEqual(discovery.parameters.wait, []);
|
|
53
|
+
}
|
|
54
|
+
process.exit(0);
|
|
55
|
+
`;
|
|
56
|
+
const child = Bun.spawn([process.execPath, "--eval", script], {
|
|
57
|
+
env: { ...process.env, SPECTEST_APP_DIR: root }, stdout: "pipe", stderr: "pipe",
|
|
58
|
+
});
|
|
59
|
+
const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
|
|
60
|
+
expect(code, stdout + stderr).toBe(0);
|
|
61
|
+
} finally { await rm(root, { recursive: true, force: true }); }
|
|
62
|
+
}, 20000);
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { annotate } from "../annotate.js";
|
|
3
|
+
import { fakeFunctions, callFakeFunction } from "./fake-functions.js";
|
|
4
|
+
|
|
5
|
+
test("discovery includes only own data functions and never runs getters", async () => {
|
|
6
|
+
const helpers = Object.create({ inherited() { throw new Error("no"); } });
|
|
7
|
+
Object.defineProperties(helpers, {
|
|
8
|
+
getter: { get() { throw new Error("no"); } },
|
|
9
|
+
value: { value: 42 },
|
|
10
|
+
own: { value() {} },
|
|
11
|
+
});
|
|
12
|
+
expect(fakeFunctions(helpers)).toEqual(["own"]);
|
|
13
|
+
for (const name of ["getter", "inherited", "constructor", "toString", "missing", "value"])
|
|
14
|
+
await expect(callFakeFunction(helpers, name, [])).rejects.toThrow("Unknown fake function");
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("calls preserve receiver, arguments, live state, and async return values", async () => {
|
|
18
|
+
const helpers = { count: 0, async add(value: { amount: number }) { this.count += value.amount; return this.count; } };
|
|
19
|
+
expect((await callFakeFunction(helpers, "add", [{ amount: 3 }])).result).toBe(3);
|
|
20
|
+
expect((await callFakeFunction(helpers, "add", [{ amount: 2 }])).result).toBe(5);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test("email and chat annotations survive separately from the raw value", async () => {
|
|
24
|
+
const value = { subject: "Receipt", html: "<p>Paid</p>" };
|
|
25
|
+
const helpers = {
|
|
26
|
+
email: () => annotate(value, "email"),
|
|
27
|
+
chat: async () => annotate({ original: 1 }, "chat", [{ side: "other", text: "Hello" }]),
|
|
28
|
+
empty() {},
|
|
29
|
+
fail() { throw new Error("declined"); },
|
|
30
|
+
async reject() { throw new Error("offline"); },
|
|
31
|
+
};
|
|
32
|
+
const email = await callFakeFunction(helpers, "email", []);
|
|
33
|
+
expect(email.result).toEqual(value);
|
|
34
|
+
expect(email.annotation).toEqual({ kind: "email", message: value });
|
|
35
|
+
const chat = await callFakeFunction(helpers, "chat", []);
|
|
36
|
+
expect(chat.result).toEqual({ original: 1 });
|
|
37
|
+
expect(chat.annotation?.kind).toBe("chat");
|
|
38
|
+
expect((await callFakeFunction(helpers, "empty", [])).result).toBeNull();
|
|
39
|
+
expect((await callFakeFunction(helpers, "fail", [])).error).toBe("declined");
|
|
40
|
+
expect((await callFakeFunction(helpers, "reject", [])).error).toBe("offline");
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("evaluates JavaScript expressions with defaults, rest and non-JSON values", async () => {
|
|
44
|
+
const helpers = {
|
|
45
|
+
values(label = "default", ...items: unknown[]) { return [label, ...items]; },
|
|
46
|
+
inspect(date: Date, callback: (n: number) => number) { return [date.getUTCFullYear(), callback(4)]; },
|
|
47
|
+
count(...items: unknown[]) { return items.length; },
|
|
48
|
+
};
|
|
49
|
+
const call = (member: string, expressions: string[]) => callFakeFunction(helpers, member, [], [], expressions);
|
|
50
|
+
expect((await call("values", ["'123'", "[1 + 2, { ok: true }, null]"])).result).toEqual(["123", 3, { ok: true }, null]);
|
|
51
|
+
expect((await call("values", ["", "[false]"])).result).toEqual(["default", false]);
|
|
52
|
+
expect((await call("values", ["undefined", ""])).result).toEqual(["default"]);
|
|
53
|
+
expect((await call("inspect", ["new Date('2026-01-01')", "n => n * 2"])).result).toEqual([2026, 8]);
|
|
54
|
+
expect((await call("count", [""])).result).toBe(0);
|
|
55
|
+
expect((await call("values", ["'x'", "42"])).error).toContain("items: Enter an array expression");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("rejects invalid expressions before evaluating fields or invoking a helper", async () => {
|
|
59
|
+
let calls = 0;
|
|
60
|
+
const helpers = { send(to: unknown, body: unknown) { calls++; return [to, body]; } };
|
|
61
|
+
const globals = globalThis as typeof globalThis & { expressionCalls?: number };
|
|
62
|
+
globals.expressionCalls = 0;
|
|
63
|
+
try {
|
|
64
|
+
const call = (expressions: string[]) => callFakeFunction(helpers, "send", [], [], expressions);
|
|
65
|
+
expect((await call(["++globalThis.expressionCalls", "{broken"])).error).toContain("body:");
|
|
66
|
+
expect(globals.expressionCalls).toBe(0);
|
|
67
|
+
expect((await call(["1); globalThis.expressionCalls++; (2", "''"])).error).toContain("to:");
|
|
68
|
+
expect((await call(["unquotedMissingName", "''"])).error).toContain("to:");
|
|
69
|
+
expect(calls).toBe(0);
|
|
70
|
+
await expect(callFakeFunction(helpers, "missing", [], [], ["++globalThis.expressionCalls"])).rejects.toThrow("Unknown fake function");
|
|
71
|
+
expect(globals.expressionCalls).toBe(0);
|
|
72
|
+
expect((await call(["++globalThis.expressionCalls", "'hello' // trailing comment"])).result).toEqual([1, "hello"]);
|
|
73
|
+
expect(globals.expressionCalls).toBe(1);
|
|
74
|
+
expect(calls).toBe(1);
|
|
75
|
+
expect((await callFakeFunction({ max: Math.max }, "max", [], [], ["1 + 2", "10"])).result).toBe(10);
|
|
76
|
+
} finally { delete globals.expressionCalls; }
|
|
77
|
+
});
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { parse } from "acorn";
|
|
2
|
+
import { functionParameters } from "./function-parameters.js";
|
|
3
|
+
import { readAnnotation } from "../annotate.js";
|
|
4
|
+
import { deepUnwrap } from "../inspect.js";
|
|
5
|
+
import { safeSerialize } from "../recorder.js";
|
|
6
|
+
import { runUninstrumented } from "./instrumentation-scope.js";
|
|
7
|
+
|
|
8
|
+
/** Only own data properties are callable. Discovery never invokes getters. */
|
|
9
|
+
export function fakeFunctions(helpers: Record<string, unknown>): string[] {
|
|
10
|
+
return Object.getOwnPropertyNames(helpers).filter((name) =>
|
|
11
|
+
typeof Object.getOwnPropertyDescriptor(helpers, name)?.value === "function",
|
|
12
|
+
).sort();
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function fakeFunctionParameters(helpers: Record<string, unknown>) {
|
|
16
|
+
return Object.fromEntries(fakeFunctions(helpers).map(name => [name,
|
|
17
|
+
functionParameters(Object.getOwnPropertyDescriptor(helpers, name)!.value),
|
|
18
|
+
]));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Use the raw helpers: the test wrappers intentionally discard annotations
|
|
22
|
+
* when there is no recorder. Preserve both the value and its presentation. */
|
|
23
|
+
export async function callFakeFunction(helpers: Record<string, unknown>, member: string, args: unknown[], omittedArgs: number[] = [], expressions?: string[]) {
|
|
24
|
+
const fn = Object.getOwnPropertyDescriptor(helpers, member)?.value;
|
|
25
|
+
if (typeof fn !== "function") throw new Error(`Unknown fake function: ${member}`);
|
|
26
|
+
if (!Array.isArray(omittedArgs) || omittedArgs.some(i => !Number.isInteger(i) || i < 0 || i >= args.length || args[i] !== null))
|
|
27
|
+
throw new Error("omitted_args must contain indices of null argument placeholders");
|
|
28
|
+
const omitted = new Set(omittedArgs);
|
|
29
|
+
const actualArgs = args.map((value, i) => omitted.has(i) ? undefined : value);
|
|
30
|
+
const start = Date.now();
|
|
31
|
+
try {
|
|
32
|
+
const value = await runUninstrumented(() => Reflect.apply(fn, helpers,
|
|
33
|
+
expressions === undefined ? actualArgs : evaluateArguments(fn, expressions)));
|
|
34
|
+
const annotated = readAnnotation(value);
|
|
35
|
+
return {
|
|
36
|
+
result: safeSerialize(deepUnwrap(annotated ? annotated.value : value)) ?? null,
|
|
37
|
+
annotation: annotated?.annotation ?? null,
|
|
38
|
+
error: null,
|
|
39
|
+
duration_ms: Date.now() - start,
|
|
40
|
+
};
|
|
41
|
+
} catch (error) {
|
|
42
|
+
return { result: null, annotation: null,
|
|
43
|
+
error: error instanceof Error ? error.message : String(error), duration_ms: Date.now() - start };
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Expressions execute only in the environment, never in either dashboard.
|
|
48
|
+
* Compile every field before evaluating any, so a syntax error cannot partially
|
|
49
|
+
* execute the call. Keep actual JS values (undefined, Date, functions, etc.). */
|
|
50
|
+
function evaluateArguments(fn: Function, expressions: string[]): unknown[] {
|
|
51
|
+
if (!Array.isArray(expressions) || expressions.some(source => typeof source !== "string"))
|
|
52
|
+
throw new Error("expressions must be an array of JavaScript expressions");
|
|
53
|
+
const parameters = functionParameters(fn);
|
|
54
|
+
const label = (index: number) => parameters?.[index]?.name ?? `Argument ${index + 1}`;
|
|
55
|
+
const withLabel = <T>(index: number, action: () => T): T => {
|
|
56
|
+
try { return action(); }
|
|
57
|
+
catch (error) { throw new Error(`${label(index)}: ${error instanceof Error ? error.message : String(error)}`); }
|
|
58
|
+
};
|
|
59
|
+
const compiled = expressions.map((source, index) => withLabel(index, () => {
|
|
60
|
+
if (!source.trim()) return null;
|
|
61
|
+
const expression = `(${source}\n)`;
|
|
62
|
+
const program = parse(expression, { ecmaVersion: "latest" });
|
|
63
|
+
if (program.body.length !== 1 || program.body[0].type !== "ExpressionStatement")
|
|
64
|
+
throw new Error("Enter a single JavaScript expression.");
|
|
65
|
+
return new Function(`"use strict"; return ${expression};`);
|
|
66
|
+
}));
|
|
67
|
+
// Preserve interior omissions but avoid inventing trailing arguments.
|
|
68
|
+
while (compiled.length && compiled.at(-1) === null) compiled.pop();
|
|
69
|
+
const args: unknown[] = [];
|
|
70
|
+
compiled.forEach((evaluate, index) => withLabel(index, () => {
|
|
71
|
+
const value = evaluate ? evaluate() : undefined;
|
|
72
|
+
if (parameters?.[index]?.rest) {
|
|
73
|
+
if (!Array.isArray(value)) throw new Error("Enter an array expression for the remaining arguments.");
|
|
74
|
+
args.push(...value);
|
|
75
|
+
} else args.push(value);
|
|
76
|
+
}));
|
|
77
|
+
return args;
|
|
78
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { functionParameters } from "./function-parameters.js";
|
|
3
|
+
import { callFakeFunction, fakeFunctionParameters } from "./fake-functions.js";
|
|
4
|
+
|
|
5
|
+
const p = (name: string, has_default = false, rest = false, destructured = false) => ({ name, has_default, rest, destructured });
|
|
6
|
+
|
|
7
|
+
test("reads runtime names from methods, arrows, async and generator functions", () => {
|
|
8
|
+
const methods = { send(to: string, subject: string, text: string) {}, async load(id: number) {}, *items(limit = 3) {} };
|
|
9
|
+
expect(fakeFunctionParameters(methods)).toEqual({ send: [p("to"), p("subject"), p("text")], load: [p("id")], items: [p("limit", true)] });
|
|
10
|
+
expect(functionParameters(value => value)).toEqual([p("value")]);
|
|
11
|
+
expect(functionParameters(async value => value)).toEqual([p("value")]);
|
|
12
|
+
expect(functionParameters(function named(ångström: string) {})).toEqual([p("ångström")]);
|
|
13
|
+
expect(functionParameters(() => 42)).toEqual([]);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test("handles nested defaults, commas, comments and patterns without evaluating anything", () => {
|
|
17
|
+
let calls = 0;
|
|
18
|
+
const fn = (first = (++calls, /[,)]/), /* not,a,param */ second = (x = [1, 2]) => x,
|
|
19
|
+
third = `a,${[1, 2].join(",")}`, { to, subject: title } = { to: "a", subject: "b" }, [a, b] = [1, 2], ...rest: unknown[]) => {};
|
|
20
|
+
const result = functionParameters(fn)!;
|
|
21
|
+
expect(result.map(v => [v.has_default, v.rest, v.destructured])).toEqual([
|
|
22
|
+
[true, false, false], [true, false, false], [true, false, false], [true, false, true], [true, false, true], [false, true, false],
|
|
23
|
+
]);
|
|
24
|
+
expect(result.slice(0, 3).map(v => v.name)).toEqual(["first", "second", "third"]);
|
|
25
|
+
expect(result[3].name).toContain("to");
|
|
26
|
+
expect(result[3].name).toContain("subject");
|
|
27
|
+
expect(result[4].name).toMatch(/^\[/);
|
|
28
|
+
expect(result[5].name).toBe("rest");
|
|
29
|
+
expect(calls).toBe(0);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test("opaque functions stay unknown and user-supplied toString is never called", () => {
|
|
33
|
+
const fn = (name: string) => name;
|
|
34
|
+
Object.defineProperty(fn, "toString", { get() { throw new Error("do not inspect"); } });
|
|
35
|
+
expect(functionParameters(fn)).toEqual([p("name")]);
|
|
36
|
+
expect(functionParameters(fn.bind(null))).toBeNull();
|
|
37
|
+
expect(functionParameters(Math.max)).toBeNull();
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("omitted slots trigger defaults while explicit null and later arguments are preserved", async () => {
|
|
41
|
+
const helpers = { send(to = "default", text: unknown, ...extra: unknown[]) { return [to, text, ...extra]; } };
|
|
42
|
+
expect((await callFakeFunction(helpers, "send", [null, "123", false, { ok: true }], [0])).result).toEqual(["default", "123", false, { ok: true }]);
|
|
43
|
+
expect((await callFakeFunction(helpers, "send", [null, "text"])).result).toEqual([null, "text"]);
|
|
44
|
+
for (const omitted of [[-1], [2], [0.5], [1]]) await expect(callFakeFunction(helpers, "send", [null, "text"], omitted)).rejects.toThrow("omitted_args");
|
|
45
|
+
});
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { parse, type Node, type Expression, type Pattern, type FunctionExpression, type ArrowFunctionExpression } from "acorn";
|
|
2
|
+
|
|
3
|
+
export interface FunctionParameter {
|
|
4
|
+
name: string;
|
|
5
|
+
has_default: boolean;
|
|
6
|
+
rest: boolean;
|
|
7
|
+
destructured: boolean;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** Parse runtime JavaScript, never execute it (including default expressions).
|
|
11
|
+
* Null means opaque/unparseable, while [] means a known zero-argument helper. */
|
|
12
|
+
export function functionParameters(fn: Function): FunctionParameter[] | null {
|
|
13
|
+
const source = Function.prototype.toString.call(fn);
|
|
14
|
+
if (source.length > 256_000) return null;
|
|
15
|
+
// Function/arrow expressions and object methods have different grammar.
|
|
16
|
+
for (const method of [false, true]) {
|
|
17
|
+
const text = method ? `({${source}})` : `(${source})`;
|
|
18
|
+
try {
|
|
19
|
+
const program = parse(text, { ecmaVersion: "latest", sourceType: "module" });
|
|
20
|
+
const statement = program.body[0];
|
|
21
|
+
if (statement?.type !== "ExpressionStatement") continue;
|
|
22
|
+
let expression: Node = statement.expression;
|
|
23
|
+
if (method) {
|
|
24
|
+
if (expression.type !== "ObjectExpression") continue;
|
|
25
|
+
const properties = (expression as Extract<Expression, { type: "ObjectExpression" }>).properties;
|
|
26
|
+
if (properties.length !== 1 || properties[0]?.type !== "Property") continue;
|
|
27
|
+
expression = properties[0].value;
|
|
28
|
+
}
|
|
29
|
+
if (expression.type !== "FunctionExpression" && expression.type !== "ArrowFunctionExpression") continue;
|
|
30
|
+
return (expression as FunctionExpression | ArrowFunctionExpression).params.map((parameter) => {
|
|
31
|
+
const rest = parameter.type === "RestElement";
|
|
32
|
+
let binding: Pattern = rest ? parameter.argument : parameter;
|
|
33
|
+
const has_default = binding.type === "AssignmentPattern";
|
|
34
|
+
if (binding.type === "AssignmentPattern") binding = binding.left;
|
|
35
|
+
return {
|
|
36
|
+
name: binding.type === "Identifier" ? binding.name : text.slice(binding.start, binding.end),
|
|
37
|
+
has_default, rest, destructured: binding.type !== "Identifier",
|
|
38
|
+
};
|
|
39
|
+
});
|
|
40
|
+
} catch { /* Native/bound functions and unsupported syntax stay opaque. */ }
|
|
41
|
+
}
|
|
42
|
+
return null;
|
|
43
|
+
}
|
package/src/harness/protocol.ts
CHANGED
|
@@ -34,6 +34,7 @@ export type Method =
|
|
|
34
34
|
| "bootstrap"
|
|
35
35
|
| "interactiveSetup"
|
|
36
36
|
| "run"
|
|
37
|
+
| "fakeFunctions"
|
|
37
38
|
| "eval"
|
|
38
39
|
| "setRecordSecrets"
|
|
39
40
|
| "teardown"
|
|
@@ -54,6 +55,7 @@ export const KNOWN_METHODS: ReadonlySet<string> = new Set<Method>([
|
|
|
54
55
|
"bootstrap",
|
|
55
56
|
"interactiveSetup",
|
|
56
57
|
"run",
|
|
58
|
+
"fakeFunctions",
|
|
57
59
|
"eval",
|
|
58
60
|
"setRecordSecrets",
|
|
59
61
|
"teardown",
|
package/src/index.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
export type { Phone, PhoneOptions, CallOptions, SpeechOptions } from "./phone.js";
|
|
2
2
|
import type { Phone, PhoneOptions } from "./phone.js";
|
|
3
3
|
import type { Android, AndroidApp } from "./android.js";
|
|
4
|
+
import { resolveEntrypoints, type InteractiveEntrypoint, type InteractiveEntrypoints } from "./interactive.js";
|
|
5
|
+
export type { InteractiveEntrypoint, InteractiveEntrypoints } from "./interactive.js";
|
|
4
6
|
export type { Android, AndroidApp, AndroidElement, AndroidRecordingOptions } from "./android.js";
|
|
5
7
|
// Spectest SDK. The user's single `spectest/index.ts` calls
|
|
6
8
|
// `defineEnvironment({ name, services })` once, defines test cases via
|
|
@@ -245,6 +247,8 @@ export interface EnvironmentConfig<S extends ServicesMap = ServicesMap> {
|
|
|
245
247
|
services: S;
|
|
246
248
|
/** Sandbox timeout in seconds (default 1h). */
|
|
247
249
|
timeoutSecs?: number;
|
|
250
|
+
/** Display metadata only; entrypoints retain their author-declared order. */
|
|
251
|
+
interactive?: { entrypoints: InteractiveEntrypoint[]; instructions?: string };
|
|
248
252
|
}
|
|
249
253
|
|
|
250
254
|
/**
|
|
@@ -2481,6 +2485,15 @@ export interface EnvironmentInput<
|
|
|
2481
2485
|
/** Services and/or service groups, keyed by name. See
|
|
2482
2486
|
* {@link EnvironmentConfig.services} and {@link ServiceGroup}. */
|
|
2483
2487
|
services: SI;
|
|
2488
|
+
/** Links highlighted in the dashboard and PR previews, in list order.
|
|
2489
|
+
* URL entries target declared service hostnames; terminal entries name services. */
|
|
2490
|
+
interactive?: {
|
|
2491
|
+
entrypoints?: InteractiveEntrypoints<keyof NoInfer<ExpandServices<SI>> & string>;
|
|
2492
|
+
/** Concise, direct plain-text sign-in instructions shown in the dashboard
|
|
2493
|
+
* and a collapsed PR comment section. Example: "Sign in as demo@example.com
|
|
2494
|
+
* and use 000000 as the code". */
|
|
2495
|
+
instructions?: string;
|
|
2496
|
+
};
|
|
2484
2497
|
/** Sandbox timeout in seconds (default 1h). */
|
|
2485
2498
|
timeoutSecs?: number;
|
|
2486
2499
|
/** Fake servers — see {@link defineFake}. Keyed by stable name; the key
|
|
@@ -2591,10 +2604,9 @@ export function defineEnvironment<
|
|
|
2591
2604
|
F extends FakesMap = FakesMap,
|
|
2592
2605
|
>(input: EnvironmentInput<SI, F>): DefinedEnvironment<ExpandServices<SI>, F> {
|
|
2593
2606
|
type S = ExpandServices<SI>;
|
|
2594
|
-
// Split fakes off the wire config
|
|
2595
|
-
//
|
|
2596
|
-
|
|
2597
|
-
const { fakes, services: rawServices, ...rest } = input;
|
|
2607
|
+
// Split fakes off the wire config, expand service groups, and validate
|
|
2608
|
+
// interactive destinations against the final service names.
|
|
2609
|
+
const { fakes, interactive, services: rawServices, ...rest } = input;
|
|
2598
2610
|
const expanded = expandServiceGroups(rawServices);
|
|
2599
2611
|
for (const [key, svc] of Object.entries(expanded)) {
|
|
2600
2612
|
for (const build of svc.imageBuilds ?? []) validateServiceImage(`${key} image ${build.reference}`, build.image);
|
|
@@ -2608,6 +2620,15 @@ export function defineEnvironment<
|
|
|
2608
2620
|
services: expanded as S,
|
|
2609
2621
|
};
|
|
2610
2622
|
validateEnvironmentConfig(config);
|
|
2623
|
+
if (interactive?.instructions !== undefined && typeof interactive.instructions !== "string") {
|
|
2624
|
+
throw new Error("interactive.instructions must be a string");
|
|
2625
|
+
}
|
|
2626
|
+
if (interactive?.entrypoints !== undefined || interactive?.instructions !== undefined) {
|
|
2627
|
+
config.interactive = {
|
|
2628
|
+
entrypoints: interactive.entrypoints === undefined ? [] : resolveEntrypoints(interactive.entrypoints, config.services),
|
|
2629
|
+
...(interactive.instructions === undefined ? {} : { instructions: interactive.instructions }),
|
|
2630
|
+
};
|
|
2631
|
+
}
|
|
2611
2632
|
if (fakes) validateFakes(config, fakes);
|
|
2612
2633
|
|
|
2613
2634
|
// Every test created via this env's `.test(...)` is registered here. A
|
|
@@ -16,7 +16,10 @@ for (const withHook of [false, true]) {
|
|
|
16
16
|
await writeFile(entry, `
|
|
17
17
|
import { defineEnvironment } from ${JSON.stringify(new URL("./index.ts", import.meta.url).href)};
|
|
18
18
|
export const calls = [];
|
|
19
|
-
const env = defineEnvironment({ name: "seed",
|
|
19
|
+
const env = defineEnvironment({ name: "seed", interactive: {
|
|
20
|
+
entrypoints: [{ name: "App shell", terminal: "app" }],
|
|
21
|
+
instructions: "Sign in as demo@example.com and use 000000 as the code",
|
|
22
|
+
}, services: {
|
|
20
23
|
app: { image: { type: "registry", reference: "busybox" },
|
|
21
24
|
helpers: () => ({ label: "typed helper" }) },
|
|
22
25
|
}});
|
|
@@ -43,7 +46,9 @@ for (const withHook of [false, true]) {
|
|
|
43
46
|
import { METHODS } from ${JSON.stringify(new URL("./daemon.ts", import.meta.url).href)};
|
|
44
47
|
import { calls } from ${JSON.stringify(entry)};
|
|
45
48
|
const nativeFetch = globalThis.fetch;
|
|
46
|
-
await METHODS.load({});
|
|
49
|
+
const loaded = await METHODS.load({});
|
|
50
|
+
assert.deepEqual(loaded.environment.interactive.entrypoints, [{ name: "App shell", terminal: "app" }]);
|
|
51
|
+
assert.equal(loaded.environment.interactive.instructions, "Sign in as demo@example.com and use 000000 as the code");
|
|
47
52
|
assert.equal((await METHODS.health({})).setupInteractive, ${withHook});
|
|
48
53
|
await METHODS.projectSetup({});
|
|
49
54
|
assert.deepEqual(calls, ["baseline"]);
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { defineEnvironment, serviceGroup, type InteractiveEntrypoints } from "./index.js";
|
|
3
|
+
|
|
4
|
+
const services = {
|
|
5
|
+
web: { image: { type: "registry" as const, reference: "nginx" }, tls: [{ hostname: "app.test", port: 80 }] },
|
|
6
|
+
api: { image: { type: "registry" as const, reference: "nginx" }, hostnames: ["api.test"] },
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
describe("interactive entrypoints", () => {
|
|
10
|
+
test("instructions survive serialization without any entrypoints", () => {
|
|
11
|
+
const instructions = "Sign in as demo@example.com and use 000000 as the code";
|
|
12
|
+
const env = defineEnvironment({ name: "app", services, interactive: { instructions } });
|
|
13
|
+
expect(JSON.parse(JSON.stringify(env.project().environment)).interactive).toEqual({ entrypoints: [], instructions });
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test("instructions must be a string", () => {
|
|
17
|
+
const invalid = () => defineEnvironment({ name: "app", services, interactive: {
|
|
18
|
+
// @ts-expect-error instructions are plain text
|
|
19
|
+
instructions: 123,
|
|
20
|
+
} });
|
|
21
|
+
expect(invalid).toThrow("interactive.instructions must be a string");
|
|
22
|
+
});
|
|
23
|
+
test("keeps mixed destinations in list order on the project wire config", () => {
|
|
24
|
+
const env = defineEnvironment({ name: "app", services, interactive: { entrypoints: [
|
|
25
|
+
{ name: "Website", url: "https://app.test/admin?tab=users#invite" },
|
|
26
|
+
{ name: "2", terminal: "api" },
|
|
27
|
+
{ name: "1", url: "http://api.test/docs" },
|
|
28
|
+
] } });
|
|
29
|
+
const config = JSON.parse(JSON.stringify(env.project().environment));
|
|
30
|
+
expect(config.interactive.entrypoints).toEqual([
|
|
31
|
+
{ name: "Website", url: "https://app.test/admin?tab=users#invite" },
|
|
32
|
+
{ name: "2", terminal: "api" },
|
|
33
|
+
{ name: "1", url: "http://api.test/docs" },
|
|
34
|
+
]);
|
|
35
|
+
expect(env.project().tests).toBeUndefined();
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("omitting entrypoints leaves existing projects unchanged", () => {
|
|
39
|
+
expect(defineEnvironment({ name: "app", services }).config.interactive).toBeUndefined();
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("terminal destinations can name expanded service-group members", () => {
|
|
43
|
+
const stack = serviceGroup({ primary: "api", services: () => ({ api: services.api, worker: services.web }) });
|
|
44
|
+
const env = defineEnvironment({ name: "app", services: { stack }, interactive: {
|
|
45
|
+
entrypoints: [{ name: "Worker shell", terminal: "stack-worker" }],
|
|
46
|
+
} });
|
|
47
|
+
expect(env.config.interactive?.entrypoints).toEqual([{ name: "Worker shell", terminal: "stack-worker" }]);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test.each([null, { App: "https://app.test" }])("requires a list: %j", (entries) => {
|
|
51
|
+
expect(() => defineEnvironment({ name: "app", services, interactive: {
|
|
52
|
+
entrypoints: entries as unknown as InteractiveEntrypoints<"web" | "api">,
|
|
53
|
+
} })).toThrow("must be a list");
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("terminal service names are checked statically and at runtime", () => {
|
|
57
|
+
const invalid = () => defineEnvironment({ name: "app", services, interactive: { entrypoints: [
|
|
58
|
+
// @ts-expect-error terminal targets must be declared services
|
|
59
|
+
{ name: "Shell", terminal: "missing" },
|
|
60
|
+
] } });
|
|
61
|
+
expect(invalid).toThrow("terminal must name a declared service");
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test.each([
|
|
65
|
+
[{ name: "App", url: "https://external.test" }],
|
|
66
|
+
[{ name: "App", url: "/relative" }],
|
|
67
|
+
[{ name: "App", url: "javascript:alert(1)" }],
|
|
68
|
+
[{ name: "App", url: "https://user:pass@app.test" }],
|
|
69
|
+
[{ name: "App", url: "https://app.test:8443" }],
|
|
70
|
+
[{ name: "App", url: "https://app.test", terminal: "web" }],
|
|
71
|
+
[{ name: "App" }],
|
|
72
|
+
[{ name: "", terminal: "web" }],
|
|
73
|
+
[{ name: "Same", terminal: "web" }, { name: "Same", terminal: "api" }],
|
|
74
|
+
])("rejects invalid destinations %j", (...entries) => {
|
|
75
|
+
expect(() => defineEnvironment({ name: "app", services, interactive: {
|
|
76
|
+
entrypoints: entries as unknown as InteractiveEntrypoints<"web" | "api">,
|
|
77
|
+
} })).toThrow("interactive.entrypoints");
|
|
78
|
+
});
|
|
79
|
+
});
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/** One destination highlighted in interactive environments and PR previews. */
|
|
2
|
+
export type InteractiveEntrypoint<Service extends string = string> = { name: string } & (
|
|
3
|
+
| { url: string; terminal?: never }
|
|
4
|
+
| { terminal: Service; url?: never }
|
|
5
|
+
);
|
|
6
|
+
|
|
7
|
+
/** Display order is exactly the order of this list. */
|
|
8
|
+
export type InteractiveEntrypoints<Service extends string = string> = readonly InteractiveEntrypoint<Service>[];
|
|
9
|
+
|
|
10
|
+
export function resolveEntrypoints(
|
|
11
|
+
entries: InteractiveEntrypoints,
|
|
12
|
+
services: Record<string, { hostnames?: readonly string[]; tls?: readonly { hostname: string }[] }>,
|
|
13
|
+
): InteractiveEntrypoint[] {
|
|
14
|
+
if (!Array.isArray(entries)) throw new Error("interactive.entrypoints must be a list");
|
|
15
|
+
const names = new Set<string>();
|
|
16
|
+
return entries.map((entry, index) => {
|
|
17
|
+
const fail = (reason: string): never => { throw new Error(`interactive.entrypoints[${index}]: ${reason}`); };
|
|
18
|
+
if (!entry || typeof entry.name !== "string" || !entry.name.trim()) fail("name must be a non-empty string");
|
|
19
|
+
const { name } = entry;
|
|
20
|
+
if (names.has(name)) fail(`duplicate name ${JSON.stringify(name)}`);
|
|
21
|
+
names.add(name);
|
|
22
|
+
if ((entry.url !== undefined) === (entry.terminal !== undefined)) fail("specify exactly one of url or terminal");
|
|
23
|
+
if (entry.terminal !== undefined) {
|
|
24
|
+
if (typeof entry.terminal !== "string" || !Object.hasOwn(services, entry.terminal)) {
|
|
25
|
+
fail("terminal must name a declared service");
|
|
26
|
+
}
|
|
27
|
+
return { name, terminal: entry.terminal };
|
|
28
|
+
}
|
|
29
|
+
if (typeof entry.url !== "string") return fail("expected an absolute HTTP or HTTPS URL");
|
|
30
|
+
let url: URL;
|
|
31
|
+
try { url = new URL(entry.url); } catch { return fail("expected an absolute HTTP or HTTPS URL"); }
|
|
32
|
+
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.port) {
|
|
33
|
+
fail("use an HTTP or HTTPS service URL without credentials or a non-default port");
|
|
34
|
+
}
|
|
35
|
+
const declared = Object.values(services).some((service) =>
|
|
36
|
+
[...(service.hostnames ?? []), ...(service.tls ?? []).map((tls) => tls.hostname)]
|
|
37
|
+
.some((host) => !host.startsWith("*.") && host.toLowerCase() === url.hostname));
|
|
38
|
+
if (!declared) fail(`hostname ${JSON.stringify(url.hostname)} is not a declared service endpoint`);
|
|
39
|
+
return { name, url: url.href };
|
|
40
|
+
});
|
|
41
|
+
}
|