@glubean/runner 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +60 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +51 -0
- package/dist/config.js.map +1 -0
- package/dist/executor.d.ts +290 -0
- package/dist/executor.d.ts.map +1 -0
- package/dist/executor.js +347 -0
- package/dist/executor.js.map +1 -0
- package/dist/harness.d.ts +9 -0
- package/dist/harness.d.ts.map +1 -0
- package/dist/harness.js +1487 -0
- package/dist/harness.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/network_budget.d.ts +9 -0
- package/dist/network_budget.d.ts.map +1 -0
- package/dist/network_budget.js +34 -0
- package/dist/network_budget.js.map +1 -0
- package/dist/network_policy.d.ts +12 -0
- package/dist/network_policy.d.ts.map +1 -0
- package/dist/network_policy.js +92 -0
- package/dist/network_policy.js.map +1 -0
- package/dist/resolve.d.ts +98 -0
- package/dist/resolve.d.ts.map +1 -0
- package/dist/resolve.js +209 -0
- package/dist/resolve.js.map +1 -0
- package/dist/thresholds.d.ts +40 -0
- package/dist/thresholds.d.ts.map +1 -0
- package/dist/thresholds.js +152 -0
- package/dist/thresholds.js.map +1 -0
- package/package.json +27 -0
package/dist/harness.js
ADDED
|
@@ -0,0 +1,1487 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Harness script - runs INSIDE the Node.js subprocess (via tsx).
|
|
3
|
+
* This is the bridge between the Runner and User Code.
|
|
4
|
+
*
|
|
5
|
+
* Usage:
|
|
6
|
+
* tsx harness.ts --testUrl=<url> --testId=<id>
|
|
7
|
+
*/
|
|
8
|
+
import { parseArgs } from "node:util";
|
|
9
|
+
import ky from "ky";
|
|
10
|
+
import { classifyHostnameBlockReason, classifyIpBlockReason, isAllowedPort, isAllowedProtocol, isIpLiteral, resolveUrlPort, } from "./network_policy.js";
|
|
11
|
+
import { applyResponseByteBudget } from "./network_budget.js";
|
|
12
|
+
import { Expectation } from "@glubean/sdk/expect";
|
|
13
|
+
// Global error handlers for async errors that escape try/catch
|
|
14
|
+
process.on("uncaughtException", (error) => {
|
|
15
|
+
console.log(JSON.stringify({
|
|
16
|
+
type: "status",
|
|
17
|
+
status: "failed",
|
|
18
|
+
error: error?.message || "Unknown error",
|
|
19
|
+
stack: error?.stack,
|
|
20
|
+
}));
|
|
21
|
+
process.exit(1);
|
|
22
|
+
});
|
|
23
|
+
process.on("unhandledRejection", (reason) => {
|
|
24
|
+
console.log(JSON.stringify({
|
|
25
|
+
type: "status",
|
|
26
|
+
status: "failed",
|
|
27
|
+
error: reason instanceof Error ? reason.message : String(reason),
|
|
28
|
+
stack: reason instanceof Error ? reason.stack : undefined,
|
|
29
|
+
}));
|
|
30
|
+
process.exit(1);
|
|
31
|
+
});
|
|
32
|
+
// Parse CLI arguments
|
|
33
|
+
const { values: args } = parseArgs({
|
|
34
|
+
args: process.argv.slice(2),
|
|
35
|
+
options: {
|
|
36
|
+
testUrl: { type: "string" },
|
|
37
|
+
testId: { type: "string" },
|
|
38
|
+
testIds: { type: "string" },
|
|
39
|
+
exportName: { type: "string" },
|
|
40
|
+
exportNames: { type: "string" },
|
|
41
|
+
emitFullTrace: { type: "boolean", default: false },
|
|
42
|
+
},
|
|
43
|
+
strict: false,
|
|
44
|
+
});
|
|
45
|
+
/** When true, auto-trace includes request/response headers and bodies. */
|
|
46
|
+
const emitFullTrace = args.emitFullTrace ?? false;
|
|
47
|
+
const testUrl = args.testUrl;
|
|
48
|
+
const testId = args.testId;
|
|
49
|
+
/**
|
|
50
|
+
* Comma-separated list of test IDs for file-level batch mode.
|
|
51
|
+
* When set, all tests run sequentially in a single process, preserving
|
|
52
|
+
* module-level state (e.g. shared `let` variables between tests).
|
|
53
|
+
*/
|
|
54
|
+
const testIds = args.testIds ? args.testIds.split(",") : undefined;
|
|
55
|
+
/** Optional export name for fallback lookup (used by test.pick/test.each). */
|
|
56
|
+
const exportName = args.exportName;
|
|
57
|
+
/** Optional testId→exportName mapping for batch mode fallback (test.pick). */
|
|
58
|
+
const exportNamesMap = {};
|
|
59
|
+
if (args.exportNames) {
|
|
60
|
+
for (const pair of args.exportNames.split(",")) {
|
|
61
|
+
const sep = pair.indexOf(":");
|
|
62
|
+
if (sep > 0) {
|
|
63
|
+
exportNamesMap[pair.slice(0, sep)] = pair.slice(sep + 1);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (!testUrl || (!testId && !testIds)) {
|
|
68
|
+
console.log(JSON.stringify({
|
|
69
|
+
type: "error",
|
|
70
|
+
message: "Missing required arguments: --testUrl and (--testId or --testIds)",
|
|
71
|
+
}));
|
|
72
|
+
process.exit(1);
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Read context data from stdin.
|
|
76
|
+
* Context is passed via stdin instead of CLI args to avoid length limits and security issues.
|
|
77
|
+
*
|
|
78
|
+
* @returns The context JSON string from stdin
|
|
79
|
+
*/
|
|
80
|
+
async function readContextFromStdin() {
|
|
81
|
+
const chunks = [];
|
|
82
|
+
for await (const chunk of process.stdin) {
|
|
83
|
+
chunks.push(chunk);
|
|
84
|
+
}
|
|
85
|
+
return Buffer.concat(chunks).toString("utf-8");
|
|
86
|
+
}
|
|
87
|
+
// Parse context data from stdin
|
|
88
|
+
const contextJson = await readContextFromStdin();
|
|
89
|
+
const contextData = contextJson ? JSON.parse(contextJson) : {};
|
|
90
|
+
const rawVars = (contextData.vars ?? {});
|
|
91
|
+
const rawSecrets = (contextData.secrets ?? {});
|
|
92
|
+
// Execution-level retry metadata injected by executor/control plane.
|
|
93
|
+
// 0 => first execution attempt.
|
|
94
|
+
const retryCount = (contextData.retryCount ?? 0);
|
|
95
|
+
function normalizeTestTags(input) {
|
|
96
|
+
if (!input)
|
|
97
|
+
return [];
|
|
98
|
+
if (Array.isArray(input))
|
|
99
|
+
return input.filter((tag) => typeof tag === "string");
|
|
100
|
+
return [input];
|
|
101
|
+
}
|
|
102
|
+
function parseRuntimeTestMetadata(input) {
|
|
103
|
+
const candidate = input && typeof input === "object" ? input : undefined;
|
|
104
|
+
const id = typeof candidate?.id === "string" ? candidate.id : (testId ?? "");
|
|
105
|
+
const tags = Array.isArray(candidate?.tags)
|
|
106
|
+
? candidate.tags.filter((tag) => typeof tag === "string")
|
|
107
|
+
: [];
|
|
108
|
+
return { id, tags };
|
|
109
|
+
}
|
|
110
|
+
const runtimeTest = parseRuntimeTestMetadata(contextData.test);
|
|
111
|
+
function parseNetworkPolicy(input) {
|
|
112
|
+
if (!input || typeof input !== "object")
|
|
113
|
+
return undefined;
|
|
114
|
+
const candidate = input;
|
|
115
|
+
if (candidate.mode !== "shared_serverless")
|
|
116
|
+
return undefined;
|
|
117
|
+
const allowedPorts = Array.isArray(candidate.allowedPorts)
|
|
118
|
+
? candidate.allowedPorts.filter((p) => typeof p === "number" && Number.isFinite(p) && p > 0 && p <= 65535).map((p) => Math.floor(p))
|
|
119
|
+
: [];
|
|
120
|
+
return {
|
|
121
|
+
mode: "shared_serverless",
|
|
122
|
+
maxRequests: Number(candidate.maxRequests) > 0 ? Math.floor(Number(candidate.maxRequests)) : 300,
|
|
123
|
+
maxConcurrentRequests: Number(candidate.maxConcurrentRequests) > 0
|
|
124
|
+
? Math.floor(Number(candidate.maxConcurrentRequests))
|
|
125
|
+
: 20,
|
|
126
|
+
requestTimeoutMs: Number(candidate.requestTimeoutMs) > 0 ? Math.floor(Number(candidate.requestTimeoutMs)) : 30_000,
|
|
127
|
+
maxResponseBytes: Number(candidate.maxResponseBytes) > 0
|
|
128
|
+
? Math.floor(Number(candidate.maxResponseBytes))
|
|
129
|
+
: 20 * 1024 * 1024,
|
|
130
|
+
allowedPorts: allowedPorts.length > 0 ? Array.from(new Set(allowedPorts)) : [80, 443, 8080, 8443],
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
const networkPolicy = parseNetworkPolicy(contextData.networkPolicy);
|
|
134
|
+
// Memory monitoring state
|
|
135
|
+
let peakMemoryBytes = 0;
|
|
136
|
+
let memoryCheckInterval;
|
|
137
|
+
// Step-level assertion tracking.
|
|
138
|
+
// Reset before each step, incremented by ctx.assert on failure.
|
|
139
|
+
let stepFailedAssertions = 0;
|
|
140
|
+
let stepAssertionTotal = 0;
|
|
141
|
+
// Current step index (null when not inside a step).
|
|
142
|
+
// Used to tag log/assertion/trace/metric events with their containing step.
|
|
143
|
+
let currentStepIndex = null;
|
|
144
|
+
// Test-level assertion and step counters.
|
|
145
|
+
// Accumulated across the entire test execution for the summary event.
|
|
146
|
+
let totalAssertions = 0;
|
|
147
|
+
let totalFailedAssertions = 0;
|
|
148
|
+
let totalSteps = 0;
|
|
149
|
+
let passedSteps = 0;
|
|
150
|
+
let failedSteps = 0;
|
|
151
|
+
let skippedSteps = 0;
|
|
152
|
+
// Warning counters — tracked separately from assertions.
|
|
153
|
+
// Warnings never affect test pass/fail status.
|
|
154
|
+
let warningTotal = 0;
|
|
155
|
+
let warningTriggered = 0;
|
|
156
|
+
// Schema validation counters.
|
|
157
|
+
let schemaValidationTotal = 0;
|
|
158
|
+
let schemaValidationFailed = 0;
|
|
159
|
+
let schemaValidationWarnings = 0;
|
|
160
|
+
/**
|
|
161
|
+
* Start monitoring memory usage.
|
|
162
|
+
* Samples memory every 100ms and tracks peak usage.
|
|
163
|
+
*/
|
|
164
|
+
function startMemoryMonitoring() {
|
|
165
|
+
const initial = process.memoryUsage();
|
|
166
|
+
peakMemoryBytes = initial.heapUsed;
|
|
167
|
+
memoryCheckInterval = setInterval(() => {
|
|
168
|
+
try {
|
|
169
|
+
const mem = process.memoryUsage();
|
|
170
|
+
peakMemoryBytes = Math.max(peakMemoryBytes, mem.heapUsed);
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
// Ignore errors during monitoring
|
|
174
|
+
}
|
|
175
|
+
}, 100);
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Stop memory monitoring and return peak usage.
|
|
179
|
+
*/
|
|
180
|
+
function stopMemoryMonitoring() {
|
|
181
|
+
if (memoryCheckInterval !== undefined) {
|
|
182
|
+
clearInterval(memoryCheckInterval);
|
|
183
|
+
memoryCheckInterval = undefined;
|
|
184
|
+
}
|
|
185
|
+
return peakMemoryBytes;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Custom error class for test skip.
|
|
189
|
+
* When thrown, the test will be marked as skipped instead of failed.
|
|
190
|
+
*/
|
|
191
|
+
class SkipError extends Error {
|
|
192
|
+
reason;
|
|
193
|
+
constructor(reason) {
|
|
194
|
+
super(reason ? `Test skipped: ${reason}` : "Test skipped");
|
|
195
|
+
this.reason = reason;
|
|
196
|
+
this.name = "SkipError";
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Sentinel error thrown by ctx.fail().
|
|
201
|
+
* Immediately aborts test execution, emitting a failed assertion before throwing.
|
|
202
|
+
*/
|
|
203
|
+
class FailError extends Error {
|
|
204
|
+
reason;
|
|
205
|
+
constructor(reason) {
|
|
206
|
+
super(reason);
|
|
207
|
+
this.reason = reason;
|
|
208
|
+
this.name = "FailError";
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Sentinel error used for step-level timeout failures.
|
|
213
|
+
* Timeouts are treated as terminal for the step and do not retry.
|
|
214
|
+
*/
|
|
215
|
+
class StepTimeoutError extends Error {
|
|
216
|
+
constructor(stepName, timeoutMs) {
|
|
217
|
+
super(`Step "${stepName}" timed out after ${timeoutMs}ms`);
|
|
218
|
+
this.name = "StepTimeoutError";
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Helper to run validator and get error message.
|
|
223
|
+
*
|
|
224
|
+
* @param result Validator result (true/false/string/void/null)
|
|
225
|
+
* @param key The variable or secret key being validated
|
|
226
|
+
* @param type Whether this is a "var" or "secret"
|
|
227
|
+
*/
|
|
228
|
+
function runValidator(result, key, type) {
|
|
229
|
+
// true, undefined, null = valid
|
|
230
|
+
if (result === true || result === undefined || result === null) {
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
// string = custom error message
|
|
234
|
+
if (typeof result === "string") {
|
|
235
|
+
throw new Error(`Invalid ${type} "${key}": ${result}`);
|
|
236
|
+
}
|
|
237
|
+
// false = generic error
|
|
238
|
+
throw new Error(`Invalid ${type} "${key}": validation failed`);
|
|
239
|
+
}
|
|
240
|
+
// ---------------------------------------------------------------------------
|
|
241
|
+
// Schema validation helper
|
|
242
|
+
// ---------------------------------------------------------------------------
|
|
243
|
+
/**
|
|
244
|
+
* Resolve a SchemaEntry to { schema, severity }.
|
|
245
|
+
*/
|
|
246
|
+
function resolveSchemaEntry(entry) {
|
|
247
|
+
if ("schema" in entry && entry.schema != null) {
|
|
248
|
+
const obj = entry;
|
|
249
|
+
return { schema: obj.schema, severity: obj.severity ?? "error" };
|
|
250
|
+
}
|
|
251
|
+
return { schema: entry, severity: "error" };
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Core schema validation logic used by both ctx.validate and HTTP hooks.
|
|
255
|
+
*
|
|
256
|
+
* Runs safeParse (preferred) or parse (fallback), emits schema_validation event,
|
|
257
|
+
* updates counters, and routes failures based on severity.
|
|
258
|
+
*
|
|
259
|
+
* Returns { success, data?, issues? }.
|
|
260
|
+
*/
|
|
261
|
+
function runSchemaValidation(data, schema, label, severity) {
|
|
262
|
+
schemaValidationTotal++;
|
|
263
|
+
let success = false;
|
|
264
|
+
let parsed;
|
|
265
|
+
let issues = [];
|
|
266
|
+
if (typeof schema.safeParse === "function") {
|
|
267
|
+
const result = schema.safeParse(data);
|
|
268
|
+
if (result.success) {
|
|
269
|
+
success = true;
|
|
270
|
+
parsed = result.data;
|
|
271
|
+
}
|
|
272
|
+
else {
|
|
273
|
+
issues = (result.error?.issues ?? []).map((i) => ({
|
|
274
|
+
message: i.message,
|
|
275
|
+
...(i.path && { path: i.path }),
|
|
276
|
+
}));
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
else if (typeof schema.parse === "function") {
|
|
280
|
+
try {
|
|
281
|
+
parsed = schema.parse(data);
|
|
282
|
+
success = true;
|
|
283
|
+
}
|
|
284
|
+
catch (err) {
|
|
285
|
+
// Try to extract structured issues from the error
|
|
286
|
+
const errAny = err;
|
|
287
|
+
if (errAny?.issues && Array.isArray(errAny.issues)) {
|
|
288
|
+
issues = errAny.issues.map((i) => ({
|
|
289
|
+
message: i.message ?? String(i),
|
|
290
|
+
...(i.path && { path: i.path }),
|
|
291
|
+
}));
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
issues = [
|
|
295
|
+
{
|
|
296
|
+
message: err instanceof Error ? err.message : String(err),
|
|
297
|
+
},
|
|
298
|
+
];
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
issues = [{ message: "Schema has neither safeParse nor parse method" }];
|
|
304
|
+
}
|
|
305
|
+
// Emit schema_validation event (always, regardless of success/severity)
|
|
306
|
+
console.log(JSON.stringify({
|
|
307
|
+
type: "schema_validation",
|
|
308
|
+
label,
|
|
309
|
+
success,
|
|
310
|
+
severity,
|
|
311
|
+
...(issues.length > 0 && { issues }),
|
|
312
|
+
...(currentStepIndex !== null && { stepIndex: currentStepIndex }),
|
|
313
|
+
}));
|
|
314
|
+
if (!success) {
|
|
315
|
+
const issuesSummary = issues
|
|
316
|
+
.map((i) => {
|
|
317
|
+
const path = i.path ? i.path.join(".") + ": " : "";
|
|
318
|
+
return path + i.message;
|
|
319
|
+
})
|
|
320
|
+
.join("; ");
|
|
321
|
+
const msg = `Schema validation failed: ${label} — ${issuesSummary}`;
|
|
322
|
+
switch (severity) {
|
|
323
|
+
case "error":
|
|
324
|
+
schemaValidationFailed++;
|
|
325
|
+
// Route through assertion pipeline so it counts as a failed assertion
|
|
326
|
+
ctx.assert(false, msg);
|
|
327
|
+
break;
|
|
328
|
+
case "warn":
|
|
329
|
+
schemaValidationWarnings++;
|
|
330
|
+
ctx.warn(false, msg);
|
|
331
|
+
break;
|
|
332
|
+
case "fatal":
|
|
333
|
+
schemaValidationFailed++;
|
|
334
|
+
// Emit failed assertion, then throw to abort
|
|
335
|
+
ctx.assert(false, msg);
|
|
336
|
+
throw new FailError(msg);
|
|
337
|
+
}
|
|
338
|
+
return { success: false, issues };
|
|
339
|
+
}
|
|
340
|
+
return { success: true, data: parsed };
|
|
341
|
+
}
|
|
342
|
+
// Helper: resolve a value from explicit context, falling back to system env.
|
|
343
|
+
// Priority: .env/.env.secrets (rawVars/rawSecrets) > system environment variable
|
|
344
|
+
function resolveValue(explicit, key) {
|
|
345
|
+
const value = explicit[key];
|
|
346
|
+
if (value !== undefined && value !== null && value !== "")
|
|
347
|
+
return value;
|
|
348
|
+
// Fallback to system environment (e.g., CI-injected vars)
|
|
349
|
+
return process.env[key] ?? undefined;
|
|
350
|
+
}
|
|
351
|
+
// Construct TestContext with streaming output
|
|
352
|
+
// (http field is attached after ky instance creation below)
|
|
353
|
+
const ctx = {
|
|
354
|
+
vars: {
|
|
355
|
+
get: (key) => resolveValue(rawVars, key),
|
|
356
|
+
require: (key, validate) => {
|
|
357
|
+
const value = resolveValue(rawVars, key);
|
|
358
|
+
if (value === undefined || value === null || value === "") {
|
|
359
|
+
throw new Error(`Missing required var: ${key}`);
|
|
360
|
+
}
|
|
361
|
+
if (validate) {
|
|
362
|
+
runValidator(validate(value), key, "var");
|
|
363
|
+
}
|
|
364
|
+
return value;
|
|
365
|
+
},
|
|
366
|
+
all: () => ({ ...rawVars }),
|
|
367
|
+
},
|
|
368
|
+
secrets: {
|
|
369
|
+
get: (key) => resolveValue(rawSecrets, key),
|
|
370
|
+
require: (key, validate) => {
|
|
371
|
+
const value = resolveValue(rawSecrets, key);
|
|
372
|
+
if (value === undefined || value === null || value === "") {
|
|
373
|
+
throw new Error(`Missing required secret: ${key}`);
|
|
374
|
+
}
|
|
375
|
+
if (validate) {
|
|
376
|
+
runValidator(validate(value), key, "secret");
|
|
377
|
+
}
|
|
378
|
+
return value;
|
|
379
|
+
},
|
|
380
|
+
},
|
|
381
|
+
// Logging function
|
|
382
|
+
log: (message, data) => {
|
|
383
|
+
console.log(JSON.stringify({
|
|
384
|
+
type: "log",
|
|
385
|
+
message,
|
|
386
|
+
data,
|
|
387
|
+
...(currentStepIndex !== null && { stepIndex: currentStepIndex }),
|
|
388
|
+
}));
|
|
389
|
+
},
|
|
390
|
+
// Assertion function with overloads
|
|
391
|
+
// Overload 1: assert(condition: boolean, message?: string, details?: AssertionDetails)
|
|
392
|
+
// Overload 2: assert(result: AssertionResultInput, message?: string)
|
|
393
|
+
assert: (arg1, arg2, arg3) => {
|
|
394
|
+
let passed;
|
|
395
|
+
let message;
|
|
396
|
+
let actual;
|
|
397
|
+
let expected;
|
|
398
|
+
if (typeof arg1 === "boolean") {
|
|
399
|
+
// Overload 1: assert(condition, message?, details?)
|
|
400
|
+
passed = arg1;
|
|
401
|
+
message = (typeof arg2 === "string" ? arg2 : undefined) ||
|
|
402
|
+
(passed ? "Assertion passed" : "Assertion failed");
|
|
403
|
+
const details = typeof arg2 === "object" ? arg2 : arg3;
|
|
404
|
+
if (details) {
|
|
405
|
+
actual = details.actual;
|
|
406
|
+
expected = details.expected;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
else {
|
|
410
|
+
// Overload 2: assert(result, message?)
|
|
411
|
+
passed = arg1.passed;
|
|
412
|
+
actual = arg1.actual;
|
|
413
|
+
expected = arg1.expected;
|
|
414
|
+
message = (typeof arg2 === "string" ? arg2 : undefined) ||
|
|
415
|
+
(passed ? "Assertion passed" : "Assertion failed");
|
|
416
|
+
}
|
|
417
|
+
// Track per-step and test-level assertion stats
|
|
418
|
+
stepAssertionTotal++;
|
|
419
|
+
totalAssertions++;
|
|
420
|
+
if (!passed) {
|
|
421
|
+
stepFailedAssertions++;
|
|
422
|
+
totalFailedAssertions++;
|
|
423
|
+
}
|
|
424
|
+
console.log(JSON.stringify({
|
|
425
|
+
type: "assertion",
|
|
426
|
+
passed,
|
|
427
|
+
message,
|
|
428
|
+
actual,
|
|
429
|
+
expected,
|
|
430
|
+
...(currentStepIndex !== null && { stepIndex: currentStepIndex }),
|
|
431
|
+
}));
|
|
432
|
+
},
|
|
433
|
+
// Fluent assertion API (Jest-style, soft-by-default)
|
|
434
|
+
expect: (actual) => {
|
|
435
|
+
return new Expectation(actual, (result) => {
|
|
436
|
+
// Route through the existing assertion pipeline
|
|
437
|
+
ctx.assert({
|
|
438
|
+
passed: result.passed,
|
|
439
|
+
actual: result.actual,
|
|
440
|
+
expected: result.expected,
|
|
441
|
+
}, result.message);
|
|
442
|
+
});
|
|
443
|
+
},
|
|
444
|
+
// Warning function — soft check, never affects test pass/fail.
|
|
445
|
+
// condition=true means OK; condition=false triggers warning.
|
|
446
|
+
warn: (condition, message) => {
|
|
447
|
+
warningTotal++;
|
|
448
|
+
if (!condition) {
|
|
449
|
+
warningTriggered++;
|
|
450
|
+
}
|
|
451
|
+
console.log(JSON.stringify({
|
|
452
|
+
type: "warning",
|
|
453
|
+
condition,
|
|
454
|
+
message,
|
|
455
|
+
...(currentStepIndex !== null && { stepIndex: currentStepIndex }),
|
|
456
|
+
}));
|
|
457
|
+
},
|
|
458
|
+
// Schema validation function
|
|
459
|
+
validate: (data, schema, label, options) => {
|
|
460
|
+
const result = runSchemaValidation(data, schema, label ?? "data", options?.severity ?? "error");
|
|
461
|
+
return result.success ? result.data : undefined;
|
|
462
|
+
},
|
|
463
|
+
// API tracing function
|
|
464
|
+
trace: (request) => {
|
|
465
|
+
console.log(JSON.stringify({
|
|
466
|
+
type: "trace",
|
|
467
|
+
data: request,
|
|
468
|
+
...(currentStepIndex !== null && { stepIndex: currentStepIndex }),
|
|
469
|
+
}));
|
|
470
|
+
// Backward compat: also emit as a typed action for timeline/filtering
|
|
471
|
+
let pathname;
|
|
472
|
+
try {
|
|
473
|
+
pathname = new URL(request.url).pathname;
|
|
474
|
+
}
|
|
475
|
+
catch {
|
|
476
|
+
pathname = request.url;
|
|
477
|
+
}
|
|
478
|
+
ctx.action({
|
|
479
|
+
category: "http:request",
|
|
480
|
+
target: `${request.method} ${pathname}`,
|
|
481
|
+
duration: request.duration,
|
|
482
|
+
status: request.status >= 400 ? "error" : "ok",
|
|
483
|
+
detail: { method: request.method, url: request.url, httpStatus: request.status },
|
|
484
|
+
});
|
|
485
|
+
},
|
|
486
|
+
// Action recording function
|
|
487
|
+
action: (a) => {
|
|
488
|
+
console.log(JSON.stringify({
|
|
489
|
+
type: "action",
|
|
490
|
+
data: a,
|
|
491
|
+
...(currentStepIndex !== null && { stepIndex: currentStepIndex }),
|
|
492
|
+
}));
|
|
493
|
+
},
|
|
494
|
+
// Structured event emission
|
|
495
|
+
event: (ev) => {
|
|
496
|
+
console.log(JSON.stringify({
|
|
497
|
+
type: "event",
|
|
498
|
+
data: ev,
|
|
499
|
+
...(currentStepIndex !== null && { stepIndex: currentStepIndex }),
|
|
500
|
+
}));
|
|
501
|
+
},
|
|
502
|
+
// Metric reporting function
|
|
503
|
+
metric: (name, value, options) => {
|
|
504
|
+
console.log(JSON.stringify({
|
|
505
|
+
type: "metric",
|
|
506
|
+
name,
|
|
507
|
+
value,
|
|
508
|
+
unit: options?.unit,
|
|
509
|
+
tags: options?.tags,
|
|
510
|
+
...(currentStepIndex !== null && { stepIndex: currentStepIndex }),
|
|
511
|
+
}));
|
|
512
|
+
},
|
|
513
|
+
/**
|
|
514
|
+
* Skip the current test with an optional reason.
|
|
515
|
+
* Throws a SkipError that will be caught and handled by the harness.
|
|
516
|
+
*
|
|
517
|
+
* @param reason Optional reason for skipping
|
|
518
|
+
*/
|
|
519
|
+
skip: (reason) => {
|
|
520
|
+
throw new SkipError(reason);
|
|
521
|
+
},
|
|
522
|
+
/**
|
|
523
|
+
* Immediately fail and abort the current test.
|
|
524
|
+
* Emits a failed assertion event, then throws to stop execution.
|
|
525
|
+
*/
|
|
526
|
+
fail: (message) => {
|
|
527
|
+
// Emit a failed assertion so the failure reason appears in events
|
|
528
|
+
console.log(JSON.stringify({
|
|
529
|
+
type: "assertion",
|
|
530
|
+
passed: false,
|
|
531
|
+
message,
|
|
532
|
+
}));
|
|
533
|
+
throw new FailError(message);
|
|
534
|
+
},
|
|
535
|
+
/**
|
|
536
|
+
* Poll a function until it returns truthy or times out.
|
|
537
|
+
*/
|
|
538
|
+
pollUntil: async (options, fn) => {
|
|
539
|
+
const { timeoutMs, intervalMs = 1000, onTimeout } = options;
|
|
540
|
+
const deadline = Date.now() + timeoutMs;
|
|
541
|
+
let lastError;
|
|
542
|
+
while (Date.now() < deadline) {
|
|
543
|
+
try {
|
|
544
|
+
const result = await fn();
|
|
545
|
+
if (result)
|
|
546
|
+
return; // truthy → done
|
|
547
|
+
}
|
|
548
|
+
catch (err) {
|
|
549
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
550
|
+
}
|
|
551
|
+
// Wait before next attempt, but don't overshoot the deadline
|
|
552
|
+
const remaining = deadline - Date.now();
|
|
553
|
+
if (remaining <= 0)
|
|
554
|
+
break;
|
|
555
|
+
await new Promise((r) => setTimeout(r, Math.min(intervalMs, remaining)));
|
|
556
|
+
}
|
|
557
|
+
// Timed out
|
|
558
|
+
if (onTimeout) {
|
|
559
|
+
onTimeout(lastError);
|
|
560
|
+
return;
|
|
561
|
+
}
|
|
562
|
+
const suffix = lastError ? `: ${lastError.message}` : "";
|
|
563
|
+
throw new Error(`pollUntil timed out after ${timeoutMs}ms${suffix}`);
|
|
564
|
+
},
|
|
565
|
+
/**
|
|
566
|
+
* Set a custom timeout for the current test.
|
|
567
|
+
* Note: This sends a timeout_update event to the runner.
|
|
568
|
+
* The runner is responsible for enforcing the timeout.
|
|
569
|
+
*
|
|
570
|
+
* @param ms Timeout in milliseconds
|
|
571
|
+
*/
|
|
572
|
+
setTimeout: (ms) => {
|
|
573
|
+
console.log(JSON.stringify({
|
|
574
|
+
type: "timeout_update",
|
|
575
|
+
timeout: ms,
|
|
576
|
+
}));
|
|
577
|
+
},
|
|
578
|
+
/**
|
|
579
|
+
* Current execution retry count (0 for first attempt).
|
|
580
|
+
* This reflects whole-test re-runs, not per-step retries.
|
|
581
|
+
*/
|
|
582
|
+
retryCount,
|
|
583
|
+
/**
|
|
584
|
+
* Get current memory usage via `process.memoryUsage()`.
|
|
585
|
+
* Useful for debugging memory issues locally.
|
|
586
|
+
*
|
|
587
|
+
* @returns Memory usage stats
|
|
588
|
+
*
|
|
589
|
+
* @example
|
|
590
|
+
* const mem = ctx.getMemoryUsage();
|
|
591
|
+
* ctx.log(`Heap used: ${(mem.heapUsed / 1024 / 1024).toFixed(2)} MB`);
|
|
592
|
+
*/
|
|
593
|
+
getMemoryUsage: () => {
|
|
594
|
+
return process.memoryUsage();
|
|
595
|
+
},
|
|
596
|
+
};
|
|
597
|
+
// ---------------------------------------------------------------------------
|
|
598
|
+
// Auto-tracing HTTP client (ctx.http) — powered by ky
|
|
599
|
+
// ---------------------------------------------------------------------------
|
|
600
|
+
// Track request start time. We use a simple variable instead of a WeakMap
|
|
601
|
+
// because ky may clone/recreate the Request object between beforeRequest and
|
|
602
|
+
// afterResponse hooks, breaking reference equality in a WeakMap.
|
|
603
|
+
let lastRequestStartTime = 0;
|
|
604
|
+
let httpRequestTotal = 0;
|
|
605
|
+
let httpErrorTotal = 0;
|
|
606
|
+
// Captured in beforeRequest when emitFullTrace is on
|
|
607
|
+
let lastRequestBody = undefined;
|
|
608
|
+
/** Max serialized body size (chars) to include in trace events. */
|
|
609
|
+
const TRACE_BODY_MAX_SIZE = 1_048_576; // 1MB
|
|
610
|
+
/**
|
|
611
|
+
* Truncate a response body if its JSON representation exceeds the size limit.
|
|
612
|
+
* Preserves structure: arrays are sliced and a count annotation is appended
|
|
613
|
+
* so the trace file stays valid JSON and diffable.
|
|
614
|
+
*/
|
|
615
|
+
function truncateBody(body) {
|
|
616
|
+
try {
|
|
617
|
+
const json = JSON.stringify(body);
|
|
618
|
+
if (json.length <= TRACE_BODY_MAX_SIZE)
|
|
619
|
+
return body;
|
|
620
|
+
if (typeof body === "object" && body !== null) {
|
|
621
|
+
// For arrays, keep first few items + count
|
|
622
|
+
if (Array.isArray(body)) {
|
|
623
|
+
const preview = body.slice(0, 3);
|
|
624
|
+
return [...preview, `(${body.length - 3} more items truncated)`];
|
|
625
|
+
}
|
|
626
|
+
// For objects with large array values, truncate those arrays
|
|
627
|
+
const pruned = {};
|
|
628
|
+
for (const [key, value] of Object.entries(body)) {
|
|
629
|
+
if (Array.isArray(value) && value.length > 3) {
|
|
630
|
+
pruned[key] = [
|
|
631
|
+
...value.slice(0, 3),
|
|
632
|
+
`(${value.length - 3} more items truncated)`,
|
|
633
|
+
];
|
|
634
|
+
}
|
|
635
|
+
else {
|
|
636
|
+
pruned[key] = value;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
const rechecked = JSON.stringify(pruned);
|
|
640
|
+
if (rechecked.length <= TRACE_BODY_MAX_SIZE * 1.5)
|
|
641
|
+
return pruned;
|
|
642
|
+
}
|
|
643
|
+
return { _truncated: true, _sizeBytes: json.length };
|
|
644
|
+
}
|
|
645
|
+
catch {
|
|
646
|
+
return "(non-serializable)";
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
let summaryEmitted = false;
|
|
650
|
+
/**
|
|
651
|
+
* Emit summary event with HTTP, assertion, and step totals.
|
|
652
|
+
* Called once before the final status event. Idempotent.
|
|
653
|
+
*/
|
|
654
|
+
/**
|
|
655
|
+
* Reset per-test counters for file-level batch mode.
|
|
656
|
+
* Called before each test when running multiple tests in a single process.
|
|
657
|
+
*/
|
|
658
|
+
function resetTestCounters() {
|
|
659
|
+
stepFailedAssertions = 0;
|
|
660
|
+
stepAssertionTotal = 0;
|
|
661
|
+
currentStepIndex = null;
|
|
662
|
+
totalAssertions = 0;
|
|
663
|
+
totalFailedAssertions = 0;
|
|
664
|
+
totalSteps = 0;
|
|
665
|
+
passedSteps = 0;
|
|
666
|
+
failedSteps = 0;
|
|
667
|
+
skippedSteps = 0;
|
|
668
|
+
warningTotal = 0;
|
|
669
|
+
warningTriggered = 0;
|
|
670
|
+
schemaValidationTotal = 0;
|
|
671
|
+
schemaValidationFailed = 0;
|
|
672
|
+
schemaValidationWarnings = 0;
|
|
673
|
+
httpRequestTotal = 0;
|
|
674
|
+
httpErrorTotal = 0;
|
|
675
|
+
summaryEmitted = false;
|
|
676
|
+
peakMemoryBytes = 0;
|
|
677
|
+
}
|
|
678
|
+
function emitSummary() {
|
|
679
|
+
if (summaryEmitted)
|
|
680
|
+
return;
|
|
681
|
+
summaryEmitted = true;
|
|
682
|
+
console.log(JSON.stringify({
|
|
683
|
+
type: "summary",
|
|
684
|
+
data: {
|
|
685
|
+
// HTTP stats (always present, 0 when no HTTP calls)
|
|
686
|
+
httpRequestTotal,
|
|
687
|
+
httpErrorTotal,
|
|
688
|
+
httpErrorRate: httpRequestTotal > 0 ? Math.round((httpErrorTotal / httpRequestTotal) * 10000) / 10000 : 0,
|
|
689
|
+
// Assertion stats
|
|
690
|
+
assertionTotal: totalAssertions,
|
|
691
|
+
assertionFailed: totalFailedAssertions,
|
|
692
|
+
// Warning stats
|
|
693
|
+
warningTotal,
|
|
694
|
+
warningTriggered,
|
|
695
|
+
// Schema validation stats
|
|
696
|
+
schemaValidationTotal,
|
|
697
|
+
schemaValidationFailed,
|
|
698
|
+
schemaValidationWarnings,
|
|
699
|
+
// Step stats (0 for simple tests without builder steps)
|
|
700
|
+
stepTotal: totalSteps,
|
|
701
|
+
stepPassed: passedSteps,
|
|
702
|
+
stepFailed: failedSteps,
|
|
703
|
+
stepSkipped: skippedSteps,
|
|
704
|
+
},
|
|
705
|
+
}));
|
|
706
|
+
}
|
|
707
|
+
const MAX_NETWORK_WARNINGS_PER_CODE = 3;
|
|
708
|
+
const networkWarningCounts = new Map();
|
|
709
|
+
let networkRequestCount = 0;
|
|
710
|
+
let networkInFlightCount = 0;
|
|
711
|
+
let networkResponseBytes = 0;
|
|
712
|
+
function emitNetworkWarning(code, message) {
|
|
713
|
+
const nextCount = (networkWarningCounts.get(code) ?? 0) + 1;
|
|
714
|
+
networkWarningCounts.set(code, nextCount);
|
|
715
|
+
if (nextCount <= MAX_NETWORK_WARNINGS_PER_CODE) {
|
|
716
|
+
ctx.warn(false, `[network_guard:${code}] ${message}`);
|
|
717
|
+
}
|
|
718
|
+
else if (nextCount === MAX_NETWORK_WARNINGS_PER_CODE + 1) {
|
|
719
|
+
ctx.warn(false, `[network_guard:${code}] further warnings suppressed`);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
async function resolveHostIps(hostname) {
|
|
723
|
+
const dns = await import("node:dns/promises");
|
|
724
|
+
const ips = new Set();
|
|
725
|
+
try {
|
|
726
|
+
const aRecords = await dns.resolve4(hostname);
|
|
727
|
+
for (const ip of aRecords)
|
|
728
|
+
ips.add(ip);
|
|
729
|
+
}
|
|
730
|
+
catch {
|
|
731
|
+
// Ignore; try AAAA next.
|
|
732
|
+
}
|
|
733
|
+
try {
|
|
734
|
+
const aaaaRecords = await dns.resolve6(hostname);
|
|
735
|
+
for (const ip of aaaaRecords)
|
|
736
|
+
ips.add(ip);
|
|
737
|
+
}
|
|
738
|
+
catch {
|
|
739
|
+
// Ignore; caller decides behavior when no records are resolved.
|
|
740
|
+
}
|
|
741
|
+
return Array.from(ips);
|
|
742
|
+
}
|
|
743
|
+
function toRequestUrl(input) {
|
|
744
|
+
if (input instanceof Request)
|
|
745
|
+
return new URL(input.url);
|
|
746
|
+
if (input instanceof URL)
|
|
747
|
+
return input;
|
|
748
|
+
return new URL(input);
|
|
749
|
+
}
|
|
750
|
+
async function enforceNetworkPolicy(url) {
|
|
751
|
+
if (!networkPolicy)
|
|
752
|
+
return;
|
|
753
|
+
if (!isAllowedProtocol(url.protocol)) {
|
|
754
|
+
emitNetworkWarning("protocol_blocked", `Blocked protocol ${url.protocol} for ${url.href}`);
|
|
755
|
+
throw new Error(`Network policy blocked protocol ${url.protocol}. Only http/https are allowed.`);
|
|
756
|
+
}
|
|
757
|
+
const port = resolveUrlPort(url);
|
|
758
|
+
if (!isAllowedPort(port, networkPolicy.allowedPorts)) {
|
|
759
|
+
emitNetworkWarning("port_blocked", `Blocked port ${port} for ${url.href}`);
|
|
760
|
+
throw new Error(`Network policy blocked destination port ${port}.`);
|
|
761
|
+
}
|
|
762
|
+
const hostname = url.hostname.toLowerCase();
|
|
763
|
+
const hostnameReason = classifyHostnameBlockReason(hostname);
|
|
764
|
+
if (hostnameReason) {
|
|
765
|
+
emitNetworkWarning(hostnameReason, `Blocked hostname ${hostname} for ${url.href}`);
|
|
766
|
+
throw new Error(`Network policy blocked sensitive hostname ${hostname}.`);
|
|
767
|
+
}
|
|
768
|
+
if (isIpLiteral(hostname)) {
|
|
769
|
+
const ipReason = classifyIpBlockReason(hostname);
|
|
770
|
+
if (ipReason) {
|
|
771
|
+
emitNetworkWarning(ipReason, `Blocked destination IP ${hostname} for ${url.href}`);
|
|
772
|
+
throw new Error(`Network policy blocked destination IP ${hostname}.`);
|
|
773
|
+
}
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
const resolvedIps = await resolveHostIps(hostname);
|
|
777
|
+
if (resolvedIps.length === 0) {
|
|
778
|
+
emitNetworkWarning("dns_resolution_failed", `Could not resolve ${hostname} for ${url.href}`);
|
|
779
|
+
throw new Error(`Network policy could not resolve host ${hostname}. Request denied.`);
|
|
780
|
+
}
|
|
781
|
+
for (const ip of resolvedIps) {
|
|
782
|
+
const ipReason = classifyIpBlockReason(ip);
|
|
783
|
+
if (ipReason) {
|
|
784
|
+
emitNetworkWarning(ipReason, `Blocked resolved IP ${ip} (${hostname}) for ${url.href}`);
|
|
785
|
+
throw new Error(`Network policy blocked resolved destination ${ip} for host ${hostname}.`);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
const originalFetch = globalThis.fetch.bind(globalThis);
|
|
790
|
+
globalThis.fetch = async (input, init) => {
|
|
791
|
+
if (!networkPolicy) {
|
|
792
|
+
return originalFetch(input, init);
|
|
793
|
+
}
|
|
794
|
+
const requestUrl = toRequestUrl(input);
|
|
795
|
+
if (networkRequestCount >= networkPolicy.maxRequests) {
|
|
796
|
+
emitNetworkWarning("request_limit_exceeded", `Request limit exceeded (${networkPolicy.maxRequests})`);
|
|
797
|
+
throw new Error(`Network policy exceeded max outbound requests (${networkPolicy.maxRequests}).`);
|
|
798
|
+
}
|
|
799
|
+
if (networkInFlightCount >= networkPolicy.maxConcurrentRequests) {
|
|
800
|
+
emitNetworkWarning("concurrency_limit_exceeded", `In-flight request limit exceeded (${networkPolicy.maxConcurrentRequests})`);
|
|
801
|
+
throw new Error(`Network policy exceeded max concurrent outbound requests (${networkPolicy.maxConcurrentRequests}).`);
|
|
802
|
+
}
|
|
803
|
+
// Reserve counters before await to avoid TOCTOU races when user code issues
|
|
804
|
+
// concurrent requests in a single Promise.all frame.
|
|
805
|
+
networkRequestCount++;
|
|
806
|
+
networkInFlightCount++;
|
|
807
|
+
const timeoutController = new AbortController();
|
|
808
|
+
let timedOutByPolicy = false;
|
|
809
|
+
const parentSignal = (() => {
|
|
810
|
+
if (!init || typeof init !== "object" || !("signal" in init)) {
|
|
811
|
+
return undefined;
|
|
812
|
+
}
|
|
813
|
+
const candidate = init.signal;
|
|
814
|
+
return candidate instanceof AbortSignal ? candidate : undefined;
|
|
815
|
+
})();
|
|
816
|
+
const onParentAbort = () => timeoutController.abort(parentSignal?.reason);
|
|
817
|
+
if (parentSignal) {
|
|
818
|
+
if (parentSignal.aborted) {
|
|
819
|
+
timeoutController.abort(parentSignal.reason);
|
|
820
|
+
}
|
|
821
|
+
else {
|
|
822
|
+
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
const timeoutId = setTimeout(() => {
|
|
826
|
+
timedOutByPolicy = true;
|
|
827
|
+
timeoutController.abort(new Error(`Network request timed out after ${networkPolicy.requestTimeoutMs}ms`));
|
|
828
|
+
}, networkPolicy.requestTimeoutMs);
|
|
829
|
+
try {
|
|
830
|
+
await enforceNetworkPolicy(requestUrl);
|
|
831
|
+
const response = await originalFetch(input, {
|
|
832
|
+
...init,
|
|
833
|
+
signal: timeoutController.signal,
|
|
834
|
+
});
|
|
835
|
+
return applyResponseByteBudget(response, {
|
|
836
|
+
requestUrl,
|
|
837
|
+
maxResponseBytes: networkPolicy.maxResponseBytes,
|
|
838
|
+
getUsedResponseBytes: () => networkResponseBytes,
|
|
839
|
+
addUsedResponseBytes: (delta) => {
|
|
840
|
+
networkResponseBytes += delta;
|
|
841
|
+
},
|
|
842
|
+
emitWarning: emitNetworkWarning,
|
|
843
|
+
});
|
|
844
|
+
}
|
|
845
|
+
catch (error) {
|
|
846
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
847
|
+
if (timedOutByPolicy) {
|
|
848
|
+
throw new Error(`Network request timed out after ${networkPolicy.requestTimeoutMs}ms`);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
throw error;
|
|
852
|
+
}
|
|
853
|
+
finally {
|
|
854
|
+
clearTimeout(timeoutId);
|
|
855
|
+
if (parentSignal) {
|
|
856
|
+
parentSignal.removeEventListener("abort", onParentAbort);
|
|
857
|
+
}
|
|
858
|
+
networkInFlightCount = Math.max(0, networkInFlightCount - 1);
|
|
859
|
+
}
|
|
860
|
+
};
|
|
861
|
+
const kyInstance = ky.create({
|
|
862
|
+
throwHttpErrors: false,
|
|
863
|
+
hooks: {
|
|
864
|
+
beforeRequest: [
|
|
865
|
+
(_request, options) => {
|
|
866
|
+
lastRequestStartTime = performance.now();
|
|
867
|
+
if (emitFullTrace) {
|
|
868
|
+
// Capture request body from ky options before the request is sent
|
|
869
|
+
lastRequestBody = options.json ?? options.body ?? undefined;
|
|
870
|
+
}
|
|
871
|
+
},
|
|
872
|
+
],
|
|
873
|
+
afterResponse: [
|
|
874
|
+
async (request, _options, response) => {
|
|
875
|
+
const duration = Math.round(performance.now() - lastRequestStartTime);
|
|
876
|
+
// Increment HTTP counters for summary
|
|
877
|
+
httpRequestTotal++;
|
|
878
|
+
if (response.status >= 400) {
|
|
879
|
+
httpErrorTotal++;
|
|
880
|
+
}
|
|
881
|
+
// Build trace data — enriched when emitFullTrace is on
|
|
882
|
+
const traceData = {
|
|
883
|
+
method: request.method,
|
|
884
|
+
url: request.url,
|
|
885
|
+
status: response.status,
|
|
886
|
+
duration,
|
|
887
|
+
};
|
|
888
|
+
// Pick up operation name from GraphQL client (X-Glubean-Op header)
|
|
889
|
+
const glubeanOp = request.headers.get("x-glubean-op");
|
|
890
|
+
if (glubeanOp) {
|
|
891
|
+
traceData.name = glubeanOp;
|
|
892
|
+
}
|
|
893
|
+
if (emitFullTrace) {
|
|
894
|
+
traceData.requestHeaders = Object.fromEntries(request.headers.entries());
|
|
895
|
+
if (lastRequestBody !== undefined) {
|
|
896
|
+
traceData.requestBody = truncateBody(lastRequestBody);
|
|
897
|
+
}
|
|
898
|
+
traceData.responseHeaders = Object.fromEntries(response.headers.entries());
|
|
899
|
+
// Clone the response to read the body without consuming the original stream
|
|
900
|
+
try {
|
|
901
|
+
const cloned = response.clone();
|
|
902
|
+
const contentType = response.headers.get("content-type") || "";
|
|
903
|
+
if (contentType.includes("json")) {
|
|
904
|
+
traceData.responseBody = truncateBody(await cloned.json());
|
|
905
|
+
}
|
|
906
|
+
else if (contentType.includes("text") ||
|
|
907
|
+
contentType.includes("xml")) {
|
|
908
|
+
const text = await cloned.text();
|
|
909
|
+
traceData.responseBody = truncateBody(text);
|
|
910
|
+
}
|
|
911
|
+
// Binary content types are intentionally skipped
|
|
912
|
+
}
|
|
913
|
+
catch {
|
|
914
|
+
// Ignore clone/parse errors — trace still emits without body
|
|
915
|
+
}
|
|
916
|
+
lastRequestBody = undefined;
|
|
917
|
+
}
|
|
918
|
+
ctx.trace(traceData);
|
|
919
|
+
// Auto-metric for response time
|
|
920
|
+
try {
|
|
921
|
+
const pathname = new URL(request.url).pathname;
|
|
922
|
+
ctx.metric("http_duration_ms", duration, {
|
|
923
|
+
unit: "ms",
|
|
924
|
+
tags: { method: request.method, path: pathname },
|
|
925
|
+
});
|
|
926
|
+
}
|
|
927
|
+
catch {
|
|
928
|
+
ctx.metric("http_duration_ms", duration, {
|
|
929
|
+
unit: "ms",
|
|
930
|
+
tags: { method: request.method },
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
return response;
|
|
934
|
+
},
|
|
935
|
+
],
|
|
936
|
+
},
|
|
937
|
+
});
|
|
938
|
+
/**
|
|
939
|
+
* Normalize URL input for ky compatibility:
|
|
940
|
+
* - Strip leading '/' from path when it's not a full URL
|
|
941
|
+
* (ky requires relative paths without leading slash when using prefixUrl)
|
|
942
|
+
*/
|
|
943
|
+
function normalizeUrl(input) {
|
|
944
|
+
if (typeof input === "string" &&
|
|
945
|
+
input.startsWith("/") &&
|
|
946
|
+
!input.startsWith("//")) {
|
|
947
|
+
return input.slice(1);
|
|
948
|
+
}
|
|
949
|
+
return input;
|
|
950
|
+
}
|
|
951
|
+
/**
|
|
952
|
+
* Normalize options to fix ky quirks:
|
|
953
|
+
* - Remove empty searchParams to prevent ky from appending bare '?'
|
|
954
|
+
*/
|
|
955
|
+
function normalizeOptions(options) {
|
|
956
|
+
if (!options)
|
|
957
|
+
return options;
|
|
958
|
+
const normalized = { ...options };
|
|
959
|
+
// Remove empty searchParams so ky doesn't append a bare '?'
|
|
960
|
+
if (normalized.searchParams != null) {
|
|
961
|
+
if (normalized.searchParams instanceof URLSearchParams) {
|
|
962
|
+
if (normalized.searchParams.toString() === "") {
|
|
963
|
+
delete normalized.searchParams;
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
else if (typeof normalized.searchParams === "object" &&
|
|
967
|
+
Object.keys(normalized.searchParams).length === 0) {
|
|
968
|
+
delete normalized.searchParams;
|
|
969
|
+
}
|
|
970
|
+
else if (typeof normalized.searchParams === "string" &&
|
|
971
|
+
normalized.searchParams === "") {
|
|
972
|
+
delete normalized.searchParams;
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
return normalized;
|
|
976
|
+
}
|
|
977
|
+
/**
|
|
978
|
+
* Run pre-request schema validations (query params, request body).
|
|
979
|
+
* Extracts schema option from the options object.
|
|
980
|
+
*/
|
|
981
|
+
function runPreRequestSchemaValidation(options) {
|
|
982
|
+
const schemaOpts = options?.schema;
|
|
983
|
+
if (!schemaOpts)
|
|
984
|
+
return;
|
|
985
|
+
// Validate query/searchParams
|
|
986
|
+
if (schemaOpts.query && options?.searchParams != null) {
|
|
987
|
+
const { schema, severity } = resolveSchemaEntry(schemaOpts.query);
|
|
988
|
+
runSchemaValidation(options.searchParams, schema, "query params", severity);
|
|
989
|
+
}
|
|
990
|
+
// Validate request body (json)
|
|
991
|
+
if (schemaOpts.request && options?.json !== undefined) {
|
|
992
|
+
const { schema, severity } = resolveSchemaEntry(schemaOpts.request);
|
|
993
|
+
runSchemaValidation(options.json, schema, "request body", severity);
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* Wrap a ky response promise to run post-response schema validation.
|
|
998
|
+
* Attaches to the .json() method so we validate the parsed body.
|
|
999
|
+
*/
|
|
1000
|
+
function wrapResponseWithSchema(responsePromise, schemaOpts) {
|
|
1001
|
+
if (!schemaOpts?.response)
|
|
1002
|
+
return responsePromise;
|
|
1003
|
+
const { schema, severity } = resolveSchemaEntry(schemaOpts.response);
|
|
1004
|
+
// Wrap the .json() method to validate after parsing
|
|
1005
|
+
const originalJson = responsePromise.json.bind(responsePromise);
|
|
1006
|
+
responsePromise.json = async () => {
|
|
1007
|
+
const body = await originalJson();
|
|
1008
|
+
runSchemaValidation(body, schema, "response body", severity);
|
|
1009
|
+
return body;
|
|
1010
|
+
};
|
|
1011
|
+
return responsePromise;
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* Wrap a ky instance so that:
|
|
1015
|
+
* 1. Leading '/' in URL paths is stripped (ky + prefixUrl compatibility)
|
|
1016
|
+
* 2. Empty searchParams are removed (no bare '?' in URL)
|
|
1017
|
+
* 3. extend() returns a wrapped instance (preserves normalization)
|
|
1018
|
+
* 4. Schema validation runs on request/response when `schema` option is provided
|
|
1019
|
+
*/
|
|
1020
|
+
function wrapKy(instance) {
|
|
1021
|
+
const methods = ["get", "post", "put", "patch", "delete", "head"];
|
|
1022
|
+
function callWithSchema(kyFn, input, options) {
|
|
1023
|
+
const normalized = normalizeOptions(options);
|
|
1024
|
+
// Run pre-request validations (query, request body)
|
|
1025
|
+
runPreRequestSchemaValidation(normalized);
|
|
1026
|
+
// Strip schema option before passing to ky (ky doesn't know about it)
|
|
1027
|
+
let kyOptions;
|
|
1028
|
+
if (normalized?.schema) {
|
|
1029
|
+
const { schema: _schema, ...rest } = normalized;
|
|
1030
|
+
kyOptions = rest;
|
|
1031
|
+
}
|
|
1032
|
+
else {
|
|
1033
|
+
kyOptions = normalized;
|
|
1034
|
+
}
|
|
1035
|
+
const responsePromise = kyFn(normalizeUrl(input), kyOptions);
|
|
1036
|
+
return wrapResponseWithSchema(responsePromise, normalized?.schema);
|
|
1037
|
+
}
|
|
1038
|
+
// The callable + methods wrapper
|
|
1039
|
+
const wrapped = function (input, options) {
|
|
1040
|
+
return callWithSchema(instance, input, options);
|
|
1041
|
+
};
|
|
1042
|
+
for (const method of methods) {
|
|
1043
|
+
wrapped[method] = (input, options) => callWithSchema(instance[method].bind(instance), input, options);
|
|
1044
|
+
}
|
|
1045
|
+
wrapped.extend = (options) => wrapKy(instance.extend(normalizeOptions(options)));
|
|
1046
|
+
return wrapped;
|
|
1047
|
+
}
|
|
1048
|
+
// Attach wrapped http client to ctx
|
|
1049
|
+
ctx.http = wrapKy(kyInstance);
|
|
1050
|
+
// Set global runtime slot for configure() API.
|
|
1051
|
+
// configure() returns lazy getters that read from this slot at test execution time.
|
|
1052
|
+
// This must be set BEFORE importing user code so the slot is available during execution.
|
|
1053
|
+
//
|
|
1054
|
+
// Wrap vars and secrets with a Proxy so that configure()'s requireVar/requireSecret
|
|
1055
|
+
// also fall back to system env (same behavior as ctx.vars/ctx.secrets above).
|
|
1056
|
+
function withEnvFallback(explicit) {
|
|
1057
|
+
return new Proxy(explicit, {
|
|
1058
|
+
get(target, prop) {
|
|
1059
|
+
const value = target[prop];
|
|
1060
|
+
if (value !== undefined && value !== null && value !== "")
|
|
1061
|
+
return value;
|
|
1062
|
+
return process.env[prop] || undefined;
|
|
1063
|
+
},
|
|
1064
|
+
has(target, prop) {
|
|
1065
|
+
return prop in target || process.env[prop] !== undefined;
|
|
1066
|
+
},
|
|
1067
|
+
});
|
|
1068
|
+
}
|
|
1069
|
+
globalThis.__glubeanRuntime = {
|
|
1070
|
+
vars: withEnvFallback(rawVars),
|
|
1071
|
+
secrets: withEnvFallback(rawSecrets),
|
|
1072
|
+
http: ctx.http,
|
|
1073
|
+
test: runtimeTest,
|
|
1074
|
+
action: ctx.action,
|
|
1075
|
+
event: ctx.event,
|
|
1076
|
+
log: ctx.log,
|
|
1077
|
+
};
|
|
1078
|
+
try {
|
|
1079
|
+
// Dynamic import - LOAD phase
|
|
1080
|
+
console.log(JSON.stringify({
|
|
1081
|
+
type: "log",
|
|
1082
|
+
message: `Loading test module: ${testUrl}`,
|
|
1083
|
+
}));
|
|
1084
|
+
const userModule = await import(testUrl);
|
|
1085
|
+
if (testIds) {
|
|
1086
|
+
// ── File-level batch mode ──
|
|
1087
|
+
// Run multiple tests sequentially in a single process.
|
|
1088
|
+
// Module-level state (let variables) is preserved between tests.
|
|
1089
|
+
let hasFailure = false;
|
|
1090
|
+
for (const id of testIds) {
|
|
1091
|
+
resetTestCounters();
|
|
1092
|
+
let testObj = findTestById(userModule, id);
|
|
1093
|
+
// Fallback for non-deterministic tests (test.pick): the testId from
|
|
1094
|
+
// discovery may differ from this run's random selection. Use the stable
|
|
1095
|
+
// exportName to locate the test.
|
|
1096
|
+
if (!testObj && exportNamesMap[id]) {
|
|
1097
|
+
testObj = findTestByExport(userModule, exportNamesMap[id]);
|
|
1098
|
+
}
|
|
1099
|
+
if (!testObj) {
|
|
1100
|
+
console.log(JSON.stringify({
|
|
1101
|
+
type: "start",
|
|
1102
|
+
id,
|
|
1103
|
+
name: id,
|
|
1104
|
+
}));
|
|
1105
|
+
console.log(JSON.stringify({
|
|
1106
|
+
type: "status",
|
|
1107
|
+
status: "failed",
|
|
1108
|
+
id,
|
|
1109
|
+
error: `Test "${id}" not found in module`,
|
|
1110
|
+
}));
|
|
1111
|
+
hasFailure = true;
|
|
1112
|
+
continue;
|
|
1113
|
+
}
|
|
1114
|
+
try {
|
|
1115
|
+
await executeNewTest(testObj);
|
|
1116
|
+
}
|
|
1117
|
+
catch (error) {
|
|
1118
|
+
emitSummary();
|
|
1119
|
+
if (error instanceof SkipError) {
|
|
1120
|
+
console.log(JSON.stringify({
|
|
1121
|
+
type: "status",
|
|
1122
|
+
status: "skipped",
|
|
1123
|
+
id,
|
|
1124
|
+
reason: error.reason,
|
|
1125
|
+
}));
|
|
1126
|
+
}
|
|
1127
|
+
else {
|
|
1128
|
+
hasFailure = true;
|
|
1129
|
+
console.log(JSON.stringify({
|
|
1130
|
+
type: "status",
|
|
1131
|
+
status: "failed",
|
|
1132
|
+
id,
|
|
1133
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1134
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
1135
|
+
}));
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
process.exit(hasFailure ? 1 : 0);
|
|
1140
|
+
}
|
|
1141
|
+
// ── Single test mode (default) ──
|
|
1142
|
+
let testObj = findTestById(userModule, testId);
|
|
1143
|
+
if (!testObj && exportName) {
|
|
1144
|
+
// Fallback: for non-deterministic tests (test.pick), the testId from
|
|
1145
|
+
// discovery may not match this run's random selection. Use the stable
|
|
1146
|
+
// exportName to locate the export and pick the first resolved test.
|
|
1147
|
+
testObj = findTestByExport(userModule, exportName);
|
|
1148
|
+
}
|
|
1149
|
+
if (testObj) {
|
|
1150
|
+
await executeNewTest(testObj);
|
|
1151
|
+
}
|
|
1152
|
+
else {
|
|
1153
|
+
throw new Error(`Test "${testId}" not found. Available exports: ${Object.keys(userModule).join(", ")}`);
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
catch (error) {
|
|
1157
|
+
// Emit HTTP summary before final status
|
|
1158
|
+
emitSummary();
|
|
1159
|
+
// Check if this is a skip error
|
|
1160
|
+
if (error instanceof SkipError) {
|
|
1161
|
+
console.log(JSON.stringify({
|
|
1162
|
+
type: "status",
|
|
1163
|
+
status: "skipped",
|
|
1164
|
+
reason: error.reason,
|
|
1165
|
+
}));
|
|
1166
|
+
process.exit(0); // Exit cleanly for skipped tests
|
|
1167
|
+
}
|
|
1168
|
+
// Regular error - report as failure
|
|
1169
|
+
console.log(JSON.stringify({
|
|
1170
|
+
type: "status",
|
|
1171
|
+
status: "failed",
|
|
1172
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1173
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
1174
|
+
}));
|
|
1175
|
+
process.exit(1);
|
|
1176
|
+
}
|
|
1177
|
+
// Resolution utilities shared with MCP and other consumers.
|
|
1178
|
+
// Extracted to resolve.ts for reuse outside the sandbox.
|
|
1179
|
+
import { findTestByExport, findTestById } from "./resolve.js";
|
|
1180
|
+
/**
|
|
1181
|
+
* Resolve test.extend() fixtures and run the test body with an augmented context.
|
|
1182
|
+
*
|
|
1183
|
+
* Simple fixtures (1-param) are resolved first; their return values are merged
|
|
1184
|
+
* into the context via prototype-linked copy. Lifecycle fixtures (2-param with
|
|
1185
|
+
* `use` callback) wrap the test execution so cleanup runs after the test completes.
|
|
1186
|
+
*
|
|
1187
|
+
* Fixture type is determined by `fn.length`:
|
|
1188
|
+
* - 1 → simple factory: `(ctx) => instance`
|
|
1189
|
+
* - 2 → lifecycle factory: `(ctx, use) => { setup; await use(instance); cleanup }`
|
|
1190
|
+
*/
|
|
1191
|
+
async function withFixtures(fixtures, baseCtx, runTest) {
|
|
1192
|
+
// Prototype-linked copy: core ctx fields (vars, secrets, http, ...) remain accessible
|
|
1193
|
+
const augmented = Object.create(baseCtx);
|
|
1194
|
+
const simple = [];
|
|
1195
|
+
const lifecycle = [];
|
|
1196
|
+
for (const [key, fn] of Object.entries(fixtures)) {
|
|
1197
|
+
if (fn.length >= 2) {
|
|
1198
|
+
lifecycle.push([key, fn]);
|
|
1199
|
+
}
|
|
1200
|
+
else {
|
|
1201
|
+
simple.push([key, fn]);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
// Resolve simple fixtures first
|
|
1205
|
+
for (const [key, fn] of simple) {
|
|
1206
|
+
augmented[key] = await fn(augmented);
|
|
1207
|
+
}
|
|
1208
|
+
// No lifecycle fixtures — run the test directly
|
|
1209
|
+
if (lifecycle.length === 0) {
|
|
1210
|
+
await runTest(augmented);
|
|
1211
|
+
return;
|
|
1212
|
+
}
|
|
1213
|
+
// Build a nested chain for lifecycle fixtures.
|
|
1214
|
+
// Each lifecycle wraps the next; the innermost call is the actual test.
|
|
1215
|
+
let innerFn = () => runTest(augmented);
|
|
1216
|
+
for (let i = lifecycle.length - 1; i >= 0; i--) {
|
|
1217
|
+
const [key, factory] = lifecycle[i];
|
|
1218
|
+
const next = innerFn;
|
|
1219
|
+
innerFn = () => {
|
|
1220
|
+
let called = false;
|
|
1221
|
+
// Capture the promise created inside use() so we can ensure the test
|
|
1222
|
+
// body completes even if the fixture forgets to `await use(...)`.
|
|
1223
|
+
let usePromise = null;
|
|
1224
|
+
return factory(augmented, (instance) => {
|
|
1225
|
+
if (called) {
|
|
1226
|
+
throw new Error(`Lifecycle fixture "${key}" called use() more than once. ` +
|
|
1227
|
+
`Each fixture must call use() exactly once.`);
|
|
1228
|
+
}
|
|
1229
|
+
called = true;
|
|
1230
|
+
augmented[key] = instance;
|
|
1231
|
+
usePromise = next();
|
|
1232
|
+
return usePromise;
|
|
1233
|
+
}).then(async () => {
|
|
1234
|
+
if (!called) {
|
|
1235
|
+
throw new Error(`Lifecycle fixture "${key}" completed without calling use(). ` +
|
|
1236
|
+
`Lifecycle fixtures must call use(instance) exactly once ` +
|
|
1237
|
+
`to run the test body.`);
|
|
1238
|
+
}
|
|
1239
|
+
// If fixture didn't await use(), wait for the test body to finish
|
|
1240
|
+
// before proceeding. When properly awaited this is a no-op.
|
|
1241
|
+
if (usePromise) {
|
|
1242
|
+
await usePromise;
|
|
1243
|
+
}
|
|
1244
|
+
});
|
|
1245
|
+
};
|
|
1246
|
+
}
|
|
1247
|
+
await innerFn();
|
|
1248
|
+
}
|
|
1249
|
+
/**
|
|
1250
|
+
* Execute a test created with the builder API.
|
|
1251
|
+
* Handles both simple tests and multi-step tests with setup/teardown.
|
|
1252
|
+
* When the test carries `fixtures` (from `test.extend()`), they are resolved
|
|
1253
|
+
* and injected into the context before the test body runs.
|
|
1254
|
+
*
|
|
1255
|
+
* @param test The Test object to execute
|
|
1256
|
+
*/
|
|
1257
|
+
async function executeNewTest(test) {
|
|
1258
|
+
const testTags = normalizeTestTags(test.meta.tags);
|
|
1259
|
+
// Keep runtime metadata aligned with the actual resolved test before user code runs.
|
|
1260
|
+
globalThis.__glubeanRuntime.test = {
|
|
1261
|
+
id: test.meta.id,
|
|
1262
|
+
tags: testTags,
|
|
1263
|
+
};
|
|
1264
|
+
console.log(JSON.stringify({
|
|
1265
|
+
type: "start",
|
|
1266
|
+
id: test.meta.id,
|
|
1267
|
+
name: test.meta.name || test.meta.id,
|
|
1268
|
+
tags: testTags,
|
|
1269
|
+
...(retryCount > 0 && { retryCount }),
|
|
1270
|
+
}));
|
|
1271
|
+
// Start memory monitoring
|
|
1272
|
+
startMemoryMonitoring();
|
|
1273
|
+
try {
|
|
1274
|
+
// Core test body — receives the effective ctx (base or fixture-augmented)
|
|
1275
|
+
const runTestBody = async (effectiveCtx) => {
|
|
1276
|
+
if (test.type === "simple") {
|
|
1277
|
+
if (!test.fn) {
|
|
1278
|
+
throw new Error(`Invalid test "${test.meta.id}": missing fn`);
|
|
1279
|
+
}
|
|
1280
|
+
await test.fn(effectiveCtx);
|
|
1281
|
+
}
|
|
1282
|
+
else {
|
|
1283
|
+
let state = undefined;
|
|
1284
|
+
let stepFailed = false;
|
|
1285
|
+
try {
|
|
1286
|
+
if (test.setup) {
|
|
1287
|
+
console.log(JSON.stringify({
|
|
1288
|
+
type: "log",
|
|
1289
|
+
message: "Running setup...",
|
|
1290
|
+
}));
|
|
1291
|
+
state = await test.setup(effectiveCtx);
|
|
1292
|
+
}
|
|
1293
|
+
if (test.steps) {
|
|
1294
|
+
totalSteps = test.steps.length;
|
|
1295
|
+
for (let i = 0; i < test.steps.length; i++) {
|
|
1296
|
+
const step = test.steps[i];
|
|
1297
|
+
// If a previous step failed, skip remaining steps
|
|
1298
|
+
if (stepFailed) {
|
|
1299
|
+
skippedSteps++;
|
|
1300
|
+
console.log(JSON.stringify({
|
|
1301
|
+
type: "step_end",
|
|
1302
|
+
index: i,
|
|
1303
|
+
name: step.meta.name,
|
|
1304
|
+
status: "skipped",
|
|
1305
|
+
durationMs: 0,
|
|
1306
|
+
assertions: 0,
|
|
1307
|
+
failedAssertions: 0,
|
|
1308
|
+
}));
|
|
1309
|
+
continue;
|
|
1310
|
+
}
|
|
1311
|
+
// Reset per-step assertion counters and set step scope
|
|
1312
|
+
stepFailedAssertions = 0;
|
|
1313
|
+
stepAssertionTotal = 0;
|
|
1314
|
+
currentStepIndex = i;
|
|
1315
|
+
const stepStart = performance.now();
|
|
1316
|
+
console.log(JSON.stringify({
|
|
1317
|
+
type: "step_start",
|
|
1318
|
+
index: i,
|
|
1319
|
+
name: step.meta.name,
|
|
1320
|
+
total: test.steps.length,
|
|
1321
|
+
}));
|
|
1322
|
+
let stepError;
|
|
1323
|
+
let stepReturnState = undefined;
|
|
1324
|
+
const retries = step.meta.retries;
|
|
1325
|
+
const configuredRetries = typeof retries === "number" && Number.isFinite(retries)
|
|
1326
|
+
? Math.max(0, Math.floor(retries))
|
|
1327
|
+
: 0;
|
|
1328
|
+
const stepTimeout = step.meta.timeout;
|
|
1329
|
+
const configuredStepTimeout = typeof stepTimeout === "number" && Number.isFinite(stepTimeout)
|
|
1330
|
+
? Math.floor(stepTimeout)
|
|
1331
|
+
: 0;
|
|
1332
|
+
const stepTimeoutMs = configuredStepTimeout > 0 ? configuredStepTimeout : undefined;
|
|
1333
|
+
const maxAttempts = configuredRetries + 1;
|
|
1334
|
+
let attemptsUsed = 0;
|
|
1335
|
+
let lastFailedAssertions = 0;
|
|
1336
|
+
let lastAssertions = 0;
|
|
1337
|
+
let timeoutFailure = false;
|
|
1338
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
1339
|
+
attemptsUsed = attempt;
|
|
1340
|
+
stepError = undefined;
|
|
1341
|
+
stepReturnState = undefined;
|
|
1342
|
+
stepFailedAssertions = 0;
|
|
1343
|
+
stepAssertionTotal = 0;
|
|
1344
|
+
timeoutFailure = false;
|
|
1345
|
+
let stepTimeoutId;
|
|
1346
|
+
try {
|
|
1347
|
+
const stepResult = step.fn(effectiveCtx, state);
|
|
1348
|
+
// Note: timed-out step bodies cannot be force-cancelled in JS.
|
|
1349
|
+
// We treat timeout as terminal (no further retries) to avoid
|
|
1350
|
+
// overlapping attempts mutating shared step context.
|
|
1351
|
+
const result = stepTimeoutMs === undefined ? await stepResult : await Promise.race([
|
|
1352
|
+
stepResult,
|
|
1353
|
+
new Promise((_, reject) => {
|
|
1354
|
+
stepTimeoutId = setTimeout(() => {
|
|
1355
|
+
reject(new StepTimeoutError(step.meta.name, stepTimeoutMs));
|
|
1356
|
+
}, stepTimeoutMs);
|
|
1357
|
+
}),
|
|
1358
|
+
]);
|
|
1359
|
+
if (result !== undefined) {
|
|
1360
|
+
state = result;
|
|
1361
|
+
stepReturnState = result;
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
catch (err) {
|
|
1365
|
+
stepError = err instanceof Error ? err.message : String(err);
|
|
1366
|
+
timeoutFailure = err instanceof StepTimeoutError;
|
|
1367
|
+
}
|
|
1368
|
+
finally {
|
|
1369
|
+
if (stepTimeoutId !== undefined) {
|
|
1370
|
+
clearTimeout(stepTimeoutId);
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
lastFailedAssertions = stepFailedAssertions;
|
|
1374
|
+
lastAssertions = stepAssertionTotal;
|
|
1375
|
+
const attemptFailed = !!stepError || stepFailedAssertions > 0;
|
|
1376
|
+
if (!attemptFailed) {
|
|
1377
|
+
break;
|
|
1378
|
+
}
|
|
1379
|
+
// Timeouts are terminal to avoid overlapping attempts from
|
|
1380
|
+
// dangling async operations in the timed-out step body.
|
|
1381
|
+
if (timeoutFailure) {
|
|
1382
|
+
break;
|
|
1383
|
+
}
|
|
1384
|
+
if (attempt < maxAttempts) {
|
|
1385
|
+
const reason = stepError ? stepError : `${stepFailedAssertions} failed assertion(s)`;
|
|
1386
|
+
console.log(JSON.stringify({
|
|
1387
|
+
type: "log",
|
|
1388
|
+
stepIndex: i,
|
|
1389
|
+
message: `Retrying step "${step.meta.name}" (${attempt + 1}/${maxAttempts}) after failure: ${reason}`,
|
|
1390
|
+
}));
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
const durationMs = Math.round(performance.now() - stepStart);
|
|
1394
|
+
const failed = !!stepError || lastFailedAssertions > 0;
|
|
1395
|
+
// Serialize return state with size guard (max 4 KB)
|
|
1396
|
+
let returnStatePayload = undefined;
|
|
1397
|
+
if (stepReturnState !== undefined) {
|
|
1398
|
+
try {
|
|
1399
|
+
const serialized = JSON.stringify(stepReturnState);
|
|
1400
|
+
if (serialized.length <= 4096) {
|
|
1401
|
+
returnStatePayload = stepReturnState;
|
|
1402
|
+
}
|
|
1403
|
+
else {
|
|
1404
|
+
returnStatePayload = `[truncated: ${serialized.length} bytes]`;
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
catch {
|
|
1408
|
+
returnStatePayload = "[non-serializable]";
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
console.log(JSON.stringify({
|
|
1412
|
+
type: "step_end",
|
|
1413
|
+
index: i,
|
|
1414
|
+
name: step.meta.name,
|
|
1415
|
+
status: failed ? "failed" : "passed",
|
|
1416
|
+
durationMs,
|
|
1417
|
+
assertions: lastAssertions,
|
|
1418
|
+
failedAssertions: lastFailedAssertions,
|
|
1419
|
+
attempts: attemptsUsed,
|
|
1420
|
+
retriesUsed: Math.max(0, attemptsUsed - 1),
|
|
1421
|
+
...(stepError && { error: stepError }),
|
|
1422
|
+
...(returnStatePayload !== undefined && {
|
|
1423
|
+
returnState: returnStatePayload,
|
|
1424
|
+
}),
|
|
1425
|
+
}));
|
|
1426
|
+
currentStepIndex = null;
|
|
1427
|
+
if (failed) {
|
|
1428
|
+
failedSteps++;
|
|
1429
|
+
stepFailed = true;
|
|
1430
|
+
// Don't throw here — let the loop continue to emit skip events
|
|
1431
|
+
}
|
|
1432
|
+
else {
|
|
1433
|
+
passedSteps++;
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
finally {
|
|
1439
|
+
if (test.teardown) {
|
|
1440
|
+
try {
|
|
1441
|
+
console.log(JSON.stringify({
|
|
1442
|
+
type: "log",
|
|
1443
|
+
message: "Running teardown...",
|
|
1444
|
+
}));
|
|
1445
|
+
await test.teardown(effectiveCtx, state);
|
|
1446
|
+
}
|
|
1447
|
+
catch (teardownError) {
|
|
1448
|
+
console.log(JSON.stringify({
|
|
1449
|
+
type: "log",
|
|
1450
|
+
message: `Teardown error: ${teardownError instanceof Error ? teardownError.message : String(teardownError)}`,
|
|
1451
|
+
}));
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
// If any step failed (assertion or throw), mark overall test as failed
|
|
1456
|
+
if (stepFailed) {
|
|
1457
|
+
// Emit summary before throwing so that step/assertion counts are reported
|
|
1458
|
+
emitSummary();
|
|
1459
|
+
throw new Error("One or more steps failed");
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
};
|
|
1463
|
+
// Resolve test.extend() fixtures (if any) and run the test body
|
|
1464
|
+
if (test.fixtures && Object.keys(test.fixtures).length > 0) {
|
|
1465
|
+
await withFixtures(test.fixtures, ctx, runTestBody);
|
|
1466
|
+
}
|
|
1467
|
+
else {
|
|
1468
|
+
await runTestBody(ctx);
|
|
1469
|
+
}
|
|
1470
|
+
// Stop monitoring and get peak memory
|
|
1471
|
+
const peakBytes = stopMemoryMonitoring();
|
|
1472
|
+
// Emit summary before final status
|
|
1473
|
+
emitSummary();
|
|
1474
|
+
console.log(JSON.stringify({
|
|
1475
|
+
type: "status",
|
|
1476
|
+
status: "completed",
|
|
1477
|
+
id: test.meta.id,
|
|
1478
|
+
peakMemoryBytes: peakBytes,
|
|
1479
|
+
peakMemoryMB: (peakBytes / 1024 / 1024).toFixed(2),
|
|
1480
|
+
}));
|
|
1481
|
+
}
|
|
1482
|
+
catch (error) {
|
|
1483
|
+
stopMemoryMonitoring();
|
|
1484
|
+
throw error;
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
//# sourceMappingURL=harness.js.map
|