fakeplane 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +12 -0
- package/README.md +225 -2
- package/demo/agents/careful.mjs +89 -0
- package/demo/agents/naive.mjs +50 -0
- package/demo/scenarios/large-refund-escalation.yaml +38 -0
- package/demo/scenarios/refund-ambiguous-timeout.yaml +40 -0
- package/demo/scenarios/refund-duplicate-webhook.yaml +40 -0
- package/dist/agent.d.ts +28 -0
- package/dist/agent.js +113 -0
- package/dist/agent.js.map +1 -0
- package/dist/assertions.d.ts +21 -0
- package/dist/assertions.js +180 -0
- package/dist/assertions.js.map +1 -0
- package/dist/backends/common.d.ts +13 -0
- package/dist/backends/common.js +37 -0
- package/dist/backends/common.js.map +1 -0
- package/dist/backends/generic.d.ts +79 -0
- package/dist/backends/generic.js +227 -0
- package/dist/backends/generic.js.map +1 -0
- package/dist/backends/graphql.d.ts +22 -0
- package/dist/backends/graphql.js +248 -0
- package/dist/backends/graphql.js.map +1 -0
- package/dist/backends/index.d.ts +24 -0
- package/dist/backends/index.js +27 -0
- package/dist/backends/index.js.map +1 -0
- package/dist/backends/linear.d.ts +123 -0
- package/dist/backends/linear.js +451 -0
- package/dist/backends/linear.js.map +1 -0
- package/dist/backends/slack.d.ts +92 -0
- package/dist/backends/slack.js +221 -0
- package/dist/backends/slack.js.map +1 -0
- package/dist/backends/zendesk.d.ts +94 -0
- package/dist/backends/zendesk.js +455 -0
- package/dist/backends/zendesk.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +340 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/matchers.d.ts +11 -0
- package/dist/matchers.js +156 -0
- package/dist/matchers.js.map +1 -0
- package/dist/mcp.d.ts +43 -0
- package/dist/mcp.js +116 -0
- package/dist/mcp.js.map +1 -0
- package/dist/pii.d.ts +17 -0
- package/dist/pii.js +79 -0
- package/dist/pii.js.map +1 -0
- package/dist/report.d.ts +13 -0
- package/dist/report.js +97 -0
- package/dist/report.js.map +1 -0
- package/dist/runner.d.ts +56 -0
- package/dist/runner.js +114 -0
- package/dist/runner.js.map +1 -0
- package/dist/scenario.d.ts +270 -0
- package/dist/scenario.js +270 -0
- package/dist/scenario.js.map +1 -0
- package/dist/server.d.ts +40 -0
- package/dist/server.js +241 -0
- package/dist/server.js.map +1 -0
- package/dist/trace.d.ts +42 -0
- package/dist/trace.js +432 -0
- package/dist/trace.js.map +1 -0
- package/dist/types.d.ts +70 -0
- package/dist/types.js +16 -0
- package/dist/types.js.map +1 -0
- package/dist/util.d.ts +13 -0
- package/dist/util.js +89 -0
- package/dist/util.js.map +1 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +2 -0
- package/dist/version.js.map +1 -0
- package/dist/world.d.ts +149 -0
- package/dist/world.js +187 -0
- package/dist/world.js.map +1 -0
- package/package.json +68 -4
package/dist/mcp.js
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { createInterface } from "node:readline";
|
|
2
|
+
import { VERSION } from "./version.js";
|
|
3
|
+
export const MCP_PROTOCOL_VERSIONS = ["2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05"];
|
|
4
|
+
/** Minimal MCP server exposing every fake operation that has a tool spec. Tools only; no resources or prompts. */
|
|
5
|
+
export class McpHandler {
|
|
6
|
+
getWorld;
|
|
7
|
+
constructor(getWorld) {
|
|
8
|
+
this.getWorld = getWorld;
|
|
9
|
+
}
|
|
10
|
+
listTools() {
|
|
11
|
+
return this.getWorld()
|
|
12
|
+
.tools()
|
|
13
|
+
.sort((a, b) => a.tool.name.localeCompare(b.tool.name))
|
|
14
|
+
.map((op) => ({
|
|
15
|
+
name: op.tool.name,
|
|
16
|
+
description: op.tool.description,
|
|
17
|
+
inputSchema: op.tool.inputSchema,
|
|
18
|
+
annotations: { readOnlyHint: !op.write, idempotentHint: false, openWorldHint: false },
|
|
19
|
+
}));
|
|
20
|
+
}
|
|
21
|
+
async callTool(name, args = {}) {
|
|
22
|
+
const world = this.getWorld();
|
|
23
|
+
const op = world.operation(name);
|
|
24
|
+
if (!op || !op.tool)
|
|
25
|
+
return undefined;
|
|
26
|
+
const key = typeof args.idempotency_key === "string" ? args.idempotency_key : undefined;
|
|
27
|
+
const res = await world.dispatch(op.id, args, { transport: "mcp", idempotencyKey: key, tool: name });
|
|
28
|
+
if (res.ok) {
|
|
29
|
+
return { content: [{ type: "text", text: JSON.stringify(res.output ?? null, null, 2) }], isError: false };
|
|
30
|
+
}
|
|
31
|
+
const err = res.error;
|
|
32
|
+
const retry = err.retryAfter !== undefined ? ` (retry after ${err.retryAfter}s)` : "";
|
|
33
|
+
return { content: [{ type: "text", text: `Error ${res.status} ${err.code}: ${err.message}${retry}` }], isError: true };
|
|
34
|
+
}
|
|
35
|
+
/** Handle one JSON-RPC message or a batch. Returns undefined when nothing should be sent back. */
|
|
36
|
+
async handle(payload) {
|
|
37
|
+
if (Array.isArray(payload)) {
|
|
38
|
+
const out = (await Promise.all(payload.map((m) => this.handleOne(m)))).filter((r) => r !== undefined);
|
|
39
|
+
return out.length ? out : undefined;
|
|
40
|
+
}
|
|
41
|
+
return this.handleOne(payload);
|
|
42
|
+
}
|
|
43
|
+
async handleOne(raw) {
|
|
44
|
+
const msg = raw;
|
|
45
|
+
if (!msg || typeof msg !== "object" || msg.jsonrpc !== "2.0" || typeof msg.method !== "string") {
|
|
46
|
+
// Responses from the client (to server requests we never send) are ignored.
|
|
47
|
+
if (msg && typeof msg === "object" && "id" in msg && ("result" in msg || "error" in msg))
|
|
48
|
+
return undefined;
|
|
49
|
+
return { jsonrpc: "2.0", id: msg?.id ?? null, error: { code: -32600, message: "Invalid Request" } };
|
|
50
|
+
}
|
|
51
|
+
const isNotification = msg.id === undefined;
|
|
52
|
+
const reply = (result) => (isNotification ? undefined : { jsonrpc: "2.0", id: msg.id ?? null, result });
|
|
53
|
+
const fail = (code, message) => (isNotification ? undefined : { jsonrpc: "2.0", id: msg.id ?? null, error: { code, message } });
|
|
54
|
+
switch (msg.method) {
|
|
55
|
+
case "initialize": {
|
|
56
|
+
const requested = msg.params?.protocolVersion;
|
|
57
|
+
const protocolVersion = MCP_PROTOCOL_VERSIONS.includes(requested) ? requested : MCP_PROTOCOL_VERSIONS[0];
|
|
58
|
+
return reply({
|
|
59
|
+
protocolVersion,
|
|
60
|
+
capabilities: { tools: { listChanged: false } },
|
|
61
|
+
serverInfo: { name: "fakeplane", version: VERSION },
|
|
62
|
+
instructions: `Simulated backends for scenario "${this.getWorld().scenario}". Writes are recorded, not sent anywhere.`,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
case "ping":
|
|
66
|
+
return reply({});
|
|
67
|
+
case "tools/list":
|
|
68
|
+
return reply({ tools: this.listTools() });
|
|
69
|
+
case "tools/call": {
|
|
70
|
+
const name = msg.params?.name;
|
|
71
|
+
if (typeof name !== "string")
|
|
72
|
+
return fail(-32602, "Missing tool name");
|
|
73
|
+
const result = await this.callTool(name, msg.params?.arguments ?? {});
|
|
74
|
+
if (!result)
|
|
75
|
+
return fail(-32602, `Unknown tool: ${name}`);
|
|
76
|
+
return reply(result);
|
|
77
|
+
}
|
|
78
|
+
case "resources/list":
|
|
79
|
+
return reply({ resources: [] });
|
|
80
|
+
case "prompts/list":
|
|
81
|
+
return reply({ prompts: [] });
|
|
82
|
+
default:
|
|
83
|
+
if (msg.method.startsWith("notifications/"))
|
|
84
|
+
return undefined;
|
|
85
|
+
return fail(-32601, `Method not found: ${msg.method}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/** Serve MCP over stdio (newline-delimited JSON-RPC). Resolves when stdin closes. */
|
|
90
|
+
export function serveStdio(handler, input = process.stdin, output = process.stdout) {
|
|
91
|
+
return new Promise((resolve) => {
|
|
92
|
+
const rl = createInterface({ input, crlfDelay: Infinity });
|
|
93
|
+
let pending = Promise.resolve();
|
|
94
|
+
rl.on("line", (line) => {
|
|
95
|
+
if (!line.trim())
|
|
96
|
+
return;
|
|
97
|
+
pending = pending.then(async () => {
|
|
98
|
+
let parsed;
|
|
99
|
+
try {
|
|
100
|
+
parsed = JSON.parse(line);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
output.write(JSON.stringify({ jsonrpc: "2.0", id: null, error: { code: -32700, message: "Parse error" } }) + "\n");
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const res = await handler.handle(parsed);
|
|
107
|
+
if (res !== undefined)
|
|
108
|
+
output.write(JSON.stringify(res) + "\n");
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
rl.on("close", () => {
|
|
112
|
+
pending.then(() => resolve());
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=mcp.js.map
|
package/dist/mcp.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mcp.js","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;AAa9F,kHAAkH;AAClH,MAAM,OAAO,UAAU;IACQ;IAA7B,YAA6B,QAAqB;QAArB,aAAQ,GAAR,QAAQ,CAAa;IAAG,CAAC;IAEtD,SAAS;QACP,OAAO,IAAI,CAAC,QAAQ,EAAE;aACnB,KAAK,EAAE;aACP,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAK,CAAC,IAAI,CAAC,CAAC;aACxD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACZ,IAAI,EAAE,EAAE,CAAC,IAAK,CAAC,IAAI;YACnB,WAAW,EAAE,EAAE,CAAC,IAAK,CAAC,WAAW;YACjC,WAAW,EAAE,EAAE,CAAC,IAAK,CAAC,WAAW;YACjC,WAAW,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE;SACtF,CAAC,CAAC,CAAC;IACR,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY,EAAE,OAAgC,EAAE;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI;YAAE,OAAO,SAAS,CAAC;QACtC,MAAM,GAAG,GAAG,OAAO,IAAI,CAAC,eAAe,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC;QACxF,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,cAAc,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACrG,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5G,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,CAAC,KAAM,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,GAAG,KAAK,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACzH,CAAC;IAED,kGAAkG;IAClG,KAAK,CAAC,MAAM,CAAC,OAAgB;QAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAwB,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;YAC5H,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAY;QAClC,MAAM,GAAG,GAAG,GAAqB,CAAC;QAClC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,OAAO,KAAK,KAAK,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/F,4EAA4E;YAC5E,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,IAAI,OAAO,IAAI,GAAG,CAAC;gBAAE,OAAO,SAAS,CAAC;YAC3G,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAG,GAAW,EAAE,EAAE,IAAI,IAAI,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,iBAAiB,EAAE,EAAE,CAAC;QAC/G,CAAC;QACD,MAAM,cAAc,GAAG,GAAG,CAAC,EAAE,KAAK,SAAS,CAAC;QAC5C,MAAM,KAAK,GAAG,CAAC,MAAe,EAA+B,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9I,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,OAAe,EAA+B,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,IAAI,IAAI,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QAE7K,QAAQ,GAAG,CAAC,MAAM,EAAE,CAAC;YACnB,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC;gBAC9C,MAAM,eAAe,GAAG,qBAAqB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;gBACzG,OAAO,KAAK,CAAC;oBACX,eAAe;oBACf,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE;oBAC/C,UAAU,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE;oBACnD,YAAY,EAAE,oCAAoC,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,4CAA4C;iBACvH,CAAC,CAAC;YACL,CAAC;YACD,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,EAAE,CAAC,CAAC;YACnB,KAAK,YAAY;gBACf,OAAO,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YAC5C,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC;gBAC9B,IAAI,OAAO,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC;gBACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;gBACtE,IAAI,CAAC,MAAM;oBAAE,OAAO,IAAI,CAAC,CAAC,KAAK,EAAE,iBAAiB,IAAI,EAAE,CAAC,CAAC;gBAC1D,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YACD,KAAK,gBAAgB;gBACnB,OAAO,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;YAClC,KAAK,cAAc;gBACjB,OAAO,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YAChC;gBACE,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC;oBAAE,OAAO,SAAS,CAAC;gBAC9D,OAAO,IAAI,CAAC,CAAC,KAAK,EAAE,qBAAqB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;CACF;AAED,qFAAqF;AACrF,MAAM,UAAU,UAAU,CAAC,OAAmB,EAAE,QAA+B,OAAO,CAAC,KAAK,EAAE,SAAgC,OAAO,CAAC,MAAM;IAC1I,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3D,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;QAChC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACrB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,OAAO;YACzB,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAChC,IAAI,MAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBACnH,OAAO;gBACT,CAAC;gBACD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACzC,IAAI,GAAG,KAAK,SAAS;oBAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;YAClE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAClB,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/pii.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface PiiFinding {
|
|
2
|
+
type: "email" | "phone" | "card" | "ssn" | "iban" | "value";
|
|
3
|
+
match: string;
|
|
4
|
+
path: string;
|
|
5
|
+
}
|
|
6
|
+
export interface PiiOptions {
|
|
7
|
+
/** Strings or `/regex/` patterns that are allowed to appear (e.g. your support address). */
|
|
8
|
+
allow?: string[];
|
|
9
|
+
/** Extra sensitive literal values to look for (e.g. a seeded customer's date of birth). */
|
|
10
|
+
values?: string[];
|
|
11
|
+
/** Detector types to run. Defaults to all. */
|
|
12
|
+
types?: PiiFinding["type"][];
|
|
13
|
+
}
|
|
14
|
+
/** Scan one string for PII-shaped values. Regex-based: expect false positives and misses. */
|
|
15
|
+
export declare function scanText(text: string, path: string, options?: PiiOptions): PiiFinding[];
|
|
16
|
+
/** Scan every string leaf of a JSON value. */
|
|
17
|
+
export declare function scanValue(value: unknown, options?: PiiOptions, path?: string): PiiFinding[];
|
package/dist/pii.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
|
|
2
|
+
const PHONE = /(?<![\w.])(?:\+?\d{1,3}[\s.-]?)?(?:\(\d{3}\)|\d{3})[\s.-]\d{3}[\s.-]\d{4}(?!\w)|(?<![\w.])\+\d{10,14}(?!\w)/g;
|
|
3
|
+
const CARD = /(?<!\d)(?:\d[ -]?){12,18}\d(?!\d)/g;
|
|
4
|
+
const SSN = /(?<!\d)\d{3}-\d{2}-\d{4}(?!\d)/g;
|
|
5
|
+
const IBAN = /\b[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){3,7}(?: ?[A-Z0-9]{1,3})?\b/g;
|
|
6
|
+
function luhn(digits) {
|
|
7
|
+
let sum = 0;
|
|
8
|
+
let double = false;
|
|
9
|
+
for (let i = digits.length - 1; i >= 0; i--) {
|
|
10
|
+
let d = digits.charCodeAt(i) - 48;
|
|
11
|
+
if (double) {
|
|
12
|
+
d *= 2;
|
|
13
|
+
if (d > 9)
|
|
14
|
+
d -= 9;
|
|
15
|
+
}
|
|
16
|
+
sum += d;
|
|
17
|
+
double = !double;
|
|
18
|
+
}
|
|
19
|
+
return sum % 10 === 0;
|
|
20
|
+
}
|
|
21
|
+
function allowed(match, allow) {
|
|
22
|
+
return allow.some((a) => {
|
|
23
|
+
const m = /^\/(.*)\/([a-z]*)$/s.exec(a);
|
|
24
|
+
if (m)
|
|
25
|
+
return new RegExp(m[1], m[2]).test(match);
|
|
26
|
+
return a.toLowerCase() === match.toLowerCase();
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
/** Scan one string for PII-shaped values. Regex-based: expect false positives and misses. */
|
|
30
|
+
export function scanText(text, path, options = {}) {
|
|
31
|
+
const types = new Set(options.types ?? ["email", "phone", "card", "ssn", "iban", "value"]);
|
|
32
|
+
const allow = options.allow ?? [];
|
|
33
|
+
const findings = [];
|
|
34
|
+
const push = (type, match) => {
|
|
35
|
+
if (!allowed(match, allow))
|
|
36
|
+
findings.push({ type, match, path });
|
|
37
|
+
};
|
|
38
|
+
if (types.has("email"))
|
|
39
|
+
for (const m of text.match(EMAIL) ?? [])
|
|
40
|
+
push("email", m);
|
|
41
|
+
if (types.has("ssn"))
|
|
42
|
+
for (const m of text.match(SSN) ?? [])
|
|
43
|
+
push("ssn", m);
|
|
44
|
+
if (types.has("card")) {
|
|
45
|
+
for (const m of text.match(CARD) ?? []) {
|
|
46
|
+
const digits = m.replace(/\D/g, "");
|
|
47
|
+
if (digits.length >= 13 && digits.length <= 19 && luhn(digits))
|
|
48
|
+
push("card", m.trim());
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (types.has("phone")) {
|
|
52
|
+
for (const m of text.match(PHONE) ?? []) {
|
|
53
|
+
if (!/^\d{3}-\d{2}-\d{4}$/.test(m.trim()))
|
|
54
|
+
push("phone", m.trim());
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (types.has("iban"))
|
|
58
|
+
for (const m of text.match(IBAN) ?? [])
|
|
59
|
+
push("iban", m);
|
|
60
|
+
if (types.has("value")) {
|
|
61
|
+
for (const v of options.values ?? []) {
|
|
62
|
+
if (v && text.toLowerCase().includes(v.toLowerCase()))
|
|
63
|
+
push("value", v);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return findings;
|
|
67
|
+
}
|
|
68
|
+
/** Scan every string leaf of a JSON value. */
|
|
69
|
+
export function scanValue(value, options = {}, path = "") {
|
|
70
|
+
if (typeof value === "string")
|
|
71
|
+
return scanText(value, path || ".", options);
|
|
72
|
+
if (Array.isArray(value))
|
|
73
|
+
return value.flatMap((v, i) => scanValue(v, options, `${path}[${i}]`));
|
|
74
|
+
if (value && typeof value === "object") {
|
|
75
|
+
return Object.entries(value).flatMap(([k, v]) => scanValue(v, options, path ? `${path}.${k}` : k));
|
|
76
|
+
}
|
|
77
|
+
return [];
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=pii.js.map
|
package/dist/pii.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pii.js","sourceRoot":"","sources":["../src/pii.ts"],"names":[],"mappings":"AAeA,MAAM,KAAK,GAAG,iDAAiD,CAAC;AAChE,MAAM,KAAK,GAAG,8GAA8G,CAAC;AAC7H,MAAM,IAAI,GAAG,oCAAoC,CAAC;AAClD,MAAM,GAAG,GAAG,iCAAiC,CAAC;AAC9C,MAAM,IAAI,GAAG,8DAA8D,CAAC;AAE5E,SAAS,IAAI,CAAC,MAAc;IAC1B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,MAAM,EAAE,CAAC;YACX,CAAC,IAAI,CAAC,CAAC;YACP,IAAI,CAAC,GAAG,CAAC;gBAAE,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QACD,GAAG,IAAI,CAAC,CAAC;QACT,MAAM,GAAG,CAAC,MAAM,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,OAAO,CAAC,KAAa,EAAE,KAAe;IAC7C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;QACtB,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC;YAAE,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,OAAO,CAAC,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,IAAY,EAAE,UAAsB,EAAE;IAC3E,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,CAAC,IAAwB,EAAE,KAAa,EAAE,EAAE;QACvD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,CAAC,CAAC;IACF,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;YAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAClF,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5E,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;QAAE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/E,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,SAAS,CAAC,KAAc,EAAE,UAAsB,EAAE,EAAE,IAAI,GAAG,EAAE;IAC3E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC,KAAK,EAAE,IAAI,IAAI,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5E,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACjG,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrG,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
package/dist/report.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RunReport, ScenarioResult } from "./runner.js";
|
|
2
|
+
export declare const color: {
|
|
3
|
+
red: (s: string) => string;
|
|
4
|
+
green: (s: string) => string;
|
|
5
|
+
yellow: (s: string) => string;
|
|
6
|
+
dim: (s: string) => string;
|
|
7
|
+
bold: (s: string) => string;
|
|
8
|
+
};
|
|
9
|
+
export declare function formatScenario(r: ScenarioResult, verbose?: boolean): string;
|
|
10
|
+
export declare function formatSummary(report: RunReport): string;
|
|
11
|
+
/** JUnit XML: one testsuite per scenario, one testcase per assertion. */
|
|
12
|
+
export declare function toJUnit(report: RunReport): string;
|
|
13
|
+
export declare function toMarkdown(report: RunReport): string;
|
package/dist/report.js
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { relative } from "node:path";
|
|
2
|
+
const useColor = () => process.stdout.isTTY && !process.env.NO_COLOR;
|
|
3
|
+
const paint = (code) => (s) => (useColor() ? `\x1b[${code}m${s}\x1b[0m` : s);
|
|
4
|
+
export const color = { red: paint(31), green: paint(32), yellow: paint(33), dim: paint(2), bold: paint(1) };
|
|
5
|
+
const rel = (file) => {
|
|
6
|
+
const r = relative(process.cwd(), file).replace(/\\/g, "/") || file;
|
|
7
|
+
return r.startsWith("../") ? r.split("/").slice(-3).join("/") : r;
|
|
8
|
+
};
|
|
9
|
+
function badge(status) {
|
|
10
|
+
if (status === "pass")
|
|
11
|
+
return color.green(" PASS ");
|
|
12
|
+
if (status === "error")
|
|
13
|
+
return color.yellow(" ERROR");
|
|
14
|
+
return color.red(" FAIL ");
|
|
15
|
+
}
|
|
16
|
+
export function formatScenario(r, verbose = false) {
|
|
17
|
+
const lines = [];
|
|
18
|
+
const reps = r.attempts.length > 1 ? ` ${Math.round(r.passRate * 100)}% of ${r.attempts.length} runs passed` : "";
|
|
19
|
+
lines.push(`${badge(r.status)} ${color.bold(r.name)} ${color.dim(`${rel(r.file)} (${r.durationMs}ms)${reps}`)}`);
|
|
20
|
+
const a = r.shown;
|
|
21
|
+
if (a.error)
|
|
22
|
+
lines.push(` ${color.yellow("!")} agent error: ${a.error}`);
|
|
23
|
+
for (const res of a.assertions) {
|
|
24
|
+
if (res.ok && !verbose && r.status === "pass")
|
|
25
|
+
continue;
|
|
26
|
+
lines.push(res.ok ? ` ${color.green("✓")} ${res.name}` : ` ${color.red("✗")} ${res.name}\n ${color.dim(res.message)}`);
|
|
27
|
+
}
|
|
28
|
+
if (r.status !== "pass" || verbose) {
|
|
29
|
+
lines.push(color.dim(` ${a.calls.length} calls, ${a.effects.length} side effects, ${a.turns.length} agent deliveries`));
|
|
30
|
+
}
|
|
31
|
+
return lines.join("\n");
|
|
32
|
+
}
|
|
33
|
+
export function formatSummary(report) {
|
|
34
|
+
const total = report.scenarios.length;
|
|
35
|
+
const parts = [color.green(`${report.passed} passed`)];
|
|
36
|
+
if (report.failed)
|
|
37
|
+
parts.push(color.red(`${report.failed} failed`));
|
|
38
|
+
if (report.errored)
|
|
39
|
+
parts.push(color.yellow(`${report.errored} errored`));
|
|
40
|
+
return `\n${parts.join(", ")} (${total} scenario${total === 1 ? "" : "s"}) in ${(report.durationMs / 1000).toFixed(1)}s`;
|
|
41
|
+
}
|
|
42
|
+
function xml(s) {
|
|
43
|
+
return s
|
|
44
|
+
.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g, "")
|
|
45
|
+
.replace(/&/g, "&")
|
|
46
|
+
.replace(/</g, "<")
|
|
47
|
+
.replace(/>/g, ">")
|
|
48
|
+
.replace(/"/g, """);
|
|
49
|
+
}
|
|
50
|
+
/** JUnit XML: one testsuite per scenario, one testcase per assertion. */
|
|
51
|
+
export function toJUnit(report) {
|
|
52
|
+
const out = ['<?xml version="1.0" encoding="UTF-8"?>'];
|
|
53
|
+
let tests = 0;
|
|
54
|
+
let failures = 0;
|
|
55
|
+
let errors = 0;
|
|
56
|
+
const suites = [];
|
|
57
|
+
for (const r of report.scenarios) {
|
|
58
|
+
const a = r.shown;
|
|
59
|
+
const cases = [];
|
|
60
|
+
const classname = xml(rel(r.file));
|
|
61
|
+
if (a.error) {
|
|
62
|
+
errors++;
|
|
63
|
+
cases.push(` <testcase classname="${classname}" name="${xml(r.name)}: agent run" time="0"><error message="${xml(a.error)}">${xml(a.error)}</error></testcase>`);
|
|
64
|
+
}
|
|
65
|
+
for (const res of a.assertions) {
|
|
66
|
+
const body = res.ok ? "" : `<failure message="${xml(res.message)}">${xml(res.message)}</failure>`;
|
|
67
|
+
if (!res.ok)
|
|
68
|
+
failures++;
|
|
69
|
+
cases.push(` <testcase classname="${classname}" name="${xml(r.name)}: ${xml(res.name)}" time="0">${body}</testcase>`);
|
|
70
|
+
}
|
|
71
|
+
if (r.attempts.length > 1) {
|
|
72
|
+
const ok = r.status === "pass";
|
|
73
|
+
if (!ok)
|
|
74
|
+
failures++;
|
|
75
|
+
cases.push(` <testcase classname="${classname}" name="${xml(r.name)}: pass rate" time="0">${ok ? "" : `<failure message="pass rate ${Math.round(r.passRate * 100)}%">pass rate ${Math.round(r.passRate * 100)}% across ${r.attempts.length} runs</failure>`}</testcase>`);
|
|
76
|
+
}
|
|
77
|
+
tests += cases.length;
|
|
78
|
+
const caseFailures = cases.filter((c) => c.includes("<failure")).length;
|
|
79
|
+
const caseErrors = cases.filter((c) => c.includes("<error")).length;
|
|
80
|
+
suites.push(` <testsuite name="${xml(r.name)}" tests="${cases.length}" failures="${caseFailures}" errors="${caseErrors}" time="${(r.durationMs / 1000).toFixed(3)}">\n${cases.join("\n")}\n </testsuite>`);
|
|
81
|
+
}
|
|
82
|
+
out.push(`<testsuites name="fakeplane" tests="${tests}" failures="${failures}" errors="${errors}" time="${(report.durationMs / 1000).toFixed(3)}">`);
|
|
83
|
+
out.push(...suites);
|
|
84
|
+
out.push("</testsuites>");
|
|
85
|
+
return out.join("\n") + "\n";
|
|
86
|
+
}
|
|
87
|
+
export function toMarkdown(report) {
|
|
88
|
+
const icon = (s) => (s === "pass" ? "✅" : s === "error" ? "⚠️" : "❌");
|
|
89
|
+
const lines = [`### fakeplane: ${report.passed}/${report.scenarios.length} scenarios passed`, "", "| | Scenario | Result |", "|---|---|---|"];
|
|
90
|
+
for (const r of report.scenarios) {
|
|
91
|
+
const failed = r.shown.assertions.filter((a) => !a.ok);
|
|
92
|
+
const detail = r.shown.error ? `agent error: ${r.shown.error}` : failed.length ? failed.map((a) => `${a.name}: ${a.message}`).join("<br>") : `${r.shown.assertions.length} assertions passed`;
|
|
93
|
+
lines.push(`| ${icon(r.status)} | ${r.name.replace(/\|/g, "\\|")} | ${detail.replace(/\|/g, "\\|").replace(/\n/g, " ").slice(0, 600)} |`);
|
|
94
|
+
}
|
|
95
|
+
return lines.join("\n") + "\n";
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AACrE,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7F,MAAM,CAAC,MAAM,KAAK,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAE5G,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE;IAC3B,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC;IACpE,OAAO,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEF,SAAS,KAAK,CAAC,MAAc;IAC3B,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAiB,EAAE,OAAO,GAAG,KAAK;IAC/D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IACjH,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;IAClB,IAAI,CAAC,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3E,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;QAC/B,IAAI,GAAG,CAAC,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM;YAAE,SAAS;QACxD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,UAAU,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACjI,CAAC;IACD,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW,CAAC,CAAC,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC,KAAK,CAAC,MAAM,mBAAmB,CAAC,CAAC,CAAC;IAC9H,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAiB;IAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;IACtC,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC;IACvD,IAAI,MAAM,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC;IACpE,IAAI,MAAM,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC;IAC1E,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,YAAY,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC3H,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,CAAC;SACL,OAAO,CAAC,2CAA2C,EAAE,EAAE,CAAC;SACxD,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,OAAO,CAAC,MAAiB;IACvC,MAAM,GAAG,GAAa,CAAC,wCAAwC,CAAC,CAAC;IACjE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;QAClB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,EAAE,CAAC;YACT,KAAK,CAAC,IAAI,CAAC,4BAA4B,SAAS,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,yCAAyC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACrK,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qBAAqB,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC;YAClG,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,QAAQ,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,4BAA4B,SAAS,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,aAAa,CAAC,CAAC;QAC3H,CAAC;QACD,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC;YAC/B,IAAI,CAAC,EAAE;gBAAE,QAAQ,EAAE,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,4BAA4B,SAAS,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,+BAA+B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,gBAAgB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,MAAM,iBAAiB,aAAa,CAAC,CAAC;QAC/Q,CAAC;QACD,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;QACtB,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;QACxE,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,CAAC,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,MAAM,eAAe,YAAY,aAAa,UAAU,WAAW,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC/M,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,uCAAuC,KAAK,eAAe,QAAQ,aAAa,MAAM,WAAW,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrJ,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IACpB,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC1B,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAiB;IAC1C,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAG,CAAC,kBAAkB,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,mBAAmB,EAAE,EAAE,EAAE,yBAAyB,EAAE,eAAe,CAAC,CAAC;IAC9I,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,oBAAoB,CAAC;QAC9L,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC5I,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC"}
|
package/dist/runner.d.ts
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { type AssertionResult } from "./assertions.js";
|
|
2
|
+
import { type AgentSpec, type Assertion, type LoadedScenario, type Scenario } from "./scenario.js";
|
|
3
|
+
import type { CallRecord, Effect, JsonObject, TurnRecord } from "./types.js";
|
|
4
|
+
import type { World } from "./world.js";
|
|
5
|
+
export type Status = "pass" | "fail" | "error";
|
|
6
|
+
export interface AttemptResult {
|
|
7
|
+
attempt: number;
|
|
8
|
+
status: Status;
|
|
9
|
+
durationMs: number;
|
|
10
|
+
assertions: AssertionResult[];
|
|
11
|
+
turns: TurnRecord[];
|
|
12
|
+
calls: CallRecord[];
|
|
13
|
+
effects: Effect[];
|
|
14
|
+
state: Record<string, unknown>;
|
|
15
|
+
error?: string;
|
|
16
|
+
}
|
|
17
|
+
export interface ScenarioResult {
|
|
18
|
+
file: string;
|
|
19
|
+
name: string;
|
|
20
|
+
status: Status;
|
|
21
|
+
agent: string;
|
|
22
|
+
durationMs: number;
|
|
23
|
+
passRate: number;
|
|
24
|
+
attempts: AttemptResult[];
|
|
25
|
+
/** The attempt the report shows: the first failing one, else the last. */
|
|
26
|
+
shown: AttemptResult;
|
|
27
|
+
}
|
|
28
|
+
export interface RunReport {
|
|
29
|
+
fakeplane: string;
|
|
30
|
+
startedAt: string;
|
|
31
|
+
durationMs: number;
|
|
32
|
+
passed: number;
|
|
33
|
+
failed: number;
|
|
34
|
+
errored: number;
|
|
35
|
+
scenarios: ScenarioResult[];
|
|
36
|
+
}
|
|
37
|
+
export interface RunOptions {
|
|
38
|
+
/** Overrides the scenario's and config's agent. */
|
|
39
|
+
agent?: AgentSpec;
|
|
40
|
+
/** Used when neither the scenario nor `agent` sets one. */
|
|
41
|
+
defaultAgent?: AgentSpec;
|
|
42
|
+
defaultAgentDir?: string;
|
|
43
|
+
/** Appended to every scenario's assertions. */
|
|
44
|
+
extraAssertions?: Assertion[];
|
|
45
|
+
repeat?: number;
|
|
46
|
+
filter?: string;
|
|
47
|
+
bail?: boolean;
|
|
48
|
+
onScenarioStart?(s: LoadedScenario): void;
|
|
49
|
+
onScenarioDone?(r: ScenarioResult): void;
|
|
50
|
+
}
|
|
51
|
+
export declare function buildWorld(scenario: Scenario): World;
|
|
52
|
+
/** Apply a user turn to the world and return the event payload the agent receives. */
|
|
53
|
+
export declare function injectTurn(world: World, turn: Scenario["turns"][number], deliveryId: string): JsonObject;
|
|
54
|
+
export declare function runAttempt(loaded: LoadedScenario, agent: AgentSpec, agentDir: string, extra: Assertion[], attempt: number): Promise<AttemptResult>;
|
|
55
|
+
export declare function runScenario(loaded: LoadedScenario, options?: RunOptions): Promise<ScenarioResult>;
|
|
56
|
+
export declare function runScenarios(list: LoadedScenario[], options?: RunOptions): Promise<RunReport>;
|
package/dist/runner.js
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { evaluateAssertions } from "./assertions.js";
|
|
2
|
+
import { describeAgent, invokeAgent } from "./agent.js";
|
|
3
|
+
import { createWorld } from "./backends/index.js";
|
|
4
|
+
import { AssertionSchema } from "./scenario.js";
|
|
5
|
+
import { FakeServer } from "./server.js";
|
|
6
|
+
import { sleep } from "./util.js";
|
|
7
|
+
import { VERSION } from "./version.js";
|
|
8
|
+
export function buildWorld(scenario) {
|
|
9
|
+
return createWorld({ scenario: scenario.name, clock: scenario.clock, seed: scenario.seed, faults: scenario.faults, backends: scenario.backends });
|
|
10
|
+
}
|
|
11
|
+
/** Apply a user turn to the world and return the event payload the agent receives. */
|
|
12
|
+
export function injectTurn(world, turn, deliveryId) {
|
|
13
|
+
if (turn.event)
|
|
14
|
+
return { ...turn.event };
|
|
15
|
+
if (turn.via !== "none") {
|
|
16
|
+
const backend = world.backends.get(turn.via);
|
|
17
|
+
if (!backend)
|
|
18
|
+
throw new Error(`turn uses via: ${turn.via}, but the ${turn.via} backend is not enabled in this scenario`);
|
|
19
|
+
if (backend.injectTurn)
|
|
20
|
+
return backend.injectTurn(turn, { world, deliveryId });
|
|
21
|
+
}
|
|
22
|
+
return { type: "message", id: deliveryId, text: turn.text ?? "" };
|
|
23
|
+
}
|
|
24
|
+
export async function runAttempt(loaded, agent, agentDir, extra, attempt) {
|
|
25
|
+
const { scenario } = loaded;
|
|
26
|
+
const started = Date.now();
|
|
27
|
+
const world = buildWorld(scenario);
|
|
28
|
+
const server = new FakeServer(world);
|
|
29
|
+
const urls = await server.start();
|
|
30
|
+
const turns = [];
|
|
31
|
+
const history = [];
|
|
32
|
+
let error;
|
|
33
|
+
try {
|
|
34
|
+
for (let i = 0; i < scenario.turns.length; i++) {
|
|
35
|
+
const turn = scenario.turns[i];
|
|
36
|
+
const turnNo = i + 1;
|
|
37
|
+
world.currentTurn = turnNo;
|
|
38
|
+
if (turn.delayMs)
|
|
39
|
+
await sleep(turn.delayMs);
|
|
40
|
+
const deliveryId = `evt_${String(turnNo).padStart(3, "0")}`;
|
|
41
|
+
const event = injectTurn(world, turn, deliveryId);
|
|
42
|
+
const copies = turn.deliver ?? 1;
|
|
43
|
+
const deliver = async (delivery) => {
|
|
44
|
+
const payload = { fakeplane: VERSION, scenario: scenario.name, turn: turnNo, delivery, deliveryId, via: turn.via, text: turn.text, event, history: [...history], urls };
|
|
45
|
+
const r = await invokeAgent(agent, payload, agentDir);
|
|
46
|
+
return { turn: turnNo, delivery, deliveryId, via: turn.via, text: turn.text, event, output: r.output, error: r.error, durationMs: r.durationMs };
|
|
47
|
+
};
|
|
48
|
+
const records = [];
|
|
49
|
+
if (turn.concurrent)
|
|
50
|
+
records.push(...(await Promise.all(Array.from({ length: copies }, (_, d) => deliver(d + 1)))));
|
|
51
|
+
else
|
|
52
|
+
for (let d = 1; d <= copies; d++)
|
|
53
|
+
records.push(await deliver(d));
|
|
54
|
+
turns.push(...records);
|
|
55
|
+
history.push({ turn: turnNo, text: turn.text, output: records.map((r) => r.output).filter(Boolean).join("\n") });
|
|
56
|
+
const failed = records.find((r) => r.error);
|
|
57
|
+
if (failed && !scenario.allowAgentErrors) {
|
|
58
|
+
error = `turn ${turnNo}${copies > 1 ? ` delivery ${failed.delivery}` : ""}: ${failed.error}`;
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
world.currentTurn = undefined;
|
|
63
|
+
const assertions = await evaluateAssertions([...scenario.assert, ...extra], { world, turns, dir: loaded.dir });
|
|
64
|
+
if (assertions.length === 0 && !error) {
|
|
65
|
+
assertions.push({ name: "scenario has assertions", ok: false, message: "add at least one entry under `assert:`; a scenario without assertions cannot fail" });
|
|
66
|
+
}
|
|
67
|
+
const status = error ? "error" : assertions.every((a) => a.ok) ? "pass" : "fail";
|
|
68
|
+
return { attempt, status, durationMs: Date.now() - started, assertions, turns, calls: world.calls, effects: world.effects, state: world.snapshot(), error };
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
await server.stop();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export async function runScenario(loaded, options = {}) {
|
|
75
|
+
const { scenario } = loaded;
|
|
76
|
+
const agent = options.agent ?? scenario.agent ?? options.defaultAgent ?? {};
|
|
77
|
+
const agentDir = options.agent ? process.cwd() : scenario.agent ? loaded.dir : (options.defaultAgentDir ?? process.cwd());
|
|
78
|
+
const repeat = options.repeat ?? scenario.repeat;
|
|
79
|
+
const extra = (options.extraAssertions ?? []).map((a) => AssertionSchema.parse(a));
|
|
80
|
+
const started = Date.now();
|
|
81
|
+
const attempts = [];
|
|
82
|
+
for (let i = 1; i <= repeat; i++)
|
|
83
|
+
attempts.push(await runAttempt(loaded, agent, agentDir, extra, i));
|
|
84
|
+
const passes = attempts.filter((a) => a.status === "pass").length;
|
|
85
|
+
const passRate = passes / attempts.length;
|
|
86
|
+
const anyError = attempts.some((a) => a.status === "error");
|
|
87
|
+
const status = passRate >= scenario.minPassRate ? "pass" : anyError && passes === 0 ? "error" : "fail";
|
|
88
|
+
const shown = attempts.find((a) => a.status !== "pass") ?? attempts[attempts.length - 1];
|
|
89
|
+
return { file: loaded.file, name: scenario.name, status, agent: describeAgent(agent), durationMs: Date.now() - started, passRate, attempts, shown };
|
|
90
|
+
}
|
|
91
|
+
export async function runScenarios(list, options = {}) {
|
|
92
|
+
const startedAt = new Date().toISOString();
|
|
93
|
+
const started = Date.now();
|
|
94
|
+
const results = [];
|
|
95
|
+
const selected = options.filter ? list.filter((s) => s.scenario.name.includes(options.filter) || s.file.includes(options.filter)) : list;
|
|
96
|
+
for (const s of selected) {
|
|
97
|
+
options.onScenarioStart?.(s);
|
|
98
|
+
const r = await runScenario(s, options);
|
|
99
|
+
results.push(r);
|
|
100
|
+
options.onScenarioDone?.(r);
|
|
101
|
+
if (options.bail && r.status !== "pass")
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
fakeplane: VERSION,
|
|
106
|
+
startedAt,
|
|
107
|
+
durationMs: Date.now() - started,
|
|
108
|
+
passed: results.filter((r) => r.status === "pass").length,
|
|
109
|
+
failed: results.filter((r) => r.status === "fail").length,
|
|
110
|
+
errored: results.filter((r) => r.status === "error").length,
|
|
111
|
+
scenarios: results,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAwB,MAAM,iBAAiB,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,WAAW,EAAoB,MAAM,YAAY,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAsE,MAAM,eAAe,CAAC;AACpH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAqDvC,MAAM,UAAU,UAAU,CAAC,QAAkB;IAC3C,OAAO,WAAW,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;AACpJ,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,UAAU,CAAC,KAAY,EAAE,IAA+B,EAAE,UAAkB;IAC1F,IAAI,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IACzC,IAAI,IAAI,CAAC,GAAG,KAAK,MAAM,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,CAAC,GAAG,aAAa,IAAI,CAAC,GAAG,0CAA0C,CAAC,CAAC;QACzH,IAAI,OAAO,CAAC,UAAU;YAAE,OAAO,OAAO,CAAC,UAAU,CAAC,IAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,MAAsB,EAAE,KAAgB,EAAE,QAAgB,EAAE,KAAkB,EAAE,OAAe;IAC9H,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IAClC,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,KAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;YACrB,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC;YAC3B,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC5C,MAAM,UAAU,GAAG,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,EAAE,QAAgB,EAAuB,EAAE;gBAC9D,MAAM,OAAO,GAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC;gBACrL,MAAM,CAAC,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACtD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC;YACnJ,CAAC,CAAC;YACF,MAAM,OAAO,GAAiB,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,UAAU;gBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;gBAC/G,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE;oBAAE,OAAO,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACtE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjH,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5C,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;gBACzC,KAAK,GAAG,QAAQ,MAAM,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC7F,MAAM;YACR,CAAC;QACH,CAAC;QACD,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC;QAC9B,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/G,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACtC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,yBAAyB,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,mFAAmF,EAAE,CAAC,CAAC;QAChK,CAAC;QACD,MAAM,MAAM,GAAW,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QACzF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC;IAC9J,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAsB,EAAE,UAAsB,EAAE;IAChF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1H,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC;IACjD,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IACrG,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAW,QAAQ,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/G,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzF,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AACtJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB,EAAE,UAAsB,EAAE;IACjF,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3I,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM;YAAE,MAAM;IACjD,CAAC;IACD,OAAO;QACL,SAAS,EAAE,OAAO;QAClB,SAAS;QACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;QAChC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACzD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACzD,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;QAC3D,SAAS,EAAE,OAAO;KACnB,CAAC;AACJ,CAAC"}
|