@emseepea/testing 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -2
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/semantic/case.mjs +16 -4
- package/semantic/test.d.mts +2 -1
- package/semantic/test.mjs +21 -7
package/README.md
CHANGED
|
@@ -30,6 +30,15 @@ never production. Resources and prompts need deterministic protocol tests;
|
|
|
30
30
|
this library does not pretend that manually injecting their content proves a
|
|
31
31
|
native user journey.
|
|
32
32
|
|
|
33
|
+
When a conversation writes to external state, pass an `environment` function
|
|
34
|
+
that returns a separate test database connection for each trial. The trial
|
|
35
|
+
number selects infrastructure only. It is never sent to the model or MCP
|
|
36
|
+
server, and it does not prepare or coach the conversation.
|
|
37
|
+
|
|
38
|
+
For ordinary integration tests, `startMcpServer(...).output()` returns the
|
|
39
|
+
server's captured `stdout` and `stderr`. Use it to prove that logs do not expose
|
|
40
|
+
credentials or provider details.
|
|
41
|
+
|
|
33
42
|
## Diagnose Failures
|
|
34
43
|
|
|
35
44
|
The evidence file contains readable test prompts, assistant responses,
|
|
@@ -38,8 +47,9 @@ meanings, and judge reasons. It also keeps hashes for comparison. A failed
|
|
|
38
47
|
meaning assertion runs and records all nine judgments, so disagreement is
|
|
39
48
|
visible without a rerun.
|
|
40
49
|
|
|
41
|
-
Failed judge invocations record a safe cause such as a
|
|
42
|
-
|
|
50
|
+
Failed answer and judge invocations record a safe cause such as a timeout,
|
|
51
|
+
process exit code, missing result event, or fixed provider error category. The
|
|
52
|
+
failed answer trial is identified even when earlier trials completed.
|
|
43
53
|
|
|
44
54
|
Treat test conversations as publishable artifact content. Use synthetic,
|
|
45
55
|
non-sensitive fixtures and never put credentials or production data in prompts,
|
package/dist/index.d.ts
CHANGED
|
@@ -10,6 +10,10 @@ export interface StartMcpServerOptions {
|
|
|
10
10
|
}
|
|
11
11
|
export interface RunningMcpServer {
|
|
12
12
|
connect(token?: string): Promise<Client>;
|
|
13
|
+
output(): Readonly<{
|
|
14
|
+
stdout: string;
|
|
15
|
+
stderr: string;
|
|
16
|
+
}>;
|
|
13
17
|
url: URL;
|
|
14
18
|
}
|
|
15
19
|
export declare function startMcpServer(test: TestCleanup, serverUrl: URL, options?: StartMcpServerOptions): Promise<RunningMcpServer>;
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAiC,MAAM,8BAA8B,CAAC;AAErF,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,GAAG,EAAE,GAAG,CAAC;CACV;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,GAAG,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAiC,MAAM,8BAA8B,CAAC;AAErF,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,IAAI,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,GAAG,EAAE,GAAG,CAAC;CACV;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,GAAG,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAiE3B"}
|
package/dist/index.js
CHANGED
|
@@ -49,6 +49,7 @@ export async function startMcpServer(test, serverUrl, options = {}) {
|
|
|
49
49
|
});
|
|
50
50
|
return {
|
|
51
51
|
url,
|
|
52
|
+
output: () => Object.freeze({ stdout: output, stderr: errors }),
|
|
52
53
|
async connect(token = options.token) {
|
|
53
54
|
const client = new Client({ name: options.clientName ?? "emseepea-test", version: "0.0.0" }, { versionNegotiation: { mode: { pin: "2026-07-28" } } });
|
|
54
55
|
await client.connect(new StreamableHTTPClientTransport(url, token ? { authProvider: { token: async () => token } } : undefined));
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAmBrF,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAiB,EACjB,SAAc,EACd,UAAiC,EAAE;IAEnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE;QAChE,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,GAAG;YACT,GAAG,OAAO,CAAC,WAAW;SACvB;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,KAAW,EAAE,EAAE;YAC5C,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC1B,IAAI,KAAK;gBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;iBACpB,IAAI,KAAK;gBAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAC7D,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC;QACF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,IAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1G,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,MAAM,EAAE,CAAC,CAAC,EAClE,OAAO,CAAC,gBAAgB,IAAI,MAAM,CACnC,CAAC;QACF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;QACzB,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAC/D,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK;YACjC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,IAAI,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE,EACjE,EAAE,kBAAkB,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,CACxD,CAAC;YACF,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,6BAA6B,CACpD,GAAG,EACH,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,OAAO,MAAM,CAAC;QAChB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAA+B;IACxD,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,OAAO;IACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,CAAC,IAAI,CAAC;QACjB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;KAC3D,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACrD,CAAC"}
|
package/package.json
CHANGED
package/semantic/case.mjs
CHANGED
|
@@ -17,11 +17,23 @@ export function validateConversationOptions(value) {
|
|
|
17
17
|
if (value.authToken !== undefined && value.authTokenEnvironment !== undefined) {
|
|
18
18
|
throw new Error("Choose one authentication source");
|
|
19
19
|
}
|
|
20
|
-
if (value.environment !== undefined &&
|
|
21
|
-
|
|
22
|
-
|| Object.values(value.environment).some((item) => typeof item !== "string"))) {
|
|
23
|
-
throw new Error("environment must contain string values");
|
|
20
|
+
if (value.environment !== undefined && typeof value.environment !== "function") {
|
|
21
|
+
validateEnvironment(value.environment);
|
|
24
22
|
}
|
|
25
23
|
const server = fileURLToPath(value.server);
|
|
26
24
|
return { ...value, server, directory: dirname(server) };
|
|
27
25
|
}
|
|
26
|
+
|
|
27
|
+
export function environmentForTrial(environment, trial) {
|
|
28
|
+
const resolved = typeof environment === "function" ? environment(trial) : environment;
|
|
29
|
+
if (resolved === undefined) return undefined;
|
|
30
|
+
validateEnvironment(resolved);
|
|
31
|
+
return resolved;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function validateEnvironment(environment) {
|
|
35
|
+
if (!environment || typeof environment !== "object" || Array.isArray(environment)
|
|
36
|
+
|| Object.values(environment).some((item) => typeof item !== "string")) {
|
|
37
|
+
throw new Error("environment must contain string values");
|
|
38
|
+
}
|
|
39
|
+
}
|
package/semantic/test.d.mts
CHANGED
|
@@ -9,7 +9,8 @@ export interface ConversationOptions {
|
|
|
9
9
|
server: URL;
|
|
10
10
|
/** Real application context supplied in production. Never use this as test guidance. */
|
|
11
11
|
context?: string;
|
|
12
|
-
environment
|
|
12
|
+
/** Server environment shared by all trials, or isolated values selected by trial number. */
|
|
13
|
+
environment?: Record<string, string> | ((trial: number) => Record<string, string>);
|
|
13
14
|
authToken?: string;
|
|
14
15
|
authTokenEnvironment?: string;
|
|
15
16
|
}
|
package/semantic/test.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { createHash } from "node:crypto";
|
|
|
3
3
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { dirname, join, resolve } from "node:path";
|
|
6
|
-
import { validateConversationOptions } from "./case.mjs";
|
|
6
|
+
import { environmentForTrial, validateConversationOptions } from "./case.mjs";
|
|
7
7
|
import {
|
|
8
8
|
listMcpTools,
|
|
9
9
|
semanticAuthToken,
|
|
@@ -60,7 +60,10 @@ export async function createConversation(testContext, options) {
|
|
|
60
60
|
|
|
61
61
|
try {
|
|
62
62
|
for (let trial = 1; trial <= 3; trial += 1) {
|
|
63
|
-
const running = await startSemanticServer(
|
|
63
|
+
const running = await startSemanticServer({
|
|
64
|
+
...specification,
|
|
65
|
+
environment: environmentForTrial(specification.environment, trial),
|
|
66
|
+
}, testContext.signal);
|
|
64
67
|
try {
|
|
65
68
|
const tools = await listMcpTools(running.url, specification, testContext.signal);
|
|
66
69
|
const record = { trial, turns: [] };
|
|
@@ -83,8 +86,10 @@ export async function createConversation(testContext, options) {
|
|
|
83
86
|
if (typeof prompt !== "string" || !prompt.trim()) throw new Error("send needs a user prompt");
|
|
84
87
|
ensureOpen(state);
|
|
85
88
|
const trials = [];
|
|
89
|
+
let activeTrial;
|
|
86
90
|
try {
|
|
87
91
|
for (const trial of state.trials) {
|
|
92
|
+
activeTrial = trial;
|
|
88
93
|
trial.model ??= startModelConversation(
|
|
89
94
|
provider,
|
|
90
95
|
trial.directory,
|
|
@@ -135,7 +140,8 @@ export async function createConversation(testContext, options) {
|
|
|
135
140
|
history: Object.freeze([...trial.history]),
|
|
136
141
|
});
|
|
137
142
|
}
|
|
138
|
-
} catch {
|
|
143
|
+
} catch (error) {
|
|
144
|
+
if (activeTrial) activeTrial.record.error = safeModelFailure(error);
|
|
139
145
|
state.failed = true;
|
|
140
146
|
evidence.failedPhase = "conversation turn";
|
|
141
147
|
throw new Error(`Semantic test failed during conversation turn: ${name}`);
|
|
@@ -235,7 +241,7 @@ export async function assertResponseMeaning(turn, expectation) {
|
|
|
235
241
|
} catch (error) {
|
|
236
242
|
record.error = error instanceof SyntaxError || error.message === "Judge returned an invalid verdict"
|
|
237
243
|
? "invalid judge verdict"
|
|
238
|
-
:
|
|
244
|
+
: safeModelFailure(error);
|
|
239
245
|
failed = true;
|
|
240
246
|
}
|
|
241
247
|
trial.evidence.judgeVerdicts.push(record);
|
|
@@ -249,7 +255,7 @@ export async function assertResponseMeaning(turn, expectation) {
|
|
|
249
255
|
}
|
|
250
256
|
}
|
|
251
257
|
|
|
252
|
-
function
|
|
258
|
+
function safeModelFailure(error) {
|
|
253
259
|
const message = error instanceof Error ? error.message : "";
|
|
254
260
|
const safeMessages = new Set([
|
|
255
261
|
"Claude subscription authentication is unavailable",
|
|
@@ -270,11 +276,19 @@ function safeJudgeFailure(error) {
|
|
|
270
276
|
"Model command used an unexpected number of turns",
|
|
271
277
|
"Model command was cancelled",
|
|
272
278
|
"Model command did not use the required model",
|
|
279
|
+
"Model command omitted MCP initialization evidence",
|
|
280
|
+
"Model command omitted an MCP tool result",
|
|
281
|
+
"Model command returned no answer",
|
|
282
|
+
"Model command used more than three tools",
|
|
283
|
+
"Model conversation already has a pending turn",
|
|
284
|
+
"Model conversation could not start",
|
|
285
|
+
"Model conversation is closed",
|
|
286
|
+
"Model conversation was cancelled",
|
|
273
287
|
]);
|
|
274
|
-
if (safeMessages.has(message) || /^Model command exited \d{1,3}$/.test(message)) {
|
|
288
|
+
if (safeMessages.has(message) || /^Model (?:command|conversation) exited \d{1,3}$/.test(message)) {
|
|
275
289
|
return message.replace(/^./, (character) => character.toLowerCase());
|
|
276
290
|
}
|
|
277
|
-
return "
|
|
291
|
+
return "model invocation failed";
|
|
278
292
|
}
|
|
279
293
|
|
|
280
294
|
async function isolatedModel(provider, prompt, prefix, signal) {
|