@emseepea/testing 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/semantic/case.mjs +16 -4
- package/semantic/test.d.mts +2 -1
- package/semantic/test.mjs +5 -2
package/README.md
CHANGED
|
@@ -30,6 +30,15 @@ never production. Resources and prompts need deterministic protocol tests;
|
|
|
30
30
|
this library does not pretend that manually injecting their content proves a
|
|
31
31
|
native user journey.
|
|
32
32
|
|
|
33
|
+
When a conversation writes to external state, pass an `environment` function
|
|
34
|
+
that returns a separate test database connection for each trial. The trial
|
|
35
|
+
number selects infrastructure only. It is never sent to the model or MCP
|
|
36
|
+
server, and it does not prepare or coach the conversation.
|
|
37
|
+
|
|
38
|
+
For ordinary integration tests, `startMcpServer(...).output()` returns the
|
|
39
|
+
server's captured `stdout` and `stderr`. Use it to prove that logs do not expose
|
|
40
|
+
credentials or provider details.
|
|
41
|
+
|
|
33
42
|
## Diagnose Failures
|
|
34
43
|
|
|
35
44
|
The evidence file contains readable test prompts, assistant responses,
|
package/dist/index.d.ts
CHANGED
|
@@ -10,6 +10,10 @@ export interface StartMcpServerOptions {
|
|
|
10
10
|
}
|
|
11
11
|
export interface RunningMcpServer {
|
|
12
12
|
connect(token?: string): Promise<Client>;
|
|
13
|
+
output(): Readonly<{
|
|
14
|
+
stdout: string;
|
|
15
|
+
stderr: string;
|
|
16
|
+
}>;
|
|
13
17
|
url: URL;
|
|
14
18
|
}
|
|
15
19
|
export declare function startMcpServer(test: TestCleanup, serverUrl: URL, options?: StartMcpServerOptions): Promise<RunningMcpServer>;
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAiC,MAAM,8BAA8B,CAAC;AAErF,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,GAAG,EAAE,GAAG,CAAC;CACV;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,GAAG,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAiC,MAAM,8BAA8B,CAAC;AAErF,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,IAAI,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,GAAG,EAAE,GAAG,CAAC;CACV;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,GAAG,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAiE3B"}
|
package/dist/index.js
CHANGED
|
@@ -49,6 +49,7 @@ export async function startMcpServer(test, serverUrl, options = {}) {
|
|
|
49
49
|
});
|
|
50
50
|
return {
|
|
51
51
|
url,
|
|
52
|
+
output: () => Object.freeze({ stdout: output, stderr: errors }),
|
|
52
53
|
async connect(token = options.token) {
|
|
53
54
|
const client = new Client({ name: options.clientName ?? "emseepea-test", version: "0.0.0" }, { versionNegotiation: { mode: { pin: "2026-07-28" } } });
|
|
54
55
|
await client.connect(new StreamableHTTPClientTransport(url, token ? { authProvider: { token: async () => token } } : undefined));
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAmBrF,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAiB,EACjB,SAAc,EACd,UAAiC,EAAE;IAEnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE;QAChE,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,GAAG;YACT,GAAG,OAAO,CAAC,WAAW;SACvB;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,KAAW,EAAE,EAAE;YAC5C,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC1B,IAAI,KAAK;gBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;iBACpB,IAAI,KAAK;gBAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAC7D,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC;QACF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,IAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1G,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,MAAM,EAAE,CAAC,CAAC,EAClE,OAAO,CAAC,gBAAgB,IAAI,MAAM,CACnC,CAAC;QACF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;QACzB,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAC/D,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK;YACjC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,IAAI,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE,EACjE,EAAE,kBAAkB,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,CACxD,CAAC;YACF,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,6BAA6B,CACpD,GAAG,EACH,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,OAAO,MAAM,CAAC;QAChB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAA+B;IACxD,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,OAAO;IACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,CAAC,IAAI,CAAC;QACjB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;KAC3D,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACrD,CAAC"}
|
package/package.json
CHANGED
package/semantic/case.mjs
CHANGED
|
@@ -17,11 +17,23 @@ export function validateConversationOptions(value) {
|
|
|
17
17
|
if (value.authToken !== undefined && value.authTokenEnvironment !== undefined) {
|
|
18
18
|
throw new Error("Choose one authentication source");
|
|
19
19
|
}
|
|
20
|
-
if (value.environment !== undefined &&
|
|
21
|
-
|
|
22
|
-
|| Object.values(value.environment).some((item) => typeof item !== "string"))) {
|
|
23
|
-
throw new Error("environment must contain string values");
|
|
20
|
+
if (value.environment !== undefined && typeof value.environment !== "function") {
|
|
21
|
+
validateEnvironment(value.environment);
|
|
24
22
|
}
|
|
25
23
|
const server = fileURLToPath(value.server);
|
|
26
24
|
return { ...value, server, directory: dirname(server) };
|
|
27
25
|
}
|
|
26
|
+
|
|
27
|
+
export function environmentForTrial(environment, trial) {
|
|
28
|
+
const resolved = typeof environment === "function" ? environment(trial) : environment;
|
|
29
|
+
if (resolved === undefined) return undefined;
|
|
30
|
+
validateEnvironment(resolved);
|
|
31
|
+
return resolved;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function validateEnvironment(environment) {
|
|
35
|
+
if (!environment || typeof environment !== "object" || Array.isArray(environment)
|
|
36
|
+
|| Object.values(environment).some((item) => typeof item !== "string")) {
|
|
37
|
+
throw new Error("environment must contain string values");
|
|
38
|
+
}
|
|
39
|
+
}
|
package/semantic/test.d.mts
CHANGED
|
@@ -9,7 +9,8 @@ export interface ConversationOptions {
|
|
|
9
9
|
server: URL;
|
|
10
10
|
/** Real application context supplied in production. Never use this as test guidance. */
|
|
11
11
|
context?: string;
|
|
12
|
-
environment
|
|
12
|
+
/** Server environment shared by all trials, or isolated values selected by trial number. */
|
|
13
|
+
environment?: Record<string, string> | ((trial: number) => Record<string, string>);
|
|
13
14
|
authToken?: string;
|
|
14
15
|
authTokenEnvironment?: string;
|
|
15
16
|
}
|
package/semantic/test.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { createHash } from "node:crypto";
|
|
|
3
3
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { dirname, join, resolve } from "node:path";
|
|
6
|
-
import { validateConversationOptions } from "./case.mjs";
|
|
6
|
+
import { environmentForTrial, validateConversationOptions } from "./case.mjs";
|
|
7
7
|
import {
|
|
8
8
|
listMcpTools,
|
|
9
9
|
semanticAuthToken,
|
|
@@ -60,7 +60,10 @@ export async function createConversation(testContext, options) {
|
|
|
60
60
|
|
|
61
61
|
try {
|
|
62
62
|
for (let trial = 1; trial <= 3; trial += 1) {
|
|
63
|
-
const running = await startSemanticServer(
|
|
63
|
+
const running = await startSemanticServer({
|
|
64
|
+
...specification,
|
|
65
|
+
environment: environmentForTrial(specification.environment, trial),
|
|
66
|
+
}, testContext.signal);
|
|
64
67
|
try {
|
|
65
68
|
const tools = await listMcpTools(running.url, specification, testContext.signal);
|
|
66
69
|
const record = { trial, turns: [] };
|