@emseepea/testing 0.5.2 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +49 -6
- package/dist/index.js.map +1 -1
- package/package.json +2 -1
- package/semantic/case.mjs +16 -4
- package/semantic/provider.mjs +10 -7
- package/semantic/test.d.mts +2 -1
- package/semantic/test.mjs +5 -2
package/README.md
CHANGED
|
@@ -30,6 +30,37 @@ never production. Resources and prompts need deterministic protocol tests;
|
|
|
30
30
|
this library does not pretend that manually injecting their content proves a
|
|
31
31
|
native user journey.
|
|
32
32
|
|
|
33
|
+
## Test Extension Composition
|
|
34
|
+
|
|
35
|
+
Use `startEmseepea` in ordinary tests when you have an app factory and want to
|
|
36
|
+
exercise it in the same process. `insecureTestAuthentication` supplies a small
|
|
37
|
+
test-only verifier for protected-access examples:
|
|
38
|
+
|
|
39
|
+
```js
|
|
40
|
+
import { insecureTestAuthentication, startEmseepea } from "@emseepea/testing";
|
|
41
|
+
|
|
42
|
+
const running = await startEmseepea(t, await createApp({
|
|
43
|
+
access: { access: "protected", requiredScopes: ["peas:read"] },
|
|
44
|
+
authentication: insecureTestAuthentication(["peas:read"]),
|
|
45
|
+
observability: [{ id: "test", emit: (event) => events.push(event) }],
|
|
46
|
+
}), { token: "test-token" });
|
|
47
|
+
|
|
48
|
+
const client = await running.connect();
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Never deploy `insecureTestAuthentication`. It accepts any supplied bearer token
|
|
52
|
+
and exists only to keep ordinary tests focused on composition and authorization
|
|
53
|
+
flow. Test a real verifier separately against its provider contract.
|
|
54
|
+
|
|
55
|
+
When a conversation writes to external state, pass an `environment` function
|
|
56
|
+
that returns a separate test database connection for each trial. The trial
|
|
57
|
+
number selects infrastructure only. It is never sent to the model or MCP
|
|
58
|
+
server, and it does not prepare or coach the conversation.
|
|
59
|
+
|
|
60
|
+
For ordinary integration tests, `startMcpServer(...).output()` returns the
|
|
61
|
+
server's captured `stdout` and `stderr`. Use it to prove that logs do not expose
|
|
62
|
+
credentials or provider details.
|
|
63
|
+
|
|
33
64
|
## Diagnose Failures
|
|
34
65
|
|
|
35
66
|
The evidence file contains readable test prompts, assistant responses,
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Client } from "@modelcontextprotocol/client";
|
|
2
|
+
import { serveEmseepea, type AuthenticationOptions } from "@emseepea/server";
|
|
2
3
|
export interface TestCleanup {
|
|
3
4
|
after(cleanup: () => Promise<void>): void;
|
|
4
5
|
}
|
|
@@ -10,7 +11,14 @@ export interface StartMcpServerOptions {
|
|
|
10
11
|
}
|
|
11
12
|
export interface RunningMcpServer {
|
|
12
13
|
connect(token?: string): Promise<Client>;
|
|
14
|
+
output(): Readonly<{
|
|
15
|
+
stdout: string;
|
|
16
|
+
stderr: string;
|
|
17
|
+
}>;
|
|
13
18
|
url: URL;
|
|
14
19
|
}
|
|
20
|
+
/** Test-only verifier. Never use it in a deployed server. */
|
|
21
|
+
export declare function insecureTestAuthentication(permissions: readonly string[], discovery?: "public" | "protected"): AuthenticationOptions;
|
|
22
|
+
export declare function startEmseepea(test: TestCleanup, app: Parameters<typeof serveEmseepea>[0], options?: Pick<StartMcpServerOptions, "clientName" | "token">): Promise<RunningMcpServer>;
|
|
15
23
|
export declare function startMcpServer(test: TestCleanup, serverUrl: URL, options?: StartMcpServerOptions): Promise<RunningMcpServer>;
|
|
16
24
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAiC,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAiC,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,KAAK,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,IAAI,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,GAAG,EAAE,GAAG,CAAC;CACV;AAED,6DAA6D;AAC7D,wBAAgB,0BAA0B,CACxC,WAAW,EAAE,SAAS,MAAM,EAAE,EAC9B,SAAS,GAAE,QAAQ,GAAG,WAAyB,GAC9C,qBAAqB,CAyBvB;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,WAAW,EACjB,GAAG,EAAE,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,EACxC,OAAO,GAAE,IAAI,CAAC,qBAAqB,EAAE,YAAY,GAAG,OAAO,CAAM,GAChE,OAAO,CAAC,gBAAgB,CAAC,CAY3B;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,GAAG,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAsD3B"}
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,47 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { fileURLToPath } from "node:url";
|
|
3
3
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
4
|
+
import { serveEmseepea } from "@emseepea/server";
|
|
5
|
+
/** Test-only verifier. Never use it in a deployed server. */
|
|
6
|
+
export function insecureTestAuthentication(permissions, discovery = "protected") {
|
|
7
|
+
const resourceServerUrl = new URL("https://test.example/mcp");
|
|
8
|
+
return {
|
|
9
|
+
discovery,
|
|
10
|
+
verifier: {
|
|
11
|
+
async verifyAccessToken(token) {
|
|
12
|
+
return {
|
|
13
|
+
token,
|
|
14
|
+
clientId: "emseepea-test-client",
|
|
15
|
+
scopes: [...permissions],
|
|
16
|
+
expiresAt: Math.floor(Date.now() / 1_000) + 60,
|
|
17
|
+
resource: resourceServerUrl,
|
|
18
|
+
};
|
|
19
|
+
},
|
|
20
|
+
},
|
|
21
|
+
metadata: {
|
|
22
|
+
resourceServerUrl,
|
|
23
|
+
oauthMetadata: {
|
|
24
|
+
issuer: "https://auth.test.example",
|
|
25
|
+
authorization_endpoint: "https://auth.test.example/authorize",
|
|
26
|
+
token_endpoint: "https://auth.test.example/token",
|
|
27
|
+
response_types_supported: ["code"],
|
|
28
|
+
},
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export async function startEmseepea(test, app, options = {}) {
|
|
33
|
+
const running = await serveEmseepea(app, { port: 0 });
|
|
34
|
+
const clients = [];
|
|
35
|
+
test.after(async () => {
|
|
36
|
+
await Promise.allSettled(clients.map((client) => client.close()));
|
|
37
|
+
await running.close();
|
|
38
|
+
});
|
|
39
|
+
return {
|
|
40
|
+
...running,
|
|
41
|
+
output: () => Object.freeze({ stdout: "", stderr: "" }),
|
|
42
|
+
connect: (token = options.token) => connect(running.url, clients, options.clientName, token),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
4
45
|
export async function startMcpServer(test, serverUrl, options = {}) {
|
|
5
46
|
const child = spawn(process.execPath, [fileURLToPath(serverUrl)], {
|
|
6
47
|
env: {
|
|
@@ -49,14 +90,16 @@ export async function startMcpServer(test, serverUrl, options = {}) {
|
|
|
49
90
|
});
|
|
50
91
|
return {
|
|
51
92
|
url,
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
await client.connect(new StreamableHTTPClientTransport(url, token ? { authProvider: { token: async () => token } } : undefined));
|
|
55
|
-
clients.push(client);
|
|
56
|
-
return client;
|
|
57
|
-
},
|
|
93
|
+
output: () => Object.freeze({ stdout: output, stderr: errors }),
|
|
94
|
+
connect: (token = options.token) => connect(url, clients, options.clientName, token),
|
|
58
95
|
};
|
|
59
96
|
}
|
|
97
|
+
async function connect(url, clients, clientName = "emseepea-test", token) {
|
|
98
|
+
const client = new Client({ name: clientName, version: "0.0.0" }, { versionNegotiation: { mode: { pin: "2026-07-28" } } });
|
|
99
|
+
await client.connect(new StreamableHTTPClientTransport(url, token ? { authProvider: { token: async () => token } } : undefined));
|
|
100
|
+
clients.push(client);
|
|
101
|
+
return client;
|
|
102
|
+
}
|
|
60
103
|
async function stopProcess(child) {
|
|
61
104
|
if (child.exitCode !== null)
|
|
62
105
|
return;
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,aAAa,EAA8B,MAAM,kBAAkB,CAAC;AAmB7E,6DAA6D;AAC7D,MAAM,UAAU,0BAA0B,CACxC,WAA8B,EAC9B,YAAoC,WAAW;IAE/C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAC9D,OAAO;QACL,SAAS;QACT,QAAQ,EAAE;YACR,KAAK,CAAC,iBAAiB,CAAC,KAAK;gBAC3B,OAAO;oBACL,KAAK;oBACL,QAAQ,EAAE,sBAAsB;oBAChC,MAAM,EAAE,CAAC,GAAG,WAAW,CAAC;oBACxB,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,EAAE;oBAC9C,QAAQ,EAAE,iBAAiB;iBAC5B,CAAC;YACJ,CAAC;SACF;QACD,QAAQ,EAAE;YACR,iBAAiB;YACjB,aAAa,EAAE;gBACb,MAAM,EAAE,2BAA2B;gBACnC,sBAAsB,EAAE,qCAAqC;gBAC7D,cAAc,EAAE,iCAAiC;gBACjD,wBAAwB,EAAE,CAAC,MAAM,CAAC;aACnC;SACF;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAiB,EACjB,GAAwC,EACxC,UAA+D,EAAE;IAEjE,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO;QACL,GAAG,OAAO;QACV,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QACvD,OAAO,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;KAC7F,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAiB,EACjB,SAAc,EACd,UAAiC,EAAE;IAEnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE;QAChE,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,GAAG;YACT,GAAG,OAAO,CAAC,WAAW;SACvB;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,KAAW,EAAE,EAAE;YAC5C,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC1B,IAAI,KAAK;gBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;iBACpB,IAAI,KAAK;gBAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAC7D,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC;QACF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,IAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1G,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,MAAM,EAAE,CAAC,CAAC,EAClE,OAAO,CAAC,gBAAgB,IAAI,MAAM,CACnC,CAAC;QACF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;QACzB,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAC/D,OAAO,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;KACrF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,GAAQ,EACR,OAAiB,EACjB,UAAU,GAAG,eAAe,EAC5B,KAAc;IAEd,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,EACtC,EAAE,kBAAkB,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,CACxD,CAAC;IACF,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,6BAA6B,CACpD,GAAG,EACH,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC,CAAC;IACH,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAA+B;IACxD,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,OAAO;IACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,CAAC,IAAI,CAAC;QACjB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;KAC3D,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACrD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@emseepea/testing",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "MCP integration and semantic testing helpers",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
"test:built": "node --test test/*.test.mjs"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
+
"@emseepea/server": "0.4.0",
|
|
44
45
|
"@modelcontextprotocol/client": "2.0.0"
|
|
45
46
|
},
|
|
46
47
|
"engines": {
|
package/semantic/case.mjs
CHANGED
|
@@ -17,11 +17,23 @@ export function validateConversationOptions(value) {
|
|
|
17
17
|
if (value.authToken !== undefined && value.authTokenEnvironment !== undefined) {
|
|
18
18
|
throw new Error("Choose one authentication source");
|
|
19
19
|
}
|
|
20
|
-
if (value.environment !== undefined &&
|
|
21
|
-
|
|
22
|
-
|| Object.values(value.environment).some((item) => typeof item !== "string"))) {
|
|
23
|
-
throw new Error("environment must contain string values");
|
|
20
|
+
if (value.environment !== undefined && typeof value.environment !== "function") {
|
|
21
|
+
validateEnvironment(value.environment);
|
|
24
22
|
}
|
|
25
23
|
const server = fileURLToPath(value.server);
|
|
26
24
|
return { ...value, server, directory: dirname(server) };
|
|
27
25
|
}
|
|
26
|
+
|
|
27
|
+
export function environmentForTrial(environment, trial) {
|
|
28
|
+
const resolved = typeof environment === "function" ? environment(trial) : environment;
|
|
29
|
+
if (resolved === undefined) return undefined;
|
|
30
|
+
validateEnvironment(resolved);
|
|
31
|
+
return resolved;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function validateEnvironment(environment) {
|
|
35
|
+
if (!environment || typeof environment !== "object" || Array.isArray(environment)
|
|
36
|
+
|| Object.values(environment).some((item) => typeof item !== "string")) {
|
|
37
|
+
throw new Error("environment must contain string values");
|
|
38
|
+
}
|
|
39
|
+
}
|
package/semantic/provider.mjs
CHANGED
|
@@ -88,10 +88,13 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
|
|
|
88
88
|
if (init) {
|
|
89
89
|
const available = [...(init.tools ?? [])].sort();
|
|
90
90
|
const expected = [...advertised.keys()].sort();
|
|
91
|
+
const expectedServer = advertised.size > 0
|
|
92
|
+
? init.mcp_servers?.length === 1
|
|
93
|
+
&& init.mcp_servers[0]?.name === mcpServerName
|
|
94
|
+
&& init.mcp_servers[0]?.status === "connected"
|
|
95
|
+
: init.mcp_servers?.length === 0;
|
|
91
96
|
if (JSON.stringify(available) !== JSON.stringify(expected)
|
|
92
|
-
||
|
|
93
|
-
|| init.mcp_servers[0]?.name !== mcpServerName
|
|
94
|
-
|| init.mcp_servers[0]?.status !== "connected") {
|
|
97
|
+
|| !expectedServer) {
|
|
95
98
|
throw new Error("Model command did not expose exactly the target MCP tools");
|
|
96
99
|
}
|
|
97
100
|
}
|
|
@@ -189,7 +192,7 @@ export function modelInvocation(provider, prompt, directory) {
|
|
|
189
192
|
|
|
190
193
|
export function conversationInvocation(provider, directory, url, tools, authToken, context) {
|
|
191
194
|
const nativeTools = tools.map(({ name }) => nativeToolName(name));
|
|
192
|
-
const config = {
|
|
195
|
+
const config = nativeTools.length ? {
|
|
193
196
|
mcpServers: {
|
|
194
197
|
[mcpServerName]: {
|
|
195
198
|
type: "http",
|
|
@@ -197,7 +200,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
|
|
|
197
200
|
...(authToken ? { headers: { Authorization: "Bearer ${EMSEEPEA_SEMANTIC_MCP_TOKEN}" } } : {}),
|
|
198
201
|
},
|
|
199
202
|
},
|
|
200
|
-
};
|
|
203
|
+
} : undefined;
|
|
201
204
|
const base = modelInvocation(provider, "", directory);
|
|
202
205
|
return {
|
|
203
206
|
...base,
|
|
@@ -210,7 +213,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
|
|
|
210
213
|
"--effort", "low",
|
|
211
214
|
"--max-turns", "4",
|
|
212
215
|
"--strict-mcp-config",
|
|
213
|
-
"--mcp-config", JSON.stringify(config),
|
|
216
|
+
...(config ? ["--mcp-config", JSON.stringify(config)] : []),
|
|
214
217
|
"--disable-slash-commands",
|
|
215
218
|
"--no-session-persistence",
|
|
216
219
|
"--permission-mode", "dontAsk",
|
|
@@ -221,7 +224,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
|
|
|
221
224
|
"--no-chrome",
|
|
222
225
|
"--prompt-suggestions", "false",
|
|
223
226
|
],
|
|
224
|
-
env: { ...base.env, ...(authToken ? { EMSEEPEA_SEMANTIC_MCP_TOKEN: authToken } : {}) },
|
|
227
|
+
env: { ...base.env, ...(config && authToken ? { EMSEEPEA_SEMANTIC_MCP_TOKEN: authToken } : {}) },
|
|
225
228
|
};
|
|
226
229
|
}
|
|
227
230
|
|
package/semantic/test.d.mts
CHANGED
|
@@ -9,7 +9,8 @@ export interface ConversationOptions {
|
|
|
9
9
|
server: URL;
|
|
10
10
|
/** Real application context supplied in production. Never use this as test guidance. */
|
|
11
11
|
context?: string;
|
|
12
|
-
environment
|
|
12
|
+
/** Server environment shared by all trials, or isolated values selected by trial number. */
|
|
13
|
+
environment?: Record<string, string> | ((trial: number) => Record<string, string>);
|
|
13
14
|
authToken?: string;
|
|
14
15
|
authTokenEnvironment?: string;
|
|
15
16
|
}
|
package/semantic/test.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { createHash } from "node:crypto";
|
|
|
3
3
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { dirname, join, resolve } from "node:path";
|
|
6
|
-
import { validateConversationOptions } from "./case.mjs";
|
|
6
|
+
import { environmentForTrial, validateConversationOptions } from "./case.mjs";
|
|
7
7
|
import {
|
|
8
8
|
listMcpTools,
|
|
9
9
|
semanticAuthToken,
|
|
@@ -60,7 +60,10 @@ export async function createConversation(testContext, options) {
|
|
|
60
60
|
|
|
61
61
|
try {
|
|
62
62
|
for (let trial = 1; trial <= 3; trial += 1) {
|
|
63
|
-
const running = await startSemanticServer(
|
|
63
|
+
const running = await startSemanticServer({
|
|
64
|
+
...specification,
|
|
65
|
+
environment: environmentForTrial(specification.environment, trial),
|
|
66
|
+
}, testContext.signal);
|
|
64
67
|
try {
|
|
65
68
|
const tools = await listMcpTools(running.url, specification, testContext.signal);
|
|
66
69
|
const record = { trial, turns: [] };
|