@buildanagent/sdk 0.1.0-alpha.11 → 0.1.0-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -1
- package/dist/errors.d.ts +6 -3
- package/dist/errors.js +3 -0
- package/dist/errors.js.map +1 -1
- package/dist/testing/auto-responder.d.ts +35 -0
- package/dist/testing/auto-responder.js +142 -0
- package/dist/testing/auto-responder.js.map +1 -0
- package/dist/testing/index.d.ts +11 -4
- package/dist/testing/index.js +10 -3
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/recording.d.ts +92 -0
- package/dist/testing/recording.js +130 -0
- package/dist/testing/recording.js.map +1 -0
- package/dist/testing/replay.d.ts +41 -0
- package/dist/testing/replay.js +134 -0
- package/dist/testing/replay.js.map +1 -0
- package/dist/testing/scripted-approvals.d.ts +32 -0
- package/dist/testing/scripted-approvals.js +31 -0
- package/dist/testing/scripted-approvals.js.map +1 -0
- package/dist/testing/scripted-mcp.d.ts +43 -0
- package/dist/testing/scripted-mcp.js +67 -0
- package/dist/testing/scripted-mcp.js.map +1 -0
- package/dist/testing/scripted-model.d.ts +11 -2
- package/dist/testing/scripted-model.js +33 -2
- package/dist/testing/scripted-model.js.map +1 -1
- package/dist/vendor/agent-safety/test-signer.d.ts +13 -0
- package/dist/vendor/agent-safety/test-signer.js +18 -0
- package/dist/vendor/agent-safety/test-signer.js.map +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recording.js","sourceRoot":"","sources":["../../src/testing/recording.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EACL,QAAQ,GAOT,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAqB,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,OAAO,EAAwB,eAAe,EAAqB,MAAM,qBAAqB,CAAC;AAsD/F;;;;;GAKG;AACH,MAAM,OAAO,cAAe,SAAQ,QAAQ;IACxB,IAAI,GAAG,gBAAgB,CAAC;IAG1C,YAAY,IAAwB,EAAE,OAAe;QACnD,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvB,CAAC;CACF;AAED,MAAM,iBAAiB,GAAwB,IAAI,GAAG,CAAqB;IACzE,OAAO;IACP,YAAY;IACZ,YAAY;IACZ,UAAU;IACV,iBAAiB;IACjB,iBAAiB;IACjB,eAAe;IACf,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,SAAS;IACT,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAmB;IACvC,OAAO,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAI,KAAQ;IAClC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAM,CAAC;AAChD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAmB,EACnB,OAAqB,EACrB,OAAyB;IAEzB,OAAO,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;AAC3F,CAAC;AAED,SAAS,MAAM,CAAC,KAAc,EAAE,IAAY;IAC1C,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IACxE,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,cAAc,CAAC,mBAAmB,EAAE,GAAG,IAAI,2BAA2B,CAAC,CAAC;IAC1G,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,OAAO,CAAC,KAAY,EAAE,OAAsB;IACnD,IAAI,OAAO,OAAO,EAAE,eAAe,KAAK,QAAQ,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC1F,MAAM,IAAI,cAAc,CAAC,mBAAmB,EAAE,uCAAuC,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpE,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,cAAc,CAAC,mBAAmB,EAAE,gEAAgE,CAAC,CAAC;IAClH,CAAC;IACD,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC;IACxC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAC/C,OAAO;QACL,UAAU,EAAE,WAAW;QACvB,GAAG;QACH,QAAQ;QACR,KAAK,EAAE,EAAE;QACT,eAAe;QACf,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,UAAU;QACV,SAAS;KACV,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,WAAW,CAAC,KAAY,EAAE,OAAsB;IAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACtC,MAAM,KAAK,GAAiC,EAAE,CAAC;IAC/C,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,OAAqB,EAAE,WAAW,GAAqB,EAAE;QAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;QACnE,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;YAC7D,IAAI,YAAY,CAAC,KAAK,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5C,8GAA8G;YAC9G,oEAAoE;YACpE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACpD,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YAC3E,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAU;QACtB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM;QACN,QAAQ,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC1E,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;KACrC,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAwB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;AAC5G,CAAC","sourcesContent":["/**\n * `recordPairs()`: records each call a model makes as a pair of normalised request and response, stamped with the\n * versions it was recorded against and an expiry (agent-SDK guide §8: record and replay at the normalised boundary).\n * A pair holds the gateway's transcript types, never wire bytes, so it survives changes to an adapter's wire format,\n * and it holds no credential: the adapter's key never reaches a call, and every property of the call's options named\n * like a credential header or an API key is left out.\n */\nimport {\n complete,\n type GatewayEvent,\n type Model,\n type ModelCallOptions,\n type ModelCaps,\n type ModelContext,\n type ModelEvent,\n} from \"@buildanagent/gateway\";\nimport { stripCredentials } from \"@buildanagent/gateway/testing\";\nimport { SdkError, type SdkErrorCode } from \"../errors.js\";\nimport { SDK_VERSION } from \"../version.js\";\nimport type { Clock } from \"./determinism.js\";\nimport { type RecordedRequest, recordedOptions, type StreamScript } from \"./scripted-model.js\";\n\n/** What a recording was made against, and until when it may be replayed. Every pair of one recording shares it. */\nexport interface RecordingStamp {\n /** The `@buildanagent/sdk` version that recorded it. */\n readonly sdkVersion: string;\n /** The model's wire protocol, such as `\"anthropic-messages\"`. */\n readonly api: string;\n /** The model's vendor, such as `\"anthropic\"`. */\n readonly provider: string;\n /** The provider's model id. */\n readonly model: string;\n /** The provider's API version, or whatever names the server build the recording was made against. */\n readonly providerVersion: string;\n /** The model's caps when it was recorded, which the replay model reports. */\n readonly caps: ModelCaps;\n /** When the recording was made, in ISO 8601. */\n readonly recordedAt: string;\n /** The time from which `replayPairs()` refuses the recording, in ISO 8601. */\n readonly expiresAt: string;\n}\n\n/** One model call: the normalised request it sent and the normalised events it streamed. */\nexport interface RecordedPair {\n readonly stamp: RecordingStamp;\n readonly request: RecordedRequest;\n /** The call's events, from `start` to its terminal, as the gateway streamed them, minus `retry` events. */\n readonly response: StreamScript;\n}\n\n/** What `recordPairs()` takes besides the model. */\nexport interface RecordOptions {\n /** The provider's API version, or another name for the server build you record against. Required. */\n readonly providerVersion: string;\n /** When the recording expires, in ISO 8601. Required, and later than `recordedAt`. */\n readonly expiresAt: string;\n /** Gives `recordedAt`. Defaults to the system time. */\n readonly clock?: Clock;\n}\n\n/** A model that records its calls, and the pairs recorded so far. */\nexport interface Recorder {\n /** Pass this model to `run()` in place of the one you recorded. */\n readonly model: Model;\n /** The pairs of every call that has reached its terminal, in the order the calls started. */\n pairs(): RecordedPair[];\n}\n\n/** The codes a `RecordingError` carries. */\nexport type RecordingErrorCode = Extract<\n SdkErrorCode,\n \"invalid_recording\" | \"recording_expired\" | \"recording_mismatch\"\n>;\n\n/**\n * Thrown by `recordPairs()` and `replayPairs()`. `invalid_recording`: the options or pairs are malformed, such as a\n * missing stamp or an expiry that isn't a time. `recording_expired`: the replay time is at or past the stamp's\n * `expiresAt`. `recording_mismatch`: the pairs disagree on their stamp, the stamp differs from what the replay\n * expects, or a call sent a request other than the recorded one. It is an `SdkError`, so `isSdkError()` recognises it.\n */\nexport class RecordingError extends SdkError {\n override readonly name = \"RecordingError\";\n declare readonly code: RecordingErrorCode;\n\n constructor(code: RecordingErrorCode, message: string) {\n super(code, message);\n }\n}\n\nconst NORMALISED_EVENTS: ReadonlySet<string> = new Set<ModelEvent[\"type\"]>([\n \"start\",\n \"text_start\",\n \"text_delta\",\n \"text_end\",\n \"reasoning_start\",\n \"reasoning_delta\",\n \"reasoning_end\",\n \"tool_input_start\",\n \"tool_input_delta\",\n \"tool_input_end\",\n \"warning\",\n \"done\",\n \"error\",\n]);\n\n/**\n * True for an event a pair stores: the model spec's own events. `retry` events, which the gateway adds, and any\n * provider event outside the spec stay out, because a replay can't reproduce them.\n */\nfunction isNormalised(event: GatewayEvent): event is ModelEvent {\n return NORMALISED_EVENTS.has(event.type);\n}\n\n/** A JSON-safe copy: exactly what a fixture file written from it would hold. */\nexport function storable<T>(value: T): T {\n return JSON.parse(JSON.stringify(value)) as T;\n}\n\n/**\n * The normalised form of one request, as a pair stores it and a replay compares it. Credentials are stripped from\n * the options only, where a key or header can live. The context is the conversation, which a replay must reproduce\n * exactly, so a tool input that happens to hold an `apiKey` property is kept as sent.\n */\nexport function normalisedRequest(\n model: Model[\"ref\"],\n context: ModelContext,\n options: ModelCallOptions,\n): RecordedRequest {\n return storable({ model, context, options: stripCredentials(recordedOptions(options)) });\n}\n\nfunction timeOf(value: unknown, name: string): number {\n const time = typeof value === \"string\" ? Date.parse(value) : Number.NaN;\n if (Number.isNaN(time)) throw new RecordingError(\"invalid_recording\", `${name} must be an ISO 8601 time`);\n return time;\n}\n\nfunction stampOf(model: Model, options: RecordOptions): RecordingStamp {\n if (typeof options?.providerVersion !== \"string\" || options.providerVersion.trim() === \"\") {\n throw new RecordingError(\"invalid_recording\", \"recordPairs() needs a providerVersion\");\n }\n const recordedAt = options.clock?.now() ?? new Date().toISOString();\n if (timeOf(options.expiresAt, \"expiresAt\") <= timeOf(recordedAt, \"recordedAt\")) {\n throw new RecordingError(\"invalid_recording\", \"recordPairs(): expiresAt must be later than the recording time\");\n }\n const { api, provider, id } = model.ref;\n const { providerVersion, expiresAt } = options;\n return {\n sdkVersion: SDK_VERSION,\n api,\n provider,\n model: id,\n providerVersion,\n caps: model.caps,\n recordedAt,\n expiresAt,\n };\n}\n\n/**\n * Wraps `model` so that each call it makes is recorded as a `RecordedPair`, and returns the wrapper with the pairs so\n * far. Write `pairs()` to a fixture file as JSON, and replay it with `replayPairs()`. Each pair holds the request the\n * call sent (model, context and options) and every event it streamed except `retry` events. The adapter's key and\n * headers are never part of a call, and every property of the options named like a credential header or an API key,\n * or ending in `token`, `secret` or `password`, is left out at any depth, so a fixture holds no key the adapter or\n * the options carried. An option such as `cancellationToken` is therefore neither recorded nor compared. The\n * conversation and the model's output are kept exactly as they were, because a replay must reproduce them. A call is recorded once it\n * reaches its terminal event; one whose reader stopped earlier is not recorded.\n *\n * The pairs are taken where the caller meets `model`: each request as the caller sent it, before any middleware\n * `model` was built with, and the events as the caller received them, after it. Replay them through `createModel()`\n * without that middleware, or the replay compares a request the middleware changed with one it didn't.\n *\n * Throws a `RecordingError` with code `invalid_recording` for a blank `providerVersion`, or an `expiresAt` that isn't\n * an ISO 8601 time later than the recording time.\n */\nexport function recordPairs(model: Model, options: RecordOptions): Recorder {\n const stamp = stampOf(model, options);\n const slots: (RecordedPair | undefined)[] = [];\n async function* stream(context: ModelContext, callOptions: ModelCallOptions = {}): AsyncGenerator<GatewayEvent> {\n const slot = slots.push(undefined) - 1;\n const request = normalisedRequest(model.ref, context, callOptions);\n const events: ModelEvent[] = [];\n for await (const event of model.stream(context, callOptions)) {\n if (isNormalised(event)) events.push(event);\n // Stored before the terminal is yielded: a reader such as `complete()` stops at the terminal, which ends this\n // generator at the `yield`, so code after the loop would never run.\n if (event.type === \"done\" || event.type === \"error\") {\n slots[slot] = { stamp, request, response: { events: storable(events) } };\n }\n yield event;\n }\n }\n const recorded: Model = {\n ref: model.ref,\n caps: model.caps,\n stream,\n complete: (context, callOptions) => complete(stream(context, callOptions)),\n price: (usage) => model.price(usage),\n };\n return { model: recorded, pairs: () => slots.filter((pair): pair is RecordedPair => pair !== undefined) };\n}\n"]}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { Clock } from "./determinism.js";
|
|
2
|
+
import { type RecordedPair, type RecordingStamp } from "./recording.js";
|
|
3
|
+
import { ScriptedModel } from "./scripted-model.js";
|
|
4
|
+
/** The stamp fields a replay can require to match exactly. */
|
|
5
|
+
export type ExpectedStamp = Partial<Pick<RecordingStamp, "sdkVersion" | "api" | "provider" | "model" | "providerVersion">>;
|
|
6
|
+
/** What `replayPairs()` takes besides the pairs. */
|
|
7
|
+
export interface ReplayOptions {
|
|
8
|
+
/** Gives the time the expiry is checked against. Defaults to the system time. */
|
|
9
|
+
readonly clock?: Clock;
|
|
10
|
+
/** Stamp fields that must match exactly, such as the `providerVersion` your adapter speaks today. */
|
|
11
|
+
readonly expect?: ExpectedStamp;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A `ScriptedModel` that plays back recorded pairs. Its `api` and `provider` are the stamp's, and `modelId` is the id
|
|
15
|
+
* to pass to `createModel()`, so the replayed calls are priced from the same catalog entry as the recorded ones.
|
|
16
|
+
*/
|
|
17
|
+
export declare class ReplayModel extends ScriptedModel {
|
|
18
|
+
#private;
|
|
19
|
+
/** The recorded model id. */
|
|
20
|
+
readonly modelId: string;
|
|
21
|
+
constructor(pairs: readonly RecordedPair[], options?: ReplayOptions);
|
|
22
|
+
/**
|
|
23
|
+
* Throws the first `RecordingError` a call raised by sending a request other than the recorded one, then does what
|
|
24
|
+
* `ScriptedModel.assertComplete()` does. Call it in teardown: behind `createModel()`, a mismatch ends the run with
|
|
25
|
+
* an `error` result, and only this check makes the test fail.
|
|
26
|
+
*/
|
|
27
|
+
assertComplete(): void;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Replays pairs from `recordPairs()` through a `ScriptedModel`: the n-th call streams the n-th pair's events exactly as
|
|
31
|
+
* they were recorded, after checking that it sent the recorded request. Wrap it with
|
|
32
|
+
* `createModel({ adapter: replay, id: replay.modelId })`, with no middleware: the recording was taken outside the
|
|
33
|
+
* recorded model's middleware, so it already holds that middleware's effect.
|
|
34
|
+
*
|
|
35
|
+
* Throws a `RecordingError`: `invalid_recording` for an empty or malformed recording, `recording_expired` once the
|
|
36
|
+
* clock reaches the stamp's `expiresAt`, and `recording_mismatch` when the pairs' stamps differ, the recording comes
|
|
37
|
+
* from another SDK release line (another minor version under 0.x), or a stamp field differs from `expect`. A call
|
|
38
|
+
* whose request differs from the recording fails with a `recording_mismatch` error, which `assertComplete()`
|
|
39
|
+
* rethrows.
|
|
40
|
+
*/
|
|
41
|
+
export declare function replayPairs(pairs: readonly RecordedPair[], options?: ReplayOptions): ReplayModel;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `replayPairs()`: plays a recording back through a `ScriptedModel`, after checking its stamp and expiry, and checks
|
|
3
|
+
* that each call sends the request that was recorded, so a replayed run that drifts from the recorded one fails.
|
|
4
|
+
*/
|
|
5
|
+
import { SDK_VERSION } from "../version.js";
|
|
6
|
+
import { normalisedRequest, RecordingError, storable } from "./recording.js";
|
|
7
|
+
import { ScriptedModel } from "./scripted-model.js";
|
|
8
|
+
/** Sorted-key JSON, so two requests compare equal whatever order their keys were written in. */
|
|
9
|
+
function canonical(value) {
|
|
10
|
+
if (Array.isArray(value))
|
|
11
|
+
return `[${value.map(canonical).join(",")}]`;
|
|
12
|
+
if (typeof value !== "object" || value === null)
|
|
13
|
+
return JSON.stringify(value);
|
|
14
|
+
const keys = Object.keys(value).sort();
|
|
15
|
+
return `{${keys.map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`).join(",")}}`;
|
|
16
|
+
}
|
|
17
|
+
/** The JSON path of the first place `a` and `b` differ, such as `$.context.messages[2].content`. */
|
|
18
|
+
function firstDifference(a, b, path = "$") {
|
|
19
|
+
if (typeof a !== "object" || typeof b !== "object" || a === null || b === null)
|
|
20
|
+
return path;
|
|
21
|
+
if (Array.isArray(a) !== Array.isArray(b))
|
|
22
|
+
return path;
|
|
23
|
+
if (Array.isArray(a) && Array.isArray(b)) {
|
|
24
|
+
if (a.length !== b.length)
|
|
25
|
+
return `${path} (length ${b.length}, recorded ${a.length})`;
|
|
26
|
+
const index = a.findIndex((item, i) => canonical(item) !== canonical(b[i]));
|
|
27
|
+
return index < 0 ? path : firstDifference(a[index], b[index], `${path}[${index}]`);
|
|
28
|
+
}
|
|
29
|
+
const left = a;
|
|
30
|
+
const right = b;
|
|
31
|
+
const keys = [...new Set([...Object.keys(left), ...Object.keys(right)])].sort();
|
|
32
|
+
const key = keys.find((name) => canonical(left[name]) !== canonical(right[name]));
|
|
33
|
+
return key === undefined ? path : firstDifference(left[key], right[key], `${path}.${key}`);
|
|
34
|
+
}
|
|
35
|
+
/** The release line: the major version from 1.0 on, the minor one under 0.x, where a minor bump may break. */
|
|
36
|
+
function releaseLine(version) {
|
|
37
|
+
const [major = "", minor = ""] = version.split(".");
|
|
38
|
+
return major === "0" ? `0.${minor}` : major;
|
|
39
|
+
}
|
|
40
|
+
// `typeof null` is "object", so a bare typeof check would let a null stamp through to a TypeError.
|
|
41
|
+
function isObject(value) {
|
|
42
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
43
|
+
}
|
|
44
|
+
function checkShape(pairs) {
|
|
45
|
+
if (!Array.isArray(pairs) || pairs.length === 0) {
|
|
46
|
+
throw new RecordingError("invalid_recording", "replayPairs() needs a non-empty array of recorded pairs");
|
|
47
|
+
}
|
|
48
|
+
for (const [i, pair] of pairs.entries()) {
|
|
49
|
+
const candidate = pair;
|
|
50
|
+
const events = candidate?.response?.events;
|
|
51
|
+
if (!isObject(candidate?.stamp) || !isObject(candidate?.request) || !Array.isArray(events)) {
|
|
52
|
+
throw new RecordingError("invalid_recording", `pair ${i + 1} needs a stamp, a request and response events`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return pairs[0].stamp;
|
|
56
|
+
}
|
|
57
|
+
function checkStamp(stamp, pairs, options) {
|
|
58
|
+
const others = pairs.findIndex((pair) => canonical(pair.stamp) !== canonical(stamp));
|
|
59
|
+
if (others >= 0) {
|
|
60
|
+
throw new RecordingError("recording_mismatch", `pair ${others + 1} has a different stamp from pair 1`);
|
|
61
|
+
}
|
|
62
|
+
const expiresAt = Date.parse(stamp.expiresAt);
|
|
63
|
+
if (Number.isNaN(expiresAt))
|
|
64
|
+
throw new RecordingError("invalid_recording", "the stamp's expiresAt isn't a time");
|
|
65
|
+
const now = options.clock?.now() ?? new Date().toISOString();
|
|
66
|
+
if (Date.parse(now) >= expiresAt) {
|
|
67
|
+
throw new RecordingError("recording_expired", `the recording expired at ${stamp.expiresAt}; record it again`);
|
|
68
|
+
}
|
|
69
|
+
if (typeof stamp.sdkVersion !== "string" || releaseLine(stamp.sdkVersion) !== releaseLine(SDK_VERSION)) {
|
|
70
|
+
throw new RecordingError("recording_mismatch", `the recording was made with SDK ${String(stamp.sdkVersion)}, which this SDK (${SDK_VERSION}) can't replay`);
|
|
71
|
+
}
|
|
72
|
+
for (const [field, value] of Object.entries(options.expect ?? {})) {
|
|
73
|
+
const recorded = stamp[field];
|
|
74
|
+
if (value !== undefined && recorded !== value) {
|
|
75
|
+
throw new RecordingError("recording_mismatch", `the stamp's ${field} is ${String(recorded)}, expected ${value}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* A `ScriptedModel` that plays back recorded pairs. Its `api` and `provider` are the stamp's, and `modelId` is the id
|
|
81
|
+
* to pass to `createModel()`, so the replayed calls are priced from the same catalog entry as the recorded ones.
|
|
82
|
+
*/
|
|
83
|
+
export class ReplayModel extends ScriptedModel {
|
|
84
|
+
/** The recorded model id. */
|
|
85
|
+
modelId;
|
|
86
|
+
#mismatches;
|
|
87
|
+
constructor(pairs, options = {}) {
|
|
88
|
+
const stamp = checkShape(pairs);
|
|
89
|
+
checkStamp(stamp, pairs, options);
|
|
90
|
+
const mismatches = [];
|
|
91
|
+
super(pairs.map((pair, i) => matchingStep(pair, i, mismatches)), { ref: { api: stamp.api, provider: stamp.provider }, caps: { ...stamp.caps } });
|
|
92
|
+
this.modelId = stamp.model;
|
|
93
|
+
this.#mismatches = mismatches;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Throws the first `RecordingError` a call raised by sending a request other than the recorded one, then does what
|
|
97
|
+
* `ScriptedModel.assertComplete()` does. Call it in teardown: behind `createModel()`, a mismatch ends the run with
|
|
98
|
+
* an `error` result, and only this check makes the test fail.
|
|
99
|
+
*/
|
|
100
|
+
assertComplete() {
|
|
101
|
+
const [first] = this.#mismatches;
|
|
102
|
+
if (first !== undefined)
|
|
103
|
+
throw first;
|
|
104
|
+
super.assertComplete();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function matchingStep(pair, index, mismatches) {
|
|
108
|
+
return (request) => {
|
|
109
|
+
const sent = normalisedRequest(request.model, request.context, request.options);
|
|
110
|
+
const recorded = storable(pair.request);
|
|
111
|
+
if (canonical(sent) !== canonical(recorded)) {
|
|
112
|
+
const error = new RecordingError("recording_mismatch", `call ${index + 1} sent a request that differs from the recording at ${firstDifference(recorded, sent)}`);
|
|
113
|
+
mismatches.push(error);
|
|
114
|
+
throw error;
|
|
115
|
+
}
|
|
116
|
+
return pair.response;
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Replays pairs from `recordPairs()` through a `ScriptedModel`: the n-th call streams the n-th pair's events exactly as
|
|
121
|
+
* they were recorded, after checking that it sent the recorded request. Wrap it with
|
|
122
|
+
* `createModel({ adapter: replay, id: replay.modelId })`, with no middleware: the recording was taken outside the
|
|
123
|
+
* recorded model's middleware, so it already holds that middleware's effect.
|
|
124
|
+
*
|
|
125
|
+
* Throws a `RecordingError`: `invalid_recording` for an empty or malformed recording, `recording_expired` once the
|
|
126
|
+
* clock reaches the stamp's `expiresAt`, and `recording_mismatch` when the pairs' stamps differ, the recording comes
|
|
127
|
+
* from another SDK release line (another minor version under 0.x), or a stamp field differs from `expect`. A call
|
|
128
|
+
* whose request differs from the recording fails with a `recording_mismatch` error, which `assertComplete()`
|
|
129
|
+
* rethrows.
|
|
130
|
+
*/
|
|
131
|
+
export function replayPairs(pairs, options = {}) {
|
|
132
|
+
return new ReplayModel(pairs, options);
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=replay.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replay.js","sourceRoot":"","sources":["../../src/testing/replay.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,OAAO,EAAE,iBAAiB,EAAqB,cAAc,EAAuB,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACrH,OAAO,EAAwB,aAAa,EAAmB,MAAM,qBAAqB,CAAC;AAe3F,gGAAgG;AAChG,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACvE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,SAAS,CAAE,KAAiC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAC5H,CAAC;AAED,oGAAoG;AACpG,SAAS,eAAe,CAAC,CAAU,EAAE,CAAU,EAAE,IAAI,GAAG,GAAG;IACzD,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC5F,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,GAAG,IAAI,YAAY,CAAC,CAAC,MAAM,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC;QACvF,MAAM,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,IAAI,GAAG,CAA4B,CAAC;IAC1C,MAAM,KAAK,GAAG,CAA4B,CAAC;IAC3C,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClF,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED,8GAA8G;AAC9G,SAAS,WAAW,CAAC,OAAe;IAClC,MAAM,CAAC,KAAK,GAAG,EAAE,EAAE,KAAK,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpD,OAAO,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9C,CAAC;AAED,mGAAmG;AACnG,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,UAAU,CAAC,KAA8B;IAChD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,cAAc,CAAC,mBAAmB,EAAE,yDAAyD,CAAC,CAAC;IAC3G,CAAC;IACD,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,IAAoC,CAAC;QACvD,MAAM,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3F,MAAM,IAAI,cAAc,CAAC,mBAAmB,EAAE,QAAQ,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;QAC9G,CAAC;IACH,CAAC;IACD,OAAQ,KAAK,CAAC,CAAC,CAAkB,CAAC,KAAK,CAAC;AAC1C,CAAC;AAED,SAAS,UAAU,CAAC,KAAqB,EAAE,KAA8B,EAAE,OAAsB;IAC/F,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACrF,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAChB,MAAM,IAAI,cAAc,CAAC,oBAAoB,EAAE,QAAQ,MAAM,GAAG,CAAC,oCAAoC,CAAC,CAAC;IACzG,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9C,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,cAAc,CAAC,mBAAmB,EAAE,oCAAoC,CAAC,CAAC;IACjH,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7D,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,IAAI,cAAc,CAAC,mBAAmB,EAAE,4BAA4B,KAAK,CAAC,SAAS,mBAAmB,CAAC,CAAC;IAChH,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,WAAW,CAAC,WAAW,CAAC,EAAE,CAAC;QACvG,MAAM,IAAI,cAAc,CACtB,oBAAoB,EACpB,mCAAmC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,qBAAqB,WAAW,gBAAgB,CAC5G,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,KAA4B,CAAC,CAAC;QACrD,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC9C,MAAM,IAAI,cAAc,CAAC,oBAAoB,EAAE,eAAe,KAAK,OAAO,MAAM,CAAC,QAAQ,CAAC,cAAc,KAAK,EAAE,CAAC,CAAC;QACnH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,WAAY,SAAQ,aAAa;IAC5C,6BAA6B;IACpB,OAAO,CAAS;IAChB,WAAW,CAAmB;IAEvC,YAAY,KAA8B,EAAE,OAAO,GAAkB,EAAE;QACrE,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAClC,MAAM,UAAU,GAAqB,EAAE,CAAC;QACxC,KAAK,CACH,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,EACzD,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,EAAE,CAC/E,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACM,cAAc;QACrB,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;QACjC,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,KAAK,CAAC;QACrC,KAAK,CAAC,cAAc,EAAE,CAAC;IACzB,CAAC;CACF;AAED,SAAS,YAAY,CAAC,IAAkB,EAAE,KAAa,EAAE,UAA4B;IACnF,OAAO,CAAC,OAAwB,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,iBAAiB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,cAAc,CAC9B,oBAAoB,EACpB,QAAQ,KAAK,GAAG,CAAC,sDAAsD,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CACzG,CAAC;YACF,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,WAAW,CAAC,KAA8B,EAAE,OAAO,GAAkB,EAAE;IACrF,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AACzC,CAAC","sourcesContent":["/**\n * `replayPairs()`: plays a recording back through a `ScriptedModel`, after checking its stamp and expiry, and checks\n * that each call sends the request that was recorded, so a replayed run that drifts from the recorded one fails.\n */\nimport { SDK_VERSION } from \"../version.js\";\nimport type { Clock } from \"./determinism.js\";\nimport { normalisedRequest, type RecordedPair, RecordingError, type RecordingStamp, storable } from \"./recording.js\";\nimport { type RecordedRequest, ScriptedModel, type ScriptStep } from \"./scripted-model.js\";\n\n/** The stamp fields a replay can require to match exactly. */\nexport type ExpectedStamp = Partial<\n Pick<RecordingStamp, \"sdkVersion\" | \"api\" | \"provider\" | \"model\" | \"providerVersion\">\n>;\n\n/** What `replayPairs()` takes besides the pairs. */\nexport interface ReplayOptions {\n /** Gives the time the expiry is checked against. Defaults to the system time. */\n readonly clock?: Clock;\n /** Stamp fields that must match exactly, such as the `providerVersion` your adapter speaks today. */\n readonly expect?: ExpectedStamp;\n}\n\n/** Sorted-key JSON, so two requests compare equal whatever order their keys were written in. */\nfunction canonical(value: unknown): string {\n if (Array.isArray(value)) return `[${value.map(canonical).join(\",\")}]`;\n if (typeof value !== \"object\" || value === null) return JSON.stringify(value);\n const keys = Object.keys(value).sort();\n return `{${keys.map((key) => `${JSON.stringify(key)}:${canonical((value as Record<string, unknown>)[key])}`).join(\",\")}}`;\n}\n\n/** The JSON path of the first place `a` and `b` differ, such as `$.context.messages[2].content`. */\nfunction firstDifference(a: unknown, b: unknown, path = \"$\"): string {\n if (typeof a !== \"object\" || typeof b !== \"object\" || a === null || b === null) return path;\n if (Array.isArray(a) !== Array.isArray(b)) return path;\n if (Array.isArray(a) && Array.isArray(b)) {\n if (a.length !== b.length) return `${path} (length ${b.length}, recorded ${a.length})`;\n const index = a.findIndex((item, i) => canonical(item) !== canonical(b[i]));\n return index < 0 ? path : firstDifference(a[index], b[index], `${path}[${index}]`);\n }\n const left = a as Record<string, unknown>;\n const right = b as Record<string, unknown>;\n const keys = [...new Set([...Object.keys(left), ...Object.keys(right)])].sort();\n const key = keys.find((name) => canonical(left[name]) !== canonical(right[name]));\n return key === undefined ? path : firstDifference(left[key], right[key], `${path}.${key}`);\n}\n\n/** The release line: the major version from 1.0 on, the minor one under 0.x, where a minor bump may break. */\nfunction releaseLine(version: string): string {\n const [major = \"\", minor = \"\"] = version.split(\".\");\n return major === \"0\" ? `0.${minor}` : major;\n}\n\n// `typeof null` is \"object\", so a bare typeof check would let a null stamp through to a TypeError.\nfunction isObject(value: unknown): value is object {\n return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction checkShape(pairs: readonly RecordedPair[]): RecordingStamp {\n if (!Array.isArray(pairs) || pairs.length === 0) {\n throw new RecordingError(\"invalid_recording\", \"replayPairs() needs a non-empty array of recorded pairs\");\n }\n for (const [i, pair] of pairs.entries()) {\n const candidate = pair as Partial<RecordedPair> | null;\n const events = candidate?.response?.events;\n if (!isObject(candidate?.stamp) || !isObject(candidate?.request) || !Array.isArray(events)) {\n throw new RecordingError(\"invalid_recording\", `pair ${i + 1} needs a stamp, a request and response events`);\n }\n }\n return (pairs[0] as RecordedPair).stamp;\n}\n\nfunction checkStamp(stamp: RecordingStamp, pairs: readonly RecordedPair[], options: ReplayOptions): void {\n const others = pairs.findIndex((pair) => canonical(pair.stamp) !== canonical(stamp));\n if (others >= 0) {\n throw new RecordingError(\"recording_mismatch\", `pair ${others + 1} has a different stamp from pair 1`);\n }\n const expiresAt = Date.parse(stamp.expiresAt);\n if (Number.isNaN(expiresAt)) throw new RecordingError(\"invalid_recording\", \"the stamp's expiresAt isn't a time\");\n const now = options.clock?.now() ?? new Date().toISOString();\n if (Date.parse(now) >= expiresAt) {\n throw new RecordingError(\"recording_expired\", `the recording expired at ${stamp.expiresAt}; record it again`);\n }\n if (typeof stamp.sdkVersion !== \"string\" || releaseLine(stamp.sdkVersion) !== releaseLine(SDK_VERSION)) {\n throw new RecordingError(\n \"recording_mismatch\",\n `the recording was made with SDK ${String(stamp.sdkVersion)}, which this SDK (${SDK_VERSION}) can't replay`,\n );\n }\n for (const [field, value] of Object.entries(options.expect ?? {})) {\n const recorded = stamp[field as keyof ExpectedStamp];\n if (value !== undefined && recorded !== value) {\n throw new RecordingError(\"recording_mismatch\", `the stamp's ${field} is ${String(recorded)}, expected ${value}`);\n }\n }\n}\n\n/**\n * A `ScriptedModel` that plays back recorded pairs. Its `api` and `provider` are the stamp's, and `modelId` is the id\n * to pass to `createModel()`, so the replayed calls are priced from the same catalog entry as the recorded ones.\n */\nexport class ReplayModel extends ScriptedModel {\n /** The recorded model id. */\n readonly modelId: string;\n readonly #mismatches: RecordingError[];\n\n constructor(pairs: readonly RecordedPair[], options: ReplayOptions = {}) {\n const stamp = checkShape(pairs);\n checkStamp(stamp, pairs, options);\n const mismatches: RecordingError[] = [];\n super(\n pairs.map((pair, i) => matchingStep(pair, i, mismatches)),\n { ref: { api: stamp.api, provider: stamp.provider }, caps: { ...stamp.caps } },\n );\n this.modelId = stamp.model;\n this.#mismatches = mismatches;\n }\n\n /**\n * Throws the first `RecordingError` a call raised by sending a request other than the recorded one, then does what\n * `ScriptedModel.assertComplete()` does. Call it in teardown: behind `createModel()`, a mismatch ends the run with\n * an `error` result, and only this check makes the test fail.\n */\n override assertComplete(): void {\n const [first] = this.#mismatches;\n if (first !== undefined) throw first;\n super.assertComplete();\n }\n}\n\nfunction matchingStep(pair: RecordedPair, index: number, mismatches: RecordingError[]): ScriptStep {\n return (request: RecordedRequest) => {\n const sent = normalisedRequest(request.model, request.context, request.options);\n const recorded = storable(pair.request);\n if (canonical(sent) !== canonical(recorded)) {\n const error = new RecordingError(\n \"recording_mismatch\",\n `call ${index + 1} sent a request that differs from the recording at ${firstDifference(recorded, sent)}`,\n );\n mismatches.push(error);\n throw error;\n }\n return pair.response;\n };\n}\n\n/**\n * Replays pairs from `recordPairs()` through a `ScriptedModel`: the n-th call streams the n-th pair's events exactly as\n * they were recorded, after checking that it sent the recorded request. Wrap it with\n * `createModel({ adapter: replay, id: replay.modelId })`, with no middleware: the recording was taken outside the\n * recorded model's middleware, so it already holds that middleware's effect.\n *\n * Throws a `RecordingError`: `invalid_recording` for an empty or malformed recording, `recording_expired` once the\n * clock reaches the stamp's `expiresAt`, and `recording_mismatch` when the pairs' stamps differ, the recording comes\n * from another SDK release line (another minor version under 0.x), or a stamp field differs from `expect`. A call\n * whose request differs from the recording fails with a `recording_mismatch` error, which `assertComplete()`\n * rethrows.\n */\nexport function replayPairs(pairs: readonly RecordedPair[], options: ReplayOptions = {}): ReplayModel {\n return new ReplayModel(pairs, options);\n}\n"]}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `scriptedApprovals()`: a strict double for the person behind an approval. It answers each approval request the run
|
|
3
|
+
* raises with the next scripted answer, through `blockingApprovals()`, so a test drives the real signing and checking
|
|
4
|
+
* path without a store or a second process.
|
|
5
|
+
*/
|
|
6
|
+
import type { Interceptors } from "../hooks.js";
|
|
7
|
+
import type { Resolution, SignedPendingRequest } from "../vendor/agent-safety/pending.js";
|
|
8
|
+
type Decision = Extract<Resolution, {
|
|
9
|
+
decision: unknown;
|
|
10
|
+
}>;
|
|
11
|
+
/**
|
|
12
|
+
* One scripted answer: allow or deny, with an optional reason and approver. Give a `requestId` only to test an
|
|
13
|
+
* answer that names the wrong request; without it, the answer names the request it answers.
|
|
14
|
+
*/
|
|
15
|
+
export type ScriptedAnswer = Omit<Decision, "requestId"> & {
|
|
16
|
+
readonly requestId?: string;
|
|
17
|
+
};
|
|
18
|
+
/** The interceptors to give a run, with what the double saw. */
|
|
19
|
+
export interface ScriptedApprovals extends Interceptors {
|
|
20
|
+
/** Every approval request the run raised so far, in order. */
|
|
21
|
+
readonly requests: readonly SignedPendingRequest[];
|
|
22
|
+
/** Throws if an answer was never used, or if the run raised more requests than there are answers. */
|
|
23
|
+
assertComplete(): void;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Answers the run's approval requests in order with `answers`, one per request. Join the result with a gate that
|
|
27
|
+
* returns `ask`, as you would `blockingApprovals()`: `composeInterceptors(gate, scriptedApprovals([{ decision:
|
|
28
|
+
* "allow" }]))`. A request past the last answer is denied, with a `hook_error` event, and `assertComplete()` then
|
|
29
|
+
* throws.
|
|
30
|
+
*/
|
|
31
|
+
export declare function scriptedApprovals(answers: readonly ScriptedAnswer[]): ScriptedApprovals;
|
|
32
|
+
export {};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { blockingApprovals } from "../pauses.js";
|
|
2
|
+
/**
|
|
3
|
+
* Answers the run's approval requests in order with `answers`, one per request. Join the result with a gate that
|
|
4
|
+
* returns `ask`, as you would `blockingApprovals()`: `composeInterceptors(gate, scriptedApprovals([{ decision:
|
|
5
|
+
* "allow" }]))`. A request past the last answer is denied, with a `hook_error` event, and `assertComplete()` then
|
|
6
|
+
* throws.
|
|
7
|
+
*/
|
|
8
|
+
export function scriptedApprovals(answers) {
|
|
9
|
+
const script = [...answers];
|
|
10
|
+
const requests = [];
|
|
11
|
+
const interceptors = blockingApprovals((request) => {
|
|
12
|
+
requests.push(request);
|
|
13
|
+
const answer = script[requests.length - 1];
|
|
14
|
+
if (answer === undefined) {
|
|
15
|
+
throw new Error(`scriptedApprovals: request ${requests.length} has no scripted answer (the script has ${script.length})`);
|
|
16
|
+
}
|
|
17
|
+
return { ...answer, requestId: answer.requestId ?? request.id };
|
|
18
|
+
});
|
|
19
|
+
return Object.assign(interceptors, {
|
|
20
|
+
requests,
|
|
21
|
+
assertComplete() {
|
|
22
|
+
if (requests.length > script.length) {
|
|
23
|
+
throw new Error(`scriptedApprovals: the run raised ${requests.length} approval requests, but the script has ${script.length}`);
|
|
24
|
+
}
|
|
25
|
+
const unused = script.length - requests.length;
|
|
26
|
+
if (unused > 0)
|
|
27
|
+
throw new Error(`scriptedApprovals: ${unused} of ${script.length} answers were never used`);
|
|
28
|
+
},
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=scripted-approvals.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scripted-approvals.js","sourceRoot":"","sources":["../../src/testing/scripted-approvals.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAmBjD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAkC;IAClE,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;IAC5B,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,MAAM,YAAY,GAAG,iBAAiB,CAAC,CAAC,OAAO,EAAE,EAAE;QACjD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,8BAA8B,QAAQ,CAAC,MAAM,2CAA2C,MAAM,CAAC,MAAM,GAAG,CACzG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE;QACjC,QAAQ;QACR,cAAc;YACZ,IAAI,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CACb,qCAAqC,QAAQ,CAAC,MAAM,0CAA0C,MAAM,CAAC,MAAM,EAAE,CAC9G,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC/C,IAAI,MAAM,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,OAAO,MAAM,CAAC,MAAM,0BAA0B,CAAC,CAAC;QAC9G,CAAC;KACF,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * `scriptedApprovals()`: a strict double for the person behind an approval. It answers each approval request the run\n * raises with the next scripted answer, through `blockingApprovals()`, so a test drives the real signing and checking\n * path without a store or a second process.\n */\nimport type { Interceptors } from \"../hooks.js\";\nimport { blockingApprovals } from \"../pauses.js\";\nimport type { Resolution, SignedPendingRequest } from \"../vendor/agent-safety/pending.js\";\n\ntype Decision = Extract<Resolution, { decision: unknown }>;\n\n/**\n * One scripted answer: allow or deny, with an optional reason and approver. Give a `requestId` only to test an\n * answer that names the wrong request; without it, the answer names the request it answers.\n */\nexport type ScriptedAnswer = Omit<Decision, \"requestId\"> & { readonly requestId?: string };\n\n/** The interceptors to give a run, with what the double saw. */\nexport interface ScriptedApprovals extends Interceptors {\n /** Every approval request the run raised so far, in order. */\n readonly requests: readonly SignedPendingRequest[];\n /** Throws if an answer was never used, or if the run raised more requests than there are answers. */\n assertComplete(): void;\n}\n\n/**\n * Answers the run's approval requests in order with `answers`, one per request. Join the result with a gate that\n * returns `ask`, as you would `blockingApprovals()`: `composeInterceptors(gate, scriptedApprovals([{ decision:\n * \"allow\" }]))`. A request past the last answer is denied, with a `hook_error` event, and `assertComplete()` then\n * throws.\n */\nexport function scriptedApprovals(answers: readonly ScriptedAnswer[]): ScriptedApprovals {\n const script = [...answers];\n const requests: SignedPendingRequest[] = [];\n const interceptors = blockingApprovals((request) => {\n requests.push(request);\n const answer = script[requests.length - 1];\n if (answer === undefined) {\n throw new Error(\n `scriptedApprovals: request ${requests.length} has no scripted answer (the script has ${script.length})`,\n );\n }\n return { ...answer, requestId: answer.requestId ?? request.id };\n });\n return Object.assign(interceptors, {\n requests,\n assertComplete(): void {\n if (requests.length > script.length) {\n throw new Error(\n `scriptedApprovals: the run raised ${requests.length} approval requests, but the script has ${script.length}`,\n );\n }\n const unused = script.length - requests.length;\n if (unused > 0) throw new Error(`scriptedApprovals: ${unused} of ${script.length} answers were never used`);\n },\n });\n}\n"]}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { MaybePromise } from "../hooks.js";
|
|
2
|
+
import type { JsonSchema, ToolDefinition, ToolResult } from "../tools/types.js";
|
|
3
|
+
/** One scripted result: a fixed result, or a function that builds one from the call's input. */
|
|
4
|
+
export type ScriptedToolStep = ToolResult | ((input: unknown) => MaybePromise<ToolResult>);
|
|
5
|
+
/** One tool the scripted server offers. */
|
|
6
|
+
export interface ScriptedMcpTool {
|
|
7
|
+
/** The tool's name on the server; the agent sees `mcp__<server>__<name>`. */
|
|
8
|
+
readonly name: string;
|
|
9
|
+
readonly description?: string;
|
|
10
|
+
/** The tool's input schema as the server sends it. */
|
|
11
|
+
readonly inputSchema: JsonSchema;
|
|
12
|
+
/** One result per call, in order. */
|
|
13
|
+
readonly results: readonly ScriptedToolStep[];
|
|
14
|
+
}
|
|
15
|
+
/** What `scriptedMcpServer()` takes. */
|
|
16
|
+
export interface ScriptedMcpServerOptions {
|
|
17
|
+
/** The server's name, used in each tool's namespaced name. */
|
|
18
|
+
readonly name: string;
|
|
19
|
+
readonly tools: readonly ScriptedMcpTool[];
|
|
20
|
+
}
|
|
21
|
+
/** One call a scripted tool received. */
|
|
22
|
+
export interface ScriptedMcpCall {
|
|
23
|
+
/** The namespaced tool name, such as `mcp__docs__search`. */
|
|
24
|
+
readonly tool: string;
|
|
25
|
+
/** The validated input. */
|
|
26
|
+
readonly input: unknown;
|
|
27
|
+
}
|
|
28
|
+
/** The server's tools, to pass to `run()`, and what they saw. */
|
|
29
|
+
export interface ScriptedMcpServer {
|
|
30
|
+
readonly tools: readonly ToolDefinition[];
|
|
31
|
+
/** Every call so far, across the server's tools, in the order they ran. */
|
|
32
|
+
readonly calls: readonly ScriptedMcpCall[];
|
|
33
|
+
/** Throws if a scripted result was never used, or if a tool was called more often than its script allows. */
|
|
34
|
+
assertComplete(): void;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Builds the tools of a scripted MCP server. Each is named `mcp__<server>__<tool>`, has `kind: "dynamic"` and
|
|
38
|
+
* `firstParty: false`, so its annotations never change scheduling, and returns its scripted results one call at a
|
|
39
|
+
* time. A call past the last result throws, which the model reads as an `execution_error` result, and
|
|
40
|
+
* `assertComplete()` then throws. Throws an `SdkError` with code `invalid_tool_definition` for a name that breaks the
|
|
41
|
+
* tool-name rule once namespaced; `run()` checks the rest of each definition, its schema included, when it starts.
|
|
42
|
+
*/
|
|
43
|
+
export declare function scriptedMcpServer(options: ScriptedMcpServerOptions): ScriptedMcpServer;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `scriptedMcpServer()`: a strict double for the tools an MCP server lends an agent. Each tool is defined the way a
|
|
3
|
+
* client would define a server's tool (a namespaced name, a schema known only at run time, not first-party) and plays
|
|
4
|
+
* scripted results, so a test covers how the loop treats tools it doesn't control without starting a server.
|
|
5
|
+
*/
|
|
6
|
+
import { assertCanonicalName } from "@buildanagent/gateway";
|
|
7
|
+
import { SdkError } from "../errors.js";
|
|
8
|
+
function checkName(name) {
|
|
9
|
+
try {
|
|
10
|
+
assertCanonicalName(name);
|
|
11
|
+
}
|
|
12
|
+
catch (cause) {
|
|
13
|
+
throw new SdkError("invalid_tool_definition", cause instanceof Error ? cause.message : String(cause), { cause });
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
// A plain frozen definition rather than `defineTool()`, which would load the schema validator into the testing
|
|
17
|
+
// export; `run()` checks the definition and compiles its schema when the run starts.
|
|
18
|
+
function scriptedTool(server, spec, state, calls) {
|
|
19
|
+
checkName(state.name);
|
|
20
|
+
return Object.freeze({
|
|
21
|
+
name: state.name,
|
|
22
|
+
description: spec.description ?? `${spec.name} from the ${server} server`,
|
|
23
|
+
kind: "dynamic",
|
|
24
|
+
firstParty: false,
|
|
25
|
+
inputSchema: spec.inputSchema,
|
|
26
|
+
async execute(input) {
|
|
27
|
+
calls.push({ tool: state.name, input });
|
|
28
|
+
const step = state.results[state.used];
|
|
29
|
+
if (step === undefined) {
|
|
30
|
+
state.overRuns += 1;
|
|
31
|
+
throw new Error(`${state.name}: call ${state.used + state.overRuns} has no scripted result`);
|
|
32
|
+
}
|
|
33
|
+
state.used += 1;
|
|
34
|
+
return typeof step === "function" ? step(input) : step;
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Builds the tools of a scripted MCP server. Each is named `mcp__<server>__<tool>`, has `kind: "dynamic"` and
|
|
40
|
+
* `firstParty: false`, so its annotations never change scheduling, and returns its scripted results one call at a
|
|
41
|
+
* time. A call past the last result throws, which the model reads as an `execution_error` result, and
|
|
42
|
+
* `assertComplete()` then throws. Throws an `SdkError` with code `invalid_tool_definition` for a name that breaks the
|
|
43
|
+
* tool-name rule once namespaced; `run()` checks the rest of each definition, its schema included, when it starts.
|
|
44
|
+
*/
|
|
45
|
+
export function scriptedMcpServer(options) {
|
|
46
|
+
const calls = [];
|
|
47
|
+
const states = options.tools.map((spec) => ({
|
|
48
|
+
spec,
|
|
49
|
+
state: { name: `mcp__${options.name}__${spec.name}`, results: [...spec.results], used: 0, overRuns: 0 },
|
|
50
|
+
}));
|
|
51
|
+
const tools = states.map(({ spec, state }) => scriptedTool(options.name, spec, state, calls));
|
|
52
|
+
return {
|
|
53
|
+
tools,
|
|
54
|
+
calls,
|
|
55
|
+
assertComplete() {
|
|
56
|
+
for (const { state } of states) {
|
|
57
|
+
if (state.overRuns > 0) {
|
|
58
|
+
throw new Error(`${state.name}: called ${state.overRuns} more time(s) than scripted`);
|
|
59
|
+
}
|
|
60
|
+
const unused = state.results.length - state.used;
|
|
61
|
+
if (unused > 0)
|
|
62
|
+
throw new Error(`${state.name}: ${unused} of ${state.results.length} results were never used`);
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=scripted-mcp.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scripted-mcp.js","sourceRoot":"","sources":["../../src/testing/scripted-mcp.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAiDxC,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,QAAQ,CAAC,yBAAyB,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IACnH,CAAC;AACH,CAAC;AAED,+GAA+G;AAC/G,qFAAqF;AACrF,SAAS,YAAY,CACnB,MAAc,EACd,IAAqB,EACrB,KAAgB,EAChB,KAAwB;IAExB,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG,IAAI,CAAC,IAAI,aAAa,MAAM,SAAS;QACzE,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,CAAC,OAAO,CAAC,KAAK;YACjB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACxC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,yBAAyB,CAAC,CAAC;YAC/F,CAAC;YACD,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;YAChB,OAAO,OAAO,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzD,CAAC;KACuB,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAiC;IACjE,MAAM,KAAK,GAAsB,EAAE,CAAC;IACpC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1C,IAAI;QACJ,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;KACxG,CAAC,CAAC,CAAC;IACJ,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IAC9F,OAAO;QACL,KAAK;QACL,KAAK;QACL,cAAc;YACZ,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,MAAM,EAAE,CAAC;gBAC/B,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,YAAY,KAAK,CAAC,QAAQ,6BAA6B,CAAC,CAAC;gBACxF,CAAC;gBACD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;gBACjD,IAAI,MAAM,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,MAAM,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,0BAA0B,CAAC,CAAC;YACjH,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC","sourcesContent":["/**\n * `scriptedMcpServer()`: a strict double for the tools an MCP server lends an agent. Each tool is defined the way a\n * client would define a server's tool (a namespaced name, a schema known only at run time, not first-party) and plays\n * scripted results, so a test covers how the loop treats tools it doesn't control without starting a server.\n */\nimport { assertCanonicalName } from \"@buildanagent/gateway\";\nimport { SdkError } from \"../errors.js\";\nimport type { MaybePromise } from \"../hooks.js\";\nimport type { JsonSchema, ToolDefinition, ToolResult } from \"../tools/types.js\";\n\n/** One scripted result: a fixed result, or a function that builds one from the call's input. */\nexport type ScriptedToolStep = ToolResult | ((input: unknown) => MaybePromise<ToolResult>);\n\n/** One tool the scripted server offers. */\nexport interface ScriptedMcpTool {\n /** The tool's name on the server; the agent sees `mcp__<server>__<name>`. */\n readonly name: string;\n readonly description?: string;\n /** The tool's input schema as the server sends it. */\n readonly inputSchema: JsonSchema;\n /** One result per call, in order. */\n readonly results: readonly ScriptedToolStep[];\n}\n\n/** What `scriptedMcpServer()` takes. */\nexport interface ScriptedMcpServerOptions {\n /** The server's name, used in each tool's namespaced name. */\n readonly name: string;\n readonly tools: readonly ScriptedMcpTool[];\n}\n\n/** One call a scripted tool received. */\nexport interface ScriptedMcpCall {\n /** The namespaced tool name, such as `mcp__docs__search`. */\n readonly tool: string;\n /** The validated input. */\n readonly input: unknown;\n}\n\n/** The server's tools, to pass to `run()`, and what they saw. */\nexport interface ScriptedMcpServer {\n readonly tools: readonly ToolDefinition[];\n /** Every call so far, across the server's tools, in the order they ran. */\n readonly calls: readonly ScriptedMcpCall[];\n /** Throws if a scripted result was never used, or if a tool was called more often than its script allows. */\n assertComplete(): void;\n}\n\ninterface ToolState {\n readonly name: string;\n readonly results: readonly ScriptedToolStep[];\n used: number;\n overRuns: number;\n}\n\nfunction checkName(name: string): void {\n try {\n assertCanonicalName(name);\n } catch (cause) {\n throw new SdkError(\"invalid_tool_definition\", cause instanceof Error ? cause.message : String(cause), { cause });\n }\n}\n\n// A plain frozen definition rather than `defineTool()`, which would load the schema validator into the testing\n// export; `run()` checks the definition and compiles its schema when the run starts.\nfunction scriptedTool(\n server: string,\n spec: ScriptedMcpTool,\n state: ToolState,\n calls: ScriptedMcpCall[],\n): ToolDefinition {\n checkName(state.name);\n return Object.freeze({\n name: state.name,\n description: spec.description ?? `${spec.name} from the ${server} server`,\n kind: \"dynamic\",\n firstParty: false,\n inputSchema: spec.inputSchema,\n async execute(input) {\n calls.push({ tool: state.name, input });\n const step = state.results[state.used];\n if (step === undefined) {\n state.overRuns += 1;\n throw new Error(`${state.name}: call ${state.used + state.overRuns} has no scripted result`);\n }\n state.used += 1;\n return typeof step === \"function\" ? step(input) : step;\n },\n } satisfies ToolDefinition);\n}\n\n/**\n * Builds the tools of a scripted MCP server. Each is named `mcp__<server>__<tool>`, has `kind: \"dynamic\"` and\n * `firstParty: false`, so its annotations never change scheduling, and returns its scripted results one call at a\n * time. A call past the last result throws, which the model reads as an `execution_error` result, and\n * `assertComplete()` then throws. Throws an `SdkError` with code `invalid_tool_definition` for a name that breaks the\n * tool-name rule once namespaced; `run()` checks the rest of each definition, its schema included, when it starts.\n */\nexport function scriptedMcpServer(options: ScriptedMcpServerOptions): ScriptedMcpServer {\n const calls: ScriptedMcpCall[] = [];\n const states = options.tools.map((spec) => ({\n spec,\n state: { name: `mcp__${options.name}__${spec.name}`, results: [...spec.results], used: 0, overRuns: 0 },\n }));\n const tools = states.map(({ spec, state }) => scriptedTool(options.name, spec, state, calls));\n return {\n tools,\n calls,\n assertComplete(): void {\n for (const { state } of states) {\n if (state.overRuns > 0) {\n throw new Error(`${state.name}: called ${state.overRuns} more time(s) than scripted`);\n }\n const unused = state.results.length - state.used;\n if (unused > 0) throw new Error(`${state.name}: ${unused} of ${state.results.length} results were never used`);\n }\n },\n };\n}\n"]}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* - **Frozen snapshots.** Each request is cloned and deeply frozen when the call is made, so later mutation of the
|
|
7
7
|
* caller's context never changes `calls`.
|
|
8
8
|
* - **Realistic streams.** Text, reasoning and tool-input JSON arrive in chunks of `chunkSize` characters, and the
|
|
9
|
-
* abort signal is checked between chunks.
|
|
9
|
+
* abort signal is checked between chunks. A `StreamScript` instead plays back recorded events exactly as they came.
|
|
10
10
|
* - **Deterministic.** Chunk sizes and generated ids come only from the options, never `Math.random()` or `Date.now()`.
|
|
11
11
|
*
|
|
12
12
|
* It implements the gateway's `ModelAdapter` and imports nothing from the loop, so gateway tests can use it too.
|
|
@@ -33,8 +33,15 @@ export interface RecordedRequest {
|
|
|
33
33
|
readonly context: ModelContext;
|
|
34
34
|
readonly options: Readonly<Pick<StreamOptions, "maxOutputTokens" | "temperature" | "cacheRetention" | "providerOptions">>;
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* A response played back event for event, such as one `recordPairs()` captured from a real model. `events` must start
|
|
38
|
+
* with `start` and end with its only terminal, `done` or `error`; the abort signal is checked between events.
|
|
39
|
+
*/
|
|
40
|
+
export interface StreamScript {
|
|
41
|
+
readonly events: readonly ModelEvent[];
|
|
42
|
+
}
|
|
36
43
|
/** A fixed response, or a responder that builds one from the recorded request. */
|
|
37
|
-
export type ScriptStep = AssistantScript | ((request: RecordedRequest) => AssistantScript | Promise<AssistantScript>);
|
|
44
|
+
export type ScriptStep = AssistantScript | StreamScript | ((request: RecordedRequest) => AssistantScript | StreamScript | Promise<AssistantScript | StreamScript>);
|
|
38
45
|
export interface ScriptedModelOptions {
|
|
39
46
|
/** Characters per delta. Default 4. */
|
|
40
47
|
chunkSize?: number;
|
|
@@ -53,6 +60,8 @@ export declare class UnexpectedModelCallError extends SdkError {
|
|
|
53
60
|
readonly name = "UnexpectedModelCallError";
|
|
54
61
|
constructor(message: string);
|
|
55
62
|
}
|
|
63
|
+
/** The serialisable stream options, with omitted ones left out rather than set to `undefined`. */
|
|
64
|
+
export declare function recordedOptions(options: StreamOptions): RecordedRequest["options"];
|
|
56
65
|
export declare class ScriptedModel implements ModelAdapter {
|
|
57
66
|
#private;
|
|
58
67
|
readonly specificationVersion: "v1";
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* - **Frozen snapshots.** Each request is cloned and deeply frozen when the call is made, so later mutation of the
|
|
7
7
|
* caller's context never changes `calls`.
|
|
8
8
|
* - **Realistic streams.** Text, reasoning and tool-input JSON arrive in chunks of `chunkSize` characters, and the
|
|
9
|
-
* abort signal is checked between chunks.
|
|
9
|
+
* abort signal is checked between chunks. A `StreamScript` instead plays back recorded events exactly as they came.
|
|
10
10
|
* - **Deterministic.** Chunk sizes and generated ids come only from the options, never `Math.random()` or `Date.now()`.
|
|
11
11
|
*
|
|
12
12
|
* It implements the gateway's `ModelAdapter` and imports nothing from the loop, so gateway tests can use it too.
|
|
@@ -41,7 +41,7 @@ function chunks(text, size) {
|
|
|
41
41
|
return out;
|
|
42
42
|
}
|
|
43
43
|
/** The serialisable stream options, with omitted ones left out rather than set to `undefined`. */
|
|
44
|
-
function recordedOptions(options) {
|
|
44
|
+
export function recordedOptions(options) {
|
|
45
45
|
const recorded = {};
|
|
46
46
|
if (options.maxOutputTokens !== undefined)
|
|
47
47
|
recorded.maxOutputTokens = options.maxOutputTokens;
|
|
@@ -71,6 +71,33 @@ function terminal(run, script) {
|
|
|
71
71
|
const stopReason = script.stopReason ?? (run.content.some((b) => b.type === "tool_call") ? "tool_calls" : "stop");
|
|
72
72
|
return { type: "done", message: messageOf(run, stopReason) };
|
|
73
73
|
}
|
|
74
|
+
function isTerminal(event) {
|
|
75
|
+
return event.type === "done" || event.type === "error";
|
|
76
|
+
}
|
|
77
|
+
function checkStreamScript(script) {
|
|
78
|
+
const { events } = script;
|
|
79
|
+
const terminals = events.filter(isTerminal).length;
|
|
80
|
+
if (events[0]?.type !== "start" || terminals !== 1 || !isTerminal(events[events.length - 1])) {
|
|
81
|
+
throw new RangeError("a StreamScript's events must start with start and end with its only done or error event");
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/** Plays a stream script's events as copies; an abort ends it with the blocks that had ended so far. */
|
|
85
|
+
async function* playEvents(script, model, signal) {
|
|
86
|
+
checkStreamScript(script);
|
|
87
|
+
const ended = [];
|
|
88
|
+
for (const event of script.events) {
|
|
89
|
+
if (signal?.aborted && event.type !== "start") {
|
|
90
|
+
const run = { model, content: ended, usage: unreportedUsage() };
|
|
91
|
+
yield { type: "error", message: messageOf(run, "aborted") };
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const copy = structuredClone(event);
|
|
95
|
+
if (copy.type === "text_end" || copy.type === "reasoning_end" || copy.type === "tool_input_end") {
|
|
96
|
+
ended.push(copy.block);
|
|
97
|
+
}
|
|
98
|
+
yield copy;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
74
101
|
export class ScriptedModel {
|
|
75
102
|
specificationVersion = "v1";
|
|
76
103
|
api;
|
|
@@ -130,6 +157,10 @@ export class ScriptedModel {
|
|
|
130
157
|
}
|
|
131
158
|
async *#run(step, request, signal) {
|
|
132
159
|
const script = typeof step === "function" ? await step(request) : step;
|
|
160
|
+
if ("events" in script) {
|
|
161
|
+
yield* playEvents(script, request.model, signal);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
133
164
|
const usage = script.usage === undefined ? unreportedUsage() : { ...emptyUsage(), ...script.usage, raw: null };
|
|
134
165
|
const run = { model: request.model, content: [], usage };
|
|
135
166
|
yield { type: "start" };
|