@withgateway/sdk 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index-DnowA8MG.d.mts +447 -0
- package/dist/index-DnowA8MG.d.ts +447 -0
- package/dist/index.d.mts +6 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +1551 -33
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1511 -27
- package/dist/index.mjs.map +1 -1
- package/dist/personas/index.d.mts +102 -0
- package/dist/personas/index.d.ts +102 -0
- package/dist/personas/index.js +205 -0
- package/dist/personas/index.js.map +1 -0
- package/dist/personas/index.mjs +180 -0
- package/dist/personas/index.mjs.map +1 -0
- package/dist/replay/index.d.mts +351 -0
- package/dist/replay/index.d.ts +351 -0
- package/dist/replay/index.js +1094 -0
- package/dist/replay/index.js.map +1 -0
- package/dist/replay/index.mjs +1057 -0
- package/dist/replay/index.mjs.map +1 -0
- package/dist/sessions/index.d.mts +143 -0
- package/dist/sessions/index.d.ts +143 -0
- package/dist/sessions/index.js +320 -0
- package/dist/sessions/index.js.map +1 -0
- package/dist/sessions/index.mjs +288 -0
- package/dist/sessions/index.mjs.map +1 -0
- package/dist/tracing/index.d.mts +3 -1
- package/dist/tracing/index.d.ts +3 -1
- package/dist/tracing/index.js +277 -31
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +266 -26
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/users/index.d.mts +81 -0
- package/dist/users/index.d.ts +81 -0
- package/dist/users/index.js +168 -0
- package/dist/users/index.js.map +1 -0
- package/dist/users/index.mjs +144 -0
- package/dist/users/index.mjs.map +1 -0
- package/package.json +21 -1
- package/dist/index-yJgb8wh6.d.mts +0 -273
- package/dist/index-yJgb8wh6.d.ts +0 -273
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
import { SessionRecording } from '../sessions/index.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Regression suites replayed from platform task sets.
|
|
5
|
+
*
|
|
6
|
+
* A TASK SET on the platform (filed by the dream agent or a human) is a
|
|
7
|
+
* runnable regression suite: each task carries its prompt, expected behavior,
|
|
8
|
+
* and provenance to the RECORDED SESSION that motivated it. This module turns
|
|
9
|
+
* that into tests:
|
|
10
|
+
*
|
|
11
|
+
* ```typescript
|
|
12
|
+
* import { suite } from "@withgateway/sdk/replay";
|
|
13
|
+
*
|
|
14
|
+
* const regressions = await suite("groceat-regressions");
|
|
15
|
+
* for (const testCase of regressions) {
|
|
16
|
+
* const stubs = await testCase.stubs(); // the recorded world
|
|
17
|
+
* const out = await myAgent(testCase.prompt, stubs.asRegistry());
|
|
18
|
+
* const result = testCase.check(out, { calledTools: stubs.calledNames() });
|
|
19
|
+
* assert.ok(result.passed, result.explain());
|
|
20
|
+
* }
|
|
21
|
+
* ```
|
|
22
|
+
*
|
|
23
|
+
* Three layers pin three different things:
|
|
24
|
+
* - tool stubs pin THE WORLD (recorded tool outputs → deterministic replay)
|
|
25
|
+
* - invariants pin THE CONTRACT (must/must-not tool calls)
|
|
26
|
+
* - your own assertions / platform judges pin THE SEMANTICS
|
|
27
|
+
*
|
|
28
|
+
* Case triage is three-way: PASS, FAIL (real regression — the world simulated
|
|
29
|
+
* fine and the contract broke), and DRIFT (the recorded world no longer
|
|
30
|
+
* matches the system — stub misses, renamed tools; the TEST is stale, not the
|
|
31
|
+
* agent).
|
|
32
|
+
*
|
|
33
|
+
* Mirrors the Python SDK's `gatewaysdk.replay` package.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
/** Base error for replay loading/config problems. */
|
|
37
|
+
declare class ReplayError extends Error {
|
|
38
|
+
constructor(message: string, options?: {
|
|
39
|
+
cause?: unknown;
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
/** Strict-mode stub lookup failed — the recorded world has no answer. */
|
|
43
|
+
declare class StubMissError extends Error {
|
|
44
|
+
constructor(message: string);
|
|
45
|
+
}
|
|
46
|
+
/** The recorded session no longer matches the system (re-baseline the case). */
|
|
47
|
+
declare class DriftError extends ReplayError {
|
|
48
|
+
constructor(message: string);
|
|
49
|
+
}
|
|
50
|
+
type RecordedToolCall = {
|
|
51
|
+
name: string;
|
|
52
|
+
input: unknown;
|
|
53
|
+
output: unknown;
|
|
54
|
+
observationId: string;
|
|
55
|
+
};
|
|
56
|
+
type MissPolicy = "error" | "reuse" | "passthrough";
|
|
57
|
+
type StubMatch = "local" | "exact" | "exact-repeat" | "ordinal" | "reuse" | "miss";
|
|
58
|
+
type StubCallLogEntry = {
|
|
59
|
+
tool: string;
|
|
60
|
+
match: StubMatch;
|
|
61
|
+
input: unknown;
|
|
62
|
+
};
|
|
63
|
+
type StubFallback = (toolName: string, toolInput: unknown) => unknown;
|
|
64
|
+
type LocalTool = (toolInput: unknown) => unknown;
|
|
65
|
+
type ToolStub = (toolInput?: unknown) => unknown;
|
|
66
|
+
/**
|
|
67
|
+
* Tool registry that synthesizes stubs for unknown names (see
|
|
68
|
+
* `ToolStubRegistry.asRegistry`). Indexing an unknown name returns a callable
|
|
69
|
+
* that routes through `lookup()`, so the call is LOGGED and follows the miss
|
|
70
|
+
* policy instead of dying on an undefined-is-not-a-function error.
|
|
71
|
+
*/
|
|
72
|
+
type StubRegistryMapping = Record<string, ToolStub>;
|
|
73
|
+
type StubRegistryOptions = {
|
|
74
|
+
missPolicy?: MissPolicy;
|
|
75
|
+
fallback?: StubFallback;
|
|
76
|
+
local?: Record<string, LocalTool>;
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Replays recorded tool outputs from a case's recorded session.
|
|
80
|
+
*
|
|
81
|
+
* Match strategies (tried in order):
|
|
82
|
+
* local — the tool was declared `local`: run its live impl (new
|
|
83
|
+
* or deterministic tools that never need recordings)
|
|
84
|
+
* exact — normalized-JSON equality of the tool input
|
|
85
|
+
* exact-repeat — same (name, input) already replayed: return that output
|
|
86
|
+
* again (idempotent world reads never exhaust)
|
|
87
|
+
* ordinal — nth un-consumed recording for that tool name (robust to
|
|
88
|
+
* arg phrasing drift across models/prompts)
|
|
89
|
+
*
|
|
90
|
+
* Miss policy:
|
|
91
|
+
* "error" — throw StubMissError (strict replay; recorded as drift).
|
|
92
|
+
* NOTE: most agent frameworks catch tool errors and feed
|
|
93
|
+
* them to the model as error output, so this does NOT
|
|
94
|
+
* hard-stop such runs — the drift still lands in the
|
|
95
|
+
* report either way.
|
|
96
|
+
* "reuse" — exhausted a tool's recordings? replay the LAST recorded
|
|
97
|
+
* output for that name again (drift recorded). Keeps
|
|
98
|
+
* research-heavy agents running when they make more calls
|
|
99
|
+
* than the recording holds. Never-recorded names still
|
|
100
|
+
* throw (or hit `fallback` when provided).
|
|
101
|
+
* "passthrough" — call the `fallback` callable (live tool) and record the
|
|
102
|
+
* miss as drift for the report
|
|
103
|
+
*/
|
|
104
|
+
declare class ToolStubRegistry {
|
|
105
|
+
readonly callLog: StubCallLogEntry[];
|
|
106
|
+
readonly driftEvents: string[];
|
|
107
|
+
private readonly recorded;
|
|
108
|
+
private readonly consumed;
|
|
109
|
+
private readonly consumptionOrder;
|
|
110
|
+
private missPolicy;
|
|
111
|
+
private fallback;
|
|
112
|
+
private readonly local;
|
|
113
|
+
constructor(calls: RecordedToolCall[], options?: StubRegistryOptions);
|
|
114
|
+
static fromSession(session: SessionRecording, options?: StubRegistryOptions): ToolStubRegistry;
|
|
115
|
+
/** Resolve a tool call against the recorded world. */
|
|
116
|
+
lookup(toolName: string, toolInput?: unknown): unknown;
|
|
117
|
+
/**
|
|
118
|
+
* Mapping of toolName → callable, drop-in for most agent tool layers.
|
|
119
|
+
*
|
|
120
|
+
* `local` merges in tools that should ALWAYS run live (new tools the
|
|
121
|
+
* recording predates, or deterministic/pure helpers) — they're logged for
|
|
122
|
+
* the contract check but never count as drift.
|
|
123
|
+
*
|
|
124
|
+
* UNKNOWN tool names also resolve (to a stub that routes through
|
|
125
|
+
* `lookup()`), so a call to an un-recorded tool is LOGGED and follows the
|
|
126
|
+
* miss policy instead of dying on an undefined lookup — that log is what
|
|
127
|
+
* lets `check()` classify a forbidden-but-unrecorded call as FAIL.
|
|
128
|
+
*/
|
|
129
|
+
asRegistry(options?: {
|
|
130
|
+
fallback?: StubFallback;
|
|
131
|
+
local?: Record<string, LocalTool>;
|
|
132
|
+
}): StubRegistryMapping;
|
|
133
|
+
calledNames(): string[];
|
|
134
|
+
/**
|
|
135
|
+
* 1:1 comparison of this replay against the recorded trace.
|
|
136
|
+
*
|
|
137
|
+
* The recording is the expectation: every recorded tool call should be
|
|
138
|
+
* replayed exactly once, in order, with nothing extra. Use for strict
|
|
139
|
+
* did-it-do-everything-the-original-did checks (`mode: "exact"`).
|
|
140
|
+
*
|
|
141
|
+
* `includeLocal`: count local-tool calls as extras (default false — local
|
|
142
|
+
* tools are declared live on purpose; their recorded calls are likewise
|
|
143
|
+
* excluded from "missing", since local bypasses recordings).
|
|
144
|
+
*/
|
|
145
|
+
compare(options?: {
|
|
146
|
+
includeLocal?: boolean;
|
|
147
|
+
}): TraceDiff;
|
|
148
|
+
get drifted(): boolean;
|
|
149
|
+
}
|
|
150
|
+
/** Replayed-vs-recorded trace comparison (see `ToolStubRegistry.compare`). */
|
|
151
|
+
declare class TraceDiff {
|
|
152
|
+
readonly matched: string[];
|
|
153
|
+
readonly missing: string[];
|
|
154
|
+
readonly extra: string[];
|
|
155
|
+
readonly outOfOrder: string[];
|
|
156
|
+
constructor(fields?: {
|
|
157
|
+
matched?: string[];
|
|
158
|
+
missing?: string[];
|
|
159
|
+
extra?: string[];
|
|
160
|
+
outOfOrder?: string[];
|
|
161
|
+
});
|
|
162
|
+
get identical(): boolean;
|
|
163
|
+
explain(): string;
|
|
164
|
+
}
|
|
165
|
+
type ReplayTaskFields = {
|
|
166
|
+
id: string;
|
|
167
|
+
name?: string | null;
|
|
168
|
+
prompt?: unknown;
|
|
169
|
+
expectedOutput?: unknown;
|
|
170
|
+
metadata?: Record<string, unknown>;
|
|
171
|
+
};
|
|
172
|
+
/** One task-set item as served by GET /task-sets/{id}/tasks. */
|
|
173
|
+
declare class ReplayTask {
|
|
174
|
+
readonly id: string;
|
|
175
|
+
readonly name: string | null;
|
|
176
|
+
readonly prompt: unknown;
|
|
177
|
+
readonly expectedOutput: unknown;
|
|
178
|
+
readonly metadata: Record<string, unknown>;
|
|
179
|
+
constructor(fields: ReplayTaskFields);
|
|
180
|
+
get sourceSessionId(): string | null;
|
|
181
|
+
get sourceTraceId(): string | null;
|
|
182
|
+
static fromApi(raw: Record<string, unknown>): ReplayTask;
|
|
183
|
+
}
|
|
184
|
+
type ReplayStatus = "PASS" | "FAIL" | "DRIFT";
|
|
185
|
+
/** Triage result: PASS | FAIL | DRIFT (+ human-readable reasons). */
|
|
186
|
+
declare class ReplayCheck {
|
|
187
|
+
readonly status: ReplayStatus;
|
|
188
|
+
readonly failures: string[];
|
|
189
|
+
readonly drift: string[];
|
|
190
|
+
readonly traceDiff: TraceDiff | null;
|
|
191
|
+
constructor(status: ReplayStatus, fields?: {
|
|
192
|
+
failures?: string[];
|
|
193
|
+
drift?: string[];
|
|
194
|
+
traceDiff?: TraceDiff | null;
|
|
195
|
+
});
|
|
196
|
+
get passed(): boolean;
|
|
197
|
+
get drifted(): boolean;
|
|
198
|
+
explain(): string;
|
|
199
|
+
}
|
|
200
|
+
type SessionLoader = (sessionId: string) => SessionRecording | Promise<SessionRecording>;
|
|
201
|
+
type ReplayMode = "contract" | "exact";
|
|
202
|
+
type CheckOptions = {
|
|
203
|
+
calledTools?: string[];
|
|
204
|
+
stubs?: ToolStubRegistry;
|
|
205
|
+
mode?: ReplayMode;
|
|
206
|
+
};
|
|
207
|
+
/** One regression case: a task bound to the session it was filed from. */
|
|
208
|
+
declare class ReplayCase {
|
|
209
|
+
readonly task: ReplayTask;
|
|
210
|
+
private readonly loader;
|
|
211
|
+
private _recording;
|
|
212
|
+
private _stubs;
|
|
213
|
+
constructor(task: ReplayTask, options: {
|
|
214
|
+
loader: SessionLoader;
|
|
215
|
+
});
|
|
216
|
+
get id(): string;
|
|
217
|
+
get name(): string;
|
|
218
|
+
get prompt(): unknown;
|
|
219
|
+
get expected(): unknown;
|
|
220
|
+
/** The full recorded session this case was filed from. */
|
|
221
|
+
recording(): Promise<SessionRecording>;
|
|
222
|
+
stubs(options?: StubRegistryOptions): Promise<ToolStubRegistry>;
|
|
223
|
+
/**
|
|
224
|
+
* The recorded conversation's user messages, one per turn — feed them to
|
|
225
|
+
* your agent in order to re-drive the full session (each turn was one
|
|
226
|
+
* trace in the recording).
|
|
227
|
+
*/
|
|
228
|
+
conversation(): Promise<string[]>;
|
|
229
|
+
/**
|
|
230
|
+
* Contract invariants + drift triage.
|
|
231
|
+
*
|
|
232
|
+
* Invariants come from task metadata (the workbench-style contract):
|
|
233
|
+
* metadata.expectTools — every listed tool MUST have been called
|
|
234
|
+
* metadata.mustNotCall — none of these may have been called
|
|
235
|
+
* Semantic grading stays with the caller (assert on `output` /
|
|
236
|
+
* `case.expected`, or run a platform judge on the replay trace).
|
|
237
|
+
*
|
|
238
|
+
* `mode` (or task `metadata.replayMode`):
|
|
239
|
+
* "contract" (default) — grade only the invariants above
|
|
240
|
+
* "exact" — true 1:1 replay: the agent must reproduce EVERY
|
|
241
|
+
* recorded tool call, in order, with nothing extra
|
|
242
|
+
* (see TraceDiff); any divergence is a FAIL. The
|
|
243
|
+
* full diff lands in the failure reasons and
|
|
244
|
+
* `traceDiff`.
|
|
245
|
+
*/
|
|
246
|
+
check(output?: unknown, options?: CheckOptions): ReplayCheck;
|
|
247
|
+
}
|
|
248
|
+
type ReplayCredentials = {
|
|
249
|
+
host: string;
|
|
250
|
+
publicKey: string;
|
|
251
|
+
secretKey: string;
|
|
252
|
+
timeoutSeconds?: number;
|
|
253
|
+
};
|
|
254
|
+
type ReplaySuiteLoadOptions = {
|
|
255
|
+
host?: string;
|
|
256
|
+
publicKey?: string;
|
|
257
|
+
secretKey?: string;
|
|
258
|
+
timeoutSeconds?: number;
|
|
259
|
+
};
|
|
260
|
+
type ReplaySuiteFields = {
|
|
261
|
+
taskSetId: string;
|
|
262
|
+
taskSetName: string;
|
|
263
|
+
cases: ReplayCase[];
|
|
264
|
+
/** Platform credentials for recording replay runs back to the task set. */
|
|
265
|
+
credentials?: ReplayCredentials | null;
|
|
266
|
+
};
|
|
267
|
+
/** All cases of one task set, iterable and indexable by name or id. */
|
|
268
|
+
declare class ReplaySuite implements Iterable<ReplayCase> {
|
|
269
|
+
readonly taskSetId: string;
|
|
270
|
+
readonly taskSetName: string;
|
|
271
|
+
readonly cases: ReplayCase[];
|
|
272
|
+
/** Credentials for recording replay runs back to the platform. */
|
|
273
|
+
readonly credentials: ReplayCredentials | null;
|
|
274
|
+
constructor(fields: ReplaySuiteFields);
|
|
275
|
+
[Symbol.iterator](): Iterator<ReplayCase>;
|
|
276
|
+
get length(): number;
|
|
277
|
+
/** Look up a case by id or name (throws when absent). */
|
|
278
|
+
get(key: string): ReplayCase;
|
|
279
|
+
static fromTaskSet(nameOrId: string, options?: ReplaySuiteLoadOptions): Promise<ReplaySuite>;
|
|
280
|
+
/** Run the whole suite against `agent` — see `runSuite()`. */
|
|
281
|
+
run(agent: ReplayAgent, options?: RunSuiteOptions): Promise<ReplayReport>;
|
|
282
|
+
/** Pin every case's recorded session into the repo as JSON fixtures. */
|
|
283
|
+
snapshot(directory: string): Promise<string[]>;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* The agent under test. Always invoked as `agent(prompt, tools, replayCase)`
|
|
287
|
+
* — declare only the parameters you need (this replaces the Python SDK's
|
|
288
|
+
* signature sniffing with idiomatic JavaScript):
|
|
289
|
+
*
|
|
290
|
+
* `(prompt) => ...` — prompt-only agents
|
|
291
|
+
* `(prompt, tools) => ...` — the common shape
|
|
292
|
+
* `(prompt, tools, replayCase) => ...` — full case access
|
|
293
|
+
* (rebuild stubs via `replayCase.stubs()`)
|
|
294
|
+
*/
|
|
295
|
+
type ReplayAgent = (prompt: unknown, tools: StubRegistryMapping, replayCase: ReplayCase) => unknown | Promise<unknown>;
|
|
296
|
+
declare class ReplayReport {
|
|
297
|
+
readonly results: Record<string, ReplayCheck>;
|
|
298
|
+
readonly runName: string | null;
|
|
299
|
+
readonly recordingErrors: string[];
|
|
300
|
+
constructor(results: Record<string, ReplayCheck>, fields?: {
|
|
301
|
+
runName?: string | null;
|
|
302
|
+
recordingErrors?: string[];
|
|
303
|
+
});
|
|
304
|
+
get passed(): number;
|
|
305
|
+
get failed(): number;
|
|
306
|
+
get drifted(): number;
|
|
307
|
+
/** True when nothing FAILED (drift is stale-test signal, not failure). */
|
|
308
|
+
get ok(): boolean;
|
|
309
|
+
explain(): string;
|
|
310
|
+
private count;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Replay traffic records under this environment — the platform hides it from
|
|
314
|
+
* the sessions/traces tables by default (users opt back in via the
|
|
315
|
+
* environment filter), so regression runs never pollute real usage views.
|
|
316
|
+
*/
|
|
317
|
+
declare const REPLAY_ENVIRONMENT = "gateway-replay";
|
|
318
|
+
type RunSuiteOptions = {
|
|
319
|
+
missPolicy?: MissPolicy;
|
|
320
|
+
record?: boolean;
|
|
321
|
+
runName?: string;
|
|
322
|
+
};
|
|
323
|
+
/**
|
|
324
|
+
* Run every case and triage. The agent is invoked as
|
|
325
|
+
* `agent(prompt, tools, replayCase)` — declare only what you need.
|
|
326
|
+
*
|
|
327
|
+
* Recording (`record` undefined → auto when the suite was loaded from the
|
|
328
|
+
* platform): each case is filed as a dataset-run item on the task set, so
|
|
329
|
+
* the replay run shows up in the task set's runs with per-case status.
|
|
330
|
+
*
|
|
331
|
+
* NOTE: unlike the Python SDK, cases are not auto-wrapped in a traced span —
|
|
332
|
+
* the TypeScript tracing module has no per-trace context primitive yet. Run
|
|
333
|
+
* items are filed with a synthetic trace id, exactly like a Python run
|
|
334
|
+
* without tracing configured.
|
|
335
|
+
*/
|
|
336
|
+
declare function runSuite(suiteToRun: ReplaySuite, agent: ReplayAgent, options?: RunSuiteOptions): Promise<ReplayReport>;
|
|
337
|
+
/**
|
|
338
|
+
* Shorthand for `ReplaySuite.fromTaskSet` — the intuitive entrypoint:
|
|
339
|
+
*
|
|
340
|
+
* ```typescript
|
|
341
|
+
* import * as replay from "@withgateway/sdk/replay";
|
|
342
|
+
* const report = await (await replay.suite("checkout-regressions")).run(myAgent);
|
|
343
|
+
* ```
|
|
344
|
+
*/
|
|
345
|
+
declare function suite(nameOrId: string, options?: ReplaySuiteLoadOptions): Promise<ReplaySuite>;
|
|
346
|
+
declare const replay: {
|
|
347
|
+
suite: typeof suite;
|
|
348
|
+
run: typeof runSuite;
|
|
349
|
+
};
|
|
350
|
+
|
|
351
|
+
export { DriftError, type LocalTool, type MissPolicy, REPLAY_ENVIRONMENT, type RecordedToolCall, type ReplayAgent, ReplayCase, ReplayCheck, type ReplayCredentials, ReplayError, type ReplayMode, ReplayReport, type ReplayStatus, ReplaySuite, type ReplaySuiteLoadOptions, ReplayTask, type RunSuiteOptions, type SessionLoader, type StubCallLogEntry, type StubFallback, type StubMatch, StubMissError, type StubRegistryMapping, type ToolStub, ToolStubRegistry, TraceDiff, replay, runSuite, suite };
|