@withgateway/sdk 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/index-DnowA8MG.d.mts +447 -0
  2. package/dist/index-DnowA8MG.d.ts +447 -0
  3. package/dist/index.d.mts +6 -1
  4. package/dist/index.d.ts +6 -1
  5. package/dist/index.js +1544 -23
  6. package/dist/index.js.map +1 -1
  7. package/dist/index.mjs +1504 -17
  8. package/dist/index.mjs.map +1 -1
  9. package/dist/personas/index.d.mts +102 -0
  10. package/dist/personas/index.d.ts +102 -0
  11. package/dist/personas/index.js +205 -0
  12. package/dist/personas/index.js.map +1 -0
  13. package/dist/personas/index.mjs +180 -0
  14. package/dist/personas/index.mjs.map +1 -0
  15. package/dist/replay/index.d.mts +351 -0
  16. package/dist/replay/index.d.ts +351 -0
  17. package/dist/replay/index.js +1094 -0
  18. package/dist/replay/index.js.map +1 -0
  19. package/dist/replay/index.mjs +1057 -0
  20. package/dist/replay/index.mjs.map +1 -0
  21. package/dist/sessions/index.d.mts +143 -0
  22. package/dist/sessions/index.d.ts +143 -0
  23. package/dist/sessions/index.js +320 -0
  24. package/dist/sessions/index.js.map +1 -0
  25. package/dist/sessions/index.mjs +288 -0
  26. package/dist/sessions/index.mjs.map +1 -0
  27. package/dist/tracing/index.d.mts +3 -1
  28. package/dist/tracing/index.d.ts +3 -1
  29. package/dist/tracing/index.js +270 -21
  30. package/dist/tracing/index.js.map +1 -1
  31. package/dist/tracing/index.mjs +259 -16
  32. package/dist/tracing/index.mjs.map +1 -1
  33. package/dist/users/index.d.mts +81 -0
  34. package/dist/users/index.d.ts +81 -0
  35. package/dist/users/index.js +168 -0
  36. package/dist/users/index.js.map +1 -0
  37. package/dist/users/index.mjs +144 -0
  38. package/dist/users/index.mjs.map +1 -0
  39. package/package.json +21 -1
  40. package/dist/index-yJgb8wh6.d.mts +0 -273
  41. package/dist/index-yJgb8wh6.d.ts +0 -273
@@ -0,0 +1,351 @@
1
+ import { SessionRecording } from '../sessions/index.mjs';
2
+
3
+ /**
4
+ * Regression suites replayed from platform task sets.
5
+ *
6
+ * A TASK SET on the platform (filed by the dream agent or a human) is a
7
+ * runnable regression suite: each task carries its prompt, expected behavior,
8
+ * and provenance to the RECORDED SESSION that motivated it. This module turns
9
+ * that into tests:
10
+ *
11
+ * ```typescript
12
+ * import { suite } from "@withgateway/sdk/replay";
13
+ *
14
+ * const regressions = await suite("groceat-regressions");
15
+ * for (const testCase of regressions) {
16
+ * const stubs = await testCase.stubs(); // the recorded world
17
+ * const out = await myAgent(testCase.prompt, stubs.asRegistry());
18
+ * const result = testCase.check(out, { calledTools: stubs.calledNames() });
19
+ * assert.ok(result.passed, result.explain());
20
+ * }
21
+ * ```
22
+ *
23
+ * Three layers pin three different things:
24
+ * - tool stubs pin THE WORLD (recorded tool outputs → deterministic replay)
25
+ * - invariants pin THE CONTRACT (must/must-not tool calls)
26
+ * - your own assertions / platform judges pin THE SEMANTICS
27
+ *
28
+ * Case triage is three-way: PASS, FAIL (real regression — the world simulated
29
+ * fine and the contract broke), and DRIFT (the recorded world no longer
30
+ * matches the system — stub misses, renamed tools; the TEST is stale, not the
31
+ * agent).
32
+ *
33
+ * Mirrors the Python SDK's `gatewaysdk.replay` package.
34
+ */
35
+
36
+ /** Base error for replay loading/config problems. */
37
+ declare class ReplayError extends Error {
38
+ constructor(message: string, options?: {
39
+ cause?: unknown;
40
+ });
41
+ }
42
+ /** Strict-mode stub lookup failed — the recorded world has no answer. */
43
+ declare class StubMissError extends Error {
44
+ constructor(message: string);
45
+ }
46
+ /** The recorded session no longer matches the system (re-baseline the case). */
47
+ declare class DriftError extends ReplayError {
48
+ constructor(message: string);
49
+ }
50
+ type RecordedToolCall = {
51
+ name: string;
52
+ input: unknown;
53
+ output: unknown;
54
+ observationId: string;
55
+ };
56
+ type MissPolicy = "error" | "reuse" | "passthrough";
57
+ type StubMatch = "local" | "exact" | "exact-repeat" | "ordinal" | "reuse" | "miss";
58
+ type StubCallLogEntry = {
59
+ tool: string;
60
+ match: StubMatch;
61
+ input: unknown;
62
+ };
63
+ type StubFallback = (toolName: string, toolInput: unknown) => unknown;
64
+ type LocalTool = (toolInput: unknown) => unknown;
65
+ type ToolStub = (toolInput?: unknown) => unknown;
66
+ /**
67
+ * Tool registry that synthesizes stubs for unknown names (see
68
+ * `ToolStubRegistry.asRegistry`). Indexing an unknown name returns a callable
69
+ * that routes through `lookup()`, so the call is LOGGED and follows the miss
70
+ * policy instead of dying on an undefined-is-not-a-function error.
71
+ */
72
+ type StubRegistryMapping = Record<string, ToolStub>;
73
+ type StubRegistryOptions = {
74
+ missPolicy?: MissPolicy;
75
+ fallback?: StubFallback;
76
+ local?: Record<string, LocalTool>;
77
+ };
78
+ /**
79
+ * Replays recorded tool outputs from a case's recorded session.
80
+ *
81
+ * Match strategies (tried in order):
82
+ * local — the tool was declared `local`: run its live impl (new
83
+ * or deterministic tools that never need recordings)
84
+ * exact — normalized-JSON equality of the tool input
85
+ * exact-repeat — same (name, input) already replayed: return that output
86
+ * again (idempotent world reads never exhaust)
87
+ * ordinal — nth un-consumed recording for that tool name (robust to
88
+ * arg phrasing drift across models/prompts)
89
+ *
90
+ * Miss policy:
91
+ * "error" — throw StubMissError (strict replay; recorded as drift).
92
+ * NOTE: most agent frameworks catch tool errors and feed
93
+ * them to the model as error output, so this does NOT
94
+ * hard-stop such runs — the drift still lands in the
95
+ * report either way.
96
+ * "reuse" — exhausted a tool's recordings? replay the LAST recorded
97
+ * output for that name again (drift recorded). Keeps
98
+ * research-heavy agents running when they make more calls
99
+ * than the recording holds. Never-recorded names still
100
+ * throw (or hit `fallback` when provided).
101
+ * "passthrough" — call the `fallback` callable (live tool) and record the
102
+ * miss as drift for the report
103
+ */
104
+ declare class ToolStubRegistry {
105
+ readonly callLog: StubCallLogEntry[];
106
+ readonly driftEvents: string[];
107
+ private readonly recorded;
108
+ private readonly consumed;
109
+ private readonly consumptionOrder;
110
+ private missPolicy;
111
+ private fallback;
112
+ private readonly local;
113
+ constructor(calls: RecordedToolCall[], options?: StubRegistryOptions);
114
+ static fromSession(session: SessionRecording, options?: StubRegistryOptions): ToolStubRegistry;
115
+ /** Resolve a tool call against the recorded world. */
116
+ lookup(toolName: string, toolInput?: unknown): unknown;
117
+ /**
118
+ * Mapping of toolName → callable, drop-in for most agent tool layers.
119
+ *
120
+ * `local` merges in tools that should ALWAYS run live (new tools the
121
+ * recording predates, or deterministic/pure helpers) — they're logged for
122
+ * the contract check but never count as drift.
123
+ *
124
+ * UNKNOWN tool names also resolve (to a stub that routes through
125
+ * `lookup()`), so a call to an un-recorded tool is LOGGED and follows the
126
+ * miss policy instead of dying on an undefined lookup — that log is what
127
+ * lets `check()` classify a forbidden-but-unrecorded call as FAIL.
128
+ */
129
+ asRegistry(options?: {
130
+ fallback?: StubFallback;
131
+ local?: Record<string, LocalTool>;
132
+ }): StubRegistryMapping;
133
+ calledNames(): string[];
134
+ /**
135
+ * 1:1 comparison of this replay against the recorded trace.
136
+ *
137
+ * The recording is the expectation: every recorded tool call should be
138
+ * replayed exactly once, in order, with nothing extra. Use for strict
139
+ * did-it-do-everything-the-original-did checks (`mode: "exact"`).
140
+ *
141
+ * `includeLocal`: count local-tool calls as extras (default false — local
142
+ * tools are declared live on purpose; their recorded calls are likewise
143
+ * excluded from "missing", since local bypasses recordings).
144
+ */
145
+ compare(options?: {
146
+ includeLocal?: boolean;
147
+ }): TraceDiff;
148
+ get drifted(): boolean;
149
+ }
150
+ /** Replayed-vs-recorded trace comparison (see `ToolStubRegistry.compare`). */
151
+ declare class TraceDiff {
152
+ readonly matched: string[];
153
+ readonly missing: string[];
154
+ readonly extra: string[];
155
+ readonly outOfOrder: string[];
156
+ constructor(fields?: {
157
+ matched?: string[];
158
+ missing?: string[];
159
+ extra?: string[];
160
+ outOfOrder?: string[];
161
+ });
162
+ get identical(): boolean;
163
+ explain(): string;
164
+ }
165
+ type ReplayTaskFields = {
166
+ id: string;
167
+ name?: string | null;
168
+ prompt?: unknown;
169
+ expectedOutput?: unknown;
170
+ metadata?: Record<string, unknown>;
171
+ };
172
+ /** One task-set item as served by GET /task-sets/{id}/tasks. */
173
+ declare class ReplayTask {
174
+ readonly id: string;
175
+ readonly name: string | null;
176
+ readonly prompt: unknown;
177
+ readonly expectedOutput: unknown;
178
+ readonly metadata: Record<string, unknown>;
179
+ constructor(fields: ReplayTaskFields);
180
+ get sourceSessionId(): string | null;
181
+ get sourceTraceId(): string | null;
182
+ static fromApi(raw: Record<string, unknown>): ReplayTask;
183
+ }
184
+ type ReplayStatus = "PASS" | "FAIL" | "DRIFT";
185
+ /** Triage result: PASS | FAIL | DRIFT (+ human-readable reasons). */
186
+ declare class ReplayCheck {
187
+ readonly status: ReplayStatus;
188
+ readonly failures: string[];
189
+ readonly drift: string[];
190
+ readonly traceDiff: TraceDiff | null;
191
+ constructor(status: ReplayStatus, fields?: {
192
+ failures?: string[];
193
+ drift?: string[];
194
+ traceDiff?: TraceDiff | null;
195
+ });
196
+ get passed(): boolean;
197
+ get drifted(): boolean;
198
+ explain(): string;
199
+ }
200
+ type SessionLoader = (sessionId: string) => SessionRecording | Promise<SessionRecording>;
201
+ type ReplayMode = "contract" | "exact";
202
+ type CheckOptions = {
203
+ calledTools?: string[];
204
+ stubs?: ToolStubRegistry;
205
+ mode?: ReplayMode;
206
+ };
207
+ /** One regression case: a task bound to the session it was filed from. */
208
+ declare class ReplayCase {
209
+ readonly task: ReplayTask;
210
+ private readonly loader;
211
+ private _recording;
212
+ private _stubs;
213
+ constructor(task: ReplayTask, options: {
214
+ loader: SessionLoader;
215
+ });
216
+ get id(): string;
217
+ get name(): string;
218
+ get prompt(): unknown;
219
+ get expected(): unknown;
220
+ /** The full recorded session this case was filed from. */
221
+ recording(): Promise<SessionRecording>;
222
+ stubs(options?: StubRegistryOptions): Promise<ToolStubRegistry>;
223
+ /**
224
+ * The recorded conversation's user messages, one per turn — feed them to
225
+ * your agent in order to re-drive the full session (each turn was one
226
+ * trace in the recording).
227
+ */
228
+ conversation(): Promise<string[]>;
229
+ /**
230
+ * Contract invariants + drift triage.
231
+ *
232
+ * Invariants come from task metadata (the workbench-style contract):
233
+ * metadata.expectTools — every listed tool MUST have been called
234
+ * metadata.mustNotCall — none of these may have been called
235
+ * Semantic grading stays with the caller (assert on `output` /
236
+ * `case.expected`, or run a platform judge on the replay trace).
237
+ *
238
+ * `mode` (or task `metadata.replayMode`):
239
+ * "contract" (default) — grade only the invariants above
240
+ * "exact" — true 1:1 replay: the agent must reproduce EVERY
241
+ * recorded tool call, in order, with nothing extra
242
+ * (see TraceDiff); any divergence is a FAIL. The
243
+ * full diff lands in the failure reasons and
244
+ * `traceDiff`.
245
+ */
246
+ check(output?: unknown, options?: CheckOptions): ReplayCheck;
247
+ }
248
+ type ReplayCredentials = {
249
+ host: string;
250
+ publicKey: string;
251
+ secretKey: string;
252
+ timeoutSeconds?: number;
253
+ };
254
+ type ReplaySuiteLoadOptions = {
255
+ host?: string;
256
+ publicKey?: string;
257
+ secretKey?: string;
258
+ timeoutSeconds?: number;
259
+ };
260
+ type ReplaySuiteFields = {
261
+ taskSetId: string;
262
+ taskSetName: string;
263
+ cases: ReplayCase[];
264
+ /** Platform credentials for recording replay runs back to the task set. */
265
+ credentials?: ReplayCredentials | null;
266
+ };
267
+ /** All cases of one task set, iterable and indexable by name or id. */
268
+ declare class ReplaySuite implements Iterable<ReplayCase> {
269
+ readonly taskSetId: string;
270
+ readonly taskSetName: string;
271
+ readonly cases: ReplayCase[];
272
+ /** Credentials for recording replay runs back to the platform. */
273
+ readonly credentials: ReplayCredentials | null;
274
+ constructor(fields: ReplaySuiteFields);
275
+ [Symbol.iterator](): Iterator<ReplayCase>;
276
+ get length(): number;
277
+ /** Look up a case by id or name (throws when absent). */
278
+ get(key: string): ReplayCase;
279
+ static fromTaskSet(nameOrId: string, options?: ReplaySuiteLoadOptions): Promise<ReplaySuite>;
280
+ /** Run the whole suite against `agent` — see `runSuite()`. */
281
+ run(agent: ReplayAgent, options?: RunSuiteOptions): Promise<ReplayReport>;
282
+ /** Pin every case's recorded session into the repo as JSON fixtures. */
283
+ snapshot(directory: string): Promise<string[]>;
284
+ }
285
+ /**
286
+ * The agent under test. Always invoked as `agent(prompt, tools, replayCase)`
287
+ * — declare only the parameters you need (this replaces the Python SDK's
288
+ * signature sniffing with idiomatic JavaScript):
289
+ *
290
+ * `(prompt) => ...` — prompt-only agents
291
+ * `(prompt, tools) => ...` — the common shape
292
+ * `(prompt, tools, replayCase) => ...` — full case access
293
+ * (rebuild stubs via `replayCase.stubs()`)
294
+ */
295
+ type ReplayAgent = (prompt: unknown, tools: StubRegistryMapping, replayCase: ReplayCase) => unknown | Promise<unknown>;
296
+ declare class ReplayReport {
297
+ readonly results: Record<string, ReplayCheck>;
298
+ readonly runName: string | null;
299
+ readonly recordingErrors: string[];
300
+ constructor(results: Record<string, ReplayCheck>, fields?: {
301
+ runName?: string | null;
302
+ recordingErrors?: string[];
303
+ });
304
+ get passed(): number;
305
+ get failed(): number;
306
+ get drifted(): number;
307
+ /** True when nothing FAILED (drift is stale-test signal, not failure). */
308
+ get ok(): boolean;
309
+ explain(): string;
310
+ private count;
311
+ }
312
+ /**
313
+ * Replay traffic records under this environment — the platform hides it from
314
+ * the sessions/traces tables by default (users opt back in via the
315
+ * environment filter), so regression runs never pollute real usage views.
316
+ */
317
+ declare const REPLAY_ENVIRONMENT = "gateway-replay";
318
+ type RunSuiteOptions = {
319
+ missPolicy?: MissPolicy;
320
+ record?: boolean;
321
+ runName?: string;
322
+ };
323
+ /**
324
+ * Run every case and triage. The agent is invoked as
325
+ * `agent(prompt, tools, replayCase)` — declare only what you need.
326
+ *
327
+ * Recording (`record` undefined → auto when the suite was loaded from the
328
+ * platform): each case is filed as a dataset-run item on the task set, so
329
+ * the replay run shows up in the task set's runs with per-case status.
330
+ *
331
+ * NOTE: unlike the Python SDK, cases are not auto-wrapped in a traced span —
332
+ * the TypeScript tracing module has no per-trace context primitive yet. Run
333
+ * items are filed with a synthetic trace id, exactly like a Python run
334
+ * without tracing configured.
335
+ */
336
+ declare function runSuite(suiteToRun: ReplaySuite, agent: ReplayAgent, options?: RunSuiteOptions): Promise<ReplayReport>;
337
+ /**
338
+ * Shorthand for `ReplaySuite.fromTaskSet` — the intuitive entrypoint:
339
+ *
340
+ * ```typescript
341
+ * import * as replay from "@withgateway/sdk/replay";
342
+ * const report = await (await replay.suite("checkout-regressions")).run(myAgent);
343
+ * ```
344
+ */
345
+ declare function suite(nameOrId: string, options?: ReplaySuiteLoadOptions): Promise<ReplaySuite>;
346
+ declare const replay: {
347
+ suite: typeof suite;
348
+ run: typeof runSuite;
349
+ };
350
+
351
+ export { DriftError, type LocalTool, type MissPolicy, REPLAY_ENVIRONMENT, type RecordedToolCall, type ReplayAgent, ReplayCase, ReplayCheck, type ReplayCredentials, ReplayError, type ReplayMode, ReplayReport, type ReplayStatus, ReplaySuite, type ReplaySuiteLoadOptions, ReplayTask, type RunSuiteOptions, type SessionLoader, type StubCallLogEntry, type StubFallback, type StubMatch, StubMissError, type StubRegistryMapping, type ToolStub, ToolStubRegistry, TraceDiff, replay, runSuite, suite };