@withgateway/sdk 0.1.13 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,508 @@
1
+ /**
2
+ * LIVE WORLD SESSIONS from TypeScript: your agent, the platform's container.
3
+ *
4
+ * The other two ways of running a world each give up something. Pulling the
5
+ * bundle and executing it locally tests your agent against a COPY — your
6
+ * python, your sqlite, your machine's idea of the environment. Dispatching a
7
+ * hosted run gives you the real container but runs the platform's agent
8
+ * inside it, so your agent never touches the world.
9
+ *
10
+ * A session is both: the platform boots the task's container and holds it
11
+ * warm, and every `call()` here executes inside THAT container. Your agent
12
+ * stays where it is — this file is just the wire.
13
+ *
14
+ * const world = await openWorld("revenue-ops-sim@main");
15
+ * const session = await openSession(world.slug, "revops-mara-dedupe-contacts");
16
+ * await session.ready(); // wait for the container
17
+ * await session.call("crm_search", { entity: "deals" });
18
+ * const { reward, rewards } = await session.grade();
19
+ * await session.close({ keepWarm: true }); // hand it back to the pool
20
+ *
21
+ * Every call is a row on the platform before it is an effect in a container,
22
+ * so the episode is on the record — and the platform traces the session
23
+ * itself, which is why these sessions show up under Sessions without the
24
+ * caller instrumenting anything.
25
+ */
26
+ type WorldToolSpec = {
27
+ name: string;
28
+ /** The world's own signature, e.g. `crm_search(q=None, entity="deals")`. */
29
+ signature: string;
30
+ doc: string | null;
31
+ };
32
+ /** What a model needs to call a world: the schemas to advertise, and the
33
+ * implementations that run them in the container. */
34
+ type WorldToolkit = {
35
+ /** OpenAI-shaped tool definitions, derived from the world's signatures. */
36
+ schemas: {
37
+ type: "function";
38
+ function: {
39
+ name: string;
40
+ description: string;
41
+ parameters: Record<string, unknown>;
42
+ };
43
+ }[];
44
+ /** name -> implementation, each one a call into THIS session. */
45
+ impls: Record<string, (args: Record<string, unknown>) => Promise<unknown>>;
46
+ specs: WorldToolSpec[];
47
+ };
48
+ type WorldSessionState = {
49
+ sessionId: string;
50
+ state: string;
51
+ ready: boolean;
52
+ versionId: string;
53
+ contentHash: string;
54
+ taskName: string | null;
55
+ /** The task's objective — present when the session was opened (not on a
56
+ * bare status read), so a client never needs the bundle on disk. */
57
+ instruction?: string | null;
58
+ /** The tools this task serves. */
59
+ tools?: WorldToolSpec[];
60
+ };
61
+ type GradeResult = {
62
+ reward: number | null;
63
+ rewards: Record<string, number>;
64
+ raw: unknown;
65
+ };
66
+ /**
67
+ * Turn a world's Python signature into a JSON schema the model can call.
68
+ *
69
+ * `crm_update_deal(id, stage=None, owner=None, amount=None)` becomes an
70
+ * object with `id` required and the rest optional. Types are read from the
71
+ * DEFAULTS the world declares — a numeric default means a number, a boolean
72
+ * default a boolean — because that is the only type information a signature
73
+ * actually carries. Everything else is a string, which is what a tool CLI
74
+ * receives anyway.
75
+ */
76
+ declare function schemaFromSignature(spec: WorldToolSpec): {
77
+ type: "function";
78
+ function: {
79
+ name: string;
80
+ description: string;
81
+ parameters: Record<string, unknown>;
82
+ };
83
+ };
84
+ declare class WorldSessionTimeout extends Error {
85
+ }
86
+ declare class WorldSessionCallError extends Error {
87
+ }
88
+ declare class WorldSession {
89
+ readonly sessionId: string;
90
+ private state;
91
+ private readonly options;
92
+ constructor(sessionId: string, state: WorldSessionState, options: WorldsClientOptions);
93
+ get task(): string | null;
94
+ /** What this task asks for — the bundle's own instruction.md. */
95
+ get instruction(): string | null;
96
+ get versionId(): string;
97
+ /** The session's latest platform state (one request, no waiting). */
98
+ status(): Promise<WorldSessionState>;
99
+ /**
100
+ * Wait until the container is warm — meaning the process inside it has
101
+ * asked the platform for work, not merely that a job was accepted.
102
+ */
103
+ ready(opts?: {
104
+ timeoutMs?: number;
105
+ }): Promise<WorldSessionState>;
106
+ /** Enqueue one interaction and wait for the container's answer. */
107
+ private exchange;
108
+ /**
109
+ * Run one of the world's tools INSIDE the container. The arguments and the
110
+ * result are the world's own contract — this layer neither validates nor
111
+ * reshapes them, so a tool error reads exactly as it would in a hosted run.
112
+ */
113
+ call(tool: string, args?: Record<string, unknown>, opts?: {
114
+ timeoutMs?: number;
115
+ }): Promise<unknown>;
116
+ /**
117
+ * THE WORLD'S TOOLS, READY TO HAND TO A MODEL. Schemas come from the
118
+ * world's own signatures and implementations call into this session, so a
119
+ * caller never writes — and never has to re-sync — a tool map of their own:
120
+ *
121
+ * const { schemas, impls } = await session.toolkit();
122
+ * await runMyAgent(impls, schemas, session.instruction);
123
+ *
124
+ * A tool added to the world shows up here on the next run; a tool renamed
125
+ * in the world renames here too, instead of failing at call time against a
126
+ * hand-written map that still remembers the old name.
127
+ */
128
+ toolkit(): Promise<WorldToolkit>;
129
+ /** Run the task's grader against the container's END STATE. */
130
+ grade(opts?: {
131
+ timeoutMs?: number;
132
+ }): Promise<GradeResult>;
133
+ /**
134
+ * End the session. `keepWarm` hands the container back to the pool instead
135
+ * of tearing it down, so the next session for the same task starts warm —
136
+ * the pooled instance still obeys the fleet's idle TTL and pinning.
137
+ */
138
+ close(opts?: {
139
+ keepWarm?: boolean;
140
+ }): Promise<{
141
+ state: string;
142
+ }>;
143
+ }
144
+ /**
145
+ * Attach to a session that already exists — resuming after a crash, or
146
+ * driving a session someone else opened (a container started by hand on a
147
+ * self-hosted install, a session opened by CI and handed to a developer).
148
+ */
149
+ declare function attachSession(sessionId: string, options?: WorldsClientOptions): Promise<WorldSession>;
150
+ /**
151
+ * Open a live session against one task of a world. Returns as soon as the
152
+ * platform has accepted it — call `ready()` to wait for the container.
153
+ */
154
+ declare function openSession(slugOrRef: string, task: string, opts?: {
155
+ versionId?: string;
156
+ reuseWarm?: boolean;
157
+ options?: WorldsClientOptions;
158
+ }): Promise<WorldSession>;
159
+ type SessionRunResult = {
160
+ task: string;
161
+ sessionId: string;
162
+ reward: number | null;
163
+ rewards: Record<string, number>;
164
+ /** Set when the task threw instead of grading. */
165
+ error?: string;
166
+ };
167
+ type SessionRunReport = {
168
+ results: SessionRunResult[];
169
+ /** Mean reward over every task, counting failures as 0. */
170
+ mean: number;
171
+ /** True when `mean >= failUnder` (or no gate was set). */
172
+ passed: boolean;
173
+ };
174
+ /**
175
+ * RUN MANY TASKS IN PARALLEL, one warm container each.
176
+ *
177
+ * Parallelism against live worlds is the normal case — a suite is a dozen
178
+ * tasks and each needs its own container, because two agents sharing one
179
+ * world state would corrupt both episodes. Every caller was writing the same
180
+ * pool, the same per-task open/ready/toolkit/grade/close, and the same
181
+ * "don't let one failure kill the run" logic, so it belongs here:
182
+ *
183
+ * const report = await runSessions({
184
+ * world: "revenue-ops-sim@main",
185
+ * tasks, // default: every task in the world
186
+ * concurrency: 4,
187
+ * failUnder: 0.7,
188
+ * agent: async ({ toolkit, instruction }) =>
189
+ * myAgent(toolkit.impls, toolkit.schemas, instruction),
190
+ * });
191
+ *
192
+ * The handler gets a ready session with its toolkit already bound; grading,
193
+ * closing (warm) and aggregation are handled. A task that throws scores 0
194
+ * and records why — the tasks that passed keep their results.
195
+ */
196
+ declare function runSessions(opts: {
197
+ world: string;
198
+ tasks?: string[];
199
+ concurrency?: number;
200
+ failUnder?: number;
201
+ /** Pin every session to one version (default: the world's head). */
202
+ versionId?: string;
203
+ /** Hand the containers back warm for the next run (default: true). */
204
+ keepWarm?: boolean;
205
+ agent: (ctx: {
206
+ session: WorldSession;
207
+ toolkit: WorldToolkit;
208
+ instruction: string;
209
+ task: string;
210
+ }) => Promise<unknown>;
211
+ onResult?: (result: SessionRunResult) => void;
212
+ options?: WorldsClientOptions;
213
+ }): Promise<SessionRunReport>;
214
+
215
+ /**
216
+ * The working-dir half of the TS CLI: push / pull / checkout / status /
217
+ * branch / tag / propose — a faithful port of the Python client
218
+ * (`gatewaysdk/benchmark_hub/client.py` + `_archive.py`).
219
+ *
220
+ * LOCKSTEP WARNING — the content hash is the version's identity and is
221
+ * byte-compatible with Prime Intellect. It now has FOUR implementations that
222
+ * must stay identical: worker `environmentAgent/tar.ts`, web
223
+ * `benchmark-hub/server/benchmarkContentHash.ts`, Python `_archive.py`, and
224
+ * this file. This one is a line-for-line port of the WEB TypeScript
225
+ * implementation (same language, same crypto) — if the rule ever changes,
226
+ * change all four or push identity splits by client.
227
+ *
228
+ * One deliberate refusal, same policy as the web upload path: a bundle with
229
+ * a ROOT `.gitignore` is refused rather than mis-hashed — the Python CLI
230
+ * honors it via `gitignore_parser`, and an imperfect reimplementation here
231
+ * would produce a wrong, Prime-incompatible hash silently.
232
+ */
233
+ declare const DEFAULT_BRANCH = "main";
234
+ declare function isIncludedInContentHash(relPath: string, includeAllRootFiles: boolean): boolean;
235
+ /** The archive-visible files of a working dir, as sorted relpaths. Throws on
236
+ * a root .gitignore (see the header). */
237
+ declare function collectArchiveRelPaths(dir: string): string[];
238
+ declare function computeContentHash(dir: string): string;
239
+ /** A deterministic .tar.gz of the source files (node's gzip stamps no mtime,
240
+ * matching the Python's mtime=0 gzip). */
241
+ declare function buildSourceTarball(dir: string): Buffer;
242
+ type LocalMeta = Record<string, unknown> & {
243
+ slug?: string;
244
+ branch?: string;
245
+ versionId?: string;
246
+ contentHash?: string;
247
+ baseFiles?: Record<string, string>;
248
+ };
249
+ declare function readLocalMeta(dir: string): LocalMeta | null;
250
+ declare function writeLocalMeta(dir: string, meta: LocalMeta): void;
251
+ declare function snapshotTreeHashes(dir: string): Record<string, string>;
252
+ declare function localTreeState(dir: string): {
253
+ meta: LocalMeta | null;
254
+ dirty: string[];
255
+ clean: boolean;
256
+ };
257
+ declare function push(dir: string, opts?: {
258
+ slug?: string;
259
+ branch?: string;
260
+ force?: boolean;
261
+ message?: string;
262
+ author?: string;
263
+ options?: WorldsClientOptions;
264
+ }): Promise<{
265
+ slug: string;
266
+ versionId: string;
267
+ contentHash: string;
268
+ branch: string;
269
+ }>;
270
+ declare function pull(slugRef: string, target?: string, options?: WorldsClientOptions): Promise<{
271
+ slug: string;
272
+ dir: string;
273
+ versionId: string;
274
+ files: number;
275
+ }>;
276
+ declare function propose(dir: string, opts: {
277
+ title: string;
278
+ reason: string;
279
+ targetBranch?: string;
280
+ options?: WorldsClientOptions;
281
+ }): Promise<{
282
+ proposalId: string;
283
+ }>;
284
+ declare function createRef(slug: string, name: string, kind: "branch" | "tag", versionId?: string, options?: WorldsClientOptions): Promise<{
285
+ name: string;
286
+ kind: string;
287
+ versionId: string;
288
+ }>;
289
+
290
+ /**
291
+ * PR validation: push the checkout's world source, run it, gate on the graded
292
+ * result — one composed call, because CI should not have to re-derive the
293
+ * choreography from primitives.
294
+ *
295
+ * The differentiating property, stated so it stays true: validation runs the
296
+ * PULL REQUEST'S OWN world source. The push files the checkout as a version on
297
+ * the PR's branch (content-hash dedup makes re-runs cheap), the dispatch pins
298
+ * that version, and the platform resolves the branch back to its open PR — so
299
+ * the run page names the PR and the PR comment names the run. Nothing here
300
+ * validates a standing deployment while pretending to validate the change.
301
+ */
302
+
303
+ type ValidatedRun = {
304
+ model: string;
305
+ runId: string;
306
+ report: RunReport | null;
307
+ /** Why this run fails the gate, or null when it passes. */
308
+ failure: string | null;
309
+ };
310
+ type ValidateReport = {
311
+ ok: boolean;
312
+ slug: string;
313
+ branch: string;
314
+ versionId: string;
315
+ contentHash: string;
316
+ runConfig: {
317
+ id: string;
318
+ name: string;
319
+ };
320
+ runs: ValidatedRun[];
321
+ /** A PR-comment-ready summary of the whole validation. */
322
+ markdown: string;
323
+ };
324
+ /**
325
+ * Push `dir` on the current (or given) branch, dispatch the named run config
326
+ * against the pushed version, wait for every run, and gate.
327
+ *
328
+ * The gate: every dispatch accepted, every run terminal COMPLETED, and — when
329
+ * `minScore` is set — every run's score meets it. A run with NO score fails a
330
+ * score gate rather than passing it: null is "not scored", never 0 and never
331
+ * fine.
332
+ */
333
+ declare function validateWorld(dir: string, opts: {
334
+ slug?: string;
335
+ runConfig: string;
336
+ branch?: string;
337
+ minScore?: number | null;
338
+ timeoutMinutes?: number;
339
+ onProgress?: (line: string) => void;
340
+ options?: WorldsClientOptions;
341
+ }): Promise<ValidateReport>;
342
+
343
+ /**
344
+ * Worlds from TypeScript: identity, tasks, and HOSTED execution.
345
+ *
346
+ * The same three-level model as the Python SDK — WorldSet → Task — with one
347
+ * deliberate difference: a world's tools are Python modules, so TypeScript
348
+ * does not materialize task environments in-process. What a TS agent
349
+ * codebase needs is exactly what this module provides: resolve a world ref
350
+ * to its pinned version, read its task list, and dispatch HOSTED runs
351
+ * (parallelism included) that execute the world server-side and file graded,
352
+ * version-pinned rollouts. Local in-process execution stays a Python
353
+ * capability by design, not by omission.
354
+ */
355
+ type WorldsClientOptions = {
356
+ host?: string;
357
+ publicKey?: string;
358
+ secretKey?: string;
359
+ };
360
+ type ResolvedWorldVersion = {
361
+ versionId: string;
362
+ semanticVersion: string | null;
363
+ contentHash: string;
364
+ state?: string;
365
+ };
366
+ type WorldTaskInfo = {
367
+ name: string;
368
+ variant: string | null;
369
+ };
370
+ type DispatchedRun = {
371
+ model: string;
372
+ runId: string;
373
+ ok: boolean;
374
+ error?: string;
375
+ };
376
+ /** Authenticated JSON request against the platform API — the primitive the
377
+ * worlds helpers and the TS CLI share. */
378
+ declare function apiRequest<T>(options: WorldsClientOptions, method: "GET" | "POST" | "PUT" | "DELETE", path: string, body?: unknown): Promise<T>;
379
+ /** Set (or rotate) one environment secret — write-only: no API returns it. */
380
+ declare function setEnvSecret(slug: string, key: string, value: string, options?: WorldsClientOptions): Promise<{
381
+ key: string;
382
+ created: boolean;
383
+ }>;
384
+ /** Keys and rotation timestamps only — never values. */
385
+ declare function listEnvSecretKeys(slug: string, options?: WorldsClientOptions): Promise<{
386
+ key: string;
387
+ updatedAt: string;
388
+ }[]>;
389
+ declare function deleteEnvSecret(slug: string, key: string, options?: WorldsClientOptions): Promise<void>;
390
+ type RunConfigSummary = {
391
+ id: string;
392
+ name: string;
393
+ runType: string;
394
+ createdAt: string;
395
+ models: string[];
396
+ parallelism: number;
397
+ autoRunOnPush: boolean;
398
+ agent: string | null;
399
+ };
400
+ /** Run configs in the project. CI resolves by NAME and dispatches by id. */
401
+ declare function listRunConfigs(options?: WorldsClientOptions): Promise<RunConfigSummary[]>;
402
+ type RunReport = {
403
+ id: string;
404
+ status: string;
405
+ terminal: boolean;
406
+ score: number | null;
407
+ summary: string | null;
408
+ model: string | null;
409
+ versionLabel: string | null;
410
+ parallelism: number;
411
+ shardsCompleted: number | null;
412
+ /** The run's page on the platform — what a PR comment should link. */
413
+ url: string | null;
414
+ evaluations: {
415
+ id: string;
416
+ status: string;
417
+ avgReward: number | null;
418
+ url: string | null;
419
+ }[];
420
+ samples: {
421
+ task: string | null;
422
+ reward: number | null;
423
+ rewards: Record<string, unknown>;
424
+ sessionId: string | null;
425
+ }[];
426
+ };
427
+ /** The read half of dispatch: status, score, per-shard evaluations, and
428
+ * per-task rewards — everything a CI gate needs. */
429
+ declare function getRun(runId: string, options?: WorldsClientOptions): Promise<RunReport>;
430
+ /** Poll a run to terminal (default every 20s, up to `timeoutMinutes`). */
431
+ declare function waitForRun(runId: string, opts?: {
432
+ timeoutMinutes?: number;
433
+ intervalSeconds?: number;
434
+ onTick?: (run: RunReport) => void;
435
+ options?: WorldsClientOptions;
436
+ }): Promise<RunReport>;
437
+ type RunProvenance = {
438
+ branch?: string;
439
+ commit?: string;
440
+ repoUrl?: string;
441
+ actor?: {
442
+ id?: string;
443
+ name?: string;
444
+ imageUrl?: string;
445
+ kind?: string;
446
+ };
447
+ };
448
+ /**
449
+ * Where a dispatch comes from, DETECTED rather than configured: the CI
450
+ * checkout's branch/commit/actor under GitHub Actions, else the developer's
451
+ * own git checkout. A run nobody can trace to a branch is a score with no
452
+ * code attached, so every dispatch carries this unless overridden.
453
+ */
454
+ declare function detectProvenance(): RunProvenance;
455
+ /** One scenario of a world, as data. */
456
+ declare class WorldTask {
457
+ readonly world: WorldSet;
458
+ readonly name: string;
459
+ readonly variant: string | null;
460
+ constructor(world: WorldSet, name: string, variant: string | null);
461
+ }
462
+ /**
463
+ * THE world, from a TS process: pinned identity + its tasks + hosted runs.
464
+ *
465
+ * const world = await openWorld("google-drive-sim@main");
466
+ * console.log(world.version.semanticVersion, world.tasks.length);
467
+ * const runs = await world.dispatch(runConfigId);
468
+ */
469
+ declare class WorldSet {
470
+ readonly slugRef: string;
471
+ readonly slug: string;
472
+ readonly ref: string | null;
473
+ readonly version: ResolvedWorldVersion;
474
+ tasks: WorldTask[];
475
+ private readonly options;
476
+ constructor(slugRef: string, slug: string, ref: string | null, version: ResolvedWorldVersion, tasks: WorldTask[], options: WorldsClientOptions);
477
+ task(variant: string): WorldTask;
478
+ /**
479
+ * Open a LIVE session against one of this world's tasks: the platform boots
480
+ * (or reuses) a warm container, and your agent calls its tools through the
481
+ * platform. See worlds/session.ts.
482
+ */
483
+ open(task: string, opts?: {
484
+ reuseWarm?: boolean;
485
+ }): Promise<WorldSession>;
486
+ /**
487
+ * Start a HOSTED run of a run config against this world — the platform
488
+ * executes the (Python) world in parallel containers per the config's
489
+ * `parallelism` and files one graded run per model. Returns those runs.
490
+ */
491
+ dispatch(runConfigId: string, opts?: {
492
+ containerId?: string;
493
+ versionId?: string;
494
+ /** Override the auto-detected branch/commit/actor. */
495
+ provenance?: RunProvenance;
496
+ }): Promise<{
497
+ versionLabel: string;
498
+ runs: DispatchedRun[];
499
+ }>;
500
+ }
501
+ /**
502
+ * Resolve `slug` / `slug@branch|tag|semver|hash` to its pinned version and
503
+ * task list. The task list comes from the version's file manifest
504
+ * (`tasks/<name>/task.toml`), so it needs no bundle download.
505
+ */
506
+ declare function openWorld(slugRef: string, options?: WorldsClientOptions): Promise<WorldSet>;
507
+
508
+ export { DEFAULT_BRANCH, type DispatchedRun, type GradeResult, type LocalMeta, type ResolvedWorldVersion, type RunConfigSummary, type RunProvenance, type RunReport, type SessionRunReport, type SessionRunResult, type ValidateReport, type ValidatedRun, WorldSession, WorldSessionCallError, type WorldSessionState, WorldSessionTimeout, WorldSet, WorldTask, type WorldTaskInfo, type WorldToolSpec, type WorldToolkit, type WorldsClientOptions, apiRequest, attachSession, buildSourceTarball, collectArchiveRelPaths, computeContentHash, createRef, deleteEnvSecret, detectProvenance, getRun, isIncludedInContentHash, listEnvSecretKeys, listRunConfigs, localTreeState, openSession, openWorld, propose, pull, push, readLocalMeta, runSessions, schemaFromSignature, setEnvSecret, snapshotTreeHashes, validateWorld, waitForRun, writeLocalMeta };