@withgateway/sdk 0.1.13 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -4
- package/dist/cli.d.mts +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +1303 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.mjs +1290 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/index.d.mts +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +612 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +593 -3
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +27 -2
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +27 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +377 -0
- package/dist/worlds/index.d.ts +377 -0
- package/dist/worlds/index.js +584 -0
- package/dist/worlds/index.js.map +1 -0
- package/dist/worlds/index.mjs +578 -0
- package/dist/worlds/index.mjs.map +1 -0
- package/package.json +27 -2
package/dist/index.js
CHANGED
|
@@ -3,6 +3,9 @@ var __defProp = Object.defineProperty;
|
|
|
3
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __esm = (fn, res) => function __init() {
|
|
7
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
8
|
+
};
|
|
6
9
|
var __commonJS = (cb, mod) => function __require() {
|
|
7
10
|
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
8
11
|
};
|
|
@@ -20,12 +23,552 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
20
23
|
};
|
|
21
24
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
22
25
|
|
|
26
|
+
// src/worlds/session.ts
|
|
27
|
+
var session_exports = {};
|
|
28
|
+
__export(session_exports, {
|
|
29
|
+
WorldSession: () => WorldSession,
|
|
30
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
31
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
32
|
+
attachSession: () => attachSession,
|
|
33
|
+
openSession: () => openSession,
|
|
34
|
+
runSessions: () => runSessions,
|
|
35
|
+
schemaFromSignature: () => schemaFromSignature
|
|
36
|
+
});
|
|
37
|
+
function schemaFromSignature(spec) {
|
|
38
|
+
const inside = /\(([\s\S]*)\)\s*$/.exec(spec.signature.trim())?.[1] ?? "";
|
|
39
|
+
const properties = {};
|
|
40
|
+
const required = [];
|
|
41
|
+
const params = [];
|
|
42
|
+
let depth = 0;
|
|
43
|
+
let current = "";
|
|
44
|
+
for (const ch of inside) {
|
|
45
|
+
if ("([{".includes(ch)) depth++;
|
|
46
|
+
if (")]}".includes(ch)) depth--;
|
|
47
|
+
if (ch === "," && depth === 0) {
|
|
48
|
+
params.push(current);
|
|
49
|
+
current = "";
|
|
50
|
+
} else current += ch;
|
|
51
|
+
}
|
|
52
|
+
if (current.trim()) params.push(current);
|
|
53
|
+
for (const raw of params) {
|
|
54
|
+
const param = raw.trim();
|
|
55
|
+
if (!param || param.startsWith("*") || param === "self") continue;
|
|
56
|
+
const eq = param.indexOf("=");
|
|
57
|
+
const name = (eq === -1 ? param : param.slice(0, eq)).split(":")[0].trim();
|
|
58
|
+
if (!name) continue;
|
|
59
|
+
if (eq === -1) {
|
|
60
|
+
properties[name] = { type: "string" };
|
|
61
|
+
required.push(name);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const value = param.slice(eq + 1).trim();
|
|
65
|
+
if (/^-?\d+(\.\d+)?$/.test(value)) properties[name] = { type: "number" };
|
|
66
|
+
else if (value === "True" || value === "False")
|
|
67
|
+
properties[name] = { type: "boolean" };
|
|
68
|
+
else properties[name] = { type: "string" };
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
type: "function",
|
|
72
|
+
function: {
|
|
73
|
+
name: spec.name,
|
|
74
|
+
description: spec.doc ?? spec.name,
|
|
75
|
+
parameters: { type: "object", properties, required }
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
async function attachSession(sessionId, options = {}) {
|
|
80
|
+
const state2 = await apiRequest(
|
|
81
|
+
options,
|
|
82
|
+
"GET",
|
|
83
|
+
`/api/public/world-sessions/${encodeURIComponent(sessionId)}`
|
|
84
|
+
);
|
|
85
|
+
return new WorldSession(sessionId, state2, options);
|
|
86
|
+
}
|
|
87
|
+
async function openSession(slugOrRef, task, opts = {}) {
|
|
88
|
+
const options = opts.options ?? {};
|
|
89
|
+
const at = slugOrRef.indexOf("@");
|
|
90
|
+
const slug = at === -1 ? slugOrRef : slugOrRef.slice(0, at);
|
|
91
|
+
const state2 = await apiRequest(
|
|
92
|
+
options,
|
|
93
|
+
"POST",
|
|
94
|
+
"/api/public/world-sessions",
|
|
95
|
+
{
|
|
96
|
+
...opts.versionId ? { versionId: opts.versionId } : { slug },
|
|
97
|
+
task,
|
|
98
|
+
...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm }
|
|
99
|
+
}
|
|
100
|
+
);
|
|
101
|
+
return new WorldSession(state2.sessionId, state2, options);
|
|
102
|
+
}
|
|
103
|
+
async function runSessions(opts) {
|
|
104
|
+
const options = opts.options ?? {};
|
|
105
|
+
const concurrency = Math.max(1, opts.concurrency ?? 3);
|
|
106
|
+
let tasks = opts.tasks;
|
|
107
|
+
let versionId = opts.versionId;
|
|
108
|
+
if (!tasks || !versionId) {
|
|
109
|
+
const { openWorld: openWorld2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
|
|
110
|
+
const world = await openWorld2(opts.world, options);
|
|
111
|
+
tasks = tasks ?? world.tasks.map((t) => t.name);
|
|
112
|
+
versionId = versionId ?? world.version.versionId;
|
|
113
|
+
}
|
|
114
|
+
const results = [];
|
|
115
|
+
let next = 0;
|
|
116
|
+
await Promise.all(
|
|
117
|
+
Array.from({ length: Math.min(concurrency, tasks.length) }, async () => {
|
|
118
|
+
while (next < tasks.length) {
|
|
119
|
+
const task = tasks[next++];
|
|
120
|
+
let session = null;
|
|
121
|
+
try {
|
|
122
|
+
session = await openSession(opts.world, task, {
|
|
123
|
+
versionId,
|
|
124
|
+
options
|
|
125
|
+
});
|
|
126
|
+
await session.ready();
|
|
127
|
+
const toolkit = await session.toolkit();
|
|
128
|
+
await opts.agent({
|
|
129
|
+
session,
|
|
130
|
+
toolkit,
|
|
131
|
+
instruction: session.instruction ?? task,
|
|
132
|
+
task
|
|
133
|
+
});
|
|
134
|
+
const graded = await session.grade();
|
|
135
|
+
const result = {
|
|
136
|
+
task,
|
|
137
|
+
sessionId: session.sessionId,
|
|
138
|
+
reward: graded.reward,
|
|
139
|
+
rewards: graded.rewards
|
|
140
|
+
};
|
|
141
|
+
results.push(result);
|
|
142
|
+
opts.onResult?.(result);
|
|
143
|
+
} catch (error) {
|
|
144
|
+
const result = {
|
|
145
|
+
task,
|
|
146
|
+
sessionId: session?.sessionId ?? "",
|
|
147
|
+
reward: 0,
|
|
148
|
+
rewards: {},
|
|
149
|
+
// An Error with an empty message would read as "no error" to any
|
|
150
|
+
// caller checking truthiness; name the type instead.
|
|
151
|
+
error: (error instanceof Error ? error.message : String(error)) || (error instanceof Error ? error.name : "Unknown error")
|
|
152
|
+
};
|
|
153
|
+
results.push(result);
|
|
154
|
+
opts.onResult?.(result);
|
|
155
|
+
} finally {
|
|
156
|
+
await session?.close({ keepWarm: opts.keepWarm ?? true }).catch(() => void 0);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
})
|
|
160
|
+
);
|
|
161
|
+
const mean = results.reduce((sum, r) => sum + (r.reward ?? 0), 0) / (results.length || 1);
|
|
162
|
+
return {
|
|
163
|
+
results: results.sort((a, b) => a.task.localeCompare(b.task)),
|
|
164
|
+
mean,
|
|
165
|
+
passed: opts.failUnder === void 0 ? true : mean >= opts.failUnder
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
var READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, sleep2, WorldSessionTimeout, WorldSessionCallError, WorldSession;
|
|
169
|
+
var init_session = __esm({
|
|
170
|
+
"src/worlds/session.ts"() {
|
|
171
|
+
"use strict";
|
|
172
|
+
init_worlds();
|
|
173
|
+
READY_TIMEOUT_MS = 10 * 6e4;
|
|
174
|
+
READY_POLL_MS = 2e3;
|
|
175
|
+
CALL_TIMEOUT_MS = 3 * 6e4;
|
|
176
|
+
CALL_POLL_MS = 400;
|
|
177
|
+
sleep2 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
178
|
+
WorldSessionTimeout = class extends Error {
|
|
179
|
+
};
|
|
180
|
+
WorldSessionCallError = class extends Error {
|
|
181
|
+
};
|
|
182
|
+
WorldSession = class {
|
|
183
|
+
constructor(sessionId, state2, options) {
|
|
184
|
+
this.sessionId = sessionId;
|
|
185
|
+
this.state = state2;
|
|
186
|
+
this.options = options;
|
|
187
|
+
}
|
|
188
|
+
get task() {
|
|
189
|
+
return this.state.taskName;
|
|
190
|
+
}
|
|
191
|
+
/** What this task asks for — the bundle's own instruction.md. */
|
|
192
|
+
get instruction() {
|
|
193
|
+
return this.state.instruction ?? null;
|
|
194
|
+
}
|
|
195
|
+
get versionId() {
|
|
196
|
+
return this.state.versionId;
|
|
197
|
+
}
|
|
198
|
+
/** The session's latest platform state (one request, no waiting). */
|
|
199
|
+
async status() {
|
|
200
|
+
const fresh = await apiRequest(
|
|
201
|
+
this.options,
|
|
202
|
+
"GET",
|
|
203
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`
|
|
204
|
+
);
|
|
205
|
+
this.state = {
|
|
206
|
+
instruction: this.state.instruction,
|
|
207
|
+
tools: this.state.tools,
|
|
208
|
+
...fresh
|
|
209
|
+
};
|
|
210
|
+
return this.state;
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Wait until the container is warm — meaning the process inside it has
|
|
214
|
+
* asked the platform for work, not merely that a job was accepted.
|
|
215
|
+
*/
|
|
216
|
+
async ready(opts = {}) {
|
|
217
|
+
const deadline = Date.now() + (opts.timeoutMs ?? READY_TIMEOUT_MS);
|
|
218
|
+
for (; ; ) {
|
|
219
|
+
const state2 = await this.status();
|
|
220
|
+
if (state2.ready) return state2;
|
|
221
|
+
if (state2.state === "FAILED" || state2.state === "STOPPED")
|
|
222
|
+
throw new WorldSessionCallError(
|
|
223
|
+
`World session ${state2.state.toLowerCase()} before it was ready.`
|
|
224
|
+
);
|
|
225
|
+
if (Date.now() > deadline)
|
|
226
|
+
throw new WorldSessionTimeout(
|
|
227
|
+
`World session still ${state2.state} after ${Math.round((opts.timeoutMs ?? READY_TIMEOUT_MS) / 1e3)}s.`
|
|
228
|
+
);
|
|
229
|
+
await sleep2(READY_POLL_MS);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/** Enqueue one interaction and wait for the container's answer. */
|
|
233
|
+
async exchange(body, timeoutMs = CALL_TIMEOUT_MS) {
|
|
234
|
+
const { callId } = await apiRequest(
|
|
235
|
+
this.options,
|
|
236
|
+
"POST",
|
|
237
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls`,
|
|
238
|
+
body
|
|
239
|
+
);
|
|
240
|
+
const deadline = Date.now() + timeoutMs;
|
|
241
|
+
for (; ; ) {
|
|
242
|
+
const call = await apiRequest(
|
|
243
|
+
this.options,
|
|
244
|
+
"GET",
|
|
245
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls/${encodeURIComponent(callId)}`
|
|
246
|
+
);
|
|
247
|
+
if (call.done) {
|
|
248
|
+
if (call.error) throw new WorldSessionCallError(call.error);
|
|
249
|
+
return call.result;
|
|
250
|
+
}
|
|
251
|
+
if (Date.now() > deadline)
|
|
252
|
+
throw new WorldSessionTimeout(
|
|
253
|
+
`Call ${callId} still ${call.state} after ${Math.round(timeoutMs / 1e3)}s \u2014 the container may have died.`
|
|
254
|
+
);
|
|
255
|
+
await sleep2(CALL_POLL_MS);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Run one of the world's tools INSIDE the container. The arguments and the
|
|
260
|
+
* result are the world's own contract — this layer neither validates nor
|
|
261
|
+
* reshapes them, so a tool error reads exactly as it would in a hosted run.
|
|
262
|
+
*/
|
|
263
|
+
async call(tool, args = {}, opts = {}) {
|
|
264
|
+
return this.exchange({ kind: "call", tool, args }, opts.timeoutMs);
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* THE WORLD'S TOOLS, READY TO HAND TO A MODEL. Schemas come from the
|
|
268
|
+
* world's own signatures and implementations call into this session, so a
|
|
269
|
+
* caller never writes — and never has to re-sync — a tool map of their own:
|
|
270
|
+
*
|
|
271
|
+
* const { schemas, impls } = await session.toolkit();
|
|
272
|
+
* await runMyAgent(impls, schemas, session.instruction);
|
|
273
|
+
*
|
|
274
|
+
* A tool added to the world shows up here on the next run; a tool renamed
|
|
275
|
+
* in the world renames here too, instead of failing at call time against a
|
|
276
|
+
* hand-written map that still remembers the old name.
|
|
277
|
+
*/
|
|
278
|
+
async toolkit() {
|
|
279
|
+
let specs = this.state.tools;
|
|
280
|
+
if (!specs) specs = (await this.status()).tools ?? [];
|
|
281
|
+
const impls = {};
|
|
282
|
+
for (const spec of specs) {
|
|
283
|
+
impls[spec.name] = (args = {}) => this.call(spec.name, args);
|
|
284
|
+
}
|
|
285
|
+
return { schemas: specs.map(schemaFromSignature), impls, specs };
|
|
286
|
+
}
|
|
287
|
+
/** Run the task's grader against the container's END STATE. */
|
|
288
|
+
async grade(opts = {}) {
|
|
289
|
+
const raw = await this.exchange(
|
|
290
|
+
{ kind: "grade" },
|
|
291
|
+
opts.timeoutMs
|
|
292
|
+
);
|
|
293
|
+
const record = raw && typeof raw === "object" ? raw : {};
|
|
294
|
+
const rewards = {};
|
|
295
|
+
for (const [key, value] of Object.entries(record)) {
|
|
296
|
+
if (key !== "reward" && typeof value === "number") rewards[key] = value;
|
|
297
|
+
}
|
|
298
|
+
return {
|
|
299
|
+
reward: typeof record.reward === "number" ? record.reward : null,
|
|
300
|
+
rewards,
|
|
301
|
+
raw
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* End the session. `keepWarm` hands the container back to the pool instead
|
|
306
|
+
* of tearing it down, so the next session for the same task starts warm —
|
|
307
|
+
* the pooled instance still obeys the fleet's idle TTL and pinning.
|
|
308
|
+
*/
|
|
309
|
+
async close(opts = {}) {
|
|
310
|
+
return apiRequest(
|
|
311
|
+
this.options,
|
|
312
|
+
"DELETE",
|
|
313
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`,
|
|
314
|
+
{ keepWarm: opts.keepWarm ?? false }
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
// src/worlds/index.ts
|
|
322
|
+
var worlds_exports = {};
|
|
323
|
+
__export(worlds_exports, {
|
|
324
|
+
WorldSession: () => WorldSession,
|
|
325
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
326
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
327
|
+
WorldSet: () => WorldSet,
|
|
328
|
+
WorldTask: () => WorldTask,
|
|
329
|
+
apiRequest: () => apiRequest,
|
|
330
|
+
attachSession: () => attachSession,
|
|
331
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
332
|
+
detectProvenance: () => detectProvenance,
|
|
333
|
+
getRun: () => getRun,
|
|
334
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
335
|
+
listRunConfigs: () => listRunConfigs,
|
|
336
|
+
openSession: () => openSession,
|
|
337
|
+
openWorld: () => openWorld,
|
|
338
|
+
runSessions: () => runSessions,
|
|
339
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
340
|
+
setEnvSecret: () => setEnvSecret,
|
|
341
|
+
waitForRun: () => waitForRun
|
|
342
|
+
});
|
|
343
|
+
function resolveEnv(options) {
|
|
344
|
+
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
345
|
+
const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
346
|
+
const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
347
|
+
if (!publicKey || !secretKey) {
|
|
348
|
+
throw new Error(
|
|
349
|
+
"Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
353
|
+
}
|
|
354
|
+
async function apiRequest(options, method, path, body) {
|
|
355
|
+
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
356
|
+
const res = await fetch(`${host}${path}`, {
|
|
357
|
+
method,
|
|
358
|
+
headers: {
|
|
359
|
+
"content-type": "application/json",
|
|
360
|
+
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
361
|
+
},
|
|
362
|
+
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
363
|
+
});
|
|
364
|
+
const text = await res.text();
|
|
365
|
+
if (!res.ok) {
|
|
366
|
+
throw new Error(
|
|
367
|
+
`${method} ${path} -> ${res.status}: ${text.slice(0, 400)}`
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
return JSON.parse(text);
|
|
371
|
+
}
|
|
372
|
+
async function setEnvSecret(slug, key, value, options = {}) {
|
|
373
|
+
return apiRequest(
|
|
374
|
+
options,
|
|
375
|
+
"PUT",
|
|
376
|
+
"/api/public/benchmark-containers/secrets",
|
|
377
|
+
{
|
|
378
|
+
slug,
|
|
379
|
+
key,
|
|
380
|
+
value
|
|
381
|
+
}
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
async function listEnvSecretKeys(slug, options = {}) {
|
|
385
|
+
const res = await apiRequest(
|
|
386
|
+
options,
|
|
387
|
+
"GET",
|
|
388
|
+
`/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
|
|
389
|
+
);
|
|
390
|
+
return res.secrets;
|
|
391
|
+
}
|
|
392
|
+
async function deleteEnvSecret(slug, key, options = {}) {
|
|
393
|
+
await apiRequest(
|
|
394
|
+
options,
|
|
395
|
+
"DELETE",
|
|
396
|
+
"/api/public/benchmark-containers/secrets",
|
|
397
|
+
{
|
|
398
|
+
slug,
|
|
399
|
+
key
|
|
400
|
+
}
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
async function listRunConfigs(options = {}) {
|
|
404
|
+
const res = await apiRequest(
|
|
405
|
+
options,
|
|
406
|
+
"GET",
|
|
407
|
+
"/api/public/benchmark-containers/run-configs"
|
|
408
|
+
);
|
|
409
|
+
return res.runConfigs;
|
|
410
|
+
}
|
|
411
|
+
async function getRun(runId, options = {}) {
|
|
412
|
+
return apiRequest(
|
|
413
|
+
options,
|
|
414
|
+
"GET",
|
|
415
|
+
`/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
async function waitForRun(runId, opts = {}) {
|
|
419
|
+
const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
|
|
420
|
+
for (; ; ) {
|
|
421
|
+
const run = await getRun(runId, opts.options ?? {});
|
|
422
|
+
if (run.terminal) return run;
|
|
423
|
+
if (Date.now() > deadline)
|
|
424
|
+
throw new Error(
|
|
425
|
+
`Run ${runId} still ${run.status} after the wait timeout.`
|
|
426
|
+
);
|
|
427
|
+
opts.onTick?.(run);
|
|
428
|
+
await new Promise(
|
|
429
|
+
(resolve) => setTimeout(resolve, (opts.intervalSeconds ?? 20) * 1e3)
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
function detectProvenance() {
|
|
434
|
+
const env = globalThis.process?.env ?? {};
|
|
435
|
+
const out = {};
|
|
436
|
+
const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
437
|
+
if (ref) out.branch = ref;
|
|
438
|
+
if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
|
|
439
|
+
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
440
|
+
if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
|
|
441
|
+
if (env.GITHUB_ACTOR) {
|
|
442
|
+
out.actor = {
|
|
443
|
+
kind: "ci",
|
|
444
|
+
id: env.GITHUB_ACTOR,
|
|
445
|
+
name: env.GITHUB_ACTOR,
|
|
446
|
+
// GitHub serves every account's avatar at /<login>.png.
|
|
447
|
+
imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
if (!out.branch || !out.commit) {
|
|
451
|
+
try {
|
|
452
|
+
const { execFileSync } = require("child_process");
|
|
453
|
+
const git = (args) => execFileSync("git", args, {
|
|
454
|
+
encoding: "utf-8",
|
|
455
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
456
|
+
}).toString().trim();
|
|
457
|
+
out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
458
|
+
out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
|
|
459
|
+
if (!out.repoUrl) {
|
|
460
|
+
const remote = git(["config", "--get", "remote.origin.url"]);
|
|
461
|
+
if (remote) out.repoUrl = remote.replace(/\.git$/, "");
|
|
462
|
+
}
|
|
463
|
+
if (!out.actor) {
|
|
464
|
+
const name = git(["config", "--get", "user.name"]);
|
|
465
|
+
if (name) out.actor = { kind: "user", name };
|
|
466
|
+
}
|
|
467
|
+
} catch {
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
return out;
|
|
471
|
+
}
|
|
472
|
+
async function openWorld(slugRef, options = {}) {
|
|
473
|
+
const at = slugRef.indexOf("@");
|
|
474
|
+
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
475
|
+
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
476
|
+
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
477
|
+
slug,
|
|
478
|
+
...ref ? { ref } : {}
|
|
479
|
+
});
|
|
480
|
+
const world = new WorldSet(
|
|
481
|
+
slugRef,
|
|
482
|
+
slug,
|
|
483
|
+
ref,
|
|
484
|
+
{
|
|
485
|
+
versionId: version.versionId,
|
|
486
|
+
semanticVersion: version.semanticVersion ?? null,
|
|
487
|
+
contentHash: version.contentHash,
|
|
488
|
+
state: version.state
|
|
489
|
+
},
|
|
490
|
+
[],
|
|
491
|
+
options
|
|
492
|
+
);
|
|
493
|
+
for (const name of version.taskNames ?? []) {
|
|
494
|
+
const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
|
|
495
|
+
world.tasks.push(new WorldTask(world, name, variant));
|
|
496
|
+
}
|
|
497
|
+
return world;
|
|
498
|
+
}
|
|
499
|
+
var WorldTask, WorldSet;
|
|
500
|
+
var init_worlds = __esm({
|
|
501
|
+
"src/worlds/index.ts"() {
|
|
502
|
+
"use strict";
|
|
503
|
+
init_session();
|
|
504
|
+
WorldTask = class {
|
|
505
|
+
constructor(world, name, variant) {
|
|
506
|
+
this.world = world;
|
|
507
|
+
this.name = name;
|
|
508
|
+
this.variant = variant;
|
|
509
|
+
}
|
|
510
|
+
};
|
|
511
|
+
WorldSet = class {
|
|
512
|
+
constructor(slugRef, slug, ref, version, tasks, options) {
|
|
513
|
+
this.slugRef = slugRef;
|
|
514
|
+
this.slug = slug;
|
|
515
|
+
this.ref = ref;
|
|
516
|
+
this.version = version;
|
|
517
|
+
this.tasks = tasks;
|
|
518
|
+
this.options = options;
|
|
519
|
+
}
|
|
520
|
+
task(variant) {
|
|
521
|
+
const hit = this.tasks.find(
|
|
522
|
+
(t) => t.variant === variant || t.name.includes(variant)
|
|
523
|
+
);
|
|
524
|
+
if (!hit) {
|
|
525
|
+
const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
|
|
526
|
+
throw new Error(`no task matches ${variant}; have: ${known}`);
|
|
527
|
+
}
|
|
528
|
+
return hit;
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Open a LIVE session against one of this world's tasks: the platform boots
|
|
532
|
+
* (or reuses) a warm container, and your agent calls its tools through the
|
|
533
|
+
* platform. See worlds/session.ts.
|
|
534
|
+
*/
|
|
535
|
+
async open(task, opts = {}) {
|
|
536
|
+
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
537
|
+
return openSession2(this.slug, task, {
|
|
538
|
+
versionId: this.version.versionId,
|
|
539
|
+
reuseWarm: opts.reuseWarm,
|
|
540
|
+
options: this.options
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Start a HOSTED run of a run config against this world — the platform
|
|
545
|
+
* executes the (Python) world in parallel containers per the config's
|
|
546
|
+
* `parallelism` and files one graded run per model. Returns those runs.
|
|
547
|
+
*/
|
|
548
|
+
async dispatch(runConfigId, opts = {}) {
|
|
549
|
+
const provenance = opts.provenance ?? detectProvenance();
|
|
550
|
+
return apiRequest(
|
|
551
|
+
this.options,
|
|
552
|
+
"POST",
|
|
553
|
+
"/api/public/benchmark-containers/run-configs/dispatch",
|
|
554
|
+
{
|
|
555
|
+
runConfigId,
|
|
556
|
+
...opts.containerId ? { containerId: opts.containerId } : {},
|
|
557
|
+
versionId: opts.versionId ?? this.version.versionId,
|
|
558
|
+
...provenance
|
|
559
|
+
}
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
});
|
|
565
|
+
|
|
23
566
|
// package.json
|
|
24
567
|
var require_package = __commonJS({
|
|
25
568
|
"package.json"(exports2, module2) {
|
|
26
569
|
module2.exports = {
|
|
27
570
|
name: "@withgateway/sdk",
|
|
28
|
-
version: "0.1.
|
|
571
|
+
version: "0.1.14",
|
|
29
572
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
30
573
|
main: "./dist/index.js",
|
|
31
574
|
module: "./dist/index.mjs",
|
|
@@ -85,6 +628,11 @@ var require_package = __commonJS({
|
|
|
85
628
|
types: "./dist/scores/index.d.ts",
|
|
86
629
|
import: "./dist/scores/index.mjs",
|
|
87
630
|
require: "./dist/scores/index.js"
|
|
631
|
+
},
|
|
632
|
+
"./worlds": {
|
|
633
|
+
types: "./dist/worlds/index.d.ts",
|
|
634
|
+
import: "./dist/worlds/index.mjs",
|
|
635
|
+
require: "./dist/worlds/index.js"
|
|
88
636
|
}
|
|
89
637
|
},
|
|
90
638
|
files: [
|
|
@@ -98,6 +646,7 @@ var require_package = __commonJS({
|
|
|
98
646
|
build: "tsup",
|
|
99
647
|
dev: "tsup --watch",
|
|
100
648
|
typecheck: "tsc --noEmit",
|
|
649
|
+
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
101
650
|
test: "vitest",
|
|
102
651
|
"test:run": "vitest run",
|
|
103
652
|
lint: "eslint src --ext .ts",
|
|
@@ -158,10 +707,29 @@ var require_package = __commonJS({
|
|
|
158
707
|
"@opentelemetry/sdk-trace-node": "^2.6.0",
|
|
159
708
|
"@opentelemetry/semantic-conventions": "^1.40.0",
|
|
160
709
|
"@types/node": "^20.0.0",
|
|
710
|
+
openai: "^4.70.0",
|
|
161
711
|
tsup: "^8.0.0",
|
|
162
712
|
tsx: "^4.0.0",
|
|
163
713
|
typescript: "^5.0.0",
|
|
164
|
-
vitest: "^2.
|
|
714
|
+
vitest: "^3.2.6"
|
|
715
|
+
},
|
|
716
|
+
overrides: {
|
|
717
|
+
protobufjs: "^8.4.1",
|
|
718
|
+
"vite@6": "^6.4.3",
|
|
719
|
+
postcss: "^8.5.18",
|
|
720
|
+
"picomatch@4": "^4.0.4"
|
|
721
|
+
},
|
|
722
|
+
pnpm: {
|
|
723
|
+
overrides: {
|
|
724
|
+
protobufjs: "^8.4.1",
|
|
725
|
+
vitest: "^3.2.6",
|
|
726
|
+
"vite@6": "^6.4.3",
|
|
727
|
+
postcss: "^8.5.18",
|
|
728
|
+
"picomatch@4": "^4.0.4"
|
|
729
|
+
}
|
|
730
|
+
},
|
|
731
|
+
bin: {
|
|
732
|
+
gateway: "./dist/cli.js"
|
|
165
733
|
}
|
|
166
734
|
};
|
|
167
735
|
}
|
|
@@ -212,9 +780,16 @@ __export(src_exports, {
|
|
|
212
780
|
TraceDiff: () => TraceDiff,
|
|
213
781
|
UserProfileError: () => UserProfileError,
|
|
214
782
|
UsersClient: () => UsersClient,
|
|
783
|
+
WorldSession: () => WorldSession,
|
|
784
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
785
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
786
|
+
WorldSet: () => WorldSet,
|
|
787
|
+
WorldTask: () => WorldTask,
|
|
215
788
|
addExperimentAttributes: () => addExperimentAttributes,
|
|
789
|
+
apiRequest: () => apiRequest,
|
|
216
790
|
assembleManifest: () => assembleManifest,
|
|
217
791
|
assignVariant: () => assignVariant,
|
|
792
|
+
attachSession: () => attachSession,
|
|
218
793
|
basicAuthHeader: () => basicAuthHeader,
|
|
219
794
|
canonicalJson: () => canonicalJson,
|
|
220
795
|
clearExperimentContext: () => clearExperimentContext,
|
|
@@ -223,6 +798,8 @@ __export(src_exports, {
|
|
|
223
798
|
context: () => context,
|
|
224
799
|
currentTraceId: () => currentTraceId,
|
|
225
800
|
defineTool: () => defineTool,
|
|
801
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
802
|
+
detectProvenance: () => detectProvenance,
|
|
226
803
|
experimentContextStorage: () => experimentContextStorage,
|
|
227
804
|
experiments: () => singleton_exports,
|
|
228
805
|
gateway: () => gateway,
|
|
@@ -231,9 +808,14 @@ __export(src_exports, {
|
|
|
231
808
|
getRegisteredPrompts: () => getRegisteredPrompts,
|
|
232
809
|
getRegisteredSkills: () => getRegisteredSkills,
|
|
233
810
|
getRegisteredTools: () => getRegisteredTools,
|
|
811
|
+
getRun: () => getRun,
|
|
234
812
|
identify: () => identify,
|
|
813
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
814
|
+
listRunConfigs: () => listRunConfigs,
|
|
235
815
|
loadSession: () => loadSession,
|
|
236
816
|
memory: () => memory,
|
|
817
|
+
openSession: () => openSession,
|
|
818
|
+
openWorld: () => openWorld,
|
|
237
819
|
personas: () => personas,
|
|
238
820
|
registerArtifact: () => registerArtifact,
|
|
239
821
|
registerPrompt: () => registerPrompt,
|
|
@@ -242,16 +824,20 @@ __export(src_exports, {
|
|
|
242
824
|
replay: () => replay,
|
|
243
825
|
resetBuildRegistry: () => resetBuildRegistry,
|
|
244
826
|
resolveUserId: () => resolveUserId,
|
|
827
|
+
runSessions: () => runSessions,
|
|
245
828
|
runSuite: () => runSuite,
|
|
246
829
|
runWithExperimentContext: () => runWithExperimentContext,
|
|
830
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
247
831
|
scores: () => scores,
|
|
248
832
|
sessionObservationFromApi: () => sessionObservationFromApi,
|
|
249
833
|
sessions: () => sessions,
|
|
834
|
+
setEnvSecret: () => setEnvSecret,
|
|
250
835
|
setExperimentContext: () => setExperimentContext,
|
|
251
836
|
success: () => success,
|
|
252
837
|
suite: () => suite,
|
|
253
838
|
tracing: () => tracing,
|
|
254
|
-
users: () => users
|
|
839
|
+
users: () => users,
|
|
840
|
+
waitForRun: () => waitForRun
|
|
255
841
|
});
|
|
256
842
|
module.exports = __toCommonJS(src_exports);
|
|
257
843
|
|
|
@@ -3053,6 +3639,10 @@ var scores = {
|
|
|
3053
3639
|
success
|
|
3054
3640
|
};
|
|
3055
3641
|
|
|
3642
|
+
// src/index.ts
|
|
3643
|
+
init_worlds();
|
|
3644
|
+
init_session();
|
|
3645
|
+
|
|
3056
3646
|
// src/build/index.ts
|
|
3057
3647
|
var import_node_crypto = require("crypto");
|
|
3058
3648
|
function canonicalJson(value) {
|
|
@@ -3976,9 +4566,16 @@ var tracing = {
|
|
|
3976
4566
|
TraceDiff,
|
|
3977
4567
|
UserProfileError,
|
|
3978
4568
|
UsersClient,
|
|
4569
|
+
WorldSession,
|
|
4570
|
+
WorldSessionCallError,
|
|
4571
|
+
WorldSessionTimeout,
|
|
4572
|
+
WorldSet,
|
|
4573
|
+
WorldTask,
|
|
3979
4574
|
addExperimentAttributes,
|
|
4575
|
+
apiRequest,
|
|
3980
4576
|
assembleManifest,
|
|
3981
4577
|
assignVariant,
|
|
4578
|
+
attachSession,
|
|
3982
4579
|
basicAuthHeader,
|
|
3983
4580
|
canonicalJson,
|
|
3984
4581
|
clearExperimentContext,
|
|
@@ -3987,6 +4584,8 @@ var tracing = {
|
|
|
3987
4584
|
context,
|
|
3988
4585
|
currentTraceId,
|
|
3989
4586
|
defineTool,
|
|
4587
|
+
deleteEnvSecret,
|
|
4588
|
+
detectProvenance,
|
|
3990
4589
|
experimentContextStorage,
|
|
3991
4590
|
experiments,
|
|
3992
4591
|
gateway,
|
|
@@ -3995,9 +4594,14 @@ var tracing = {
|
|
|
3995
4594
|
getRegisteredPrompts,
|
|
3996
4595
|
getRegisteredSkills,
|
|
3997
4596
|
getRegisteredTools,
|
|
4597
|
+
getRun,
|
|
3998
4598
|
identify,
|
|
4599
|
+
listEnvSecretKeys,
|
|
4600
|
+
listRunConfigs,
|
|
3999
4601
|
loadSession,
|
|
4000
4602
|
memory,
|
|
4603
|
+
openSession,
|
|
4604
|
+
openWorld,
|
|
4001
4605
|
personas,
|
|
4002
4606
|
registerArtifact,
|
|
4003
4607
|
registerPrompt,
|
|
@@ -4006,15 +4610,19 @@ var tracing = {
|
|
|
4006
4610
|
replay,
|
|
4007
4611
|
resetBuildRegistry,
|
|
4008
4612
|
resolveUserId,
|
|
4613
|
+
runSessions,
|
|
4009
4614
|
runSuite,
|
|
4010
4615
|
runWithExperimentContext,
|
|
4616
|
+
schemaFromSignature,
|
|
4011
4617
|
scores,
|
|
4012
4618
|
sessionObservationFromApi,
|
|
4013
4619
|
sessions,
|
|
4620
|
+
setEnvSecret,
|
|
4014
4621
|
setExperimentContext,
|
|
4015
4622
|
success,
|
|
4016
4623
|
suite,
|
|
4017
4624
|
tracing,
|
|
4018
|
-
users
|
|
4625
|
+
users,
|
|
4626
|
+
waitForRun
|
|
4019
4627
|
});
|
|
4020
4628
|
//# sourceMappingURL=index.js.map
|