@withgateway/sdk 0.1.12 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -0
- package/dist/cli.d.mts +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +1303 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.mjs +1290 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/{index-CTc8iOrx.d.ts → index-B5HUQQDE.d.mts} +9 -0
- package/dist/{index-CTc8iOrx.d.mts → index-Dpo-t8iO.d.ts} +9 -0
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +878 -32
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +845 -25
- package/dist/index.mjs.map +1 -1
- package/dist/scores/index.d.mts +165 -0
- package/dist/scores/index.d.ts +165 -0
- package/dist/scores/index.js +252 -0
- package/dist/scores/index.js.map +1 -0
- package/dist/scores/index.mjs +223 -0
- package/dist/scores/index.mjs.map +1 -0
- package/dist/tracing/index.d.mts +2 -1
- package/dist/tracing/index.d.ts +2 -1
- package/dist/tracing/index.js +262 -15
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +256 -9
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +377 -0
- package/dist/worlds/index.d.ts +377 -0
- package/dist/worlds/index.js +584 -0
- package/dist/worlds/index.js.map +1 -0
- package/dist/worlds/index.mjs +578 -0
- package/dist/worlds/index.mjs.map +1 -0
- package/package.json +32 -2
package/dist/index.js
CHANGED
|
@@ -3,6 +3,9 @@ var __defProp = Object.defineProperty;
|
|
|
3
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __esm = (fn, res) => function __init() {
|
|
7
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
8
|
+
};
|
|
6
9
|
var __commonJS = (cb, mod) => function __require() {
|
|
7
10
|
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
8
11
|
};
|
|
@@ -20,12 +23,552 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
20
23
|
};
|
|
21
24
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
22
25
|
|
|
26
|
+
// src/worlds/session.ts
|
|
27
|
+
var session_exports = {};
|
|
28
|
+
__export(session_exports, {
|
|
29
|
+
WorldSession: () => WorldSession,
|
|
30
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
31
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
32
|
+
attachSession: () => attachSession,
|
|
33
|
+
openSession: () => openSession,
|
|
34
|
+
runSessions: () => runSessions,
|
|
35
|
+
schemaFromSignature: () => schemaFromSignature
|
|
36
|
+
});
|
|
37
|
+
function schemaFromSignature(spec) {
|
|
38
|
+
const inside = /\(([\s\S]*)\)\s*$/.exec(spec.signature.trim())?.[1] ?? "";
|
|
39
|
+
const properties = {};
|
|
40
|
+
const required = [];
|
|
41
|
+
const params = [];
|
|
42
|
+
let depth = 0;
|
|
43
|
+
let current = "";
|
|
44
|
+
for (const ch of inside) {
|
|
45
|
+
if ("([{".includes(ch)) depth++;
|
|
46
|
+
if (")]}".includes(ch)) depth--;
|
|
47
|
+
if (ch === "," && depth === 0) {
|
|
48
|
+
params.push(current);
|
|
49
|
+
current = "";
|
|
50
|
+
} else current += ch;
|
|
51
|
+
}
|
|
52
|
+
if (current.trim()) params.push(current);
|
|
53
|
+
for (const raw of params) {
|
|
54
|
+
const param = raw.trim();
|
|
55
|
+
if (!param || param.startsWith("*") || param === "self") continue;
|
|
56
|
+
const eq = param.indexOf("=");
|
|
57
|
+
const name = (eq === -1 ? param : param.slice(0, eq)).split(":")[0].trim();
|
|
58
|
+
if (!name) continue;
|
|
59
|
+
if (eq === -1) {
|
|
60
|
+
properties[name] = { type: "string" };
|
|
61
|
+
required.push(name);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const value = param.slice(eq + 1).trim();
|
|
65
|
+
if (/^-?\d+(\.\d+)?$/.test(value)) properties[name] = { type: "number" };
|
|
66
|
+
else if (value === "True" || value === "False")
|
|
67
|
+
properties[name] = { type: "boolean" };
|
|
68
|
+
else properties[name] = { type: "string" };
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
type: "function",
|
|
72
|
+
function: {
|
|
73
|
+
name: spec.name,
|
|
74
|
+
description: spec.doc ?? spec.name,
|
|
75
|
+
parameters: { type: "object", properties, required }
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
async function attachSession(sessionId, options = {}) {
|
|
80
|
+
const state2 = await apiRequest(
|
|
81
|
+
options,
|
|
82
|
+
"GET",
|
|
83
|
+
`/api/public/world-sessions/${encodeURIComponent(sessionId)}`
|
|
84
|
+
);
|
|
85
|
+
return new WorldSession(sessionId, state2, options);
|
|
86
|
+
}
|
|
87
|
+
async function openSession(slugOrRef, task, opts = {}) {
|
|
88
|
+
const options = opts.options ?? {};
|
|
89
|
+
const at = slugOrRef.indexOf("@");
|
|
90
|
+
const slug = at === -1 ? slugOrRef : slugOrRef.slice(0, at);
|
|
91
|
+
const state2 = await apiRequest(
|
|
92
|
+
options,
|
|
93
|
+
"POST",
|
|
94
|
+
"/api/public/world-sessions",
|
|
95
|
+
{
|
|
96
|
+
...opts.versionId ? { versionId: opts.versionId } : { slug },
|
|
97
|
+
task,
|
|
98
|
+
...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm }
|
|
99
|
+
}
|
|
100
|
+
);
|
|
101
|
+
return new WorldSession(state2.sessionId, state2, options);
|
|
102
|
+
}
|
|
103
|
+
async function runSessions(opts) {
|
|
104
|
+
const options = opts.options ?? {};
|
|
105
|
+
const concurrency = Math.max(1, opts.concurrency ?? 3);
|
|
106
|
+
let tasks = opts.tasks;
|
|
107
|
+
let versionId = opts.versionId;
|
|
108
|
+
if (!tasks || !versionId) {
|
|
109
|
+
const { openWorld: openWorld2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
|
|
110
|
+
const world = await openWorld2(opts.world, options);
|
|
111
|
+
tasks = tasks ?? world.tasks.map((t) => t.name);
|
|
112
|
+
versionId = versionId ?? world.version.versionId;
|
|
113
|
+
}
|
|
114
|
+
const results = [];
|
|
115
|
+
let next = 0;
|
|
116
|
+
await Promise.all(
|
|
117
|
+
Array.from({ length: Math.min(concurrency, tasks.length) }, async () => {
|
|
118
|
+
while (next < tasks.length) {
|
|
119
|
+
const task = tasks[next++];
|
|
120
|
+
let session = null;
|
|
121
|
+
try {
|
|
122
|
+
session = await openSession(opts.world, task, {
|
|
123
|
+
versionId,
|
|
124
|
+
options
|
|
125
|
+
});
|
|
126
|
+
await session.ready();
|
|
127
|
+
const toolkit = await session.toolkit();
|
|
128
|
+
await opts.agent({
|
|
129
|
+
session,
|
|
130
|
+
toolkit,
|
|
131
|
+
instruction: session.instruction ?? task,
|
|
132
|
+
task
|
|
133
|
+
});
|
|
134
|
+
const graded = await session.grade();
|
|
135
|
+
const result = {
|
|
136
|
+
task,
|
|
137
|
+
sessionId: session.sessionId,
|
|
138
|
+
reward: graded.reward,
|
|
139
|
+
rewards: graded.rewards
|
|
140
|
+
};
|
|
141
|
+
results.push(result);
|
|
142
|
+
opts.onResult?.(result);
|
|
143
|
+
} catch (error) {
|
|
144
|
+
const result = {
|
|
145
|
+
task,
|
|
146
|
+
sessionId: session?.sessionId ?? "",
|
|
147
|
+
reward: 0,
|
|
148
|
+
rewards: {},
|
|
149
|
+
// An Error with an empty message would read as "no error" to any
|
|
150
|
+
// caller checking truthiness; name the type instead.
|
|
151
|
+
error: (error instanceof Error ? error.message : String(error)) || (error instanceof Error ? error.name : "Unknown error")
|
|
152
|
+
};
|
|
153
|
+
results.push(result);
|
|
154
|
+
opts.onResult?.(result);
|
|
155
|
+
} finally {
|
|
156
|
+
await session?.close({ keepWarm: opts.keepWarm ?? true }).catch(() => void 0);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
})
|
|
160
|
+
);
|
|
161
|
+
const mean = results.reduce((sum, r) => sum + (r.reward ?? 0), 0) / (results.length || 1);
|
|
162
|
+
return {
|
|
163
|
+
results: results.sort((a, b) => a.task.localeCompare(b.task)),
|
|
164
|
+
mean,
|
|
165
|
+
passed: opts.failUnder === void 0 ? true : mean >= opts.failUnder
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
var READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, sleep2, WorldSessionTimeout, WorldSessionCallError, WorldSession;
|
|
169
|
+
var init_session = __esm({
|
|
170
|
+
"src/worlds/session.ts"() {
|
|
171
|
+
"use strict";
|
|
172
|
+
init_worlds();
|
|
173
|
+
READY_TIMEOUT_MS = 10 * 6e4;
|
|
174
|
+
READY_POLL_MS = 2e3;
|
|
175
|
+
CALL_TIMEOUT_MS = 3 * 6e4;
|
|
176
|
+
CALL_POLL_MS = 400;
|
|
177
|
+
sleep2 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
178
|
+
WorldSessionTimeout = class extends Error {
|
|
179
|
+
};
|
|
180
|
+
WorldSessionCallError = class extends Error {
|
|
181
|
+
};
|
|
182
|
+
WorldSession = class {
|
|
183
|
+
constructor(sessionId, state2, options) {
|
|
184
|
+
this.sessionId = sessionId;
|
|
185
|
+
this.state = state2;
|
|
186
|
+
this.options = options;
|
|
187
|
+
}
|
|
188
|
+
get task() {
|
|
189
|
+
return this.state.taskName;
|
|
190
|
+
}
|
|
191
|
+
/** What this task asks for — the bundle's own instruction.md. */
|
|
192
|
+
get instruction() {
|
|
193
|
+
return this.state.instruction ?? null;
|
|
194
|
+
}
|
|
195
|
+
get versionId() {
|
|
196
|
+
return this.state.versionId;
|
|
197
|
+
}
|
|
198
|
+
/** The session's latest platform state (one request, no waiting). */
|
|
199
|
+
async status() {
|
|
200
|
+
const fresh = await apiRequest(
|
|
201
|
+
this.options,
|
|
202
|
+
"GET",
|
|
203
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`
|
|
204
|
+
);
|
|
205
|
+
this.state = {
|
|
206
|
+
instruction: this.state.instruction,
|
|
207
|
+
tools: this.state.tools,
|
|
208
|
+
...fresh
|
|
209
|
+
};
|
|
210
|
+
return this.state;
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Wait until the container is warm — meaning the process inside it has
|
|
214
|
+
* asked the platform for work, not merely that a job was accepted.
|
|
215
|
+
*/
|
|
216
|
+
async ready(opts = {}) {
|
|
217
|
+
const deadline = Date.now() + (opts.timeoutMs ?? READY_TIMEOUT_MS);
|
|
218
|
+
for (; ; ) {
|
|
219
|
+
const state2 = await this.status();
|
|
220
|
+
if (state2.ready) return state2;
|
|
221
|
+
if (state2.state === "FAILED" || state2.state === "STOPPED")
|
|
222
|
+
throw new WorldSessionCallError(
|
|
223
|
+
`World session ${state2.state.toLowerCase()} before it was ready.`
|
|
224
|
+
);
|
|
225
|
+
if (Date.now() > deadline)
|
|
226
|
+
throw new WorldSessionTimeout(
|
|
227
|
+
`World session still ${state2.state} after ${Math.round((opts.timeoutMs ?? READY_TIMEOUT_MS) / 1e3)}s.`
|
|
228
|
+
);
|
|
229
|
+
await sleep2(READY_POLL_MS);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/** Enqueue one interaction and wait for the container's answer. */
|
|
233
|
+
async exchange(body, timeoutMs = CALL_TIMEOUT_MS) {
|
|
234
|
+
const { callId } = await apiRequest(
|
|
235
|
+
this.options,
|
|
236
|
+
"POST",
|
|
237
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls`,
|
|
238
|
+
body
|
|
239
|
+
);
|
|
240
|
+
const deadline = Date.now() + timeoutMs;
|
|
241
|
+
for (; ; ) {
|
|
242
|
+
const call = await apiRequest(
|
|
243
|
+
this.options,
|
|
244
|
+
"GET",
|
|
245
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls/${encodeURIComponent(callId)}`
|
|
246
|
+
);
|
|
247
|
+
if (call.done) {
|
|
248
|
+
if (call.error) throw new WorldSessionCallError(call.error);
|
|
249
|
+
return call.result;
|
|
250
|
+
}
|
|
251
|
+
if (Date.now() > deadline)
|
|
252
|
+
throw new WorldSessionTimeout(
|
|
253
|
+
`Call ${callId} still ${call.state} after ${Math.round(timeoutMs / 1e3)}s \u2014 the container may have died.`
|
|
254
|
+
);
|
|
255
|
+
await sleep2(CALL_POLL_MS);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Run one of the world's tools INSIDE the container. The arguments and the
|
|
260
|
+
* result are the world's own contract — this layer neither validates nor
|
|
261
|
+
* reshapes them, so a tool error reads exactly as it would in a hosted run.
|
|
262
|
+
*/
|
|
263
|
+
async call(tool, args = {}, opts = {}) {
|
|
264
|
+
return this.exchange({ kind: "call", tool, args }, opts.timeoutMs);
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* THE WORLD'S TOOLS, READY TO HAND TO A MODEL. Schemas come from the
|
|
268
|
+
* world's own signatures and implementations call into this session, so a
|
|
269
|
+
* caller never writes — and never has to re-sync — a tool map of their own:
|
|
270
|
+
*
|
|
271
|
+
* const { schemas, impls } = await session.toolkit();
|
|
272
|
+
* await runMyAgent(impls, schemas, session.instruction);
|
|
273
|
+
*
|
|
274
|
+
* A tool added to the world shows up here on the next run; a tool renamed
|
|
275
|
+
* in the world renames here too, instead of failing at call time against a
|
|
276
|
+
* hand-written map that still remembers the old name.
|
|
277
|
+
*/
|
|
278
|
+
async toolkit() {
|
|
279
|
+
let specs = this.state.tools;
|
|
280
|
+
if (!specs) specs = (await this.status()).tools ?? [];
|
|
281
|
+
const impls = {};
|
|
282
|
+
for (const spec of specs) {
|
|
283
|
+
impls[spec.name] = (args = {}) => this.call(spec.name, args);
|
|
284
|
+
}
|
|
285
|
+
return { schemas: specs.map(schemaFromSignature), impls, specs };
|
|
286
|
+
}
|
|
287
|
+
/** Run the task's grader against the container's END STATE. */
|
|
288
|
+
async grade(opts = {}) {
|
|
289
|
+
const raw = await this.exchange(
|
|
290
|
+
{ kind: "grade" },
|
|
291
|
+
opts.timeoutMs
|
|
292
|
+
);
|
|
293
|
+
const record = raw && typeof raw === "object" ? raw : {};
|
|
294
|
+
const rewards = {};
|
|
295
|
+
for (const [key, value] of Object.entries(record)) {
|
|
296
|
+
if (key !== "reward" && typeof value === "number") rewards[key] = value;
|
|
297
|
+
}
|
|
298
|
+
return {
|
|
299
|
+
reward: typeof record.reward === "number" ? record.reward : null,
|
|
300
|
+
rewards,
|
|
301
|
+
raw
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* End the session. `keepWarm` hands the container back to the pool instead
|
|
306
|
+
* of tearing it down, so the next session for the same task starts warm —
|
|
307
|
+
* the pooled instance still obeys the fleet's idle TTL and pinning.
|
|
308
|
+
*/
|
|
309
|
+
async close(opts = {}) {
|
|
310
|
+
return apiRequest(
|
|
311
|
+
this.options,
|
|
312
|
+
"DELETE",
|
|
313
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`,
|
|
314
|
+
{ keepWarm: opts.keepWarm ?? false }
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
// src/worlds/index.ts
|
|
322
|
+
var worlds_exports = {};
|
|
323
|
+
__export(worlds_exports, {
|
|
324
|
+
WorldSession: () => WorldSession,
|
|
325
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
326
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
327
|
+
WorldSet: () => WorldSet,
|
|
328
|
+
WorldTask: () => WorldTask,
|
|
329
|
+
apiRequest: () => apiRequest,
|
|
330
|
+
attachSession: () => attachSession,
|
|
331
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
332
|
+
detectProvenance: () => detectProvenance,
|
|
333
|
+
getRun: () => getRun,
|
|
334
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
335
|
+
listRunConfigs: () => listRunConfigs,
|
|
336
|
+
openSession: () => openSession,
|
|
337
|
+
openWorld: () => openWorld,
|
|
338
|
+
runSessions: () => runSessions,
|
|
339
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
340
|
+
setEnvSecret: () => setEnvSecret,
|
|
341
|
+
waitForRun: () => waitForRun
|
|
342
|
+
});
|
|
343
|
+
function resolveEnv(options) {
|
|
344
|
+
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
345
|
+
const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
346
|
+
const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
347
|
+
if (!publicKey || !secretKey) {
|
|
348
|
+
throw new Error(
|
|
349
|
+
"Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
353
|
+
}
|
|
354
|
+
async function apiRequest(options, method, path, body) {
|
|
355
|
+
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
356
|
+
const res = await fetch(`${host}${path}`, {
|
|
357
|
+
method,
|
|
358
|
+
headers: {
|
|
359
|
+
"content-type": "application/json",
|
|
360
|
+
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
361
|
+
},
|
|
362
|
+
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
363
|
+
});
|
|
364
|
+
const text = await res.text();
|
|
365
|
+
if (!res.ok) {
|
|
366
|
+
throw new Error(
|
|
367
|
+
`${method} ${path} -> ${res.status}: ${text.slice(0, 400)}`
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
return JSON.parse(text);
|
|
371
|
+
}
|
|
372
|
+
async function setEnvSecret(slug, key, value, options = {}) {
|
|
373
|
+
return apiRequest(
|
|
374
|
+
options,
|
|
375
|
+
"PUT",
|
|
376
|
+
"/api/public/benchmark-containers/secrets",
|
|
377
|
+
{
|
|
378
|
+
slug,
|
|
379
|
+
key,
|
|
380
|
+
value
|
|
381
|
+
}
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
async function listEnvSecretKeys(slug, options = {}) {
|
|
385
|
+
const res = await apiRequest(
|
|
386
|
+
options,
|
|
387
|
+
"GET",
|
|
388
|
+
`/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
|
|
389
|
+
);
|
|
390
|
+
return res.secrets;
|
|
391
|
+
}
|
|
392
|
+
async function deleteEnvSecret(slug, key, options = {}) {
|
|
393
|
+
await apiRequest(
|
|
394
|
+
options,
|
|
395
|
+
"DELETE",
|
|
396
|
+
"/api/public/benchmark-containers/secrets",
|
|
397
|
+
{
|
|
398
|
+
slug,
|
|
399
|
+
key
|
|
400
|
+
}
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
async function listRunConfigs(options = {}) {
|
|
404
|
+
const res = await apiRequest(
|
|
405
|
+
options,
|
|
406
|
+
"GET",
|
|
407
|
+
"/api/public/benchmark-containers/run-configs"
|
|
408
|
+
);
|
|
409
|
+
return res.runConfigs;
|
|
410
|
+
}
|
|
411
|
+
async function getRun(runId, options = {}) {
|
|
412
|
+
return apiRequest(
|
|
413
|
+
options,
|
|
414
|
+
"GET",
|
|
415
|
+
`/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
async function waitForRun(runId, opts = {}) {
|
|
419
|
+
const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
|
|
420
|
+
for (; ; ) {
|
|
421
|
+
const run = await getRun(runId, opts.options ?? {});
|
|
422
|
+
if (run.terminal) return run;
|
|
423
|
+
if (Date.now() > deadline)
|
|
424
|
+
throw new Error(
|
|
425
|
+
`Run ${runId} still ${run.status} after the wait timeout.`
|
|
426
|
+
);
|
|
427
|
+
opts.onTick?.(run);
|
|
428
|
+
await new Promise(
|
|
429
|
+
(resolve) => setTimeout(resolve, (opts.intervalSeconds ?? 20) * 1e3)
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
function detectProvenance() {
|
|
434
|
+
const env = globalThis.process?.env ?? {};
|
|
435
|
+
const out = {};
|
|
436
|
+
const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
437
|
+
if (ref) out.branch = ref;
|
|
438
|
+
if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
|
|
439
|
+
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
440
|
+
if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
|
|
441
|
+
if (env.GITHUB_ACTOR) {
|
|
442
|
+
out.actor = {
|
|
443
|
+
kind: "ci",
|
|
444
|
+
id: env.GITHUB_ACTOR,
|
|
445
|
+
name: env.GITHUB_ACTOR,
|
|
446
|
+
// GitHub serves every account's avatar at /<login>.png.
|
|
447
|
+
imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
if (!out.branch || !out.commit) {
|
|
451
|
+
try {
|
|
452
|
+
const { execFileSync } = require("child_process");
|
|
453
|
+
const git = (args) => execFileSync("git", args, {
|
|
454
|
+
encoding: "utf-8",
|
|
455
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
456
|
+
}).toString().trim();
|
|
457
|
+
out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
458
|
+
out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
|
|
459
|
+
if (!out.repoUrl) {
|
|
460
|
+
const remote = git(["config", "--get", "remote.origin.url"]);
|
|
461
|
+
if (remote) out.repoUrl = remote.replace(/\.git$/, "");
|
|
462
|
+
}
|
|
463
|
+
if (!out.actor) {
|
|
464
|
+
const name = git(["config", "--get", "user.name"]);
|
|
465
|
+
if (name) out.actor = { kind: "user", name };
|
|
466
|
+
}
|
|
467
|
+
} catch {
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
return out;
|
|
471
|
+
}
|
|
472
|
+
async function openWorld(slugRef, options = {}) {
|
|
473
|
+
const at = slugRef.indexOf("@");
|
|
474
|
+
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
475
|
+
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
476
|
+
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
477
|
+
slug,
|
|
478
|
+
...ref ? { ref } : {}
|
|
479
|
+
});
|
|
480
|
+
const world = new WorldSet(
|
|
481
|
+
slugRef,
|
|
482
|
+
slug,
|
|
483
|
+
ref,
|
|
484
|
+
{
|
|
485
|
+
versionId: version.versionId,
|
|
486
|
+
semanticVersion: version.semanticVersion ?? null,
|
|
487
|
+
contentHash: version.contentHash,
|
|
488
|
+
state: version.state
|
|
489
|
+
},
|
|
490
|
+
[],
|
|
491
|
+
options
|
|
492
|
+
);
|
|
493
|
+
for (const name of version.taskNames ?? []) {
|
|
494
|
+
const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
|
|
495
|
+
world.tasks.push(new WorldTask(world, name, variant));
|
|
496
|
+
}
|
|
497
|
+
return world;
|
|
498
|
+
}
|
|
499
|
+
var WorldTask, WorldSet;
|
|
500
|
+
var init_worlds = __esm({
|
|
501
|
+
"src/worlds/index.ts"() {
|
|
502
|
+
"use strict";
|
|
503
|
+
init_session();
|
|
504
|
+
WorldTask = class {
|
|
505
|
+
constructor(world, name, variant) {
|
|
506
|
+
this.world = world;
|
|
507
|
+
this.name = name;
|
|
508
|
+
this.variant = variant;
|
|
509
|
+
}
|
|
510
|
+
};
|
|
511
|
+
WorldSet = class {
|
|
512
|
+
constructor(slugRef, slug, ref, version, tasks, options) {
|
|
513
|
+
this.slugRef = slugRef;
|
|
514
|
+
this.slug = slug;
|
|
515
|
+
this.ref = ref;
|
|
516
|
+
this.version = version;
|
|
517
|
+
this.tasks = tasks;
|
|
518
|
+
this.options = options;
|
|
519
|
+
}
|
|
520
|
+
task(variant) {
|
|
521
|
+
const hit = this.tasks.find(
|
|
522
|
+
(t) => t.variant === variant || t.name.includes(variant)
|
|
523
|
+
);
|
|
524
|
+
if (!hit) {
|
|
525
|
+
const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
|
|
526
|
+
throw new Error(`no task matches ${variant}; have: ${known}`);
|
|
527
|
+
}
|
|
528
|
+
return hit;
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Open a LIVE session against one of this world's tasks: the platform boots
|
|
532
|
+
* (or reuses) a warm container, and your agent calls its tools through the
|
|
533
|
+
* platform. See worlds/session.ts.
|
|
534
|
+
*/
|
|
535
|
+
async open(task, opts = {}) {
|
|
536
|
+
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
537
|
+
return openSession2(this.slug, task, {
|
|
538
|
+
versionId: this.version.versionId,
|
|
539
|
+
reuseWarm: opts.reuseWarm,
|
|
540
|
+
options: this.options
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Start a HOSTED run of a run config against this world — the platform
|
|
545
|
+
* executes the (Python) world in parallel containers per the config's
|
|
546
|
+
* `parallelism` and files one graded run per model. Returns those runs.
|
|
547
|
+
*/
|
|
548
|
+
async dispatch(runConfigId, opts = {}) {
|
|
549
|
+
const provenance = opts.provenance ?? detectProvenance();
|
|
550
|
+
return apiRequest(
|
|
551
|
+
this.options,
|
|
552
|
+
"POST",
|
|
553
|
+
"/api/public/benchmark-containers/run-configs/dispatch",
|
|
554
|
+
{
|
|
555
|
+
runConfigId,
|
|
556
|
+
...opts.containerId ? { containerId: opts.containerId } : {},
|
|
557
|
+
versionId: opts.versionId ?? this.version.versionId,
|
|
558
|
+
...provenance
|
|
559
|
+
}
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
});
|
|
565
|
+
|
|
23
566
|
// package.json
|
|
24
567
|
var require_package = __commonJS({
|
|
25
568
|
"package.json"(exports2, module2) {
|
|
26
569
|
module2.exports = {
|
|
27
570
|
name: "@withgateway/sdk",
|
|
28
|
-
version: "0.1.
|
|
571
|
+
version: "0.1.14",
|
|
29
572
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
30
573
|
main: "./dist/index.js",
|
|
31
574
|
module: "./dist/index.mjs",
|
|
@@ -80,6 +623,16 @@ var require_package = __commonJS({
|
|
|
80
623
|
types: "./dist/users/index.d.ts",
|
|
81
624
|
import: "./dist/users/index.mjs",
|
|
82
625
|
require: "./dist/users/index.js"
|
|
626
|
+
},
|
|
627
|
+
"./scores": {
|
|
628
|
+
types: "./dist/scores/index.d.ts",
|
|
629
|
+
import: "./dist/scores/index.mjs",
|
|
630
|
+
require: "./dist/scores/index.js"
|
|
631
|
+
},
|
|
632
|
+
"./worlds": {
|
|
633
|
+
types: "./dist/worlds/index.d.ts",
|
|
634
|
+
import: "./dist/worlds/index.mjs",
|
|
635
|
+
require: "./dist/worlds/index.js"
|
|
83
636
|
}
|
|
84
637
|
},
|
|
85
638
|
files: [
|
|
@@ -93,6 +646,7 @@ var require_package = __commonJS({
|
|
|
93
646
|
build: "tsup",
|
|
94
647
|
dev: "tsup --watch",
|
|
95
648
|
typecheck: "tsc --noEmit",
|
|
649
|
+
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
96
650
|
test: "vitest",
|
|
97
651
|
"test:run": "vitest run",
|
|
98
652
|
lint: "eslint src --ext .ts",
|
|
@@ -153,10 +707,29 @@ var require_package = __commonJS({
|
|
|
153
707
|
"@opentelemetry/sdk-trace-node": "^2.6.0",
|
|
154
708
|
"@opentelemetry/semantic-conventions": "^1.40.0",
|
|
155
709
|
"@types/node": "^20.0.0",
|
|
710
|
+
openai: "^4.70.0",
|
|
156
711
|
tsup: "^8.0.0",
|
|
157
712
|
tsx: "^4.0.0",
|
|
158
713
|
typescript: "^5.0.0",
|
|
159
|
-
vitest: "^2.
|
|
714
|
+
vitest: "^3.2.6"
|
|
715
|
+
},
|
|
716
|
+
overrides: {
|
|
717
|
+
protobufjs: "^8.4.1",
|
|
718
|
+
"vite@6": "^6.4.3",
|
|
719
|
+
postcss: "^8.5.18",
|
|
720
|
+
"picomatch@4": "^4.0.4"
|
|
721
|
+
},
|
|
722
|
+
pnpm: {
|
|
723
|
+
overrides: {
|
|
724
|
+
protobufjs: "^8.4.1",
|
|
725
|
+
vitest: "^3.2.6",
|
|
726
|
+
"vite@6": "^6.4.3",
|
|
727
|
+
postcss: "^8.5.18",
|
|
728
|
+
"picomatch@4": "^4.0.4"
|
|
729
|
+
}
|
|
730
|
+
},
|
|
731
|
+
bin: {
|
|
732
|
+
gateway: "./dist/cli.js"
|
|
160
733
|
}
|
|
161
734
|
};
|
|
162
735
|
}
|
|
@@ -165,6 +738,7 @@ var require_package = __commonJS({
|
|
|
165
738
|
// src/index.ts
|
|
166
739
|
var src_exports = {};
|
|
167
740
|
__export(src_exports, {
|
|
741
|
+
AGENT_ID_KEY: () => AGENT_ID_KEY,
|
|
168
742
|
DEFAULT_TIMEOUT_SECONDS: () => DEFAULT_TIMEOUT_SECONDS,
|
|
169
743
|
DriftError: () => DriftError,
|
|
170
744
|
ENV_GATEWAY_HOST: () => ENV_GATEWAY_HOST,
|
|
@@ -194,6 +768,10 @@ __export(src_exports, {
|
|
|
194
768
|
ReplayReport: () => ReplayReport,
|
|
195
769
|
ReplaySuite: () => ReplaySuite,
|
|
196
770
|
ReplayTask: () => ReplayTask,
|
|
771
|
+
SUCCESS_METRIC_NAME_KEY: () => SUCCESS_METRIC_NAME_KEY,
|
|
772
|
+
SUCCESS_SIGNAL_KEY: () => SUCCESS_SIGNAL_KEY,
|
|
773
|
+
ScoreError: () => ScoreError,
|
|
774
|
+
ScoresClient: () => ScoresClient,
|
|
197
775
|
SessionLoadError: () => SessionLoadError,
|
|
198
776
|
SessionRecording: () => SessionRecording,
|
|
199
777
|
SessionTrace: () => SessionTrace,
|
|
@@ -202,16 +780,26 @@ __export(src_exports, {
|
|
|
202
780
|
TraceDiff: () => TraceDiff,
|
|
203
781
|
UserProfileError: () => UserProfileError,
|
|
204
782
|
UsersClient: () => UsersClient,
|
|
783
|
+
WorldSession: () => WorldSession,
|
|
784
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
785
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
786
|
+
WorldSet: () => WorldSet,
|
|
787
|
+
WorldTask: () => WorldTask,
|
|
205
788
|
addExperimentAttributes: () => addExperimentAttributes,
|
|
789
|
+
apiRequest: () => apiRequest,
|
|
206
790
|
assembleManifest: () => assembleManifest,
|
|
207
791
|
assignVariant: () => assignVariant,
|
|
792
|
+
attachSession: () => attachSession,
|
|
208
793
|
basicAuthHeader: () => basicAuthHeader,
|
|
209
794
|
canonicalJson: () => canonicalJson,
|
|
210
795
|
clearExperimentContext: () => clearExperimentContext,
|
|
211
796
|
computeBuildHash: () => computeBuildHash,
|
|
212
797
|
computeHash: () => computeHash,
|
|
213
798
|
context: () => context,
|
|
799
|
+
currentTraceId: () => currentTraceId,
|
|
214
800
|
defineTool: () => defineTool,
|
|
801
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
802
|
+
detectProvenance: () => detectProvenance,
|
|
215
803
|
experimentContextStorage: () => experimentContextStorage,
|
|
216
804
|
experiments: () => singleton_exports,
|
|
217
805
|
gateway: () => gateway,
|
|
@@ -220,9 +808,14 @@ __export(src_exports, {
|
|
|
220
808
|
getRegisteredPrompts: () => getRegisteredPrompts,
|
|
221
809
|
getRegisteredSkills: () => getRegisteredSkills,
|
|
222
810
|
getRegisteredTools: () => getRegisteredTools,
|
|
811
|
+
getRun: () => getRun,
|
|
223
812
|
identify: () => identify,
|
|
813
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
814
|
+
listRunConfigs: () => listRunConfigs,
|
|
224
815
|
loadSession: () => loadSession,
|
|
225
816
|
memory: () => memory,
|
|
817
|
+
openSession: () => openSession,
|
|
818
|
+
openWorld: () => openWorld,
|
|
226
819
|
personas: () => personas,
|
|
227
820
|
registerArtifact: () => registerArtifact,
|
|
228
821
|
registerPrompt: () => registerPrompt,
|
|
@@ -231,14 +824,20 @@ __export(src_exports, {
|
|
|
231
824
|
replay: () => replay,
|
|
232
825
|
resetBuildRegistry: () => resetBuildRegistry,
|
|
233
826
|
resolveUserId: () => resolveUserId,
|
|
827
|
+
runSessions: () => runSessions,
|
|
234
828
|
runSuite: () => runSuite,
|
|
235
829
|
runWithExperimentContext: () => runWithExperimentContext,
|
|
830
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
831
|
+
scores: () => scores,
|
|
236
832
|
sessionObservationFromApi: () => sessionObservationFromApi,
|
|
237
833
|
sessions: () => sessions,
|
|
834
|
+
setEnvSecret: () => setEnvSecret,
|
|
238
835
|
setExperimentContext: () => setExperimentContext,
|
|
836
|
+
success: () => success,
|
|
239
837
|
suite: () => suite,
|
|
240
838
|
tracing: () => tracing,
|
|
241
|
-
users: () => users
|
|
839
|
+
users: () => users,
|
|
840
|
+
waitForRun: () => waitForRun
|
|
242
841
|
});
|
|
243
842
|
module.exports = __toCommonJS(src_exports);
|
|
244
843
|
|
|
@@ -1472,7 +2071,7 @@ var SessionRecording = class _SessionRecording {
|
|
|
1472
2071
|
* to re-drive the recorded conversation.
|
|
1473
2072
|
*/
|
|
1474
2073
|
userMessages() {
|
|
1475
|
-
return this.turns().map((
|
|
2074
|
+
return this.turns().map((trace5) => trace5.userMessage());
|
|
1476
2075
|
}
|
|
1477
2076
|
/** Output of the last chronological turn (null when empty). */
|
|
1478
2077
|
finalOutput() {
|
|
@@ -1490,8 +2089,8 @@ var SessionRecording = class _SessionRecording {
|
|
|
1490
2089
|
/** Filter observations across all turns (chronological order). */
|
|
1491
2090
|
select(filter = {}) {
|
|
1492
2091
|
const selected = [];
|
|
1493
|
-
for (const
|
|
1494
|
-
for (const observation of
|
|
2092
|
+
for (const trace5 of this.turns()) {
|
|
2093
|
+
for (const observation of trace5.observations) {
|
|
1495
2094
|
if (filter.type && (observation.type ?? "").toUpperCase() !== filter.type.toUpperCase()) {
|
|
1496
2095
|
continue;
|
|
1497
2096
|
}
|
|
@@ -1511,15 +2110,15 @@ var SessionRecording = class _SessionRecording {
|
|
|
1511
2110
|
sessionId: this.sessionId,
|
|
1512
2111
|
projectId: this.projectId,
|
|
1513
2112
|
environment: this.environment,
|
|
1514
|
-
traces: this.turns().map((
|
|
1515
|
-
id:
|
|
1516
|
-
name:
|
|
1517
|
-
timestamp:
|
|
1518
|
-
input:
|
|
1519
|
-
output:
|
|
1520
|
-
metadata:
|
|
1521
|
-
tags:
|
|
1522
|
-
observations:
|
|
2113
|
+
traces: this.turns().map((trace5) => ({
|
|
2114
|
+
id: trace5.id,
|
|
2115
|
+
name: trace5.name,
|
|
2116
|
+
timestamp: trace5.timestamp,
|
|
2117
|
+
input: trace5.input,
|
|
2118
|
+
output: trace5.output,
|
|
2119
|
+
metadata: trace5.metadata,
|
|
2120
|
+
tags: trace5.tags,
|
|
2121
|
+
observations: trace5.observations.map((o) => ({ ...o }))
|
|
1523
2122
|
}))
|
|
1524
2123
|
};
|
|
1525
2124
|
}
|
|
@@ -2830,6 +3429,220 @@ var users = {
|
|
|
2830
3429
|
fromEnv: UsersClient.fromEnv
|
|
2831
3430
|
};
|
|
2832
3431
|
|
|
3432
|
+
// src/scores/index.ts
|
|
3433
|
+
var import_api = require("@opentelemetry/api");
|
|
3434
|
+
var SUCCESS_SIGNAL_KEY = "gateway.success.signal";
|
|
3435
|
+
var SUCCESS_METRIC_NAME_KEY = "gateway.success.metricName";
|
|
3436
|
+
var AGENT_ID_KEY = "gateway.agent.id";
|
|
3437
|
+
var ScoreError = class extends Error {
|
|
3438
|
+
constructor(message, options) {
|
|
3439
|
+
super(message);
|
|
3440
|
+
this.name = "ScoreError";
|
|
3441
|
+
this.cause = options?.cause;
|
|
3442
|
+
}
|
|
3443
|
+
};
|
|
3444
|
+
function currentTraceId() {
|
|
3445
|
+
try {
|
|
3446
|
+
const span = import_api.trace.getActiveSpan();
|
|
3447
|
+
if (!span) return void 0;
|
|
3448
|
+
const ctx = span.spanContext();
|
|
3449
|
+
if (!ctx || !(0, import_api.isSpanContextValid)(ctx)) return void 0;
|
|
3450
|
+
return ctx.traceId;
|
|
3451
|
+
} catch {
|
|
3452
|
+
return void 0;
|
|
3453
|
+
}
|
|
3454
|
+
}
|
|
3455
|
+
var ScoresClient = class _ScoresClient {
|
|
3456
|
+
host;
|
|
3457
|
+
publicKey;
|
|
3458
|
+
secretKey;
|
|
3459
|
+
timeoutSeconds;
|
|
3460
|
+
debug;
|
|
3461
|
+
constructor(config) {
|
|
3462
|
+
this.host = config.host.replace(/\/$/, "");
|
|
3463
|
+
this.publicKey = config.publicKey;
|
|
3464
|
+
this.secretKey = config.secretKey;
|
|
3465
|
+
this.timeoutSeconds = config.timeoutSeconds ?? 15;
|
|
3466
|
+
this.debug = config.debug ?? false;
|
|
3467
|
+
}
|
|
3468
|
+
/** Create a client from GATEWAY_HOST / GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY. */
|
|
3469
|
+
static fromEnv(config = {}) {
|
|
3470
|
+
const host = config.host ?? process.env.GATEWAY_HOST;
|
|
3471
|
+
const publicKey = config.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
3472
|
+
const secretKey = config.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
3473
|
+
const missing = [
|
|
3474
|
+
["host", host],
|
|
3475
|
+
["publicKey", publicKey],
|
|
3476
|
+
["secretKey", secretKey]
|
|
3477
|
+
].filter(([, value]) => !value).map(([name]) => name);
|
|
3478
|
+
if (missing.length > 0) {
|
|
3479
|
+
throw new ScoreError(
|
|
3480
|
+
`Missing Gateway scores configuration: ${missing.join(", ")}`
|
|
3481
|
+
);
|
|
3482
|
+
}
|
|
3483
|
+
return new _ScoresClient({
|
|
3484
|
+
...config,
|
|
3485
|
+
host,
|
|
3486
|
+
publicKey,
|
|
3487
|
+
secretKey
|
|
3488
|
+
});
|
|
3489
|
+
}
|
|
3490
|
+
/**
|
|
3491
|
+
* Post a score, throwing {@link ScoreError} on any failure. Builds the exact
|
|
3492
|
+
* `POST /api/public/scores` body the Python SDK sends: exactly one of
|
|
3493
|
+
* `sessionId` / `traceId` (+ optional `observationId`) / `datasetRunId`.
|
|
3494
|
+
* When no target is provided, the active OTel trace id is used; if none is
|
|
3495
|
+
* available the call is a no-op (returns without posting), matching Python.
|
|
3496
|
+
*/
|
|
3497
|
+
async postScore(name, value, options = {}) {
|
|
3498
|
+
const body = this.buildScoreBody(name, value, options);
|
|
3499
|
+
if (!body) {
|
|
3500
|
+
this.log(
|
|
3501
|
+
"score '%s' dropped: no sessionId, traceId or datasetRunId available",
|
|
3502
|
+
name
|
|
3503
|
+
);
|
|
3504
|
+
return;
|
|
3505
|
+
}
|
|
3506
|
+
await this.request("POST", "/api/public/scores", body);
|
|
3507
|
+
}
|
|
3508
|
+
/**
|
|
3509
|
+
* Build the score request body, or null when no target can be resolved.
|
|
3510
|
+
* Split out so it is unit-testable and shared by score()/success().
|
|
3511
|
+
*/
|
|
3512
|
+
buildScoreBody(name, value, options = {}) {
|
|
3513
|
+
const traceId = options.traceId ?? currentTraceId();
|
|
3514
|
+
const target = {};
|
|
3515
|
+
if (options.sessionId) {
|
|
3516
|
+
target.sessionId = options.sessionId;
|
|
3517
|
+
} else if (options.datasetRunId) {
|
|
3518
|
+
target.datasetRunId = options.datasetRunId;
|
|
3519
|
+
} else if (traceId) {
|
|
3520
|
+
target.traceId = traceId;
|
|
3521
|
+
if (options.observationId) target.observationId = options.observationId;
|
|
3522
|
+
} else {
|
|
3523
|
+
return null;
|
|
3524
|
+
}
|
|
3525
|
+
const metadata = {
|
|
3526
|
+
source: "gatewaysdk",
|
|
3527
|
+
...options.metadata ?? {}
|
|
3528
|
+
};
|
|
3529
|
+
return {
|
|
3530
|
+
name,
|
|
3531
|
+
value,
|
|
3532
|
+
dataType: options.dataType ?? "NUMERIC",
|
|
3533
|
+
environment: options.environment ?? "default",
|
|
3534
|
+
...options.comment ? { comment: options.comment } : {},
|
|
3535
|
+
metadata,
|
|
3536
|
+
...target
|
|
3537
|
+
};
|
|
3538
|
+
}
|
|
3539
|
+
/**
|
|
3540
|
+
* Post a score, FAIL-SOFT: warns and resolves on any failure so a broken
|
|
3541
|
+
* transport never throws into (or rejects out of) the caller.
|
|
3542
|
+
*/
|
|
3543
|
+
async score(name, value, options = {}) {
|
|
3544
|
+
try {
|
|
3545
|
+
await this.postScore(name, value, options);
|
|
3546
|
+
} catch (error) {
|
|
3547
|
+
this.warn(`score '${name}' failed`, error);
|
|
3548
|
+
}
|
|
3549
|
+
}
|
|
3550
|
+
/**
|
|
3551
|
+
* Emit a SUCCESS SIGNAL that feeds a Gateway Success Metric — a score tagged
|
|
3552
|
+
* with `gateway.success.signal` + `gateway.success.metricName` (and the agent
|
|
3553
|
+
* identity when `agentId` is set). A boolean `value` records a 1/0 BOOLEAN
|
|
3554
|
+
* score (pass/fail); a number records a NUMERIC score. FAIL-SOFT.
|
|
3555
|
+
*
|
|
3556
|
+
* The metadata shape matches the Python SDK's `Run.success()` exactly, so the
|
|
3557
|
+
* platform's Success Metrics aggregation counts signals from either SDK
|
|
3558
|
+
* without change.
|
|
3559
|
+
*
|
|
3560
|
+
* @example
|
|
3561
|
+
* ```ts
|
|
3562
|
+
* await scores.success("resolved_ticket", true, { agentId: "support-triage", sessionId });
|
|
3563
|
+
* await scores.success("handle_time_s", 42, { agentId: "support-triage", sessionId });
|
|
3564
|
+
* ```
|
|
3565
|
+
*/
|
|
3566
|
+
async success(name, value = true, options = {}) {
|
|
3567
|
+
const { numericValue, inferredType } = coerceSignalValue(value);
|
|
3568
|
+
const { agentId, metricName, metadata, dataType, ...rest } = options;
|
|
3569
|
+
const signalMetadata = {
|
|
3570
|
+
...metadata ?? {},
|
|
3571
|
+
[SUCCESS_SIGNAL_KEY]: true,
|
|
3572
|
+
[SUCCESS_METRIC_NAME_KEY]: metricName ?? name,
|
|
3573
|
+
...agentId ? { [AGENT_ID_KEY]: agentId } : {}
|
|
3574
|
+
};
|
|
3575
|
+
await this.score(name, numericValue, {
|
|
3576
|
+
...rest,
|
|
3577
|
+
dataType: dataType ?? inferredType,
|
|
3578
|
+
metadata: signalMetadata
|
|
3579
|
+
});
|
|
3580
|
+
}
|
|
3581
|
+
async request(method, path, payload) {
|
|
3582
|
+
let response;
|
|
3583
|
+
try {
|
|
3584
|
+
response = await fetch(`${this.host}${path}`, {
|
|
3585
|
+
method,
|
|
3586
|
+
headers: {
|
|
3587
|
+
Authorization: this.authHeader(),
|
|
3588
|
+
"Content-Type": "application/json"
|
|
3589
|
+
},
|
|
3590
|
+
body: JSON.stringify(payload),
|
|
3591
|
+
signal: AbortSignal.timeout(this.timeoutSeconds * 1e3)
|
|
3592
|
+
});
|
|
3593
|
+
} catch (error) {
|
|
3594
|
+
throw new ScoreError(
|
|
3595
|
+
`${method} ${path} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
3596
|
+
{ cause: error }
|
|
3597
|
+
);
|
|
3598
|
+
}
|
|
3599
|
+
const text = await response.text();
|
|
3600
|
+
if (!response.ok) {
|
|
3601
|
+
throw new ScoreError(
|
|
3602
|
+
`${method} ${path} failed (${response.status}): ${text.slice(0, 500)}`
|
|
3603
|
+
);
|
|
3604
|
+
}
|
|
3605
|
+
this.log("%s %s -> %s", method, path, response.status);
|
|
3606
|
+
}
|
|
3607
|
+
authHeader() {
|
|
3608
|
+
return `Basic ${Buffer.from(`${this.publicKey}:${this.secretKey}`).toString(
|
|
3609
|
+
"base64"
|
|
3610
|
+
)}`;
|
|
3611
|
+
}
|
|
3612
|
+
log(message, ...args) {
|
|
3613
|
+
if (this.debug) {
|
|
3614
|
+
console.log(`[gateway.scores] ${message}`, ...args);
|
|
3615
|
+
}
|
|
3616
|
+
}
|
|
3617
|
+
warn(message, ...args) {
|
|
3618
|
+
console.warn(`[gateway.scores] ${message}`, ...args);
|
|
3619
|
+
}
|
|
3620
|
+
};
|
|
3621
|
+
function coerceSignalValue(value) {
|
|
3622
|
+
if (typeof value === "boolean") {
|
|
3623
|
+
return { numericValue: value ? 1 : 0, inferredType: "BOOLEAN" };
|
|
3624
|
+
}
|
|
3625
|
+
return { numericValue: value, inferredType: "NUMERIC" };
|
|
3626
|
+
}
|
|
3627
|
+
async function success(name, value = true, options = {}) {
|
|
3628
|
+
let client2;
|
|
3629
|
+
try {
|
|
3630
|
+
client2 = ScoresClient.fromEnv();
|
|
3631
|
+
} catch (error) {
|
|
3632
|
+
console.warn(`[gateway.scores] success '${name}' skipped`, error);
|
|
3633
|
+
return;
|
|
3634
|
+
}
|
|
3635
|
+
await client2.success(name, value, options);
|
|
3636
|
+
}
|
|
3637
|
+
var scores = {
|
|
3638
|
+
fromEnv: ScoresClient.fromEnv,
|
|
3639
|
+
success
|
|
3640
|
+
};
|
|
3641
|
+
|
|
3642
|
+
// src/index.ts
|
|
3643
|
+
init_worlds();
|
|
3644
|
+
init_session();
|
|
3645
|
+
|
|
2833
3646
|
// src/build/index.ts
|
|
2834
3647
|
var import_node_crypto = require("crypto");
|
|
2835
3648
|
function canonicalJson(value) {
|
|
@@ -3113,7 +3926,7 @@ async function registerBuild(params) {
|
|
|
3113
3926
|
|
|
3114
3927
|
// src/tracing/identity.ts
|
|
3115
3928
|
var import_async_hooks2 = require("async_hooks");
|
|
3116
|
-
var
|
|
3929
|
+
var import_api2 = require("@opentelemetry/api");
|
|
3117
3930
|
var USER_ID_ATTRIBUTE = "gateway.user.id";
|
|
3118
3931
|
var currentUserStorage = new import_async_hooks2.AsyncLocalStorage();
|
|
3119
3932
|
function resolveUserId() {
|
|
@@ -3127,8 +3940,8 @@ function identify(userId, options = {}) {
|
|
|
3127
3940
|
if (!cleaned) return;
|
|
3128
3941
|
currentUserStorage.enterWith(cleaned);
|
|
3129
3942
|
try {
|
|
3130
|
-
const span =
|
|
3131
|
-
if (span && (0,
|
|
3943
|
+
const span = import_api2.trace.getActiveSpan();
|
|
3944
|
+
if (span && (0, import_api2.isSpanContextValid)(span.spanContext())) {
|
|
3132
3945
|
span.setAttribute(USER_ID_ATTRIBUTE, cleaned);
|
|
3133
3946
|
}
|
|
3134
3947
|
} catch (error) {
|
|
@@ -3406,12 +4219,19 @@ function initIsolated(config = {}) {
|
|
|
3406
4219
|
const provider = built.provider;
|
|
3407
4220
|
scheduleBuildRegistration(config, built);
|
|
3408
4221
|
const tracer = provider.getTracer("@withgateway/sdk");
|
|
4222
|
+
const scores2 = new ScoresClient({
|
|
4223
|
+
host: built.connection.host,
|
|
4224
|
+
publicKey: built.connection.publicKey,
|
|
4225
|
+
secretKey: built.connection.secretKey,
|
|
4226
|
+
debug: config.debug ?? false
|
|
4227
|
+
});
|
|
3409
4228
|
debugLog2(
|
|
3410
4229
|
"Isolated tracing provider created (service=%s)",
|
|
3411
4230
|
config.serviceName ?? "gatewaysdk-tracing"
|
|
3412
4231
|
);
|
|
3413
4232
|
return {
|
|
3414
4233
|
tracer,
|
|
4234
|
+
scores: scores2,
|
|
3415
4235
|
async flush() {
|
|
3416
4236
|
try {
|
|
3417
4237
|
await provider.forceFlush();
|
|
@@ -3465,12 +4285,12 @@ function isGatewayConfigured() {
|
|
|
3465
4285
|
}
|
|
3466
4286
|
|
|
3467
4287
|
// src/tracing/observe.ts
|
|
3468
|
-
var
|
|
4288
|
+
var import_api3 = require("@opentelemetry/api");
|
|
3469
4289
|
function asAttr(value) {
|
|
3470
4290
|
return typeof value === "string" ? value : JSON.stringify(value);
|
|
3471
4291
|
}
|
|
3472
4292
|
async function observe(name, fn, options = {}) {
|
|
3473
|
-
const tracer = options.tracer ??
|
|
4293
|
+
const tracer = options.tracer ?? import_api3.trace.getTracer("@withgateway/sdk");
|
|
3474
4294
|
const span = tracer.startSpan(name, {
|
|
3475
4295
|
attributes: {
|
|
3476
4296
|
"openinference.span.kind": options.kind ?? "AGENT",
|
|
@@ -3479,18 +4299,18 @@ async function observe(name, fn, options = {}) {
|
|
|
3479
4299
|
}
|
|
3480
4300
|
});
|
|
3481
4301
|
try {
|
|
3482
|
-
const result = await
|
|
3483
|
-
|
|
4302
|
+
const result = await import_api3.context.with(
|
|
4303
|
+
import_api3.trace.setSpan(import_api3.context.active(), span),
|
|
3484
4304
|
fn
|
|
3485
4305
|
);
|
|
3486
4306
|
if (options.captureOutput !== false && result !== void 0) {
|
|
3487
4307
|
span.setAttribute("output.value", asAttr(result));
|
|
3488
4308
|
}
|
|
3489
|
-
span.setStatus({ code:
|
|
4309
|
+
span.setStatus({ code: import_api3.SpanStatusCode.OK });
|
|
3490
4310
|
return result;
|
|
3491
4311
|
} catch (error) {
|
|
3492
4312
|
span.setStatus({
|
|
3493
|
-
code:
|
|
4313
|
+
code: import_api3.SpanStatusCode.ERROR,
|
|
3494
4314
|
message: error instanceof Error ? error.message : String(error)
|
|
3495
4315
|
});
|
|
3496
4316
|
throw error;
|
|
@@ -3537,7 +4357,7 @@ function observeAnthropic(client2) {
|
|
|
3537
4357
|
}
|
|
3538
4358
|
|
|
3539
4359
|
// src/tracing/claude.ts
|
|
3540
|
-
var
|
|
4360
|
+
var import_api4 = require("@opentelemetry/api");
|
|
3541
4361
|
function cleanToolName(name) {
|
|
3542
4362
|
return name.replace(/^mcp__[^_]+__/, "");
|
|
3543
4363
|
}
|
|
@@ -3551,7 +4371,7 @@ function toolResultText(content) {
|
|
|
3551
4371
|
}
|
|
3552
4372
|
var TURN_INPUT_MAX = 4e3;
|
|
3553
4373
|
async function* observeClaude(stream, options) {
|
|
3554
|
-
const tracer = options.tracer ??
|
|
4374
|
+
const tracer = options.tracer ?? import_api4.trace.getTracer("@withgateway/sdk");
|
|
3555
4375
|
const model = options.model ?? "claude";
|
|
3556
4376
|
const openTools = /* @__PURE__ */ new Map();
|
|
3557
4377
|
let root = null;
|
|
@@ -3564,7 +4384,7 @@ async function* observeClaude(stream, options) {
|
|
|
3564
4384
|
...options.attributes ?? {}
|
|
3565
4385
|
}
|
|
3566
4386
|
});
|
|
3567
|
-
rootCtx =
|
|
4387
|
+
rootCtx = import_api4.trace.setSpan(import_api4.context.active(), root);
|
|
3568
4388
|
}
|
|
3569
4389
|
let turnRoot = null;
|
|
3570
4390
|
let turnCtx = null;
|
|
@@ -3575,7 +4395,7 @@ async function* observeClaude(stream, options) {
|
|
|
3575
4395
|
const endTurn = () => {
|
|
3576
4396
|
if (!turnRoot) return;
|
|
3577
4397
|
if (turnText) turnRoot.setAttribute("output.value", turnText);
|
|
3578
|
-
turnRoot.setStatus({ code:
|
|
4398
|
+
turnRoot.setStatus({ code: import_api4.SpanStatusCode.OK });
|
|
3579
4399
|
turnRoot.end();
|
|
3580
4400
|
turnRoot = null;
|
|
3581
4401
|
turnCtx = null;
|
|
@@ -3597,7 +4417,7 @@ async function* observeClaude(stream, options) {
|
|
|
3597
4417
|
...options.attributes ?? {}
|
|
3598
4418
|
}
|
|
3599
4419
|
});
|
|
3600
|
-
turnCtx =
|
|
4420
|
+
turnCtx = import_api4.trace.setSpan(import_api4.context.active(), turnRoot);
|
|
3601
4421
|
pendingTurnInput = "";
|
|
3602
4422
|
}
|
|
3603
4423
|
const childCtx = options.perTurn ? turnCtx : rootCtx;
|
|
@@ -3659,7 +4479,7 @@ async function* observeClaude(stream, options) {
|
|
|
3659
4479
|
const span = openTools.get(block.tool_use_id);
|
|
3660
4480
|
if (span) {
|
|
3661
4481
|
span.setAttribute("output.value", resultText);
|
|
3662
|
-
span.setStatus({ code:
|
|
4482
|
+
span.setStatus({ code: import_api4.SpanStatusCode.OK });
|
|
3663
4483
|
span.end();
|
|
3664
4484
|
openTools.delete(block.tool_use_id);
|
|
3665
4485
|
}
|
|
@@ -3679,7 +4499,7 @@ async function* observeClaude(stream, options) {
|
|
|
3679
4499
|
endTurn();
|
|
3680
4500
|
} else if (root) {
|
|
3681
4501
|
root.setAttribute("output.value", finalText);
|
|
3682
|
-
root.setStatus({ code:
|
|
4502
|
+
root.setStatus({ code: import_api4.SpanStatusCode.OK });
|
|
3683
4503
|
root.end();
|
|
3684
4504
|
}
|
|
3685
4505
|
}
|
|
@@ -3704,6 +4524,7 @@ var tracing = {
|
|
|
3704
4524
|
};
|
|
3705
4525
|
// Annotate the CommonJS export names for ESM import in node:
|
|
3706
4526
|
0 && (module.exports = {
|
|
4527
|
+
AGENT_ID_KEY,
|
|
3707
4528
|
DEFAULT_TIMEOUT_SECONDS,
|
|
3708
4529
|
DriftError,
|
|
3709
4530
|
ENV_GATEWAY_HOST,
|
|
@@ -3733,6 +4554,10 @@ var tracing = {
|
|
|
3733
4554
|
ReplayReport,
|
|
3734
4555
|
ReplaySuite,
|
|
3735
4556
|
ReplayTask,
|
|
4557
|
+
SUCCESS_METRIC_NAME_KEY,
|
|
4558
|
+
SUCCESS_SIGNAL_KEY,
|
|
4559
|
+
ScoreError,
|
|
4560
|
+
ScoresClient,
|
|
3736
4561
|
SessionLoadError,
|
|
3737
4562
|
SessionRecording,
|
|
3738
4563
|
SessionTrace,
|
|
@@ -3741,16 +4566,26 @@ var tracing = {
|
|
|
3741
4566
|
TraceDiff,
|
|
3742
4567
|
UserProfileError,
|
|
3743
4568
|
UsersClient,
|
|
4569
|
+
WorldSession,
|
|
4570
|
+
WorldSessionCallError,
|
|
4571
|
+
WorldSessionTimeout,
|
|
4572
|
+
WorldSet,
|
|
4573
|
+
WorldTask,
|
|
3744
4574
|
addExperimentAttributes,
|
|
4575
|
+
apiRequest,
|
|
3745
4576
|
assembleManifest,
|
|
3746
4577
|
assignVariant,
|
|
4578
|
+
attachSession,
|
|
3747
4579
|
basicAuthHeader,
|
|
3748
4580
|
canonicalJson,
|
|
3749
4581
|
clearExperimentContext,
|
|
3750
4582
|
computeBuildHash,
|
|
3751
4583
|
computeHash,
|
|
3752
4584
|
context,
|
|
4585
|
+
currentTraceId,
|
|
3753
4586
|
defineTool,
|
|
4587
|
+
deleteEnvSecret,
|
|
4588
|
+
detectProvenance,
|
|
3754
4589
|
experimentContextStorage,
|
|
3755
4590
|
experiments,
|
|
3756
4591
|
gateway,
|
|
@@ -3759,9 +4594,14 @@ var tracing = {
|
|
|
3759
4594
|
getRegisteredPrompts,
|
|
3760
4595
|
getRegisteredSkills,
|
|
3761
4596
|
getRegisteredTools,
|
|
4597
|
+
getRun,
|
|
3762
4598
|
identify,
|
|
4599
|
+
listEnvSecretKeys,
|
|
4600
|
+
listRunConfigs,
|
|
3763
4601
|
loadSession,
|
|
3764
4602
|
memory,
|
|
4603
|
+
openSession,
|
|
4604
|
+
openWorld,
|
|
3765
4605
|
personas,
|
|
3766
4606
|
registerArtifact,
|
|
3767
4607
|
registerPrompt,
|
|
@@ -3770,13 +4610,19 @@ var tracing = {
|
|
|
3770
4610
|
replay,
|
|
3771
4611
|
resetBuildRegistry,
|
|
3772
4612
|
resolveUserId,
|
|
4613
|
+
runSessions,
|
|
3773
4614
|
runSuite,
|
|
3774
4615
|
runWithExperimentContext,
|
|
4616
|
+
schemaFromSignature,
|
|
4617
|
+
scores,
|
|
3775
4618
|
sessionObservationFromApi,
|
|
3776
4619
|
sessions,
|
|
4620
|
+
setEnvSecret,
|
|
3777
4621
|
setExperimentContext,
|
|
4622
|
+
success,
|
|
3778
4623
|
suite,
|
|
3779
4624
|
tracing,
|
|
3780
|
-
users
|
|
4625
|
+
users,
|
|
4626
|
+
waitForRun
|
|
3781
4627
|
});
|
|
3782
4628
|
//# sourceMappingURL=index.js.map
|