@withgateway/sdk 0.1.12 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -0
- package/dist/cli.d.mts +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +1303 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.mjs +1290 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/{index-CTc8iOrx.d.ts → index-B5HUQQDE.d.mts} +9 -0
- package/dist/{index-CTc8iOrx.d.mts → index-Dpo-t8iO.d.ts} +9 -0
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +878 -32
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +845 -25
- package/dist/index.mjs.map +1 -1
- package/dist/scores/index.d.mts +165 -0
- package/dist/scores/index.d.ts +165 -0
- package/dist/scores/index.js +252 -0
- package/dist/scores/index.js.map +1 -0
- package/dist/scores/index.mjs +223 -0
- package/dist/scores/index.mjs.map +1 -0
- package/dist/tracing/index.d.mts +2 -1
- package/dist/tracing/index.d.ts +2 -1
- package/dist/tracing/index.js +262 -15
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +256 -9
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +377 -0
- package/dist/worlds/index.d.ts +377 -0
- package/dist/worlds/index.js +584 -0
- package/dist/worlds/index.js.map +1 -0
- package/dist/worlds/index.mjs +578 -0
- package/dist/worlds/index.mjs.map +1 -0
- package/package.json +32 -2
package/dist/index.mjs
CHANGED
|
@@ -8,6 +8,9 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
|
|
|
8
8
|
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
9
9
|
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
10
10
|
});
|
|
11
|
+
var __esm = (fn, res) => function __init() {
|
|
12
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
13
|
+
};
|
|
11
14
|
var __commonJS = (cb, mod) => function __require2() {
|
|
12
15
|
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
13
16
|
};
|
|
@@ -16,12 +19,552 @@ var __export = (target, all) => {
|
|
|
16
19
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
17
20
|
};
|
|
18
21
|
|
|
22
|
+
// src/worlds/session.ts
|
|
23
|
+
var session_exports = {};
|
|
24
|
+
__export(session_exports, {
|
|
25
|
+
WorldSession: () => WorldSession,
|
|
26
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
27
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
28
|
+
attachSession: () => attachSession,
|
|
29
|
+
openSession: () => openSession,
|
|
30
|
+
runSessions: () => runSessions,
|
|
31
|
+
schemaFromSignature: () => schemaFromSignature
|
|
32
|
+
});
|
|
33
|
+
function schemaFromSignature(spec) {
|
|
34
|
+
const inside = /\(([\s\S]*)\)\s*$/.exec(spec.signature.trim())?.[1] ?? "";
|
|
35
|
+
const properties = {};
|
|
36
|
+
const required = [];
|
|
37
|
+
const params = [];
|
|
38
|
+
let depth = 0;
|
|
39
|
+
let current = "";
|
|
40
|
+
for (const ch of inside) {
|
|
41
|
+
if ("([{".includes(ch)) depth++;
|
|
42
|
+
if (")]}".includes(ch)) depth--;
|
|
43
|
+
if (ch === "," && depth === 0) {
|
|
44
|
+
params.push(current);
|
|
45
|
+
current = "";
|
|
46
|
+
} else current += ch;
|
|
47
|
+
}
|
|
48
|
+
if (current.trim()) params.push(current);
|
|
49
|
+
for (const raw of params) {
|
|
50
|
+
const param = raw.trim();
|
|
51
|
+
if (!param || param.startsWith("*") || param === "self") continue;
|
|
52
|
+
const eq = param.indexOf("=");
|
|
53
|
+
const name = (eq === -1 ? param : param.slice(0, eq)).split(":")[0].trim();
|
|
54
|
+
if (!name) continue;
|
|
55
|
+
if (eq === -1) {
|
|
56
|
+
properties[name] = { type: "string" };
|
|
57
|
+
required.push(name);
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
const value = param.slice(eq + 1).trim();
|
|
61
|
+
if (/^-?\d+(\.\d+)?$/.test(value)) properties[name] = { type: "number" };
|
|
62
|
+
else if (value === "True" || value === "False")
|
|
63
|
+
properties[name] = { type: "boolean" };
|
|
64
|
+
else properties[name] = { type: "string" };
|
|
65
|
+
}
|
|
66
|
+
return {
|
|
67
|
+
type: "function",
|
|
68
|
+
function: {
|
|
69
|
+
name: spec.name,
|
|
70
|
+
description: spec.doc ?? spec.name,
|
|
71
|
+
parameters: { type: "object", properties, required }
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
async function attachSession(sessionId, options = {}) {
|
|
76
|
+
const state2 = await apiRequest(
|
|
77
|
+
options,
|
|
78
|
+
"GET",
|
|
79
|
+
`/api/public/world-sessions/${encodeURIComponent(sessionId)}`
|
|
80
|
+
);
|
|
81
|
+
return new WorldSession(sessionId, state2, options);
|
|
82
|
+
}
|
|
83
|
+
async function openSession(slugOrRef, task, opts = {}) {
|
|
84
|
+
const options = opts.options ?? {};
|
|
85
|
+
const at = slugOrRef.indexOf("@");
|
|
86
|
+
const slug = at === -1 ? slugOrRef : slugOrRef.slice(0, at);
|
|
87
|
+
const state2 = await apiRequest(
|
|
88
|
+
options,
|
|
89
|
+
"POST",
|
|
90
|
+
"/api/public/world-sessions",
|
|
91
|
+
{
|
|
92
|
+
...opts.versionId ? { versionId: opts.versionId } : { slug },
|
|
93
|
+
task,
|
|
94
|
+
...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm }
|
|
95
|
+
}
|
|
96
|
+
);
|
|
97
|
+
return new WorldSession(state2.sessionId, state2, options);
|
|
98
|
+
}
|
|
99
|
+
async function runSessions(opts) {
|
|
100
|
+
const options = opts.options ?? {};
|
|
101
|
+
const concurrency = Math.max(1, opts.concurrency ?? 3);
|
|
102
|
+
let tasks = opts.tasks;
|
|
103
|
+
let versionId = opts.versionId;
|
|
104
|
+
if (!tasks || !versionId) {
|
|
105
|
+
const { openWorld: openWorld2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
|
|
106
|
+
const world = await openWorld2(opts.world, options);
|
|
107
|
+
tasks = tasks ?? world.tasks.map((t) => t.name);
|
|
108
|
+
versionId = versionId ?? world.version.versionId;
|
|
109
|
+
}
|
|
110
|
+
const results = [];
|
|
111
|
+
let next = 0;
|
|
112
|
+
await Promise.all(
|
|
113
|
+
Array.from({ length: Math.min(concurrency, tasks.length) }, async () => {
|
|
114
|
+
while (next < tasks.length) {
|
|
115
|
+
const task = tasks[next++];
|
|
116
|
+
let session = null;
|
|
117
|
+
try {
|
|
118
|
+
session = await openSession(opts.world, task, {
|
|
119
|
+
versionId,
|
|
120
|
+
options
|
|
121
|
+
});
|
|
122
|
+
await session.ready();
|
|
123
|
+
const toolkit = await session.toolkit();
|
|
124
|
+
await opts.agent({
|
|
125
|
+
session,
|
|
126
|
+
toolkit,
|
|
127
|
+
instruction: session.instruction ?? task,
|
|
128
|
+
task
|
|
129
|
+
});
|
|
130
|
+
const graded = await session.grade();
|
|
131
|
+
const result = {
|
|
132
|
+
task,
|
|
133
|
+
sessionId: session.sessionId,
|
|
134
|
+
reward: graded.reward,
|
|
135
|
+
rewards: graded.rewards
|
|
136
|
+
};
|
|
137
|
+
results.push(result);
|
|
138
|
+
opts.onResult?.(result);
|
|
139
|
+
} catch (error) {
|
|
140
|
+
const result = {
|
|
141
|
+
task,
|
|
142
|
+
sessionId: session?.sessionId ?? "",
|
|
143
|
+
reward: 0,
|
|
144
|
+
rewards: {},
|
|
145
|
+
// An Error with an empty message would read as "no error" to any
|
|
146
|
+
// caller checking truthiness; name the type instead.
|
|
147
|
+
error: (error instanceof Error ? error.message : String(error)) || (error instanceof Error ? error.name : "Unknown error")
|
|
148
|
+
};
|
|
149
|
+
results.push(result);
|
|
150
|
+
opts.onResult?.(result);
|
|
151
|
+
} finally {
|
|
152
|
+
await session?.close({ keepWarm: opts.keepWarm ?? true }).catch(() => void 0);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
})
|
|
156
|
+
);
|
|
157
|
+
const mean = results.reduce((sum, r) => sum + (r.reward ?? 0), 0) / (results.length || 1);
|
|
158
|
+
return {
|
|
159
|
+
results: results.sort((a, b) => a.task.localeCompare(b.task)),
|
|
160
|
+
mean,
|
|
161
|
+
passed: opts.failUnder === void 0 ? true : mean >= opts.failUnder
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
var READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, sleep2, WorldSessionTimeout, WorldSessionCallError, WorldSession;
|
|
165
|
+
var init_session = __esm({
|
|
166
|
+
"src/worlds/session.ts"() {
|
|
167
|
+
"use strict";
|
|
168
|
+
init_worlds();
|
|
169
|
+
READY_TIMEOUT_MS = 10 * 6e4;
|
|
170
|
+
READY_POLL_MS = 2e3;
|
|
171
|
+
CALL_TIMEOUT_MS = 3 * 6e4;
|
|
172
|
+
CALL_POLL_MS = 400;
|
|
173
|
+
sleep2 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
174
|
+
WorldSessionTimeout = class extends Error {
|
|
175
|
+
};
|
|
176
|
+
WorldSessionCallError = class extends Error {
|
|
177
|
+
};
|
|
178
|
+
WorldSession = class {
|
|
179
|
+
constructor(sessionId, state2, options) {
|
|
180
|
+
this.sessionId = sessionId;
|
|
181
|
+
this.state = state2;
|
|
182
|
+
this.options = options;
|
|
183
|
+
}
|
|
184
|
+
get task() {
|
|
185
|
+
return this.state.taskName;
|
|
186
|
+
}
|
|
187
|
+
/** What this task asks for — the bundle's own instruction.md. */
|
|
188
|
+
get instruction() {
|
|
189
|
+
return this.state.instruction ?? null;
|
|
190
|
+
}
|
|
191
|
+
get versionId() {
|
|
192
|
+
return this.state.versionId;
|
|
193
|
+
}
|
|
194
|
+
/** The session's latest platform state (one request, no waiting). */
|
|
195
|
+
async status() {
|
|
196
|
+
const fresh = await apiRequest(
|
|
197
|
+
this.options,
|
|
198
|
+
"GET",
|
|
199
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`
|
|
200
|
+
);
|
|
201
|
+
this.state = {
|
|
202
|
+
instruction: this.state.instruction,
|
|
203
|
+
tools: this.state.tools,
|
|
204
|
+
...fresh
|
|
205
|
+
};
|
|
206
|
+
return this.state;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Wait until the container is warm — meaning the process inside it has
|
|
210
|
+
* asked the platform for work, not merely that a job was accepted.
|
|
211
|
+
*/
|
|
212
|
+
async ready(opts = {}) {
|
|
213
|
+
const deadline = Date.now() + (opts.timeoutMs ?? READY_TIMEOUT_MS);
|
|
214
|
+
for (; ; ) {
|
|
215
|
+
const state2 = await this.status();
|
|
216
|
+
if (state2.ready) return state2;
|
|
217
|
+
if (state2.state === "FAILED" || state2.state === "STOPPED")
|
|
218
|
+
throw new WorldSessionCallError(
|
|
219
|
+
`World session ${state2.state.toLowerCase()} before it was ready.`
|
|
220
|
+
);
|
|
221
|
+
if (Date.now() > deadline)
|
|
222
|
+
throw new WorldSessionTimeout(
|
|
223
|
+
`World session still ${state2.state} after ${Math.round((opts.timeoutMs ?? READY_TIMEOUT_MS) / 1e3)}s.`
|
|
224
|
+
);
|
|
225
|
+
await sleep2(READY_POLL_MS);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
/** Enqueue one interaction and wait for the container's answer. */
|
|
229
|
+
async exchange(body, timeoutMs = CALL_TIMEOUT_MS) {
|
|
230
|
+
const { callId } = await apiRequest(
|
|
231
|
+
this.options,
|
|
232
|
+
"POST",
|
|
233
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls`,
|
|
234
|
+
body
|
|
235
|
+
);
|
|
236
|
+
const deadline = Date.now() + timeoutMs;
|
|
237
|
+
for (; ; ) {
|
|
238
|
+
const call = await apiRequest(
|
|
239
|
+
this.options,
|
|
240
|
+
"GET",
|
|
241
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls/${encodeURIComponent(callId)}`
|
|
242
|
+
);
|
|
243
|
+
if (call.done) {
|
|
244
|
+
if (call.error) throw new WorldSessionCallError(call.error);
|
|
245
|
+
return call.result;
|
|
246
|
+
}
|
|
247
|
+
if (Date.now() > deadline)
|
|
248
|
+
throw new WorldSessionTimeout(
|
|
249
|
+
`Call ${callId} still ${call.state} after ${Math.round(timeoutMs / 1e3)}s \u2014 the container may have died.`
|
|
250
|
+
);
|
|
251
|
+
await sleep2(CALL_POLL_MS);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Run one of the world's tools INSIDE the container. The arguments and the
|
|
256
|
+
* result are the world's own contract — this layer neither validates nor
|
|
257
|
+
* reshapes them, so a tool error reads exactly as it would in a hosted run.
|
|
258
|
+
*/
|
|
259
|
+
async call(tool, args = {}, opts = {}) {
|
|
260
|
+
return this.exchange({ kind: "call", tool, args }, opts.timeoutMs);
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* THE WORLD'S TOOLS, READY TO HAND TO A MODEL. Schemas come from the
|
|
264
|
+
* world's own signatures and implementations call into this session, so a
|
|
265
|
+
* caller never writes — and never has to re-sync — a tool map of their own:
|
|
266
|
+
*
|
|
267
|
+
* const { schemas, impls } = await session.toolkit();
|
|
268
|
+
* await runMyAgent(impls, schemas, session.instruction);
|
|
269
|
+
*
|
|
270
|
+
* A tool added to the world shows up here on the next run; a tool renamed
|
|
271
|
+
* in the world renames here too, instead of failing at call time against a
|
|
272
|
+
* hand-written map that still remembers the old name.
|
|
273
|
+
*/
|
|
274
|
+
async toolkit() {
|
|
275
|
+
let specs = this.state.tools;
|
|
276
|
+
if (!specs) specs = (await this.status()).tools ?? [];
|
|
277
|
+
const impls = {};
|
|
278
|
+
for (const spec of specs) {
|
|
279
|
+
impls[spec.name] = (args = {}) => this.call(spec.name, args);
|
|
280
|
+
}
|
|
281
|
+
return { schemas: specs.map(schemaFromSignature), impls, specs };
|
|
282
|
+
}
|
|
283
|
+
/** Run the task's grader against the container's END STATE. */
|
|
284
|
+
async grade(opts = {}) {
|
|
285
|
+
const raw = await this.exchange(
|
|
286
|
+
{ kind: "grade" },
|
|
287
|
+
opts.timeoutMs
|
|
288
|
+
);
|
|
289
|
+
const record = raw && typeof raw === "object" ? raw : {};
|
|
290
|
+
const rewards = {};
|
|
291
|
+
for (const [key, value] of Object.entries(record)) {
|
|
292
|
+
if (key !== "reward" && typeof value === "number") rewards[key] = value;
|
|
293
|
+
}
|
|
294
|
+
return {
|
|
295
|
+
reward: typeof record.reward === "number" ? record.reward : null,
|
|
296
|
+
rewards,
|
|
297
|
+
raw
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* End the session. `keepWarm` hands the container back to the pool instead
|
|
302
|
+
* of tearing it down, so the next session for the same task starts warm —
|
|
303
|
+
* the pooled instance still obeys the fleet's idle TTL and pinning.
|
|
304
|
+
*/
|
|
305
|
+
async close(opts = {}) {
|
|
306
|
+
return apiRequest(
|
|
307
|
+
this.options,
|
|
308
|
+
"DELETE",
|
|
309
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`,
|
|
310
|
+
{ keepWarm: opts.keepWarm ?? false }
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
// src/worlds/index.ts
|
|
318
|
+
var worlds_exports = {};
|
|
319
|
+
__export(worlds_exports, {
|
|
320
|
+
WorldSession: () => WorldSession,
|
|
321
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
322
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
323
|
+
WorldSet: () => WorldSet,
|
|
324
|
+
WorldTask: () => WorldTask,
|
|
325
|
+
apiRequest: () => apiRequest,
|
|
326
|
+
attachSession: () => attachSession,
|
|
327
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
328
|
+
detectProvenance: () => detectProvenance,
|
|
329
|
+
getRun: () => getRun,
|
|
330
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
331
|
+
listRunConfigs: () => listRunConfigs,
|
|
332
|
+
openSession: () => openSession,
|
|
333
|
+
openWorld: () => openWorld,
|
|
334
|
+
runSessions: () => runSessions,
|
|
335
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
336
|
+
setEnvSecret: () => setEnvSecret,
|
|
337
|
+
waitForRun: () => waitForRun
|
|
338
|
+
});
|
|
339
|
+
function resolveEnv(options) {
|
|
340
|
+
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
341
|
+
const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
342
|
+
const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
343
|
+
if (!publicKey || !secretKey) {
|
|
344
|
+
throw new Error(
|
|
345
|
+
"Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
|
|
346
|
+
);
|
|
347
|
+
}
|
|
348
|
+
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
349
|
+
}
|
|
350
|
+
async function apiRequest(options, method, path, body) {
|
|
351
|
+
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
352
|
+
const res = await fetch(`${host}${path}`, {
|
|
353
|
+
method,
|
|
354
|
+
headers: {
|
|
355
|
+
"content-type": "application/json",
|
|
356
|
+
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
357
|
+
},
|
|
358
|
+
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
359
|
+
});
|
|
360
|
+
const text = await res.text();
|
|
361
|
+
if (!res.ok) {
|
|
362
|
+
throw new Error(
|
|
363
|
+
`${method} ${path} -> ${res.status}: ${text.slice(0, 400)}`
|
|
364
|
+
);
|
|
365
|
+
}
|
|
366
|
+
return JSON.parse(text);
|
|
367
|
+
}
|
|
368
|
+
async function setEnvSecret(slug, key, value, options = {}) {
|
|
369
|
+
return apiRequest(
|
|
370
|
+
options,
|
|
371
|
+
"PUT",
|
|
372
|
+
"/api/public/benchmark-containers/secrets",
|
|
373
|
+
{
|
|
374
|
+
slug,
|
|
375
|
+
key,
|
|
376
|
+
value
|
|
377
|
+
}
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
async function listEnvSecretKeys(slug, options = {}) {
|
|
381
|
+
const res = await apiRequest(
|
|
382
|
+
options,
|
|
383
|
+
"GET",
|
|
384
|
+
`/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
|
|
385
|
+
);
|
|
386
|
+
return res.secrets;
|
|
387
|
+
}
|
|
388
|
+
async function deleteEnvSecret(slug, key, options = {}) {
|
|
389
|
+
await apiRequest(
|
|
390
|
+
options,
|
|
391
|
+
"DELETE",
|
|
392
|
+
"/api/public/benchmark-containers/secrets",
|
|
393
|
+
{
|
|
394
|
+
slug,
|
|
395
|
+
key
|
|
396
|
+
}
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
async function listRunConfigs(options = {}) {
|
|
400
|
+
const res = await apiRequest(
|
|
401
|
+
options,
|
|
402
|
+
"GET",
|
|
403
|
+
"/api/public/benchmark-containers/run-configs"
|
|
404
|
+
);
|
|
405
|
+
return res.runConfigs;
|
|
406
|
+
}
|
|
407
|
+
async function getRun(runId, options = {}) {
|
|
408
|
+
return apiRequest(
|
|
409
|
+
options,
|
|
410
|
+
"GET",
|
|
411
|
+
`/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
async function waitForRun(runId, opts = {}) {
|
|
415
|
+
const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
|
|
416
|
+
for (; ; ) {
|
|
417
|
+
const run = await getRun(runId, opts.options ?? {});
|
|
418
|
+
if (run.terminal) return run;
|
|
419
|
+
if (Date.now() > deadline)
|
|
420
|
+
throw new Error(
|
|
421
|
+
`Run ${runId} still ${run.status} after the wait timeout.`
|
|
422
|
+
);
|
|
423
|
+
opts.onTick?.(run);
|
|
424
|
+
await new Promise(
|
|
425
|
+
(resolve) => setTimeout(resolve, (opts.intervalSeconds ?? 20) * 1e3)
|
|
426
|
+
);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
function detectProvenance() {
|
|
430
|
+
const env = globalThis.process?.env ?? {};
|
|
431
|
+
const out = {};
|
|
432
|
+
const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
433
|
+
if (ref) out.branch = ref;
|
|
434
|
+
if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
|
|
435
|
+
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
436
|
+
if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
|
|
437
|
+
if (env.GITHUB_ACTOR) {
|
|
438
|
+
out.actor = {
|
|
439
|
+
kind: "ci",
|
|
440
|
+
id: env.GITHUB_ACTOR,
|
|
441
|
+
name: env.GITHUB_ACTOR,
|
|
442
|
+
// GitHub serves every account's avatar at /<login>.png.
|
|
443
|
+
imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
if (!out.branch || !out.commit) {
|
|
447
|
+
try {
|
|
448
|
+
const { execFileSync } = __require("child_process");
|
|
449
|
+
const git = (args) => execFileSync("git", args, {
|
|
450
|
+
encoding: "utf-8",
|
|
451
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
452
|
+
}).toString().trim();
|
|
453
|
+
out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
454
|
+
out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
|
|
455
|
+
if (!out.repoUrl) {
|
|
456
|
+
const remote = git(["config", "--get", "remote.origin.url"]);
|
|
457
|
+
if (remote) out.repoUrl = remote.replace(/\.git$/, "");
|
|
458
|
+
}
|
|
459
|
+
if (!out.actor) {
|
|
460
|
+
const name = git(["config", "--get", "user.name"]);
|
|
461
|
+
if (name) out.actor = { kind: "user", name };
|
|
462
|
+
}
|
|
463
|
+
} catch {
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
return out;
|
|
467
|
+
}
|
|
468
|
+
async function openWorld(slugRef, options = {}) {
|
|
469
|
+
const at = slugRef.indexOf("@");
|
|
470
|
+
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
471
|
+
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
472
|
+
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
473
|
+
slug,
|
|
474
|
+
...ref ? { ref } : {}
|
|
475
|
+
});
|
|
476
|
+
const world = new WorldSet(
|
|
477
|
+
slugRef,
|
|
478
|
+
slug,
|
|
479
|
+
ref,
|
|
480
|
+
{
|
|
481
|
+
versionId: version.versionId,
|
|
482
|
+
semanticVersion: version.semanticVersion ?? null,
|
|
483
|
+
contentHash: version.contentHash,
|
|
484
|
+
state: version.state
|
|
485
|
+
},
|
|
486
|
+
[],
|
|
487
|
+
options
|
|
488
|
+
);
|
|
489
|
+
for (const name of version.taskNames ?? []) {
|
|
490
|
+
const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
|
|
491
|
+
world.tasks.push(new WorldTask(world, name, variant));
|
|
492
|
+
}
|
|
493
|
+
return world;
|
|
494
|
+
}
|
|
495
|
+
var WorldTask, WorldSet;
|
|
496
|
+
var init_worlds = __esm({
|
|
497
|
+
"src/worlds/index.ts"() {
|
|
498
|
+
"use strict";
|
|
499
|
+
init_session();
|
|
500
|
+
WorldTask = class {
|
|
501
|
+
constructor(world, name, variant) {
|
|
502
|
+
this.world = world;
|
|
503
|
+
this.name = name;
|
|
504
|
+
this.variant = variant;
|
|
505
|
+
}
|
|
506
|
+
};
|
|
507
|
+
WorldSet = class {
|
|
508
|
+
constructor(slugRef, slug, ref, version, tasks, options) {
|
|
509
|
+
this.slugRef = slugRef;
|
|
510
|
+
this.slug = slug;
|
|
511
|
+
this.ref = ref;
|
|
512
|
+
this.version = version;
|
|
513
|
+
this.tasks = tasks;
|
|
514
|
+
this.options = options;
|
|
515
|
+
}
|
|
516
|
+
task(variant) {
|
|
517
|
+
const hit = this.tasks.find(
|
|
518
|
+
(t) => t.variant === variant || t.name.includes(variant)
|
|
519
|
+
);
|
|
520
|
+
if (!hit) {
|
|
521
|
+
const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
|
|
522
|
+
throw new Error(`no task matches ${variant}; have: ${known}`);
|
|
523
|
+
}
|
|
524
|
+
return hit;
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Open a LIVE session against one of this world's tasks: the platform boots
|
|
528
|
+
* (or reuses) a warm container, and your agent calls its tools through the
|
|
529
|
+
* platform. See worlds/session.ts.
|
|
530
|
+
*/
|
|
531
|
+
async open(task, opts = {}) {
|
|
532
|
+
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
533
|
+
return openSession2(this.slug, task, {
|
|
534
|
+
versionId: this.version.versionId,
|
|
535
|
+
reuseWarm: opts.reuseWarm,
|
|
536
|
+
options: this.options
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Start a HOSTED run of a run config against this world — the platform
|
|
541
|
+
* executes the (Python) world in parallel containers per the config's
|
|
542
|
+
* `parallelism` and files one graded run per model. Returns those runs.
|
|
543
|
+
*/
|
|
544
|
+
async dispatch(runConfigId, opts = {}) {
|
|
545
|
+
const provenance = opts.provenance ?? detectProvenance();
|
|
546
|
+
return apiRequest(
|
|
547
|
+
this.options,
|
|
548
|
+
"POST",
|
|
549
|
+
"/api/public/benchmark-containers/run-configs/dispatch",
|
|
550
|
+
{
|
|
551
|
+
runConfigId,
|
|
552
|
+
...opts.containerId ? { containerId: opts.containerId } : {},
|
|
553
|
+
versionId: opts.versionId ?? this.version.versionId,
|
|
554
|
+
...provenance
|
|
555
|
+
}
|
|
556
|
+
);
|
|
557
|
+
}
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
});
|
|
561
|
+
|
|
19
562
|
// package.json
|
|
20
563
|
var require_package = __commonJS({
|
|
21
564
|
"package.json"(exports, module) {
|
|
22
565
|
module.exports = {
|
|
23
566
|
name: "@withgateway/sdk",
|
|
24
|
-
version: "0.1.
|
|
567
|
+
version: "0.1.14",
|
|
25
568
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
26
569
|
main: "./dist/index.js",
|
|
27
570
|
module: "./dist/index.mjs",
|
|
@@ -76,6 +619,16 @@ var require_package = __commonJS({
|
|
|
76
619
|
types: "./dist/users/index.d.ts",
|
|
77
620
|
import: "./dist/users/index.mjs",
|
|
78
621
|
require: "./dist/users/index.js"
|
|
622
|
+
},
|
|
623
|
+
"./scores": {
|
|
624
|
+
types: "./dist/scores/index.d.ts",
|
|
625
|
+
import: "./dist/scores/index.mjs",
|
|
626
|
+
require: "./dist/scores/index.js"
|
|
627
|
+
},
|
|
628
|
+
"./worlds": {
|
|
629
|
+
types: "./dist/worlds/index.d.ts",
|
|
630
|
+
import: "./dist/worlds/index.mjs",
|
|
631
|
+
require: "./dist/worlds/index.js"
|
|
79
632
|
}
|
|
80
633
|
},
|
|
81
634
|
files: [
|
|
@@ -89,6 +642,7 @@ var require_package = __commonJS({
|
|
|
89
642
|
build: "tsup",
|
|
90
643
|
dev: "tsup --watch",
|
|
91
644
|
typecheck: "tsc --noEmit",
|
|
645
|
+
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
92
646
|
test: "vitest",
|
|
93
647
|
"test:run": "vitest run",
|
|
94
648
|
lint: "eslint src --ext .ts",
|
|
@@ -149,10 +703,29 @@ var require_package = __commonJS({
|
|
|
149
703
|
"@opentelemetry/sdk-trace-node": "^2.6.0",
|
|
150
704
|
"@opentelemetry/semantic-conventions": "^1.40.0",
|
|
151
705
|
"@types/node": "^20.0.0",
|
|
706
|
+
openai: "^4.70.0",
|
|
152
707
|
tsup: "^8.0.0",
|
|
153
708
|
tsx: "^4.0.0",
|
|
154
709
|
typescript: "^5.0.0",
|
|
155
|
-
vitest: "^2.
|
|
710
|
+
vitest: "^3.2.6"
|
|
711
|
+
},
|
|
712
|
+
overrides: {
|
|
713
|
+
protobufjs: "^8.4.1",
|
|
714
|
+
"vite@6": "^6.4.3",
|
|
715
|
+
postcss: "^8.5.18",
|
|
716
|
+
"picomatch@4": "^4.0.4"
|
|
717
|
+
},
|
|
718
|
+
pnpm: {
|
|
719
|
+
overrides: {
|
|
720
|
+
protobufjs: "^8.4.1",
|
|
721
|
+
vitest: "^3.2.6",
|
|
722
|
+
"vite@6": "^6.4.3",
|
|
723
|
+
postcss: "^8.5.18",
|
|
724
|
+
"picomatch@4": "^4.0.4"
|
|
725
|
+
}
|
|
726
|
+
},
|
|
727
|
+
bin: {
|
|
728
|
+
gateway: "./dist/cli.js"
|
|
156
729
|
}
|
|
157
730
|
};
|
|
158
731
|
}
|
|
@@ -1388,7 +1961,7 @@ var SessionRecording = class _SessionRecording {
|
|
|
1388
1961
|
* to re-drive the recorded conversation.
|
|
1389
1962
|
*/
|
|
1390
1963
|
userMessages() {
|
|
1391
|
-
return this.turns().map((
|
|
1964
|
+
return this.turns().map((trace5) => trace5.userMessage());
|
|
1392
1965
|
}
|
|
1393
1966
|
/** Output of the last chronological turn (null when empty). */
|
|
1394
1967
|
finalOutput() {
|
|
@@ -1406,8 +1979,8 @@ var SessionRecording = class _SessionRecording {
|
|
|
1406
1979
|
/** Filter observations across all turns (chronological order). */
|
|
1407
1980
|
select(filter = {}) {
|
|
1408
1981
|
const selected = [];
|
|
1409
|
-
for (const
|
|
1410
|
-
for (const observation of
|
|
1982
|
+
for (const trace5 of this.turns()) {
|
|
1983
|
+
for (const observation of trace5.observations) {
|
|
1411
1984
|
if (filter.type && (observation.type ?? "").toUpperCase() !== filter.type.toUpperCase()) {
|
|
1412
1985
|
continue;
|
|
1413
1986
|
}
|
|
@@ -1427,15 +2000,15 @@ var SessionRecording = class _SessionRecording {
|
|
|
1427
2000
|
sessionId: this.sessionId,
|
|
1428
2001
|
projectId: this.projectId,
|
|
1429
2002
|
environment: this.environment,
|
|
1430
|
-
traces: this.turns().map((
|
|
1431
|
-
id:
|
|
1432
|
-
name:
|
|
1433
|
-
timestamp:
|
|
1434
|
-
input:
|
|
1435
|
-
output:
|
|
1436
|
-
metadata:
|
|
1437
|
-
tags:
|
|
1438
|
-
observations:
|
|
2003
|
+
traces: this.turns().map((trace5) => ({
|
|
2004
|
+
id: trace5.id,
|
|
2005
|
+
name: trace5.name,
|
|
2006
|
+
timestamp: trace5.timestamp,
|
|
2007
|
+
input: trace5.input,
|
|
2008
|
+
output: trace5.output,
|
|
2009
|
+
metadata: trace5.metadata,
|
|
2010
|
+
tags: trace5.tags,
|
|
2011
|
+
observations: trace5.observations.map((o) => ({ ...o }))
|
|
1439
2012
|
}))
|
|
1440
2013
|
};
|
|
1441
2014
|
}
|
|
@@ -2746,6 +3319,220 @@ var users = {
|
|
|
2746
3319
|
fromEnv: UsersClient.fromEnv
|
|
2747
3320
|
};
|
|
2748
3321
|
|
|
3322
|
+
// src/scores/index.ts
|
|
3323
|
+
import { isSpanContextValid, trace } from "@opentelemetry/api";
|
|
3324
|
+
var SUCCESS_SIGNAL_KEY = "gateway.success.signal";
|
|
3325
|
+
var SUCCESS_METRIC_NAME_KEY = "gateway.success.metricName";
|
|
3326
|
+
var AGENT_ID_KEY = "gateway.agent.id";
|
|
3327
|
+
var ScoreError = class extends Error {
|
|
3328
|
+
constructor(message, options) {
|
|
3329
|
+
super(message);
|
|
3330
|
+
this.name = "ScoreError";
|
|
3331
|
+
this.cause = options?.cause;
|
|
3332
|
+
}
|
|
3333
|
+
};
|
|
3334
|
+
function currentTraceId() {
|
|
3335
|
+
try {
|
|
3336
|
+
const span = trace.getActiveSpan();
|
|
3337
|
+
if (!span) return void 0;
|
|
3338
|
+
const ctx = span.spanContext();
|
|
3339
|
+
if (!ctx || !isSpanContextValid(ctx)) return void 0;
|
|
3340
|
+
return ctx.traceId;
|
|
3341
|
+
} catch {
|
|
3342
|
+
return void 0;
|
|
3343
|
+
}
|
|
3344
|
+
}
|
|
3345
|
+
var ScoresClient = class _ScoresClient {
|
|
3346
|
+
host;
|
|
3347
|
+
publicKey;
|
|
3348
|
+
secretKey;
|
|
3349
|
+
timeoutSeconds;
|
|
3350
|
+
debug;
|
|
3351
|
+
constructor(config) {
|
|
3352
|
+
this.host = config.host.replace(/\/$/, "");
|
|
3353
|
+
this.publicKey = config.publicKey;
|
|
3354
|
+
this.secretKey = config.secretKey;
|
|
3355
|
+
this.timeoutSeconds = config.timeoutSeconds ?? 15;
|
|
3356
|
+
this.debug = config.debug ?? false;
|
|
3357
|
+
}
|
|
3358
|
+
/** Create a client from GATEWAY_HOST / GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY. */
|
|
3359
|
+
static fromEnv(config = {}) {
|
|
3360
|
+
const host = config.host ?? process.env.GATEWAY_HOST;
|
|
3361
|
+
const publicKey = config.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
3362
|
+
const secretKey = config.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
3363
|
+
const missing = [
|
|
3364
|
+
["host", host],
|
|
3365
|
+
["publicKey", publicKey],
|
|
3366
|
+
["secretKey", secretKey]
|
|
3367
|
+
].filter(([, value]) => !value).map(([name]) => name);
|
|
3368
|
+
if (missing.length > 0) {
|
|
3369
|
+
throw new ScoreError(
|
|
3370
|
+
`Missing Gateway scores configuration: ${missing.join(", ")}`
|
|
3371
|
+
);
|
|
3372
|
+
}
|
|
3373
|
+
return new _ScoresClient({
|
|
3374
|
+
...config,
|
|
3375
|
+
host,
|
|
3376
|
+
publicKey,
|
|
3377
|
+
secretKey
|
|
3378
|
+
});
|
|
3379
|
+
}
|
|
3380
|
+
/**
|
|
3381
|
+
* Post a score, throwing {@link ScoreError} on any failure. Builds the exact
|
|
3382
|
+
* `POST /api/public/scores` body the Python SDK sends: exactly one of
|
|
3383
|
+
* `sessionId` / `traceId` (+ optional `observationId`) / `datasetRunId`.
|
|
3384
|
+
* When no target is provided, the active OTel trace id is used; if none is
|
|
3385
|
+
* available the call is a no-op (returns without posting), matching Python.
|
|
3386
|
+
*/
|
|
3387
|
+
async postScore(name, value, options = {}) {
|
|
3388
|
+
const body = this.buildScoreBody(name, value, options);
|
|
3389
|
+
if (!body) {
|
|
3390
|
+
this.log(
|
|
3391
|
+
"score '%s' dropped: no sessionId, traceId or datasetRunId available",
|
|
3392
|
+
name
|
|
3393
|
+
);
|
|
3394
|
+
return;
|
|
3395
|
+
}
|
|
3396
|
+
await this.request("POST", "/api/public/scores", body);
|
|
3397
|
+
}
|
|
3398
|
+
/**
|
|
3399
|
+
* Build the score request body, or null when no target can be resolved.
|
|
3400
|
+
* Split out so it is unit-testable and shared by score()/success().
|
|
3401
|
+
*/
|
|
3402
|
+
buildScoreBody(name, value, options = {}) {
|
|
3403
|
+
const traceId = options.traceId ?? currentTraceId();
|
|
3404
|
+
const target = {};
|
|
3405
|
+
if (options.sessionId) {
|
|
3406
|
+
target.sessionId = options.sessionId;
|
|
3407
|
+
} else if (options.datasetRunId) {
|
|
3408
|
+
target.datasetRunId = options.datasetRunId;
|
|
3409
|
+
} else if (traceId) {
|
|
3410
|
+
target.traceId = traceId;
|
|
3411
|
+
if (options.observationId) target.observationId = options.observationId;
|
|
3412
|
+
} else {
|
|
3413
|
+
return null;
|
|
3414
|
+
}
|
|
3415
|
+
const metadata = {
|
|
3416
|
+
source: "gatewaysdk",
|
|
3417
|
+
...options.metadata ?? {}
|
|
3418
|
+
};
|
|
3419
|
+
return {
|
|
3420
|
+
name,
|
|
3421
|
+
value,
|
|
3422
|
+
dataType: options.dataType ?? "NUMERIC",
|
|
3423
|
+
environment: options.environment ?? "default",
|
|
3424
|
+
...options.comment ? { comment: options.comment } : {},
|
|
3425
|
+
metadata,
|
|
3426
|
+
...target
|
|
3427
|
+
};
|
|
3428
|
+
}
|
|
3429
|
+
/**
|
|
3430
|
+
* Post a score, FAIL-SOFT: warns and resolves on any failure so a broken
|
|
3431
|
+
* transport never throws into (or rejects out of) the caller.
|
|
3432
|
+
*/
|
|
3433
|
+
async score(name, value, options = {}) {
|
|
3434
|
+
try {
|
|
3435
|
+
await this.postScore(name, value, options);
|
|
3436
|
+
} catch (error) {
|
|
3437
|
+
this.warn(`score '${name}' failed`, error);
|
|
3438
|
+
}
|
|
3439
|
+
}
|
|
3440
|
+
/**
|
|
3441
|
+
* Emit a SUCCESS SIGNAL that feeds a Gateway Success Metric — a score tagged
|
|
3442
|
+
* with `gateway.success.signal` + `gateway.success.metricName` (and the agent
|
|
3443
|
+
* identity when `agentId` is set). A boolean `value` records a 1/0 BOOLEAN
|
|
3444
|
+
* score (pass/fail); a number records a NUMERIC score. FAIL-SOFT.
|
|
3445
|
+
*
|
|
3446
|
+
* The metadata shape matches the Python SDK's `Run.success()` exactly, so the
|
|
3447
|
+
* platform's Success Metrics aggregation counts signals from either SDK
|
|
3448
|
+
* without change.
|
|
3449
|
+
*
|
|
3450
|
+
* @example
|
|
3451
|
+
* ```ts
|
|
3452
|
+
* await scores.success("resolved_ticket", true, { agentId: "support-triage", sessionId });
|
|
3453
|
+
* await scores.success("handle_time_s", 42, { agentId: "support-triage", sessionId });
|
|
3454
|
+
* ```
|
|
3455
|
+
*/
|
|
3456
|
+
async success(name, value = true, options = {}) {
|
|
3457
|
+
const { numericValue, inferredType } = coerceSignalValue(value);
|
|
3458
|
+
const { agentId, metricName, metadata, dataType, ...rest } = options;
|
|
3459
|
+
const signalMetadata = {
|
|
3460
|
+
...metadata ?? {},
|
|
3461
|
+
[SUCCESS_SIGNAL_KEY]: true,
|
|
3462
|
+
[SUCCESS_METRIC_NAME_KEY]: metricName ?? name,
|
|
3463
|
+
...agentId ? { [AGENT_ID_KEY]: agentId } : {}
|
|
3464
|
+
};
|
|
3465
|
+
await this.score(name, numericValue, {
|
|
3466
|
+
...rest,
|
|
3467
|
+
dataType: dataType ?? inferredType,
|
|
3468
|
+
metadata: signalMetadata
|
|
3469
|
+
});
|
|
3470
|
+
}
|
|
3471
|
+
async request(method, path, payload) {
|
|
3472
|
+
let response;
|
|
3473
|
+
try {
|
|
3474
|
+
response = await fetch(`${this.host}${path}`, {
|
|
3475
|
+
method,
|
|
3476
|
+
headers: {
|
|
3477
|
+
Authorization: this.authHeader(),
|
|
3478
|
+
"Content-Type": "application/json"
|
|
3479
|
+
},
|
|
3480
|
+
body: JSON.stringify(payload),
|
|
3481
|
+
signal: AbortSignal.timeout(this.timeoutSeconds * 1e3)
|
|
3482
|
+
});
|
|
3483
|
+
} catch (error) {
|
|
3484
|
+
throw new ScoreError(
|
|
3485
|
+
`${method} ${path} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
3486
|
+
{ cause: error }
|
|
3487
|
+
);
|
|
3488
|
+
}
|
|
3489
|
+
const text = await response.text();
|
|
3490
|
+
if (!response.ok) {
|
|
3491
|
+
throw new ScoreError(
|
|
3492
|
+
`${method} ${path} failed (${response.status}): ${text.slice(0, 500)}`
|
|
3493
|
+
);
|
|
3494
|
+
}
|
|
3495
|
+
this.log("%s %s -> %s", method, path, response.status);
|
|
3496
|
+
}
|
|
3497
|
+
authHeader() {
|
|
3498
|
+
return `Basic ${Buffer.from(`${this.publicKey}:${this.secretKey}`).toString(
|
|
3499
|
+
"base64"
|
|
3500
|
+
)}`;
|
|
3501
|
+
}
|
|
3502
|
+
log(message, ...args) {
|
|
3503
|
+
if (this.debug) {
|
|
3504
|
+
console.log(`[gateway.scores] ${message}`, ...args);
|
|
3505
|
+
}
|
|
3506
|
+
}
|
|
3507
|
+
warn(message, ...args) {
|
|
3508
|
+
console.warn(`[gateway.scores] ${message}`, ...args);
|
|
3509
|
+
}
|
|
3510
|
+
};
|
|
3511
|
+
function coerceSignalValue(value) {
|
|
3512
|
+
if (typeof value === "boolean") {
|
|
3513
|
+
return { numericValue: value ? 1 : 0, inferredType: "BOOLEAN" };
|
|
3514
|
+
}
|
|
3515
|
+
return { numericValue: value, inferredType: "NUMERIC" };
|
|
3516
|
+
}
|
|
3517
|
+
async function success(name, value = true, options = {}) {
|
|
3518
|
+
let client2;
|
|
3519
|
+
try {
|
|
3520
|
+
client2 = ScoresClient.fromEnv();
|
|
3521
|
+
} catch (error) {
|
|
3522
|
+
console.warn(`[gateway.scores] success '${name}' skipped`, error);
|
|
3523
|
+
return;
|
|
3524
|
+
}
|
|
3525
|
+
await client2.success(name, value, options);
|
|
3526
|
+
}
|
|
3527
|
+
var scores = {
|
|
3528
|
+
fromEnv: ScoresClient.fromEnv,
|
|
3529
|
+
success
|
|
3530
|
+
};
|
|
3531
|
+
|
|
3532
|
+
// src/index.ts
|
|
3533
|
+
init_worlds();
|
|
3534
|
+
init_session();
|
|
3535
|
+
|
|
2749
3536
|
// src/build/index.ts
|
|
2750
3537
|
import { createHash as createHash2 } from "crypto";
|
|
2751
3538
|
function canonicalJson(value) {
|
|
@@ -3029,7 +3816,7 @@ async function registerBuild(params) {
|
|
|
3029
3816
|
|
|
3030
3817
|
// src/tracing/identity.ts
|
|
3031
3818
|
import { AsyncLocalStorage as AsyncLocalStorage2 } from "async_hooks";
|
|
3032
|
-
import { isSpanContextValid, trace } from "@opentelemetry/api";
|
|
3819
|
+
import { isSpanContextValid as isSpanContextValid2, trace as trace2 } from "@opentelemetry/api";
|
|
3033
3820
|
var USER_ID_ATTRIBUTE = "gateway.user.id";
|
|
3034
3821
|
var currentUserStorage = new AsyncLocalStorage2();
|
|
3035
3822
|
function resolveUserId() {
|
|
@@ -3043,8 +3830,8 @@ function identify(userId, options = {}) {
|
|
|
3043
3830
|
if (!cleaned) return;
|
|
3044
3831
|
currentUserStorage.enterWith(cleaned);
|
|
3045
3832
|
try {
|
|
3046
|
-
const span =
|
|
3047
|
-
if (span &&
|
|
3833
|
+
const span = trace2.getActiveSpan();
|
|
3834
|
+
if (span && isSpanContextValid2(span.spanContext())) {
|
|
3048
3835
|
span.setAttribute(USER_ID_ATTRIBUTE, cleaned);
|
|
3049
3836
|
}
|
|
3050
3837
|
} catch (error) {
|
|
@@ -3322,12 +4109,19 @@ function initIsolated(config = {}) {
|
|
|
3322
4109
|
const provider = built.provider;
|
|
3323
4110
|
scheduleBuildRegistration(config, built);
|
|
3324
4111
|
const tracer = provider.getTracer("@withgateway/sdk");
|
|
4112
|
+
const scores2 = new ScoresClient({
|
|
4113
|
+
host: built.connection.host,
|
|
4114
|
+
publicKey: built.connection.publicKey,
|
|
4115
|
+
secretKey: built.connection.secretKey,
|
|
4116
|
+
debug: config.debug ?? false
|
|
4117
|
+
});
|
|
3325
4118
|
debugLog2(
|
|
3326
4119
|
"Isolated tracing provider created (service=%s)",
|
|
3327
4120
|
config.serviceName ?? "gatewaysdk-tracing"
|
|
3328
4121
|
);
|
|
3329
4122
|
return {
|
|
3330
4123
|
tracer,
|
|
4124
|
+
scores: scores2,
|
|
3331
4125
|
async flush() {
|
|
3332
4126
|
try {
|
|
3333
4127
|
await provider.forceFlush();
|
|
@@ -3381,12 +4175,12 @@ function isGatewayConfigured() {
|
|
|
3381
4175
|
}
|
|
3382
4176
|
|
|
3383
4177
|
// src/tracing/observe.ts
|
|
3384
|
-
import { trace as
|
|
4178
|
+
import { trace as trace3, context as context2, SpanStatusCode } from "@opentelemetry/api";
|
|
3385
4179
|
function asAttr(value) {
|
|
3386
4180
|
return typeof value === "string" ? value : JSON.stringify(value);
|
|
3387
4181
|
}
|
|
3388
4182
|
async function observe(name, fn, options = {}) {
|
|
3389
|
-
const tracer = options.tracer ??
|
|
4183
|
+
const tracer = options.tracer ?? trace3.getTracer("@withgateway/sdk");
|
|
3390
4184
|
const span = tracer.startSpan(name, {
|
|
3391
4185
|
attributes: {
|
|
3392
4186
|
"openinference.span.kind": options.kind ?? "AGENT",
|
|
@@ -3396,7 +4190,7 @@ async function observe(name, fn, options = {}) {
|
|
|
3396
4190
|
});
|
|
3397
4191
|
try {
|
|
3398
4192
|
const result = await context2.with(
|
|
3399
|
-
|
|
4193
|
+
trace3.setSpan(context2.active(), span),
|
|
3400
4194
|
fn
|
|
3401
4195
|
);
|
|
3402
4196
|
if (options.captureOutput !== false && result !== void 0) {
|
|
@@ -3454,7 +4248,7 @@ function observeAnthropic(client2) {
|
|
|
3454
4248
|
|
|
3455
4249
|
// src/tracing/claude.ts
|
|
3456
4250
|
import {
|
|
3457
|
-
trace as
|
|
4251
|
+
trace as trace4,
|
|
3458
4252
|
context as context3,
|
|
3459
4253
|
SpanStatusCode as SpanStatusCode2
|
|
3460
4254
|
} from "@opentelemetry/api";
|
|
@@ -3471,7 +4265,7 @@ function toolResultText(content) {
|
|
|
3471
4265
|
}
|
|
3472
4266
|
var TURN_INPUT_MAX = 4e3;
|
|
3473
4267
|
async function* observeClaude(stream, options) {
|
|
3474
|
-
const tracer = options.tracer ??
|
|
4268
|
+
const tracer = options.tracer ?? trace4.getTracer("@withgateway/sdk");
|
|
3475
4269
|
const model = options.model ?? "claude";
|
|
3476
4270
|
const openTools = /* @__PURE__ */ new Map();
|
|
3477
4271
|
let root = null;
|
|
@@ -3484,7 +4278,7 @@ async function* observeClaude(stream, options) {
|
|
|
3484
4278
|
...options.attributes ?? {}
|
|
3485
4279
|
}
|
|
3486
4280
|
});
|
|
3487
|
-
rootCtx =
|
|
4281
|
+
rootCtx = trace4.setSpan(context3.active(), root);
|
|
3488
4282
|
}
|
|
3489
4283
|
let turnRoot = null;
|
|
3490
4284
|
let turnCtx = null;
|
|
@@ -3517,7 +4311,7 @@ async function* observeClaude(stream, options) {
|
|
|
3517
4311
|
...options.attributes ?? {}
|
|
3518
4312
|
}
|
|
3519
4313
|
});
|
|
3520
|
-
turnCtx =
|
|
4314
|
+
turnCtx = trace4.setSpan(context3.active(), turnRoot);
|
|
3521
4315
|
pendingTurnInput = "";
|
|
3522
4316
|
}
|
|
3523
4317
|
const childCtx = options.perTurn ? turnCtx : rootCtx;
|
|
@@ -3623,6 +4417,7 @@ var tracing = {
|
|
|
3623
4417
|
}
|
|
3624
4418
|
};
|
|
3625
4419
|
export {
|
|
4420
|
+
AGENT_ID_KEY,
|
|
3626
4421
|
DEFAULT_TIMEOUT_SECONDS,
|
|
3627
4422
|
DriftError,
|
|
3628
4423
|
ENV_GATEWAY_HOST,
|
|
@@ -3652,6 +4447,10 @@ export {
|
|
|
3652
4447
|
ReplayReport,
|
|
3653
4448
|
ReplaySuite,
|
|
3654
4449
|
ReplayTask,
|
|
4450
|
+
SUCCESS_METRIC_NAME_KEY,
|
|
4451
|
+
SUCCESS_SIGNAL_KEY,
|
|
4452
|
+
ScoreError,
|
|
4453
|
+
ScoresClient,
|
|
3655
4454
|
SessionLoadError,
|
|
3656
4455
|
SessionRecording,
|
|
3657
4456
|
SessionTrace,
|
|
@@ -3660,16 +4459,26 @@ export {
|
|
|
3660
4459
|
TraceDiff,
|
|
3661
4460
|
UserProfileError,
|
|
3662
4461
|
UsersClient,
|
|
4462
|
+
WorldSession,
|
|
4463
|
+
WorldSessionCallError,
|
|
4464
|
+
WorldSessionTimeout,
|
|
4465
|
+
WorldSet,
|
|
4466
|
+
WorldTask,
|
|
3663
4467
|
addExperimentAttributes,
|
|
4468
|
+
apiRequest,
|
|
3664
4469
|
assembleManifest,
|
|
3665
4470
|
assignVariant,
|
|
4471
|
+
attachSession,
|
|
3666
4472
|
basicAuthHeader,
|
|
3667
4473
|
canonicalJson,
|
|
3668
4474
|
clearExperimentContext,
|
|
3669
4475
|
computeBuildHash,
|
|
3670
4476
|
computeHash,
|
|
3671
4477
|
context,
|
|
4478
|
+
currentTraceId,
|
|
3672
4479
|
defineTool,
|
|
4480
|
+
deleteEnvSecret,
|
|
4481
|
+
detectProvenance,
|
|
3673
4482
|
experimentContextStorage,
|
|
3674
4483
|
singleton_exports as experiments,
|
|
3675
4484
|
gateway,
|
|
@@ -3678,9 +4487,14 @@ export {
|
|
|
3678
4487
|
getRegisteredPrompts,
|
|
3679
4488
|
getRegisteredSkills,
|
|
3680
4489
|
getRegisteredTools,
|
|
4490
|
+
getRun,
|
|
3681
4491
|
identify,
|
|
4492
|
+
listEnvSecretKeys,
|
|
4493
|
+
listRunConfigs,
|
|
3682
4494
|
loadSession,
|
|
3683
4495
|
memory,
|
|
4496
|
+
openSession,
|
|
4497
|
+
openWorld,
|
|
3684
4498
|
personas,
|
|
3685
4499
|
registerArtifact,
|
|
3686
4500
|
registerPrompt,
|
|
@@ -3689,13 +4503,19 @@ export {
|
|
|
3689
4503
|
replay,
|
|
3690
4504
|
resetBuildRegistry,
|
|
3691
4505
|
resolveUserId,
|
|
4506
|
+
runSessions,
|
|
3692
4507
|
runSuite,
|
|
3693
4508
|
runWithExperimentContext,
|
|
4509
|
+
schemaFromSignature,
|
|
4510
|
+
scores,
|
|
3694
4511
|
sessionObservationFromApi,
|
|
3695
4512
|
sessions,
|
|
4513
|
+
setEnvSecret,
|
|
3696
4514
|
setExperimentContext,
|
|
4515
|
+
success,
|
|
3697
4516
|
suite,
|
|
3698
4517
|
tracing,
|
|
3699
|
-
users
|
|
4518
|
+
users,
|
|
4519
|
+
waitForRun
|
|
3700
4520
|
};
|
|
3701
4521
|
//# sourceMappingURL=index.mjs.map
|