@withgateway/sdk 0.1.19 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -2
- package/dist/cli.js +8677 -1320
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +8683 -1327
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1005 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +969 -47
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +9 -2
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +9 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +738 -28
- package/dist/worlds/index.d.ts +738 -28
- package/dist/worlds/index.js +966 -51
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +960 -45
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +9 -2
- package/runtime/gatewaysdk/__init__.py +0 -0
- package/runtime/gatewaysdk/environment/__init__.py +108 -0
- package/runtime/gatewaysdk/environment/_bundle.py +281 -0
- package/runtime/gatewaysdk/environment/_harbor.py +276 -0
- package/runtime/gatewaysdk/environment/_materialize.py +97 -0
- package/runtime/gatewaysdk/environment/_tar.py +33 -0
- package/runtime/gatewaysdk/environment/_world.py +346 -0
- package/runtime/gatewaysdk/environment/core.py +527 -0
- package/runtime/gatewaysdk/environment/errors.py +58 -0
- package/runtime/gatewaysdk/environment/runtime.py +150 -0
- package/runtime/gatewaysdk/environment/schema/__init__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +225 -0
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +116 -0
- package/runtime/gatewaysdk/environment/schema/_slack_scenario.py +183 -0
- package/runtime/gatewaysdk/environment/schema/api.py +1628 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +755 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +615 -0
- package/runtime/gatewaysdk/environment/schema/conform.py +428 -0
- package/runtime/gatewaysdk/environment/schema/connector.py +777 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/handlers.py +17 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/parity.json +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/provenance.json +233 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/scope.toml +112 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/world.json +1380 -0
- package/runtime/gatewaysdk/environment/schema/connectors/base.json +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +18 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/world.json +123 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/handlers.py +81 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/parity.json +37 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/provenance.json +134 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/scope.toml +158 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +5869 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-calendar/handlers.py +23 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-calendar/world.json +209 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/handlers.py +18 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/world.json +286 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/handlers.py +34 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/parity.json +49 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +181 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +128 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/world.json +2218 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jira/handlers.py +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jira/world.json +389 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/handlers.py +38 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/parity.json +29 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/provenance.json +152 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/scope.toml +120 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/world.json +765 -0
- package/runtime/gatewaysdk/environment/schema/connectors/linear/handlers.py +20 -0
- package/runtime/gatewaysdk/environment/schema/connectors/linear/world.json +363 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/fidelity.json +45 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/handlers.py +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/provenance.json +851 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/scope.md +231 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/teams-types.json +1710 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/world.json +858 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/handlers.py +32 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/parity.json +50 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/provenance.json +137 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/scope.toml +78 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/world.json +16243 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/handlers.py +806 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/parity.json +46 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/provenance.json +143 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/scope.toml +174 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/world.json +4695 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/MODEL.md +236 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/capabilities.json +1039 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/handlers.py +762 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/parity.json +65 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/provenance.json +294 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/scope.toml +44 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/world.json +4993 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/handlers.py +75 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/parity.json +36 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/provenance.json +179 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/scope.toml +180 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/world.json +806 -0
- package/runtime/gatewaysdk/environment/schema/data.py +593 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +263 -0
- package/runtime/gatewaysdk/environment/schema/host.py +2407 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +275 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +1272 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +250 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +118 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +202 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +331 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +313 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +223 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +293 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +390 -0
- package/runtime/gatewaysdk/environment/schema/slack.py +757 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +479 -0
- package/runtime/gatewaysdk/environment/schema/store.py +904 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +525 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +322 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +577 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +78 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +500 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +336 -0
- package/skills/agents/connector-template-author.md +202 -0
- package/skills/connector-schema-authoring/CONNECTORS.md +331 -0
- package/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
- package/skills/connector-schema-authoring/SKILL.md +313 -0
- package/skills/world-data-ingestion/ROWS.md +223 -0
- package/skills/world-data-ingestion/SKILL.md +293 -0
- package/skills/worlds-getting-started/SKILL.md +390 -0
package/dist/worlds/index.js
CHANGED
|
@@ -30,6 +30,483 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
30
30
|
));
|
|
31
31
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
32
32
|
|
|
33
|
+
// src/worlds/browser.ts
|
|
34
|
+
function surfaceKinds(s) {
|
|
35
|
+
const kinds = ["tools"];
|
|
36
|
+
if (s.api) kinds.push("api");
|
|
37
|
+
if (s.ui) kinds.push("ui");
|
|
38
|
+
if (s.browser) kinds.push("browser");
|
|
39
|
+
return kinds;
|
|
40
|
+
}
|
|
41
|
+
function uiUrl(ui, path2) {
|
|
42
|
+
const base = ui.url.replace(/\/+$/, "");
|
|
43
|
+
const rel = path2.startsWith("/") ? path2 : `/${path2}`;
|
|
44
|
+
if (!ui.token) return base + rel;
|
|
45
|
+
const [beforeHash, hash] = splitHash(rel);
|
|
46
|
+
const joiner = beforeHash.includes("?") ? "&" : "?";
|
|
47
|
+
const query = `${UI_TOKEN_QUERY}=${encodeURIComponent(ui.token)}`;
|
|
48
|
+
return base + beforeHash + joiner + query + hash;
|
|
49
|
+
}
|
|
50
|
+
function uiEntryUrl(ui) {
|
|
51
|
+
return uiUrl(ui, ui.entry || "/");
|
|
52
|
+
}
|
|
53
|
+
function withoutUiToken(url) {
|
|
54
|
+
const [beforeHash, hash] = splitHash(url);
|
|
55
|
+
const q = beforeHash.indexOf("?");
|
|
56
|
+
if (q < 0) return url;
|
|
57
|
+
const kept = beforeHash.slice(q + 1).split("&").filter((pair) => pair !== "" && pair.split("=")[0] !== UI_TOKEN_QUERY);
|
|
58
|
+
return beforeHash.slice(0, q) + (kept.length ? `?${kept.join("&")}` : "") + hash;
|
|
59
|
+
}
|
|
60
|
+
function splitHash(url) {
|
|
61
|
+
const h = url.indexOf("#");
|
|
62
|
+
return h < 0 ? [url, ""] : [url.slice(0, h), url.slice(h)];
|
|
63
|
+
}
|
|
64
|
+
function redactActionArgs(action, args) {
|
|
65
|
+
const secretField = action === "browser_type" && SECRET_KEY.test(String(args.selector ?? ""));
|
|
66
|
+
const out = {};
|
|
67
|
+
for (const [key, value] of Object.entries(args)) {
|
|
68
|
+
if (value === null || value === void 0) out[key] = null;
|
|
69
|
+
else if (typeof value === "boolean" || typeof value === "number")
|
|
70
|
+
out[key] = value;
|
|
71
|
+
else if (typeof value === "string") {
|
|
72
|
+
if (SECRET_KEY.test(key) || SECRET_VALUE.test(value) || secretField && key === "text")
|
|
73
|
+
out[key] = "[redacted]";
|
|
74
|
+
else
|
|
75
|
+
out[key] = value.length <= EVENT_MAX_STRING ? value : value.slice(0, EVENT_MAX_STRING) + "\u2026";
|
|
76
|
+
} else out[key] = "[omitted]";
|
|
77
|
+
}
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
async function loadPlaywright() {
|
|
81
|
+
try {
|
|
82
|
+
const name = "playwright";
|
|
83
|
+
return await import(name);
|
|
84
|
+
} catch (error) {
|
|
85
|
+
throw new Error(
|
|
86
|
+
"Driving a world UI needs Playwright: npm install playwright (and `npx playwright install chromium` for local mode).",
|
|
87
|
+
{ cause: error }
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function resolveBrowserMode(surfaces, mode) {
|
|
92
|
+
if (!surfaces.ui && !surfaces.browser)
|
|
93
|
+
throw new WorldSurfaceUnavailable(
|
|
94
|
+
"ui",
|
|
95
|
+
"open the session with surfaces: ['ui'] on a world that declares [ui]"
|
|
96
|
+
);
|
|
97
|
+
const resolved = mode === "auto" ? surfaces.browser ? "hosted" : "local" : mode;
|
|
98
|
+
if (resolved === "hosted" && !surfaces.browser)
|
|
99
|
+
throw new WorldSurfaceUnavailable(
|
|
100
|
+
"browser",
|
|
101
|
+
"open the session with surfaces: ['browser'] to get a hosted browser"
|
|
102
|
+
);
|
|
103
|
+
return resolved;
|
|
104
|
+
}
|
|
105
|
+
async function openBrowser(surfaces, opts = {}) {
|
|
106
|
+
const mode = resolveBrowserMode(surfaces, opts.mode ?? "auto");
|
|
107
|
+
const pw = opts.playwright ?? await loadPlaywright();
|
|
108
|
+
const viewport = surfaces.browser?.viewport ?? DEFAULT_VIEWPORT;
|
|
109
|
+
const browser = mode === "hosted" ? surfaces.browser.protocol === "playwright" ? await pw.chromium.connect(surfaces.browser.wsUrl) : await pw.chromium.connectOverCDP(surfaces.browser.wsUrl) : await pw.chromium.launch({ headless: opts.headless ?? true });
|
|
110
|
+
const context = await browser.newContext({ viewport });
|
|
111
|
+
const page = await context.newPage();
|
|
112
|
+
return new WorldBrowser(
|
|
113
|
+
page,
|
|
114
|
+
surfaces.ui,
|
|
115
|
+
mode,
|
|
116
|
+
async () => {
|
|
117
|
+
await context.close();
|
|
118
|
+
await browser.close();
|
|
119
|
+
},
|
|
120
|
+
viewport,
|
|
121
|
+
opts.recorder ?? null
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
function computerUseToolkit(browser, opts = {}) {
|
|
125
|
+
const maxText = opts.maxText ?? 2e4;
|
|
126
|
+
const observe = async (selector = "body") => {
|
|
127
|
+
const text = await browser.text(selector);
|
|
128
|
+
return {
|
|
129
|
+
url: browser.url,
|
|
130
|
+
title: await browser.title(),
|
|
131
|
+
text: text.slice(0, maxText),
|
|
132
|
+
truncated: text.length > maxText
|
|
133
|
+
};
|
|
134
|
+
};
|
|
135
|
+
const impls = {
|
|
136
|
+
browser_goto: async (a) => {
|
|
137
|
+
await browser.goto(typeof a.path === "string" ? a.path : void 0);
|
|
138
|
+
return observe();
|
|
139
|
+
},
|
|
140
|
+
browser_click: async (a) => {
|
|
141
|
+
await browser.click(str(a.selector, ""));
|
|
142
|
+
return observe();
|
|
143
|
+
},
|
|
144
|
+
browser_type: async (a) => {
|
|
145
|
+
await browser.type(str(a.selector, ""), String(a.text ?? ""));
|
|
146
|
+
return { ok: true, url: browser.url };
|
|
147
|
+
},
|
|
148
|
+
browser_press: async (a) => {
|
|
149
|
+
await browser.press(str(a.key, "Enter"));
|
|
150
|
+
return observe();
|
|
151
|
+
},
|
|
152
|
+
browser_scroll: async (a) => {
|
|
153
|
+
await browser.scroll(typeof a.dy === "number" ? a.dy : 600);
|
|
154
|
+
return { ok: true, url: browser.url };
|
|
155
|
+
},
|
|
156
|
+
browser_screenshot: async (a) => ({
|
|
157
|
+
image_base64: await browser.screenshotBase64(a.full_page === true),
|
|
158
|
+
url: browser.url
|
|
159
|
+
}),
|
|
160
|
+
browser_read: async (a) => observe(str(a.selector, "body")),
|
|
161
|
+
browser_back: async () => {
|
|
162
|
+
await browser.back();
|
|
163
|
+
return observe();
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
const specs = COMPUTER_USE_SPECS.map((s) => ({ ...s }));
|
|
167
|
+
return {
|
|
168
|
+
schemas: specs.map((spec) => schemaFromSignature(spec)),
|
|
169
|
+
impls,
|
|
170
|
+
specs
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
var SURFACE_KINDS, DEFAULT_VIEWPORT, SETTLE_TIMEOUT_MS, SETTLE_QUIET_MS, SETTLE_POLL_MS, EVENT_BATCH_SIZE, EVENT_FLUSH_INTERVAL_MS, EVENT_MAX_RETRIES, EVENT_SCREENSHOT_QUALITY, EVENT_MAX_STRING, SECRET_KEY, SECRET_VALUE, WorldSurfaceUnavailable, UI_TOKEN_QUERY, BrowserActionRecorder, WorldBrowser, COMPUTER_USE_SPECS, str;
|
|
174
|
+
var init_browser = __esm({
|
|
175
|
+
"src/worlds/browser.ts"() {
|
|
176
|
+
"use strict";
|
|
177
|
+
init_session();
|
|
178
|
+
SURFACE_KINDS = ["tools", "api", "ui", "browser"];
|
|
179
|
+
DEFAULT_VIEWPORT = { width: 1280, height: 800 };
|
|
180
|
+
SETTLE_TIMEOUT_MS = 5e3;
|
|
181
|
+
SETTLE_QUIET_MS = 150;
|
|
182
|
+
SETTLE_POLL_MS = 25;
|
|
183
|
+
EVENT_BATCH_SIZE = 10;
|
|
184
|
+
EVENT_FLUSH_INTERVAL_MS = 1e3;
|
|
185
|
+
EVENT_MAX_RETRIES = 3;
|
|
186
|
+
EVENT_SCREENSHOT_QUALITY = 60;
|
|
187
|
+
EVENT_MAX_STRING = 500;
|
|
188
|
+
SECRET_KEY = /pass(word|wd)?|secret|token|api[-_]?key|authorization|cookie|(^|[-_ #.\["'=])(otp|pin|cvv|ssn)([-_ \]"']|$)/i;
|
|
189
|
+
SECRET_VALUE = /^((sk|pk|ghp|xox[abp])[-_][-_A-Za-z0-9]{8,}|AKIA[0-9A-Z]{12,}|Bearer [-_.A-Za-z0-9]{8,})/;
|
|
190
|
+
WorldSurfaceUnavailable = class extends Error {
|
|
191
|
+
constructor(surface, reason) {
|
|
192
|
+
super(`World surface '${surface}' is unavailable: ${reason}`);
|
|
193
|
+
this.surface = surface;
|
|
194
|
+
}
|
|
195
|
+
};
|
|
196
|
+
UI_TOKEN_QUERY = "token";
|
|
197
|
+
BrowserActionRecorder = class {
|
|
198
|
+
constructor(post, agentId, opts = {}) {
|
|
199
|
+
this.post = post;
|
|
200
|
+
this.agentId = agentId;
|
|
201
|
+
this.opts = opts;
|
|
202
|
+
}
|
|
203
|
+
seq = 0;
|
|
204
|
+
base = null;
|
|
205
|
+
queue = [];
|
|
206
|
+
timer = null;
|
|
207
|
+
inflight = Promise.resolve();
|
|
208
|
+
sent = 0;
|
|
209
|
+
dropped = 0;
|
|
210
|
+
/** This recorder's next position; the wire seq is `resumeFrom() + this`. */
|
|
211
|
+
nextSeq() {
|
|
212
|
+
return this.seq++;
|
|
213
|
+
}
|
|
214
|
+
resolveBase() {
|
|
215
|
+
if (this.base === null) {
|
|
216
|
+
const resolve2 = this.opts.resumeFrom ?? (async () => 0);
|
|
217
|
+
this.base = resolve2().then((n) => {
|
|
218
|
+
if (!Number.isInteger(n) || n < 0)
|
|
219
|
+
throw new Error(
|
|
220
|
+
`browser event base seq must be a non-negative integer, got ${String(n)}`
|
|
221
|
+
);
|
|
222
|
+
return n;
|
|
223
|
+
});
|
|
224
|
+
this.base.catch(() => {
|
|
225
|
+
this.base = null;
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
return this.base;
|
|
229
|
+
}
|
|
230
|
+
record(event) {
|
|
231
|
+
this.queue.push(event);
|
|
232
|
+
if (this.queue.length >= (this.opts.batchSize ?? EVENT_BATCH_SIZE)) {
|
|
233
|
+
this.drain();
|
|
234
|
+
} else if (this.timer === null) {
|
|
235
|
+
this.timer = setTimeout(
|
|
236
|
+
() => this.drain(),
|
|
237
|
+
this.opts.flushIntervalMs ?? EVENT_FLUSH_INTERVAL_MS
|
|
238
|
+
);
|
|
239
|
+
if (typeof this.timer === "object" && "unref" in this.timer)
|
|
240
|
+
this.timer.unref();
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
/** Resolve once everything recorded so far has been posted (or dropped). */
|
|
244
|
+
async flush() {
|
|
245
|
+
this.drain();
|
|
246
|
+
await this.inflight;
|
|
247
|
+
}
|
|
248
|
+
async close() {
|
|
249
|
+
await this.flush();
|
|
250
|
+
}
|
|
251
|
+
drain() {
|
|
252
|
+
if (this.timer !== null) {
|
|
253
|
+
clearTimeout(this.timer);
|
|
254
|
+
this.timer = null;
|
|
255
|
+
}
|
|
256
|
+
if (this.queue.length === 0) return;
|
|
257
|
+
const batch = this.queue;
|
|
258
|
+
this.queue = [];
|
|
259
|
+
this.inflight = this.inflight.then(() => this.send(batch));
|
|
260
|
+
}
|
|
261
|
+
async send(batch) {
|
|
262
|
+
const retries = this.opts.maxRetries ?? EVENT_MAX_RETRIES;
|
|
263
|
+
const sleep2 = this.opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
264
|
+
for (let attempt = 0; attempt < retries; attempt++) {
|
|
265
|
+
try {
|
|
266
|
+
const base = await this.resolveBase();
|
|
267
|
+
await this.post(
|
|
268
|
+
base === 0 ? batch : batch.map((e) => ({ ...e, seq: base + e.seq }))
|
|
269
|
+
);
|
|
270
|
+
this.sent += batch.length;
|
|
271
|
+
return;
|
|
272
|
+
} catch (error) {
|
|
273
|
+
if (attempt + 1 === retries) {
|
|
274
|
+
this.dropped += batch.length;
|
|
275
|
+
(this.opts.warn ?? console.warn)(
|
|
276
|
+
`dropping ${batch.length} browser events after ${attempt + 1} attempts: ${String(error)}`
|
|
277
|
+
);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
await sleep2(500 * 2 ** attempt);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
};
|
|
285
|
+
WorldBrowser = class {
|
|
286
|
+
constructor(page, ui, mode, teardown, viewport = DEFAULT_VIEWPORT, recorder = null) {
|
|
287
|
+
this.page = page;
|
|
288
|
+
this.ui = ui;
|
|
289
|
+
this.mode = mode;
|
|
290
|
+
this.teardown = teardown;
|
|
291
|
+
this.viewport = viewport;
|
|
292
|
+
this.recorder = recorder;
|
|
293
|
+
page.on("request", () => {
|
|
294
|
+
this.inflight += 1;
|
|
295
|
+
this.lastNetwork = Date.now();
|
|
296
|
+
});
|
|
297
|
+
const done = () => {
|
|
298
|
+
this.inflight = Math.max(0, this.inflight - 1);
|
|
299
|
+
this.lastNetwork = Date.now();
|
|
300
|
+
};
|
|
301
|
+
page.on("requestfinished", done);
|
|
302
|
+
page.on("requestfailed", done);
|
|
303
|
+
}
|
|
304
|
+
inflight = 0;
|
|
305
|
+
lastNetwork = Date.now();
|
|
306
|
+
get agentId() {
|
|
307
|
+
return this.recorder?.agentId ?? null;
|
|
308
|
+
}
|
|
309
|
+
/** Run one action and report it: the event carries what was asked,
|
|
310
|
+
* whether it worked, and the screen after — for failures too, so a
|
|
311
|
+
* reviewer sees what the agent saw when the selector did not match. */
|
|
312
|
+
async act(action, args, fn) {
|
|
313
|
+
const recorder = this.recorder;
|
|
314
|
+
if (!recorder) return fn();
|
|
315
|
+
const started = Date.now();
|
|
316
|
+
let ok = true;
|
|
317
|
+
let error = null;
|
|
318
|
+
try {
|
|
319
|
+
return await fn();
|
|
320
|
+
} catch (err) {
|
|
321
|
+
ok = false;
|
|
322
|
+
error = (err instanceof Error ? `${err.name}: ${err.message}` : String(err)).slice(0, 4e3);
|
|
323
|
+
throw err;
|
|
324
|
+
} finally {
|
|
325
|
+
recorder.record({
|
|
326
|
+
agentId: recorder.agentId,
|
|
327
|
+
seq: recorder.nextSeq(),
|
|
328
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
329
|
+
action,
|
|
330
|
+
args: redactActionArgs(action, args),
|
|
331
|
+
ok,
|
|
332
|
+
error,
|
|
333
|
+
durationMs: Date.now() - started,
|
|
334
|
+
mode: this.mode,
|
|
335
|
+
viewport: { ...this.viewport },
|
|
336
|
+
...await this.observeForEvent()
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
async observeForEvent() {
|
|
341
|
+
const out = {
|
|
342
|
+
url: null,
|
|
343
|
+
title: null,
|
|
344
|
+
screenshot: null
|
|
345
|
+
};
|
|
346
|
+
try {
|
|
347
|
+
out.url = this.page.url();
|
|
348
|
+
out.title = await this.page.title();
|
|
349
|
+
const jpeg = await this.page.screenshot({
|
|
350
|
+
type: "jpeg",
|
|
351
|
+
quality: EVENT_SCREENSHOT_QUALITY
|
|
352
|
+
});
|
|
353
|
+
out.screenshot = jpeg.toString("base64");
|
|
354
|
+
} catch {
|
|
355
|
+
}
|
|
356
|
+
return out;
|
|
357
|
+
}
|
|
358
|
+
/** Open a UI path (`/customers`), a full URL, or — with no argument — the
|
|
359
|
+
* UI's declared entry page. */
|
|
360
|
+
async goto(pathOrUrl) {
|
|
361
|
+
let target;
|
|
362
|
+
if (pathOrUrl === void 0) {
|
|
363
|
+
if (!this.ui)
|
|
364
|
+
throw new WorldSurfaceUnavailable(
|
|
365
|
+
"ui",
|
|
366
|
+
"this session has no UI to navigate to"
|
|
367
|
+
);
|
|
368
|
+
target = uiEntryUrl(this.ui);
|
|
369
|
+
} else if (/^https?:\/\//.test(pathOrUrl)) {
|
|
370
|
+
target = pathOrUrl;
|
|
371
|
+
} else {
|
|
372
|
+
if (!this.ui)
|
|
373
|
+
throw new WorldSurfaceUnavailable(
|
|
374
|
+
"ui",
|
|
375
|
+
"relative paths need a UI surface"
|
|
376
|
+
);
|
|
377
|
+
target = uiUrl(this.ui, pathOrUrl);
|
|
378
|
+
}
|
|
379
|
+
return this.act(
|
|
380
|
+
"browser_goto",
|
|
381
|
+
{ path: pathOrUrl ?? null, url: withoutUiToken(target) },
|
|
382
|
+
async () => {
|
|
383
|
+
await this.page.goto(target, { waitUntil: "load" });
|
|
384
|
+
await this.settle();
|
|
385
|
+
return this.page.url();
|
|
386
|
+
}
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
async back() {
|
|
390
|
+
return this.act("browser_back", {}, async () => {
|
|
391
|
+
await this.page.goBack({ waitUntil: "load" });
|
|
392
|
+
await this.settle();
|
|
393
|
+
return this.page.url();
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
/** Wait for the page to finish reacting to an action. `load` alone is not
|
|
397
|
+
* enough for client-rendered UIs (React et al.): a click returns before the
|
|
398
|
+
* fetch that re-renders the page, so an agent would read stale text.
|
|
399
|
+
* Playwright's `networkidle` is per navigation and resolves at once after
|
|
400
|
+
* the first one, so this tracks requests itself: done when nothing is in
|
|
401
|
+
* flight and the network has been quiet for `SETTLE_QUIET_MS`, bounded so
|
|
402
|
+
* long-polling UIs cannot hang us. */
|
|
403
|
+
async settle() {
|
|
404
|
+
await this.page.waitForLoadState("load");
|
|
405
|
+
const deadline = Date.now() + SETTLE_TIMEOUT_MS;
|
|
406
|
+
while (Date.now() < deadline) {
|
|
407
|
+
const quiet = Date.now() - this.lastNetwork >= SETTLE_QUIET_MS;
|
|
408
|
+
if (this.inflight === 0 && quiet) return;
|
|
409
|
+
await this.page.waitForTimeout(SETTLE_POLL_MS);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
async click(selector) {
|
|
413
|
+
await this.act("browser_click", { selector }, async () => {
|
|
414
|
+
await this.page.click(selector);
|
|
415
|
+
await this.settle();
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
async type(selector, text) {
|
|
419
|
+
await this.act(
|
|
420
|
+
"browser_type",
|
|
421
|
+
{ selector, text },
|
|
422
|
+
() => this.page.fill(selector, text)
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
async press(key) {
|
|
426
|
+
await this.act("browser_press", { key }, async () => {
|
|
427
|
+
await this.page.keyboard.press(key);
|
|
428
|
+
await this.settle();
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
async scroll(dy) {
|
|
432
|
+
await this.act(
|
|
433
|
+
"browser_scroll",
|
|
434
|
+
{ dy },
|
|
435
|
+
() => this.page.mouse.wheel(0, dy)
|
|
436
|
+
);
|
|
437
|
+
}
|
|
438
|
+
async screenshot(fullPage = false) {
|
|
439
|
+
return this.page.screenshot({ type: "png", fullPage });
|
|
440
|
+
}
|
|
441
|
+
async screenshotBase64(fullPage = false) {
|
|
442
|
+
return (await this.screenshot(fullPage)).toString("base64");
|
|
443
|
+
}
|
|
444
|
+
async text(selector = "body") {
|
|
445
|
+
return this.page.innerText(selector);
|
|
446
|
+
}
|
|
447
|
+
async html() {
|
|
448
|
+
return this.page.content();
|
|
449
|
+
}
|
|
450
|
+
get url() {
|
|
451
|
+
return this.page.url();
|
|
452
|
+
}
|
|
453
|
+
async title() {
|
|
454
|
+
return this.page.title();
|
|
455
|
+
}
|
|
456
|
+
async close() {
|
|
457
|
+
try {
|
|
458
|
+
await this.teardown();
|
|
459
|
+
} finally {
|
|
460
|
+
await this.recorder?.close();
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
};
|
|
464
|
+
COMPUTER_USE_SPECS = [
|
|
465
|
+
{
|
|
466
|
+
name: "browser_goto",
|
|
467
|
+
signature: "browser_goto(path=None)",
|
|
468
|
+
doc: "Open a page of the world's UI. `path` is a UI path like /customers, a full URL, or omitted for the entry page. Returns the URL and page text."
|
|
469
|
+
},
|
|
470
|
+
{
|
|
471
|
+
name: "browser_click",
|
|
472
|
+
signature: "browser_click(selector)",
|
|
473
|
+
doc: "Click an element by Playwright selector (CSS, `text=Refund`, `role=button[name=Save]`). Returns the page text after the click."
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
name: "browser_type",
|
|
477
|
+
signature: "browser_type(selector, text)",
|
|
478
|
+
doc: "Replace the value of an input/textarea found by selector with `text`."
|
|
479
|
+
},
|
|
480
|
+
{
|
|
481
|
+
name: "browser_press",
|
|
482
|
+
signature: "browser_press(key)",
|
|
483
|
+
doc: "Press a key (Enter, Tab, Escape, ArrowDown, ...). Returns the page text after."
|
|
484
|
+
},
|
|
485
|
+
{
|
|
486
|
+
name: "browser_scroll",
|
|
487
|
+
signature: "browser_scroll(dy=600)",
|
|
488
|
+
doc: "Scroll the page vertically by `dy` pixels (negative scrolls up)."
|
|
489
|
+
},
|
|
490
|
+
{
|
|
491
|
+
name: "browser_screenshot",
|
|
492
|
+
signature: "browser_screenshot(full_page=False)",
|
|
493
|
+
doc: "Take a PNG screenshot; returns {image_base64, url}."
|
|
494
|
+
},
|
|
495
|
+
{
|
|
496
|
+
name: "browser_read",
|
|
497
|
+
signature: "browser_read(selector='body')",
|
|
498
|
+
doc: "Return the visible text of an element (default: whole page) plus the URL and title."
|
|
499
|
+
},
|
|
500
|
+
{
|
|
501
|
+
name: "browser_back",
|
|
502
|
+
signature: "browser_back()",
|
|
503
|
+
doc: "Go back one page in history."
|
|
504
|
+
}
|
|
505
|
+
];
|
|
506
|
+
str = (v, fallback) => typeof v === "string" && v.length > 0 ? v : fallback;
|
|
507
|
+
}
|
|
508
|
+
});
|
|
509
|
+
|
|
33
510
|
// src/worlds/session.ts
|
|
34
511
|
var session_exports = {};
|
|
35
512
|
__export(session_exports, {
|
|
@@ -37,11 +514,23 @@ __export(session_exports, {
|
|
|
37
514
|
WorldSessionCallError: () => WorldSessionCallError,
|
|
38
515
|
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
39
516
|
attachSession: () => attachSession,
|
|
517
|
+
isTaskIdRef: () => isTaskIdRef,
|
|
518
|
+
isTaskSpec: () => isTaskSpec,
|
|
40
519
|
openSession: () => openSession,
|
|
41
520
|
runSessions: () => runSessions,
|
|
42
|
-
schemaFromSignature: () => schemaFromSignature
|
|
521
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
522
|
+
taskLabel: () => taskLabel
|
|
43
523
|
});
|
|
44
524
|
function schemaFromSignature(spec) {
|
|
525
|
+
if (spec.inputSchema)
|
|
526
|
+
return {
|
|
527
|
+
type: "function",
|
|
528
|
+
function: {
|
|
529
|
+
name: modelToolName(spec.name),
|
|
530
|
+
description: spec.doc ?? spec.name,
|
|
531
|
+
parameters: spec.inputSchema
|
|
532
|
+
}
|
|
533
|
+
};
|
|
45
534
|
const inside = /\(([\s\S]*)\)\s*$/.exec(spec.signature.trim())?.[1] ?? "";
|
|
46
535
|
const properties = {};
|
|
47
536
|
const required = [];
|
|
@@ -77,7 +566,7 @@ function schemaFromSignature(spec) {
|
|
|
77
566
|
return {
|
|
78
567
|
type: "function",
|
|
79
568
|
function: {
|
|
80
|
-
name: spec.name,
|
|
569
|
+
name: modelToolName(spec.name),
|
|
81
570
|
description: spec.doc ?? spec.name,
|
|
82
571
|
parameters: { type: "object", properties, required }
|
|
83
572
|
}
|
|
@@ -91,8 +580,20 @@ async function attachSession(sessionId, options = {}) {
|
|
|
91
580
|
);
|
|
92
581
|
return new WorldSession(sessionId, state, options);
|
|
93
582
|
}
|
|
94
|
-
async function openSession(slugOrRef,
|
|
583
|
+
async function openSession(slugOrRef, taskOrOpts, positionalOpts = {}) {
|
|
584
|
+
const { task, ...opts } = typeof taskOrOpts === "object" && "task" in taskOrOpts ? taskOrOpts : { task: taskOrOpts, ...positionalOpts };
|
|
585
|
+
if (task === void 0 || task === null || typeof task === "string" && !task || isTaskIdRef(task) && !task.id || isTaskSpec(task) && !task.instruction || typeof task === "object" && !isTaskIdRef(task) && !isTaskSpec(task))
|
|
586
|
+
throw new Error(
|
|
587
|
+
"openSession needs a task: a bundled task name, a stored task { id, version? } or { instruction, seed?, grader?, metadata? }"
|
|
588
|
+
);
|
|
95
589
|
const options = opts.options ?? {};
|
|
590
|
+
const unknown = (opts.surfaces ?? []).filter(
|
|
591
|
+
(s) => !SURFACE_KINDS.includes(s)
|
|
592
|
+
);
|
|
593
|
+
if (unknown.length)
|
|
594
|
+
throw new Error(
|
|
595
|
+
`Unknown world surfaces: ${unknown.join(", ")}; choose from ${SURFACE_KINDS.join(", ")}`
|
|
596
|
+
);
|
|
96
597
|
const at = slugOrRef.indexOf("@");
|
|
97
598
|
const slug = at === -1 ? slugOrRef : slugOrRef.slice(0, at);
|
|
98
599
|
const state = await apiRequest(
|
|
@@ -102,7 +603,8 @@ async function openSession(slugOrRef, task, opts = {}) {
|
|
|
102
603
|
{
|
|
103
604
|
...opts.versionId ? { versionId: opts.versionId } : { slug },
|
|
104
605
|
task,
|
|
105
|
-
...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm }
|
|
606
|
+
...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm },
|
|
607
|
+
...opts.surfaces?.length ? { surfaces: opts.surfaces } : {}
|
|
106
608
|
}
|
|
107
609
|
);
|
|
108
610
|
return new WorldSession(state.sessionId, state, options);
|
|
@@ -123,20 +625,24 @@ async function runSessions(opts) {
|
|
|
123
625
|
await Promise.all(
|
|
124
626
|
Array.from({ length: Math.min(concurrency, tasks.length) }, async () => {
|
|
125
627
|
while (next < tasks.length) {
|
|
126
|
-
const
|
|
628
|
+
const taskRef = tasks[next++];
|
|
629
|
+
let task = taskLabel(taskRef);
|
|
127
630
|
let session = null;
|
|
128
631
|
try {
|
|
129
|
-
session = await openSession(opts.world,
|
|
632
|
+
session = await openSession(opts.world, {
|
|
633
|
+
task: taskRef,
|
|
130
634
|
versionId,
|
|
131
635
|
options
|
|
132
636
|
});
|
|
637
|
+
task = taskLabel(taskRef, session.taskInfo);
|
|
133
638
|
await session.ready();
|
|
134
639
|
const toolkit = await session.toolkit();
|
|
135
640
|
await opts.agent({
|
|
136
641
|
session,
|
|
137
642
|
toolkit,
|
|
138
|
-
instruction: session.instruction ?? task,
|
|
139
|
-
task
|
|
643
|
+
instruction: session.instruction ?? (isTaskSpec(taskRef) ? taskRef.instruction : task),
|
|
644
|
+
task,
|
|
645
|
+
taskRef
|
|
140
646
|
});
|
|
141
647
|
const graded = await session.grade();
|
|
142
648
|
const result = {
|
|
@@ -172,35 +678,143 @@ async function runSessions(opts) {
|
|
|
172
678
|
passed: opts.failUnder === void 0 ? true : mean >= opts.failUnder
|
|
173
679
|
};
|
|
174
680
|
}
|
|
175
|
-
var READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, sleep, WorldSessionTimeout, WorldSessionCallError, WorldSession;
|
|
681
|
+
var import_node_crypto, isTaskIdRef, isTaskSpec, taskLabel, READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, JUDGED_GRADE_TIMEOUT_MS, JUDGED_GRADE_POLL_MS, EXPORT_PAGE, sleep, modelToolName, WorldSessionTimeout, WorldSessionCallError, WorldSession;
|
|
176
682
|
var init_session = __esm({
|
|
177
683
|
"src/worlds/session.ts"() {
|
|
178
684
|
"use strict";
|
|
685
|
+
import_node_crypto = require("crypto");
|
|
179
686
|
init_worlds();
|
|
687
|
+
init_browser();
|
|
688
|
+
isTaskIdRef = (task) => typeof task === "object" && task !== null && "id" in task;
|
|
689
|
+
isTaskSpec = (task) => typeof task === "object" && task !== null && "instruction" in task;
|
|
690
|
+
taskLabel = (task, info) => {
|
|
691
|
+
if (typeof task === "string") return task;
|
|
692
|
+
if (isTaskIdRef(task)) return info?.name ?? task.id;
|
|
693
|
+
if (task.name) return task.name;
|
|
694
|
+
return info?.hash ? `inline:${info.hash.slice(0, 12)}` : "inline";
|
|
695
|
+
};
|
|
180
696
|
READY_TIMEOUT_MS = 10 * 6e4;
|
|
181
697
|
READY_POLL_MS = 2e3;
|
|
182
698
|
CALL_TIMEOUT_MS = 3 * 6e4;
|
|
183
699
|
CALL_POLL_MS = 400;
|
|
700
|
+
JUDGED_GRADE_TIMEOUT_MS = 15 * 6e4;
|
|
701
|
+
JUDGED_GRADE_POLL_MS = 2e3;
|
|
702
|
+
EXPORT_PAGE = 500;
|
|
184
703
|
sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
704
|
+
modelToolName = (name) => name.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 64) || "tool";
|
|
185
705
|
WorldSessionTimeout = class extends Error {
|
|
186
706
|
};
|
|
187
707
|
WorldSessionCallError = class extends Error {
|
|
188
708
|
};
|
|
189
709
|
WorldSession = class {
|
|
190
|
-
constructor(sessionId,
|
|
710
|
+
constructor(sessionId, current, options) {
|
|
191
711
|
this.sessionId = sessionId;
|
|
192
|
-
this.
|
|
712
|
+
this.current = current;
|
|
193
713
|
this.options = options;
|
|
194
714
|
}
|
|
715
|
+
/** The task's display name: a bundled scenario's, or the inline spec's
|
|
716
|
+
* `name` (null when the spec was unnamed). Same as `taskName`. */
|
|
195
717
|
get task() {
|
|
196
|
-
return this.
|
|
718
|
+
return this.current.taskName;
|
|
719
|
+
}
|
|
720
|
+
/** The task's display name: a bundled scenario's, or the inline spec's
|
|
721
|
+
* `name` (null when the spec was unnamed). */
|
|
722
|
+
get taskName() {
|
|
723
|
+
return this.current.taskName;
|
|
724
|
+
}
|
|
725
|
+
/** The task this session runs: bundled scenario, stored task or inline
|
|
726
|
+
* spec, its name, hash and the caller's metadata. */
|
|
727
|
+
get taskInfo() {
|
|
728
|
+
return this.current.task ?? {
|
|
729
|
+
source: "bundled",
|
|
730
|
+
name: this.current.taskName,
|
|
731
|
+
hash: null,
|
|
732
|
+
metadata: {}
|
|
733
|
+
};
|
|
197
734
|
}
|
|
198
|
-
/** What this task asks for — the bundle's
|
|
735
|
+
/** What this task asks for — the bundle's instruction.md, or the inline
|
|
736
|
+
* spec's `instruction`. */
|
|
199
737
|
get instruction() {
|
|
200
|
-
return this.
|
|
738
|
+
return this.current.instruction ?? null;
|
|
201
739
|
}
|
|
202
740
|
get versionId() {
|
|
203
|
-
return this.
|
|
741
|
+
return this.current.versionId;
|
|
742
|
+
}
|
|
743
|
+
/** Whether the session answered ready as of the last state read (the open,
|
|
744
|
+
* a `status()`, or the poll inside `ready()`); no request is made. */
|
|
745
|
+
get isReady() {
|
|
746
|
+
return this.current.ready;
|
|
747
|
+
}
|
|
748
|
+
/** `api` / `ui` / `browser` are present exactly when the world declares
|
|
749
|
+
* them AND the session was opened asking for them. */
|
|
750
|
+
async surfaces() {
|
|
751
|
+
if (!this.current.surfaces) await this.status();
|
|
752
|
+
return this.current.surfaces ?? { tools: { kind: "rendezvous" } };
|
|
753
|
+
}
|
|
754
|
+
/** Base URL of the world's own HTTP API for this session. */
|
|
755
|
+
async api() {
|
|
756
|
+
const { api } = await this.surfaces();
|
|
757
|
+
if (!api)
|
|
758
|
+
throw new WorldSurfaceUnavailable(
|
|
759
|
+
"api",
|
|
760
|
+
"open the session with surfaces: ['api'] on an api-type world"
|
|
761
|
+
);
|
|
762
|
+
return api.url;
|
|
763
|
+
}
|
|
764
|
+
/** Headers every request to `api()` needs. A World Host session gates its
|
|
765
|
+
* api surface on a per-session token; a pod or hosted twin answers with the
|
|
766
|
+
* world's own declared auth and this is `{}`. */
|
|
767
|
+
async apiHeaders() {
|
|
768
|
+
const { api } = await this.surfaces();
|
|
769
|
+
return api?.token ? { Authorization: `Bearer ${api.token}` } : {};
|
|
770
|
+
}
|
|
771
|
+
/** URL of the world's UI entry page for this session. */
|
|
772
|
+
async ui() {
|
|
773
|
+
const { ui } = await this.surfaces();
|
|
774
|
+
if (!ui)
|
|
775
|
+
throw new WorldSurfaceUnavailable(
|
|
776
|
+
"ui",
|
|
777
|
+
"open the session with surfaces: ['ui'] on a world that declares [ui]"
|
|
778
|
+
);
|
|
779
|
+
return uiEntryUrl(ui);
|
|
780
|
+
}
|
|
781
|
+
/** A Playwright browser on this session's UI: `hosted` drives the one the
|
|
782
|
+
* platform runs next to the world, `local` launches one here, `auto`
|
|
783
|
+
* prefers hosted. Call `close()` when done.
|
|
784
|
+
*
|
|
785
|
+
* Every action is reported to the platform under `agentId` (default:
|
|
786
|
+
* `GATEWAY_AGENT_ID` or a fresh `agent-<hex>`) so the session page shows
|
|
787
|
+
* this agent's own timeline — several agents may share one session and
|
|
788
|
+
* one hosted browser. `record: false` opts out. */
|
|
789
|
+
async browser(opts = {}) {
|
|
790
|
+
const { agentId, record = true, ...rest } = opts;
|
|
791
|
+
return openBrowser(await this.surfaces(), {
|
|
792
|
+
...rest,
|
|
793
|
+
recorder: record ? this.browserRecorder(agentId) : null
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
/** The reporter `browser()` uses; exposed for a page you drive yourself. */
|
|
797
|
+
browserRecorder(agentId) {
|
|
798
|
+
const path2 = `/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/browser-events`;
|
|
799
|
+
const id = agentId ?? process.env.GATEWAY_AGENT_ID ?? `agent-${(0, import_node_crypto.randomUUID)().replace(/-/g, "").slice(0, 8)}`;
|
|
800
|
+
return new BrowserActionRecorder(
|
|
801
|
+
async (events) => {
|
|
802
|
+
await apiRequest(this.options, "POST", path2, { events });
|
|
803
|
+
},
|
|
804
|
+
id,
|
|
805
|
+
{
|
|
806
|
+
// Continue this agent's timeline where the platform has it, so the
|
|
807
|
+
// same id reopened later never re-numbers from zero.
|
|
808
|
+
resumeFrom: async () => {
|
|
809
|
+
const summary = await apiRequest(this.options, "GET", path2);
|
|
810
|
+
return summary.agents.find((a) => a.agentId === id)?.nextSeq ?? 0;
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
);
|
|
814
|
+
}
|
|
815
|
+
/** Browser actions as model tools, in the same shape as `toolkit()`. */
|
|
816
|
+
computerUseTools(browser) {
|
|
817
|
+
return computerUseToolkit(browser);
|
|
204
818
|
}
|
|
205
819
|
/** The session's latest platform state (one request, no waiting). */
|
|
206
820
|
async status() {
|
|
@@ -209,12 +823,14 @@ var init_session = __esm({
|
|
|
209
823
|
"GET",
|
|
210
824
|
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`
|
|
211
825
|
);
|
|
212
|
-
this.
|
|
213
|
-
instruction: this.
|
|
214
|
-
tools: this.
|
|
826
|
+
this.current = {
|
|
827
|
+
instruction: this.current.instruction,
|
|
828
|
+
tools: this.current.tools,
|
|
829
|
+
surfaces: this.current.surfaces,
|
|
830
|
+
task: this.current.task,
|
|
215
831
|
...fresh
|
|
216
832
|
};
|
|
217
|
-
return this.
|
|
833
|
+
return this.current;
|
|
218
834
|
}
|
|
219
835
|
/**
|
|
220
836
|
* Wait until the container is warm — meaning the process inside it has
|
|
@@ -237,14 +853,18 @@ var init_session = __esm({
|
|
|
237
853
|
}
|
|
238
854
|
}
|
|
239
855
|
/** Enqueue one interaction and wait for the container's answer. */
|
|
240
|
-
async exchange(body, timeoutMs
|
|
241
|
-
const
|
|
856
|
+
async exchange(body, timeoutMs) {
|
|
857
|
+
const enqueued = await apiRequest(
|
|
242
858
|
this.options,
|
|
243
859
|
"POST",
|
|
244
860
|
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls`,
|
|
245
861
|
body
|
|
246
862
|
);
|
|
247
|
-
const
|
|
863
|
+
const { callId } = enqueued;
|
|
864
|
+
const judged = enqueued.pending === true;
|
|
865
|
+
const budgetMs = timeoutMs ?? (judged ? JUDGED_GRADE_TIMEOUT_MS : CALL_TIMEOUT_MS);
|
|
866
|
+
const pollMs = judged ? JUDGED_GRADE_POLL_MS : CALL_POLL_MS;
|
|
867
|
+
const deadline = Date.now() + budgetMs;
|
|
248
868
|
for (; ; ) {
|
|
249
869
|
const call = await apiRequest(
|
|
250
870
|
this.options,
|
|
@@ -257,9 +877,9 @@ var init_session = __esm({
|
|
|
257
877
|
}
|
|
258
878
|
if (Date.now() > deadline)
|
|
259
879
|
throw new WorldSessionTimeout(
|
|
260
|
-
`Call ${callId} still ${call.state} after ${Math.round(
|
|
880
|
+
judged ? `Grade ${callId} still ${call.state} after ${Math.round(budgetMs / 1e3)}s \u2014 the judge has not answered; the platform finalizes the call on its own, read it back with export().` : `Call ${callId} still ${call.state} after ${Math.round(budgetMs / 1e3)}s \u2014 the container may have died.`
|
|
261
881
|
);
|
|
262
|
-
await sleep(
|
|
882
|
+
await sleep(pollMs);
|
|
263
883
|
}
|
|
264
884
|
}
|
|
265
885
|
/**
|
|
@@ -283,15 +903,27 @@ var init_session = __esm({
|
|
|
283
903
|
* hand-written map that still remembers the old name.
|
|
284
904
|
*/
|
|
285
905
|
async toolkit() {
|
|
286
|
-
let specs = this.
|
|
906
|
+
let specs = this.current.tools;
|
|
287
907
|
if (!specs) specs = (await this.status()).tools ?? [];
|
|
288
|
-
const impls =
|
|
908
|
+
const impls = /* @__PURE__ */ Object.create(null);
|
|
289
909
|
for (const spec of specs) {
|
|
290
|
-
|
|
910
|
+
const alias = modelToolName(spec.name);
|
|
911
|
+
if (Object.hasOwn(impls, alias))
|
|
912
|
+
throw new WorldSessionCallError(`Model tool name collision: ${alias}`);
|
|
913
|
+
impls[alias] = (args = {}) => this.call(spec.name, args);
|
|
291
914
|
}
|
|
292
915
|
return { schemas: specs.map(schemaFromSignature), impls, specs };
|
|
293
916
|
}
|
|
294
|
-
/** Run the task's grader against the container's END STATE.
|
|
917
|
+
/** Run the task's grader against the container's END STATE. Graders answer
|
|
918
|
+
* in one of two shapes: the scalar with named rewards beside it
|
|
919
|
+
* (`{reward, check_a, check_b}`, a task's own `grade.py`), or the named
|
|
920
|
+
* rewards nested under `rewards` (the api, mock-tools and World Host
|
|
921
|
+
* runners). Both land in `rewards`.
|
|
922
|
+
*
|
|
923
|
+
* A rubric grade is finalized by the platform's judge, not the container:
|
|
924
|
+
* the platform answers the enqueue with `pending: true`, and this waits on
|
|
925
|
+
* the judge's budget (15 min) instead of a call's. `timeoutMs` overrides
|
|
926
|
+
* either. */
|
|
295
927
|
async grade(opts = {}) {
|
|
296
928
|
const raw = await this.exchange(
|
|
297
929
|
{ kind: "grade" },
|
|
@@ -299,6 +931,12 @@ var init_session = __esm({
|
|
|
299
931
|
);
|
|
300
932
|
const record = raw && typeof raw === "object" ? raw : {};
|
|
301
933
|
const rewards = {};
|
|
934
|
+
const nested = record.rewards;
|
|
935
|
+
if (nested && typeof nested === "object") {
|
|
936
|
+
for (const [key, value] of Object.entries(nested)) {
|
|
937
|
+
if (typeof value === "number") rewards[key] = value;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
302
940
|
for (const [key, value] of Object.entries(record)) {
|
|
303
941
|
if (key !== "reward" && typeof value === "number") rewards[key] = value;
|
|
304
942
|
}
|
|
@@ -308,6 +946,91 @@ var init_session = __esm({
|
|
|
308
946
|
raw
|
|
309
947
|
};
|
|
310
948
|
}
|
|
949
|
+
/**
|
|
950
|
+
* Put rows into the LIVE world through its contract, in one batch:
|
|
951
|
+
* `append` (default) upserts by primary key, `replace` makes the named
|
|
952
|
+
* entities hold exactly these rows. Atomic — a refused row leaves the
|
|
953
|
+
* session unchanged and the error names the entity, row and field.
|
|
954
|
+
*/
|
|
955
|
+
async seed(rows, opts = {}) {
|
|
956
|
+
const mode = opts.mode ?? "append";
|
|
957
|
+
return await this.exchange(
|
|
958
|
+
{ kind: "seed", args: { mode, rows } },
|
|
959
|
+
opts.timeoutMs
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
/** Every entity's rows as the world holds them now — the same dump a
|
|
963
|
+
* grader reads. The shape is the world runtime's: a schema world answers
|
|
964
|
+
* `{<entity>: {<key>: row}, snapshot}`, a mock-tools world
|
|
965
|
+
* `{entities: {<entity>: [row]}}`. */
|
|
966
|
+
async state(opts = {}) {
|
|
967
|
+
return this.exchange({ kind: "state" }, opts.timeoutMs);
|
|
968
|
+
}
|
|
969
|
+
/** Back to the opening state: the version's data plus the bundled
|
|
970
|
+
* scenario's or the inline task's seed. Rows written since are gone; the
|
|
971
|
+
* call log stays. */
|
|
972
|
+
async reset(opts = {}) {
|
|
973
|
+
await this.exchange({ kind: "reset" }, opts.timeoutMs);
|
|
974
|
+
}
|
|
975
|
+
/** The session's call log, oldest first: every tool call, seed, reset,
|
|
976
|
+
* grade with its result — the record of the episode. */
|
|
977
|
+
async export() {
|
|
978
|
+
const calls = [];
|
|
979
|
+
let cursor;
|
|
980
|
+
do {
|
|
981
|
+
const params = new URLSearchParams({ limit: String(EXPORT_PAGE) });
|
|
982
|
+
if (cursor) params.set("cursor", cursor);
|
|
983
|
+
const page = await apiRequest(
|
|
984
|
+
this.options,
|
|
985
|
+
"GET",
|
|
986
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls?${params.toString()}`
|
|
987
|
+
);
|
|
988
|
+
for (const call of page.calls) {
|
|
989
|
+
calls.push({
|
|
990
|
+
seq: Number(call.seq),
|
|
991
|
+
kind: String(call.kind ?? "call"),
|
|
992
|
+
tool: typeof call.tool === "string" ? call.tool : null,
|
|
993
|
+
args: call.args ?? null,
|
|
994
|
+
result: call.result ?? null,
|
|
995
|
+
error: typeof call.error === "string" ? call.error : null,
|
|
996
|
+
completedAt: typeof call.completedAt === "string" ? call.completedAt : null
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
cursor = page.nextCursor ?? void 0;
|
|
1000
|
+
} while (cursor);
|
|
1001
|
+
return calls;
|
|
1002
|
+
}
|
|
1003
|
+
/**
|
|
1004
|
+
* Promote this session's inline task into a stored task of the project
|
|
1005
|
+
* (version 1, pinned to the session's world unless `world` says otherwise),
|
|
1006
|
+
* so a task written for one end user can be reused and its grades
|
|
1007
|
+
* compared. The session then records the new task's id. A session on a
|
|
1008
|
+
* bundled task has nothing to save; one on a stored task is already saved.
|
|
1009
|
+
*/
|
|
1010
|
+
async saveTask(opts = {}) {
|
|
1011
|
+
const task = await apiRequest(
|
|
1012
|
+
this.options,
|
|
1013
|
+
"POST",
|
|
1014
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/task`,
|
|
1015
|
+
{
|
|
1016
|
+
...opts.name !== void 0 ? { name: opts.name } : {},
|
|
1017
|
+
...opts.world !== void 0 ? { world: opts.world } : {},
|
|
1018
|
+
...opts.message !== void 0 ? { message: opts.message } : {}
|
|
1019
|
+
}
|
|
1020
|
+
);
|
|
1021
|
+
this.current = {
|
|
1022
|
+
...this.current,
|
|
1023
|
+
taskName: task.name,
|
|
1024
|
+
task: {
|
|
1025
|
+
...this.taskInfo,
|
|
1026
|
+
source: "stored",
|
|
1027
|
+
name: task.name,
|
|
1028
|
+
id: task.id,
|
|
1029
|
+
version: task.version
|
|
1030
|
+
}
|
|
1031
|
+
};
|
|
1032
|
+
return task;
|
|
1033
|
+
}
|
|
311
1034
|
/**
|
|
312
1035
|
* End the session. `keepWarm` hands the container back to the pool instead
|
|
313
1036
|
* of tearing it down, so the next session for the same task starts warm —
|
|
@@ -325,6 +1048,83 @@ var init_session = __esm({
|
|
|
325
1048
|
}
|
|
326
1049
|
});
|
|
327
1050
|
|
|
1051
|
+
// src/worlds/tasks.ts
|
|
1052
|
+
async function createTask(input, options = {}) {
|
|
1053
|
+
return apiRequest(
|
|
1054
|
+
options,
|
|
1055
|
+
"POST",
|
|
1056
|
+
TASKS,
|
|
1057
|
+
compact({
|
|
1058
|
+
spec: input.spec,
|
|
1059
|
+
name: input.name,
|
|
1060
|
+
world: input.world,
|
|
1061
|
+
message: input.message
|
|
1062
|
+
})
|
|
1063
|
+
);
|
|
1064
|
+
}
|
|
1065
|
+
async function updateTask(taskId, input, options = {}) {
|
|
1066
|
+
if (input.spec === void 0 && input.name === void 0 && input.world === void 0)
|
|
1067
|
+
throw new Error("updateTask: nothing to update \u2014 pass spec, name or world");
|
|
1068
|
+
return apiRequest(
|
|
1069
|
+
options,
|
|
1070
|
+
"PATCH",
|
|
1071
|
+
taskRoute(taskId),
|
|
1072
|
+
compact({
|
|
1073
|
+
spec: input.spec,
|
|
1074
|
+
name: input.name,
|
|
1075
|
+
world: input.world,
|
|
1076
|
+
message: input.message
|
|
1077
|
+
})
|
|
1078
|
+
);
|
|
1079
|
+
}
|
|
1080
|
+
async function listTasks(filter = {}, options = {}) {
|
|
1081
|
+
const params = new URLSearchParams();
|
|
1082
|
+
if (filter.world) params.set("world", filter.world);
|
|
1083
|
+
if (filter.page !== void 0) params.set("page", String(filter.page));
|
|
1084
|
+
if (filter.limit !== void 0) params.set("limit", String(filter.limit));
|
|
1085
|
+
for (const [key, value] of Object.entries(filter.metadata ?? {}))
|
|
1086
|
+
params.set(`metadata.${key}`, value);
|
|
1087
|
+
const query = params.toString();
|
|
1088
|
+
return apiRequest(
|
|
1089
|
+
options,
|
|
1090
|
+
"GET",
|
|
1091
|
+
query ? `${TASKS}?${query}` : TASKS
|
|
1092
|
+
);
|
|
1093
|
+
}
|
|
1094
|
+
async function getTask(taskId, opts = {}, options = {}) {
|
|
1095
|
+
const query = opts.version === void 0 ? "" : `?version=${String(opts.version)}`;
|
|
1096
|
+
return apiRequest(
|
|
1097
|
+
options,
|
|
1098
|
+
"GET",
|
|
1099
|
+
`${taskRoute(taskId)}${query}`
|
|
1100
|
+
);
|
|
1101
|
+
}
|
|
1102
|
+
async function deleteTask(taskId, options = {}) {
|
|
1103
|
+
return apiRequest(options, "DELETE", taskRoute(taskId));
|
|
1104
|
+
}
|
|
1105
|
+
async function validateTask(taskId, target = {}, options = {}) {
|
|
1106
|
+
return apiRequest(
|
|
1107
|
+
options,
|
|
1108
|
+
"POST",
|
|
1109
|
+
`${taskRoute(taskId)}/validate`,
|
|
1110
|
+
compact({
|
|
1111
|
+
version: target.version,
|
|
1112
|
+
slug: target.slug,
|
|
1113
|
+
versionId: target.versionId
|
|
1114
|
+
})
|
|
1115
|
+
);
|
|
1116
|
+
}
|
|
1117
|
+
var TASKS, taskRoute, compact;
|
|
1118
|
+
var init_tasks = __esm({
|
|
1119
|
+
"src/worlds/tasks.ts"() {
|
|
1120
|
+
"use strict";
|
|
1121
|
+
init_worlds();
|
|
1122
|
+
TASKS = "/api/public/world-tasks";
|
|
1123
|
+
taskRoute = (id) => `${TASKS}/${encodeURIComponent(id)}`;
|
|
1124
|
+
compact = (body) => Object.fromEntries(Object.entries(body).filter(([, v]) => v !== void 0));
|
|
1125
|
+
}
|
|
1126
|
+
});
|
|
1127
|
+
|
|
328
1128
|
// src/worlds/hub.ts
|
|
329
1129
|
function isExcludedDirComponent(component) {
|
|
330
1130
|
return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
|
|
@@ -367,7 +1167,7 @@ function collectArchiveRelPaths(dir) {
|
|
|
367
1167
|
return out.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
368
1168
|
}
|
|
369
1169
|
function computeContentHash(dir) {
|
|
370
|
-
const hasher = (0,
|
|
1170
|
+
const hasher = (0, import_node_crypto2.createHash)("sha256");
|
|
371
1171
|
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
372
1172
|
hasher.update(Buffer.from(`file:${relPath}`, "utf-8"));
|
|
373
1173
|
hasher.update(fs.readFileSync(path.join(dir, relPath)));
|
|
@@ -441,7 +1241,7 @@ function writeLocalMeta(dir, meta) {
|
|
|
441
1241
|
function snapshotTreeHashes(dir) {
|
|
442
1242
|
const out = {};
|
|
443
1243
|
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
444
|
-
out[relPath] = (0,
|
|
1244
|
+
out[relPath] = (0, import_node_crypto2.createHash)("sha256").update(fs.readFileSync(path.join(dir, relPath))).digest("hex");
|
|
445
1245
|
}
|
|
446
1246
|
return out;
|
|
447
1247
|
}
|
|
@@ -484,7 +1284,7 @@ function buildFileManifest(dir) {
|
|
|
484
1284
|
path: childRel,
|
|
485
1285
|
isDirectory: false,
|
|
486
1286
|
size: payload.length,
|
|
487
|
-
sha256: (0,
|
|
1287
|
+
sha256: (0, import_node_crypto2.createHash)("sha256").update(payload).digest("hex")
|
|
488
1288
|
});
|
|
489
1289
|
}
|
|
490
1290
|
}
|
|
@@ -518,7 +1318,13 @@ async function push(dir, opts = {}) {
|
|
|
518
1318
|
options,
|
|
519
1319
|
"POST",
|
|
520
1320
|
"/api/public/benchmark-containers/resolve",
|
|
521
|
-
{
|
|
1321
|
+
{
|
|
1322
|
+
slug,
|
|
1323
|
+
name: opts.name ?? pyproject.name ?? slug,
|
|
1324
|
+
...opts.description ? { description: opts.description } : {},
|
|
1325
|
+
...opts.tags?.length ? { tags: opts.tags } : {},
|
|
1326
|
+
...opts.visibility ? { visibility: opts.visibility } : {}
|
|
1327
|
+
}
|
|
522
1328
|
);
|
|
523
1329
|
const tarball = buildSourceTarball(dir);
|
|
524
1330
|
const bundleSha256 = sha256b64(tarball);
|
|
@@ -534,10 +1340,11 @@ async function push(dir, opts = {}) {
|
|
|
534
1340
|
baseVersionId,
|
|
535
1341
|
...opts.force ? { force: true } : {},
|
|
536
1342
|
...opts.message ? { changeReason: opts.message } : {},
|
|
537
|
-
...opts.author ? { author: opts.author } : {}
|
|
1343
|
+
...opts.author ? { author: opts.author } : {},
|
|
1344
|
+
...opts.imageRef ? { imageRef: opts.imageRef } : {}
|
|
538
1345
|
});
|
|
539
1346
|
if (version.uploadUrl) await putObject(version.uploadUrl, tarball, bundleSha256);
|
|
540
|
-
await apiRequest(
|
|
1347
|
+
const final = await apiRequest(
|
|
541
1348
|
options,
|
|
542
1349
|
"POST",
|
|
543
1350
|
`/api/public/benchmark-containers/versions/${version.versionId}/finalize`,
|
|
@@ -554,7 +1361,15 @@ async function push(dir, opts = {}) {
|
|
|
554
1361
|
forkedFromVersionId: baseVersionId,
|
|
555
1362
|
baseFiles: snapshotTreeHashes(dir)
|
|
556
1363
|
});
|
|
557
|
-
return {
|
|
1364
|
+
return {
|
|
1365
|
+
containerId: resolve2.containerId,
|
|
1366
|
+
slug,
|
|
1367
|
+
versionId: version.versionId,
|
|
1368
|
+
contentHash,
|
|
1369
|
+
branch: final.branch ?? branch,
|
|
1370
|
+
state: final.state ?? null,
|
|
1371
|
+
alreadyUploaded: Boolean(version.alreadyUploaded)
|
|
1372
|
+
};
|
|
558
1373
|
}
|
|
559
1374
|
function extractBundle(bundle, dir) {
|
|
560
1375
|
const tar = (0, import_node_zlib.gunzipSync)(bundle);
|
|
@@ -663,11 +1478,11 @@ async function createRef(slug, name, kind, versionId, options = {}) {
|
|
|
663
1478
|
...versionId ? { versionId } : {}
|
|
664
1479
|
});
|
|
665
1480
|
}
|
|
666
|
-
var
|
|
1481
|
+
var import_node_crypto2, import_node_zlib, fs, path, DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, BROAD_ROOT_MARKERS, sha256b64;
|
|
667
1482
|
var init_hub = __esm({
|
|
668
1483
|
"src/worlds/hub.ts"() {
|
|
669
1484
|
"use strict";
|
|
670
|
-
|
|
1485
|
+
import_node_crypto2 = require("crypto");
|
|
671
1486
|
import_node_zlib = require("zlib");
|
|
672
1487
|
fs = __toESM(require("fs"));
|
|
673
1488
|
path = __toESM(require("path"));
|
|
@@ -686,7 +1501,7 @@ var init_hub = __esm({
|
|
|
686
1501
|
]);
|
|
687
1502
|
ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
688
1503
|
BROAD_ROOT_MARKERS = ["gateway-env.toml", "gateway-api-world.toml"];
|
|
689
|
-
sha256b64 = (data) => (0,
|
|
1504
|
+
sha256b64 = (data) => (0, import_node_crypto2.createHash)("sha256").update(data).digest("base64");
|
|
690
1505
|
}
|
|
691
1506
|
});
|
|
692
1507
|
|
|
@@ -694,14 +1509,15 @@ var init_hub = __esm({
|
|
|
694
1509
|
function fmtScore(score) {
|
|
695
1510
|
return score === null ? "no score" : score.toFixed(3);
|
|
696
1511
|
}
|
|
697
|
-
function buildMarkdown(report, minScore) {
|
|
1512
|
+
function buildMarkdown(report, minScore, source) {
|
|
698
1513
|
const lines = [];
|
|
699
1514
|
lines.push(
|
|
700
1515
|
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
701
1516
|
);
|
|
702
1517
|
lines.push("");
|
|
1518
|
+
const commitLink = source?.repoUrl && source?.commit ? `[\`${source.commit.slice(0, 12)}\`](${source.repoUrl}/commit/${source.commit})` : null;
|
|
703
1519
|
lines.push(
|
|
704
|
-
`Ran the pull request's own world source (
|
|
1520
|
+
`Ran the pull request's own world source` + (commitLink ? ` (${commitLink} on \`${report.branch}\`` : ` (`) + `${commitLink ? ", " : ""}pinned as \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
705
1521
|
);
|
|
706
1522
|
lines.push("");
|
|
707
1523
|
lines.push("| Model | Status | Score | Run |");
|
|
@@ -721,19 +1537,29 @@ function buildMarkdown(report, minScore) {
|
|
|
721
1537
|
reward: sample.reward
|
|
722
1538
|
}))
|
|
723
1539
|
);
|
|
724
|
-
if (taskRows.length > 0
|
|
1540
|
+
if (taskRows.length > 0) {
|
|
1541
|
+
const fold = taskRows.length > 12;
|
|
725
1542
|
lines.push("");
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
1543
|
+
if (fold) {
|
|
1544
|
+
lines.push(
|
|
1545
|
+
`<details><summary>Per-scenario rewards (${taskRows.length})</summary>`
|
|
1546
|
+
);
|
|
1547
|
+
lines.push("");
|
|
1548
|
+
} else {
|
|
1549
|
+
lines.push(`**Scenarios (${taskRows.length})**`);
|
|
1550
|
+
lines.push("");
|
|
1551
|
+
}
|
|
1552
|
+
lines.push("| Scenario | Model | Reward |");
|
|
729
1553
|
lines.push("| --- | --- | ---: |");
|
|
730
1554
|
for (const row of taskRows) {
|
|
731
1555
|
lines.push(
|
|
732
1556
|
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
733
1557
|
);
|
|
734
1558
|
}
|
|
735
|
-
|
|
736
|
-
|
|
1559
|
+
if (fold) {
|
|
1560
|
+
lines.push("");
|
|
1561
|
+
lines.push("</details>");
|
|
1562
|
+
}
|
|
737
1563
|
}
|
|
738
1564
|
return lines.join("\n");
|
|
739
1565
|
}
|
|
@@ -847,7 +1673,13 @@ async function validateWorld(dir, opts) {
|
|
|
847
1673
|
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
848
1674
|
runs
|
|
849
1675
|
};
|
|
850
|
-
return {
|
|
1676
|
+
return {
|
|
1677
|
+
...base,
|
|
1678
|
+
markdown: buildMarkdown(base, minScore, {
|
|
1679
|
+
repoUrl: provenance.repoUrl,
|
|
1680
|
+
commit: provenance.commit
|
|
1681
|
+
})
|
|
1682
|
+
};
|
|
851
1683
|
}
|
|
852
1684
|
var init_validate = __esm({
|
|
853
1685
|
"src/worlds/validate.ts"() {
|
|
@@ -860,37 +1692,65 @@ var init_validate = __esm({
|
|
|
860
1692
|
// src/worlds/index.ts
|
|
861
1693
|
var worlds_exports = {};
|
|
862
1694
|
__export(worlds_exports, {
|
|
1695
|
+
BrowserActionRecorder: () => BrowserActionRecorder,
|
|
1696
|
+
COMPUTER_USE_SPECS: () => COMPUTER_USE_SPECS,
|
|
863
1697
|
DEFAULT_BRANCH: () => DEFAULT_BRANCH,
|
|
1698
|
+
DEFAULT_VIEWPORT: () => DEFAULT_VIEWPORT,
|
|
1699
|
+
EVENT_BATCH_SIZE: () => EVENT_BATCH_SIZE,
|
|
1700
|
+
EVENT_FLUSH_INTERVAL_MS: () => EVENT_FLUSH_INTERVAL_MS,
|
|
1701
|
+
EVENT_MAX_RETRIES: () => EVENT_MAX_RETRIES,
|
|
1702
|
+
SURFACE_KINDS: () => SURFACE_KINDS,
|
|
1703
|
+
WorldBrowser: () => WorldBrowser,
|
|
864
1704
|
WorldSession: () => WorldSession,
|
|
865
1705
|
WorldSessionCallError: () => WorldSessionCallError,
|
|
866
1706
|
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
867
1707
|
WorldSet: () => WorldSet,
|
|
1708
|
+
WorldSurfaceUnavailable: () => WorldSurfaceUnavailable,
|
|
868
1709
|
WorldTask: () => WorldTask,
|
|
869
1710
|
apiRequest: () => apiRequest,
|
|
870
1711
|
attachSession: () => attachSession,
|
|
871
1712
|
buildSourceTarball: () => buildSourceTarball,
|
|
872
1713
|
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
873
1714
|
computeContentHash: () => computeContentHash,
|
|
1715
|
+
computerUseToolkit: () => computerUseToolkit,
|
|
874
1716
|
createRef: () => createRef,
|
|
1717
|
+
createTask: () => createTask,
|
|
875
1718
|
deleteEnvSecret: () => deleteEnvSecret,
|
|
1719
|
+
deleteTask: () => deleteTask,
|
|
876
1720
|
detectProvenance: () => detectProvenance,
|
|
877
1721
|
getRun: () => getRun,
|
|
1722
|
+
getTask: () => getTask,
|
|
878
1723
|
isIncludedInContentHash: () => isIncludedInContentHash,
|
|
1724
|
+
isTaskIdRef: () => isTaskIdRef,
|
|
1725
|
+
isTaskSpec: () => isTaskSpec,
|
|
879
1726
|
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
880
1727
|
listRunConfigs: () => listRunConfigs,
|
|
1728
|
+
listTasks: () => listTasks,
|
|
881
1729
|
localTreeState: () => localTreeState,
|
|
1730
|
+
openBrowser: () => openBrowser,
|
|
882
1731
|
openSession: () => openSession,
|
|
883
1732
|
openWorld: () => openWorld,
|
|
1733
|
+
parseWorldTaskRefs: () => parseWorldTaskRefs,
|
|
884
1734
|
propose: () => propose,
|
|
885
1735
|
pull: () => pull,
|
|
886
1736
|
push: () => push,
|
|
887
1737
|
readLocalMeta: () => readLocalMeta,
|
|
1738
|
+
redactActionArgs: () => redactActionArgs,
|
|
1739
|
+
resolveBrowserMode: () => resolveBrowserMode,
|
|
888
1740
|
runSessions: () => runSessions,
|
|
889
1741
|
schemaFromSignature: () => schemaFromSignature,
|
|
890
1742
|
setEnvSecret: () => setEnvSecret,
|
|
891
1743
|
snapshotTreeHashes: () => snapshotTreeHashes,
|
|
1744
|
+
surfaceKinds: () => surfaceKinds,
|
|
1745
|
+
taskLabel: () => taskLabel,
|
|
1746
|
+
uiEntryUrl: () => uiEntryUrl,
|
|
1747
|
+
uiUrl: () => uiUrl,
|
|
1748
|
+
updateTask: () => updateTask,
|
|
1749
|
+
upsertRunConfig: () => upsertRunConfig,
|
|
1750
|
+
validateTask: () => validateTask,
|
|
892
1751
|
validateWorld: () => validateWorld,
|
|
893
1752
|
waitForRun: () => waitForRun,
|
|
1753
|
+
withoutUiToken: () => withoutUiToken,
|
|
894
1754
|
writeLocalMeta: () => writeLocalMeta
|
|
895
1755
|
});
|
|
896
1756
|
module.exports = __toCommonJS(worlds_exports);
|
|
@@ -962,6 +1822,28 @@ async function listRunConfigs(options = {}) {
|
|
|
962
1822
|
);
|
|
963
1823
|
return res.runConfigs;
|
|
964
1824
|
}
|
|
1825
|
+
async function upsertRunConfig(input, options = {}) {
|
|
1826
|
+
return apiRequest(
|
|
1827
|
+
options,
|
|
1828
|
+
"POST",
|
|
1829
|
+
"/api/public/benchmark-containers/run-configs",
|
|
1830
|
+
input
|
|
1831
|
+
);
|
|
1832
|
+
}
|
|
1833
|
+
function parseWorldTaskRefs(spec) {
|
|
1834
|
+
const items = (Array.isArray(spec) ? spec : spec.split(",")).map((s) => s.trim()).filter(Boolean);
|
|
1835
|
+
return items.map((item) => {
|
|
1836
|
+
const at = item.lastIndexOf("@");
|
|
1837
|
+
if (at < 0) return { id: item };
|
|
1838
|
+
const id = item.slice(0, at);
|
|
1839
|
+
const version = Number(item.slice(at + 1));
|
|
1840
|
+
if (!id || !Number.isInteger(version) || version < 1)
|
|
1841
|
+
throw new Error(
|
|
1842
|
+
`stored task ref '${item}' is not <id> or <id>@<version> (version a positive integer)`
|
|
1843
|
+
);
|
|
1844
|
+
return { id, version };
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
965
1847
|
async function getRun(runId, options = {}) {
|
|
966
1848
|
return apiRequest(
|
|
967
1849
|
options,
|
|
@@ -1054,6 +1936,8 @@ var WorldTask, WorldSet;
|
|
|
1054
1936
|
var init_worlds = __esm({
|
|
1055
1937
|
"src/worlds/index.ts"() {
|
|
1056
1938
|
init_session();
|
|
1939
|
+
init_tasks();
|
|
1940
|
+
init_browser();
|
|
1057
1941
|
init_hub();
|
|
1058
1942
|
init_validate();
|
|
1059
1943
|
WorldTask = class {
|
|
@@ -1083,15 +1967,18 @@ var init_worlds = __esm({
|
|
|
1083
1967
|
return hit;
|
|
1084
1968
|
}
|
|
1085
1969
|
/**
|
|
1086
|
-
* Open a LIVE session against one of this world's tasks
|
|
1087
|
-
*
|
|
1088
|
-
* platform. See
|
|
1970
|
+
* Open a LIVE session against one of this world's tasks — a bundled task's
|
|
1971
|
+
* name or an inline `WorldTaskSpec`: the platform boots (or reuses) a warm
|
|
1972
|
+
* container, and your agent calls its tools through the platform. See
|
|
1973
|
+
* worlds/session.ts.
|
|
1089
1974
|
*/
|
|
1090
1975
|
async open(task, opts = {}) {
|
|
1091
1976
|
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
1092
|
-
return openSession2(this.slug,
|
|
1977
|
+
return openSession2(this.slug, {
|
|
1978
|
+
task,
|
|
1093
1979
|
versionId: this.version.versionId,
|
|
1094
1980
|
reuseWarm: opts.reuseWarm,
|
|
1981
|
+
surfaces: opts.surfaces,
|
|
1095
1982
|
options: this.options
|
|
1096
1983
|
});
|
|
1097
1984
|
}
|
|
@@ -1120,37 +2007,65 @@ var init_worlds = __esm({
|
|
|
1120
2007
|
init_worlds();
|
|
1121
2008
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1122
2009
|
0 && (module.exports = {
|
|
2010
|
+
BrowserActionRecorder,
|
|
2011
|
+
COMPUTER_USE_SPECS,
|
|
1123
2012
|
DEFAULT_BRANCH,
|
|
2013
|
+
DEFAULT_VIEWPORT,
|
|
2014
|
+
EVENT_BATCH_SIZE,
|
|
2015
|
+
EVENT_FLUSH_INTERVAL_MS,
|
|
2016
|
+
EVENT_MAX_RETRIES,
|
|
2017
|
+
SURFACE_KINDS,
|
|
2018
|
+
WorldBrowser,
|
|
1124
2019
|
WorldSession,
|
|
1125
2020
|
WorldSessionCallError,
|
|
1126
2021
|
WorldSessionTimeout,
|
|
1127
2022
|
WorldSet,
|
|
2023
|
+
WorldSurfaceUnavailable,
|
|
1128
2024
|
WorldTask,
|
|
1129
2025
|
apiRequest,
|
|
1130
2026
|
attachSession,
|
|
1131
2027
|
buildSourceTarball,
|
|
1132
2028
|
collectArchiveRelPaths,
|
|
1133
2029
|
computeContentHash,
|
|
2030
|
+
computerUseToolkit,
|
|
1134
2031
|
createRef,
|
|
2032
|
+
createTask,
|
|
1135
2033
|
deleteEnvSecret,
|
|
2034
|
+
deleteTask,
|
|
1136
2035
|
detectProvenance,
|
|
1137
2036
|
getRun,
|
|
2037
|
+
getTask,
|
|
1138
2038
|
isIncludedInContentHash,
|
|
2039
|
+
isTaskIdRef,
|
|
2040
|
+
isTaskSpec,
|
|
1139
2041
|
listEnvSecretKeys,
|
|
1140
2042
|
listRunConfigs,
|
|
2043
|
+
listTasks,
|
|
1141
2044
|
localTreeState,
|
|
2045
|
+
openBrowser,
|
|
1142
2046
|
openSession,
|
|
1143
2047
|
openWorld,
|
|
2048
|
+
parseWorldTaskRefs,
|
|
1144
2049
|
propose,
|
|
1145
2050
|
pull,
|
|
1146
2051
|
push,
|
|
1147
2052
|
readLocalMeta,
|
|
2053
|
+
redactActionArgs,
|
|
2054
|
+
resolveBrowserMode,
|
|
1148
2055
|
runSessions,
|
|
1149
2056
|
schemaFromSignature,
|
|
1150
2057
|
setEnvSecret,
|
|
1151
2058
|
snapshotTreeHashes,
|
|
2059
|
+
surfaceKinds,
|
|
2060
|
+
taskLabel,
|
|
2061
|
+
uiEntryUrl,
|
|
2062
|
+
uiUrl,
|
|
2063
|
+
updateTask,
|
|
2064
|
+
upsertRunConfig,
|
|
2065
|
+
validateTask,
|
|
1152
2066
|
validateWorld,
|
|
1153
2067
|
waitForRun,
|
|
2068
|
+
withoutUiToken,
|
|
1154
2069
|
writeLocalMeta
|
|
1155
2070
|
});
|
|
1156
2071
|
//# sourceMappingURL=index.js.map
|