@withgateway/sdk 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +13267 -5869
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +13272 -5874
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +83 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -28
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -0
- package/dist/hooks/index.d.ts +1 -0
- package/dist/hooks/index.js +4355 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/hooks/index.mjs +4243 -0
- package/dist/hooks/index.mjs.map +1 -0
- package/dist/{index-Dpo-t8iO.d.ts → index-1xnxt0Cc.d.ts} +58 -2
- package/dist/index-B0R3IgsV.d.mts +1279 -0
- package/dist/index-B0R3IgsV.d.ts +1279 -0
- package/dist/{index-B5HUQQDE.d.mts → index-C1wDEwH4.d.mts} +58 -2
- package/dist/index.d.mts +4 -3
- package/dist/index.d.ts +4 -3
- package/dist/index.js +6698 -1606
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6793 -1713
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +72 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +68 -17
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +47 -3
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +42 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +390 -27
- package/dist/worlds/index.d.ts +390 -27
- package/dist/worlds/index.js +2291 -366
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +2281 -356
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +7 -1
- package/runtime/gatewaysdk/environment/_bundle.py +14 -24
- package/runtime/gatewaysdk/environment/_materialize.py +3 -3
- package/runtime/gatewaysdk/environment/schema/api.py +153 -20
- package/runtime/gatewaysdk/environment/schema/compiler.py +14 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +171 -535
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/platform.py +14 -22
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +156 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +6 -5
- package/runtime/gatewaysdk/environment/schema/tasks.py +12 -2
- package/runtime/gatewaysdk/environment/schema/tools_world.py +75 -23
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +44 -13
- package/runtime/gatewaysdk/environment/world.py +391 -0
- package/skills/hooks-getting-started/SKILL.md +156 -0
package/dist/worlds/index.d.ts
CHANGED
|
@@ -58,19 +58,32 @@ type WorldTaskPage = {
|
|
|
58
58
|
totalPages: number;
|
|
59
59
|
};
|
|
60
60
|
};
|
|
61
|
+
type WorldTaskWorldValidation = {
|
|
62
|
+
slug: string;
|
|
63
|
+
ref: string;
|
|
64
|
+
versionId: string;
|
|
65
|
+
ok: boolean;
|
|
66
|
+
checked: "entities" | "none";
|
|
67
|
+
problems: string[];
|
|
68
|
+
entities: string[] | null;
|
|
69
|
+
};
|
|
61
70
|
type WorldTaskValidation = {
|
|
62
71
|
taskId: string;
|
|
63
72
|
version: number;
|
|
64
|
-
/** The world version the task was checked against
|
|
65
|
-
|
|
73
|
+
/** The world version the task was checked against; null for a task with
|
|
74
|
+
* `worlds`, whose versions are per alias under `worlds`. */
|
|
75
|
+
versionId: string | null;
|
|
66
76
|
ok: boolean;
|
|
67
77
|
/** `entities` when the world's contract was read and every seed and check
|
|
68
78
|
* entity was matched against it; `none` when the version has no contract
|
|
69
79
|
* to check (a mock-tools world), in which case `ok` says only that the
|
|
70
|
-
* version exists and is READY
|
|
71
|
-
|
|
80
|
+
* version exists and is READY; `worlds` for a multi-world task, checked
|
|
81
|
+
* alias by alias against the ref each names. */
|
|
82
|
+
checked: "entities" | "none" | "worlds";
|
|
83
|
+
/** For a multi-world task each problem is prefixed with its alias. */
|
|
72
84
|
problems: string[];
|
|
73
85
|
entities: string[] | null;
|
|
86
|
+
worlds?: Record<string, WorldTaskWorldValidation>;
|
|
74
87
|
};
|
|
75
88
|
/** Store a task. `name` defaults to `spec.name`; one of the two is needed. */
|
|
76
89
|
declare function createTask(input: {
|
|
@@ -268,6 +281,18 @@ type WorldTaskSpec = {
|
|
|
268
281
|
grader?: WorldTaskGrader;
|
|
269
282
|
tools?: string[];
|
|
270
283
|
metadata?: Record<string, unknown>;
|
|
284
|
+
/** A MULTI-WORLD task: one session per alias, each on its own world with
|
|
285
|
+
* its own seed, grader and tools (the top-level three are then refused).
|
|
286
|
+
* Brought up together by `openTask` / `gateway worlds task up`; see
|
|
287
|
+
* worlds/taskSessions.ts. */
|
|
288
|
+
worlds?: Record<string, {
|
|
289
|
+
slug: string;
|
|
290
|
+
/** One of the world's refs; default `main`. */
|
|
291
|
+
ref?: string;
|
|
292
|
+
seed?: WorldTaskSeed;
|
|
293
|
+
grader?: WorldTaskGrader;
|
|
294
|
+
tools?: string[];
|
|
295
|
+
}>;
|
|
271
296
|
};
|
|
272
297
|
/** A stored task of the project (see worlds/tasks.ts), opened by id; the
|
|
273
298
|
* session runs a copy of one version's spec — the current one by default. */
|
|
@@ -345,6 +370,7 @@ type GradeResult = {
|
|
|
345
370
|
rewards: Record<string, number>;
|
|
346
371
|
raw: unknown;
|
|
347
372
|
};
|
|
373
|
+
declare const modelToolName: (name: string) => string;
|
|
348
374
|
/**
|
|
349
375
|
* Turn a world's Python signature into a JSON schema the model can call.
|
|
350
376
|
*
|
|
@@ -462,9 +488,12 @@ declare class WorldSession {
|
|
|
462
488
|
* A rubric grade is finalized by the platform's judge, not the container:
|
|
463
489
|
* the platform answers the enqueue with `pending: true`, and this waits on
|
|
464
490
|
* the judge's budget (15 min) instead of a call's. `timeoutMs` overrides
|
|
465
|
-
* either.
|
|
491
|
+
* either. `report` is the agent's own account of what it did and found,
|
|
492
|
+
* handed to the judge as its claim beside the evidence (the state and the
|
|
493
|
+
* calls); assertions and python graders ignore it. */
|
|
466
494
|
grade(opts?: {
|
|
467
495
|
timeoutMs?: number;
|
|
496
|
+
report?: string;
|
|
468
497
|
}): Promise<GradeResult>;
|
|
469
498
|
/**
|
|
470
499
|
* Put rows into the LIVE world through its contract, in one batch:
|
|
@@ -926,18 +955,340 @@ declare function computerUseToolkit(browser: WorldBrowser, opts?: {
|
|
|
926
955
|
maxText?: number;
|
|
927
956
|
}): WorldToolkit;
|
|
928
957
|
|
|
958
|
+
/** The per-world entry of a `WorldTaskSpec.worlds` table. */
|
|
959
|
+
type WorldTaskWorld = {
|
|
960
|
+
slug: string;
|
|
961
|
+
ref?: string;
|
|
962
|
+
seed?: WorldTaskSpec["seed"];
|
|
963
|
+
grader?: WorldTaskSpec["grader"];
|
|
964
|
+
tools?: string[];
|
|
965
|
+
};
|
|
966
|
+
/** One world's session as the platform answered it at up. */
|
|
967
|
+
type TaskWorldSessionState = WorldSessionState & {
|
|
968
|
+
alias: string;
|
|
969
|
+
slug: string;
|
|
970
|
+
ref: string;
|
|
971
|
+
reusedWarm: boolean;
|
|
972
|
+
};
|
|
973
|
+
/** What `POST /world-task-sessions` answers. */
|
|
974
|
+
type TaskSessionsResponse = {
|
|
975
|
+
taskId: string | null;
|
|
976
|
+
taskVersion: number | null;
|
|
977
|
+
name: string | null;
|
|
978
|
+
hash: string;
|
|
979
|
+
instruction: string;
|
|
980
|
+
sessions: Record<string, TaskWorldSessionState>;
|
|
981
|
+
};
|
|
982
|
+
/** One world in the manifest: where it answers, in the shape
|
|
983
|
+
* `gateway worlds session open` prints for a single session. */
|
|
984
|
+
type TaskManifestSession = {
|
|
985
|
+
slug: string;
|
|
986
|
+
ref: string | null;
|
|
987
|
+
versionId: string;
|
|
988
|
+
sessionId: string;
|
|
989
|
+
ready: boolean;
|
|
990
|
+
state: string;
|
|
991
|
+
surfaces: WorldSurfaceKind[];
|
|
992
|
+
api?: {
|
|
993
|
+
url: string;
|
|
994
|
+
token?: string;
|
|
995
|
+
};
|
|
996
|
+
ui?: {
|
|
997
|
+
url: string;
|
|
998
|
+
entry: string;
|
|
999
|
+
apiPath: string | null;
|
|
1000
|
+
};
|
|
1001
|
+
browser?: {
|
|
1002
|
+
wsUrl: string;
|
|
1003
|
+
viewport?: unknown;
|
|
1004
|
+
};
|
|
1005
|
+
};
|
|
1006
|
+
/** The manifest: every session of the task, keyed by alias. The file
|
|
1007
|
+
* `task up --out` writes and `task down|status|grade|export` read. */
|
|
1008
|
+
type TaskManifest = {
|
|
1009
|
+
taskId: string | null;
|
|
1010
|
+
taskVersion: number | null;
|
|
1011
|
+
name: string | null;
|
|
1012
|
+
instruction: string | null;
|
|
1013
|
+
sessions: Record<string, TaskManifestSession>;
|
|
1014
|
+
};
|
|
1015
|
+
/** A stored task's live sessions, as `GET /world-task-sessions` lists them. */
|
|
1016
|
+
type LiveTaskSession = {
|
|
1017
|
+
alias: string | null;
|
|
1018
|
+
slug: string | null;
|
|
1019
|
+
sessionId: string;
|
|
1020
|
+
state: string;
|
|
1021
|
+
ready: boolean;
|
|
1022
|
+
versionId: string;
|
|
1023
|
+
taskVersion: number | null;
|
|
1024
|
+
createdAt: string;
|
|
1025
|
+
};
|
|
1026
|
+
/** The task's grade: every world's own `GradeResult` under its alias, and
|
|
1027
|
+
* the aggregate. `reward` is the mean of the worlds' numeric rewards — null
|
|
1028
|
+
* when no world answered one (no grader anywhere, or a judge still
|
|
1029
|
+
* pending); a world with no reward is listed under `ungraded`, never
|
|
1030
|
+
* counted as 0. `rewards` flattens each world's named rewards as
|
|
1031
|
+
* `<alias>.<key>` beside `<alias>` for the world's own reward. */
|
|
1032
|
+
type TaskGrade = {
|
|
1033
|
+
reward: number | null;
|
|
1034
|
+
rewards: Record<string, number>;
|
|
1035
|
+
worlds: Record<string, GradeResult>;
|
|
1036
|
+
ungraded: string[];
|
|
1037
|
+
};
|
|
1038
|
+
/** Every world's call log, keyed by alias. */
|
|
1039
|
+
type TaskExport = Record<string, WorldSessionCall[]>;
|
|
1040
|
+
/** Combine per-world grades into the task's. Pure, so the CLI and the tests
|
|
1041
|
+
* share it with `TaskSessions.grade()`. */
|
|
1042
|
+
declare function aggregateTaskGrade(worlds: Record<string, GradeResult>): TaskGrade;
|
|
1043
|
+
declare class TaskSessions {
|
|
1044
|
+
/** The stored task's id, or null for a task given as a spec. */
|
|
1045
|
+
readonly taskId: string | null;
|
|
1046
|
+
readonly taskVersion: number | null;
|
|
1047
|
+
readonly name: string | null;
|
|
1048
|
+
/** The instruction every world's session shares. */
|
|
1049
|
+
readonly instruction: string | null;
|
|
1050
|
+
/** One live session per alias, in the spec's order. */
|
|
1051
|
+
readonly sessions: Record<string, WorldSession>;
|
|
1052
|
+
private readonly worlds;
|
|
1053
|
+
/** The spec's content hash — set only by `openTask` (the raw platform
|
|
1054
|
+
* response carries it); null for a `TaskSessions` rebuilt with
|
|
1055
|
+
* `attachManifest`/`attachTask`, which have no spec to hash. */
|
|
1056
|
+
readonly hash: string | null;
|
|
1057
|
+
constructor(
|
|
1058
|
+
/** The stored task's id, or null for a task given as a spec. */
|
|
1059
|
+
taskId: string | null, taskVersion: number | null, name: string | null,
|
|
1060
|
+
/** The instruction every world's session shares. */
|
|
1061
|
+
instruction: string | null,
|
|
1062
|
+
/** One live session per alias, in the spec's order. */
|
|
1063
|
+
sessions: Record<string, WorldSession>, worlds: Record<string, {
|
|
1064
|
+
slug: string;
|
|
1065
|
+
ref: string | null;
|
|
1066
|
+
}>,
|
|
1067
|
+
/** The spec's content hash — set only by `openTask` (the raw platform
|
|
1068
|
+
* response carries it); null for a `TaskSessions` rebuilt with
|
|
1069
|
+
* `attachManifest`/`attachTask`, which have no spec to hash. */
|
|
1070
|
+
hash?: string | null);
|
|
1071
|
+
/** This alias's `{slug, ref}` as recorded at open/attach — the world a
|
|
1072
|
+
* `TaskRun` filing needs to name, without exposing the whole map. */
|
|
1073
|
+
worldRef(alias: string): {
|
|
1074
|
+
slug: string;
|
|
1075
|
+
ref: string | null;
|
|
1076
|
+
};
|
|
1077
|
+
/** The aliases, in the spec's order. */
|
|
1078
|
+
get aliases(): string[];
|
|
1079
|
+
/** The alias's session, or a clear error naming the aliases there are. */
|
|
1080
|
+
session(alias: string): WorldSession;
|
|
1081
|
+
private each;
|
|
1082
|
+
/** Wait until every world answers ready. A world that never does throws
|
|
1083
|
+
* `WorldSessionTimeout` naming it; the others are left up. */
|
|
1084
|
+
ready(opts?: {
|
|
1085
|
+
timeoutMs?: number;
|
|
1086
|
+
}): Promise<void>;
|
|
1087
|
+
/** Every world's current state, keyed by alias. */
|
|
1088
|
+
status(): Promise<Record<string, WorldSessionState>>;
|
|
1089
|
+
/** The manifest: where every world answers, keyed by alias. */
|
|
1090
|
+
manifest(): Promise<TaskManifest>;
|
|
1091
|
+
/** Grade every world and aggregate — see `TaskGrade`. One `report` (the
|
|
1092
|
+
* agent's own account of the whole investigation) goes to every world's
|
|
1093
|
+
* judge; a world graded by assertions or python ignores it. */
|
|
1094
|
+
grade(opts?: {
|
|
1095
|
+
timeoutMs?: number;
|
|
1096
|
+
report?: string;
|
|
1097
|
+
}): Promise<TaskGrade>;
|
|
1098
|
+
/** Every world's call log, keyed by alias. */
|
|
1099
|
+
export(): Promise<TaskExport>;
|
|
1100
|
+
/** Close every world. Closes are attempted for all of them even when one
|
|
1101
|
+
* fails; the error then names every world that did not close. */
|
|
1102
|
+
close(opts?: {
|
|
1103
|
+
keepWarm?: boolean;
|
|
1104
|
+
}): Promise<Record<string, {
|
|
1105
|
+
state: string;
|
|
1106
|
+
}>>;
|
|
1107
|
+
}
|
|
1108
|
+
type OpenTaskOptions = {
|
|
1109
|
+
/** With a task id: open that version of the task instead of the current. */
|
|
1110
|
+
version?: number;
|
|
1111
|
+
/** Asked of every world; the platform refuses (400) rather than serving
|
|
1112
|
+
* fewer surfaces than asked, and then nothing is up. */
|
|
1113
|
+
surfaces?: WorldSurfaceKind[];
|
|
1114
|
+
reuseWarm?: boolean;
|
|
1115
|
+
options?: WorldsClientOptions;
|
|
1116
|
+
};
|
|
1117
|
+
/**
|
|
1118
|
+
* Bring every world of a task up. `ref` is a stored task's id, its
|
|
1119
|
+
* `{ id, version? }`, or the spec itself (one with `worlds`). Returns as
|
|
1120
|
+
* soon as the platform has opened every session — `ready()` waits for the
|
|
1121
|
+
* containers. All or nothing: a world that fails to open closes the rest
|
|
1122
|
+
* and the error names it.
|
|
1123
|
+
*/
|
|
1124
|
+
declare function openTask(ref: string | WorldTaskIdRef | WorldTaskSpec, opts?: OpenTaskOptions): Promise<TaskSessions>;
|
|
1125
|
+
/** A stored task's live sessions, newest first — every session the project
|
|
1126
|
+
* has opened on it that is not stopped or failed. */
|
|
1127
|
+
declare function listTaskSessions(taskId: string, options?: WorldsClientOptions): Promise<LiveTaskSession[]>;
|
|
1128
|
+
/**
|
|
1129
|
+
* Attach to the sessions a stored task has up — after a crash, or from a
|
|
1130
|
+
* process other than the one that ran `up`. Refuses when an alias is live
|
|
1131
|
+
* more than once (two `up`s of the same task): which set is meant is not
|
|
1132
|
+
* knowable, so pass the manifest (`attachManifest`) or close the extras.
|
|
1133
|
+
*/
|
|
1134
|
+
declare function attachTask(taskId: string, options?: WorldsClientOptions): Promise<TaskSessions>;
|
|
1135
|
+
/** Attach to the sessions a manifest names — the file `task up --out`
|
|
1136
|
+
* wrote. Each session is read back from the platform, so a closed one is a
|
|
1137
|
+
* clear error rather than a stale handle. */
|
|
1138
|
+
declare function attachManifest(manifest: TaskManifest, options?: WorldsClientOptions): Promise<TaskSessions>;
|
|
1139
|
+
/** Whether a parsed JSON value is a task manifest (what `task up` wrote)
|
|
1140
|
+
* rather than a task spec. */
|
|
1141
|
+
declare function isTaskManifest(value: unknown): value is TaskManifest;
|
|
1142
|
+
|
|
1143
|
+
/**
|
|
1144
|
+
* RUN A TASK END TO END: open, agent, grade, file, close.
|
|
1145
|
+
*
|
|
1146
|
+
* `runTask` is `gateway worlds task run` in one call: bring every world of a
|
|
1147
|
+
* task up, hand the agent a small `TaskRun` (its sessions, a merged
|
|
1148
|
+
* toolkit, a place to record notes), grade every world with the agent's own
|
|
1149
|
+
* report, file one rollout the same way the Python SDK's `World.run` does
|
|
1150
|
+
* for a single in-process world, and close everything down — a run
|
|
1151
|
+
* directory on disk is the paper trail (`manifest.json`, `grades.json`,
|
|
1152
|
+
* `report.md`, `run.json`). Mirrors `gatewaysdk.world_tasks.run_task`.
|
|
1153
|
+
*
|
|
1154
|
+
* const summary = await runTask("wt_1", async (run) => {
|
|
1155
|
+
* const { url, token } = await run.api("spycloud");
|
|
1156
|
+
* // ... call the agent, using run.sessions / run.toolkit() ...
|
|
1157
|
+
* return "found and locked the account";
|
|
1158
|
+
* }, { model: "claude-sonnet-4-5" });
|
|
1159
|
+
*/
|
|
1160
|
+
/** What the agent callable given to `runTask` is called with: the run's
|
|
1161
|
+
* identity, every world's live session, and a place to leave notes that
|
|
1162
|
+
* land in the run's `run.json`. */
|
|
1163
|
+
declare class TaskRun {
|
|
1164
|
+
readonly runId: string;
|
|
1165
|
+
readonly instruction: string;
|
|
1166
|
+
readonly model: string;
|
|
1167
|
+
readonly sessions: Record<string, WorldSession>;
|
|
1168
|
+
readonly manifest: TaskManifest;
|
|
1169
|
+
readonly notes: Record<string, unknown>;
|
|
1170
|
+
constructor(runId: string, instruction: string, model: string, sessions: Record<string, WorldSession>, manifest: TaskManifest);
|
|
1171
|
+
/** The alias's session, or a clear error naming the aliases there are. */
|
|
1172
|
+
session(alias: string): WorldSession;
|
|
1173
|
+
/** `alias`'s HTTP API. Raises the same error the session itself would
|
|
1174
|
+
* when it was not opened with `surfaces: ["api"]`. */
|
|
1175
|
+
api(alias: string): Promise<{
|
|
1176
|
+
url: string;
|
|
1177
|
+
token?: string;
|
|
1178
|
+
}>;
|
|
1179
|
+
/** Every world's tools merged into one toolkit, each name prefixed
|
|
1180
|
+
* `<alias>.<tool>` so a model can tell which world a call goes to. A
|
|
1181
|
+
* world that exposes no tools contributes nothing — there is nothing to
|
|
1182
|
+
* merge. */
|
|
1183
|
+
toolkit(): Promise<WorldToolkit>;
|
|
1184
|
+
/** Log arbitrary notes into this run's `run.json` (merged; a later call
|
|
1185
|
+
* overrides a key an earlier one set). */
|
|
1186
|
+
record(notes: Record<string, unknown>): void;
|
|
1187
|
+
}
|
|
1188
|
+
type RunTaskOptions = {
|
|
1189
|
+
/** Model label recorded on the filed run. */
|
|
1190
|
+
model: string;
|
|
1191
|
+
/** Asked of every world besides tools: api, ui, browser. */
|
|
1192
|
+
surfaces?: WorldSurfaceKind[];
|
|
1193
|
+
/** Directory the run's own `<runId>/` folder is created under. Default `"runs"`. */
|
|
1194
|
+
outDir?: string;
|
|
1195
|
+
/** Initialize tracing when GATEWAY_* credentials are configured. Default `true`. */
|
|
1196
|
+
trace?: boolean;
|
|
1197
|
+
/** File one rollout when the run ends. Default `true`. */
|
|
1198
|
+
file?: boolean;
|
|
1199
|
+
/** Leave every session up instead of closing it when the run ends. Default `false`. */
|
|
1200
|
+
keepUp?: boolean;
|
|
1201
|
+
reuseWarm?: boolean;
|
|
1202
|
+
options?: WorldsClientOptions;
|
|
1203
|
+
};
|
|
1204
|
+
/** What `runTask` returns and writes to `<runDir>/run.json`. */
|
|
1205
|
+
type RunTaskSummary = {
|
|
1206
|
+
runId: string;
|
|
1207
|
+
taskId: string | null;
|
|
1208
|
+
taskVersion: number | null;
|
|
1209
|
+
name: string | null;
|
|
1210
|
+
instruction: string;
|
|
1211
|
+
model: string;
|
|
1212
|
+
aliases: string[];
|
|
1213
|
+
reward: number | null;
|
|
1214
|
+
rewards: Record<string, number>;
|
|
1215
|
+
ungraded: string[];
|
|
1216
|
+
agentError: string | null;
|
|
1217
|
+
report: string | null;
|
|
1218
|
+
traced: boolean;
|
|
1219
|
+
runDir: string;
|
|
1220
|
+
durationSeconds: number;
|
|
1221
|
+
notes: Record<string, unknown>;
|
|
1222
|
+
filed: boolean;
|
|
1223
|
+
evaluationId?: string;
|
|
1224
|
+
[extra: string]: unknown;
|
|
1225
|
+
};
|
|
1226
|
+
/** The report an agent callable may return: a string, or an object with a
|
|
1227
|
+
* `report` key (any other keys are kept in `run.json` as-is). */
|
|
1228
|
+
type TaskRunResult = string | {
|
|
1229
|
+
report?: unknown;
|
|
1230
|
+
[key: string]: unknown;
|
|
1231
|
+
} | null | undefined;
|
|
1232
|
+
/**
|
|
1233
|
+
* Open every world of a task, run `agent` once against all of them, grade
|
|
1234
|
+
* with its report, file one rollout and close — the engine behind `gateway
|
|
1235
|
+
* worlds task run`. `agent` is called as `agent(taskRun)` with a `TaskRun`;
|
|
1236
|
+
* its return value becomes the report handed to every world's grader.
|
|
1237
|
+
*
|
|
1238
|
+
* Every step has a flag and a default, nothing implicit: tracing runs when
|
|
1239
|
+
* `GATEWAY_*` credentials are configured unless `trace: false`; filing
|
|
1240
|
+
* happens unless `file: false`; every session closes unless `keepUp: true`
|
|
1241
|
+
* — even when the agent throws, so grading and filing still happen with the
|
|
1242
|
+
* error recorded (`run.json.agentError`, the filed evaluation's status
|
|
1243
|
+
* FAILED).
|
|
1244
|
+
*
|
|
1245
|
+
* Writes `<outDir>/<runId>/{manifest,grades,run}.json` (and `report.md`
|
|
1246
|
+
* when the agent gave one).
|
|
1247
|
+
*/
|
|
1248
|
+
declare function runTask(ref: string | WorldTaskIdRef | WorldTaskSpec, agent: (run: TaskRun) => TaskRunResult | Promise<TaskRunResult>, opts: RunTaskOptions): Promise<RunTaskSummary>;
|
|
1249
|
+
|
|
1250
|
+
/** One rollout, in the platform's sample shape. */
|
|
1251
|
+
type EvalSample = {
|
|
1252
|
+
exampleId: string;
|
|
1253
|
+
task?: string;
|
|
1254
|
+
rolloutIndex?: number;
|
|
1255
|
+
prompt?: unknown;
|
|
1256
|
+
completion?: unknown;
|
|
1257
|
+
reward?: number;
|
|
1258
|
+
score?: number;
|
|
1259
|
+
correct?: boolean;
|
|
1260
|
+
rewards?: Record<string, number>;
|
|
1261
|
+
info?: Record<string, unknown>;
|
|
1262
|
+
};
|
|
1263
|
+
type CreateEvaluationOptions = {
|
|
1264
|
+
model?: string;
|
|
1265
|
+
rolloutsPerExample?: number;
|
|
1266
|
+
metadata?: Record<string, unknown>;
|
|
1267
|
+
};
|
|
1268
|
+
type FinalizeEvaluationOptions = {
|
|
1269
|
+
status?: "COMPLETED" | "FAILED";
|
|
1270
|
+
metrics?: Record<string, unknown>;
|
|
1271
|
+
};
|
|
1272
|
+
declare class GatewayEvalsClient {
|
|
1273
|
+
private readonly options;
|
|
1274
|
+
constructor(options?: WorldsClientOptions);
|
|
1275
|
+
/** Anchor a new evaluation to `versionId`; returns its id. */
|
|
1276
|
+
createEvaluation(versionId: string, opts?: CreateEvaluationOptions): Promise<string>;
|
|
1277
|
+
/** Push one batch of samples; returns how many the platform inserted. */
|
|
1278
|
+
pushSamples(evaluationId: string, samples: EvalSample[]): Promise<number>;
|
|
1279
|
+
/** Close the evaluation out; returns its final status. */
|
|
1280
|
+
finalizeEvaluation(evaluationId: string, opts?: FinalizeEvaluationOptions): Promise<string>;
|
|
1281
|
+
}
|
|
1282
|
+
|
|
929
1283
|
/**
|
|
930
1284
|
* The working-dir half of the TS CLI: push / pull / checkout / status /
|
|
931
1285
|
* branch / tag / propose — a faithful port of the Python client
|
|
932
1286
|
* (`gatewaysdk/benchmark_hub/client.py` + `_archive.py`).
|
|
933
1287
|
*
|
|
934
|
-
*
|
|
935
|
-
*
|
|
936
|
-
*
|
|
937
|
-
*
|
|
938
|
-
* this file. This one is a line-for-line port of the WEB TypeScript
|
|
939
|
-
* implementation (same language, same crypto) — if the rule ever changes,
|
|
940
|
-
* change all four or push identity splits by client.
|
|
1288
|
+
* The content hash is the version's identity and is byte-compatible with
|
|
1289
|
+
* Prime Intellect: `WorldDir.contentHash` (./worldDir.ts) computes it here,
|
|
1290
|
+
* `WorldTree.contentHash` on the platform and `_archive.compute_content_hash`
|
|
1291
|
+
* in the Python SDK; the conformance corpus pins the three to the same bytes.
|
|
941
1292
|
*
|
|
942
1293
|
* One deliberate refusal, same policy as the web upload path: a bundle with
|
|
943
1294
|
* a ROOT `.gitignore` is refused rather than mis-hashed — the Python CLI
|
|
@@ -945,9 +1296,8 @@ declare function computerUseToolkit(browser: WorldBrowser, opts?: {
|
|
|
945
1296
|
* would produce a wrong, Prime-incompatible hash silently.
|
|
946
1297
|
*/
|
|
947
1298
|
declare const DEFAULT_BRANCH = "main";
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
* a root .gitignore (see the header). */
|
|
1299
|
+
/** The archive-visible files of a working dir, as sorted relpaths
|
|
1300
|
+
* (`WorldDir.archivePaths`). Throws on a root .gitignore (see the header). */
|
|
951
1301
|
declare function collectArchiveRelPaths(dir: string): string[];
|
|
952
1302
|
declare function computeContentHash(dir: string): string;
|
|
953
1303
|
/** A deterministic .tar.gz of the source files (node's gzip stamps no mtime,
|
|
@@ -961,6 +1311,11 @@ type LocalMeta = Record<string, unknown> & {
|
|
|
961
1311
|
baseFiles?: Record<string, string>;
|
|
962
1312
|
};
|
|
963
1313
|
declare function readLocalMeta(dir: string): LocalMeta | null;
|
|
1314
|
+
/** The slug a push of `dir` lands on: an explicit one, else the slug the dir
|
|
1315
|
+
* was pushed to or pulled from (its `.gateway` meta), else the pyproject
|
|
1316
|
+
* name, else the directory's name — so a tracked checkout never creates a
|
|
1317
|
+
* second world named after its folder. */
|
|
1318
|
+
declare function pushSlug(dir: string, explicit?: string): string;
|
|
964
1319
|
declare function writeLocalMeta(dir: string, meta: LocalMeta): void;
|
|
965
1320
|
declare function snapshotTreeHashes(dir: string): Record<string, string>;
|
|
966
1321
|
declare function localTreeState(dir: string): {
|
|
@@ -1275,9 +1630,9 @@ declare function parseTransform(name: string): Transform;
|
|
|
1275
1630
|
declare function applyTransforms(value: string | number, chain: readonly string[]): string;
|
|
1276
1631
|
|
|
1277
1632
|
/**
|
|
1278
|
-
* `queryRows(slug, entity, {where, resolve, limit, offset})` — the rows
|
|
1279
|
-
* one entity that match a predicate, from
|
|
1280
|
-
* `GET /api/public/worlds/{slug}/data?entity=&where=&limit=&offset=&resolve=`.
|
|
1633
|
+
* `queryRows(slug[@ref], entity, {where, resolve, limit, offset})` — the rows
|
|
1634
|
+
* of one entity that match a predicate, from
|
|
1635
|
+
* `GET /api/public/worlds/{slug}/data?ref=&entity=&where=&limit=&offset=&resolve=`.
|
|
1281
1636
|
* `where` is field → value, or field → list of values (any of); values match
|
|
1282
1637
|
* the STORED form as given. With `resolve: true` the values are plaintext and
|
|
1283
1638
|
* the runtime maps each to its stored form under the world's `[redact]`
|
|
@@ -1385,11 +1740,13 @@ type WorldQueryAcrossResult = {
|
|
|
1385
1740
|
entityCounts: Record<string, number>;
|
|
1386
1741
|
snapshot: WorldQuerySnapshot | null;
|
|
1387
1742
|
};
|
|
1388
|
-
/** The query string the route reads; `where` travels as URL-encoded JSON
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
*
|
|
1743
|
+
/** The query string the route reads; `where` travels as URL-encoded JSON and
|
|
1744
|
+
* `ref` (the `@ref` of `slug@ref`) picks the version. */
|
|
1745
|
+
declare function queryRowsSearch(entity: string, opts?: WorldQueryOptions, ref?: string): string;
|
|
1746
|
+
/** One version of the world, queried: the main tip for a bare slug, or the
|
|
1747
|
+
* version `slug@ref` names (branch, tag, semver pin or hash prefix — the
|
|
1748
|
+
* `bench pull` grammar; a ref nothing answers to is the platform's 404, never
|
|
1749
|
+
* the head). With `across: true` the answer is the groups shape. */
|
|
1393
1750
|
declare function queryRows(slug: string, entity: string, opts: WorldQueryAcrossOptions, options?: WorldsClientOptions): Promise<WorldQueryAcrossResult>;
|
|
1394
1751
|
declare function queryRows(slug: string, entity: string, opts?: WorldQueryOptions, options?: WorldsClientOptions): Promise<WorldQueryResult>;
|
|
1395
1752
|
|
|
@@ -1597,7 +1954,11 @@ declare function captureToolCalls<T extends ToolFunctions>(tools: T, sink: ToolC
|
|
|
1597
1954
|
source?: string;
|
|
1598
1955
|
}): T;
|
|
1599
1956
|
/** What a TOOL observation carries, as `GET /api/public/observations` and
|
|
1600
|
-
* `GET /api/public/sessions/{id}?includeIO=true` return it.
|
|
1957
|
+
* `GET /api/public/sessions/{id}?includeIO=true` return it. Both routes decode
|
|
1958
|
+
* stored JSON text once on the way out (`parseIoAsJson` defaults to true on the
|
|
1959
|
+
* v1 route; the session route renders with the same default), so `input` and
|
|
1960
|
+
* `output` arrive as the values the tool saw — an object, or a string when the
|
|
1961
|
+
* tool really produced text. Nothing here parses them again. */
|
|
1601
1962
|
type ObservationLike = {
|
|
1602
1963
|
id?: string;
|
|
1603
1964
|
traceId?: string | null;
|
|
@@ -1619,8 +1980,10 @@ type ObservationConversion = {
|
|
|
1619
1980
|
/**
|
|
1620
1981
|
* TOOL observations → tool-call records: `name` is the tool, `input` the args,
|
|
1621
1982
|
* `output` the result, `startTime` the time, `trace:<traceId>/<id>` the source.
|
|
1622
|
-
*
|
|
1623
|
-
*
|
|
1983
|
+
* `input`/`output` arrive from the public API already decoded (it parses the
|
|
1984
|
+
* stored text once), so they are taken as they are: a string `output` stays a
|
|
1985
|
+
* string even when it looks like JSON; an `input` that is not an object leaves
|
|
1986
|
+
* `args` empty rather than guessed.
|
|
1624
1987
|
* Only `type: "TOOL"` converts. With `tools` (the world's operation names) an
|
|
1625
1988
|
* observation for any other tool is counted in `unknownTools`, not converted.
|
|
1626
1989
|
* An `ERROR`-level observation converts with `error` set, so ingest counts it
|
|
@@ -1910,4 +2273,4 @@ declare class WorldSet {
|
|
|
1910
2273
|
*/
|
|
1911
2274
|
declare function openWorld(slugRef: string, options?: WorldsClientOptions): Promise<WorldSet>;
|
|
1912
2275
|
|
|
1913
|
-
export { ApiError, type BrowserActionEvent, BrowserActionRecorder, type BrowserMode, type BrowserPage, COMPUTER_USE_SPECS, DEFAULT_BRANCH, DEFAULT_VIEWPORT, type DispatchedRun, EVENT_BATCH_SIZE, EVENT_FLUSH_INTERVAL_MS, EVENT_MAX_RETRIES, type GradeResult, type IngestExplode, type IngestFieldSpec, type IngestMapping, type IngestMode, type IngestOptions, type IngestRefusal, type IngestReport, type IngestSource, type IngestSourceReport, type IngestToolCallsOptions, type IngestToolCallsReport, type LocalMeta, type MigrateWorldOptions, type ObservationConversion, type ObservationLike, type OpenSessionOptions, type ParsedToolCall, type Plaintext, type PlaywrightBrowserLike, type PlaywrightLike, type RedactMode, type RedactPolicy, type Redaction, type ResolvedWorldVersion, type RunConfigInput, type RunConfigSummary, type RunProvenance, type RunReport, STRIP_PREFIX, SURFACE_KINDS, type SeedResult, type SessionRunReport, type SessionRunResult, type SessionSurfaces, TRANSFORMS, type ToolCallRecord, type ToolCallSink, type ToolFunctions, type Transform, type ValidateReport, type ValidatedRun, WHITESPACE, WorldBrowser, type WorldDependency, type WorldDeprecation, type WorldDescription, type WorldEntity, type WorldEntityMapping, type WorldField, type WorldMigrationRefusal, type WorldMigrationReport, type WorldQueryAcrossOptions, type WorldQueryAcrossResult, type WorldQueryCompare, type WorldQueryCompareStatus, type WorldQueryGroup, type WorldQueryOptions, type WorldQueryResult, type WorldQuerySnapshot, type WorldQueryWhere, type WorldRelationship, WorldSession, type WorldSessionCall, WorldSessionCallError, type WorldSessionState, type WorldSessionTask, WorldSessionTimeout, WorldSet, type WorldSurfaceKind, WorldSurfaceUnavailable, WorldTask, type WorldTaskCheck, type WorldTaskDetail, type WorldTaskFieldOp, type WorldTaskGrader, type WorldTaskIdRef, type WorldTaskInfo, type WorldTaskPage, type WorldTaskRef, type WorldTaskRubricCriterion, type WorldTaskRubricGrader, type WorldTaskScalar, type WorldTaskSeed, type WorldTaskSpec, type WorldTaskSummary, type WorldTaskValidation, type WorldTaskVersionSummary, type WorldToolSpec, type WorldToolkit, type WorldsClientOptions, apiRequest, applyTransforms, attachSession, buildSourceTarball, captureToolCalls, collectArchiveRelPaths, computeContentHash, computerUseToolkit, createRef, createTask, deleteEnvSecret, deleteTask, describeWorld, detectProvenance, getRun, getTask, hashForm, ingest, ingestToolCalls,
|
|
2276
|
+
export { ApiError, type BrowserActionEvent, BrowserActionRecorder, type BrowserMode, type BrowserPage, COMPUTER_USE_SPECS, type CreateEvaluationOptions, DEFAULT_BRANCH, DEFAULT_VIEWPORT, type DispatchedRun, EVENT_BATCH_SIZE, EVENT_FLUSH_INTERVAL_MS, EVENT_MAX_RETRIES, type EvalSample, type FinalizeEvaluationOptions, GatewayEvalsClient, type GradeResult, type IngestExplode, type IngestFieldSpec, type IngestMapping, type IngestMode, type IngestOptions, type IngestRefusal, type IngestReport, type IngestSource, type IngestSourceReport, type IngestToolCallsOptions, type IngestToolCallsReport, type LiveTaskSession, type LocalMeta, type MigrateWorldOptions, type ObservationConversion, type ObservationLike, type OpenSessionOptions, type OpenTaskOptions, type ParsedToolCall, type Plaintext, type PlaywrightBrowserLike, type PlaywrightLike, type RedactMode, type RedactPolicy, type Redaction, type ResolvedWorldVersion, type RunConfigInput, type RunConfigSummary, type RunProvenance, type RunReport, type RunTaskOptions, type RunTaskSummary, STRIP_PREFIX, SURFACE_KINDS, type SeedResult, type SessionRunReport, type SessionRunResult, type SessionSurfaces, TRANSFORMS, type TaskExport, type TaskGrade, type TaskManifest, type TaskManifestSession, TaskRun, type TaskRunResult, TaskSessions, type TaskSessionsResponse, type TaskWorldSessionState, type ToolCallRecord, type ToolCallSink, type ToolFunctions, type Transform, type ValidateReport, type ValidatedRun, WHITESPACE, WorldBrowser, type WorldDependency, type WorldDeprecation, type WorldDescription, type WorldEntity, type WorldEntityMapping, type WorldField, type WorldMigrationRefusal, type WorldMigrationReport, type WorldQueryAcrossOptions, type WorldQueryAcrossResult, type WorldQueryCompare, type WorldQueryCompareStatus, type WorldQueryGroup, type WorldQueryOptions, type WorldQueryResult, type WorldQuerySnapshot, type WorldQueryWhere, type WorldRelationship, WorldSession, type WorldSessionCall, WorldSessionCallError, type WorldSessionState, type WorldSessionTask, WorldSessionTimeout, WorldSet, type WorldSurfaceKind, WorldSurfaceUnavailable, WorldTask, type WorldTaskCheck, type WorldTaskDetail, type WorldTaskFieldOp, type WorldTaskGrader, type WorldTaskIdRef, type WorldTaskInfo, type WorldTaskPage, type WorldTaskRef, type WorldTaskRubricCriterion, type WorldTaskRubricGrader, type WorldTaskScalar, type WorldTaskSeed, type WorldTaskSpec, type WorldTaskSummary, type WorldTaskValidation, type WorldTaskVersionSummary, type WorldTaskWorld, type WorldTaskWorldValidation, type WorldToolSpec, type WorldToolkit, type WorldsClientOptions, aggregateTaskGrade, apiRequest, applyTransforms, attachManifest, attachSession, attachTask, buildSourceTarball, captureToolCalls, collectArchiveRelPaths, computeContentHash, computerUseToolkit, createRef, createTask, deleteEnvSecret, deleteTask, describeWorld, detectProvenance, getRun, getTask, hashForm, ingest, ingestToolCalls, isTaskIdRef, isTaskManifest, isTaskSpec, listEnvSecretKeys, listRunConfigs, listTaskSessions, listTasks, localTreeState, migrateProjectWorlds, migrateWorld, modelToolName, openBrowser, openSession, openTask, openWorld, parseToolCallRecord, parseTransform, parseWorldTaskRefs, preserveForm, propose, pull, push, pushSlug, pyText, queryRows, queryRowsSearch, readLocalMeta, readToolCallRecords, redactActionArgs, redactionOf, resolveBrowserMode, roundForm, runSessions, runTask, schemaFromSignature, sessionToolObservations, setEnvSecret, shapeText, snapshotTreeHashes, storedForms, surfaceKinds, taskLabel, toolCallRecordsIn, toolCallsFromObservations, uiEntryUrl, uiUrl, updateTask, upsertRunConfig, validateTask, validateWorld, waitForRun, withoutUiToken, writeLocalMeta };
|