@withgateway/sdk 0.2.5 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/cli.js +3121 -1283
  2. package/dist/cli.js.map +1 -1
  3. package/dist/cli.mjs +3185 -1347
  4. package/dist/cli.mjs.map +1 -1
  5. package/dist/files/index.js +9 -0
  6. package/dist/files/index.js.map +1 -1
  7. package/dist/files/index.mjs +9 -0
  8. package/dist/files/index.mjs.map +1 -1
  9. package/dist/hooks/index.d.mts +1 -1
  10. package/dist/hooks/index.d.ts +1 -1
  11. package/dist/hooks/index.js +280 -8
  12. package/dist/hooks/index.js.map +1 -1
  13. package/dist/hooks/index.mjs +272 -8
  14. package/dist/hooks/index.mjs.map +1 -1
  15. package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
  16. package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
  17. package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
  18. package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
  19. package/dist/index.d.mts +4 -4
  20. package/dist/index.d.ts +4 -4
  21. package/dist/index.js +726 -167
  22. package/dist/index.js.map +1 -1
  23. package/dist/index.mjs +718 -167
  24. package/dist/index.mjs.map +1 -1
  25. package/dist/traces/index.js +9 -0
  26. package/dist/traces/index.js.map +1 -1
  27. package/dist/traces/index.mjs +9 -0
  28. package/dist/traces/index.mjs.map +1 -1
  29. package/dist/tracing/index.d.mts +1 -1
  30. package/dist/tracing/index.d.ts +1 -1
  31. package/dist/tracing/index.js +97 -61
  32. package/dist/tracing/index.js.map +1 -1
  33. package/dist/tracing/index.mjs +97 -61
  34. package/dist/tracing/index.mjs.map +1 -1
  35. package/dist/worlds/index.d.mts +154 -4
  36. package/dist/worlds/index.d.ts +154 -4
  37. package/dist/worlds/index.js +348 -97
  38. package/dist/worlds/index.js.map +1 -1
  39. package/dist/worlds/index.mjs +348 -97
  40. package/dist/worlds/index.mjs.map +1 -1
  41. package/package.json +1 -1
  42. package/runtime/gatewaysdk/environment/_bundle.py +1 -1
  43. package/runtime/gatewaysdk/environment/_materialize.py +53 -9
  44. package/runtime/gatewaysdk/environment/_toml.py +13 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +3 -7
  46. package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
  47. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
  48. package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
  49. package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
  51. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
  52. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
  53. package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
  54. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
  55. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  56. package/runtime/gatewaysdk/environment/schema/platform.py +16 -4
  57. package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
  58. package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
  59. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
  60. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +61 -2
  61. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
  62. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
  63. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
  64. package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
  65. package/runtime/gatewaysdk/environment/schema/tools_world.py +1 -5
  66. package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
  67. package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
  68. package/runtime/gatewaysdk/environment/world.py +120 -4
  69. package/skills/benchmarks-getting-started/SKILL.md +67 -9
  70. package/skills/connector-schema-authoring/connector-worlds.md +8 -1
  71. package/skills/connector-schema-authoring/world-schema.md +1 -1
  72. package/skills/hooks-getting-started/SKILL.md +61 -2
  73. package/skills/world-data-ingestion/SKILL.md +4 -1
  74. package/skills/worlds-getting-started/SKILL.md +11 -11
  75. package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
@@ -587,7 +587,11 @@ type SessionRunReport = {
587
587
  results: SessionRunResult[];
588
588
  /** Mean reward over every task, counting failures as 0. */
589
589
  mean: number;
590
- /** True when `mean >= failUnder` (or no gate was set). */
590
+ /** Tasks that never graded: the session did not open or become ready, the
591
+ * agent threw, or grading failed. Any failure fails the run — a 0 an agent
592
+ * earned and a session that never opened are not the same result. */
593
+ failed: number;
594
+ /** True when no task failed and `mean >= failUnder` (or no gate was set). */
591
595
  passed: boolean;
592
596
  };
593
597
  /**
@@ -1223,8 +1227,10 @@ type RunTaskSummary = {
1223
1227
  evaluationId?: string;
1224
1228
  [extra: string]: unknown;
1225
1229
  };
1226
- /** The report an agent callable may return: a string, or an object with a
1227
- * `report` key (any other keys are kept in `run.json` as-is). */
1230
+ /** The report an agent callable may return: a string, or an object with an
1231
+ * optional string `report` key (any other keys are kept in `run.json` as-is),
1232
+ * or nothing. Anything else is an agent error — a number, a bigint or an
1233
+ * array is never stringified into a graded completion. */
1228
1234
  type TaskRunResult = string | {
1229
1235
  report?: unknown;
1230
1236
  [key: string]: unknown;
@@ -1300,6 +1306,7 @@ declare const DEFAULT_BRANCH = "main";
1300
1306
  * (`WorldDir.archivePaths`). Throws on a root .gitignore (see the header). */
1301
1307
  declare function collectArchiveRelPaths(dir: string): string[];
1302
1308
  declare function computeContentHash(dir: string): string;
1309
+ declare function archiveMode(mode: number): number;
1303
1310
  /** A deterministic .tar.gz of the source files (node's gzip stamps no mtime,
1304
1311
  * matching the Python's mtime=0 gzip). */
1305
1312
  declare function buildSourceTarball(dir: string): Buffer;
@@ -1316,6 +1323,39 @@ declare function readLocalMeta(dir: string): LocalMeta | null;
1316
1323
  * name, else the directory's name — so a tracked checkout never creates a
1317
1324
  * second world named after its folder. */
1318
1325
  declare function pushSlug(dir: string, explicit?: string): string;
1326
+ type SlugDeclaration = {
1327
+ source: string;
1328
+ slug: string;
1329
+ };
1330
+ type PushSlugResolution = {
1331
+ slug: string;
1332
+ /** Where the slug came from (`--slug`, `.gateway metadata`, `benchmark.json`, `pyproject.toml`, `directory name`). */
1333
+ source: string;
1334
+ /** The slug the dir's `.gateway/.env-metadata.json` tracks from an earlier push or pull, if any. */
1335
+ tracked: string | null;
1336
+ /** What the tree itself declares (a caller's benchmark.json, then pyproject.toml), in precedence order. */
1337
+ declared: SlugDeclaration[];
1338
+ /** A declaration the tracked slug does not match — set only when the tracked slug won without `--slug`. */
1339
+ conflict: SlugDeclaration | null;
1340
+ };
1341
+ /** `pushSlug` with its provenance. `declared` are the caller's own slug sources
1342
+ * (e.g. `{ "benchmark.json": "x" }`), consulted before pyproject. A tracked slug
1343
+ * that disagrees with what the tree declares is reported as `conflict` so a
1344
+ * front door can refuse: that is how a directory pushed once with `--slug X`
1345
+ * silently landed every later push on X while its benchmark.json said Y. The
1346
+ * platform's collision suffix (`acme-2` minted for a taken `acme`) is not a
1347
+ * conflict; any other difference is (`acme-e2e` for `acme` was the mix-up). */
1348
+ declare function resolvePushSlug(dir: string, opts?: {
1349
+ explicit?: string;
1350
+ declared?: Record<string, string | null | undefined>;
1351
+ }): PushSlugResolution;
1352
+ /** The refusal a front door raises on `PushSlugResolution.conflict`: both slugs, both sources, every way out. */
1353
+ declare class PushSlugConflict extends Error {
1354
+ readonly dir: string;
1355
+ readonly tracked: string;
1356
+ readonly declared: SlugDeclaration;
1357
+ constructor(dir: string, tracked: string, declared: SlugDeclaration);
1358
+ }
1319
1359
  declare function writeLocalMeta(dir: string, meta: LocalMeta): void;
1320
1360
  declare function snapshotTreeHashes(dir: string): Record<string, string>;
1321
1361
  declare function localTreeState(dir: string): {
@@ -1323,6 +1363,16 @@ declare function localTreeState(dir: string): {
1323
1363
  dirty: string[];
1324
1364
  clean: boolean;
1325
1365
  };
1366
+ /** The rows the platform previews for a pushed DRAFT: the archive set, one
1367
+ * entry per file the content hash covers, so hash, tarball, manifest and
1368
+ * `pull` agree on what the world is. Finalize replaces them with rows
1369
+ * derived from the stored bytes. */
1370
+ declare function buildFileManifest(dir: string): {
1371
+ path: string;
1372
+ isDirectory: boolean;
1373
+ size: number;
1374
+ sha256: string;
1375
+ }[];
1326
1376
  declare function push(dir: string, opts?: {
1327
1377
  slug?: string;
1328
1378
  branch?: string;
@@ -1352,6 +1402,8 @@ declare function push(dir: string, opts?: {
1352
1402
  state: string | null;
1353
1403
  alreadyUploaded: boolean;
1354
1404
  }>;
1405
+ /** Extract a downloaded bundle into `dir` (paths sanitized, executable bit kept). */
1406
+ declare function extractBundle(bundle: Buffer, dir: string): number;
1355
1407
  declare function pull(slugRef: string, target?: string, options?: WorldsClientOptions): Promise<{
1356
1408
  slug: string;
1357
1409
  dir: string;
@@ -2088,6 +2140,97 @@ declare function ingestToolCalls(world: string, records: unknown[], options?: In
2088
2140
  /** The TOOL observations of one session, every trace's, from `GET /api/public/sessions/{id}?includeIO=true`. */
2089
2141
  declare function sessionToolObservations(sessionId: string, options?: WorldsClientOptions): Promise<ObservationLike[]>;
2090
2142
 
2143
+ /**
2144
+ * The CLI sandbox: a server-side workspace opened on a READY version of a
2145
+ * world (`/workspace/env` in the console agent's sandbox), edited and run
2146
+ * there, and committed straight into the version service — the source
2147
+ * never lands on the caller's disk. One client under both `gateway worlds
2148
+ * sandbox …` and `gateway bench sandbox …`; the Python twin is
2149
+ * `gatewaysdk.benchmark_hub.sandbox.WorldSandboxClient`.
2150
+ */
2151
+ /** Mirrors `SandboxSummaryResponse` (web/features/public-api/types/world-sandboxes.ts). */
2152
+ type SandboxSummary = {
2153
+ sandboxId: string;
2154
+ slug: string;
2155
+ containerId: string;
2156
+ /** The version the next commit is validated against; advances on commit. */
2157
+ baseVersionId: string;
2158
+ status: "open" | "closed";
2159
+ lastCommitVersionId: string | null;
2160
+ createdAt: string;
2161
+ };
2162
+ type SandboxOpened = SandboxSummary & {
2163
+ baseContentHash: string;
2164
+ branch: string;
2165
+ dataSnapshotId: string | null;
2166
+ fileCount: number;
2167
+ workdir: string;
2168
+ };
2169
+ type SandboxExecResult = {
2170
+ exitCode: number;
2171
+ stdout: string;
2172
+ stderr: string;
2173
+ timedOut: boolean;
2174
+ truncated: boolean;
2175
+ durationMs: number;
2176
+ };
2177
+ type SandboxChanges = {
2178
+ added: string[];
2179
+ modified: string[];
2180
+ removed: string[];
2181
+ };
2182
+ type SandboxCheckResult = {
2183
+ ok: boolean;
2184
+ contentHash: string | null;
2185
+ noChange: boolean;
2186
+ fileCount: number;
2187
+ changes: SandboxChanges;
2188
+ warnings: string[];
2189
+ error: string | null;
2190
+ };
2191
+ type SandboxCommitResult = {
2192
+ versionId: string;
2193
+ contentHash: string;
2194
+ branch: string;
2195
+ deduped: boolean;
2196
+ noChange: boolean;
2197
+ changes: SandboxChanges;
2198
+ };
2199
+ /** `slug` or `slug@ref` → the open body; a bare slug opens the latest READY version. */
2200
+ declare function parseSlugRef(slugRef: string): {
2201
+ slug: string;
2202
+ ref?: string;
2203
+ };
2204
+ declare class WorldSandboxClient {
2205
+ private readonly options;
2206
+ constructor(options?: WorldsClientOptions);
2207
+ open(target: {
2208
+ slug: string;
2209
+ ref?: string;
2210
+ }): Promise<SandboxOpened>;
2211
+ status(sandboxId: string): Promise<SandboxSummary>;
2212
+ read(sandboxId: string, filePath: string): Promise<{
2213
+ path: string;
2214
+ size: number;
2215
+ bytes: Buffer;
2216
+ }>;
2217
+ write(sandboxId: string, filePath: string, bytes: Buffer): Promise<{
2218
+ path: string;
2219
+ size: number;
2220
+ }>;
2221
+ exec(sandboxId: string, input: {
2222
+ command: string;
2223
+ cwd?: string;
2224
+ timeoutMs?: number;
2225
+ }): Promise<SandboxExecResult>;
2226
+ check(sandboxId: string): Promise<SandboxCheckResult>;
2227
+ commit(sandboxId: string, input: {
2228
+ message: string;
2229
+ branch?: string;
2230
+ }): Promise<SandboxCommitResult>;
2231
+ close(sandboxId: string): Promise<SandboxSummary>;
2232
+ }
2233
+
2091
2234
  type WorldsClientOptions = {
2092
2235
  host?: string;
2093
2236
  publicKey?: string;
@@ -2272,5 +2415,12 @@ declare class WorldSet {
2272
2415
  * (`tasks/<name>/task.toml`), so it needs no bundle download.
2273
2416
  */
2274
2417
  declare function openWorld(slugRef: string, options?: WorldsClientOptions): Promise<WorldSet>;
2418
+ /**
2419
+ * Resolve `slug` at one pinned READY `versionId` — the task list is THAT
2420
+ * version's, never the branch tip's. `worlds run|ci --version-id` come
2421
+ * through here; a version of another container, a draft or an unknown id
2422
+ * is a 404 from the platform, not a silent fall-through to head.
2423
+ */
2424
+ declare function openWorldVersion(slug: string, versionId: string, options?: WorldsClientOptions): Promise<WorldSet>;
2275
2425
 
2276
- export { ApiError, type BrowserActionEvent, BrowserActionRecorder, type BrowserMode, type BrowserPage, COMPUTER_USE_SPECS, type CreateEvaluationOptions, DEFAULT_BRANCH, DEFAULT_VIEWPORT, type DispatchedRun, EVENT_BATCH_SIZE, EVENT_FLUSH_INTERVAL_MS, EVENT_MAX_RETRIES, type EvalSample, type FinalizeEvaluationOptions, GatewayEvalsClient, type GradeResult, type IngestExplode, type IngestFieldSpec, type IngestMapping, type IngestMode, type IngestOptions, type IngestRefusal, type IngestReport, type IngestSource, type IngestSourceReport, type IngestToolCallsOptions, type IngestToolCallsReport, type LiveTaskSession, type LocalMeta, type MigrateWorldOptions, type ObservationConversion, type ObservationLike, type OpenSessionOptions, type OpenTaskOptions, type ParsedToolCall, type Plaintext, type PlaywrightBrowserLike, type PlaywrightLike, type RedactMode, type RedactPolicy, type Redaction, type ResolvedWorldVersion, type RunConfigInput, type RunConfigSummary, type RunProvenance, type RunReport, type RunTaskOptions, type RunTaskSummary, STRIP_PREFIX, SURFACE_KINDS, type SeedResult, type SessionRunReport, type SessionRunResult, type SessionSurfaces, TRANSFORMS, type TaskExport, type TaskGrade, type TaskManifest, type TaskManifestSession, TaskRun, type TaskRunResult, TaskSessions, type TaskSessionsResponse, type TaskWorldSessionState, type ToolCallRecord, type ToolCallSink, type ToolFunctions, type Transform, type ValidateReport, type ValidatedRun, WHITESPACE, WorldBrowser, type WorldDependency, type WorldDeprecation, type WorldDescription, type WorldEntity, type WorldEntityMapping, type WorldField, type WorldMigrationRefusal, type WorldMigrationReport, type WorldQueryAcrossOptions, type WorldQueryAcrossResult, type WorldQueryCompare, type WorldQueryCompareStatus, type WorldQueryGroup, type WorldQueryOptions, type WorldQueryResult, type WorldQuerySnapshot, type WorldQueryWhere, type WorldRelationship, WorldSession, type WorldSessionCall, WorldSessionCallError, type WorldSessionState, type WorldSessionTask, WorldSessionTimeout, WorldSet, type WorldSurfaceKind, WorldSurfaceUnavailable, WorldTask, type WorldTaskCheck, type WorldTaskDetail, type WorldTaskFieldOp, type WorldTaskGrader, type WorldTaskIdRef, type WorldTaskInfo, type WorldTaskPage, type WorldTaskRef, type WorldTaskRubricCriterion, type WorldTaskRubricGrader, type WorldTaskScalar, type WorldTaskSeed, type WorldTaskSpec, type WorldTaskSummary, type WorldTaskValidation, type WorldTaskVersionSummary, type WorldTaskWorld, type WorldTaskWorldValidation, type WorldToolSpec, type WorldToolkit, type WorldsClientOptions, aggregateTaskGrade, apiRequest, applyTransforms, attachManifest, attachSession, attachTask, buildSourceTarball, captureToolCalls, collectArchiveRelPaths, computeContentHash, computerUseToolkit, createRef, createTask, deleteEnvSecret, deleteTask, describeWorld, detectProvenance, getRun, getTask, hashForm, ingest, ingestToolCalls, isTaskIdRef, isTaskManifest, isTaskSpec, listEnvSecretKeys, listRunConfigs, listTaskSessions, listTasks, localTreeState, migrateProjectWorlds, migrateWorld, modelToolName, openBrowser, openSession, openTask, openWorld, parseToolCallRecord, parseTransform, parseWorldTaskRefs, preserveForm, propose, pull, push, pushSlug, pyText, queryRows, queryRowsSearch, readLocalMeta, readToolCallRecords, redactActionArgs, redactionOf, resolveBrowserMode, roundForm, runSessions, runTask, schemaFromSignature, sessionToolObservations, setEnvSecret, shapeText, snapshotTreeHashes, storedForms, surfaceKinds, taskLabel, toolCallRecordsIn, toolCallsFromObservations, uiEntryUrl, uiUrl, updateTask, upsertRunConfig, validateTask, validateWorld, waitForRun, withoutUiToken, writeLocalMeta };
2426
+ export { ApiError, type BrowserActionEvent, BrowserActionRecorder, type BrowserMode, type BrowserPage, COMPUTER_USE_SPECS, type CreateEvaluationOptions, DEFAULT_BRANCH, DEFAULT_VIEWPORT, type DispatchedRun, EVENT_BATCH_SIZE, EVENT_FLUSH_INTERVAL_MS, EVENT_MAX_RETRIES, type EvalSample, type FinalizeEvaluationOptions, GatewayEvalsClient, type GradeResult, type IngestExplode, type IngestFieldSpec, type IngestMapping, type IngestMode, type IngestOptions, type IngestRefusal, type IngestReport, type IngestSource, type IngestSourceReport, type IngestToolCallsOptions, type IngestToolCallsReport, type LiveTaskSession, type LocalMeta, type MigrateWorldOptions, type ObservationConversion, type ObservationLike, type OpenSessionOptions, type OpenTaskOptions, type ParsedToolCall, type Plaintext, type PlaywrightBrowserLike, type PlaywrightLike, PushSlugConflict, type PushSlugResolution, type RedactMode, type RedactPolicy, type Redaction, type ResolvedWorldVersion, type RunConfigInput, type RunConfigSummary, type RunProvenance, type RunReport, type RunTaskOptions, type RunTaskSummary, STRIP_PREFIX, SURFACE_KINDS, type SandboxChanges, type SandboxCheckResult, type SandboxCommitResult, type SandboxExecResult, type SandboxOpened, type SandboxSummary, type SeedResult, type SessionRunReport, type SessionRunResult, type SessionSurfaces, type SlugDeclaration, TRANSFORMS, type TaskExport, type TaskGrade, type TaskManifest, type TaskManifestSession, TaskRun, type TaskRunResult, TaskSessions, type TaskSessionsResponse, type TaskWorldSessionState, type ToolCallRecord, type ToolCallSink, type ToolFunctions, type Transform, type ValidateReport, type ValidatedRun, WHITESPACE, WorldBrowser, type WorldDependency, type WorldDeprecation, type WorldDescription, type WorldEntity, type WorldEntityMapping, type WorldField, type WorldMigrationRefusal, type WorldMigrationReport, type WorldQueryAcrossOptions, type WorldQueryAcrossResult, type WorldQueryCompare, type WorldQueryCompareStatus, type WorldQueryGroup, type WorldQueryOptions, type WorldQueryResult, type WorldQuerySnapshot, type WorldQueryWhere, type WorldRelationship, WorldSandboxClient, WorldSession, type WorldSessionCall, WorldSessionCallError, type WorldSessionState, type WorldSessionTask, WorldSessionTimeout, WorldSet, type WorldSurfaceKind, WorldSurfaceUnavailable, WorldTask, type WorldTaskCheck, type WorldTaskDetail, type WorldTaskFieldOp, type WorldTaskGrader, type WorldTaskIdRef, type WorldTaskInfo, type WorldTaskPage, type WorldTaskRef, type WorldTaskRubricCriterion, type WorldTaskRubricGrader, type WorldTaskScalar, type WorldTaskSeed, type WorldTaskSpec, type WorldTaskSummary, type WorldTaskValidation, type WorldTaskVersionSummary, type WorldTaskWorld, type WorldTaskWorldValidation, type WorldToolSpec, type WorldToolkit, type WorldsClientOptions, aggregateTaskGrade, apiRequest, applyTransforms, archiveMode, attachManifest, attachSession, attachTask, buildFileManifest, buildSourceTarball, captureToolCalls, collectArchiveRelPaths, computeContentHash, computerUseToolkit, createRef, createTask, deleteEnvSecret, deleteTask, describeWorld, detectProvenance, extractBundle, getRun, getTask, hashForm, ingest, ingestToolCalls, isTaskIdRef, isTaskManifest, isTaskSpec, listEnvSecretKeys, listRunConfigs, listTaskSessions, listTasks, localTreeState, migrateProjectWorlds, migrateWorld, modelToolName, openBrowser, openSession, openTask, openWorld, openWorldVersion, parseSlugRef, parseToolCallRecord, parseTransform, parseWorldTaskRefs, preserveForm, propose, pull, push, pushSlug, pyText, queryRows, queryRowsSearch, readLocalMeta, readToolCallRecords, redactActionArgs, redactionOf, resolveBrowserMode, resolvePushSlug, roundForm, runSessions, runTask, schemaFromSignature, sessionToolObservations, setEnvSecret, shapeText, snapshotTreeHashes, storedForms, surfaceKinds, taskLabel, toolCallRecordsIn, toolCallsFromObservations, uiEntryUrl, uiUrl, updateTask, upsertRunConfig, validateTask, validateWorld, waitForRun, withoutUiToken, writeLocalMeta };
@@ -587,7 +587,11 @@ type SessionRunReport = {
587
587
  results: SessionRunResult[];
588
588
  /** Mean reward over every task, counting failures as 0. */
589
589
  mean: number;
590
- /** True when `mean >= failUnder` (or no gate was set). */
590
+ /** Tasks that never graded: the session did not open or become ready, the
591
+ * agent threw, or grading failed. Any failure fails the run — a 0 an agent
592
+ * earned and a session that never opened are not the same result. */
593
+ failed: number;
594
+ /** True when no task failed and `mean >= failUnder` (or no gate was set). */
591
595
  passed: boolean;
592
596
  };
593
597
  /**
@@ -1223,8 +1227,10 @@ type RunTaskSummary = {
1223
1227
  evaluationId?: string;
1224
1228
  [extra: string]: unknown;
1225
1229
  };
1226
- /** The report an agent callable may return: a string, or an object with a
1227
- * `report` key (any other keys are kept in `run.json` as-is). */
1230
+ /** The report an agent callable may return: a string, or an object with an
1231
+ * optional string `report` key (any other keys are kept in `run.json` as-is),
1232
+ * or nothing. Anything else is an agent error — a number, a bigint or an
1233
+ * array is never stringified into a graded completion. */
1228
1234
  type TaskRunResult = string | {
1229
1235
  report?: unknown;
1230
1236
  [key: string]: unknown;
@@ -1300,6 +1306,7 @@ declare const DEFAULT_BRANCH = "main";
1300
1306
  * (`WorldDir.archivePaths`). Throws on a root .gitignore (see the header). */
1301
1307
  declare function collectArchiveRelPaths(dir: string): string[];
1302
1308
  declare function computeContentHash(dir: string): string;
1309
+ declare function archiveMode(mode: number): number;
1303
1310
  /** A deterministic .tar.gz of the source files (node's gzip stamps no mtime,
1304
1311
  * matching the Python's mtime=0 gzip). */
1305
1312
  declare function buildSourceTarball(dir: string): Buffer;
@@ -1316,6 +1323,39 @@ declare function readLocalMeta(dir: string): LocalMeta | null;
1316
1323
  * name, else the directory's name — so a tracked checkout never creates a
1317
1324
  * second world named after its folder. */
1318
1325
  declare function pushSlug(dir: string, explicit?: string): string;
1326
+ type SlugDeclaration = {
1327
+ source: string;
1328
+ slug: string;
1329
+ };
1330
+ type PushSlugResolution = {
1331
+ slug: string;
1332
+ /** Where the slug came from (`--slug`, `.gateway metadata`, `benchmark.json`, `pyproject.toml`, `directory name`). */
1333
+ source: string;
1334
+ /** The slug the dir's `.gateway/.env-metadata.json` tracks from an earlier push or pull, if any. */
1335
+ tracked: string | null;
1336
+ /** What the tree itself declares (a caller's benchmark.json, then pyproject.toml), in precedence order. */
1337
+ declared: SlugDeclaration[];
1338
+ /** A declaration the tracked slug does not match — set only when the tracked slug won without `--slug`. */
1339
+ conflict: SlugDeclaration | null;
1340
+ };
1341
+ /** `pushSlug` with its provenance. `declared` are the caller's own slug sources
1342
+ * (e.g. `{ "benchmark.json": "x" }`), consulted before pyproject. A tracked slug
1343
+ * that disagrees with what the tree declares is reported as `conflict` so a
1344
+ * front door can refuse: that is how a directory pushed once with `--slug X`
1345
+ * silently landed every later push on X while its benchmark.json said Y. The
1346
+ * platform's collision suffix (`acme-2` minted for a taken `acme`) is not a
1347
+ * conflict; any other difference is (`acme-e2e` for `acme` was the mix-up). */
1348
+ declare function resolvePushSlug(dir: string, opts?: {
1349
+ explicit?: string;
1350
+ declared?: Record<string, string | null | undefined>;
1351
+ }): PushSlugResolution;
1352
+ /** The refusal a front door raises on `PushSlugResolution.conflict`: both slugs, both sources, every way out. */
1353
+ declare class PushSlugConflict extends Error {
1354
+ readonly dir: string;
1355
+ readonly tracked: string;
1356
+ readonly declared: SlugDeclaration;
1357
+ constructor(dir: string, tracked: string, declared: SlugDeclaration);
1358
+ }
1319
1359
  declare function writeLocalMeta(dir: string, meta: LocalMeta): void;
1320
1360
  declare function snapshotTreeHashes(dir: string): Record<string, string>;
1321
1361
  declare function localTreeState(dir: string): {
@@ -1323,6 +1363,16 @@ declare function localTreeState(dir: string): {
1323
1363
  dirty: string[];
1324
1364
  clean: boolean;
1325
1365
  };
1366
+ /** The rows the platform previews for a pushed DRAFT: the archive set, one
1367
+ * entry per file the content hash covers, so hash, tarball, manifest and
1368
+ * `pull` agree on what the world is. Finalize replaces them with rows
1369
+ * derived from the stored bytes. */
1370
+ declare function buildFileManifest(dir: string): {
1371
+ path: string;
1372
+ isDirectory: boolean;
1373
+ size: number;
1374
+ sha256: string;
1375
+ }[];
1326
1376
  declare function push(dir: string, opts?: {
1327
1377
  slug?: string;
1328
1378
  branch?: string;
@@ -1352,6 +1402,8 @@ declare function push(dir: string, opts?: {
1352
1402
  state: string | null;
1353
1403
  alreadyUploaded: boolean;
1354
1404
  }>;
1405
+ /** Extract a downloaded bundle into `dir` (paths sanitized, executable bit kept). */
1406
+ declare function extractBundle(bundle: Buffer, dir: string): number;
1355
1407
  declare function pull(slugRef: string, target?: string, options?: WorldsClientOptions): Promise<{
1356
1408
  slug: string;
1357
1409
  dir: string;
@@ -2088,6 +2140,97 @@ declare function ingestToolCalls(world: string, records: unknown[], options?: In
2088
2140
  /** The TOOL observations of one session, every trace's, from `GET /api/public/sessions/{id}?includeIO=true`. */
2089
2141
  declare function sessionToolObservations(sessionId: string, options?: WorldsClientOptions): Promise<ObservationLike[]>;
2090
2142
 
2143
+ /**
2144
+ * The CLI sandbox: a server-side workspace opened on a READY version of a
2145
+ * world (`/workspace/env` in the console agent's sandbox), edited and run
2146
+ * there, and committed straight into the version service — the source
2147
+ * never lands on the caller's disk. One client under both `gateway worlds
2148
+ * sandbox …` and `gateway bench sandbox …`; the Python twin is
2149
+ * `gatewaysdk.benchmark_hub.sandbox.WorldSandboxClient`.
2150
+ */
2151
+ /** Mirrors `SandboxSummaryResponse` (web/features/public-api/types/world-sandboxes.ts). */
2152
+ type SandboxSummary = {
2153
+ sandboxId: string;
2154
+ slug: string;
2155
+ containerId: string;
2156
+ /** The version the next commit is validated against; advances on commit. */
2157
+ baseVersionId: string;
2158
+ status: "open" | "closed";
2159
+ lastCommitVersionId: string | null;
2160
+ createdAt: string;
2161
+ };
2162
+ type SandboxOpened = SandboxSummary & {
2163
+ baseContentHash: string;
2164
+ branch: string;
2165
+ dataSnapshotId: string | null;
2166
+ fileCount: number;
2167
+ workdir: string;
2168
+ };
2169
+ type SandboxExecResult = {
2170
+ exitCode: number;
2171
+ stdout: string;
2172
+ stderr: string;
2173
+ timedOut: boolean;
2174
+ truncated: boolean;
2175
+ durationMs: number;
2176
+ };
2177
+ type SandboxChanges = {
2178
+ added: string[];
2179
+ modified: string[];
2180
+ removed: string[];
2181
+ };
2182
+ type SandboxCheckResult = {
2183
+ ok: boolean;
2184
+ contentHash: string | null;
2185
+ noChange: boolean;
2186
+ fileCount: number;
2187
+ changes: SandboxChanges;
2188
+ warnings: string[];
2189
+ error: string | null;
2190
+ };
2191
+ type SandboxCommitResult = {
2192
+ versionId: string;
2193
+ contentHash: string;
2194
+ branch: string;
2195
+ deduped: boolean;
2196
+ noChange: boolean;
2197
+ changes: SandboxChanges;
2198
+ };
2199
+ /** `slug` or `slug@ref` → the open body; a bare slug opens the latest READY version. */
2200
+ declare function parseSlugRef(slugRef: string): {
2201
+ slug: string;
2202
+ ref?: string;
2203
+ };
2204
+ declare class WorldSandboxClient {
2205
+ private readonly options;
2206
+ constructor(options?: WorldsClientOptions);
2207
+ open(target: {
2208
+ slug: string;
2209
+ ref?: string;
2210
+ }): Promise<SandboxOpened>;
2211
+ status(sandboxId: string): Promise<SandboxSummary>;
2212
+ read(sandboxId: string, filePath: string): Promise<{
2213
+ path: string;
2214
+ size: number;
2215
+ bytes: Buffer;
2216
+ }>;
2217
+ write(sandboxId: string, filePath: string, bytes: Buffer): Promise<{
2218
+ path: string;
2219
+ size: number;
2220
+ }>;
2221
+ exec(sandboxId: string, input: {
2222
+ command: string;
2223
+ cwd?: string;
2224
+ timeoutMs?: number;
2225
+ }): Promise<SandboxExecResult>;
2226
+ check(sandboxId: string): Promise<SandboxCheckResult>;
2227
+ commit(sandboxId: string, input: {
2228
+ message: string;
2229
+ branch?: string;
2230
+ }): Promise<SandboxCommitResult>;
2231
+ close(sandboxId: string): Promise<SandboxSummary>;
2232
+ }
2233
+
2091
2234
  type WorldsClientOptions = {
2092
2235
  host?: string;
2093
2236
  publicKey?: string;
@@ -2272,5 +2415,12 @@ declare class WorldSet {
2272
2415
  * (`tasks/<name>/task.toml`), so it needs no bundle download.
2273
2416
  */
2274
2417
  declare function openWorld(slugRef: string, options?: WorldsClientOptions): Promise<WorldSet>;
2418
+ /**
2419
+ * Resolve `slug` at one pinned READY `versionId` — the task list is THAT
2420
+ * version's, never the branch tip's. `worlds run|ci --version-id` come
2421
+ * through here; a version of another container, a draft or an unknown id
2422
+ * is a 404 from the platform, not a silent fall-through to head.
2423
+ */
2424
+ declare function openWorldVersion(slug: string, versionId: string, options?: WorldsClientOptions): Promise<WorldSet>;
2275
2425
 
2276
- export { ApiError, type BrowserActionEvent, BrowserActionRecorder, type BrowserMode, type BrowserPage, COMPUTER_USE_SPECS, type CreateEvaluationOptions, DEFAULT_BRANCH, DEFAULT_VIEWPORT, type DispatchedRun, EVENT_BATCH_SIZE, EVENT_FLUSH_INTERVAL_MS, EVENT_MAX_RETRIES, type EvalSample, type FinalizeEvaluationOptions, GatewayEvalsClient, type GradeResult, type IngestExplode, type IngestFieldSpec, type IngestMapping, type IngestMode, type IngestOptions, type IngestRefusal, type IngestReport, type IngestSource, type IngestSourceReport, type IngestToolCallsOptions, type IngestToolCallsReport, type LiveTaskSession, type LocalMeta, type MigrateWorldOptions, type ObservationConversion, type ObservationLike, type OpenSessionOptions, type OpenTaskOptions, type ParsedToolCall, type Plaintext, type PlaywrightBrowserLike, type PlaywrightLike, type RedactMode, type RedactPolicy, type Redaction, type ResolvedWorldVersion, type RunConfigInput, type RunConfigSummary, type RunProvenance, type RunReport, type RunTaskOptions, type RunTaskSummary, STRIP_PREFIX, SURFACE_KINDS, type SeedResult, type SessionRunReport, type SessionRunResult, type SessionSurfaces, TRANSFORMS, type TaskExport, type TaskGrade, type TaskManifest, type TaskManifestSession, TaskRun, type TaskRunResult, TaskSessions, type TaskSessionsResponse, type TaskWorldSessionState, type ToolCallRecord, type ToolCallSink, type ToolFunctions, type Transform, type ValidateReport, type ValidatedRun, WHITESPACE, WorldBrowser, type WorldDependency, type WorldDeprecation, type WorldDescription, type WorldEntity, type WorldEntityMapping, type WorldField, type WorldMigrationRefusal, type WorldMigrationReport, type WorldQueryAcrossOptions, type WorldQueryAcrossResult, type WorldQueryCompare, type WorldQueryCompareStatus, type WorldQueryGroup, type WorldQueryOptions, type WorldQueryResult, type WorldQuerySnapshot, type WorldQueryWhere, type WorldRelationship, WorldSession, type WorldSessionCall, WorldSessionCallError, type WorldSessionState, type WorldSessionTask, WorldSessionTimeout, WorldSet, type WorldSurfaceKind, WorldSurfaceUnavailable, WorldTask, type WorldTaskCheck, type WorldTaskDetail, type WorldTaskFieldOp, type WorldTaskGrader, type WorldTaskIdRef, type WorldTaskInfo, type WorldTaskPage, type WorldTaskRef, type WorldTaskRubricCriterion, type WorldTaskRubricGrader, type WorldTaskScalar, type WorldTaskSeed, type WorldTaskSpec, type WorldTaskSummary, type WorldTaskValidation, type WorldTaskVersionSummary, type WorldTaskWorld, type WorldTaskWorldValidation, type WorldToolSpec, type WorldToolkit, type WorldsClientOptions, aggregateTaskGrade, apiRequest, applyTransforms, attachManifest, attachSession, attachTask, buildSourceTarball, captureToolCalls, collectArchiveRelPaths, computeContentHash, computerUseToolkit, createRef, createTask, deleteEnvSecret, deleteTask, describeWorld, detectProvenance, getRun, getTask, hashForm, ingest, ingestToolCalls, isTaskIdRef, isTaskManifest, isTaskSpec, listEnvSecretKeys, listRunConfigs, listTaskSessions, listTasks, localTreeState, migrateProjectWorlds, migrateWorld, modelToolName, openBrowser, openSession, openTask, openWorld, parseToolCallRecord, parseTransform, parseWorldTaskRefs, preserveForm, propose, pull, push, pushSlug, pyText, queryRows, queryRowsSearch, readLocalMeta, readToolCallRecords, redactActionArgs, redactionOf, resolveBrowserMode, roundForm, runSessions, runTask, schemaFromSignature, sessionToolObservations, setEnvSecret, shapeText, snapshotTreeHashes, storedForms, surfaceKinds, taskLabel, toolCallRecordsIn, toolCallsFromObservations, uiEntryUrl, uiUrl, updateTask, upsertRunConfig, validateTask, validateWorld, waitForRun, withoutUiToken, writeLocalMeta };
2426
+ export { ApiError, type BrowserActionEvent, BrowserActionRecorder, type BrowserMode, type BrowserPage, COMPUTER_USE_SPECS, type CreateEvaluationOptions, DEFAULT_BRANCH, DEFAULT_VIEWPORT, type DispatchedRun, EVENT_BATCH_SIZE, EVENT_FLUSH_INTERVAL_MS, EVENT_MAX_RETRIES, type EvalSample, type FinalizeEvaluationOptions, GatewayEvalsClient, type GradeResult, type IngestExplode, type IngestFieldSpec, type IngestMapping, type IngestMode, type IngestOptions, type IngestRefusal, type IngestReport, type IngestSource, type IngestSourceReport, type IngestToolCallsOptions, type IngestToolCallsReport, type LiveTaskSession, type LocalMeta, type MigrateWorldOptions, type ObservationConversion, type ObservationLike, type OpenSessionOptions, type OpenTaskOptions, type ParsedToolCall, type Plaintext, type PlaywrightBrowserLike, type PlaywrightLike, PushSlugConflict, type PushSlugResolution, type RedactMode, type RedactPolicy, type Redaction, type ResolvedWorldVersion, type RunConfigInput, type RunConfigSummary, type RunProvenance, type RunReport, type RunTaskOptions, type RunTaskSummary, STRIP_PREFIX, SURFACE_KINDS, type SandboxChanges, type SandboxCheckResult, type SandboxCommitResult, type SandboxExecResult, type SandboxOpened, type SandboxSummary, type SeedResult, type SessionRunReport, type SessionRunResult, type SessionSurfaces, type SlugDeclaration, TRANSFORMS, type TaskExport, type TaskGrade, type TaskManifest, type TaskManifestSession, TaskRun, type TaskRunResult, TaskSessions, type TaskSessionsResponse, type TaskWorldSessionState, type ToolCallRecord, type ToolCallSink, type ToolFunctions, type Transform, type ValidateReport, type ValidatedRun, WHITESPACE, WorldBrowser, type WorldDependency, type WorldDeprecation, type WorldDescription, type WorldEntity, type WorldEntityMapping, type WorldField, type WorldMigrationRefusal, type WorldMigrationReport, type WorldQueryAcrossOptions, type WorldQueryAcrossResult, type WorldQueryCompare, type WorldQueryCompareStatus, type WorldQueryGroup, type WorldQueryOptions, type WorldQueryResult, type WorldQuerySnapshot, type WorldQueryWhere, type WorldRelationship, WorldSandboxClient, WorldSession, type WorldSessionCall, WorldSessionCallError, type WorldSessionState, type WorldSessionTask, WorldSessionTimeout, WorldSet, type WorldSurfaceKind, WorldSurfaceUnavailable, WorldTask, type WorldTaskCheck, type WorldTaskDetail, type WorldTaskFieldOp, type WorldTaskGrader, type WorldTaskIdRef, type WorldTaskInfo, type WorldTaskPage, type WorldTaskRef, type WorldTaskRubricCriterion, type WorldTaskRubricGrader, type WorldTaskScalar, type WorldTaskSeed, type WorldTaskSpec, type WorldTaskSummary, type WorldTaskValidation, type WorldTaskVersionSummary, type WorldTaskWorld, type WorldTaskWorldValidation, type WorldToolSpec, type WorldToolkit, type WorldsClientOptions, aggregateTaskGrade, apiRequest, applyTransforms, archiveMode, attachManifest, attachSession, attachTask, buildFileManifest, buildSourceTarball, captureToolCalls, collectArchiveRelPaths, computeContentHash, computerUseToolkit, createRef, createTask, deleteEnvSecret, deleteTask, describeWorld, detectProvenance, extractBundle, getRun, getTask, hashForm, ingest, ingestToolCalls, isTaskIdRef, isTaskManifest, isTaskSpec, listEnvSecretKeys, listRunConfigs, listTaskSessions, listTasks, localTreeState, migrateProjectWorlds, migrateWorld, modelToolName, openBrowser, openSession, openTask, openWorld, openWorldVersion, parseSlugRef, parseToolCallRecord, parseTransform, parseWorldTaskRefs, preserveForm, propose, pull, push, pushSlug, pyText, queryRows, queryRowsSearch, readLocalMeta, readToolCallRecords, redactActionArgs, redactionOf, resolveBrowserMode, resolvePushSlug, roundForm, runSessions, runTask, schemaFromSignature, sessionToolObservations, setEnvSecret, shapeText, snapshotTreeHashes, storedForms, surfaceKinds, taskLabel, toolCallRecordsIn, toolCallsFromObservations, uiEntryUrl, uiUrl, updateTask, upsertRunConfig, validateTask, validateWorld, waitForRun, withoutUiToken, writeLocalMeta };