@nylorun/runtime 0.15.1-beta → 0.16.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +33 -6
- package/dist/adapters/execution/restate.d.ts +9 -1
- package/dist/adapters/execution/restate.js +64 -5
- package/dist/adapters/replication/pgoutput.js +3 -1
- package/dist/adapters/sandbox/local.d.ts +10 -0
- package/dist/adapters/sandbox/local.js +195 -0
- package/dist/adapters/sandbox/virtual.js +36 -1
- package/dist/admin-openapi.json +37 -3
- package/dist/api/a2a/tasks.js +6 -1
- package/dist/api/ag-ui/history.js +6 -1
- package/dist/api/ag-ui/routes.js +5 -1
- package/dist/api/components.d.ts +2673 -655
- package/dist/api/components.js +17 -1
- package/dist/api/gate/keys.d.ts +4 -3
- package/dist/api/gate/routes.d.ts +19 -3
- package/dist/api/gate/routes.js +162 -53
- package/dist/api/http/app.js +4 -0
- package/dist/api/http/define.d.ts +10 -0
- package/dist/api/http/define.js +2 -1
- package/dist/api/http/routes/artifacts.d.ts +11 -0
- package/dist/api/http/routes/artifacts.js +518 -0
- package/dist/api/http/routes/sandboxes.d.ts +9 -0
- package/dist/api/http/routes/sandboxes.js +152 -0
- package/dist/api/http/routes/sessions.js +3 -1
- package/dist/api/http/routes/tenant.js +36 -2
- package/dist/artifacts/config.d.ts +23 -0
- package/dist/artifacts/config.js +27 -0
- package/dist/artifacts/export.d.ts +8 -0
- package/dist/artifacts/export.js +205 -0
- package/dist/artifacts/files.d.ts +36 -0
- package/dist/artifacts/files.js +45 -0
- package/dist/artifacts/folders.d.ts +60 -0
- package/dist/artifacts/folders.js +209 -0
- package/dist/artifacts/links.d.ts +28 -0
- package/dist/artifacts/links.js +92 -0
- package/dist/artifacts/media-types.d.ts +13 -0
- package/dist/artifacts/media-types.js +80 -0
- package/dist/artifacts/parts.d.ts +34 -0
- package/dist/artifacts/parts.js +38 -0
- package/dist/artifacts/service.d.ts +74 -0
- package/dist/artifacts/service.js +301 -0
- package/dist/artifacts/workspace.d.ts +60 -0
- package/dist/artifacts/workspace.js +57 -0
- package/dist/artifacts/zip.d.ts +10 -0
- package/dist/artifacts/zip.js +126 -0
- package/dist/blob/fs.d.ts +13 -0
- package/dist/blob/fs.js +194 -0
- package/dist/blob/index.d.ts +9 -0
- package/dist/blob/index.js +9 -0
- package/dist/blob/s3.d.ts +42 -0
- package/dist/blob/s3.js +366 -0
- package/dist/blob/types.d.ts +108 -0
- package/dist/blob/types.js +85 -0
- package/dist/core/sandbox-routes.d.ts +2 -2
- package/dist/core/sandbox-routes.js +4 -3
- package/dist/execution/memory.d.ts +11 -1
- package/dist/execution/memory.js +77 -3
- package/dist/execution/types.d.ts +54 -1
- package/dist/execution/types.js +31 -0
- package/dist/gates/contract.d.ts +14 -13
- package/dist/gates/contract.js +13 -12
- package/dist/gates/egress.d.ts +72 -0
- package/dist/gates/egress.js +319 -0
- package/dist/gates/handler.d.ts +3 -0
- package/dist/gates/handler.js +3 -2
- package/dist/gates/http-client.d.ts +3 -2
- package/dist/gates/http-client.js +17 -6
- package/dist/gates/in-process.d.ts +13 -1
- package/dist/gates/in-process.js +11 -0
- package/dist/gates/inflight.d.ts +23 -4
- package/dist/gates/inflight.js +18 -4
- package/dist/gates/model-gate.d.ts +6 -2
- package/dist/gates/tool-calls.d.ts +9 -5
- package/dist/gates/tool-calls.js +6 -6
- package/dist/gates/tool-client.d.ts +9 -2
- package/dist/gates/tool-client.js +56 -22
- package/dist/gates/tool-contract.d.ts +12 -7
- package/dist/gates/tool-contract.js +10 -4
- package/dist/gates/tool-gate.d.ts +1 -0
- package/dist/harness/calls.d.ts +52 -0
- package/dist/harness/calls.js +113 -0
- package/dist/harness/executors.d.ts +22 -0
- package/dist/harness/executors.js +96 -0
- package/dist/harness/main.d.ts +6 -0
- package/dist/harness/main.js +103 -0
- package/dist/harness/pod.d.ts +21 -0
- package/dist/harness/pod.js +119 -0
- package/dist/harness/run-tokens.d.ts +18 -0
- package/dist/harness/run-tokens.js +15 -0
- package/dist/harness/service.d.ts +46 -0
- package/dist/harness/service.js +179 -0
- package/dist/harness/ws-client.d.ts +43 -0
- package/dist/harness/ws-client.js +122 -0
- package/dist/harness/ws-port.d.ts +22 -0
- package/dist/harness/ws-port.js +78 -0
- package/dist/harness-api/in-process.d.ts +24 -0
- package/dist/harness-api/in-process.js +119 -0
- package/dist/harness-api/record.d.ts +44 -0
- package/dist/harness-api/record.js +250 -0
- package/dist/harness-api/renew.d.ts +37 -0
- package/dist/harness-api/renew.js +77 -0
- package/dist/harness-api/server.d.ts +104 -0
- package/dist/harness-api/server.js +408 -0
- package/dist/harness-api/start.d.ts +36 -0
- package/dist/harness-api/start.js +118 -0
- package/dist/harness-api/workspace.d.ts +53 -0
- package/dist/harness-api/workspace.js +201 -0
- package/dist/harness-api/ws-server.d.ts +34 -0
- package/dist/harness-api/ws-server.js +227 -0
- package/dist/host/app.js +23 -2
- package/dist/host/gates.d.ts +27 -2
- package/dist/host/gates.js +46 -1
- package/dist/host/http.d.ts +4 -1
- package/dist/host/http.js +6 -1
- package/dist/host/main.js +135 -27
- package/dist/host/stack-config.d.ts +101 -1
- package/dist/host/stack-config.js +199 -8
- package/dist/mcp/snapshot.d.ts +8 -1
- package/dist/mcp/snapshot.js +28 -0
- package/dist/media.d.ts +5 -24
- package/dist/media.js +5 -0
- package/dist/model/pi-model.d.ts +6 -2
- package/dist/model/pi-model.js +39 -22
- package/dist/node/index.d.ts +1 -1
- package/dist/node/index.js +1 -1
- package/dist/openapi.json +15245 -8850
- package/dist/record/index.d.ts +1 -0
- package/dist/record/index.js +1 -0
- package/dist/record/sandbox.d.ts +25 -0
- package/dist/record/sandbox.js +43 -0
- package/dist/sandbox/capability.d.ts +3 -0
- package/dist/sandbox/capability.js +10 -0
- package/dist/sandbox/egress-token.d.ts +26 -0
- package/dist/sandbox/egress-token.js +8 -0
- package/dist/sandbox/join.d.ts +37 -0
- package/dist/sandbox/join.js +165 -0
- package/dist/sandbox/manager.d.ts +99 -34
- package/dist/sandbox/manager.js +216 -84
- package/dist/sandbox/placement.d.ts +27 -0
- package/dist/sandbox/placement.js +44 -0
- package/dist/sandbox/pods/client.d.ts +90 -0
- package/dist/sandbox/pods/client.js +79 -0
- package/dist/sandbox/pods/lifecycle.d.ts +91 -0
- package/dist/sandbox/pods/lifecycle.js +178 -0
- package/dist/sandbox/pods/name.d.ts +1 -0
- package/dist/sandbox/pods/name.js +28 -0
- package/dist/sandbox/pods/network-gate.d.ts +25 -0
- package/dist/sandbox/pods/network-gate.js +71 -0
- package/dist/sandbox/pods/reconcile.d.ts +11 -0
- package/dist/sandbox/pods/reconcile.js +152 -0
- package/dist/sandbox/pods/spec.d.ts +45 -0
- package/dist/sandbox/pods/spec.js +62 -0
- package/dist/sandbox/records.d.ts +29 -0
- package/dist/sandbox/records.js +79 -0
- package/dist/sandbox/resolve.d.ts +5 -0
- package/dist/sandbox/resolve.js +3 -2
- package/dist/sandbox/session-sandbox.js +7 -5
- package/dist/sandbox/share.d.ts +11 -0
- package/dist/sandbox/share.js +10 -0
- package/dist/sandbox/tenant-config.d.ts +16 -1
- package/dist/sandbox/tenant-config.js +9 -0
- package/dist/sandbox/types.d.ts +29 -1
- package/dist/sandbox/types.js +13 -1
- package/dist/store/postgres/drizzle/0004_sandbox_resources.sql +20 -0
- package/dist/store/postgres/drizzle/0005_artifacts.sql +27 -0
- package/dist/store/postgres/drizzle/0006_folder_artifacts.sql +12 -0
- package/dist/store/postgres/drizzle/0007_sandbox_pods.sql +14 -0
- package/dist/store/postgres/drizzle/meta/_journal.json +28 -0
- package/dist/store/postgres/record-writer.d.ts +6 -1
- package/dist/store/postgres/record-writer.js +20 -2
- package/dist/store/postgres/schema.d.ts +894 -0
- package/dist/store/postgres/schema.js +130 -2
- package/dist/store/postgres/store.js +289 -4
- package/dist/store/types.d.ts +143 -6
- package/dist/streams/control.d.ts +2 -0
- package/dist/streams/control.js +5 -0
- package/dist/streams/types.d.ts +10 -1
- package/dist/tenant/advance.js +218 -160
- package/dist/tenant/artifact-tool.d.ts +5 -0
- package/dist/tenant/artifact-tool.js +119 -0
- package/dist/tenant/commands.js +30 -5
- package/dist/tenant/context.d.ts +64 -5
- package/dist/tenant/effects.d.ts +19 -13
- package/dist/tenant/effects.js +5 -382
- package/dist/tenant/history.d.ts +13 -23
- package/dist/tenant/history.js +14 -62
- package/dist/tenant/host-token.d.ts +92 -0
- package/dist/tenant/host-token.js +179 -0
- package/dist/tenant/module.js +1 -0
- package/dist/tenant/paths.d.ts +2 -1
- package/dist/tenant/paths.js +3 -1
- package/dist/tenant/reset.d.ts +5 -2
- package/dist/tenant/reset.js +22 -2
- package/dist/tenant/run-grants.d.ts +49 -0
- package/dist/tenant/run-grants.js +62 -0
- package/dist/tenant/run-token.d.ts +70 -0
- package/dist/tenant/run-token.js +138 -0
- package/dist/tenant/runtime.d.ts +38 -0
- package/dist/tenant/runtime.js +120 -32
- package/dist/tenant/sandboxes.d.ts +119 -0
- package/dist/tenant/sandboxes.js +551 -0
- package/dist/tenant/session.d.ts +2 -13
- package/dist/tenant/session.js +4 -28
- package/dist/tenant/sessions.d.ts +5 -0
- package/dist/tenant/sessions.js +28 -1
- package/dist/tenant/status.d.ts +8 -3
- package/dist/tenant/status.js +5 -0
- package/dist/tenant/streams.d.ts +7 -0
- package/dist/tenant/streams.js +24 -1
- package/dist/tenant/sweep.js +37 -4
- package/dist/tenant/tokens.js +9 -2
- package/dist/tenant/types.d.ts +18 -1
- package/dist/tenant/worker.d.ts +11 -5
- package/dist/tenant/worker.js +12 -6
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +13 -10
- package/dist/adapters/media.d.ts +0 -24
- package/dist/adapters/media.js +0 -111
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File artifacts (blueprint D35, F8.1): an opaque id, a name, numbered immutable versions,
|
|
3
|
+
* metadata in Postgres and bytes in the Object store behind the `BlobStore` seam. Folders
|
|
4
|
+
* (F8.2) are written by the turn-end export (`./export.ts`, `./folders.ts`) and read here.
|
|
5
|
+
*
|
|
6
|
+
* - **Uploads stream.** The body goes to the store as it arrives, at a fresh random key, counted
|
|
7
|
+
* against the per-file cap and what is left of the Tenant total (`maxBytes`): nothing is
|
|
8
|
+
* buffered whole, and a body past either limit stores nothing (`413 limit_exceeded`).
|
|
9
|
+
* - **Postgres is the commit point.** One transaction writes the version row and, for a session's
|
|
10
|
+
* artifact, its `artifact.created` or `artifact.version.created` event through the record
|
|
11
|
+
* module, after re-checking the Tenant total under the quota lock. Until it commits the object
|
|
12
|
+
* is garbage, and a failed commit deletes it.
|
|
13
|
+
* - **Access mirrors sessions.** An application principal reaches every artifact. A request acting
|
|
14
|
+
* for a person (a subject token or subject headers) reaches only artifacts of that person's
|
|
15
|
+
* sessions; another person's, or a Tenant-wide one, is the same 404 as a missing artifact.
|
|
16
|
+
*
|
|
17
|
+
* Lock order, the same in every write: the session, then the quota lock, then the artifact row.
|
|
18
|
+
*/
|
|
19
|
+
import { randomUUID } from "node:crypto";
|
|
20
|
+
import { ARTIFACT_NAME_MAX } from "@nylorun/core/contracts";
|
|
21
|
+
import { isArtifactId, newArtifactId } from "@nylorun/core/compatibility";
|
|
22
|
+
import { BlobTooLargeError } from "../blob/index.js";
|
|
23
|
+
import { lockedSession, sessionOf, } from "../tenant/context.js";
|
|
24
|
+
import { fail } from "../tenant/http.js";
|
|
25
|
+
import { readArtifactLimits } from "./config.js";
|
|
26
|
+
import { collectContent, readManifest } from "./folders.js";
|
|
27
|
+
import { contentTypeFor } from "./media-types.js";
|
|
28
|
+
const notFound = () => fail(404, "Artifact not found");
|
|
29
|
+
function tooLarge(message, limit) {
|
|
30
|
+
return fail(413, message, { code: "limit_exceeded", details: { limitBytes: limit } });
|
|
31
|
+
}
|
|
32
|
+
/** A body past `maxBytes`: past the per-file cap, or past what is left of the Tenant total. */
|
|
33
|
+
function tooLargeFor(maxBytes, limits) {
|
|
34
|
+
return maxBytes === limits.fileBytes
|
|
35
|
+
? tooLarge(`A file may hold at most ${limits.fileBytes} bytes`, limits.fileBytes)
|
|
36
|
+
: tooLarge(`The Tenant's artifacts may hold at most ${limits.totalBytes} bytes; this file does not fit`, limits.totalBytes);
|
|
37
|
+
}
|
|
38
|
+
export function versionView(row) {
|
|
39
|
+
return {
|
|
40
|
+
version: row.version,
|
|
41
|
+
size: row.size,
|
|
42
|
+
sha256: row.sha256,
|
|
43
|
+
contentType: row.contentType,
|
|
44
|
+
source: row.source,
|
|
45
|
+
createdAt: row.createdAt,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
export function artifactView(row, versions) {
|
|
49
|
+
return {
|
|
50
|
+
artifactId: row.id,
|
|
51
|
+
kind: row.kind,
|
|
52
|
+
name: row.name,
|
|
53
|
+
contentType: row.contentType,
|
|
54
|
+
...(row.sessionId === null ? {} : { sessionId: row.sessionId }),
|
|
55
|
+
latestVersion: row.latestVersion,
|
|
56
|
+
...(row.labelsJson === null ? {} : { labels: JSON.parse(row.labelsJson) }),
|
|
57
|
+
createdAt: row.createdAt,
|
|
58
|
+
updatedAt: row.updatedAt,
|
|
59
|
+
...(versions === undefined ? {} : { versions: versions.map(versionView) }),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/** An artifact name: 1–255 characters, no path separators or control characters. */
|
|
63
|
+
export function checkArtifactName(name) {
|
|
64
|
+
if (name === undefined ||
|
|
65
|
+
name.trim() === "" ||
|
|
66
|
+
name.length > ARTIFACT_NAME_MAX ||
|
|
67
|
+
/[/\\\u0000-\u001f\u007f]/u.test(name) ||
|
|
68
|
+
name === "." ||
|
|
69
|
+
name === "..")
|
|
70
|
+
return fail(400, `An artifact name is 1 to ${ARTIFACT_NAME_MAX} characters, without '/', '\\' or control characters`);
|
|
71
|
+
return name;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* The artifact if the caller may reach it, else the 404 of a missing one. Reads its session
|
|
75
|
+
* (without locking) to check whose it is.
|
|
76
|
+
*/
|
|
77
|
+
export async function readableArtifact(t, id, access, options = {}) {
|
|
78
|
+
if (!isArtifactId(id))
|
|
79
|
+
return notFound();
|
|
80
|
+
const row = await t.artifact(id, options);
|
|
81
|
+
if (!row)
|
|
82
|
+
return notFound();
|
|
83
|
+
await checkReach(t, row, access);
|
|
84
|
+
return row;
|
|
85
|
+
}
|
|
86
|
+
async function checkReach(t, row, access) {
|
|
87
|
+
if (access === undefined)
|
|
88
|
+
return;
|
|
89
|
+
if (row.sessionId === null)
|
|
90
|
+
notFound();
|
|
91
|
+
try {
|
|
92
|
+
await sessionOf(t, row.sessionId, access);
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
notFound();
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** `version`, or the latest when it is undefined; a missing one is a 404. */
|
|
99
|
+
export async function versionOf(t, row, version) {
|
|
100
|
+
const found = await t.artifactVersion(row.id, version ?? row.latestVersion);
|
|
101
|
+
return found ?? fail(404, "Artifact version not found");
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Stores `body` as a new artifact, or a new version of one, within the Tenant's limits. See the
|
|
105
|
+
* module comment for how it streams and commits.
|
|
106
|
+
*/
|
|
107
|
+
export async function uploadArtifact(ctx, input, body, access, signal) {
|
|
108
|
+
const name = input.artifactId === undefined ? checkArtifactName(input.name) : undefined;
|
|
109
|
+
// Where it goes and how much room is left, before a byte is stored.
|
|
110
|
+
const { limits, used, sessionId, fileName } = await ctx.store.tx(async (t) => {
|
|
111
|
+
let sessionId;
|
|
112
|
+
let fileName = name;
|
|
113
|
+
if (input.artifactId !== undefined) {
|
|
114
|
+
const prior = await readableArtifact(t, input.artifactId, access);
|
|
115
|
+
if (prior.kind !== "file")
|
|
116
|
+
fail(400, `Artifact ${prior.id} is a folder; its versions come from turn-end exports`);
|
|
117
|
+
sessionId = prior.sessionId;
|
|
118
|
+
fileName = prior.name;
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
if (input.sessionId === undefined && access !== undefined)
|
|
122
|
+
fail(400, "Acting for a person, an artifact belongs to one of their sessions: name sessionId");
|
|
123
|
+
if (input.sessionId !== undefined)
|
|
124
|
+
await sessionOf(t, input.sessionId, access);
|
|
125
|
+
sessionId = input.sessionId ?? null;
|
|
126
|
+
}
|
|
127
|
+
return { limits: await readArtifactLimits(t), used: await t.artifactBytes(), sessionId, fileName };
|
|
128
|
+
});
|
|
129
|
+
const contentType = input.contentType ?? contentTypeFor(fileName);
|
|
130
|
+
const room = limits.totalBytes - used;
|
|
131
|
+
if (room <= 0)
|
|
132
|
+
tooLarge(`The Tenant's artifacts hold ${used} of ${limits.totalBytes} bytes; delete some first`, limits.totalBytes);
|
|
133
|
+
const maxBytes = Math.min(limits.fileBytes, room);
|
|
134
|
+
if (input.declaredBytes !== undefined && input.declaredBytes > maxBytes)
|
|
135
|
+
tooLargeFor(maxBytes, limits);
|
|
136
|
+
const artifactId = input.artifactId ?? newArtifactId();
|
|
137
|
+
const key = `artifacts/${artifactId}/${randomUUID()}`;
|
|
138
|
+
let stored;
|
|
139
|
+
try {
|
|
140
|
+
stored = await ctx.blobs.put(key, body, {
|
|
141
|
+
contentType,
|
|
142
|
+
maxBytes,
|
|
143
|
+
...(signal ? { signal } : {}),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
if (error instanceof BlobTooLargeError)
|
|
148
|
+
tooLargeFor(maxBytes, limits);
|
|
149
|
+
throw error;
|
|
150
|
+
}
|
|
151
|
+
const createdAt = new Date().toISOString();
|
|
152
|
+
try {
|
|
153
|
+
return await ctx.store.tx(async (t) => {
|
|
154
|
+
// The session first, then the quota, then the artifact (the module's lock order).
|
|
155
|
+
let session;
|
|
156
|
+
if (sessionId !== null)
|
|
157
|
+
session = await lockedSession(t, sessionId, access);
|
|
158
|
+
await t.lockArtifactQuota();
|
|
159
|
+
const total = await t.artifactBytes();
|
|
160
|
+
if (total + stored.size > limits.totalBytes)
|
|
161
|
+
tooLarge(`The Tenant's artifacts may hold at most ${limits.totalBytes} bytes; this file does not fit`, limits.totalBytes);
|
|
162
|
+
let row;
|
|
163
|
+
let version;
|
|
164
|
+
if (input.artifactId !== undefined) {
|
|
165
|
+
const prior = await readableArtifact(t, input.artifactId, access, { lock: true });
|
|
166
|
+
version = {
|
|
167
|
+
artifactId: prior.id,
|
|
168
|
+
version: prior.latestVersion + 1,
|
|
169
|
+
blobKey: key,
|
|
170
|
+
size: stored.size,
|
|
171
|
+
sha256: stored.sha256,
|
|
172
|
+
contentType,
|
|
173
|
+
source: input.source,
|
|
174
|
+
createdAt,
|
|
175
|
+
};
|
|
176
|
+
await t.insertArtifactVersion(version);
|
|
177
|
+
row = {
|
|
178
|
+
...prior,
|
|
179
|
+
latestVersion: version.version,
|
|
180
|
+
contentType: version.contentType,
|
|
181
|
+
updatedAt: createdAt,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
row = {
|
|
186
|
+
id: artifactId,
|
|
187
|
+
kind: "file",
|
|
188
|
+
name: name,
|
|
189
|
+
contentType,
|
|
190
|
+
sessionId,
|
|
191
|
+
latestVersion: 1,
|
|
192
|
+
labelsJson: input.labels === undefined || Object.keys(input.labels).length === 0
|
|
193
|
+
? null
|
|
194
|
+
: JSON.stringify(input.labels),
|
|
195
|
+
createdAt,
|
|
196
|
+
updatedAt: createdAt,
|
|
197
|
+
};
|
|
198
|
+
version = {
|
|
199
|
+
artifactId,
|
|
200
|
+
version: 1,
|
|
201
|
+
blobKey: key,
|
|
202
|
+
size: stored.size,
|
|
203
|
+
sha256: stored.sha256,
|
|
204
|
+
contentType,
|
|
205
|
+
source: input.source,
|
|
206
|
+
createdAt,
|
|
207
|
+
};
|
|
208
|
+
await t.insertArtifact(row, version);
|
|
209
|
+
}
|
|
210
|
+
if (session)
|
|
211
|
+
await t.event(session.id, input.turnId ?? null, version.version === 1 ? "artifact.created" : "artifact.version.created", {
|
|
212
|
+
artifactId: row.id,
|
|
213
|
+
kind: "file",
|
|
214
|
+
name: row.name,
|
|
215
|
+
contentType: version.contentType,
|
|
216
|
+
version: version.version,
|
|
217
|
+
size: version.size,
|
|
218
|
+
sha256: version.sha256,
|
|
219
|
+
source: version.source,
|
|
220
|
+
...(input.callId === undefined ? {} : { callId: input.callId }),
|
|
221
|
+
});
|
|
222
|
+
return { artifact: artifactView(row), version: versionView(version) };
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
// Nothing references the object: it was garbage from the start.
|
|
227
|
+
await ctx.blobs.delete(key).catch(() => undefined);
|
|
228
|
+
throw error;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/** The caller's artifacts: of one session, or of every session it reaches. */
|
|
232
|
+
export async function listArtifacts(ctx, filter, access) {
|
|
233
|
+
return ctx.store.tx(async (t) => {
|
|
234
|
+
if (filter.sessionId !== undefined) {
|
|
235
|
+
await sessionOf(t, filter.sessionId, access);
|
|
236
|
+
return { artifacts: (await t.listArtifacts({ sessionId: filter.sessionId })).map((row) => artifactView(row)) };
|
|
237
|
+
}
|
|
238
|
+
const rows = await t.listArtifacts();
|
|
239
|
+
if (access === undefined)
|
|
240
|
+
return { artifacts: rows.map((row) => artifactView(row)) };
|
|
241
|
+
const owned = new Map();
|
|
242
|
+
const reachable = [];
|
|
243
|
+
for (const row of rows) {
|
|
244
|
+
if (row.sessionId === null)
|
|
245
|
+
continue;
|
|
246
|
+
let ok = owned.get(row.sessionId);
|
|
247
|
+
if (ok === undefined) {
|
|
248
|
+
ok = await sessionOf(t, row.sessionId, access).then(() => true, () => false);
|
|
249
|
+
owned.set(row.sessionId, ok);
|
|
250
|
+
}
|
|
251
|
+
if (ok)
|
|
252
|
+
reachable.push(row);
|
|
253
|
+
}
|
|
254
|
+
return { artifacts: reachable.map((row) => artifactView(row)) };
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
/** One artifact with every version. */
|
|
258
|
+
export async function getArtifact(ctx, id, access) {
|
|
259
|
+
return ctx.store.tx(async (t) => {
|
|
260
|
+
const row = await readableArtifact(t, id, access);
|
|
261
|
+
return artifactView(row, await t.artifactVersions(row.id));
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
/** One version's row, for a download. */
|
|
265
|
+
export async function artifactContent(ctx, id, version, access) {
|
|
266
|
+
return ctx.store.tx(async (t) => {
|
|
267
|
+
const artifact = await readableArtifact(t, id, access);
|
|
268
|
+
return { artifact, version: await versionOf(t, artifact, version) };
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
/** One version's row, its artifact and, for a folder, its manifest; a file is a 400. */
|
|
272
|
+
export async function folderContent(ctx, id, version, access) {
|
|
273
|
+
const found = await artifactContent(ctx, id, version, access);
|
|
274
|
+
if (found.artifact.kind !== "folder")
|
|
275
|
+
fail(400, `Artifact ${found.artifact.id} is a file; download its content`);
|
|
276
|
+
return { ...found, manifest: await readManifest(ctx.blobs, found.version) };
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Deletes the artifact and every version: its rows and, for a session's artifact, its
|
|
280
|
+
* `artifact.deleted` event in one transaction, then its bytes. A folder's content-addressed
|
|
281
|
+
* files go too, those no other version names, under the quota lock (`folders.ts`).
|
|
282
|
+
*/
|
|
283
|
+
export async function deleteArtifact(ctx, id, access) {
|
|
284
|
+
const keys = await ctx.store.tx(async (t) => {
|
|
285
|
+
const found = await readableArtifact(t, id, access);
|
|
286
|
+
if (found.sessionId !== null)
|
|
287
|
+
await lockedSession(t, found.sessionId, access);
|
|
288
|
+
if (found.kind === "folder")
|
|
289
|
+
await t.lockArtifactQuota();
|
|
290
|
+
const row = await readableArtifact(t, id, access, { lock: true });
|
|
291
|
+
const content = row.kind === "folder" ? await t.artifactContentShas({ artifactId: row.id }) : [];
|
|
292
|
+
const keys = await t.deleteArtifact(row.id);
|
|
293
|
+
await collectContent(ctx.blobs, t, content);
|
|
294
|
+
if (row.sessionId !== null)
|
|
295
|
+
await t.event(row.sessionId, null, "artifact.deleted", { artifactId: row.id, name: row.name });
|
|
296
|
+
return keys;
|
|
297
|
+
});
|
|
298
|
+
for (const key of keys)
|
|
299
|
+
await ctx.blobs.delete(key).catch(() => undefined);
|
|
300
|
+
return { artifactId: id, deleted: true };
|
|
301
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `WorkspaceReader` seam (F8.2): how core reads a session's sandbox workspace at turn end, to
|
|
3
|
+
* export its outputs as a folder artifact. Two calls, a listing and one file's bytes, both the
|
|
4
|
+
* claim of whoever holds the workspace: the Runtime stores what it is given without observing it.
|
|
5
|
+
*
|
|
6
|
+
* One implementation, `sandboxWorkspaceReader`, over the workspace capability (F6.2,
|
|
7
|
+
* `harness-api/workspace.ts`): the in-process `SandboxManager`, or the harness that serves the
|
|
8
|
+
* Tenant's workspaces (`workspace.read`). F6.2 reimplements it over the Harness API's `workspace.read`, and F7.2's pod
|
|
9
|
+
* sandboxes are read through the same seam, so the export never knows which kind of sandbox it
|
|
10
|
+
* reads.
|
|
11
|
+
*/
|
|
12
|
+
import { type AgentManifest } from "@nylorun/core/define";
|
|
13
|
+
import { type SandboxManager } from "../sandbox/manager.js";
|
|
14
|
+
/** What the reader needs of a workspace: a `SandboxManager`, or the workspace capability. */
|
|
15
|
+
export type WorkspaceFiles = Pick<SandboxManager, "listFiles" | "readBytes">;
|
|
16
|
+
/** Whose workspace to read: the session whose turn ended, resolved to the workspace it uses. */
|
|
17
|
+
export interface WorkspaceSession {
|
|
18
|
+
/** The session whose turn ended. */
|
|
19
|
+
readonly sessionId: string;
|
|
20
|
+
/** The turn that ended. */
|
|
21
|
+
readonly turnId: string | null;
|
|
22
|
+
/** The session that owns the workspace (one hop through `sandboxOwnerId`). */
|
|
23
|
+
readonly ownerId: string;
|
|
24
|
+
/** The sandbox resource the workspace belongs to, when the owner is attached to one. */
|
|
25
|
+
readonly sandboxId?: string;
|
|
26
|
+
/** The session's pinned manifest: its sandbox capability carries the sandbox's spec. */
|
|
27
|
+
readonly manifest: AgentManifest;
|
|
28
|
+
}
|
|
29
|
+
/** One regular file under the listed directory. */
|
|
30
|
+
export interface WorkspaceEntry {
|
|
31
|
+
/** Relative to the directory listed, `/`-separated, never `.` or `..` segments. */
|
|
32
|
+
readonly path: string;
|
|
33
|
+
readonly size: number;
|
|
34
|
+
}
|
|
35
|
+
export interface WorkspaceListing {
|
|
36
|
+
/** The files in path order, symbolic links left out. */
|
|
37
|
+
readonly entries: readonly WorkspaceEntry[];
|
|
38
|
+
/** More than `maxEntries` files: `entries` holds `maxEntries + 1` of them. */
|
|
39
|
+
readonly truncated: boolean;
|
|
40
|
+
}
|
|
41
|
+
export interface WorkspaceReader {
|
|
42
|
+
/**
|
|
43
|
+
* The regular files under `dir` (absolute, e.g. `/workspace/outputs`), recursively. Undefined
|
|
44
|
+
* when the session has no sandbox, its sandbox was never created, or `dir` is not a directory.
|
|
45
|
+
*/
|
|
46
|
+
list(session: WorkspaceSession, dir: string, options: {
|
|
47
|
+
readonly maxEntries: number;
|
|
48
|
+
readonly signal: AbortSignal;
|
|
49
|
+
}): Promise<WorkspaceListing | undefined>;
|
|
50
|
+
/**
|
|
51
|
+
* One file's bytes (absolute path), or undefined when it is gone. A file larger than
|
|
52
|
+
* `maxBytes` throws `SandboxFileTooLargeError`; a sandbox that fails throws.
|
|
53
|
+
*/
|
|
54
|
+
readBytes(session: WorkspaceSession, path: string, options: {
|
|
55
|
+
readonly maxBytes: number;
|
|
56
|
+
readonly signal: AbortSignal;
|
|
57
|
+
}): Promise<Uint8Array | undefined>;
|
|
58
|
+
}
|
|
59
|
+
/** The reader over the workspaces, in each sandbox's queue of tool calls. */
|
|
60
|
+
export declare function sandboxWorkspaceReader(sandbox: WorkspaceFiles): WorkspaceReader;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `WorkspaceReader` seam (F8.2): how core reads a session's sandbox workspace at turn end, to
|
|
3
|
+
* export its outputs as a folder artifact. Two calls, a listing and one file's bytes, both the
|
|
4
|
+
* claim of whoever holds the workspace: the Runtime stores what it is given without observing it.
|
|
5
|
+
*
|
|
6
|
+
* One implementation, `sandboxWorkspaceReader`, over the workspace capability (F6.2,
|
|
7
|
+
* `harness-api/workspace.ts`): the in-process `SandboxManager`, or the harness that serves the
|
|
8
|
+
* Tenant's workspaces (`workspace.read`). F6.2 reimplements it over the Harness API's `workspace.read`, and F7.2's pod
|
|
9
|
+
* sandboxes are read through the same seam, so the export never knows which kind of sandbox it
|
|
10
|
+
* reads.
|
|
11
|
+
*/
|
|
12
|
+
import { SANDBOX_CAPABILITY_ID, } from "@nylorun/core/define";
|
|
13
|
+
import { sandboxCapabilityOf } from "../sandbox/manager.js";
|
|
14
|
+
import { SandboxFileTooLargeError } from "../sandbox/types.js";
|
|
15
|
+
/** The reader over the workspaces, in each sandbox's queue of tool calls. */
|
|
16
|
+
export function sandboxWorkspaceReader(sandbox) {
|
|
17
|
+
const target = (session) => {
|
|
18
|
+
const capability = sandboxCapabilityOf(session.manifest, SANDBOX_CAPABILITY_ID, "read");
|
|
19
|
+
if (!capability)
|
|
20
|
+
return undefined;
|
|
21
|
+
return {
|
|
22
|
+
capability,
|
|
23
|
+
ref: {
|
|
24
|
+
id: session.ownerId,
|
|
25
|
+
activeTurnId: session.turnId,
|
|
26
|
+
manifest: session.manifest,
|
|
27
|
+
...(session.sandboxId === undefined ? {} : { sandboxId: session.sandboxId }),
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
return {
|
|
32
|
+
async list(session, dir, options) {
|
|
33
|
+
const found = target(session);
|
|
34
|
+
if (!found)
|
|
35
|
+
return undefined;
|
|
36
|
+
const listed = await sandbox.listFiles(found.ref, found.capability, dir, options.maxEntries, options.signal);
|
|
37
|
+
if (listed.kind === "missing")
|
|
38
|
+
return undefined;
|
|
39
|
+
if (listed.kind === "failed")
|
|
40
|
+
throw new Error(listed.message);
|
|
41
|
+
return listed.listing;
|
|
42
|
+
},
|
|
43
|
+
async readBytes(session, path, options) {
|
|
44
|
+
const found = target(session);
|
|
45
|
+
if (!found)
|
|
46
|
+
return undefined;
|
|
47
|
+
const read = await sandbox.readBytes(found.ref, found.capability, path, options.maxBytes, options.signal);
|
|
48
|
+
if (read.kind === "read")
|
|
49
|
+
return read.bytes;
|
|
50
|
+
if (read.kind === "missing")
|
|
51
|
+
return undefined;
|
|
52
|
+
if (read.code === "artifact.too_large")
|
|
53
|
+
throw new SandboxFileTooLargeError(path, options.maxBytes + 1, options.maxBytes);
|
|
54
|
+
throw new Error(read.message);
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface ZipEntry {
|
|
2
|
+
/** The path in the archive, `/`-separated. */
|
|
3
|
+
readonly name: string;
|
|
4
|
+
/** The file's bytes, opened only when the writer reaches it. */
|
|
5
|
+
readonly open: () => Promise<AsyncIterable<Uint8Array>>;
|
|
6
|
+
}
|
|
7
|
+
/** Why a folder cannot be zipped without ZIP64, or undefined when it can. */
|
|
8
|
+
export declare function zipTooLarge(files: number, bytes: number): string | undefined;
|
|
9
|
+
/** The archive's bytes, as the writer produces them. */
|
|
10
|
+
export declare function zipStream(entries: readonly ZipEntry[], modified?: Date): AsyncGenerator<Uint8Array>;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A streaming zip writer (PKWARE APPNOTE 6.3), enough for a folder artifact's download: each
|
|
3
|
+
* file deflated as it streams, its CRC-32 and sizes in a data descriptor after it (general
|
|
4
|
+
* purpose bit 3) and again in the central directory, names in UTF-8 (bit 11). Nothing is
|
|
5
|
+
* buffered beyond one chunk. No ZIP64: a folder this serves stays under 4 GiB and 65,535 files
|
|
6
|
+
* (the export's limits), and `zipTooLarge` says when one would not.
|
|
7
|
+
*/
|
|
8
|
+
import { constants, crc32, createDeflateRaw } from "node:zlib";
|
|
9
|
+
const LOCAL = 0x04034b50;
|
|
10
|
+
const DESCRIPTOR = 0x08074b50;
|
|
11
|
+
const CENTRAL = 0x02014b50;
|
|
12
|
+
const END = 0x06054b50;
|
|
13
|
+
const FLAGS = 0x0808; // bit 3: sizes in a data descriptor; bit 11: UTF-8 names
|
|
14
|
+
const DEFLATE = 8;
|
|
15
|
+
const VERSION = 20;
|
|
16
|
+
/** Made by Unix (3), so the external attributes below are a Unix mode. */
|
|
17
|
+
const MADE_BY_UNIX = (3 << 8) | VERSION;
|
|
18
|
+
/** A regular file, `rw-r--r--`, in the high 16 bits. */
|
|
19
|
+
const REGULAR_FILE = (0o100644 << 16) >>> 0;
|
|
20
|
+
const MAX_32 = 0xffffffff;
|
|
21
|
+
/** Why a folder cannot be zipped without ZIP64, or undefined when it can. */
|
|
22
|
+
export function zipTooLarge(files, bytes) {
|
|
23
|
+
if (files > 0xffff)
|
|
24
|
+
return `A zip holds at most 65535 files; this folder has ${files}`;
|
|
25
|
+
// Deflate adds at most 5 bytes per 16 KiB block; keep well clear of 4 GiB.
|
|
26
|
+
if (bytes > 3.5 * 1024 * 1024 * 1024)
|
|
27
|
+
return `A zip holds at most 3.5 GiB; this folder has ${bytes} bytes`;
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
/** MS-DOS date and time, as zip headers store them (local time is not known: UTC). */
|
|
31
|
+
function dosDateTime(date) {
|
|
32
|
+
const year = Math.max(1980, date.getUTCFullYear());
|
|
33
|
+
return {
|
|
34
|
+
time: (date.getUTCHours() << 11) | (date.getUTCMinutes() << 5) | (date.getUTCSeconds() >> 1),
|
|
35
|
+
date: ((year - 1980) << 9) | ((date.getUTCMonth() + 1) << 5) | date.getUTCDate(),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/** The archive's bytes, as the writer produces them. */
|
|
39
|
+
export async function* zipStream(entries, modified = new Date()) {
|
|
40
|
+
const stamp = dosDateTime(modified);
|
|
41
|
+
const central = [];
|
|
42
|
+
let offset = 0;
|
|
43
|
+
for (const entry of entries) {
|
|
44
|
+
const name = Buffer.from(entry.name, "utf8");
|
|
45
|
+
const local = Buffer.alloc(30);
|
|
46
|
+
local.writeUInt32LE(LOCAL, 0);
|
|
47
|
+
local.writeUInt16LE(VERSION, 4);
|
|
48
|
+
local.writeUInt16LE(FLAGS, 6);
|
|
49
|
+
local.writeUInt16LE(DEFLATE, 8);
|
|
50
|
+
local.writeUInt16LE(stamp.time, 10);
|
|
51
|
+
local.writeUInt16LE(stamp.date, 12);
|
|
52
|
+
// CRC and sizes (14–25) are zero: they follow in the data descriptor.
|
|
53
|
+
local.writeUInt16LE(name.length, 26);
|
|
54
|
+
local.writeUInt16LE(0, 28);
|
|
55
|
+
const headerAt = offset;
|
|
56
|
+
yield local;
|
|
57
|
+
yield name;
|
|
58
|
+
offset += local.length + name.length;
|
|
59
|
+
let crc = 0;
|
|
60
|
+
let size = 0;
|
|
61
|
+
let compressed = 0;
|
|
62
|
+
const deflate = createDeflateRaw({ level: constants.Z_DEFAULT_COMPRESSION });
|
|
63
|
+
const source = await entry.open();
|
|
64
|
+
const feed = (async () => {
|
|
65
|
+
try {
|
|
66
|
+
for await (const chunk of source) {
|
|
67
|
+
// The reader went away (a cancelled download): stop reading the file.
|
|
68
|
+
if (deflate.destroyed)
|
|
69
|
+
break;
|
|
70
|
+
crc = crc32(chunk, crc);
|
|
71
|
+
size += chunk.length;
|
|
72
|
+
if (!deflate.write(chunk))
|
|
73
|
+
await new Promise((resolve) => {
|
|
74
|
+
deflate.once("drain", resolve);
|
|
75
|
+
deflate.once("close", resolve);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
if (!deflate.destroyed)
|
|
79
|
+
deflate.end();
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
deflate.destroy(error);
|
|
83
|
+
}
|
|
84
|
+
})();
|
|
85
|
+
for await (const chunk of deflate) {
|
|
86
|
+
compressed += chunk.length;
|
|
87
|
+
yield chunk;
|
|
88
|
+
}
|
|
89
|
+
await feed;
|
|
90
|
+
offset += compressed;
|
|
91
|
+
if (offset > MAX_32 || size > MAX_32)
|
|
92
|
+
throw new Error("The folder is too large to zip without ZIP64");
|
|
93
|
+
const descriptor = Buffer.alloc(16);
|
|
94
|
+
descriptor.writeUInt32LE(DESCRIPTOR, 0);
|
|
95
|
+
descriptor.writeUInt32LE(crc >>> 0, 4);
|
|
96
|
+
descriptor.writeUInt32LE(compressed, 8);
|
|
97
|
+
descriptor.writeUInt32LE(size, 12);
|
|
98
|
+
yield descriptor;
|
|
99
|
+
offset += descriptor.length;
|
|
100
|
+
const record = Buffer.alloc(46);
|
|
101
|
+
record.writeUInt32LE(CENTRAL, 0);
|
|
102
|
+
record.writeUInt16LE(MADE_BY_UNIX, 4);
|
|
103
|
+
record.writeUInt16LE(VERSION, 6);
|
|
104
|
+
record.writeUInt16LE(FLAGS, 8);
|
|
105
|
+
record.writeUInt16LE(DEFLATE, 10);
|
|
106
|
+
record.writeUInt16LE(stamp.time, 12);
|
|
107
|
+
record.writeUInt16LE(stamp.date, 14);
|
|
108
|
+
record.writeUInt32LE(crc >>> 0, 16);
|
|
109
|
+
record.writeUInt32LE(compressed, 20);
|
|
110
|
+
record.writeUInt32LE(size, 24);
|
|
111
|
+
record.writeUInt16LE(name.length, 28);
|
|
112
|
+
// Extra, comment, disk and internal attributes (30–37) stay zero.
|
|
113
|
+
record.writeUInt32LE(REGULAR_FILE, 38);
|
|
114
|
+
record.writeUInt32LE(headerAt, 42);
|
|
115
|
+
central.push(record, name);
|
|
116
|
+
}
|
|
117
|
+
const directory = Buffer.concat(central);
|
|
118
|
+
yield directory;
|
|
119
|
+
const end = Buffer.alloc(22);
|
|
120
|
+
end.writeUInt32LE(END, 0);
|
|
121
|
+
end.writeUInt16LE(entries.length, 8);
|
|
122
|
+
end.writeUInt16LE(entries.length, 10);
|
|
123
|
+
end.writeUInt32LE(directory.length, 12);
|
|
124
|
+
end.writeUInt32LE(offset, 16);
|
|
125
|
+
yield end;
|
|
126
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type BlobStore } from "./types.js";
|
|
2
|
+
export interface FsBlobStoreOptions {
|
|
3
|
+
/** Absolute directory the store owns; created on the first write. */
|
|
4
|
+
root: string;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* The `fs` adapter: objects as files under `root`, for embedding (`startEphemeralRuntime`, a
|
|
8
|
+
* Host without an Object store) and tests. `objects/<key>` holds the bytes, `meta/<key>.json`
|
|
9
|
+
* the content type; a write goes to `tmp/` first and is renamed into place, so a reader sees
|
|
10
|
+
* the old object or the new one, never part of one. A key cannot also be the directory of
|
|
11
|
+
* another key (`a` and `a/b`); the Runtime's keys never are.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createFsBlobStore(options: FsBlobStoreOptions): BlobStore;
|