@foldrun/core 8.0.1 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/history.d.ts +5 -0
- package/dist/src/history.js +31 -4
- package/dist/src/hooks.d.ts +2 -0
- package/dist/src/hooks.js +13 -0
- package/dist/src/library.js +4 -0
- package/dist/src/notify.d.ts +15 -1
- package/dist/src/notify.js +81 -11
- package/dist/src/runner.js +12 -7
- package/dist/src/secrets.js +8 -2
- package/dist/src/storage.d.ts +76 -4
- package/dist/src/storage.js +219 -22
- package/dist/src/store.d.ts +20 -2
- package/dist/src/store.js +50 -15
- package/dist/src/webhook.js +5 -1
- package/package.json +1 -1
- package/src/history.ts +31 -4
- package/src/hooks.ts +14 -0
- package/src/library.ts +3 -0
- package/src/notify.ts +82 -12
- package/src/runner.ts +12 -6
- package/src/secrets.ts +7 -2
- package/src/storage.ts +223 -19
- package/src/store.ts +49 -14
- package/src/webhook.ts +4 -1
package/src/notify.ts
CHANGED
|
@@ -34,7 +34,8 @@
|
|
|
34
34
|
import crypto from "node:crypto";
|
|
35
35
|
import { readAgentsMd } from "./runner.ts";
|
|
36
36
|
import { accountDir, workspaceDir, runCost, readFlow, runVerdict, type RunRecord } from "./store.ts";
|
|
37
|
-
import { getSecret } from "./secrets.ts";
|
|
37
|
+
import { getSecret, secretSites } from "./secrets.ts";
|
|
38
|
+
import { siteCovers } from "./network.ts";
|
|
38
39
|
import { publicUrl } from "./webhook.ts";
|
|
39
40
|
import { approveLinkPath, approveLinkTtlMs } from "./approvals.ts";
|
|
40
41
|
import { noteSecretUse, healthKey } from "./secret-health.ts";
|
|
@@ -209,15 +210,80 @@ export function newDeliveryId(): string {
|
|
|
209
210
|
}
|
|
210
211
|
|
|
211
212
|
/** The URL with ${SECRET}s resolved, and how it may be shown: a Slack hook
|
|
212
|
-
* URL is itself a credential, so it is never echoed resolved.
|
|
213
|
-
|
|
213
|
+
* URL is itself a credential, so it is never echoed resolved.
|
|
214
|
+
*
|
|
215
|
+
* A secret bound to sites (`foldrun secrets sites`) is filled only when the
|
|
216
|
+
* resolved URL's host is one of them — the rule the egress proxy and the
|
|
217
|
+
* tool tester keep (30 Oct 2026). Before, notify.url filled any secret into
|
|
218
|
+
* any host, so an editor could write `https://their.server/?k=${STRIPE_KEY}`
|
|
219
|
+
* and press Test. The host is read AFTER filling, because the usual shape is
|
|
220
|
+
* the whole URL as one secret (`${SLACK_WEBHOOK_URL}`, bound to
|
|
221
|
+
* hooks.slack.com). An unbound secret fills as it does for an agent's
|
|
222
|
+
* `secrets:` and a tool test — workspace first, then the account — since
|
|
223
|
+
* binding is how an account says where a value may go. `refused` says why
|
|
224
|
+
* `url` is null when a binding stopped it; `filled` maps each value put
|
|
225
|
+
* into the URL to its name, so an error that echoes the URL can be scrubbed. */
|
|
226
|
+
export function webhookTarget(
|
|
227
|
+
tenant: string,
|
|
228
|
+
workspace: string,
|
|
229
|
+
config: NotifyConfig | null = notifyConfig(tenant, workspace),
|
|
230
|
+
): { url: string | null; shown: string; refused?: string; filled: Record<string, string> } {
|
|
214
231
|
const raw = config?.url ?? "";
|
|
232
|
+
const filled: Record<string, string> = {};
|
|
233
|
+
// Every name put in, kept apart from `filled` (value → name, for
|
|
234
|
+
// scrubbing): two secrets with the same value would otherwise leave only
|
|
235
|
+
// the last name there, and the first one's binding would go unchecked.
|
|
236
|
+
const used = new Set<string>();
|
|
215
237
|
const url = raw.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => {
|
|
216
238
|
const hit = getSecret(tenant, name, workspace);
|
|
239
|
+
if (hit) {
|
|
240
|
+
filled[hit.value] = name;
|
|
241
|
+
used.add(name);
|
|
242
|
+
}
|
|
217
243
|
return hit ? hit.value : whole;
|
|
218
244
|
});
|
|
219
245
|
const shown = raw.includes("${") ? raw : raw ? `${raw.slice(0, 40)}${raw.length > 40 ? "…" : ""}` : "none";
|
|
220
|
-
|
|
246
|
+
if (!url || url.includes("${")) return { url: null, shown, filled: {} };
|
|
247
|
+
const bound = secretSites(tenant, [...used], workspace);
|
|
248
|
+
if (Object.keys(bound).length) {
|
|
249
|
+
let host = "";
|
|
250
|
+
try {
|
|
251
|
+
host = new URL(url).hostname;
|
|
252
|
+
} catch {
|
|
253
|
+
// Not a URL: no host can be one of its sites.
|
|
254
|
+
}
|
|
255
|
+
const offSite = Object.entries(bound).filter(([, sites]) => !host || !sites.some((site) => siteCovers(site, host))).map(([n]) => n);
|
|
256
|
+
if (offSite.length) {
|
|
257
|
+
return {
|
|
258
|
+
url: null,
|
|
259
|
+
shown,
|
|
260
|
+
filled: {},
|
|
261
|
+
// The host is not echoed: it is read from the FILLED url, so
|
|
262
|
+
// `https://${KEY}.their.server/` would print the key back.
|
|
263
|
+
refused: `${offSite.join(", ")} ${offSite.length === 1 ? "is" : "are"} bound to other sites than notify.url's host — not sent`,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return { url, shown, filled };
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** An error about a request to a resolved notify.url, with every secret value
|
|
271
|
+
* put back as its ${NAME} — a redirect refusal quotes the URL it was sent to,
|
|
272
|
+
* and the delivery log that keeps it is read by every viewer. */
|
|
273
|
+
function scrubTarget(text: string, filled: Record<string, string>): string {
|
|
274
|
+
let out = text;
|
|
275
|
+
// Longest first, so a value inside another is not half-replaced.
|
|
276
|
+
for (const value of Object.keys(filled).sort((a, b) => b.length - a.length)) {
|
|
277
|
+
if (!value) continue;
|
|
278
|
+
const variants = new Set([value]);
|
|
279
|
+
try {
|
|
280
|
+
variants.add(encodeURI(value)).add(encodeURIComponent(value));
|
|
281
|
+
} catch {
|
|
282
|
+
// A lone surrogate cannot be encoded, so it cannot appear encoded.
|
|
283
|
+
}
|
|
284
|
+
for (const v of variants) out = out.split(v).join(`\${${filled[value]}}`);
|
|
285
|
+
}
|
|
286
|
+
return out;
|
|
221
287
|
}
|
|
222
288
|
|
|
223
289
|
export interface WebhookAttempt {
|
|
@@ -252,10 +318,13 @@ export async function attemptWebhook(
|
|
|
252
318
|
if (!config?.url) {
|
|
253
319
|
return { ...base, ok: false, statusCode: null, durationMs: 0, response: null, error: "this workspace's notify: has no url any more" };
|
|
254
320
|
}
|
|
255
|
-
const { url } = webhookTarget(tenant, workspace, config);
|
|
321
|
+
const { url, refused, filled } = webhookTarget(tenant, workspace, config);
|
|
256
322
|
if (!url) {
|
|
257
|
-
return { ...base, ok: false, statusCode: null, durationMs: 0, response: null, error: "the secret named in notify.url is not in this account's vault" };
|
|
323
|
+
return { ...base, ok: false, statusCode: null, durationMs: 0, response: null, error: refused ?? "the secret named in notify.url is not in this account's vault" };
|
|
258
324
|
}
|
|
325
|
+
// A URL that carries a secret is a credential: a redirect is followed only
|
|
326
|
+
// within the origin it was sent to, as the tool tester does.
|
|
327
|
+
const carriesSecret = Object.keys(filled).length > 0;
|
|
259
328
|
try {
|
|
260
329
|
const res = await fetchUntrusted(url, {
|
|
261
330
|
method: "POST",
|
|
@@ -270,7 +339,7 @@ export async function attemptWebhook(
|
|
|
270
339
|
},
|
|
271
340
|
body: d.body,
|
|
272
341
|
signal: AbortSignal.timeout(Number(process.env.FOLDRUN_WEBHOOK_TIMEOUT_MS) || 8000),
|
|
273
|
-
});
|
|
342
|
+
}, { sameOrigin: carriesSecret });
|
|
274
343
|
const text = (await res.text().catch(() => "")).slice(0, 500);
|
|
275
344
|
return {
|
|
276
345
|
...base,
|
|
@@ -282,7 +351,7 @@ export async function attemptWebhook(
|
|
|
282
351
|
};
|
|
283
352
|
} catch (err) {
|
|
284
353
|
const why = err instanceof Error ? (err.name === "TimeoutError" ? "timed out" : err.message) : String(err);
|
|
285
|
-
return { ...base, ok: false, statusCode: null, durationMs: Date.now() - started, response: null, error: why.slice(0, 500) };
|
|
354
|
+
return { ...base, ok: false, statusCode: null, durationMs: Date.now() - started, response: null, error: scrubTarget(why, filled).slice(0, 500) };
|
|
286
355
|
}
|
|
287
356
|
}
|
|
288
357
|
|
|
@@ -486,9 +555,9 @@ export async function sendTestNotification(
|
|
|
486
555
|
// The destination is echoed back with the secret still unresolved: a
|
|
487
556
|
// Slack webhook URL is itself a credential, and a page that prints it is
|
|
488
557
|
// a page that leaks it into a screenshot.
|
|
489
|
-
const { url, shown } = webhookTarget(tenant, workspace, config);
|
|
558
|
+
const { url, shown, refused } = webhookTarget(tenant, workspace, config);
|
|
490
559
|
if (!url) {
|
|
491
|
-
return { ok: false, destination: shown, detail: `the secret named in notify.url is not in this account's vault` };
|
|
560
|
+
return { ok: false, destination: shown, detail: refused ?? `the secret named in notify.url is not in this account's vault` };
|
|
492
561
|
}
|
|
493
562
|
const payload = JSON.stringify({ text: `${headline} — ${detail}`, workspace, status: "test", summary: detail });
|
|
494
563
|
// One attempt, recorded in the delivery log like any other — and the way
|
|
@@ -635,8 +704,9 @@ export async function sendRunNotification(
|
|
|
635
704
|
|
|
636
705
|
// ${SECRET} so the Slack URL — itself a credential — can live in the
|
|
637
706
|
// vault instead of in a file that gets committed.
|
|
638
|
-
|
|
639
|
-
|
|
707
|
+
const target = webhookTarget(tenant, workspace, config);
|
|
708
|
+
if (!target.url) {
|
|
709
|
+
console.error(`[foldrun] notify: ${target.refused ?? "secret in URL not set"} for ${tenant}/${workspace}`);
|
|
640
710
|
return false;
|
|
641
711
|
}
|
|
642
712
|
const event = blocked ? "blocked" : run.status;
|
package/src/runner.ts
CHANGED
|
@@ -90,6 +90,7 @@ import {
|
|
|
90
90
|
STORAGE_DIR,
|
|
91
91
|
adoptLegacyVersionKey,
|
|
92
92
|
listAgents,
|
|
93
|
+
isOneSegment,
|
|
93
94
|
} from "./store.ts";
|
|
94
95
|
import { canReadBack } from "./tool-output.ts";
|
|
95
96
|
import { PromptLayers, promptCacheKey } from "./prompt-layers.ts";
|
|
@@ -4926,8 +4927,12 @@ function driveRunInner(
|
|
|
4926
4927
|
// finished steps to `failed`, destroying the history it was
|
|
4927
4928
|
// resuming around.
|
|
4928
4929
|
if (step.status !== "pending") continue;
|
|
4929
|
-
|
|
4930
|
-
|
|
4930
|
+
// One folder under agents/, never a path (30 Oct 2026): an ad-hoc
|
|
4931
|
+
// run's agent came from the URL, and `../../<tenant>/…` joined here
|
|
4932
|
+
// ran another account's agent. Refused as not found, before the
|
|
4933
|
+
// existence check could answer for a path outside.
|
|
4934
|
+
const aDir = isOneSegment(step.agent) ? path.join(pDir, "agents", step.agent) : null;
|
|
4935
|
+
if (!aDir || !fs.existsSync(path.join(aDir, "agent.md"))) {
|
|
4931
4936
|
step.status = "failed";
|
|
4932
4937
|
step.events.push({
|
|
4933
4938
|
t: new Date().toISOString(),
|
|
@@ -5156,8 +5161,9 @@ function driveRunInner(
|
|
|
5156
5161
|
// step of the flow (isSplicedRecord). It gets no delegateJob of its
|
|
5157
5162
|
// own: depth one.
|
|
5158
5163
|
const delegateFor = (parent: StepRecord) => async (agent: string, job: string, cancelled: () => boolean): Promise<DelegationResult> => {
|
|
5159
|
-
|
|
5160
|
-
|
|
5164
|
+
// One folder under agents/, never a path — as the group check above.
|
|
5165
|
+
const dir = isOneSegment(agent) ? path.join(pDir, "agents", agent) : null;
|
|
5166
|
+
if (!dir || !fs.existsSync(path.join(dir, "agent.md"))) return { status: "failed", result: `${agent} is not an agent in this workspace` };
|
|
5161
5167
|
// A stable id on both records: an index into run.steps moves when a
|
|
5162
5168
|
// rescue or a fan-out is spliced in before the child.
|
|
5163
5169
|
const id = `d_${crypto.randomBytes(5).toString("hex")}`;
|
|
@@ -5597,8 +5603,8 @@ function driveRunInner(
|
|
|
5597
5603
|
};
|
|
5598
5604
|
const rescueStep = async (step: StepRecord): Promise<void> => {
|
|
5599
5605
|
if (!step.onFail) return;
|
|
5600
|
-
const rescuerDir = path.join(pDir, "agents", step.onFail);
|
|
5601
|
-
if (!fs.existsSync(path.join(rescuerDir, "agent.md"))) {
|
|
5606
|
+
const rescuerDir = isOneSegment(step.onFail) ? path.join(pDir, "agents", step.onFail) : null;
|
|
5607
|
+
if (!rescuerDir || !fs.existsSync(path.join(rescuerDir, "agent.md"))) {
|
|
5602
5608
|
step.events.push({
|
|
5603
5609
|
t: new Date().toISOString(),
|
|
5604
5610
|
type: "error",
|
package/src/secrets.ts
CHANGED
|
@@ -291,7 +291,10 @@ export function setSecret(
|
|
|
291
291
|
workspace?: string,
|
|
292
292
|
kind?: SecretKind,
|
|
293
293
|
) {
|
|
294
|
-
|
|
294
|
+
// A string, not anything that prints as one: RegExp.test and the vault's
|
|
295
|
+
// keys both coerce, so ["RESEND_API_KEY"] from a JSON body was stored as
|
|
296
|
+
// RESEND_API_KEY past a caller's check on the name (30 Oct 2026).
|
|
297
|
+
if (typeof name !== "string" || !SECRET_NAME.test(name)) {
|
|
295
298
|
throw new Error(`secret name "${name}" must be UPPER_SNAKE_CASE`);
|
|
296
299
|
}
|
|
297
300
|
if (value.length > 8192) throw new Error("secret too long");
|
|
@@ -314,6 +317,8 @@ export function setSecret(
|
|
|
314
317
|
* scheme, path or port. Returns the list as stored.
|
|
315
318
|
*/
|
|
316
319
|
export function setSecretSites(tenant: string, name: string, sites: string[], workspace?: string): string[] {
|
|
320
|
+
// all[name] would find ["NAME"] as NAME: see setSecret.
|
|
321
|
+
if (typeof name !== "string") throw new Error("a secret's name is a string");
|
|
317
322
|
const clean = [...new Set(sites.map((x) => x.trim().toLowerCase()).filter(Boolean))];
|
|
318
323
|
for (const site of clean) {
|
|
319
324
|
if (!SITE.test(site)) throw new Error(`"${site}" is not a site — a domain like api.example.com or *.example.com, no scheme, path or port`);
|
|
@@ -1157,7 +1162,7 @@ export function setClientCertSecret(tenant: string, name: string, config: Client
|
|
|
1157
1162
|
...(config.passphrase ? { passphrase: config.passphrase } : {}),
|
|
1158
1163
|
...(config.export ? { export: true } : {}),
|
|
1159
1164
|
};
|
|
1160
|
-
if (!SECRET_NAME.test(name)) throw new Error(`secret name "${name}" must be UPPER_SNAKE_CASE`);
|
|
1165
|
+
if (typeof name !== "string" || !SECRET_NAME.test(name)) throw new Error(`secret name "${name}" must be UPPER_SNAKE_CASE`);
|
|
1161
1166
|
const plainCert = CLIENT_CERT_PREFIX + JSON.stringify(stored);
|
|
1162
1167
|
// Written under the vault's own lock, with the binding checked in the same
|
|
1163
1168
|
// step: a value set over a secret keeps its site binding, and a wildcard
|
package/src/storage.ts
CHANGED
|
@@ -51,6 +51,7 @@
|
|
|
51
51
|
import fs from "node:fs";
|
|
52
52
|
import path from "node:path";
|
|
53
53
|
import crypto from "node:crypto";
|
|
54
|
+
import { Readable } from "node:stream";
|
|
54
55
|
import { dataRoot, singleWorkspace } from "./paths.ts";
|
|
55
56
|
import { assertSafeName, workspaceDir, STORAGE_DIR, adoptLegacyFilesDir } from "./store.ts";
|
|
56
57
|
|
|
@@ -85,6 +86,30 @@ interface Index {
|
|
|
85
86
|
const maxBytes = () => Number(process.env.FOLDRUN_STORAGE_MAX_MB ?? 512) * 1024 * 1024;
|
|
86
87
|
const quotaBytes = () => Number(process.env.FOLDRUN_STORAGE_QUOTA_MB ?? 5120) * 1024 * 1024;
|
|
87
88
|
|
|
89
|
+
/** The bytes a set of records occupies: distinct blobs, not records — two
|
|
90
|
+
* names for identical bytes are one object. A size that is not a positive
|
|
91
|
+
* number counts as nothing: indexes written before 30 Oct 2026 hold sizes a
|
|
92
|
+
* browser claimed, and a negative one used to lower the total. */
|
|
93
|
+
function bytesOf(files: Pick<FileRecord, "sha" | "size">[]): number {
|
|
94
|
+
const seen = new Map<string, number>();
|
|
95
|
+
for (const f of files) seen.set(f.sha, Number.isFinite(f.size) && f.size > 0 ? f.size : 0);
|
|
96
|
+
return [...seen.values()].reduce((a, b) => a + b, 0);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Refuse a workspace that is not there, as a 404 (30 Oct 2026).
|
|
101
|
+
*
|
|
102
|
+
* The store is keyed by workspace *name* and used to be created for any safe
|
|
103
|
+
* name on first write, so an editor got a fresh quota for every name they
|
|
104
|
+
* invented — stores no workspace list shows and no delete reaches. Every
|
|
105
|
+
* write here, and the routes that read, ask this first.
|
|
106
|
+
*/
|
|
107
|
+
export function requireWorkspace(tenant: string, workspace: string): void {
|
|
108
|
+
if (!fs.existsSync(workspaceDir(tenant, workspace))) {
|
|
109
|
+
throw Object.assign(new Error(`workspace ${workspace} not found`), { status: 404 });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
88
113
|
// ---------- paths ----------
|
|
89
114
|
|
|
90
115
|
/**
|
|
@@ -212,9 +237,7 @@ export function getFile(tenant: string, workspace: string, rel: string): FileRec
|
|
|
212
237
|
export function usedBytes(tenant: string, workspace: string): number {
|
|
213
238
|
// Distinct blobs, not records: two names for identical bytes occupy one
|
|
214
239
|
// object, and billing the tenant twice for it would be a lie.
|
|
215
|
-
|
|
216
|
-
for (const f of readIndex(tenant, workspace).files) seen.set(f.sha, f.size);
|
|
217
|
-
return [...seen.values()].reduce((a, b) => a + b, 0);
|
|
240
|
+
return bytesOf(readIndex(tenant, workspace).files);
|
|
218
241
|
}
|
|
219
242
|
|
|
220
243
|
// ---------- the driver interface ----------
|
|
@@ -224,6 +247,15 @@ export interface Driver {
|
|
|
224
247
|
put(key: string, body: Buffer, mime: string): Promise<void>;
|
|
225
248
|
get(key: string): Promise<Buffer>;
|
|
226
249
|
has(key: string): Promise<boolean>;
|
|
250
|
+
/** The object's real length, or null when there is none — what the store
|
|
251
|
+
* holds, never what a browser said it sent (30 Oct 2026). */
|
|
252
|
+
size(key: string): Promise<number | null>;
|
|
253
|
+
/** The bytes as a stream, optionally one inclusive byte range — so a route
|
|
254
|
+
* hands a file on without holding all of it in memory. */
|
|
255
|
+
open(key: string, range?: { start: number; end: number }): Promise<ReadableStream<Uint8Array>>;
|
|
256
|
+
/** Server-side copy to another key (s3 only — the fs driver's blobs move
|
|
257
|
+
* with their directory). Used when a workspace is renamed. */
|
|
258
|
+
copy?(from: string, to: string): Promise<void>;
|
|
227
259
|
remove(key: string): Promise<void>;
|
|
228
260
|
/** A URL a browser can GET directly, or null when the driver has no such
|
|
229
261
|
* thing (fs) and the route must stream the bytes itself. */
|
|
@@ -232,7 +264,9 @@ export interface Driver {
|
|
|
232
264
|
presignGet(key: string, filename: string, mime: string, ttlSec?: number, inlineType?: string): Promise<string | null>;
|
|
233
265
|
/** A URL a browser can PUT directly to, so upload bytes never transit the
|
|
234
266
|
* dashboard process. Null on fs, where the route takes the body. */
|
|
235
|
-
|
|
267
|
+
/** `size`: sign the length in, so the store refuses a PUT of any other
|
|
268
|
+
* length (30 Oct 2026) — the per-file limit then holds at the bucket. */
|
|
269
|
+
presignPut(key: string, mime: string, ttlSec?: number, size?: number): Promise<string | null>;
|
|
236
270
|
}
|
|
237
271
|
|
|
238
272
|
// ---------- fs driver ----------
|
|
@@ -264,6 +298,18 @@ function fsDriver(tenant: string, workspace: string): Driver {
|
|
|
264
298
|
async has(key) {
|
|
265
299
|
return fs.existsSync(at(key));
|
|
266
300
|
},
|
|
301
|
+
async size(key) {
|
|
302
|
+
try {
|
|
303
|
+
const st = fs.statSync(at(key));
|
|
304
|
+
return st.isFile() ? st.size : null;
|
|
305
|
+
} catch {
|
|
306
|
+
return null;
|
|
307
|
+
}
|
|
308
|
+
},
|
|
309
|
+
async open(key, range) {
|
|
310
|
+
const stream = fs.createReadStream(at(key), range ? { start: range.start, end: range.end } : {});
|
|
311
|
+
return Readable.toWeb(stream) as unknown as ReadableStream<Uint8Array>;
|
|
312
|
+
},
|
|
267
313
|
async remove(key) {
|
|
268
314
|
fs.rmSync(at(key), { force: true });
|
|
269
315
|
},
|
|
@@ -435,10 +481,17 @@ function presign(
|
|
|
435
481
|
ttlSec: number,
|
|
436
482
|
extraQuery: Record<string, string> = {},
|
|
437
483
|
now = new Date(),
|
|
484
|
+
extraHeaders: Record<string, string> = {},
|
|
438
485
|
): string {
|
|
439
486
|
const { host, origin, canonicalUri } = target(cfg, key);
|
|
440
487
|
const { amzDate, dateStamp } = stamps(now);
|
|
441
488
|
const scope = `${dateStamp}/${cfg.region}/s3/aws4_request`;
|
|
489
|
+
// Headers the request must carry exactly as signed. `content-length` on an
|
|
490
|
+
// upload is the one that matters: the store then refuses a body of any
|
|
491
|
+
// other length, so a presigned PUT cannot be used to park 5 GB under a key
|
|
492
|
+
// that was asked for as 2 MB (30 Oct 2026).
|
|
493
|
+
const headers: Record<string, string> = { host, ...Object.fromEntries(Object.entries(extraHeaders).map(([k, v]) => [k.toLowerCase(), v])) };
|
|
494
|
+
const signed = Object.keys(headers).sort();
|
|
442
495
|
|
|
443
496
|
const query: Record<string, string> = {
|
|
444
497
|
...extraQuery,
|
|
@@ -446,7 +499,7 @@ function presign(
|
|
|
446
499
|
"X-Amz-Credential": `${creds.accessKeyId}/${scope}`,
|
|
447
500
|
"X-Amz-Date": amzDate,
|
|
448
501
|
"X-Amz-Expires": String(ttlSec),
|
|
449
|
-
"X-Amz-SignedHeaders": "
|
|
502
|
+
"X-Amz-SignedHeaders": signed.join(";"),
|
|
450
503
|
// A role's credentials are only valid with their token, and the token
|
|
451
504
|
// is part of what is signed.
|
|
452
505
|
...(creds.sessionToken ? { "X-Amz-Security-Token": creds.sessionToken } : {}),
|
|
@@ -460,8 +513,8 @@ function presign(
|
|
|
460
513
|
method,
|
|
461
514
|
canonicalUri,
|
|
462
515
|
canonicalQuery,
|
|
463
|
-
|
|
464
|
-
"
|
|
516
|
+
signed.map((h) => `${h}:${headers[h].trim()}\n`).join(""),
|
|
517
|
+
signed.join(";"),
|
|
465
518
|
"UNSIGNED-PAYLOAD",
|
|
466
519
|
].join("\n");
|
|
467
520
|
|
|
@@ -483,6 +536,7 @@ async function s3Request(
|
|
|
483
536
|
key: string,
|
|
484
537
|
body?: Buffer,
|
|
485
538
|
mime?: string,
|
|
539
|
+
extra: Record<string, string> = {},
|
|
486
540
|
): Promise<Response> {
|
|
487
541
|
const creds = await credentials(cfg);
|
|
488
542
|
const { host, origin, canonicalUri } = target(cfg, key);
|
|
@@ -496,6 +550,9 @@ async function s3Request(
|
|
|
496
550
|
"x-amz-date": amzDate,
|
|
497
551
|
...(creds.sessionToken ? { "x-amz-security-token": creds.sessionToken } : {}),
|
|
498
552
|
...(mime ? { "content-type": mime } : {}),
|
|
553
|
+
// range on a streamed read, x-amz-copy-source on a copy. Lower-case,
|
|
554
|
+
// because the canonical form is.
|
|
555
|
+
...Object.fromEntries(Object.entries(extra).map(([k, v]) => [k.toLowerCase(), v])),
|
|
499
556
|
};
|
|
500
557
|
const signed = Object.keys(headers).sort();
|
|
501
558
|
const canonicalHeaders = signed.map((h) => `${h}:${headers[h].trim()}\n`).join("");
|
|
@@ -545,6 +602,27 @@ function s3Driver(cfg: S3Config): Driver {
|
|
|
545
602
|
const res = await s3Request(cfg, "HEAD", key);
|
|
546
603
|
return res.ok;
|
|
547
604
|
},
|
|
605
|
+
async size(key) {
|
|
606
|
+
const res = await s3Request(cfg, "HEAD", key);
|
|
607
|
+
if (res.status === 404) return null;
|
|
608
|
+
if (!res.ok) await fail(res, "head");
|
|
609
|
+
const n = Number(res.headers.get("content-length"));
|
|
610
|
+
return Number.isSafeInteger(n) && n >= 0 ? n : null;
|
|
611
|
+
},
|
|
612
|
+
async open(key, range) {
|
|
613
|
+
const res = await s3Request(cfg, "GET", key, undefined, undefined, range ? { range: `bytes=${range.start}-${range.end}` } : {});
|
|
614
|
+
if (!res.ok || !res.body) await fail(res, "get");
|
|
615
|
+
return res.body as ReadableStream<Uint8Array>;
|
|
616
|
+
},
|
|
617
|
+
async copy(from, to) {
|
|
618
|
+
// CopyObject: the bytes move inside the store, never through here.
|
|
619
|
+
const res = await s3Request(cfg, "PUT", to, undefined, undefined, {
|
|
620
|
+
"x-amz-copy-source": `/${cfg.bucket}/${uriEncode(from, true)}`,
|
|
621
|
+
});
|
|
622
|
+
// S3 can answer 200 with an <Error> body when a copy fails part-way.
|
|
623
|
+
const text = res.ok ? await res.text() : "";
|
|
624
|
+
if (!res.ok || /<Error>/.test(text)) throw new Error(`s3 copy failed: ${res.status} ${text.slice(0, 300)}`);
|
|
625
|
+
},
|
|
548
626
|
async remove(key) {
|
|
549
627
|
const res = await s3Request(cfg, "DELETE", key);
|
|
550
628
|
// 404 on delete is the desired end state, not an error.
|
|
@@ -560,8 +638,8 @@ function s3Driver(cfg: S3Config): Driver {
|
|
|
560
638
|
"response-content-type": safe ?? mime,
|
|
561
639
|
});
|
|
562
640
|
},
|
|
563
|
-
async presignPut(key, _mime, ttlSec = 600) {
|
|
564
|
-
return presign(cfg, await credentials(cfg), "PUT", key, ttlSec);
|
|
641
|
+
async presignPut(key, _mime, ttlSec = 600, size) {
|
|
642
|
+
return presign(cfg, await credentials(cfg), "PUT", key, ttlSec, {}, new Date(), size === undefined ? {} : { "content-length": String(size) });
|
|
565
643
|
},
|
|
566
644
|
};
|
|
567
645
|
}
|
|
@@ -619,6 +697,9 @@ export async function putFile(
|
|
|
619
697
|
body: Buffer,
|
|
620
698
|
by: string,
|
|
621
699
|
): Promise<FileRecord> {
|
|
700
|
+
// A person's upload. A run's harvest writes through putFileQuietly, into a
|
|
701
|
+
// workspace that was there when the run began.
|
|
702
|
+
requireWorkspace(tenant, workspace);
|
|
622
703
|
const record = await putFileQuietly(tenant, workspace, rel, body, by);
|
|
623
704
|
// A run's copy-back announces once, for all its files, from harvestFiles.
|
|
624
705
|
if (!by.startsWith("run:")) await announceFiles(tenant, workspace, [record.path], by);
|
|
@@ -666,20 +747,29 @@ async function putFileQuietly(
|
|
|
666
747
|
}
|
|
667
748
|
|
|
668
749
|
/** Register bytes a browser PUT straight to the bucket. Same bookkeeping as
|
|
669
|
-
* putFile, minus the bytes — the platform never saw them.
|
|
750
|
+
* putFile, minus the bytes — the platform never saw them.
|
|
751
|
+
*
|
|
752
|
+
* `_claimed` is the size the browser says it sent, kept for the callers that
|
|
753
|
+
* pass it and never believed (30 Oct 2026): the record, the per-file limit
|
|
754
|
+
* and the quota all use the length the store reports. It used to store the
|
|
755
|
+
* claim, so an object registered as 0 bytes (or -1e12) sat outside every
|
|
756
|
+
* limit and lowered the quota line for everything after it. */
|
|
670
757
|
export async function registerUpload(
|
|
671
758
|
tenant: string,
|
|
672
759
|
workspace: string,
|
|
673
760
|
rel: string,
|
|
674
761
|
sha: string,
|
|
675
|
-
|
|
762
|
+
_claimed: number,
|
|
676
763
|
by: string,
|
|
677
764
|
): Promise<FileRecord> {
|
|
678
765
|
const norm = assertFilePath(rel);
|
|
679
766
|
if (!/^[0-9a-f]{64}$/.test(sha)) throw new Error("expected a sha256");
|
|
767
|
+
requireWorkspace(tenant, workspace);
|
|
680
768
|
const driver = driverFor(tenant, workspace);
|
|
681
|
-
|
|
682
|
-
|
|
769
|
+
const key = blobKey(tenant, workspace, sha);
|
|
770
|
+
// Trust nothing the browser said: not that the PUT landed, not its size.
|
|
771
|
+
const size = await driver.size(key);
|
|
772
|
+
if (size === null) {
|
|
683
773
|
throw new Error("no object at that key — the upload did not complete");
|
|
684
774
|
}
|
|
685
775
|
const index = readIndex(tenant, workspace);
|
|
@@ -692,10 +782,21 @@ export async function registerUpload(
|
|
|
692
782
|
updatedAt: new Date().toISOString(),
|
|
693
783
|
by,
|
|
694
784
|
};
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
785
|
+
const files = [...index.files.filter((f) => f.path !== norm), record];
|
|
786
|
+
// putFile's two limits, on the real length. A refused object is removed:
|
|
787
|
+
// nothing names it, so it would sit in the bucket unreachable and billed —
|
|
788
|
+
// unless a record already names these bytes, which are then not ours to drop.
|
|
789
|
+
const refuse = async (message: string): Promise<never> => {
|
|
790
|
+
if (!index.files.some((f) => f.sha === sha)) await driver.remove(key).catch(() => {});
|
|
791
|
+
throw new Error(message);
|
|
792
|
+
};
|
|
793
|
+
if (size > maxBytes()) {
|
|
794
|
+
await refuse(`${norm} is ${Math.round(size / 1e6)}MB — the limit is ${maxBytes() / 1e6}MB`);
|
|
795
|
+
}
|
|
796
|
+
if (bytesOf(files) > quotaBytes() && bytesOf(files) > bytesOf(index.files)) {
|
|
797
|
+
await refuse(`workspace file quota exceeded (${quotaBytes() / 1e6}MB)`);
|
|
798
|
+
}
|
|
799
|
+
writeIndex(tenant, workspace, { version: 1, files });
|
|
699
800
|
if (previous && previous.sha !== sha) await collect(tenant, workspace, previous.sha);
|
|
700
801
|
await announceFiles(tenant, workspace, [record.path], by);
|
|
701
802
|
return record;
|
|
@@ -711,6 +812,36 @@ export async function readFileBytes(
|
|
|
711
812
|
return driverFor(tenant, workspace).get(blobKey(tenant, workspace, record.sha));
|
|
712
813
|
}
|
|
713
814
|
|
|
815
|
+
/** A stored file's real length, from the store — a stat or a HEAD, never a
|
|
816
|
+
* read. Null when there is no such record or no object behind it. */
|
|
817
|
+
export async function storedSize(tenant: string, workspace: string, rel: string): Promise<number | null> {
|
|
818
|
+
const record = getFile(tenant, workspace, rel);
|
|
819
|
+
if (!record) return null;
|
|
820
|
+
return driverFor(tenant, workspace).size(blobKey(tenant, workspace, record.sha));
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* A stored file as a stream, with its real length — for a route that passes
|
|
825
|
+
* the bytes on (a download, a public link) rather than reading them whole
|
|
826
|
+
* into memory. Null when there is no such record or no object behind it.
|
|
827
|
+
* `range` is inclusive and must lie inside the file (storedSize first, then
|
|
828
|
+
* byteRange, when the caller honours a Range header).
|
|
829
|
+
*/
|
|
830
|
+
export async function openFile(
|
|
831
|
+
tenant: string,
|
|
832
|
+
workspace: string,
|
|
833
|
+
rel: string,
|
|
834
|
+
range?: { start: number; end: number } | null,
|
|
835
|
+
): Promise<{ record: FileRecord; size: number; body: ReadableStream<Uint8Array> } | null> {
|
|
836
|
+
const record = getFile(tenant, workspace, rel);
|
|
837
|
+
if (!record) return null;
|
|
838
|
+
const driver = driverFor(tenant, workspace);
|
|
839
|
+
const key = blobKey(tenant, workspace, record.sha);
|
|
840
|
+
const size = await driver.size(key);
|
|
841
|
+
if (size === null) return null;
|
|
842
|
+
return { record, size, body: await driver.open(key, range ?? undefined) };
|
|
843
|
+
}
|
|
844
|
+
|
|
714
845
|
export async function deleteFile(tenant: string, workspace: string, rel: string): Promise<boolean> {
|
|
715
846
|
const norm = assertFilePath(rel);
|
|
716
847
|
const index = readIndex(tenant, workspace);
|
|
@@ -779,10 +910,83 @@ export async function uploadUrl(
|
|
|
779
910
|
workspace: string,
|
|
780
911
|
rel: string,
|
|
781
912
|
sha: string,
|
|
913
|
+
size?: number,
|
|
782
914
|
): Promise<string | null> {
|
|
783
|
-
assertFilePath(rel);
|
|
915
|
+
const norm = assertFilePath(rel);
|
|
784
916
|
if (!/^[0-9a-f]{64}$/.test(sha)) throw new Error("expected a sha256");
|
|
785
|
-
|
|
917
|
+
requireWorkspace(tenant, workspace);
|
|
918
|
+
// The limits, before a byte moves, and the length signed into the URL so
|
|
919
|
+
// the bucket refuses any other (30 Oct 2026). registerUpload checks the
|
|
920
|
+
// real object again; this is what keeps an oversized one from ever landing
|
|
921
|
+
// — an object that is PUT and never registered was otherwise unbounded.
|
|
922
|
+
if (size !== undefined) {
|
|
923
|
+
if (!Number.isSafeInteger(size) || size <= 0) throw new Error("size must be the file's length in bytes, a whole number above 0");
|
|
924
|
+
if (size > maxBytes()) throw new Error(`${norm} is ${Math.round(size / 1e6)}MB — the limit is ${maxBytes() / 1e6}MB`);
|
|
925
|
+
const files = readIndex(tenant, workspace).files;
|
|
926
|
+
const next = [...files.filter((f) => f.path !== norm), { sha, size }];
|
|
927
|
+
if (bytesOf(next) > quotaBytes() && bytesOf(next) > bytesOf(files)) {
|
|
928
|
+
throw new Error(`workspace file quota exceeded (${quotaBytes() / 1e6}MB)`);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
return driverFor(tenant, workspace).presignPut(blobKey(tenant, workspace, sha), mimeFor(norm), undefined, size);
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
// ---------- a workspace deleted or renamed ----------
|
|
935
|
+
|
|
936
|
+
/** The store's directory for a name, without the workspace check —
|
|
937
|
+
* for the two calls below, which run when the folder is gone or moving. */
|
|
938
|
+
function storeDirOf(tenant: string, workspace: string): string | null {
|
|
939
|
+
if (!fileStoreEnabled()) return null;
|
|
940
|
+
return storeDir(tenant, workspace);
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* Drop everything a workspace stored: every object its index names, then the
|
|
945
|
+
* index (30 Oct 2026). Called when a workspace is deleted — the store is kept
|
|
946
|
+
* by name beside the folder, so it used to outlive it: the files stayed
|
|
947
|
+
* downloadable under the old name, public links kept serving them, and a new
|
|
948
|
+
* workspace given the same name inherited them. Best effort per object (a
|
|
949
|
+
* leaked object costs storage; the index going is what makes them
|
|
950
|
+
* unreachable), and a no-op when there is no store.
|
|
951
|
+
*/
|
|
952
|
+
export async function purgeWorkspaceStorage(tenant: string, workspace: string): Promise<void> {
|
|
953
|
+
const dir = storeDirOf(tenant, workspace);
|
|
954
|
+
if (!dir || !fs.existsSync(dir)) return;
|
|
955
|
+
const driver = driverFor(tenant, workspace);
|
|
956
|
+
if (driver.kind !== "fs") {
|
|
957
|
+
const shas = new Set(readIndex(tenant, workspace).files.map((f) => f.sha));
|
|
958
|
+
for (const sha of shas) await driver.remove(blobKey(tenant, workspace, sha)).catch(() => {});
|
|
959
|
+
}
|
|
960
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
/**
|
|
964
|
+
* Move a workspace's stored files to its new name (30 Oct 2026). A rename
|
|
965
|
+
* used to move the folder only: the renamed workspace lost its files, and
|
|
966
|
+
* the old name kept them for whoever took it next.
|
|
967
|
+
*
|
|
968
|
+
* On fs the blobs live under the store's directory, so moving the directory
|
|
969
|
+
* is the whole move. On s3 an object's key carries the workspace name, so
|
|
970
|
+
* every object is copied first; only once all copies landed does the index
|
|
971
|
+
* move and the old objects go. A copy that fails leaves the store under the
|
|
972
|
+
* old name, whole, and throws — the caller can put the folder back.
|
|
973
|
+
*
|
|
974
|
+
* Whatever a store under the new name held is dropped first: it belonged to a
|
|
975
|
+
* workspace of that name that no longer exists, never to this one.
|
|
976
|
+
*/
|
|
977
|
+
export async function moveWorkspaceStorage(tenant: string, from: string, to: string): Promise<void> {
|
|
978
|
+
const src = storeDirOf(tenant, from);
|
|
979
|
+
const dest = storeDirOf(tenant, to);
|
|
980
|
+
if (!src || !dest || src === dest) return;
|
|
981
|
+
if (fs.existsSync(dest)) await purgeWorkspaceStorage(tenant, to);
|
|
982
|
+
if (!fs.existsSync(src)) return;
|
|
983
|
+
const driver = driverFor(tenant, from);
|
|
984
|
+
const shas = driver.kind === "fs" ? [] : [...new Set(readIndex(tenant, from).files.map((f) => f.sha))];
|
|
985
|
+
if (shas.length && !driver.copy) throw new Error(`the ${driver.kind} store cannot move files to a new workspace name`);
|
|
986
|
+
for (const sha of shas) await driver.copy!(blobKey(tenant, from, sha), blobKey(tenant, to, sha));
|
|
987
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
988
|
+
fs.renameSync(src, dest);
|
|
989
|
+
for (const sha of shas) await driver.remove(blobKey(tenant, from, sha)).catch(() => {});
|
|
786
990
|
}
|
|
787
991
|
|
|
788
992
|
// ---------- the run mirror ----------
|