@foldrun/core 5.0.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/evals.js +11 -2
- package/dist/src/model-env.d.ts +12 -3
- package/dist/src/model-env.js +44 -6
- package/dist/src/platform.d.ts +24 -0
- package/dist/src/platform.js +35 -1
- package/dist/src/runner.d.ts +2 -0
- package/dist/src/secrets.d.ts +22 -0
- package/dist/src/secrets.js +174 -25
- package/dist/src/translator.d.ts +8 -2
- package/dist/src/translator.js +73 -0
- package/dist/src/webhook.js +46 -6
- package/package.json +1 -1
- package/src/evals.ts +9 -2
- package/src/model-env.ts +42 -7
- package/src/platform.ts +45 -1
- package/src/secrets.ts +189 -26
- package/src/translator.ts +83 -2
- package/src/webhook.ts +44 -5
package/src/secrets.ts
CHANGED
|
@@ -63,6 +63,12 @@ export type SecretScope = "account" | "workspace";
|
|
|
63
63
|
export function installKeyMaterial(): string {
|
|
64
64
|
const fromEnv = process.env.FOLDRUN_SECRET_KEY;
|
|
65
65
|
if (fromEnv) return fromEnv;
|
|
66
|
+
// A process that must never hold the install key (the egress proxy) says
|
|
67
|
+
// so; without this, asking for it would quietly mint a NEW random key on
|
|
68
|
+
// the shared data volume — a second root key nobody chose.
|
|
69
|
+
if (process.env.FOLDRUN_INSTALL_KEY_ABSENT === "1") {
|
|
70
|
+
throw new Error("this process runs without the install key (FOLDRUN_INSTALL_KEY_ABSENT=1) — whatever asked for it belongs in the worker");
|
|
71
|
+
}
|
|
66
72
|
fs.mkdirSync(dataRoot(), { recursive: true });
|
|
67
73
|
if (!fs.existsSync(keyFile())) {
|
|
68
74
|
fs.writeFileSync(keyFile(), crypto.randomBytes(32).toString("hex"), { mode: 0o600 });
|
|
@@ -105,10 +111,100 @@ interface StoredSecret {
|
|
|
105
111
|
* these sites alone. Metadata, not a value, so it is stored in the clear
|
|
106
112
|
* beside the ciphertext and survives the value being set again. */
|
|
107
113
|
sites?: string[];
|
|
114
|
+
/** 2: sealed with the record's place as AAD — account, scope, name, kind,
|
|
115
|
+
* key and sites (vaultAad). Absent: the older, unbound form. */
|
|
116
|
+
v?: 2;
|
|
108
117
|
}
|
|
109
118
|
|
|
110
119
|
type SecretsFile = Record<string, StoredSecret>;
|
|
111
120
|
|
|
121
|
+
// ------------------------------------------------------------ binding
|
|
122
|
+
//
|
|
123
|
+
// A v2 record is sealed with WHERE it belongs as additional authenticated
|
|
124
|
+
// data: the account, the scope (account or which workspace), the name, its
|
|
125
|
+
// kind, which key sealed it, and the sites it is bound to. Before 29 Oct 2026
|
|
126
|
+
// (OWASP review) a record was bound to nothing: anyone who could write the
|
|
127
|
+
// data volume or a backup could move one secret's ciphertext under another
|
|
128
|
+
// name, scope or account, or rewrite its clear `sites` and have the egress
|
|
129
|
+
// proxy fill it for their own host. Now any of those makes it fail to open,
|
|
130
|
+
// which reads as "not set" — never as unbound.
|
|
131
|
+
//
|
|
132
|
+
// The place comes from the FILE the record is read from, never from fields
|
|
133
|
+
// inside it. Sites stay in clear beside the ciphertext (listing needs no
|
|
134
|
+
// key, and a pod from before v2 reads them there) and are authenticated by
|
|
135
|
+
// the AAD.
|
|
136
|
+
//
|
|
137
|
+
// Rolled out in steps: every release since reads v2; FOLDRUN_VAULT_WRITE_BOUND=1
|
|
138
|
+
// writes it and migrates existing records at boot; FOLDRUN_VAULT_REQUIRE_BOUND=1
|
|
139
|
+
// refuses the old form once none is left.
|
|
140
|
+
|
|
141
|
+
// The scope is account or workspace, not WHICH workspace: a workspace can be
|
|
142
|
+
// renamed (its folder moves with its vault), and binding the name would make
|
|
143
|
+
// every rename a re-seal racing other writers. What that leaves open — a
|
|
144
|
+
// ciphertext moved between two workspaces of the same account by someone who
|
|
145
|
+
// can already write the volume — is the residual, and stays inside one account.
|
|
146
|
+
function vaultAad(tenant: string, workspace: string | undefined, name: string, rec: Pick<StoredSecret, "kind" | "k" | "sites">): Buffer {
|
|
147
|
+
if (rec.sites !== undefined && (!Array.isArray(rec.sites) || rec.sites.some((x) => typeof x !== "string"))) {
|
|
148
|
+
throw new Error("malformed sites");
|
|
149
|
+
}
|
|
150
|
+
return Buffer.from(
|
|
151
|
+
JSON.stringify(["foldrun-vault/2", tenant, workspace ? "w" : "a", name, rec.kind ?? "", rec.k ?? "install", [...(rec.sites ?? [])].sort()]),
|
|
152
|
+
"utf8",
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
const writeBound = () => process.env.FOLDRUN_VAULT_WRITE_BOUND === "1";
|
|
156
|
+
const requireBound = () => process.env.FOLDRUN_VAULT_REQUIRE_BOUND === "1";
|
|
157
|
+
|
|
158
|
+
/** One record, sealed for its place. Bound (v2) when the install writes bound
|
|
159
|
+
* records or `bound` says so; the unbound form otherwise. */
|
|
160
|
+
function sealRecord(
|
|
161
|
+
tenant: string,
|
|
162
|
+
workspace: string | undefined,
|
|
163
|
+
name: string,
|
|
164
|
+
value: string,
|
|
165
|
+
meta: { kind?: SecretKind; sites?: string[]; updatedAt?: string },
|
|
166
|
+
bound = writeBound(),
|
|
167
|
+
): StoredSecret {
|
|
168
|
+
const { key, mark } = keyFor(tenant);
|
|
169
|
+
const rec: StoredSecret = {
|
|
170
|
+
iv: "",
|
|
171
|
+
tag: "",
|
|
172
|
+
data: "",
|
|
173
|
+
updatedAt: meta.updatedAt ?? new Date().toISOString(),
|
|
174
|
+
...(meta.kind ? { kind: meta.kind } : {}),
|
|
175
|
+
...(mark ? { k: mark } : {}),
|
|
176
|
+
...(meta.sites?.length ? { sites: [...meta.sites] } : {}),
|
|
177
|
+
};
|
|
178
|
+
const iv = crypto.randomBytes(12);
|
|
179
|
+
const cipher = crypto.createCipheriv("aes-256-gcm", key, iv, GCM);
|
|
180
|
+
if (bound) {
|
|
181
|
+
cipher.setAAD(vaultAad(tenant, workspace, name, rec));
|
|
182
|
+
rec.v = 2;
|
|
183
|
+
}
|
|
184
|
+
const enc = Buffer.concat([cipher.update(value, "utf8"), cipher.final()]);
|
|
185
|
+
rec.iv = iv.toString("base64");
|
|
186
|
+
rec.tag = cipher.getAuthTag().toString("base64");
|
|
187
|
+
rec.data = enc.toString("base64");
|
|
188
|
+
return rec;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** A record's value, opened for its place; null when it will not open. */
|
|
192
|
+
function openRecord(tenant: string, workspace: string | undefined, name: string, rec: StoredSecret | undefined): string | null {
|
|
193
|
+
if (!rec) return null;
|
|
194
|
+
if (rec.v === 2) {
|
|
195
|
+
// Metadata someone rewrote into a shape that is not a list fails closed,
|
|
196
|
+
// like a bad tag — never as an exception up the read path.
|
|
197
|
+
let aad: Buffer;
|
|
198
|
+
try {
|
|
199
|
+
aad = vaultAad(tenant, workspace, name, rec);
|
|
200
|
+
} catch {
|
|
201
|
+
return null;
|
|
202
|
+
}
|
|
203
|
+
return decrypt(tenant, rec, aad);
|
|
204
|
+
}
|
|
205
|
+
return requireBound() ? null : decrypt(tenant, rec);
|
|
206
|
+
}
|
|
207
|
+
|
|
112
208
|
function read(tenant: string, workspace?: string): SecretsFile {
|
|
113
209
|
return readFile(secretsFile(tenant, workspace));
|
|
114
210
|
}
|
|
@@ -199,18 +295,15 @@ export function setSecret(
|
|
|
199
295
|
throw new Error(`secret name "${name}" must be UPPER_SNAKE_CASE`);
|
|
200
296
|
}
|
|
201
297
|
if (value.length > 8192) throw new Error("secret too long");
|
|
202
|
-
const sealed = encryptValue(tenant, value);
|
|
203
298
|
mutateVaultFile(secretsFile(tenant, workspace), (all) => {
|
|
204
299
|
// A new value keeps the sites the old one was bound to: binding is a
|
|
205
300
|
// property of the credential's use, not of its current value, and a
|
|
206
301
|
// rotation that silently unbound it would put the new value in pods.
|
|
302
|
+
// Sealed here, under the lock: a bound record's AAD names those sites.
|
|
207
303
|
const sites = all[name]?.sites;
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
...(kind ? { kind } : {}),
|
|
212
|
-
...(sites?.length ? { sites } : {}),
|
|
213
|
-
};
|
|
304
|
+
// Once bound, always bound: a re-save with the switch off must not
|
|
305
|
+
// quietly drop the record's authentication.
|
|
306
|
+
all[name] = sealRecord(tenant, workspace, name, value, { kind, sites }, all[name]?.v === 2 || writeBound());
|
|
214
307
|
return all;
|
|
215
308
|
});
|
|
216
309
|
}
|
|
@@ -241,6 +334,15 @@ export function setSecretSites(tenant: string, name: string, sites: string[], wo
|
|
|
241
334
|
if (rec.kind === "client-cert" && clean.some((s) => s.startsWith("*."))) {
|
|
242
335
|
throw new Error(`${name} is a client certificate — bind it to exact hosts, not a wildcard`);
|
|
243
336
|
}
|
|
337
|
+
// A bound record names its sites in its seal: open it with the old
|
|
338
|
+
// binding and seal it again with the new one. One that will not open is
|
|
339
|
+
// refused, never relabelled.
|
|
340
|
+
if (rec.v === 2) {
|
|
341
|
+
const value = openRecord(tenant, workspace, name, rec);
|
|
342
|
+
if (value === null) throw new Error(`${name} cannot be read with this install's keys — set its value again before binding it`);
|
|
343
|
+
all[name] = sealRecord(tenant, workspace, name, value, { kind: rec.kind, sites: clean, updatedAt: rec.updatedAt }, true);
|
|
344
|
+
return all;
|
|
345
|
+
}
|
|
244
346
|
if (clean.length) rec.sites = clean;
|
|
245
347
|
else delete rec.sites;
|
|
246
348
|
return all;
|
|
@@ -259,7 +361,11 @@ export function secretSites(tenant: string, names: string[], workspace?: string)
|
|
|
259
361
|
const own = workspace ? read(tenant, workspace) : {};
|
|
260
362
|
const account = read(tenant);
|
|
261
363
|
for (const name of names) {
|
|
262
|
-
|
|
364
|
+
// The record getSecret would take the VALUE from — the workspace's only
|
|
365
|
+
// when it opens, else the account's. Picking by presence paired an
|
|
366
|
+
// account value with a workspace record's (unauthenticated) sites.
|
|
367
|
+
const ownRec = own[name];
|
|
368
|
+
const rec = ownRec && openRecord(tenant, workspace, name, ownRec) !== null ? ownRec : account[name];
|
|
263
369
|
if (rec?.sites?.length) out[name] = [...rec.sites];
|
|
264
370
|
}
|
|
265
371
|
return out;
|
|
@@ -303,8 +409,10 @@ export function decryptValue(
|
|
|
303
409
|
return decrypt(tenant, rec as StoredSecret | undefined);
|
|
304
410
|
}
|
|
305
411
|
|
|
306
|
-
function decrypt(tenant: string, rec: StoredSecret | undefined): string | null {
|
|
412
|
+
function decrypt(tenant: string, rec: StoredSecret | undefined, aad?: Buffer): string | null {
|
|
307
413
|
if (!rec) return null;
|
|
414
|
+
// A bound record opens only with its binding: one without is not a guess.
|
|
415
|
+
if (rec.v === 2 && !aad) return null;
|
|
308
416
|
try {
|
|
309
417
|
// The record says which key wrote it. An unmarked one predates per-account
|
|
310
418
|
// keys and is under the install key; guessing either way would turn a
|
|
@@ -334,6 +442,7 @@ function decrypt(tenant: string, rec: StoredSecret | undefined): string | null {
|
|
|
334
442
|
GCM,
|
|
335
443
|
);
|
|
336
444
|
decipher.setAuthTag(Buffer.from(rec.tag, "base64"));
|
|
445
|
+
if (aad) decipher.setAAD(aad);
|
|
337
446
|
return Buffer.concat([
|
|
338
447
|
decipher.update(Buffer.from(rec.data, "base64")),
|
|
339
448
|
decipher.final(),
|
|
@@ -350,10 +459,10 @@ export function getSecret(
|
|
|
350
459
|
workspace?: string,
|
|
351
460
|
): { value: string; scope: SecretScope } | null {
|
|
352
461
|
if (workspace) {
|
|
353
|
-
const own =
|
|
462
|
+
const own = openRecord(tenant, workspace, name, read(tenant, workspace)[name]);
|
|
354
463
|
if (own !== null) return { value: own, scope: "workspace" };
|
|
355
464
|
}
|
|
356
|
-
const shared =
|
|
465
|
+
const shared = openRecord(tenant, undefined, name, read(tenant)[name]);
|
|
357
466
|
return shared === null ? null : { value: shared, scope: "account" };
|
|
358
467
|
}
|
|
359
468
|
|
|
@@ -490,7 +599,7 @@ export function rotateMasterKey(
|
|
|
490
599
|
const rewritten: RotationResult["rewritten"] = [];
|
|
491
600
|
const unreadable: string[] = [];
|
|
492
601
|
|
|
493
|
-
for (const { file } of secretFiles(tenant)) {
|
|
602
|
+
for (const { file, workspace } of secretFiles(tenant)) {
|
|
494
603
|
let count = 0;
|
|
495
604
|
mutateVaultFile(file, (all) => {
|
|
496
605
|
const next: SecretsFile = {};
|
|
@@ -505,10 +614,14 @@ export function rotateMasterKey(
|
|
|
505
614
|
next[name] = rec;
|
|
506
615
|
continue;
|
|
507
616
|
}
|
|
617
|
+
// A bound record keeps its binding: same place, same AAD.
|
|
618
|
+
let aad: Buffer | null = null;
|
|
508
619
|
let plain: string;
|
|
509
620
|
try {
|
|
621
|
+
if (rec.v === 2) aad = vaultAad(tenant, workspace, name, rec);
|
|
510
622
|
const d = crypto.createDecipheriv("aes-256-gcm", oldKey, Buffer.from(rec.iv, "base64"), GCM);
|
|
511
623
|
d.setAuthTag(Buffer.from(rec.tag, "base64"));
|
|
624
|
+
if (aad) d.setAAD(aad);
|
|
512
625
|
plain = Buffer.concat([d.update(Buffer.from(rec.data, "base64")), d.final()]).toString("utf8");
|
|
513
626
|
} catch {
|
|
514
627
|
unreadable.push(name);
|
|
@@ -517,6 +630,7 @@ export function rotateMasterKey(
|
|
|
517
630
|
}
|
|
518
631
|
const iv = crypto.randomBytes(12);
|
|
519
632
|
const c = crypto.createCipheriv("aes-256-gcm", newKey, iv, GCM);
|
|
633
|
+
if (aad) c.setAAD(aad);
|
|
520
634
|
const enc = Buffer.concat([c.update(plain, "utf8"), c.final()]);
|
|
521
635
|
next[name] = {
|
|
522
636
|
iv: iv.toString("base64"),
|
|
@@ -526,6 +640,10 @@ export function rotateMasterKey(
|
|
|
526
640
|
// Kept: without it an oauth2 secret stops being refreshed after a
|
|
527
641
|
// rotation, and a file/ssh/api secret stops being materialised.
|
|
528
642
|
...(rec.kind ? { kind: rec.kind } : {}),
|
|
643
|
+
// Kept too (it was dropped until 29 Oct 2026): a rotation that
|
|
644
|
+
// unbinds every bound secret puts their values into pods.
|
|
645
|
+
...(rec.sites?.length ? { sites: rec.sites } : {}),
|
|
646
|
+
...(aad ? { v: 2 as const } : {}),
|
|
529
647
|
};
|
|
530
648
|
count++;
|
|
531
649
|
}
|
|
@@ -637,8 +755,11 @@ export function replaceOAuth2RefreshToken(tenant: string, name: string, workspac
|
|
|
637
755
|
let replaced = false;
|
|
638
756
|
mutateVaultFile(secretsFile(tenant, scope), (all) => {
|
|
639
757
|
const rec = all[name];
|
|
640
|
-
if (rec &&
|
|
641
|
-
|
|
758
|
+
if (rec && openRecord(tenant, scope, name, rec) === usedValue) {
|
|
759
|
+
// Built whole, not spread over the old record: a spread kept a stale
|
|
760
|
+
// `k: "tenant"` when this seal fell back to the install key, and the
|
|
761
|
+
// record then opened with neither.
|
|
762
|
+
all[name] = sealRecord(tenant, scope, name, next, { kind: rec.kind, sites: rec.sites }, rec.v === 2 || writeBound());
|
|
642
763
|
replaced = true;
|
|
643
764
|
}
|
|
644
765
|
return all;
|
|
@@ -747,7 +868,15 @@ export async function materializeSecrets(
|
|
|
747
868
|
env: Record<string, string>,
|
|
748
869
|
/** Whose secrets these are, when the caller knows: lets the platform keep
|
|
749
870
|
* each OAuth connection's health and quote a reconnect link on failure. */
|
|
750
|
-
ctx?: {
|
|
871
|
+
ctx?: {
|
|
872
|
+
tenant: string;
|
|
873
|
+
workspace?: string;
|
|
874
|
+
/** A rotated refresh token goes here INSTEAD of the vault: a caller that
|
|
875
|
+
* cannot open the vault (the egress proxy without the install key)
|
|
876
|
+
* keeps it itself and has it written by a process that can. `next` is
|
|
877
|
+
* the stored value with the new refresh token. */
|
|
878
|
+
onRotate?: (name: string, usedValue: string, next: string) => unknown;
|
|
879
|
+
},
|
|
751
880
|
): Promise<Record<string, string>> {
|
|
752
881
|
const out: Record<string, string> = { ...env };
|
|
753
882
|
for (const [name, value] of Object.entries(env)) {
|
|
@@ -769,7 +898,8 @@ export async function materializeSecrets(
|
|
|
769
898
|
// the next run does not refresh again.
|
|
770
899
|
const next = OAUTH2_PREFIX + JSON.stringify({ ...config, refresh_token: live.rotated });
|
|
771
900
|
try {
|
|
772
|
-
if (ctx)
|
|
901
|
+
if (ctx?.onRotate) await ctx.onRotate(name, value, next);
|
|
902
|
+
else if (ctx) replaceOAuth2RefreshToken(ctx.tenant, name, ctx.workspace, value, live.rotated);
|
|
773
903
|
} catch {
|
|
774
904
|
// The access token is good either way; the next refresh says if not.
|
|
775
905
|
}
|
|
@@ -1028,7 +1158,7 @@ export function setClientCertSecret(tenant: string, name: string, config: Client
|
|
|
1028
1158
|
...(config.export ? { export: true } : {}),
|
|
1029
1159
|
};
|
|
1030
1160
|
if (!SECRET_NAME.test(name)) throw new Error(`secret name "${name}" must be UPPER_SNAKE_CASE`);
|
|
1031
|
-
const
|
|
1161
|
+
const plainCert = CLIENT_CERT_PREFIX + JSON.stringify(stored);
|
|
1032
1162
|
// Written under the vault's own lock, with the binding checked in the same
|
|
1033
1163
|
// step: a value set over a secret keeps its site binding, and a wildcard
|
|
1034
1164
|
// one would present the certificate to a whole domain.
|
|
@@ -1037,7 +1167,7 @@ export function setClientCertSecret(tenant: string, name: string, config: Client
|
|
|
1037
1167
|
if (sites?.some((site) => site.startsWith("*."))) {
|
|
1038
1168
|
throw new Error(`${name} is bound to ${sites.join(", ")} — a client certificate is bound to exact hosts; unbind the wildcard first`);
|
|
1039
1169
|
}
|
|
1040
|
-
all[name] =
|
|
1170
|
+
all[name] = sealRecord(tenant, workspace, name, plainCert, { kind: "client-cert", sites }, all[name]?.v === 2 || writeBound());
|
|
1041
1171
|
return all;
|
|
1042
1172
|
});
|
|
1043
1173
|
}
|
|
@@ -1143,6 +1273,44 @@ export function apiConfigOf(value: string): ApiConfig {
|
|
|
1143
1273
|
}
|
|
1144
1274
|
|
|
1145
1275
|
|
|
1276
|
+
/**
|
|
1277
|
+
* Seal every unbound record of an account for its place (v2). Run at boot
|
|
1278
|
+
* once the install writes bound records (FOLDRUN_VAULT_WRITE_BOUND=1), and by
|
|
1279
|
+
* scripts/seal-vault-bindings.mjs. Per file, all or nothing: every unbound
|
|
1280
|
+
* value is opened first, and a file holding one that will not open is left
|
|
1281
|
+
* exactly as it is and reported. Idempotent: bound records are skipped.
|
|
1282
|
+
*/
|
|
1283
|
+
export function sealVaultBindings(tenant: string): { sealed: number; unreadable: string[] } {
|
|
1284
|
+
assertSafeName(tenant, "tenant");
|
|
1285
|
+
let sealed = 0;
|
|
1286
|
+
const unreadable: string[] = [];
|
|
1287
|
+
for (const { file, workspace } of secretFiles(tenant)) {
|
|
1288
|
+
mutateVaultFile(file, (all) => {
|
|
1289
|
+
const plain = new Map<string, string>();
|
|
1290
|
+
const bad: string[] = [];
|
|
1291
|
+
for (const [name, rec] of Object.entries(all)) {
|
|
1292
|
+
if (rec.v === 2) continue;
|
|
1293
|
+
const value = decrypt(tenant, rec);
|
|
1294
|
+
if (value === null) bad.push(workspace ? `${workspace}/${name}` : name);
|
|
1295
|
+
else plain.set(name, value);
|
|
1296
|
+
}
|
|
1297
|
+
unreadable.push(...bad);
|
|
1298
|
+
if (plain.size === 0 || bad.length) return null;
|
|
1299
|
+
const next: SecretsFile = { ...all };
|
|
1300
|
+
for (const [name, value] of plain) {
|
|
1301
|
+
const rec = all[name];
|
|
1302
|
+
next[name] = sealRecord(tenant, workspace, name, value, { kind: rec.kind, sites: rec.sites, updatedAt: rec.updatedAt }, true);
|
|
1303
|
+
}
|
|
1304
|
+
sealed += plain.size;
|
|
1305
|
+
return next;
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
return { sealed, unreadable };
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
/** Whether this install writes bound vault records (and migrates at boot). */
|
|
1312
|
+
export const vaultWritesBound = writeBound;
|
|
1313
|
+
|
|
1146
1314
|
/**
|
|
1147
1315
|
* Move an account's secrets from the install key onto its own key.
|
|
1148
1316
|
*
|
|
@@ -1158,7 +1326,7 @@ export function rewrapTenantSecrets(tenant: string): { moved: number; unreadable
|
|
|
1158
1326
|
let moved = 0;
|
|
1159
1327
|
const unreadable: string[] = [];
|
|
1160
1328
|
|
|
1161
|
-
for (const { file } of secretFiles(tenant)) {
|
|
1329
|
+
for (const { file, workspace } of secretFiles(tenant)) {
|
|
1162
1330
|
// Under the vault's lock: every pod runs this at boot, and a save made
|
|
1163
1331
|
// between our read and our write must not be lost.
|
|
1164
1332
|
mutateVaultFile(file, (all) => {
|
|
@@ -1166,7 +1334,7 @@ export function rewrapTenantSecrets(tenant: string): { moved: number; unreadable
|
|
|
1166
1334
|
const unreadableHere: string[] = [];
|
|
1167
1335
|
for (const [name, rec] of Object.entries(all)) {
|
|
1168
1336
|
if (rec.k === "tenant") continue; // already moved
|
|
1169
|
-
const value =
|
|
1337
|
+
const value = openRecord(tenant, workspace, name, rec);
|
|
1170
1338
|
if (value === null) {
|
|
1171
1339
|
unreadableHere.push(name);
|
|
1172
1340
|
continue;
|
|
@@ -1180,12 +1348,7 @@ export function rewrapTenantSecrets(tenant: string): { moved: number; unreadable
|
|
|
1180
1348
|
if (plain.size === 0 || unreadableHere.length) return null;
|
|
1181
1349
|
const next: SecretsFile = { ...all };
|
|
1182
1350
|
for (const [name, value] of plain) {
|
|
1183
|
-
next[name] = {
|
|
1184
|
-
...encryptValue(tenant, value),
|
|
1185
|
-
updatedAt: all[name].updatedAt,
|
|
1186
|
-
...(all[name].kind ? { kind: all[name].kind } : {}),
|
|
1187
|
-
...(all[name].sites?.length ? { sites: all[name].sites } : {}),
|
|
1188
|
-
};
|
|
1351
|
+
next[name] = sealRecord(tenant, workspace, name, value, { kind: all[name].kind, sites: all[name].sites, updatedAt: all[name].updatedAt }, all[name].v === 2 || writeBound());
|
|
1189
1352
|
}
|
|
1190
1353
|
moved += plain.size;
|
|
1191
1354
|
return next;
|
package/src/translator.ts
CHANGED
|
@@ -68,7 +68,11 @@ function usageBlock(u: { input: number; cached: number; output: number } | null)
|
|
|
68
68
|
};
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
/** `anthropic` is no translation at all: the host-side passthrough for an
|
|
72
|
+
* endpoint that already speaks Messages, so the SDK on the host talks only
|
|
73
|
+
* to loopback and every byte to the tenant's URL goes through the guard
|
|
74
|
+
* (FOL-95). A step never gets it (translatorSpecFor returns null there). */
|
|
75
|
+
export type TranslatorWire = "chat" | "responses" | "anthropic";
|
|
72
76
|
|
|
73
77
|
export interface TranslatorSpec {
|
|
74
78
|
/** The provider's base, e.g. https://api.openai.com/v1 — `/chat/completions`
|
|
@@ -76,8 +80,10 @@ export interface TranslatorSpec {
|
|
|
76
80
|
upstreamBase: string;
|
|
77
81
|
/** Which OpenAI shape the endpoint speaks. Default chat. */
|
|
78
82
|
wire?: TranslatorWire;
|
|
79
|
-
/** Already resolved. Sent as `Authorization: Bearer
|
|
83
|
+
/** Already resolved. Sent as `Authorization: Bearer`, or as `x-api-key`
|
|
84
|
+
* where `upstreamAuth` says so (the anthropic wire only). */
|
|
80
85
|
upstreamKey: string;
|
|
86
|
+
upstreamAuth?: "bearer" | "x-api-key";
|
|
81
87
|
/** Extra headers the provider wants, already resolved. */
|
|
82
88
|
headers?: Record<string, string>;
|
|
83
89
|
maxTokensParam?: "max_tokens" | "max_completion_tokens";
|
|
@@ -943,6 +949,8 @@ export async function startTranslator(spec: TranslatorSpec): Promise<RunningTran
|
|
|
943
949
|
req.on("error", reject);
|
|
944
950
|
});
|
|
945
951
|
|
|
952
|
+
if (spec.wire === "anthropic") return await passthrough(spec, base, label, req, url, raw, res, log, started);
|
|
953
|
+
|
|
946
954
|
if (req.method === "POST" && url.pathname === "/v1/messages/count_tokens") {
|
|
947
955
|
// No endpoint on the other side counts for us; four characters a
|
|
948
956
|
// token is the estimate every tokenizer lands near for English.
|
|
@@ -1105,6 +1113,79 @@ export async function startTranslator(spec: TranslatorSpec): Promise<RunningTran
|
|
|
1105
1113
|
};
|
|
1106
1114
|
}
|
|
1107
1115
|
|
|
1116
|
+
/** Request headers the SDK's own call carries that the endpoint is owed:
|
|
1117
|
+
* the content type, and Anthropic's version and beta flags. Its credential
|
|
1118
|
+
* (the translator's key) is not one of them. */
|
|
1119
|
+
const PASSED_HEADERS = /^(content-type|accept|anthropic-[a-z0-9-]+)$/;
|
|
1120
|
+
/** Response headers that describe the bytes as they came off the wire —
|
|
1121
|
+
* fetch has already decoded them, so they would lie on the way back. */
|
|
1122
|
+
const HOP_HEADERS = new Set(["content-encoding", "content-length", "transfer-encoding", "connection", "keep-alive"]);
|
|
1123
|
+
|
|
1124
|
+
/**
|
|
1125
|
+
* The anthropic wire: the SDK's request, as it is, to the same path under the
|
|
1126
|
+
* tenant's base URL, with the tenant's key put on here — through
|
|
1127
|
+
* fetchUntrusted, so no private or cluster address, no redirect that takes
|
|
1128
|
+
* the key to another origin, and the answer read to UPSTREAM_MAX_BYTES. The
|
|
1129
|
+
* answer streams back as it arrives, status and headers included.
|
|
1130
|
+
*/
|
|
1131
|
+
async function passthrough(
|
|
1132
|
+
spec: TranslatorSpec,
|
|
1133
|
+
base: string,
|
|
1134
|
+
label: string,
|
|
1135
|
+
req: http.IncomingMessage,
|
|
1136
|
+
url: URL,
|
|
1137
|
+
raw: string,
|
|
1138
|
+
res: http.ServerResponse,
|
|
1139
|
+
log: string[],
|
|
1140
|
+
started: number,
|
|
1141
|
+
): Promise<void> {
|
|
1142
|
+
// Lowercase keys, so a header named twice in different case is one header,
|
|
1143
|
+
// and the provider's own headers last: one that carries its credential
|
|
1144
|
+
// itself (an Authorization or x-api-key in `headers:`) wins over the key
|
|
1145
|
+
// slot, as on the Chat Completions wire above.
|
|
1146
|
+
const headers: Record<string, string> = {};
|
|
1147
|
+
for (const [k, v] of Object.entries(req.headers)) if (PASSED_HEADERS.test(k) && typeof v === "string") headers[k] = v;
|
|
1148
|
+
if (spec.upstreamKey) {
|
|
1149
|
+
if (spec.upstreamAuth === "x-api-key") headers["x-api-key"] = spec.upstreamKey;
|
|
1150
|
+
else headers.authorization = `Bearer ${spec.upstreamKey}`;
|
|
1151
|
+
}
|
|
1152
|
+
for (const [k, v] of Object.entries(spec.headers ?? {})) headers[k.toLowerCase()] = v;
|
|
1153
|
+
const method = req.method ?? "GET";
|
|
1154
|
+
const target = `${base}${url.pathname}${url.search}`;
|
|
1155
|
+
let upstream: Response;
|
|
1156
|
+
try {
|
|
1157
|
+
upstream = await fetchUntrusted(target, { method, headers, body: method === "GET" || method === "HEAD" ? undefined : raw }, { sameOrigin: true, maxBytes: UPSTREAM_MAX_BYTES });
|
|
1158
|
+
} catch (err) {
|
|
1159
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1160
|
+
log.push(`${method} ${label}${url.pathname} → unreachable: ${message}`);
|
|
1161
|
+
res.writeHead(502, { "content-type": "application/json" });
|
|
1162
|
+
res.end(errorBody(502, `translator: could not reach ${label}: ${message}`));
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
const out: Record<string, string> = {};
|
|
1166
|
+
upstream.headers.forEach((v, k) => {
|
|
1167
|
+
if (!HOP_HEADERS.has(k)) out[k] = v;
|
|
1168
|
+
});
|
|
1169
|
+
res.writeHead(upstream.status, out);
|
|
1170
|
+
log.push(`${method} ${label}${url.pathname} → ${upstream.status} (${Date.now() - started}ms)`);
|
|
1171
|
+
const reader = upstream.body?.getReader();
|
|
1172
|
+
if (!reader) return void res.end();
|
|
1173
|
+
try {
|
|
1174
|
+
for (;;) {
|
|
1175
|
+
const { value, done } = await reader.read();
|
|
1176
|
+
if (done) break;
|
|
1177
|
+
res.write(value);
|
|
1178
|
+
}
|
|
1179
|
+
res.end();
|
|
1180
|
+
} catch (err) {
|
|
1181
|
+
// Mid-answer (the cap, a dropped connection): an Anthropic error event,
|
|
1182
|
+
// so a stream reads as failed, not as a reply that merely stopped.
|
|
1183
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1184
|
+
log.push(`translator error: ${message}`);
|
|
1185
|
+
res.end(`event: error\ndata: ${errorBody(500, `translator: ${message}`)}\n\n`);
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1108
1189
|
/**
|
|
1109
1190
|
* What a provider block needs from the translator, or null when the block
|
|
1110
1191
|
* speaks Anthropic and needs none. Values only — this crosses into the run
|
package/src/webhook.ts
CHANGED
|
@@ -163,7 +163,7 @@ const KEEP_DELIVERIES = 500;
|
|
|
163
163
|
export function recordDelivery(tenant: string, workspace: string, delivery: HookDelivery) {
|
|
164
164
|
const file = deliveriesFile(tenant, workspace);
|
|
165
165
|
if (!fs.existsSync(path.dirname(file))) return; // no workspace, nothing to log against
|
|
166
|
-
if (UNAUTHENTICATED.includes(delivery.outcome)
|
|
166
|
+
if (UNAUTHENTICATED.includes(delivery.outcome)) return recordRefusal(file, tenant, workspace, delivery);
|
|
167
167
|
fs.appendFileSync(file, JSON.stringify(delivery) + "\n");
|
|
168
168
|
// Bounded: compact to the newest KEEP when it doubles. Amortised cheap,
|
|
169
169
|
// and a log that grows forever is a disk-full incident with a delay.
|
|
@@ -179,9 +179,42 @@ export function recordDelivery(tenant: string, workspace: string, delivery: Hook
|
|
|
179
179
|
}
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
+
/**
|
|
183
|
+
* A refusal — anyone's to send — never changes the workspace's directory: no
|
|
184
|
+
* new file, no tmp file, no rename. The directory's mtime is the workspace's
|
|
185
|
+
* deployedAt (store.ts listWorkspaces), and a stranger posting a bad token
|
|
186
|
+
* moved it (FOL-102). So a refusal goes into a log that already exists, in
|
|
187
|
+
* place (an append, or the last line rewritten), and is never the write that
|
|
188
|
+
* compacts it; a workspace with no log yet gets a server log line instead,
|
|
189
|
+
* at most one a minute per workspace.
|
|
190
|
+
*/
|
|
191
|
+
function recordRefusal(file: string, tenant: string, workspace: string, delivery: HookDelivery) {
|
|
192
|
+
if (!fs.existsSync(file)) return noteRefusal(tenant, workspace, delivery);
|
|
193
|
+
if (collapseInto(file, delivery)) return;
|
|
194
|
+
fs.appendFileSync(file, JSON.stringify(delivery) + "\n");
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const REFUSAL_NOTE_MS = 60_000;
|
|
198
|
+
const refusalNotes = new Map<string, { at: number; since: number }>();
|
|
199
|
+
|
|
200
|
+
function noteRefusal(tenant: string, workspace: string, delivery: HookDelivery) {
|
|
201
|
+
const key = `${tenant}/${workspace}`;
|
|
202
|
+
const now = Date.now();
|
|
203
|
+
const seen = refusalNotes.get(key);
|
|
204
|
+
if (seen && now - seen.at < REFUSAL_NOTE_MS) {
|
|
205
|
+
seen.since++;
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
const more = seen?.since ? ` (and ${seen.since} more since the last note)` : "";
|
|
209
|
+
console.warn(`[hooks] refused a delivery to ${key}/${delivery.flow}: ${delivery.outcome}${more}`);
|
|
210
|
+
refusalNotes.set(key, { at: now, since: 0 });
|
|
211
|
+
if (refusalNotes.size > 10_000) refusalNotes.clear(); // bounded: a note lost beats memory
|
|
212
|
+
}
|
|
213
|
+
|
|
182
214
|
/** Fold a refusal into the log's last line when that line is a refusal
|
|
183
215
|
* too. The flow name is not part of the match: it comes from the URL, so
|
|
184
|
-
* a flood that varies it would otherwise never collapse. True when folded.
|
|
216
|
+
* a flood that varies it would otherwise never collapse. True when folded.
|
|
217
|
+
* Rewritten in place, never through a rename (recordRefusal says why). */
|
|
185
218
|
function collapseInto(file: string, delivery: HookDelivery): boolean {
|
|
186
219
|
let text: string;
|
|
187
220
|
try {
|
|
@@ -204,9 +237,15 @@ function collapseInto(file: string, delivery: HookDelivery): boolean {
|
|
|
204
237
|
first: last.first ?? last.t,
|
|
205
238
|
...(last.detail === "more than one flow name" || last.flow !== delivery.flow ? { detail: "more than one flow name" } : {}),
|
|
206
239
|
};
|
|
207
|
-
const
|
|
208
|
-
|
|
209
|
-
fs.
|
|
240
|
+
const line = Buffer.from(JSON.stringify(merged) + "\n");
|
|
241
|
+
const at = Buffer.byteLength(end.slice(0, cut));
|
|
242
|
+
const fd = fs.openSync(file, "r+");
|
|
243
|
+
try {
|
|
244
|
+
fs.writeSync(fd, line, 0, line.length, at);
|
|
245
|
+
fs.ftruncateSync(fd, at + line.length);
|
|
246
|
+
} finally {
|
|
247
|
+
fs.closeSync(fd);
|
|
248
|
+
}
|
|
210
249
|
return true;
|
|
211
250
|
}
|
|
212
251
|
|