@runuai/host 0.9.148 → 0.9.150
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0024_task_env_mode.sql +9 -0
- package/db/migrations/0025_project_env_authority.sql +147 -0
- package/db/migrations/0026_project_env_replica.sql +150 -0
- package/db/migrations/meta/_journal.json +21 -0
- package/db/schema.ts +312 -2
- package/images/standard/Dockerfile +44 -6
- package/images/standard/container/uai-init +85 -2
- package/images/standard/container/uai-task-forwarder.mjs +303 -0
- package/lib/agent-cli.ts +18 -1
- package/lib/agent.ts +99 -34
- package/lib/agents/codex.ts +9 -1
- package/lib/agents/cursor.ts +15 -1
- package/lib/agents/grok.ts +8 -0
- package/lib/agents/kimi.ts +1 -0
- package/lib/agents/machine-durable.ts +2 -0
- package/lib/agents/opencode.ts +1 -0
- package/lib/agents/proc.ts +6 -1
- package/lib/agents/transport.ts +9 -5
- package/lib/browser-testing.ts +6 -0
- package/lib/codex-auth.ts +2 -0
- package/lib/credential-store.ts +141 -6
- package/lib/engine-accounts.ts +1 -0
- package/lib/env-file.ts +7 -0
- package/lib/env-schema.ts +841 -51
- package/lib/git-diff.ts +1 -0
- package/lib/git-identity.ts +2 -0
- package/lib/github-tokens.ts +1 -0
- package/lib/host-env.ts +98 -18
- package/lib/lease.ts +11 -3
- package/lib/machine-gateway-tunnel.ts +195 -17
- package/lib/machine-project-env-loader.ts +152 -0
- package/lib/managed-runtime.ts +270 -51
- package/lib/mcp-gateway.ts +777 -26
- package/lib/orchestrator.ts +408 -81
- package/lib/project-env-authority.ts +1319 -0
- package/lib/project-env-feature.ts +74 -0
- package/lib/project-env-loader.ts +237 -0
- package/lib/project-env-replica.ts +2094 -0
- package/lib/project-env-schema-capture.ts +342 -0
- package/lib/project-env-staging.ts +148 -0
- package/lib/project-env-task-policy.ts +176 -0
- package/lib/runtime-archive.ts +79 -16
- package/lib/runtime-authority.ts +44 -0
- package/lib/runtime-state.ts +175 -10
- package/lib/secrets-proxy/address.ts +72 -0
- package/lib/secrets-proxy/audit.ts +30 -0
- package/lib/secrets-proxy/ca-store.ts +200 -0
- package/lib/secrets-proxy/ca.ts +470 -0
- package/lib/secrets-proxy/container-steps.ts +138 -0
- package/lib/secrets-proxy/der.ts +273 -0
- package/lib/secrets-proxy/engine.ts +1633 -0
- package/lib/secrets-proxy/host.ts +93 -0
- package/lib/secrets-proxy/hostname.ts +60 -0
- package/lib/secrets-proxy/placeholders.ts +196 -0
- package/lib/secrets-proxy/policy.ts +141 -0
- package/lib/secrets-proxy/readiness.ts +148 -0
- package/lib/secrets-proxy/runtime.ts +582 -0
- package/lib/secrets-proxy/scrub.ts +559 -0
- package/lib/secrets-proxy/sidecar-view.ts +139 -0
- package/lib/secrets-proxy/task-scripts.ts +262 -0
- package/lib/secrets-proxy/task-trust.ts +574 -0
- package/lib/secrets-proxy/trust-watch.ts +80 -0
- package/lib/shared-files-s3.ts +3 -0
- package/lib/sidecar-guard.ts +58 -0
- package/lib/sidecar-resume.ts +51 -3
- package/lib/standard-image.ts +50 -12
- package/lib/task-env-forwarder.ts +228 -0
- package/lib/task-env-source.ts +193 -0
- package/lib/task-environment/apple-container.ts +45 -11
- package/lib/task-environment/docker.ts +51 -12
- package/lib/task-environment/index.ts +6 -2
- package/lib/task-environment/machine-task-up.ts +258 -25
- package/lib/task-environment/machine.ts +80 -22
- package/lib/task-environment/registry.ts +3 -2
- package/lib/task-environment/self-identity.ts +20 -0
- package/lib/task-environment/self.ts +92 -16
- package/lib/task-environment/types.ts +63 -0
- package/lib/task-gateway-port.ts +22 -0
- package/lib/task-gateway-protocol.ts +190 -0
- package/lib/task-gateway-recovery.ts +44 -0
- package/lib/task-gateway-state.ts +497 -0
- package/lib/transcript.ts +1 -0
- package/package.json +13 -1
- package/scripts/agent/env-hook.sh +626 -0
- package/scripts/agent/task-up.sh +476 -27
- package/scripts/agent/uai-env-exec.sh +26 -0
- package/src/cli.ts +26 -3
- package/src/index.ts +273 -19
- package/src/main.ts +939 -48
- package/src/project-env-latest.ts +130 -0
- package/src/project-env-policy.ts +281 -0
- package/src/project-env-wording.ts +47 -0
- package/src/protocol.ts +2219 -15
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Response scrubbing for the secrets proxy (ADR-153 §3, rule 7).
|
|
3
|
+
*
|
|
4
|
+
* Once a request carried a real value, anything the upstream sends back may
|
|
5
|
+
* echo it: an error message quoting the key, an echo endpoint, a redirect URL.
|
|
6
|
+
* Every header and body byte of that response is scanned and each value is
|
|
7
|
+
* replaced by its placeholder before the task sees it.
|
|
8
|
+
*
|
|
9
|
+
* A value is matched character by character, and each character in every
|
|
10
|
+
* form an upstream writes it in: raw (UTF-8); percent-encoded, in either
|
|
11
|
+
* case, so encodeURIComponent, RFC 3986, form encoding and every partial or
|
|
12
|
+
* mixed style are one rule rather than a list of library quirks; `+` for a
|
|
13
|
+
* space; and the JSON escapes (`\/`, `\"`, `\\`, `\n` and friends, `\uXXXX`
|
|
14
|
+
* in either case, surrogate pairs above the BMP). A replacement is written
|
|
15
|
+
* in the style of what it replaces, so a placeholder lands percent-encoded
|
|
16
|
+
* in a URL and raw in a body.
|
|
17
|
+
*
|
|
18
|
+
* Matching is leftmost-longest over raw bytes, whatever the MIME type, and
|
|
19
|
+
* streaming gives byte-for-byte the same result as scrubbing the whole body
|
|
20
|
+
* at once. A stream holds back only from the earliest position where a
|
|
21
|
+
* match could still be completing, so an SSE event that cannot be the start
|
|
22
|
+
* of a value is never delayed.
|
|
23
|
+
*
|
|
24
|
+
* A guard re-checks every output chunk (with enough of the previous output
|
|
25
|
+
* to catch a value straddling the boundary) before it is released.
|
|
26
|
+
* Replacement alone cannot always guarantee a value is gone — a very short
|
|
27
|
+
* value can occur inside its own placeholder — and in that case the
|
|
28
|
+
* response fails (502, or a torn-down stream) rather than going out.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { Transform, type TransformCallback } from "node:stream";
|
|
32
|
+
|
|
33
|
+
const EMPTY = Buffer.alloc(0);
|
|
34
|
+
|
|
35
|
+
/** One value to find (`from`, decoded) and what replaces it (`to`, raw). */
|
|
36
|
+
export interface ScrubRule {
|
|
37
|
+
readonly from: string;
|
|
38
|
+
readonly to: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface InjectedValue {
|
|
42
|
+
readonly value: string;
|
|
43
|
+
readonly placeholder: string;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export class ScrubError extends Error {
|
|
47
|
+
constructor(
|
|
48
|
+
readonly code: "value_survived" | "too_large" | "too_complex",
|
|
49
|
+
message: string,
|
|
50
|
+
) {
|
|
51
|
+
super(message);
|
|
52
|
+
this.name = "ScrubError";
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The rules for one request: every injected value back to its placeholder,
|
|
58
|
+
* plus caller-built pairs such as a re-encoded Basic credential. Empty values
|
|
59
|
+
* are skipped (there is nothing to find), and when two keys share a value the
|
|
60
|
+
* first placeholder wins.
|
|
61
|
+
*/
|
|
62
|
+
export function scrubRules(
|
|
63
|
+
injected: readonly InjectedValue[],
|
|
64
|
+
extra: ReadonlyArray<{ from: string; to: string }> = [],
|
|
65
|
+
): ScrubRule[] {
|
|
66
|
+
const rules = new Map<string, ScrubRule>();
|
|
67
|
+
const add = (from: string, to: string): void => {
|
|
68
|
+
if (from.length === 0 || rules.has(from)) return;
|
|
69
|
+
rules.set(from, { from, to });
|
|
70
|
+
};
|
|
71
|
+
for (const { value, placeholder } of injected) add(value, placeholder);
|
|
72
|
+
for (const { from, to } of extra) add(from, to);
|
|
73
|
+
return [...rules.values()];
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// --- compiled rules -----------------------------------------------------------
|
|
77
|
+
|
|
78
|
+
/** One position of an alternative: the bytes accepted there. */
|
|
79
|
+
type ByteChoice = readonly number[];
|
|
80
|
+
|
|
81
|
+
type Form = "raw" | "percent" | "plus" | "json";
|
|
82
|
+
|
|
83
|
+
interface Alternative {
|
|
84
|
+
readonly form: Form;
|
|
85
|
+
readonly bytes: readonly ByteChoice[];
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** How a match was written, so its replacement is written the same way. */
|
|
89
|
+
interface Style {
|
|
90
|
+
percent: boolean;
|
|
91
|
+
/** The case of the first hex letter seen in an escape, if any was. */
|
|
92
|
+
hexCase: "upper" | "lower" | null;
|
|
93
|
+
jsonSlash: boolean;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
interface CompiledRule {
|
|
97
|
+
readonly to: string;
|
|
98
|
+
/** One unit per character of the value; each unit lists the forms that
|
|
99
|
+
* character may take. */
|
|
100
|
+
readonly units: ReadonlyArray<readonly Alternative[]>;
|
|
101
|
+
readonly firstBytes: ReadonlySet<number>;
|
|
102
|
+
readonly maxEncodedLength: number;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const HEX = "0123456789ABCDEF";
|
|
106
|
+
|
|
107
|
+
function exact(text: string): ByteChoice[] {
|
|
108
|
+
return [...Buffer.from(text, "latin1")].map((byte) => [byte]);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** A hex digit in either case. */
|
|
112
|
+
function hexDigit(nibble: number): ByteChoice {
|
|
113
|
+
const upper = HEX.charCodeAt(nibble);
|
|
114
|
+
const lower = HEX.toLowerCase().charCodeAt(nibble);
|
|
115
|
+
return upper === lower ? [upper] : [upper, lower];
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function percentOf(byte: number): ByteChoice[] {
|
|
119
|
+
return [[0x25], hexDigit(byte >> 4), hexDigit(byte & 0x0f)];
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function unicodeEscape(codeUnit: number): ByteChoice[] {
|
|
123
|
+
return [[0x5c], [0x75], hexDigit((codeUnit >> 12) & 0xf), hexDigit((codeUnit >> 8) & 0xf), hexDigit((codeUnit >> 4) & 0xf), hexDigit(codeUnit & 0xf)];
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const JSON_SHORT: Readonly<Record<string, string>> = {
|
|
127
|
+
'"': '\\"',
|
|
128
|
+
"\\": "\\\\",
|
|
129
|
+
"/": "\\/",
|
|
130
|
+
"\b": "\\b",
|
|
131
|
+
"\f": "\\f",
|
|
132
|
+
"\n": "\\n",
|
|
133
|
+
"\r": "\\r",
|
|
134
|
+
"\t": "\\t",
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
function alternativesFor(character: string): Alternative[] {
|
|
138
|
+
const utf8 = [...Buffer.from(character, "utf8")];
|
|
139
|
+
const alternatives: Alternative[] = [
|
|
140
|
+
{ form: "raw", bytes: utf8.map((byte) => [byte]) },
|
|
141
|
+
{ form: "percent", bytes: utf8.flatMap(percentOf) },
|
|
142
|
+
];
|
|
143
|
+
if (character === " ") alternatives.push({ form: "plus", bytes: [[0x2b]] });
|
|
144
|
+
const short = JSON_SHORT[character];
|
|
145
|
+
if (short !== undefined) alternatives.push({ form: "json", bytes: exact(short) });
|
|
146
|
+
const units: number[] = [];
|
|
147
|
+
for (let index = 0; index < character.length; index += 1) units.push(character.charCodeAt(index));
|
|
148
|
+
alternatives.push({ form: "json", bytes: units.flatMap(unicodeEscape) });
|
|
149
|
+
return alternatives;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function compile(rule: ScrubRule): CompiledRule {
|
|
153
|
+
const units = [...rule.from].map(alternativesFor);
|
|
154
|
+
const firstBytes = new Set<number>();
|
|
155
|
+
for (const alternative of units[0] ?? []) for (const byte of alternative.bytes[0] ?? []) firstBytes.add(byte);
|
|
156
|
+
const maxEncodedLength = units.reduce(
|
|
157
|
+
(total, alternatives) => total + Math.max(...alternatives.map((alternative) => alternative.bytes.length)),
|
|
158
|
+
0,
|
|
159
|
+
);
|
|
160
|
+
return { to: rule.to, units, firstBytes, maxEncodedLength };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Rules compiled once, then shared by every scrubber and guard one response
|
|
164
|
+
* needs. */
|
|
165
|
+
export class PreparedRules {
|
|
166
|
+
readonly rules: readonly CompiledRule[];
|
|
167
|
+
readonly firstBytes: ReadonlySet<number>;
|
|
168
|
+
/** The longest form any value can take: the guard's look-behind. */
|
|
169
|
+
readonly maxEncodedLength: number;
|
|
170
|
+
|
|
171
|
+
constructor(rules: readonly ScrubRule[]) {
|
|
172
|
+
this.rules = rules.filter((rule) => rule.from.length > 0).map(compile);
|
|
173
|
+
const firstBytes = new Set<number>();
|
|
174
|
+
for (const rule of this.rules) for (const byte of rule.firstBytes) firstBytes.add(byte);
|
|
175
|
+
this.firstBytes = firstBytes;
|
|
176
|
+
this.maxEncodedLength = Math.max(0, ...this.rules.map((rule) => rule.maxEncodedLength));
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function prepared(rules: readonly ScrubRule[] | PreparedRules): PreparedRules {
|
|
181
|
+
return rules instanceof PreparedRules ? rules : new PreparedRules(rules);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
interface MatchResult {
|
|
185
|
+
/** End of the longest complete match from the start, or -1. */
|
|
186
|
+
end: number;
|
|
187
|
+
style: Style | null;
|
|
188
|
+
/** Some way of matching ran out of input: more bytes might still make a
|
|
189
|
+
* (longer) match here. */
|
|
190
|
+
partial: boolean;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Work allowed for one scan, counted in (state × alternative) steps and
|
|
194
|
+
* proportional to the bytes scanned. Realistic values cost a handful of
|
|
195
|
+
* steps per byte — every value has a JSON `\uXXXX` form, so each backslash
|
|
196
|
+
* in a body is a candidate that dies a byte later. An adversarial pairing
|
|
197
|
+
* (a value full of backslashes against a body full of them) costs steps
|
|
198
|
+
* quadratic in the value's length per byte and runs out instead; running
|
|
199
|
+
* out fails the response closed. */
|
|
200
|
+
const SCRUB_STEPS_PER_BYTE = 64;
|
|
201
|
+
const SCRUB_MIN_STEPS = 1_000_000;
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
/** A stream's allowance: a floor once, then SCRUB_STEPS_PER_BYTE for every
|
|
206
|
+
* byte that arrives. It never resets per chunk, so an upstream that dribbles
|
|
207
|
+
* a body one byte at a time — each push rescanning the held-back tail —
|
|
208
|
+
* cannot buy itself a fresh floor per byte. */
|
|
209
|
+
class Budget {
|
|
210
|
+
private remaining = SCRUB_MIN_STEPS;
|
|
211
|
+
accrue(bytes: number): void {
|
|
212
|
+
this.remaining += SCRUB_STEPS_PER_BYTE * bytes;
|
|
213
|
+
}
|
|
214
|
+
spend(steps: number): void {
|
|
215
|
+
this.remaining -= steps;
|
|
216
|
+
if (this.remaining < 0) throw new ScrubError("too_complex", "the response is too costly to scrub");
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Every way `rule` can match `input` from `start`, evaluated unit by unit
|
|
222
|
+
* over the set of input offsets reachable so far — iterative, so a long value
|
|
223
|
+
* cannot exhaust the stack, and deduplicated by offset, so values
|
|
224
|
+
* whose characters have overlapping forms (`\` raw or `\\` escaped) cost
|
|
225
|
+
* polynomially, never exponentially. The budget bounds what remains.
|
|
226
|
+
*/
|
|
227
|
+
function matchAt(rule: CompiledRule, input: Buffer, start: number, budget: Budget): MatchResult {
|
|
228
|
+
const result: MatchResult = { end: -1, style: null, partial: false };
|
|
229
|
+
// Offset → the style of the first way found to reach it. Style only shapes
|
|
230
|
+
// the replacement, so one per offset is enough, and keying by offset alone
|
|
231
|
+
// keeps the frontier no larger than the bytes a unit can span.
|
|
232
|
+
let frontier = new Map<number, Style>([[start, { percent: false, hexCase: null, jsonSlash: false }]]);
|
|
233
|
+
for (const alternatives of rule.units) {
|
|
234
|
+
const next = new Map<number, Style>();
|
|
235
|
+
for (const [at, style] of frontier) {
|
|
236
|
+
budget.spend(alternatives.length);
|
|
237
|
+
if (at === input.length) {
|
|
238
|
+
result.partial = true;
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
for (const alternative of alternatives) {
|
|
242
|
+
let position = at;
|
|
243
|
+
let matched = true;
|
|
244
|
+
for (const choice of alternative.bytes) {
|
|
245
|
+
if (position === input.length) {
|
|
246
|
+
result.partial = true;
|
|
247
|
+
matched = false;
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
if (!choice.includes(input[position] ?? -1)) {
|
|
251
|
+
matched = false;
|
|
252
|
+
break;
|
|
253
|
+
}
|
|
254
|
+
position += 1;
|
|
255
|
+
}
|
|
256
|
+
if (!matched) continue;
|
|
257
|
+
const nextStyle: Style = { ...style };
|
|
258
|
+
if (alternative.form === "percent") {
|
|
259
|
+
nextStyle.percent = true;
|
|
260
|
+
// The first escape with a hex letter decides the replacement's case.
|
|
261
|
+
if (nextStyle.hexCase === null) {
|
|
262
|
+
const hex = input.subarray(at, position).toString("latin1");
|
|
263
|
+
if (/[a-f]/.test(hex)) nextStyle.hexCase = "lower";
|
|
264
|
+
else if (/[A-F]/.test(hex)) nextStyle.hexCase = "upper";
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
if (alternative.form === "json" && input[at + 1] === 0x2f) nextStyle.jsonSlash = true;
|
|
268
|
+
if (!next.has(position)) next.set(position, nextStyle);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
frontier = next;
|
|
272
|
+
if (frontier.size === 0) return result;
|
|
273
|
+
}
|
|
274
|
+
for (const [at, style] of frontier) {
|
|
275
|
+
if (at > result.end) {
|
|
276
|
+
result.end = at;
|
|
277
|
+
result.style = style;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return result;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const UNRESERVED = /^[A-Za-z0-9._~-]$/;
|
|
284
|
+
|
|
285
|
+
/** The replacement, written the way the match was. */
|
|
286
|
+
function encodeLike(to: string, style: Style | null): Buffer {
|
|
287
|
+
if (style === null || (!style.percent && !style.jsonSlash)) return Buffer.from(to, "utf8");
|
|
288
|
+
let output = "";
|
|
289
|
+
for (const character of to) {
|
|
290
|
+
if (UNRESERVED.test(character)) {
|
|
291
|
+
output += character;
|
|
292
|
+
} else if (style.percent) {
|
|
293
|
+
for (const byte of Buffer.from(character, "utf8")) {
|
|
294
|
+
const hex = byte.toString(16).padStart(2, "0");
|
|
295
|
+
output += `%${style.hexCase === "lower" ? hex : hex.toUpperCase()}`;
|
|
296
|
+
}
|
|
297
|
+
} else if (character === "/") {
|
|
298
|
+
output += "\\/";
|
|
299
|
+
} else {
|
|
300
|
+
output += character;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return Buffer.from(output, "utf8");
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** Leftmost-longest streaming replacer. `push` returns what is safe to emit;
|
|
307
|
+
* `end` releases the held-back tail. */
|
|
308
|
+
export class Scrubber {
|
|
309
|
+
private readonly prepared: PreparedRules;
|
|
310
|
+
private readonly budget = new Budget();
|
|
311
|
+
private carry: Buffer = EMPTY;
|
|
312
|
+
|
|
313
|
+
constructor(rules: readonly ScrubRule[] | PreparedRules) {
|
|
314
|
+
this.prepared = prepared(rules);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
push(chunk: Buffer): Buffer {
|
|
318
|
+
this.budget.accrue(chunk.length);
|
|
319
|
+
const input = this.carry.length === 0 ? chunk : Buffer.concat([this.carry, chunk]);
|
|
320
|
+
const { output, held } = this.scan(input, false);
|
|
321
|
+
// Copy: `held` is a view that would pin the whole input buffer.
|
|
322
|
+
this.carry = held.length === 0 ? EMPTY : Buffer.from(held);
|
|
323
|
+
return output;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
end(): Buffer {
|
|
327
|
+
const { output } = this.scan(this.carry, true);
|
|
328
|
+
this.carry = EMPTY;
|
|
329
|
+
return output;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
private scan(input: Buffer, final: boolean): { output: Buffer; held: Buffer } {
|
|
333
|
+
const { rules, firstBytes } = this.prepared;
|
|
334
|
+
if (rules.length === 0) return { output: input, held: EMPTY };
|
|
335
|
+
const budget = this.budget;
|
|
336
|
+
// Every byte scanned (and copied in with the held-back tail) costs a step,
|
|
337
|
+
// so rescanning a tail is paid for, not free.
|
|
338
|
+
budget.spend(input.length);
|
|
339
|
+
const pieces: Buffer[] = [];
|
|
340
|
+
let cursor = 0;
|
|
341
|
+
let position = 0;
|
|
342
|
+
while (position < input.length) {
|
|
343
|
+
if (!firstBytes.has(input[position] ?? -1)) {
|
|
344
|
+
position += 1;
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
let best: { end: number; rule: CompiledRule; style: Style | null } | null = null;
|
|
348
|
+
let pending = false;
|
|
349
|
+
for (const rule of rules) {
|
|
350
|
+
if (!rule.firstBytes.has(input[position] ?? -1)) continue;
|
|
351
|
+
const match = matchAt(rule, input, position, budget);
|
|
352
|
+
if (match.partial && !final) pending = true;
|
|
353
|
+
if (match.end > position && (best === null || match.end > best.end)) {
|
|
354
|
+
best = { end: match.end, rule, style: match.style };
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
// A match still arriving here may beat (or extend) one already
|
|
358
|
+
// complete: everything from here waits for more bytes.
|
|
359
|
+
if (pending) break;
|
|
360
|
+
if (best !== null) {
|
|
361
|
+
pieces.push(input.subarray(cursor, position), encodeLike(best.rule.to, best.style));
|
|
362
|
+
cursor = best.end;
|
|
363
|
+
position = best.end;
|
|
364
|
+
continue;
|
|
365
|
+
}
|
|
366
|
+
position += 1;
|
|
367
|
+
}
|
|
368
|
+
pieces.push(input.subarray(cursor, position));
|
|
369
|
+
return {
|
|
370
|
+
output: pieces.length === 1 ? (pieces[0] ?? EMPTY) : Buffer.concat(pieces),
|
|
371
|
+
held: input.subarray(position),
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Whether any rule matches completely anywhere in `input`, and the earliest
|
|
377
|
+
* position from which a match could still be forming when more bytes
|
|
378
|
+
* arrive (input.length when none). */
|
|
379
|
+
function inspect(input: Buffer, ready: PreparedRules, budget: Budget): { found: boolean; pendingFrom: number } {
|
|
380
|
+
let pendingFrom = input.length;
|
|
381
|
+
for (let position = 0; position < input.length; position += 1) {
|
|
382
|
+
if (!ready.firstBytes.has(input[position] ?? -1)) continue;
|
|
383
|
+
for (const rule of ready.rules) {
|
|
384
|
+
if (!rule.firstBytes.has(input[position] ?? -1)) continue;
|
|
385
|
+
const match = matchAt(rule, input, position, budget);
|
|
386
|
+
if (match.end > position) return { found: true, pendingFrom };
|
|
387
|
+
if (match.partial && position < pendingFrom) pendingFrom = position;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
return { found: false, pendingFrom };
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* Refuses output containing any value in any form, including across the
|
|
395
|
+
* boundary with what was already released. Stateful: it keeps only the suffix
|
|
396
|
+
* from which a value could still be forming — the scrubber's own hold-back
|
|
397
|
+
* rule — not a fixed look-behind, so released bytes that cannot begin a
|
|
398
|
+
* value are never scanned twice and a harmless stream costs a step a byte.
|
|
399
|
+
*/
|
|
400
|
+
class ScrubGuard {
|
|
401
|
+
private readonly ready: PreparedRules;
|
|
402
|
+
private readonly budget = new Budget();
|
|
403
|
+
private pending: Buffer = EMPTY;
|
|
404
|
+
|
|
405
|
+
constructor(rules: readonly ScrubRule[] | PreparedRules) {
|
|
406
|
+
this.ready = prepared(rules);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
accepts(output: Buffer): boolean {
|
|
410
|
+
if (output.length === 0 || this.ready.rules.length === 0) return true;
|
|
411
|
+
const window = this.pending.length === 0 ? output : Buffer.concat([this.pending, output]);
|
|
412
|
+
this.budget.accrue(output.length);
|
|
413
|
+
let result: { found: boolean; pendingFrom: number };
|
|
414
|
+
try {
|
|
415
|
+
this.budget.spend(window.length);
|
|
416
|
+
result = inspect(window, this.ready, this.budget);
|
|
417
|
+
} catch {
|
|
418
|
+
// Could not prove the output clean: refuse it.
|
|
419
|
+
return false;
|
|
420
|
+
}
|
|
421
|
+
if (result.found) return false;
|
|
422
|
+
this.pending = result.pendingFrom >= window.length ? EMPTY : Buffer.from(window.subarray(result.pendingFrom));
|
|
423
|
+
return true;
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** Whole-buffer scrub; null when a value would survive (answer 502). */
|
|
428
|
+
export function scrubBuffer(input: Buffer, rules: readonly ScrubRule[] | PreparedRules): Buffer | null {
|
|
429
|
+
const ready = prepared(rules);
|
|
430
|
+
try {
|
|
431
|
+
const scrubber = new Scrubber(ready);
|
|
432
|
+
const output = Buffer.concat([scrubber.push(input), scrubber.end()]);
|
|
433
|
+
return new ScrubGuard(ready).accepts(output) ? output : null;
|
|
434
|
+
} catch {
|
|
435
|
+
// A budget exhausted (or any matcher failure) is a scrub that did not
|
|
436
|
+
// happen: the caller answers 502, never the unscanned bytes.
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/** Header and status-line scrub. Node hands header bytes over as latin1
|
|
442
|
+
* strings, so a latin1 round trip compares the exact bytes on the wire.
|
|
443
|
+
* Pass PreparedRules when scrubbing many headers of one response. */
|
|
444
|
+
export function scrubHeaderText(text: string, rules: readonly ScrubRule[] | PreparedRules): string | null {
|
|
445
|
+
const output = scrubBuffer(Buffer.from(text, "latin1"), rules);
|
|
446
|
+
return output === null ? null : output.toString("latin1");
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** A stream scans once this many bytes have gathered, or after
|
|
450
|
+
* COALESCE_MS, whichever comes first. Every scan rereads the held-back tail
|
|
451
|
+
* and the guard's look-behind, so scanning per arriving chunk would let an
|
|
452
|
+
* upstream that dribbles a body a byte at a time multiply the work by the
|
|
453
|
+
* tail's length; gathering bounds scans by bytes and by time, at the cost
|
|
454
|
+
* of at most COALESCE_MS of added latency on a stream of small events. */
|
|
455
|
+
const COALESCE_BYTES = 4_096;
|
|
456
|
+
const COALESCE_MS = 5;
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* The streaming form. Errors with ScrubError when a value would survive, when
|
|
460
|
+
* scrubbing would cost more than the stream's allowance, or when more than
|
|
461
|
+
* `maxInputBytes` arrive (the bound on decoding a compressed body that was
|
|
462
|
+
* chosen for buffering).
|
|
463
|
+
*/
|
|
464
|
+
export function createScrubStream(
|
|
465
|
+
rules: readonly ScrubRule[] | PreparedRules,
|
|
466
|
+
options: { maxInputBytes?: number } = {},
|
|
467
|
+
): Transform {
|
|
468
|
+
const ready = prepared(rules);
|
|
469
|
+
const scrubber = new Scrubber(ready);
|
|
470
|
+
const guard = new ScrubGuard(ready);
|
|
471
|
+
let seen = 0;
|
|
472
|
+
let gathered: Buffer[] = [];
|
|
473
|
+
let gatheredBytes = 0;
|
|
474
|
+
let timer: NodeJS.Timeout | null = null;
|
|
475
|
+
|
|
476
|
+
const stopTimer = (): void => {
|
|
477
|
+
if (timer !== null) {
|
|
478
|
+
clearTimeout(timer);
|
|
479
|
+
timer = null;
|
|
480
|
+
}
|
|
481
|
+
};
|
|
482
|
+
/** Scans what has gathered. Returns what may be released, or the error
|
|
483
|
+
* that ends the stream. A throw never leaves here: inside _transform it
|
|
484
|
+
* would escape the stream machinery and take the runner down. */
|
|
485
|
+
const scanGathered = (): Buffer | ScrubError => {
|
|
486
|
+
const input = gathered.length === 1 ? (gathered[0] ?? EMPTY) : Buffer.concat(gathered);
|
|
487
|
+
gathered = [];
|
|
488
|
+
gatheredBytes = 0;
|
|
489
|
+
let output: Buffer;
|
|
490
|
+
try {
|
|
491
|
+
output = scrubber.push(input);
|
|
492
|
+
} catch (error) {
|
|
493
|
+
return error instanceof ScrubError ? error : new ScrubError("too_complex", "scrubbing failed");
|
|
494
|
+
}
|
|
495
|
+
return guard.accepts(output) ? output : new ScrubError("value_survived", "a proxied value survived scrubbing");
|
|
496
|
+
};
|
|
497
|
+
|
|
498
|
+
const stream: Transform = new Transform({
|
|
499
|
+
transform(chunk: Buffer, _encoding, callback) {
|
|
500
|
+
seen += chunk.length;
|
|
501
|
+
if (options.maxInputBytes !== undefined && seen > options.maxInputBytes) {
|
|
502
|
+
stopTimer();
|
|
503
|
+
callback(new ScrubError("too_large", "body exceeds the scrub limit"));
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
gathered.push(chunk);
|
|
507
|
+
gatheredBytes += chunk.length;
|
|
508
|
+
// Gathering acknowledges a write without pushing, and Node completes
|
|
509
|
+
// such a write at once even when the readable side is full; the timer's
|
|
510
|
+
// later push would then grow that buffer without bound behind a slow
|
|
511
|
+
// reader. Once the reader is behind, scan now through the callback,
|
|
512
|
+
// whose push lets Node hold the write — and the upstream — back.
|
|
513
|
+
const readerBehind = stream.readableLength >= stream.readableHighWaterMark;
|
|
514
|
+
if (gatheredBytes >= COALESCE_BYTES || readerBehind) {
|
|
515
|
+
stopTimer();
|
|
516
|
+
const result = scanGathered();
|
|
517
|
+
if (result instanceof ScrubError) callback(result);
|
|
518
|
+
else callback(null, result.length > 0 ? result : undefined);
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
if (timer === null) {
|
|
522
|
+
timer = setTimeout(() => {
|
|
523
|
+
timer = null;
|
|
524
|
+
if (gatheredBytes === 0) return;
|
|
525
|
+
const result = scanGathered();
|
|
526
|
+
if (result instanceof ScrubError) stream.destroy(result);
|
|
527
|
+
else if (result.length > 0) stream.push(result);
|
|
528
|
+
}, COALESCE_MS);
|
|
529
|
+
}
|
|
530
|
+
callback();
|
|
531
|
+
},
|
|
532
|
+
flush(callback) {
|
|
533
|
+
stopTimer();
|
|
534
|
+
const result = gatheredBytes > 0 ? scanGathered() : EMPTY;
|
|
535
|
+
if (result instanceof ScrubError) {
|
|
536
|
+
callback(result);
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
let tail: Buffer;
|
|
540
|
+
try {
|
|
541
|
+
tail = scrubber.end();
|
|
542
|
+
} catch (error) {
|
|
543
|
+
callback(error instanceof ScrubError ? error : new ScrubError("too_complex", "scrubbing failed"));
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
if (!guard.accepts(tail)) {
|
|
547
|
+
callback(new ScrubError("value_survived", "a proxied value survived scrubbing"));
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
if (result.length > 0) stream.push(result);
|
|
551
|
+
callback(null, tail.length > 0 ? tail : undefined);
|
|
552
|
+
},
|
|
553
|
+
destroy(error, callback) {
|
|
554
|
+
stopTimer();
|
|
555
|
+
callback(error);
|
|
556
|
+
},
|
|
557
|
+
});
|
|
558
|
+
return stream;
|
|
559
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether every process of a Uai Cloud machine's own task sees the task's
|
|
3
|
+
* hosts view (ADR-153 "Machines"), so its pins can count. Kept as a record
|
|
4
|
+
* per task rather than read off its processes: a process can sit in a mount
|
|
5
|
+
* namespace of its own and still see the machine's file, or hand itself off
|
|
6
|
+
* to a child mid-look, so no look at them proves they all see it.
|
|
7
|
+
*
|
|
8
|
+
* Only a fresh admission proves a task: task-up for a task with no row yet,
|
|
9
|
+
* so that nothing of it can have run before, recorded before its row and so
|
|
10
|
+
* before its first process. The task stays proven in that boot and every
|
|
11
|
+
* boot after. A task with a row and no record (a runner without the view
|
|
12
|
+
* started it, whether or not it ever came up) is held instead: its
|
|
13
|
+
* processes may run outside the view until its machine boots again, which
|
|
14
|
+
* none of them survives.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
|
|
20
|
+
import { env } from "../env";
|
|
21
|
+
import { isSafeHostTaskId } from "../task-identity";
|
|
22
|
+
|
|
23
|
+
/** The kernel's boot ids are lowercase UUIDs; nothing else is one. */
|
|
24
|
+
const BOOT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
25
|
+
|
|
26
|
+
interface ViewRecord {
|
|
27
|
+
/** A boot in which every process of the task came through the wrapper. */
|
|
28
|
+
readonly viewBoot?: string;
|
|
29
|
+
/** The boot in which the task was held: its processes may predate the view. */
|
|
30
|
+
readonly preViewBoot?: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface SidecarViewDeps {
|
|
34
|
+
bootId(): string;
|
|
35
|
+
/** The stored record, as parsed; null when there is none. */
|
|
36
|
+
read(taskId: string): unknown;
|
|
37
|
+
write(taskId: string, record: ViewRecord): void;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function recordPath(taskId: string): string {
|
|
41
|
+
if (!isSafeHostTaskId(taskId)) throw new Error("invalid host task identity");
|
|
42
|
+
return join(env.dataDir, "secrets-proxy", "task-views", `${taskId}.json`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const defaults: SidecarViewDeps = {
|
|
46
|
+
bootId: () => readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(),
|
|
47
|
+
read: (taskId) => {
|
|
48
|
+
try {
|
|
49
|
+
return JSON.parse(readFileSync(recordPath(taskId), "utf8")) as unknown;
|
|
50
|
+
} catch {
|
|
51
|
+
// Missing or unreadable: no record.
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
write: (taskId, record) => {
|
|
56
|
+
const path = recordPath(taskId);
|
|
57
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
58
|
+
writeFileSync(`${path}.tmp`, JSON.stringify(record), { mode: 0o600 });
|
|
59
|
+
renameSync(`${path}.tmp`, path);
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
let deps: SidecarViewDeps = defaults;
|
|
64
|
+
|
|
65
|
+
/** Test seam: the boot id and the record store; what is left out keeps its
|
|
66
|
+
* default. */
|
|
67
|
+
export function setSidecarViewDeps(next: Partial<SidecarViewDeps> | null): void {
|
|
68
|
+
deps = next === null ? defaults : { ...defaults, ...next };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function remember(taskId: string, record: ViewRecord): void {
|
|
72
|
+
try {
|
|
73
|
+
deps.write(taskId, record);
|
|
74
|
+
} catch {
|
|
75
|
+
// Not kept: the next look decides again, from the same facts.
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** This boot's id; null when it cannot be read, or what is read is not one. */
|
|
80
|
+
function currentBoot(): string | null {
|
|
81
|
+
try {
|
|
82
|
+
const boot = deps.bootId();
|
|
83
|
+
return BOOT_ID.test(boot) ? boot : null;
|
|
84
|
+
} catch {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** The task's record, if it is one this module writes: a single boot id
|
|
90
|
+
* under one of its two names. Anything else counts as no record. */
|
|
91
|
+
function recordOf(taskId: string): ViewRecord | null {
|
|
92
|
+
const stored = deps.read(taskId);
|
|
93
|
+
if (stored === null || typeof stored !== "object" || Array.isArray(stored)) return null;
|
|
94
|
+
const entries = Object.entries(stored);
|
|
95
|
+
if (entries.length !== 1) return null;
|
|
96
|
+
const [name, boot] = entries[0]!;
|
|
97
|
+
if (typeof boot !== "string" || !BOOT_ID.test(boot)) return null;
|
|
98
|
+
if (name === "viewBoot") return { viewBoot: boot };
|
|
99
|
+
if (name === "preViewBoot") return { preViewBoot: boot };
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Task-up for a task with no row yet: nothing of it has run on this machine,
|
|
105
|
+
* so everything it runs from here comes through the wrapper. Called before
|
|
106
|
+
* task-up records the task, and so before its first process. Never lifts a
|
|
107
|
+
* hold in this boot, and records nothing without a boot id.
|
|
108
|
+
*/
|
|
109
|
+
export function admitFreshTaskView(taskId: string): void {
|
|
110
|
+
const boot = currentBoot();
|
|
111
|
+
if (boot === null) return;
|
|
112
|
+
if (recordOf(taskId)?.preViewBoot === boot) return;
|
|
113
|
+
remember(taskId, { viewBoot: boot });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Whether every process of the task sees its hosts view now. Never true on
|
|
117
|
+
* a doubt: no boot id, or a task held in this boot. A task without a record
|
|
118
|
+
* is held from here on, for this boot. */
|
|
119
|
+
export function taskViewProven(taskId: string): boolean {
|
|
120
|
+
const boot = currentBoot();
|
|
121
|
+
if (boot === null) return false;
|
|
122
|
+
const record = recordOf(taskId);
|
|
123
|
+
if (record?.viewBoot !== undefined) {
|
|
124
|
+
// Admitted fresh in this boot, or rebooted since: every process running
|
|
125
|
+
// now came through the wrapper.
|
|
126
|
+
if (record.viewBoot !== boot) remember(taskId, { viewBoot: boot });
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
if (record?.preViewBoot !== undefined && record.preViewBoot !== boot) {
|
|
130
|
+
// Held in an earlier boot: none of those processes survived the reboot.
|
|
131
|
+
remember(taskId, { viewBoot: boot });
|
|
132
|
+
return true;
|
|
133
|
+
}
|
|
134
|
+
// Never admitted fresh: a runner without the view started the task, and
|
|
135
|
+
// its processes may still run. Remembered, so a runner restart in this
|
|
136
|
+
// boot does not forget it.
|
|
137
|
+
if (record?.preViewBoot !== boot) remember(taskId, { preViewBoot: boot });
|
|
138
|
+
return false;
|
|
139
|
+
}
|