@jr2/orchestrator 0.2.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/bin/server.ts +7 -4
  2. package/console/canvas.ts +27 -14
  3. package/console/components/app.ts +10 -1
  4. package/console/components/fleet.ts +39 -6
  5. package/console/components/machine-pane.ts +89 -2
  6. package/console/components/nav.ts +1 -1
  7. package/console/components/schema-form.ts +12 -4
  8. package/console/json-field.ts +23 -0
  9. package/console/main.ts +110 -39
  10. package/console/store.ts +221 -17
  11. package/console/style.css +90 -0
  12. package/package.json +4 -5
  13. package/src/actor.ts +507 -54
  14. package/src/agent.ts +2 -2
  15. package/src/ambient.ts +2 -0
  16. package/src/assembly-checks.ts +165 -0
  17. package/src/assembly-tar.ts +494 -0
  18. package/src/assembly.ts +721 -0
  19. package/src/chart.ts +2050 -0
  20. package/src/charts.ts +720 -0
  21. package/src/config.ts +233 -22
  22. package/src/content-hash.ts +105 -0
  23. package/src/custodian.lua +313 -0
  24. package/src/custodian.ts +951 -0
  25. package/src/customize.ts +216 -25
  26. package/src/effect-checks.ts +557 -0
  27. package/src/effect.ts +905 -0
  28. package/src/effector-wire.ts +525 -0
  29. package/src/fingerprint.ts +4 -3
  30. package/src/gate.ts +8 -0
  31. package/src/git-clone.ts +65 -0
  32. package/src/git-push.ts +336 -0
  33. package/src/harness-client.ts +247 -25
  34. package/src/held-secrets.ts +643 -0
  35. package/src/holds.ts +156 -0
  36. package/src/http-effect.ts +981 -0
  37. package/src/http.ts +514 -119
  38. package/src/images.ts +58 -31
  39. package/src/index.ts +49 -4
  40. package/src/instance.ts +445 -59
  41. package/src/kube-client.ts +333 -0
  42. package/src/machine-doc.ts +88 -6
  43. package/src/model-keys.ts +92 -0
  44. package/src/mount-digest.ts +53 -0
  45. package/src/names.ts +89 -7
  46. package/src/notices.ts +88 -0
  47. package/src/open-body.ts +1444 -0
  48. package/src/open-effect.ts +144 -0
  49. package/src/open.ts +4 -4
  50. package/src/parts.ts +530 -33
  51. package/src/party.ts +468 -0
  52. package/src/pick.ts +309 -0
  53. package/src/pool.ts +1 -1
  54. package/src/registration.ts +301 -17
  55. package/src/repo-fetch.ts +61 -86
  56. package/src/repos.ts +63 -74
  57. package/src/rest.ts +133 -0
  58. package/src/run-host.ts +1841 -105
  59. package/src/sandbox-effect.ts +891 -0
  60. package/src/sandbox-effector.ts +111 -0
  61. package/src/sandbox-kube.ts +1719 -0
  62. package/src/sandbox-watch.ts +260 -0
  63. package/src/secret-placement.ts +494 -0
  64. package/src/server.ts +285 -89
  65. package/src/setup.ts +589 -70
  66. package/src/size.ts +342 -0
  67. package/src/snapshot-store.ts +64 -13
  68. package/src/stub-harness.ts +5 -0
  69. package/src/sync-client.ts +54 -0
  70. package/src/timers.ts +178 -0
  71. package/src/tokens.ts +44 -12
  72. package/src/vocabulary.ts +66 -1
  73. package/src/wire.ts +156 -2
  74. package/src/workspace.ts +529 -194
  75. package/src/sandbox-kubectl.ts +0 -1144
package/src/agent.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  // The Agent definition (ADR-0018/0027): the part of an Agent a user genuinely owns — model +
2
- // instructions + workspace access — as SERIALIZABLE DATA. Everything mechanical (the Adapter
3
- // leash, the Working-tool assembly, the wire) lives in the stock Harness image (`@jr2/harness`),
2
+ // instructions + workspace access — as SERIALIZABLE DATA. Everything mechanical (the Menu,
3
+ // the Working-tool assembly, the wire) lives in the stock Harness image (`@jr2/harness`),
4
4
  // which runs the definition it is handed and re-reads it per Submission.
5
5
  //
6
6
  // A definition is NOT an Instance roster entry: a Machine CARRIES it as an actor slot —
package/src/ambient.ts CHANGED
@@ -23,6 +23,8 @@ export type AmbientHandles = {
23
23
  sandbox: string;
24
24
  /** Every Repo Slot's branch-worktree path, by slot (ADR-0051). */
25
25
  repos: Record<string, string>;
26
+ /** Every Repo Slot's Binding url, by slot (ADR-0051): where a push goes. */
27
+ bindings?: Record<string, string>;
26
28
  branch: string;
27
29
  /** Detached review-worktree paths, when the spec carried `reviewSha` (ADR-0028). */
28
30
  review?: Record<string, string>;
@@ -0,0 +1,165 @@
1
+ // The walk's refusals, as a LISTING (ADR-0069). The walk runs twice: `jr2 up` runs it on the host
2
+ // before any cluster work, and the door runs it again over the unpacked Assembly before the
3
+ // publish lands. Both ends call these functions, so the two walks cannot drift. They return the
4
+ // lines and print nothing: `jr2 up` prints them after `refusing: `, and the door returns them in
5
+ // its `422` report.
6
+
7
+ import { isAbsolute, relative, sep } from "node:path";
8
+ import type { AnyStateMachine } from "xstate";
9
+ import type { JR2Config } from "./config.ts";
10
+ import { imageContextDigest, type ImageRefs } from "./images.ts";
11
+ import { openPartsText } from "./open-effect.ts";
12
+ import { customizeLine, partsOf, placedEffectsOf, workspacePodsOf } from "./parts.ts";
13
+ import { refuseEffectSecrets, type PlacementWalk } from "./secret-placement.ts";
14
+ import { chartFileRefusals, type ChartFile } from "./charts.ts";
15
+
16
+ /** One registered Workflow, as `loadWorkflows` gives it. */
17
+ export type LoadedWorkflow = { name: string; machine: AnyStateMachine };
18
+
19
+ /**
20
+ * Every part a registered Machine leaves Open (ADR-0051, ADR-0054): a Repo Slot with no url, an
21
+ * Agent with no model, an Effect's url, argv or secret. One line per part, in walk order within
22
+ * each kind, so a composer fixes all of them in one edit. Each line names the Workflow and the
23
+ * `customize()` line that binds the part. A part on a Machine invoked inline has no fix line,
24
+ * because no `customize()` reaches an actor without a slot key.
25
+ *
26
+ * The line text follows `refusing: ` in `jr2 up`'s output; the prefix is the caller's.
27
+ */
28
+ export function openPartRefusals(workflows: readonly LoadedWorkflow[]): string[] {
29
+ const lines: string[] = [];
30
+ for (const w of workflows) {
31
+ const walked = partsOf([w.machine]);
32
+ const opened: Array<{
33
+ slot: string | undefined;
34
+ path: readonly string[] | undefined;
35
+ part: Parameters<typeof customizeLine>[3];
36
+ what: string;
37
+ adr: string;
38
+ }> = [
39
+ ...walked.openSlots.map((o) => ({
40
+ ...o,
41
+ part: "repo" as const,
42
+ // An Open MAP names no slot (the composer does), so it is refused as "the Repo Slots".
43
+ what: o.slot === undefined ? "its Repo Slots" : `Repo Slot "${o.slot}"`,
44
+ adr: "ADR-0051",
45
+ })),
46
+ ...walked.openAgents.map((o) => ({ ...o, part: "agent" as const, what: `Agent "${o.slot}"`, adr: "ADR-0054" })),
47
+ // An Effect's Open url, argv or secret (ADR-0054): one binding object for an Effect with two.
48
+ ...walked.openEffects.map(({ parts, ...o }) => ({
49
+ ...o,
50
+ part: parts,
51
+ what: `the ${openPartsText(parts)} of Effect "${o.slot}"`,
52
+ adr: "ADR-0054",
53
+ })),
54
+ ];
55
+ for (const { slot, path, part, what, adr } of opened) {
56
+ if (path === undefined) {
57
+ lines.push(
58
+ `workflow "${w.name}" leaves ${what} open on a Machine invoked inline — no ` +
59
+ "customize() reaches an actor without a slot key; declare that Machine under setup({ actors }) and " +
60
+ `bind it there (ADR-0049, ${adr})`,
61
+ );
62
+ continue;
63
+ }
64
+ const where = path.length ? ` (on the Machine composed as ${path.map((k) => `"${k}"`).join(" → ")})` : "";
65
+ lines.push(
66
+ `workflow "${w.name}" leaves ${what} open${where} — bind it where the Machine is ` +
67
+ `registered: export const machine = ${customizeLine("<import>", path, slot, part)} (${adr})`,
68
+ );
69
+ }
70
+ }
71
+ return lines;
72
+ }
73
+
74
+ /**
75
+ * What the registered Machines name for secret placement (ADR-0059, ADR-0068): every Effect and
76
+ * every Workspace's pod, per route, and the models the Instance Harness hosts: those of the `"none"`
77
+ * Agents and of every Open body that allows `"none"` (ADR-0070). `jr2 up` and the door build the same walk from the same Workflows.
78
+ */
79
+ export function placementWalkOf(workflows: readonly LoadedWorkflow[]): PlacementWalk {
80
+ // The walk's own `instanceHarness` (ADR-0068, ADR-0070): the models of the "none" Agents and of
81
+ // every Open body that allows "none", so a Chart's minted "none" Agent has its key placed, or the
82
+ // publish is refused, never a 401 mid-run.
83
+ const { instanceHarness } = partsOf(workflows.map((w) => w.machine));
84
+ return {
85
+ // Per route, not per Machine object: a Machine placed twice is checked where each copy runs.
86
+ effects: workflows.flatMap((w) => placedEffectsOf(w.machine).map((effect) => ({ workflow: w.name, effect }))),
87
+ workspaces: workflows.flatMap((w) => workspacePodsOf(w.machine).map((pod) => ({ workflow: w.name, pod }))),
88
+ ...(instanceHarness ? { instanceHarness: { models: [...instanceHarness.models] } } : {}),
89
+ };
90
+ }
91
+
92
+ export type AssemblyRefusalInput = {
93
+ /** The Assembly's `jr2.config.ts`, loaded; `undefined` when it ships none. */
94
+ config: JR2Config | undefined;
95
+ /** Every Workflow the Assembly registers, loaded. */
96
+ workflows: readonly LoadedWorkflow[];
97
+ /**
98
+ * The Assembly's Chart files (ADR-0070), read: `workflows/<wrapper>/<name>.chart.json`. Each is
99
+ * admitted against its wrapper's Open body by the same door a run's Chart passes. Absent, none.
100
+ */
101
+ charts?: readonly ChartFile[];
102
+ /**
103
+ * The image map the Orchestrator reads per provision (ADR-0038), or a reader of it. Present
104
+ * only when deployed: a host-booted process has no map, and no Sandbox to provision. When
105
+ * present, every `file:` context a Machine names must have its digest in the map.
106
+ */
107
+ images?: () => Promise<ImageRefs>;
108
+ /**
109
+ * The directory the Assembly was imported from. A refusal names a `file:` context relative to
110
+ * it (`images/box`), as the composer's own folder holds it: at the door that directory is a
111
+ * staging path, which is gone when the refusal arrives.
112
+ */
113
+ root?: string;
114
+ };
115
+
116
+ /**
117
+ * The door's walk (ADR-0069): the refusals the Orchestrator can see in an Assembly it has already
118
+ * imported. An import error, a missing `export const machine` and a Vocabulary error are the
119
+ * caller's: they throw at import, before this runs. Here: every Open part, a secret the config does
120
+ * not declare (or a literal one too short to scrub), a `file:` image context whose digest the
121
+ * mounted image map does not hold, and a Chart file that does not admit under its wrapper
122
+ * (ADR-0070), each line naming the file. The door runs this after `jr2 up` did its cluster work, so each
123
+ * check is true when the Assembly lands.
124
+ */
125
+ export async function assemblyRefusals(input: AssemblyRefusalInput): Promise<string[]> {
126
+ const lines = openPartRefusals(input.workflows);
127
+ // A Chart file is the Instance's own (ADR-0070): one that fails refuses the publish.
128
+ if (input.charts?.length) lines.push(...chartFileRefusals(input.charts, input.workflows));
129
+ try {
130
+ refuseEffectSecrets(input.config ?? {}, placementWalkOf(input.workflows));
131
+ } catch (err) {
132
+ lines.push(err instanceof Error ? err.message : String(err));
133
+ }
134
+ const images = partsOf(input.workflows.map((w) => w.machine)).images;
135
+ const where = (dir: string): string => {
136
+ const rel = input.root === undefined ? undefined : relative(input.root, dir);
137
+ return rel !== undefined && rel !== "" && !rel.startsWith("..") && !isAbsolute(rel)
138
+ ? rel.split(sep).join("/")
139
+ : dir;
140
+ };
141
+ if (input.images && images.length) {
142
+ let refs: ImageRefs | undefined;
143
+ try {
144
+ refs = await input.images();
145
+ } catch (err) {
146
+ lines.push(err instanceof Error ? err.message : String(err));
147
+ }
148
+ if (refs) {
149
+ for (const image of images) {
150
+ const digest = await imageContextDigest(image.dir);
151
+ if (refs.sandbox[digest] === undefined) {
152
+ const named = input.workflows
153
+ .filter((w) => partsOf([w.machine]).images.some((i) => i.dir === image.dir))
154
+ .map((w) => `"${w.name}"`);
155
+ const by = named.length ? ` that workflow ${named.join(", ")} names` : "";
156
+ lines.push(
157
+ `the Sandbox Image context ${where(image.dir)}${by} (digest ${digest}) is not in the image map — ` +
158
+ "`jr2 up` builds every `file:` context and writes the map before it publishes (ADR-0038, ADR-0069)",
159
+ );
160
+ }
161
+ }
162
+ }
163
+ }
164
+ return lines;
165
+ }
@@ -0,0 +1,494 @@
1
+ // The Assembly's wire form (ADR-0069): one ustar archive, packed by `jr2 up` and unpacked at the
2
+ // door. The codec is in-process on both ends. No `tar` binary and no subprocess: the Kit image
3
+ // need not ship one, and the host and the door then read one format the same way.
4
+ //
5
+ // The archive is deterministic. Entries are in path order, every mtime is 0, every uid and gid is
6
+ // 0, and owner names are empty. So two packs of one tree are the same bytes. The content address
7
+ // is `assemblyHash` over the tree, not over these bytes, but a stable archive keeps a repeat
8
+ // publish cheap to compare and easy to debug.
9
+ //
10
+ // Modes are kept, so that the unpacked Assembly is a faithful copy of the staged tree. Nothing at
11
+ // the door reads them: not the Orchestrator, not `assemblyHash`, not `imageContextDigest`.
12
+ //
13
+ // Only three kinds of entry exist: directories, regular files, and symlinks with RELATIVE targets
14
+ // that stay inside the tree. An absolute target names host geography, and a target that escapes
15
+ // the tree reaches outside the Assembly. Both are refused, on pack and again on unpack. The door
16
+ // is the trust boundary, so unpack also refuses what pack can never write: a `..` segment, a
17
+ // leading `/`, a duplicate, an entry under a non-directory, a link that escapes through another
18
+ // link.
19
+ //
20
+ // A path that the ustar prefix+name split cannot hold, or a link target over 100 bytes, rides a
21
+ // POSIX pax extended header (typeflag `x`) with only `path` and `linkpath` records. Real npm
22
+ // packages ship such names (`@mistralai/mistralai` has a 130-byte file name). The pax header is
23
+ // deterministic too: sorted records, no time. Unpack accepts an `x` header only immediately before
24
+ // its entry, and checks its values as it checks any path and target. A global header (`g`) and the
25
+ // GNU long-name entries (`L`, `K`) are refused: pack never writes them.
26
+
27
+ import { chmod, lstat, mkdir, open, readdir, readFile, readlink, symlink } from "node:fs/promises";
28
+ import { join } from "node:path";
29
+
30
+ /** What `unpackAssembly` reads: the whole archive, or a stream of it (an HTTP body, a file). */
31
+ export type AssemblyTarInput = Uint8Array | AsyncIterable<Uint8Array>;
32
+
33
+ /** A refusal of one entry, on pack or on unpack. `entry` is the entry's path in the archive, or
34
+ * `""` when the refusal is of the archive as a whole (it ends too early), which names no entry. */
35
+ export class AssemblyTarRefusal extends Error {
36
+ readonly entry: string;
37
+ constructor(entry: string, reason: string) {
38
+ super(
39
+ entry === ""
40
+ ? `the Assembly archive is refused: ${reason} (ADR-0069)`
41
+ : `the Assembly entry ${JSON.stringify(entry)} is refused: ${reason} (ADR-0069)`,
42
+ );
43
+ this.name = "AssemblyTarRefusal";
44
+ this.entry = entry;
45
+ }
46
+ }
47
+
48
+ type EntryType = "dir" | "file" | "symlink";
49
+
50
+ const BLOCK = 512;
51
+ const NAME_MAX = 100;
52
+ const PREFIX_MAX = 155;
53
+ const LINK_MAX = 100;
54
+ // The bounds of one pax value and one pax header. Linux's PATH_MAX, and room for two such records.
55
+ const PAX_VALUE_MAX = 4096;
56
+ const PAX_BODY_MAX = 16 * 1024;
57
+ // The keys pack writes, so the only keys unpack accepts.
58
+ const PAX_KEYS = new Set(["path", "linkpath"]);
59
+ // The size field is 11 octal digits: 8 GiB less one byte.
60
+ const SIZE_MAX = 0o77777777777;
61
+ // More expansions than this while a link target resolves is a loop. Linux stops at 40 too.
62
+ const LINK_EXPANSIONS_MAX = 40;
63
+
64
+ const TYPEFLAG: Record<EntryType, string> = { file: "0", dir: "5", symlink: "2" };
65
+
66
+ // ---------------------------------------------------------------------------------------------
67
+ // Paths
68
+
69
+ /** Byte order per segment, so a directory sorts before its children and the order holds in any
70
+ * locale. */
71
+ function comparePaths(a: string, b: string): number {
72
+ const as = a.split("/");
73
+ const bs = b.split("/");
74
+ for (let i = 0; i < Math.min(as.length, bs.length); i++) {
75
+ const c = Buffer.compare(Buffer.from(as[i]!, "utf8"), Buffer.from(bs[i]!, "utf8"));
76
+ if (c !== 0) return c;
77
+ }
78
+ return as.length - bs.length;
79
+ }
80
+
81
+ /** Refuse a path that is not a plain relative path inside the tree. */
82
+ function checkEntryPath(path: string): void {
83
+ if (path === "") throw new AssemblyTarRefusal(path, "the path is empty");
84
+ if (path.includes("\0")) throw new AssemblyTarRefusal(path, "the path holds a NUL byte");
85
+ if (path.startsWith("/")) throw new AssemblyTarRefusal(path, "the path starts with '/'");
86
+ const segments = path.split("/");
87
+ if (segments.includes("..")) throw new AssemblyTarRefusal(path, "the path holds a '..' segment");
88
+ if (segments.some((s) => s === "" || s === "."))
89
+ throw new AssemblyTarRefusal(path, "the path holds an empty or '.' segment");
90
+ }
91
+
92
+ /** Refuse a symlink target that is absolute, or that is not a usable string. */
93
+ function checkLinkTarget(path: string, target: string): void {
94
+ if (target === "") throw new AssemblyTarRefusal(path, "the symlink has an empty target");
95
+ if (target.includes("\0")) throw new AssemblyTarRefusal(path, "the symlink target holds a NUL byte");
96
+ if (target.startsWith("/"))
97
+ throw new AssemblyTarRefusal(
98
+ path,
99
+ `the symlink target ${JSON.stringify(target)} is absolute; only relative targets travel`,
100
+ );
101
+ if (Buffer.byteLength(target, "utf8") > PAX_VALUE_MAX)
102
+ throw new AssemblyTarRefusal(path, `the symlink target is longer than ${PAX_VALUE_MAX} bytes`);
103
+ }
104
+
105
+ /**
106
+ * Refuse a symlink whose target resolves outside the tree. The walk follows every link the tree
107
+ * holds, so a chain of links that are each inside cannot compose an escape (`s -> .` then
108
+ * `l -> s/..`). A segment that names nothing in the tree is a dangling link, which is allowed.
109
+ */
110
+ function checkLinkInside(path: string, links: Map<string, string>): void {
111
+ const stack = path.split("/").slice(0, -1);
112
+ let pending = (links.get(path) ?? "").split("/");
113
+ let expansions = 0;
114
+ while (pending.length > 0) {
115
+ const seg = pending.shift()!;
116
+ if (seg === "" || seg === ".") continue;
117
+ if (seg === "..") {
118
+ if (stack.length === 0) throw new AssemblyTarRefusal(path, "the symlink target resolves outside the Assembly");
119
+ stack.pop();
120
+ continue;
121
+ }
122
+ stack.push(seg);
123
+ const target = links.get(stack.join("/"));
124
+ if (target === undefined) continue;
125
+ if (++expansions > LINK_EXPANSIONS_MAX) throw new AssemblyTarRefusal(path, "the symlink target is a loop");
126
+ if (target.startsWith("/"))
127
+ throw new AssemblyTarRefusal(path, "the symlink target resolves through an absolute link");
128
+ stack.pop();
129
+ pending = [...target.split("/"), ...pending];
130
+ }
131
+ }
132
+
133
+ /** The ustar prefix+name split of `full` (a directory carries its trailing '/'), or undefined
134
+ * when no split fits. The shortest prefix that fits wins, so the split is deterministic. */
135
+ function splitName(full: string): { prefix: Buffer; name: Buffer } | undefined {
136
+ const bytes = Buffer.from(full, "utf8");
137
+ if (bytes.length <= NAME_MAX) return { prefix: Buffer.alloc(0), name: bytes };
138
+ for (let i = 0; i < bytes.length - 1; i++) {
139
+ if (bytes[i] !== 0x2f) continue;
140
+ if (i > PREFIX_MAX) return undefined;
141
+ const name = bytes.subarray(i + 1);
142
+ // The name part after a trailing-'/' directory split must not be empty.
143
+ if (name.length <= NAME_MAX && name.length > 0 && !(name.length === 1 && name[0] === 0x2f))
144
+ return { prefix: bytes.subarray(0, i), name };
145
+ }
146
+ return undefined;
147
+ }
148
+
149
+ // ---------------------------------------------------------------------------------------------
150
+ // Pack
151
+
152
+ type PackEntry = { path: string; abs: string; type: EntryType; mode: number; target?: string };
153
+
154
+ async function collect(dir: string): Promise<PackEntry[]> {
155
+ const out: PackEntry[] = [];
156
+ const walk = async (absDir: string, relDir: string): Promise<void> => {
157
+ for (const name of await readdir(absDir)) {
158
+ const path = relDir === "" ? name : `${relDir}/${name}`;
159
+ const abs = join(absDir, name);
160
+ const st = await lstat(abs);
161
+ if (st.isDirectory()) {
162
+ out.push({ path, abs, type: "dir", mode: st.mode & 0o7777 });
163
+ await walk(abs, path);
164
+ } else if (st.isFile()) {
165
+ out.push({ path, abs, type: "file", mode: st.mode & 0o7777 });
166
+ } else if (st.isSymbolicLink()) {
167
+ out.push({ path, abs, type: "symlink", mode: 0o777, target: await readlink(abs) });
168
+ } else {
169
+ throw new AssemblyTarRefusal(path, "it is not a directory, a regular file or a symlink");
170
+ }
171
+ }
172
+ };
173
+ await walk(dir, "");
174
+ return out.sort((a, b) => comparePaths(a.path, b.path));
175
+ }
176
+
177
+ function writeOctal(b: Buffer, offset: number, length: number, value: number): void {
178
+ const s = value.toString(8).padStart(length - 1, "0");
179
+ b.write(`${s}\0`, offset, length, "latin1");
180
+ }
181
+
182
+ /** One pax record, `<length> <key>=<value>\n`, where `<length>` counts the whole record. */
183
+ function paxRecord(key: string, value: string): string {
184
+ const rest = ` ${key}=${value}\n`;
185
+ const restBytes = Buffer.byteLength(rest, "utf8");
186
+ let length = restBytes + 1;
187
+ while (String(length).length + restBytes !== length) length = String(length).length + restBytes;
188
+ return `${length}${rest}`;
189
+ }
190
+
191
+ /** The blocks of one entry: a pax header first when the path or the target does not fit ustar. */
192
+ function header(path: string, type: EntryType, mode: number, size: number, target: string): Buffer[] {
193
+ if (Buffer.byteLength(path, "utf8") > PAX_VALUE_MAX)
194
+ throw new AssemblyTarRefusal(path, `the path is longer than ${PAX_VALUE_MAX} bytes`);
195
+ if (size > SIZE_MAX) throw new AssemblyTarRefusal(path, "the file is larger than ustar can hold (8 GiB)");
196
+ const full = type === "dir" ? `${path}/` : path;
197
+ const fits = splitName(full);
198
+ const longTarget = Buffer.byteLength(target, "utf8") > LINK_MAX;
199
+ const out: Buffer[] = [];
200
+ if (fits === undefined || longTarget) {
201
+ // Records in key order, so the bytes are deterministic.
202
+ const body = Buffer.from(
203
+ (longTarget ? paxRecord("linkpath", target) : "") + (fits === undefined ? paxRecord("path", path) : ""),
204
+ "utf8",
205
+ );
206
+ out.push(ustarBlock({ prefix: Buffer.alloc(0), name: Buffer.from("././@PaxHeader") }, "x", 0o644, body.length, ""));
207
+ out.push(body);
208
+ const pad = (BLOCK - (body.length % BLOCK)) % BLOCK;
209
+ if (pad > 0) out.push(Buffer.alloc(pad));
210
+ }
211
+ // The ustar fields then hold a truncated copy. A reader that knows pax ignores it.
212
+ const split = fits ?? { prefix: Buffer.alloc(0), name: Buffer.from(full, "utf8").subarray(-NAME_MAX) };
213
+ const linkField = longTarget ? Buffer.from(target, "utf8").subarray(0, LINK_MAX).toString("latin1") : target;
214
+ out.push(ustarBlock(split, TYPEFLAG[type], mode, size, linkField, longTarget));
215
+ return out;
216
+ }
217
+
218
+ function ustarBlock(
219
+ split: { prefix: Buffer; name: Buffer },
220
+ flag: string,
221
+ mode: number,
222
+ size: number,
223
+ target: string,
224
+ targetIsLatin1 = false,
225
+ ): Buffer {
226
+ const b = Buffer.alloc(BLOCK);
227
+ split.name.copy(b, 0);
228
+ writeOctal(b, 100, 8, mode);
229
+ writeOctal(b, 108, 8, 0); // uid
230
+ writeOctal(b, 116, 8, 0); // gid
231
+ writeOctal(b, 124, 12, size);
232
+ writeOctal(b, 136, 12, 0); // mtime
233
+ b.fill(0x20, 148, 156); // the checksum field counts as spaces while the sum is taken
234
+ b.write(flag, 156, 1, "latin1");
235
+ Buffer.from(target, targetIsLatin1 ? "latin1" : "utf8").copy(b, 157);
236
+ b.write("ustar\0", 257, 6, "latin1");
237
+ b.write("00", 263, 2, "latin1");
238
+ writeOctal(b, 329, 8, 0); // devmajor
239
+ writeOctal(b, 337, 8, 0); // devminor
240
+ split.prefix.copy(b, 345);
241
+ let sum = 0;
242
+ for (const byte of b) sum += byte;
243
+ b.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148, 8, "latin1");
244
+ return b;
245
+ }
246
+
247
+ /**
248
+ * Pack the tree at `dir` into one ustar archive (ADR-0069). The root itself is not an entry.
249
+ * Refuses, naming the entry, anything the door would refuse: a special file, an absolute or
250
+ * escaping symlink target, a path or a target longer than 4096 bytes.
251
+ */
252
+ export async function packAssembly(dir: string): Promise<Buffer> {
253
+ const entries = await collect(dir);
254
+ const links = new Map<string, string>();
255
+ for (const e of entries) if (e.type === "symlink") links.set(e.path, e.target!);
256
+ const parts: Buffer[] = [];
257
+ for (const e of entries) {
258
+ checkEntryPath(e.path);
259
+ if (e.type === "symlink") {
260
+ checkLinkTarget(e.path, e.target!);
261
+ checkLinkInside(e.path, links);
262
+ parts.push(...header(e.path, "symlink", e.mode, 0, e.target!));
263
+ } else if (e.type === "dir") {
264
+ parts.push(...header(e.path, "dir", e.mode, 0, ""));
265
+ } else {
266
+ const body = await readFile(e.abs);
267
+ parts.push(...header(e.path, "file", e.mode, body.length, ""), body);
268
+ const pad = (BLOCK - (body.length % BLOCK)) % BLOCK;
269
+ if (pad > 0) parts.push(Buffer.alloc(pad));
270
+ }
271
+ }
272
+ parts.push(Buffer.alloc(BLOCK * 2));
273
+ return Buffer.concat(parts);
274
+ }
275
+
276
+ // ---------------------------------------------------------------------------------------------
277
+ // Unpack
278
+
279
+ /** Exact-length reads over a chunked source. */
280
+ class ByteReader {
281
+ private chunks: Buffer[] = [];
282
+ private length = 0;
283
+ private done = false;
284
+ private readonly it: AsyncIterator<Uint8Array>;
285
+
286
+ constructor(input: AssemblyTarInput) {
287
+ const source: AsyncIterable<Uint8Array> =
288
+ input instanceof Uint8Array
289
+ ? (async function* () {
290
+ yield input;
291
+ })()
292
+ : input;
293
+ this.it = source[Symbol.asyncIterator]();
294
+ }
295
+
296
+ private async fill(n: number): Promise<void> {
297
+ while (this.length < n && !this.done) {
298
+ const r = await this.it.next();
299
+ if (r.done) {
300
+ this.done = true;
301
+ break;
302
+ }
303
+ const b = Buffer.from(r.value.buffer, r.value.byteOffset, r.value.byteLength);
304
+ if (b.length === 0) continue;
305
+ this.chunks.push(b);
306
+ this.length += b.length;
307
+ }
308
+ }
309
+
310
+ private take(n: number): Buffer {
311
+ const out: Buffer[] = [];
312
+ let need = n;
313
+ while (need > 0) {
314
+ const head = this.chunks[0]!;
315
+ if (head.length <= need) {
316
+ out.push(head);
317
+ this.chunks.shift();
318
+ need -= head.length;
319
+ } else {
320
+ out.push(head.subarray(0, need));
321
+ this.chunks[0] = head.subarray(need);
322
+ need = 0;
323
+ }
324
+ }
325
+ this.length -= n;
326
+ return out.length === 1 ? out[0]! : Buffer.concat(out);
327
+ }
328
+
329
+ /** `n` bytes, or undefined when the source ends first. */
330
+ async read(n: number): Promise<Buffer | undefined> {
331
+ await this.fill(n);
332
+ return this.length >= n ? this.take(n) : undefined;
333
+ }
334
+
335
+ /** At most `max` bytes (at least one), or undefined at the end of the source. */
336
+ async readSome(max: number): Promise<Buffer | undefined> {
337
+ await this.fill(1);
338
+ if (this.length === 0) return undefined;
339
+ return this.take(Math.min(max, this.chunks[0]!.length));
340
+ }
341
+ }
342
+
343
+ function field(b: Buffer, offset: number, length: number): Buffer {
344
+ const raw = b.subarray(offset, offset + length);
345
+ const nul = raw.indexOf(0);
346
+ return nul === -1 ? raw : raw.subarray(0, nul);
347
+ }
348
+
349
+ function readOctal(b: Buffer, offset: number, length: number, path: string, what: string): number {
350
+ const s = field(b, offset, length).toString("latin1").trim();
351
+ if (!/^[0-7]+$/.test(s)) throw new AssemblyTarRefusal(path, `the header's ${what} is not an octal number`);
352
+ return parseInt(s, 8);
353
+ }
354
+
355
+ type Parsed = { path: string; type: EntryType | "pax"; mode: number; size: number; target: string };
356
+
357
+ /** The values a pax header carries for the next entry. */
358
+ type Pax = { path?: string; linkpath?: string };
359
+
360
+ /** Parse a pax header's body. Only `path` and `linkpath`, once each, as pack writes them. */
361
+ function parsePax(body: Buffer, at: string): Pax {
362
+ const pax: Pax = {};
363
+ let rest = body;
364
+ while (rest.length > 0) {
365
+ const space = rest.indexOf(0x20);
366
+ const lengthText = space > 0 ? rest.toString("latin1", 0, space) : "";
367
+ if (!/^[1-9][0-9]*$/.test(lengthText)) throw new AssemblyTarRefusal(at, "a pax record has no length");
368
+ const length = Number(lengthText);
369
+ if (length > rest.length || rest[length - 1] !== 0x0a)
370
+ throw new AssemblyTarRefusal(at, "a pax record's length does not match its bytes");
371
+ const record = rest.subarray(space + 1, length - 1).toString("utf8");
372
+ rest = rest.subarray(length);
373
+ const eq = record.indexOf("=");
374
+ const key = eq > 0 ? record.slice(0, eq) : record;
375
+ if (!PAX_KEYS.has(key))
376
+ throw new AssemblyTarRefusal(at, `the pax key ${JSON.stringify(key)} is not one an Assembly carries`);
377
+ if (pax[key as keyof Pax] !== undefined) throw new AssemblyTarRefusal(at, `the pax key ${key} comes twice`);
378
+ pax[key as keyof Pax] = record.slice(eq + 1);
379
+ }
380
+ return pax;
381
+ }
382
+
383
+ function parseHeader(b: Buffer, pax?: Pax): Parsed {
384
+ const name = field(b, 0, 100).toString("utf8");
385
+ const prefix = field(b, 345, 155).toString("utf8");
386
+ let path = pax?.path ?? (prefix === "" ? name : `${prefix}/${name}`);
387
+ if (b.toString("latin1", 257, 263) !== "ustar\0" || b.toString("latin1", 263, 265) !== "00")
388
+ throw new AssemblyTarRefusal(path, "the header is not ustar");
389
+ let sum = 0;
390
+ for (let i = 0; i < BLOCK; i++) sum += i >= 148 && i < 156 ? 0x20 : b[i]!;
391
+ if (readOctal(b, 148, 8, path, "checksum") !== sum)
392
+ throw new AssemblyTarRefusal(path, "the header checksum does not match");
393
+ const flag = b.toString("latin1", 156, 157);
394
+ let type: Parsed["type"];
395
+ if (flag === "x") type = "pax";
396
+ else if (flag === "0" || flag === "\0") type = "file";
397
+ else if (flag === "5") type = "dir";
398
+ else if (flag === "2") type = "symlink";
399
+ else
400
+ throw new AssemblyTarRefusal(
401
+ path,
402
+ `the entry type ${JSON.stringify(flag)} is not a directory, a regular file or a symlink`,
403
+ );
404
+ if (type === "dir" && path.endsWith("/")) path = path.slice(0, -1);
405
+ const size = readOctal(b, 124, 12, path, "size");
406
+ if (type === "pax") {
407
+ if (pax !== undefined) throw new AssemblyTarRefusal(path, "a pax header follows another pax header");
408
+ if (size > PAX_BODY_MAX) throw new AssemblyTarRefusal(path, `the pax header is larger than ${PAX_BODY_MAX} bytes`);
409
+ } else if (type !== "file" && size !== 0) {
410
+ throw new AssemblyTarRefusal(path, "a directory or symlink entry carries a body");
411
+ }
412
+ if (pax?.linkpath !== undefined && type !== "symlink")
413
+ throw new AssemblyTarRefusal(path, "a pax linkpath names an entry that is not a symlink");
414
+ return {
415
+ path,
416
+ type,
417
+ mode: readOctal(b, 100, 8, path, "mode"),
418
+ size,
419
+ target: pax?.linkpath ?? field(b, 157, 100).toString("utf8"),
420
+ };
421
+ }
422
+
423
+ /**
424
+ * Unpack an Assembly archive into `dir` (ADR-0069). `dir` must be absent or empty. Directories and
425
+ * files land as they come; symlinks land last, after every target has been checked to stay inside
426
+ * `dir`, so no entry is ever written through a link. A refusal names the entry and leaves `dir`
427
+ * partly written: the caller owns `dir` and removes it (the door unpacks into its staging).
428
+ */
429
+ export async function unpackAssembly(input: AssemblyTarInput, dir: string): Promise<void> {
430
+ await mkdir(dir, { recursive: true });
431
+ if ((await readdir(dir)).length > 0) throw new Error(`unpackAssembly: ${dir} is not empty`);
432
+ const reader = new ByteReader(input);
433
+ const seen = new Map<string, EntryType>();
434
+ const links = new Map<string, string>();
435
+ const dirModes: { abs: string; mode: number }[] = [];
436
+ let pax: Pax | undefined;
437
+ for (;;) {
438
+ const block = await reader.read(BLOCK);
439
+ if (block === undefined) throw new AssemblyTarRefusal("", "the archive ends before its end-of-archive blocks");
440
+ if (block.every((byte) => byte === 0)) {
441
+ if (pax !== undefined) throw new AssemblyTarRefusal(pax.path ?? "", "a pax header is the last entry");
442
+ break;
443
+ }
444
+ const e = parseHeader(block, pax);
445
+ pax = undefined;
446
+ if (e.type === "pax") {
447
+ const body = await reader.read(e.size + ((BLOCK - (e.size % BLOCK)) % BLOCK));
448
+ if (body === undefined) throw new AssemblyTarRefusal(e.path, "the archive ends inside a pax header");
449
+ pax = parsePax(body.subarray(0, e.size), e.path);
450
+ continue;
451
+ }
452
+ checkEntryPath(e.path);
453
+ if (seen.has(e.path)) throw new AssemblyTarRefusal(e.path, "the archive holds the path twice");
454
+ const parent = e.path.includes("/") ? e.path.slice(0, e.path.lastIndexOf("/")) : undefined;
455
+ if (parent !== undefined && seen.get(parent) !== "dir")
456
+ throw new AssemblyTarRefusal(
457
+ e.path,
458
+ `its parent ${JSON.stringify(parent)} is not a directory earlier in the archive`,
459
+ );
460
+ seen.set(e.path, e.type);
461
+ const abs = join(dir, ...e.path.split("/"));
462
+ if (e.type === "dir") {
463
+ await mkdir(abs, { mode: 0o700 });
464
+ dirModes.push({ abs, mode: e.mode & 0o7777 });
465
+ } else if (e.type === "symlink") {
466
+ checkLinkTarget(e.path, e.target);
467
+ links.set(e.path, e.target);
468
+ } else {
469
+ const fh = await open(abs, "wx", 0o600);
470
+ try {
471
+ let left = e.size;
472
+ while (left > 0) {
473
+ const chunk = await reader.readSome(left);
474
+ if (chunk === undefined) throw new AssemblyTarRefusal(e.path, "the archive ends inside the file's body");
475
+ await fh.write(chunk);
476
+ left -= chunk.length;
477
+ }
478
+ } finally {
479
+ await fh.close();
480
+ }
481
+ await chmod(abs, e.mode & 0o7777);
482
+ const pad = (BLOCK - (e.size % BLOCK)) % BLOCK;
483
+ if (pad > 0 && (await reader.read(pad)) === undefined)
484
+ throw new AssemblyTarRefusal(e.path, "the archive ends inside the file's padding");
485
+ }
486
+ }
487
+ for (const path of [...links.keys()].sort(comparePaths)) {
488
+ checkLinkInside(path, links);
489
+ await symlink(links.get(path)!, join(dir, ...path.split("/")));
490
+ }
491
+ // Directory modes land last, deepest first, so a read-only directory does not block its own
492
+ // children.
493
+ for (const { abs, mode } of dirModes.reverse()) await chmod(abs, mode);
494
+ }