@xemahq/biome-bundle-pack 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +175 -0
- package/README.md +15 -2
- package/package.json +52 -4
- package/src/biome-content.mjs +49 -0
- package/src/bundle-archive.mjs +140 -0
- package/src/bundle-archive.test.mjs +97 -0
- package/src/bundle-build-source.mjs +58 -0
- package/src/bundle-build-source.test.mjs +53 -0
- package/src/bundle-content.mjs +30 -0
- package/src/bundle-manifest.mjs +27 -0
- package/src/bundle-unpack.mjs +114 -0
- package/src/bundle-unpack.test.mjs +63 -0
- package/src/canonical-json.mjs +9 -0
- package/src/host-provided.mjs +160 -0
- package/src/host-provided.test.mjs +128 -0
- package/src/member-required-env.mjs +32 -0
- package/src/pack-directory.mjs +148 -0
- package/src/pack-directory.test.mjs +144 -0
- package/src/seal-bundle.mjs +38 -0
- package/src/stage.mjs +60 -0
- package/src/stage.test.mjs +36 -0
- package/src/target-platform.mjs +35 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// The content digest of a bundle tree, as the kernel contracts define it (`encodeBiomeBundleContentPreimage`): one
|
|
2
|
+
// `{ path, sha256 }` per regular FILE, the manifest excluded because it carries the digest. Symlinks and directories are
|
|
3
|
+
// not listed: the preimage knows files only, so the pnpm layout's links are covered by the archive digest, not by this one.
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
5
|
+
import { readFileSync, readdirSync } from 'node:fs';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
|
|
8
|
+
import { BIOME_BUNDLE_MANIFEST_FILE, encodeBiomeBundleContentPreimage } from '@xemahq/contracts/biome-bundle';
|
|
9
|
+
|
|
10
|
+
/** Lowercase SHA-256 hex of `bytes`. */
|
|
11
|
+
export const sha256Hex = (bytes) => createHash('sha256').update(bytes).digest('hex');
|
|
12
|
+
|
|
13
|
+
/** Every regular file under `dir` except the manifest, as `{ path, sha256 }`, sorted by path. */
|
|
14
|
+
export function bundleContentFiles(dir) {
|
|
15
|
+
const files = [];
|
|
16
|
+
const walk = (prefix) => {
|
|
17
|
+
for (const entry of readdirSync(join(dir, prefix), { withFileTypes: true })) {
|
|
18
|
+
const rel = prefix === '' ? entry.name : `${prefix}/${entry.name}`;
|
|
19
|
+
if (entry.isDirectory()) {walk(rel);}
|
|
20
|
+
else if (entry.isFile() && rel !== BIOME_BUNDLE_MANIFEST_FILE) {files.push({ path: rel, sha256: sha256Hex(readFileSync(join(dir, rel))) });}
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
walk('');
|
|
24
|
+
return files.sort((a, b) => (a.path < b.path ? -1 : 1));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** `sha256:<hex>` of the UTF-8 bytes of the content preimage of the bundle tree at `dir`. */
|
|
28
|
+
export function bundleContentDigest(dir) {
|
|
29
|
+
return `sha256:${sha256Hex(Buffer.from(encodeBiomeBundleContentPreimage(bundleContentFiles(dir)), 'utf8'))}`;
|
|
30
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// The `xema-bundle.json` at a bundle's root: what a host needs to load, migrate and register the bundle's members without
|
|
2
|
+
// opening the tree. One bundle is one biome with all its members. It names the build source (repository and commit) but no
|
|
3
|
+
// timestamp or host path; the commit is outside the content digest, which covers every file but this one. The object is graded by the contracts' own
|
|
4
|
+
// `BiomeBundleManifestSchema`, so the packer cannot write a manifest a host would refuse.
|
|
5
|
+
import { BIOME_BUNDLE_MANIFEST_SCHEMA_VERSION, BiomeBundleManifestSchema } from '@xemahq/contracts/biome-bundle';
|
|
6
|
+
|
|
7
|
+
import { canonicalJson } from './canonical-json.mjs';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The manifest text (canonical JSON, newline-terminated) of the parsed manifest. `members` are `{ member: <componentKey>, entry, migrations?, requiredEnvVars?, kind? }` (the NAMES the entry requires and its `http`/`worker` kind, captured by `memberEntryFacts`) ascending by member, where
|
|
11
|
+
* `migrations` is `{ engine, path }` (bundle-relative); `buildSource` is `{ repository, commit }`; `hostAbi` is `{ id, kernelContracts, identityPackages }`;
|
|
12
|
+
* `platform` is `{ os, cpu, libc }` or absent; `build` is the biome's build facts (`deriveBiomeBuildFacts`), required. Absent optional keys are omitted, never written as null. Paths are bundle-relative.
|
|
13
|
+
*/
|
|
14
|
+
export function bundleManifest({ biome, members, buildSource, hostAbi, contributionKinds, platform, build, contentDigest }) {
|
|
15
|
+
const manifest = BiomeBundleManifestSchema.parse({
|
|
16
|
+
schemaVersion: BIOME_BUNDLE_MANIFEST_SCHEMA_VERSION,
|
|
17
|
+
biome,
|
|
18
|
+
members: members.map(({ member, entry, migrations, requiredEnvVars, kind }) => ({ member, entry, ...(migrations === undefined ? {} : { migrations }), ...(requiredEnvVars === undefined ? {} : { requiredEnvVars }), ...(kind === undefined ? {} : { kind }) })),
|
|
19
|
+
buildSource,
|
|
20
|
+
hostAbi,
|
|
21
|
+
contributionKinds,
|
|
22
|
+
...(platform === undefined ? {} : { platform }),
|
|
23
|
+
build,
|
|
24
|
+
contentDigest,
|
|
25
|
+
});
|
|
26
|
+
return `${canonicalJson(manifest)}\n`;
|
|
27
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// The reader of a bundle archive: the inverse of `canonicalTar` + `gzipCanonical`. It reads exactly what that writer
|
|
2
|
+
// writes (ustar headers, pax `path` and `linkpath` records, directories, files, symlinks) and refuses everything else,
|
|
3
|
+
// so there is ONE bundle format with one writer and one reader. Nothing leaves `destDir`: an absolute or `..` path, a
|
|
4
|
+
// duplicate path, an entry through a symlink and a symlink whose target resolves outside the tree are refused.
|
|
5
|
+
// Loads only `node:` builtins.
|
|
6
|
+
import { mkdirSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
7
|
+
import { dirname, isAbsolute, join, normalize, posix, resolve, sep } from 'node:path';
|
|
8
|
+
import { gunzipSync } from 'node:zlib';
|
|
9
|
+
|
|
10
|
+
import { archiveDigest } from './bundle-archive.mjs';
|
|
11
|
+
|
|
12
|
+
const BLOCK = 512;
|
|
13
|
+
|
|
14
|
+
/** The ustar type flags this reader accepts: what the writer writes. */
|
|
15
|
+
const TypeFlag = Object.freeze({ File: '0', Symlink: '2', Directory: '5', Pax: 'x' });
|
|
16
|
+
|
|
17
|
+
/** The archive bytes do not hash to the digest they were fetched under. */
|
|
18
|
+
export class BundleArchiveDigestMismatchError extends Error {
|
|
19
|
+
constructor(expected, actual) {
|
|
20
|
+
super(`the bundle archive hashes to ${actual}, not the expected ${expected}`);
|
|
21
|
+
this.name = 'BundleArchiveDigestMismatchError';
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const readString = (header, offset, width) => {
|
|
26
|
+
const end = header.indexOf(0, offset);
|
|
27
|
+
return header.subarray(offset, end === -1 || end > offset + width ? offset + width : end).toString('utf8');
|
|
28
|
+
};
|
|
29
|
+
const readOctal = (header, offset, width) => parseInt(readString(header, offset, width).trim() || '0', 8);
|
|
30
|
+
|
|
31
|
+
/** `{ key: value }` of the pax records in `data` (`"<length> <key>=<value>\n"` each). */
|
|
32
|
+
function parsePax(data) {
|
|
33
|
+
const records = {};
|
|
34
|
+
let at = 0;
|
|
35
|
+
while (at < data.length) {
|
|
36
|
+
const space = data.indexOf(0x20, at);
|
|
37
|
+
const length = Number(data.subarray(at, space).toString('latin1'));
|
|
38
|
+
if (!Number.isInteger(length) || length <= 0) {throw new Error('the bundle archive has a malformed pax record');}
|
|
39
|
+
const body = data.subarray(space + 1, at + length - 1).toString('utf8');
|
|
40
|
+
const equals = body.indexOf('=');
|
|
41
|
+
records[body.slice(0, equals)] = body.slice(equals + 1);
|
|
42
|
+
at += length;
|
|
43
|
+
}
|
|
44
|
+
return records;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The entries of an uncompressed tar, in order. */
|
|
48
|
+
function* readEntries(tar) {
|
|
49
|
+
let at = 0;
|
|
50
|
+
let pax = {};
|
|
51
|
+
while (at + BLOCK <= tar.length) {
|
|
52
|
+
const header = tar.subarray(at, at + BLOCK);
|
|
53
|
+
if (header.every((byte) => byte === 0)) {return;}
|
|
54
|
+
const size = readOctal(header, 124, 12);
|
|
55
|
+
const type = String.fromCharCode(header[156]);
|
|
56
|
+
const prefix = readString(header, 345, 155);
|
|
57
|
+
const name = readString(header, 0, 100);
|
|
58
|
+
const data = tar.subarray(at + BLOCK, at + BLOCK + size);
|
|
59
|
+
at += BLOCK + Math.ceil(size / BLOCK) * BLOCK;
|
|
60
|
+
if (type === TypeFlag.Pax) {
|
|
61
|
+
pax = parsePax(data);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (![TypeFlag.File, TypeFlag.Symlink, TypeFlag.Directory].includes(type)) {
|
|
65
|
+
throw new Error(`the bundle archive has an entry of type "${type}", which a bundle never holds`);
|
|
66
|
+
}
|
|
67
|
+
const path = (pax.path ?? (prefix === '' ? name : `${prefix}/${name}`)).replace(/\/$/, '');
|
|
68
|
+
const linkname = pax.linkpath ?? readString(header, 157, 100);
|
|
69
|
+
yield { path, type, mode: readOctal(header, 100, 8), data, linkname };
|
|
70
|
+
pax = {};
|
|
71
|
+
}
|
|
72
|
+
throw new Error('the bundle archive ends without its end-of-archive marker');
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Unpack the canonical bundle archive `archiveBytes` (the `.tgz`) into the existing, empty directory `destDir`.
|
|
77
|
+
* When `expectedDigest` is given the bytes are hashed first and a mismatch throws
|
|
78
|
+
* {@link BundleArchiveDigestMismatchError} before anything is written.
|
|
79
|
+
*/
|
|
80
|
+
export function unpackBundle(archiveBytes, destDir, { expectedDigest } = {}) {
|
|
81
|
+
if (expectedDigest !== undefined) {
|
|
82
|
+
const actual = archiveDigest(archiveBytes);
|
|
83
|
+
if (actual !== expectedDigest) {throw new BundleArchiveDigestMismatchError(expectedDigest, actual);}
|
|
84
|
+
}
|
|
85
|
+
const root = resolve(destDir);
|
|
86
|
+
const seen = new Set();
|
|
87
|
+
const links = new Set();
|
|
88
|
+
for (const entry of readEntries(gunzipSync(archiveBytes))) {
|
|
89
|
+
const relative = normalize(entry.path);
|
|
90
|
+
if (entry.path === '' || isAbsolute(entry.path) || relative === '..' || relative.startsWith(`..${sep}`)) {
|
|
91
|
+
throw new Error(`the bundle archive has an entry that leaves the bundle: "${entry.path}"`);
|
|
92
|
+
}
|
|
93
|
+
if (seen.has(relative)) {throw new Error(`the bundle archive lists "${entry.path}" twice`);}
|
|
94
|
+
seen.add(relative);
|
|
95
|
+
for (let parent = dirname(relative); parent !== '.'; parent = dirname(parent)) {
|
|
96
|
+
if (links.has(parent)) {throw new Error(`the bundle archive writes "${entry.path}" through the symlink "${parent}"`);}
|
|
97
|
+
}
|
|
98
|
+
const target = join(root, relative);
|
|
99
|
+
if (entry.type === TypeFlag.Directory) {
|
|
100
|
+
mkdirSync(target, { recursive: true, mode: 0o755 });
|
|
101
|
+
} else if (entry.type === TypeFlag.Symlink) {
|
|
102
|
+
const resolved = posix.normalize(posix.join(posix.dirname(entry.path), entry.linkname));
|
|
103
|
+
if (posix.isAbsolute(entry.linkname) || resolved === '..' || resolved.startsWith('../')) {
|
|
104
|
+
throw new Error(`the bundle archive has a symlink that leaves the bundle: "${entry.path}" -> "${entry.linkname}"`);
|
|
105
|
+
}
|
|
106
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
107
|
+
symlinkSync(entry.linkname, target);
|
|
108
|
+
links.add(relative);
|
|
109
|
+
} else {
|
|
110
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
111
|
+
writeFileSync(target, entry.data, { mode: (entry.mode & 0o111) === 0 ? 0o644 : 0o755 });
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import test from 'node:test';
|
|
6
|
+
|
|
7
|
+
import { archiveDigest, canonicalTar, gzipCanonical } from './bundle-archive.mjs';
|
|
8
|
+
import { bundleContentDigest } from './bundle-content.mjs';
|
|
9
|
+
import { BundleArchiveDigestMismatchError, unpackBundle } from './bundle-unpack.mjs';
|
|
10
|
+
|
|
11
|
+
const scratch = () => fs.mkdtempSync(path.join(os.tmpdir(), 'xema-unpack-'));
|
|
12
|
+
|
|
13
|
+
function put(root, rel, content, mode) {
|
|
14
|
+
fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true });
|
|
15
|
+
fs.writeFileSync(path.join(root, rel), content);
|
|
16
|
+
if (mode !== undefined) {fs.chmodSync(path.join(root, rel), mode);}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function sampleTree() {
|
|
20
|
+
const tree = scratch();
|
|
21
|
+
put(tree, 'dist/main.js', 'console.log(1);\n');
|
|
22
|
+
put(tree, 'bin/run.sh', '#!/bin/sh\n', 0o755);
|
|
23
|
+
put(tree, `deep/${'long-directory-name-'.repeat(8)}/${'file-name-'.repeat(12)}.txt`, 'long path\n');
|
|
24
|
+
put(tree, 'unicode/café.txt', 'non-ascii path\n');
|
|
25
|
+
fs.mkdirSync(path.join(tree, 'empty'));
|
|
26
|
+
fs.symlinkSync('../dist/main.js', path.join(tree, 'bin', 'main-link'));
|
|
27
|
+
fs.symlinkSync(`x/${'y'.repeat(150)}`, path.join(tree, 'long-link'));
|
|
28
|
+
return tree;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const pack = (tree) => gzipCanonical(canonicalTar(tree));
|
|
32
|
+
|
|
33
|
+
test('a packed tree unpacks to the same content digest, the same modes, links and empty directories, and packs to the same bytes again', () => {
|
|
34
|
+
const tree = sampleTree();
|
|
35
|
+
const archive = pack(tree);
|
|
36
|
+
const out = scratch();
|
|
37
|
+
|
|
38
|
+
unpackBundle(archive, out, { expectedDigest: archiveDigest(archive) });
|
|
39
|
+
|
|
40
|
+
assert.equal(bundleContentDigest(out), bundleContentDigest(tree));
|
|
41
|
+
assert.equal(fs.statSync(path.join(out, 'bin/run.sh')).mode & 0o111, 0o111);
|
|
42
|
+
assert.equal(fs.statSync(path.join(out, 'dist/main.js')).mode & 0o111, 0);
|
|
43
|
+
assert.equal(fs.readlinkSync(path.join(out, 'bin/main-link')), '../dist/main.js');
|
|
44
|
+
assert.equal(fs.readlinkSync(path.join(out, 'long-link')), `x/${'y'.repeat(150)}`);
|
|
45
|
+
assert.deepEqual(fs.readdirSync(path.join(out, 'empty')), []);
|
|
46
|
+
assert.deepEqual(pack(out), archive);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('bytes that do not hash to the expected digest are refused before anything is written', () => {
|
|
50
|
+
const archive = pack(sampleTree());
|
|
51
|
+
const out = scratch();
|
|
52
|
+
|
|
53
|
+
assert.throws(() => unpackBundle(archive, out, { expectedDigest: `sha256:${'0'.repeat(64)}` }), BundleArchiveDigestMismatchError);
|
|
54
|
+
assert.deepEqual(fs.readdirSync(out), []);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('an archive that reaches outside the bundle is refused', () => {
|
|
58
|
+
const tree = scratch();
|
|
59
|
+
fs.symlinkSync('../../etc/passwd', path.join(tree, 'escape'));
|
|
60
|
+
const out = scratch();
|
|
61
|
+
|
|
62
|
+
assert.throws(() => unpackBundle(pack(tree), out), /leaves the bundle/);
|
|
63
|
+
});
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Canonical JSON: keys sorted recursively, no whitespace. */
|
|
2
|
+
export function canonicalJson(value) {
|
|
3
|
+
if (Array.isArray(value)) {return `[${value.map(canonicalJson).join(',')}]`;}
|
|
4
|
+
if (value !== null && typeof value === 'object') {
|
|
5
|
+
return `{${Object.keys(value).sort().map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;
|
|
6
|
+
}
|
|
7
|
+
return JSON.stringify(value);
|
|
8
|
+
}
|
|
9
|
+
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
// What a member bundle must NOT carry: the packages the host provides (`IDENTITY_PACKAGES` of the in-image module:
|
|
2
|
+
// `@nestjs/common`, `@nestjs/core`, `@nestjs/config`, `reflect-metadata` and the service SDK), and everything only they reach.
|
|
3
|
+
// A bundle that shipped its own copy would be a second copy of a singleton beside the host's: `instanceof`, Nest's DI
|
|
4
|
+
// metadata and module identity all break. So the host's packages are cut out of the member's pnpm install tree and the
|
|
5
|
+
// rest is pruned to what the member's own dependencies still reach.
|
|
6
|
+
//
|
|
7
|
+
// The set is the caller's (`hostProvided`): the contracts package names it, this file only cuts it.
|
|
8
|
+
// The cut is by symlink, not by directory name: in a pnpm tree a package is a real directory in
|
|
9
|
+
// `node_modules/.pnpm/<entry>/node_modules/<name>` and every use of it is a symlink to there. Four steps:
|
|
10
|
+
// 1. every `.pnpm` entry whose package is host-provided is deleted, injected `@file+` copies included;
|
|
11
|
+
// 2. every symlink to a host-provided package, in any `node_modules` of the tree (the member's, each entry's, the hidden
|
|
12
|
+
// hoist, the root), is deleted;
|
|
13
|
+
// 3. every `.pnpm` entry the member's `node_modules` no longer reaches by following symlinks is pruned (the host-provided
|
|
14
|
+
// packages' own dependencies, unless the member or another kept package shares them);
|
|
15
|
+
// 4. a symlink left dangling is deleted, and a symlink whose target resolves outside the tree is refused: a bundle must
|
|
16
|
+
// not depend on the machine that packed it.
|
|
17
|
+
// Every walk is sorted, so the same tree gives the same result.
|
|
18
|
+
import { existsSync, lstatSync, readdirSync, readlinkSync, realpathSync, rmSync, unlinkSync } from 'node:fs';
|
|
19
|
+
import { dirname, join, relative, resolve, sep } from 'node:path';
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
/** `<name>` of a `node_modules/.pnpm` entry (`@scope+name@1.0.0_peer@2…` → `@scope/name`). */
|
|
23
|
+
export function pnpmEntryName(entry) {
|
|
24
|
+
const at = entry.indexOf('@', entry.startsWith('@') ? 1 : 0);
|
|
25
|
+
if (at <= 0) {return undefined;}
|
|
26
|
+
const flat = entry.slice(0, at);
|
|
27
|
+
return flat.startsWith('@') ? flat.replace('+', '/') : flat;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const NODE_MODULES = 'node_modules';
|
|
31
|
+
const PNPM_DIRECTORY = '.pnpm';
|
|
32
|
+
|
|
33
|
+
const byName = (a, b) => (a.name < b.name ? -1 : 1);
|
|
34
|
+
const entriesOf = (dir) => readdirSync(dir, { withFileTypes: true }).sort(byName);
|
|
35
|
+
|
|
36
|
+
/** Every entry under `dir`, symlinks reported and never followed, directories first-visited in sorted order: `visit(path, dirent)`. */
|
|
37
|
+
function walk(dir, visit) {
|
|
38
|
+
for (const entry of entriesOf(dir)) {
|
|
39
|
+
const path = join(dir, entry.name);
|
|
40
|
+
visit(path, entry);
|
|
41
|
+
if (entry.isDirectory()) {walk(path, visit);}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The package slots of a `node_modules` directory: `[path, name]` for `<name>` and `@scope/<name>`, sorted; a scope directory is not a slot. */
|
|
46
|
+
function slotsOf(modulesDir) {
|
|
47
|
+
const slots = [];
|
|
48
|
+
for (const entry of entriesOf(modulesDir)) {
|
|
49
|
+
if (entry.name.startsWith('.')) {continue;}
|
|
50
|
+
const path = join(modulesDir, entry.name);
|
|
51
|
+
if (entry.name.startsWith('@') && entry.isDirectory()) {
|
|
52
|
+
for (const inner of entriesOf(path)) {slots.push([join(path, inner.name), `${entry.name}/${inner.name}`]);}
|
|
53
|
+
} else {
|
|
54
|
+
slots.push([path, entry.name]);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return slots;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Where the symlink at `link` points: its target resolved from its real parent, then fully when it exists. */
|
|
61
|
+
function linkTarget(link) {
|
|
62
|
+
const lexical = resolve(realpathSync(dirname(link)), readlinkSync(link));
|
|
63
|
+
return existsSync(link) ? realpathSync(link) : lexical;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const isInside = (root, path) => path === root || path.startsWith(root + sep);
|
|
67
|
+
|
|
68
|
+
/** The `.pnpm` entry a path lies in, or `undefined` (the hidden hoist and anything outside `.pnpm` are no entry). */
|
|
69
|
+
function entryOf(pnpmDir, path) {
|
|
70
|
+
if (!isInside(pnpmDir, path) || path === pnpmDir) {return undefined;}
|
|
71
|
+
const first = relative(pnpmDir, path).split(sep)[0];
|
|
72
|
+
return first === NODE_MODULES ? undefined : first;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Delete every `.pnpm` entry whose package is in `hostProvided`; returns the count. */
|
|
76
|
+
function removeHostEntries(pnpmDir, hostProvided) {
|
|
77
|
+
let removed = 0;
|
|
78
|
+
for (const entry of entriesOf(pnpmDir)) {
|
|
79
|
+
if (!entry.isDirectory() || entry.name === NODE_MODULES || !hostProvided.has(pnpmEntryName(entry.name))) {continue;}
|
|
80
|
+
rmSync(join(pnpmDir, entry.name), { recursive: true });
|
|
81
|
+
removed += 1;
|
|
82
|
+
}
|
|
83
|
+
return removed;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Delete every symlink in a `node_modules` slot of the tree that names a host-provided package; returns the count. */
|
|
87
|
+
function removeHostLinks(treeRoot, hostProvided) {
|
|
88
|
+
const links = [];
|
|
89
|
+
walk(treeRoot, (path, entry) => {
|
|
90
|
+
if (entry.name !== NODE_MODULES || !entry.isDirectory()) {return;}
|
|
91
|
+
for (const [slot, name] of slotsOf(path)) {
|
|
92
|
+
if (hostProvided.has(name) && lstatSync(slot).isSymbolicLink()) {links.push(slot);}
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
for (const link of links) {unlinkSync(link);}
|
|
96
|
+
return links.length;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The `.pnpm` entries the members' `node_modules` (`memberModules`) reach through symlinks, a reached entry's own `node_modules` siblings being its dependencies. */
|
|
100
|
+
function reachableEntries({ pnpmDir, memberModules }) {
|
|
101
|
+
const reached = new Set();
|
|
102
|
+
const pending = [];
|
|
103
|
+
const follow = (slotPath) => {
|
|
104
|
+
if (!lstatSync(slotPath).isSymbolicLink()) {return;}
|
|
105
|
+
const entry = entryOf(pnpmDir, linkTarget(slotPath));
|
|
106
|
+
if (entry !== undefined && !reached.has(entry)) {
|
|
107
|
+
reached.add(entry);
|
|
108
|
+
pending.push(entry);
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
for (const modules of memberModules.filter((dir) => existsSync(dir))) {for (const [slot] of slotsOf(modules)) {follow(slot);}}
|
|
112
|
+
while (pending.length > 0) {
|
|
113
|
+
const modules = join(pnpmDir, pending.pop(), NODE_MODULES);
|
|
114
|
+
if (existsSync(modules)) {for (const [slot] of slotsOf(modules)) {follow(slot);}}
|
|
115
|
+
}
|
|
116
|
+
return reached;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Delete the `.pnpm` entries not in `reached`; returns the count. */
|
|
120
|
+
function pruneUnreached(pnpmDir, reached) {
|
|
121
|
+
let pruned = 0;
|
|
122
|
+
for (const entry of entriesOf(pnpmDir)) {
|
|
123
|
+
if (!entry.isDirectory() || entry.name === NODE_MODULES || reached.has(entry.name)) {continue;}
|
|
124
|
+
rmSync(join(pnpmDir, entry.name), { recursive: true });
|
|
125
|
+
pruned += 1;
|
|
126
|
+
}
|
|
127
|
+
return pruned;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Delete every dangling symlink of the tree and refuse one that points outside it; returns the deleted count. */
|
|
131
|
+
function removeDangling(treeRoot) {
|
|
132
|
+
const real = realpathSync(treeRoot);
|
|
133
|
+
const dangling = [];
|
|
134
|
+
const escaping = [];
|
|
135
|
+
walk(treeRoot, (path, entry) => {
|
|
136
|
+
if (!entry.isSymbolicLink()) {return;}
|
|
137
|
+
if (!isInside(real, linkTarget(path))) {escaping.push(`${relative(treeRoot, path)} -> ${readlinkSync(path)}`);}
|
|
138
|
+
else if (!existsSync(path)) {dangling.push(path);}
|
|
139
|
+
});
|
|
140
|
+
if (escaping.length > 0) {
|
|
141
|
+
throw new Error(`the bundle tree holds ${escaping.length} symlink(s) that point outside it:\n${escaping.map((p) => ` - ${p}`).join('\n')}`);
|
|
142
|
+
}
|
|
143
|
+
for (const link of dangling) {unlinkSync(link);}
|
|
144
|
+
return dangling.length;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Cut the host-provided packages (`hostProvided`: package names, the caller's: this module owns no default) out of the pnpm
|
|
149
|
+
* install tree under `treeRoot` and prune what only they reached from every `<treeRoot>/<memberDirectory>/node_modules`.
|
|
150
|
+
* Returns `{ removedLinks, removedEntries, pruned }`.
|
|
151
|
+
*/
|
|
152
|
+
export function excludeHostProvided({ treeRoot, memberDirectories, hostProvided }) {
|
|
153
|
+
const provided = new Set(hostProvided);
|
|
154
|
+
const pnpmDir = join(realpathSync(treeRoot), NODE_MODULES, PNPM_DIRECTORY);
|
|
155
|
+
const hasStore = existsSync(pnpmDir);
|
|
156
|
+
const removedEntries = hasStore ? removeHostEntries(pnpmDir, provided) : 0;
|
|
157
|
+
const removedLinks = removeHostLinks(treeRoot, provided);
|
|
158
|
+
const pruned = hasStore ? pruneUnreached(pnpmDir, reachableEntries({ pnpmDir, memberModules: memberDirectories.map((directory) => join(realpathSync(treeRoot), directory, NODE_MODULES)) })) : 0;
|
|
159
|
+
return { removedLinks: removedLinks + removeDangling(treeRoot), removedEntries, pruned };
|
|
160
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import test from 'node:test';
|
|
6
|
+
|
|
7
|
+
import { excludeHostProvided, pnpmEntryName } from './host-provided.mjs';
|
|
8
|
+
|
|
9
|
+
const IDENTITY_PACKAGES = ['@nestjs/common', '@nestjs/core', '@nestjs/config', 'reflect-metadata', '@xemahq/xema-service-nest'];
|
|
10
|
+
|
|
11
|
+
const MEMBER = 'biomes/alpha/api/alpha-api';
|
|
12
|
+
|
|
13
|
+
function pkg(tree, entry, name) {
|
|
14
|
+
const dir = path.join(tree, 'node_modules/.pnpm', entry, 'node_modules', name);
|
|
15
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
16
|
+
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name }));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** `<from>/<name>` links to `<to>`, a path relative to the directory holding the link. */
|
|
20
|
+
function link(from, name, to) {
|
|
21
|
+
const at = path.join(from, name);
|
|
22
|
+
fs.mkdirSync(path.dirname(at), { recursive: true });
|
|
23
|
+
fs.symlinkSync(to, at);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const inEntry = (tree, entry) => path.join(tree, 'node_modules/.pnpm', entry, 'node_modules');
|
|
27
|
+
|
|
28
|
+
/** A pnpm-shaped tree: the member links its deps; @nestjs/core reaches `core-only` and the shared dep; the SDK reaches `sdk-only`. */
|
|
29
|
+
function fixture() {
|
|
30
|
+
const tree = fs.mkdtempSync(path.join(os.tmpdir(), 'xema-host-provided-'));
|
|
31
|
+
const store = path.join(tree, 'node_modules/.pnpm');
|
|
32
|
+
const entries = {
|
|
33
|
+
'app-dep@1.0.0': 'app-dep',
|
|
34
|
+
'shared@1.0.0': 'shared',
|
|
35
|
+
'with-peer@1.0.0_@nestjs+common@10.0.0': 'with-peer',
|
|
36
|
+
'@nestjs+common@10.0.0': '@nestjs/common',
|
|
37
|
+
'@nestjs+core@10.0.0': '@nestjs/core',
|
|
38
|
+
'core-only@1.0.0': 'core-only',
|
|
39
|
+
'common-only@1.0.0': 'common-only',
|
|
40
|
+
'@xemahq+xema-service-nest@file+packages+sdk': '@xemahq/xema-service-nest',
|
|
41
|
+
'sdk-only@1.0.0': 'sdk-only',
|
|
42
|
+
};
|
|
43
|
+
for (const [entry, name] of Object.entries(entries)) {pkg(tree, entry, name);}
|
|
44
|
+
const dep = (entry, name, target, targetName = target.split('@')[0]) =>
|
|
45
|
+
link(inEntry(tree, entry), name, `${'../'.repeat(name.split('/').length)}../${target}/node_modules/${targetName}`);
|
|
46
|
+
dep('app-dep@1.0.0', 'shared', 'shared@1.0.0');
|
|
47
|
+
dep('with-peer@1.0.0_@nestjs+common@10.0.0', '@nestjs/common', '@nestjs+common@10.0.0', '@nestjs/common');
|
|
48
|
+
dep('@nestjs+common@10.0.0', 'common-only', 'common-only@1.0.0');
|
|
49
|
+
dep('@nestjs+core@10.0.0', '@nestjs/common', '@nestjs+common@10.0.0', '@nestjs/common');
|
|
50
|
+
dep('@nestjs+core@10.0.0', 'core-only', 'core-only@1.0.0');
|
|
51
|
+
dep('@nestjs+core@10.0.0', 'shared', 'shared@1.0.0');
|
|
52
|
+
dep('@xemahq+xema-service-nest@file+packages+sdk', 'sdk-only', 'sdk-only@1.0.0');
|
|
53
|
+
dep('@xemahq+xema-service-nest@file+packages+sdk', '@nestjs/core', '@nestjs+core@10.0.0', '@nestjs/core');
|
|
54
|
+
|
|
55
|
+
const member = path.join(tree, MEMBER, 'node_modules');
|
|
56
|
+
const up = '../'.repeat(MEMBER.split('/').length + 1);
|
|
57
|
+
const fromMember = (name, entry, target = name) => link(member, name, `${up}node_modules/.pnpm/${entry}/node_modules/${target}`);
|
|
58
|
+
fromMember('app-dep', 'app-dep@1.0.0');
|
|
59
|
+
fromMember('shared', 'shared@1.0.0');
|
|
60
|
+
fromMember('with-peer', 'with-peer@1.0.0_@nestjs+common@10.0.0');
|
|
61
|
+
fromMember('@nestjs/common', '@nestjs+common@10.0.0');
|
|
62
|
+
fromMember('@nestjs/core', '@nestjs+core@10.0.0');
|
|
63
|
+
fromMember('@xemahq/xema-service-nest', '@xemahq+xema-service-nest@file+packages+sdk');
|
|
64
|
+
fs.mkdirSync(path.join(member, '.prisma/client'), { recursive: true });
|
|
65
|
+
|
|
66
|
+
const hoist = path.join(store, 'node_modules');
|
|
67
|
+
link(hoist, 'shared', '../shared@1.0.0/node_modules/shared');
|
|
68
|
+
link(hoist, '@nestjs/common', '../../@nestjs+common@10.0.0/node_modules/@nestjs/common');
|
|
69
|
+
link(hoist, 'core-only', '../core-only@1.0.0/node_modules/core-only');
|
|
70
|
+
link(path.join(tree, 'node_modules'), '@nestjs/core', '../.pnpm/@nestjs+core@10.0.0/node_modules/@nestjs/core');
|
|
71
|
+
return tree;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const exists = (p) => fs.lstatSync(p, { throwIfNoEntry: false }) !== undefined;
|
|
75
|
+
|
|
76
|
+
function danglingLinks(dir) {
|
|
77
|
+
const found = [];
|
|
78
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
79
|
+
const p = path.join(dir, entry.name);
|
|
80
|
+
if (entry.isSymbolicLink()) {if (!fs.existsSync(p)) {found.push(p);}}
|
|
81
|
+
else if (entry.isDirectory()) {found.push(...danglingLinks(p));}
|
|
82
|
+
}
|
|
83
|
+
return found;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
test('the host-provided packages, their links and what only they reached are gone; ordinary deps stay', () => {
|
|
87
|
+
const tree = fixture();
|
|
88
|
+
const counts = excludeHostProvided({ treeRoot: tree, memberDirectories: [MEMBER], hostProvided: IDENTITY_PACKAGES });
|
|
89
|
+
const store = path.join(tree, 'node_modules/.pnpm');
|
|
90
|
+
const member = path.join(tree, MEMBER, 'node_modules');
|
|
91
|
+
|
|
92
|
+
for (const gone of ['@nestjs+common@10.0.0', '@nestjs+core@10.0.0', '@xemahq+xema-service-nest@file+packages+sdk']) {assert.ok(!exists(path.join(store, gone)), gone);}
|
|
93
|
+
for (const gone of ['@nestjs/common', '@nestjs/core', '@xemahq/xema-service-nest']) {assert.ok(!exists(path.join(member, gone)), `member link ${gone}`);}
|
|
94
|
+
assert.ok(!exists(path.join(store, 'node_modules/@nestjs/common')), 'hidden hoist link');
|
|
95
|
+
assert.ok(!exists(path.join(tree, 'node_modules/@nestjs/core')), 'root link');
|
|
96
|
+
assert.ok(!exists(path.join(inEntry(tree, 'with-peer@1.0.0_@nestjs+common@10.0.0'), '@nestjs/common')), 'peer link');
|
|
97
|
+
|
|
98
|
+
for (const pruned of ['core-only@1.0.0', 'common-only@1.0.0', 'sdk-only@1.0.0']) {assert.ok(!exists(path.join(store, pruned)), `${pruned} was reached only through a host package`);}
|
|
99
|
+
assert.ok(!exists(path.join(store, 'node_modules/core-only')), 'its hoist link is not left dangling');
|
|
100
|
+
|
|
101
|
+
for (const kept of ['app-dep@1.0.0', 'shared@1.0.0', 'with-peer@1.0.0_@nestjs+common@10.0.0']) {assert.ok(exists(path.join(store, kept)), kept);}
|
|
102
|
+
for (const kept of ['app-dep', 'shared', 'with-peer']) {assert.ok(fs.existsSync(path.join(member, kept, 'package.json')), `member keeps ${kept}`);}
|
|
103
|
+
assert.ok(fs.existsSync(path.join(inEntry(tree, 'app-dep@1.0.0'), 'shared/package.json')), 'a dep shared with @nestjs/core stays reachable');
|
|
104
|
+
assert.ok(exists(path.join(member, '.prisma/client')));
|
|
105
|
+
assert.deepEqual(danglingLinks(tree), []);
|
|
106
|
+
assert.equal(counts.removedEntries, 3);
|
|
107
|
+
assert.equal(counts.pruned, 3);
|
|
108
|
+
assert.ok(counts.removedLinks >= 6);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test('the result is the same when run twice', () => {
|
|
112
|
+
const tree = fixture();
|
|
113
|
+
excludeHostProvided({ treeRoot: tree, memberDirectories: [MEMBER], hostProvided: IDENTITY_PACKAGES });
|
|
114
|
+
assert.deepEqual(excludeHostProvided({ treeRoot: tree, memberDirectories: [MEMBER], hostProvided: IDENTITY_PACKAGES }), { removedLinks: 0, removedEntries: 0, pruned: 0 });
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test('a symlink that points outside the tree is refused', () => {
|
|
118
|
+
const tree = fixture();
|
|
119
|
+
const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'xema-outside-'));
|
|
120
|
+
fs.symlinkSync(outside, path.join(tree, MEMBER, 'node_modules/escape'));
|
|
121
|
+
assert.throws(() => excludeHostProvided({ treeRoot: tree, memberDirectories: [MEMBER], hostProvided: IDENTITY_PACKAGES }), /point outside it:\n {2}- .*escape/);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test('pnpmEntryName reads scoped, unscoped and peer-suffixed .pnpm entries', () => {
|
|
125
|
+
assert.equal(pnpmEntryName('prisma@7.8.0_@types+react@19.3.0'), 'prisma');
|
|
126
|
+
assert.equal(pnpmEntryName('@prisma+client@7.8.0_prisma@7.8.0'), '@prisma/client');
|
|
127
|
+
assert.equal(pnpmEntryName('node_modules'), undefined);
|
|
128
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// The NAMES of the environment a built member entry requires, read from the entry itself: its `main.js` passes them to
|
|
2
|
+
// `bootstrapXemaService`, and the member's own copy of `@xemahq/xema-service-nest` captures that call without booting
|
|
3
|
+
// (`readServiceEntry`). The same capture the seed closure derivation uses, so the code stays the one declaration.
|
|
4
|
+
import { createRequire } from 'node:module';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
|
|
7
|
+
import { MemberRequiredEnvVarsSchema, ProcessMemberKindSchema, processMemberKindOfCapture } from '@xemahq/contracts/process-topology';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* What ONE capture of the entry at `entryPath` yields, resolved through the SDK copy installed beside `packageRoot` (the
|
|
11
|
+
* directory holding the member's `package.json` and `node_modules`): the sorted, de-duplicated `requiredEnvVars` (names only; the
|
|
12
|
+
* schema refuses a NAME=value) and the member's `kind`, from which bootstrap function the entry calls (`entry.kind`).
|
|
13
|
+
*/
|
|
14
|
+
export function memberEntryFacts({ packageRoot, entryPath }) {
|
|
15
|
+
const memberRequire = createRequire(join(packageRoot, 'package.json'));
|
|
16
|
+
let serviceNest;
|
|
17
|
+
try {
|
|
18
|
+
serviceNest = memberRequire('@xemahq/xema-service-nest');
|
|
19
|
+
} catch (cause) {
|
|
20
|
+
throw new Error(`cannot read the required environment of ${entryPath}: @xemahq/xema-service-nest is not installed beside ${packageRoot}`, { cause });
|
|
21
|
+
}
|
|
22
|
+
const captured = serviceNest.readServiceEntry(entryPath, {});
|
|
23
|
+
return {
|
|
24
|
+
requiredEnvVars: MemberRequiredEnvVarsSchema.parse([...new Set(captured.entry.options.requiredEnvVars ?? [])].sort()),
|
|
25
|
+
kind: ProcessMemberKindSchema.parse(processMemberKindOfCapture(captured.entry.kind)),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The `requiredEnvVars` of {@link memberEntryFacts}. */
|
|
30
|
+
export function memberRequiredEnvVars(input) {
|
|
31
|
+
return memberEntryFacts(input).requiredEnvVars;
|
|
32
|
+
}
|