@packall/core 0.0.1 → 0.2.0

package/src/layout.ts

@@ -0,0 +1,137 @@
+/**
+ * The on-disk shape of a bundle.
+ *
+ * Everything is laid out exactly the way a registry serves it:
+ *
+ *     lodash/-/lodash-4.17.21.tgz
+ *     @babel/core/-/core-7.24.0.tgz
+ *
+ * That is not an Artifactory convention — it is the path structure in every
+ * `dist.tarball` URL npm publishes. See `notes/requirements.md` § FR-12 for
+ * what that buys.
+ *
+ * The one shape worth noticing before editing anything here: a scoped
+ * package's file name drops the scope, so `@babel/core` becomes
+ * `core-7.24.0.tgz` under an `@babel/core/-/` directory.
+ */
+
+/** Splits `@scope/name` into its parts. `scope` is undefined when unscoped. */
+export const splitName = (
+	name: string,
+): { readonly scope: string | undefined; readonly bare: string } => {
+	if (!name.startsWith("@")) return { scope: undefined, bare: name };
+	const slash = name.indexOf("/");
+	if (slash === -1) return { scope: undefined, bare: name };
+	return { scope: name.slice(0, slash), bare: name.slice(slash + 1) };
+};
+
+/** `@babel/core` + `7.24.0` -> `core-7.24.0.tgz` */
+export const tarballFileName = (name: string, version: string): string => {
+	const { bare } = splitName(name);
+	return `${bare}-${version}.tgz`;
+};
+
+/**
+ * The path of one package tarball within a bundle, using `/` separators
+ * regardless of host platform — these become tar entry names, and tar entries
+ * are always POSIX.
+ */
+export const packagePath = (name: string, version: string): string =>
+	`${name}/-/${tarballFileName(name, version)}`;
+
+/** File name of the manifest that ships inside every bundle. */
+export const MANIFEST_FILE = "bundle-manifest.json";
+
+/** File name of the short import guide that ships inside every bundle. */
+export const README_FILE = "IMPORT.md";
+
+/**
+ * Name of the archive produced for one root spec in `per-spec` layout.
+ *
+ * It is the spec, with the one character a file name cannot hold taken out:
+ * `@types/react@19.2.18` becomes `@types-react@19.2.18.tgz`. So the name of the
+ * file answers the question somebody has when they are looking at a directory
+ * of them — which package, and which version — in the notation they would have
+ * typed to ask for it.
+ *
+ * The version is joined with `@` rather than `-` for the same reason. A
+ * hyphen made `foo-bar-1.0.0` ambiguous by eye: package `foo` at `bar-1.0.0`,
+ * or `foo-bar` at `1.0.0`? npm's own syntax has no such problem, because a
+ * package name may not contain `@` beyond its scope.
+ *
+ * **The scope's `@` stays**, and it is the difference between a correct bundle
+ * and a quietly incomplete one.
+ *
+ * Dropping it — which this did, matching what `npm pack` produces — meant
+ * `@something/somewhat` and `something-somewhat` were both
+ * `something-somewhat-1.0.0.tgz`. Two different packages, one path: the second
+ * landed on the first, and the run reported two artifacts for the one file that
+ * survived. A bundle short a package, called complete, found at the far end of
+ * an air gap.
+ *
+ * Keeping the `@` makes that impossible by construction rather than by a check,
+ * because an unscoped npm name may not begin with one. It is also the more
+ * honest name: the scope is part of what the package *is*, and a directory of
+ * archives that has forgotten which ones were scoped is a directory somebody
+ * has to cross-reference against a manifest.
+ *
+ * It does not make every collision impossible — `@a/b-c` and `@a-b/c` still
+ * meet — so `assertDistinctOutputs` still runs. That pair is pathological; the
+ * scoped-against-unscoped one is `@babel/core` beside `babel-core`, which is a
+ * real migration somebody could be bundling.
+ *
+ * The leading `@` is safe here in a way it is not inside the archive: tar's
+ * splice convention applies to the *file list* it is given, which is what
+ * `archive.ts` guards with `./`. This is the output path, not an entry.
+ */
+export const perSpecArchiveName = (name: string, version: string): string => {
+	const { scope, bare } = splitName(name);
+	const prefix = scope === undefined ? bare : `${scope}-${bare}`;
+	return `${prefix}@${version}.tgz`;
+};
+
+/** Name of the archive produced in `single` layout. */
+export const singleArchiveName = (base = "bundle"): string => `${base}.tgz`;
+
+/**
+ * The import guide written into each bundle.
+ *
+ * Kept short and copy-pasteable on purpose: whoever opens this is mid-task on a
+ * restricted network and does not want prose.
+ */
+export const importGuide = (options: {
+	readonly packageCount: number;
+	readonly createdAt: string;
+	readonly toolVersion: string;
+}): string =>
+	`# Importing this bundle
+
+${options.packageCount} package tarball(s), laid out exactly as an npm registry serves them:
+
+    <package-name>/-/<file>.tgz
+    @<scope>/<name>/-/<file>.tgz
+
+Created ${options.createdAt} by packall ${options.toolVersion}.
+See ${MANIFEST_FILE} for the full list with checksums.
+
+## JFrog Artifactory
+
+Unpack, then upload the tree into a **local npm** repository. The layout already
+matches what Artifactory expects, so no path rewriting is needed:
+
+    tar -xzf <this-bundle>.tgz -C ./bundle
+    jf rt upload "bundle/(**)" "<npm-local-repo>/{1}" --flat=false
+
+## Verdaccio / Nexus / any npm registry
+
+Publish each tarball individually:
+
+    find ./bundle -name '*.tgz' -exec npm publish --registry <url> {} \\;
+
+## Verifying before import
+
+    node -e "const m=require('./bundle/${MANIFEST_FILE}');console.log(m.packages.length+' packages')"
+
+Every entry in ${MANIFEST_FILE} carries the integrity string the source registry
+advertised, and each tarball was checked against it at download time.
+`;

package/src/locked-resolve.ts

@@ -0,0 +1,471 @@
+/**
+ * Resolution driven by a lockfile instead of by semver.
+ *
+ * The distinction matters more than it first looks. Feeding a lockfile's
+ * versions in as exact root specs and letting the ordinary walk run would still
+ * produce the wrong bundle: every *transitive* edge is a range, and
+ * `Resolve.walk` answers a range with whatever is newest today. Lock `tslib` at
+ * 2.6.0 and let something depending on `tslib@^2` be re-resolved, and the
+ * bundle grows a second copy at 2.8.1 that CI never installed.
+ *
+ * So the lockfile is treated as the graph, not as a set of hints. The package
+ * set *is* what the file pins, and every edge is satisfied from inside it.
+ * Nothing here calls `selectVersions`; the only thing the registry is asked for
+ * is the metadata that a lockfile does not record — `dist`, and the `os`/`cpu`
+ * constraints that `--platform` narrows on.
+ *
+ * The result is an ordinary `Resolution`, so layout, archiving, the manifest
+ * and the summary cannot tell which path produced it.
+ */
+import * as Effect from "effect/Effect";
+import semver from "semver";
+
+import type { EdgeKind } from "./enums/edge-kind.js";
+import { LockedRootKind } from "./enums/edge-kind.js";
+import { Phase } from "./enums/phase.js";
+import type { BundlerError } from "./errors.js";
+import { LockfileIncompleteError, LockfileOutOfDateError } from "./errors.js";
+import type { LockedRoot, LockedTree } from "./lockfile/types.js";
+import type { ResolveOptions } from "./options.js";
+import { isIncluded } from "./platform.js";
+import * as Progress from "./progress.js";
+import type { PackageManifest } from "./registry.js";
+import { Registry } from "./registry.js";
+import type {
+	PackageKey,
+	Reason,
+	Resolution,
+	ResolutionWarning,
+	ResolvedPackage,
+	RootResolution,
+} from "./resolve.js";
+import { packageKey } from "./resolve.js";
+import type { PackageSpec } from "./spec.js";
+import { formatSpec } from "./spec.js";
+
+/** Options for a lockfile-pinned resolution. */
+export type LockedResolveOptions = ResolveOptions & {
+	/**
+	 * Include the project's `devDependencies`.
+	 *
+	 * Matches `--file package.json`: on by default, because a project without its
+	 * dev tooling does not build behind the firewall either.
+	 */
+	readonly includeDev: boolean;
+	/**
+	 * Direct dependency names declared by the accompanying package.json.
+	 *
+	 * When present, anything declared there but absent from the lockfile's roots
+	 * means the two have drifted — which is exactly the silent-mismatch case this
+	 * whole feature exists to catch, so it is reported rather than papered over.
+	 */
+	readonly declared?: ReadonlyArray<string> | undefined;
+};
+
+/** Turns a parsed lockfile into a resolution, pinning every version. */
+export const resolveLocked = (
+	lock: LockedTree,
+	options: LockedResolveOptions,
+): Effect.Effect<Resolution, BundlerError, Registry | Progress.Progress> =>
+	Effect.gen(function* () {
+		const roots = selectRoots(lock, options);
+
+		yield* Progress.emit({ _tag: "PhaseStarted", phase: Phase.Resolve, total: roots.length });
+
+		const gap = findGaps(lock, roots, options.declared);
+		if (gap !== null) return yield* Effect.fail(gap);
+
+		// Everything the lockfile can reach, before the platform filter has had a
+		// say. Cheap — no network at all — and it is what decides the fetch list.
+		const reachable = reachableKeys(lock, roots, options);
+		const manifests = yield* fetchManifests(lock, reachable, options);
+
+		const warnings: Array<ResolutionWarning> = lock.warnings.map((message) => ({ message }));
+		const resolved = new Map<PackageKey, ResolvedPackage>();
+		const seenWarnings = new Set<string>();
+
+		const addWarning = (warning: ResolutionWarning): void => {
+			const key = `${warning.from ?? ""}|${warning.message}`;
+			if (seenWarnings.has(key)) return;
+			seenWarnings.add(key);
+			warnings.push(warning);
+		};
+
+		const rootResolutions: Array<RootResolution> = [];
+
+		for (const root of roots) {
+			const key = packageKey(root.name, root.version);
+			if (!manifests.has(key)) continue;
+
+			const spec: PackageSpec = {
+				name: root.name,
+				selector: { _tag: "Exact", version: root.version },
+				raw: `${root.name}@${root.version}`,
+			};
+
+			// Platform narrowing applies to optional roots for the same reason it
+			// applies to optional edges: a native binding for another OS is exactly
+			// what --platform is there to drop.
+			if (root.kind === LockedRootKind.Optional && !platformAllows(manifests, key, options))
+				continue;
+
+			const closure = walkLocked({
+				lock,
+				manifests,
+				options,
+				seed: key,
+				rootSpec: formatSpec(spec),
+				resolved,
+				addWarning,
+			});
+
+			rootResolutions.push({
+				spec,
+				versions: [root.version],
+				closures: new Map([[root.version, closure]]),
+				closure,
+			});
+		}
+
+		for (const pkg of resolved.values()) {
+			const deprecated = pkg.manifest.deprecated;
+			if (deprecated !== undefined && deprecated !== "") {
+				addWarning({
+					from: packageKey(pkg.name, pkg.version),
+					message: `${packageKey(pkg.name, pkg.version)} is deprecated: ${deprecated}`,
+				});
+			}
+		}
+
+		yield* Progress.emit({ _tag: "PhaseCompleted", phase: Phase.Resolve });
+
+		return {
+			roots: rootResolutions,
+			packages: [...resolved.values()].toSorted(compareResolved),
+			warnings,
+		};
+	});
+
+/* -------------------------------------------------------------------------- */
+/* Roots                                                                      */
+/* -------------------------------------------------------------------------- */
+
+/** Which of the project's direct dependencies this run should bundle. */
+const selectRoots = (lock: LockedTree, options: LockedResolveOptions): ReadonlyArray<LockedRoot> =>
+	lock.roots
+		.filter((root) => {
+			if (root.kind === LockedRootKind.Dev) return options.includeDev;
+			if (root.kind === LockedRootKind.Optional) return options.scope.optional;
+			if (root.kind === LockedRootKind.Peer) return options.scope.peer;
+			return true;
+		})
+		// Sorted so the artifact order is a property of the dependency set rather
+		// than of which manager wrote the file.
+		.toSorted((a, b) => (a.name === b.name ? 0 : a.name < b.name ? -1 : 1));
+
+/**
+ * The two ways a lockfile can be partial, checked before anything is fetched.
+ *
+ * Both mean the same thing to whoever is waiting on the bundle — a package they
+ * expected will not be in it — and both are invisible unless something says so,
+ * because the run otherwise succeeds and simply produces less.
+ */
+const findGaps = (
+	lock: LockedTree,
+	roots: ReadonlyArray<LockedRoot>,
+	declared: ReadonlyArray<string> | undefined,
+): LockfileIncompleteError | null => {
+	const remedy = `Run \`${lock.format} install\` to bring the lockfile up to date.`;
+
+	// A dependency added to package.json without re-running the install. The
+	// commonest way the two drift, and the cheapest to detect.
+	//
+	// Checked against everything the file pins rather than against its roots,
+	// because the managers disagree about which blocks appear in the root entry —
+	// pnpm's importers omit peers, npm's `packages[""]` includes them — and a
+	// package that is pinned somewhere will be in the bundle either way.
+	if (declared !== undefined && declared.length > 0) {
+		const locked = new Set<string>();
+		for (const pkg of lock.packages.values()) locked.add(pkg.name);
+		for (const root of roots) locked.add(root.name);
+
+		const missing = declared.filter((name) => !locked.has(name));
+		if (missing.length > 0) {
+			return new LockfileIncompleteError(
+				lock.path,
+				missing,
+				`the package.json declares ${missing.length} ` +
+					`dependenc${missing.length === 1 ? "y" : "ies"} it does not pin`,
+				remedy,
+			);
+		}
+	}
+
+	// A required edge inside the file pointing at nothing the file pins.
+	if (lock.incomplete.length > 0) {
+		return new LockfileIncompleteError(
+			lock.path,
+			lock.incomplete,
+			`${lock.incomplete.length} required ` +
+				`dependenc${lock.incomplete.length === 1 ? "y is" : "ies are"} not pinned`,
+			remedy,
+		);
+	}
+
+	return null;
+};
+
+/* -------------------------------------------------------------------------- */
+/* Reachability                                                               */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Every package the roots can reach through the locked edges.
+ *
+ * Scope filters apply here — `--no-optional` genuinely removes a subtree — but
+ * the platform filter does not, because deciding it needs the `os`/`cpu` fields
+ * that only the registry has.
+ */
+const reachableKeys = (
+	lock: LockedTree,
+	roots: ReadonlyArray<LockedRoot>,
+	options: LockedResolveOptions,
+): ReadonlySet<PackageKey> => {
+	const seen = new Set<PackageKey>();
+	const queue = roots.map((root) => packageKey(root.name, root.version));
+
+	while (queue.length > 0) {
+		const key = queue.pop();
+		if (key === undefined || seen.has(key)) continue;
+		seen.add(key);
+
+		for (const edge of lock.packages.get(key)?.dependencies ?? []) {
+			if (!followsEdge(edge.kind, options)) continue;
+			queue.push(packageKey(edge.name, edge.version));
+		}
+	}
+
+	return seen;
+};
+
+const followsEdge = (kind: EdgeKind, options: LockedResolveOptions): boolean => {
+	if (kind === LockedRootKind.Optional) return options.scope.optional;
+	if (kind === LockedRootKind.Peer) return options.scope.peer;
+	return true;
+};
+
+/* -------------------------------------------------------------------------- */
+/* Metadata                                                                   */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Fetches the published manifest for every pinned version.
+ *
+ * One packument per distinct *name*, not per version — a lockfile holding four
+ * versions of `tslib` costs one request. Every pinned version that the registry
+ * does not serve is collected and reported together, because "refresh your
+ * lockfile" is a single action and finding out about the gaps one run at a time
+ * would be miserable.
+ */
+const fetchManifests = (
+	lock: LockedTree,
+	reachable: ReadonlySet<PackageKey>,
+	options: LockedResolveOptions,
+): Effect.Effect<
+	ReadonlyMap<PackageKey, PackageManifest>,
+	BundlerError,
+	Registry | Progress.Progress
+> =>
+	Effect.gen(function* () {
+		const registry = yield* Registry;
+
+		const wanted = new Map<string, Array<string>>();
+		for (const key of reachable) {
+			const pkg = lock.packages.get(key);
+			if (pkg === undefined) continue;
+			const versions = wanted.get(pkg.name);
+			if (versions === undefined) wanted.set(pkg.name, [pkg.version]);
+			else versions.push(pkg.version);
+		}
+
+		const manifests = new Map<PackageKey, PackageManifest>();
+		const missing: Array<{ name: string; version: string; detail: string }> = [];
+
+		const fetched = yield* Effect.forEach(
+			[...wanted.entries()],
+			([name, versions]) =>
+				registry.packument(name).pipe(
+					Effect.map((packument) => ({ name, versions, packument })),
+					// A package the registry has never heard of is the same problem as a
+					// version it no longer serves, and belongs in the same report.
+					Effect.catch((error) =>
+						error._tag === "PackageNotFoundError"
+							? Effect.succeed({ name, versions, packument: null })
+							: Effect.fail(error),
+					),
+				),
+			{ concurrency: options.concurrency },
+		);
+
+		for (const { name, versions, packument } of fetched) {
+			for (const version of versions) {
+				if (packument === null) {
+					missing.push({ name, version, detail: "package not found" });
+					continue;
+				}
+
+				const manifest = packument.versions[version];
+				if (manifest === undefined) {
+					missing.push({
+						name,
+						version,
+						detail: describeAvailable(Object.keys(packument.versions)),
+					});
+					continue;
+				}
+
+				manifests.set(packageKey(name, version), manifest);
+				yield* Progress.emit({
+					_tag: "PackageResolved",
+					name,
+					version,
+					resolvedCount: manifests.size,
+					pendingCount: reachable.size - manifests.size,
+				});
+			}
+		}
+
+		if (missing.length > 0) {
+			return yield* Effect.fail(
+				new LockfileOutOfDateError(
+					lock.path,
+					registry.registryFor(missing[0]?.name ?? ""),
+					missing,
+				),
+			);
+		}
+
+		return manifests;
+	});
+
+const describeAvailable = (available: ReadonlyArray<string>): string => {
+	const recent = available
+		.toSorted((a, b) =>
+			semver.valid(a) && semver.valid(b) ? semver.compare(a, b) : a.localeCompare(b),
+		)
+		.slice(-3);
+	return recent.length === 0
+		? "no published versions"
+		: `no longer published (latest: ${recent.join(", ")})`;
+};
+
+/* -------------------------------------------------------------------------- */
+/* The walk                                                                   */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Collects one root's closure by following locked edges.
+ *
+ * Pure and synchronous — every manifest is already in hand — so a per-root
+ * closure costs nothing beyond the traversal, which is what makes `per-spec`
+ * layout as cheap here as it is for a range-resolved run.
+ */
+const walkLocked = (input: {
+	readonly lock: LockedTree;
+	readonly manifests: ReadonlyMap<PackageKey, PackageManifest>;
+	readonly options: LockedResolveOptions;
+	readonly seed: PackageKey;
+	readonly rootSpec: string;
+	readonly resolved: Map<PackageKey, ResolvedPackage>;
+	readonly addWarning: (warning: ResolutionWarning) => void;
+}): ReadonlyArray<PackageKey> => {
+	const closure = new Set<PackageKey>();
+	const queue: Array<{ key: PackageKey; reason: Reason }> = [
+		{ key: input.seed, reason: { _tag: "Root", spec: input.rootSpec } },
+	];
+
+	while (queue.length > 0) {
+		const next = queue.pop();
+		if (next === undefined) continue;
+
+		const manifest = input.manifests.get(next.key);
+		const locked = input.lock.packages.get(next.key);
+		if (manifest === undefined || locked === undefined) continue;
+
+		record(input.resolved, next.key, manifest, locked, next.reason);
+
+		if (closure.has(next.key)) continue;
+		closure.add(next.key);
+
+		for (const edge of locked.dependencies) {
+			if (!followsEdge(edge.kind, input.options)) continue;
+
+			const target = packageKey(edge.name, edge.version);
+			if (!input.manifests.has(target)) continue;
+
+			if (
+				edge.kind === LockedRootKind.Optional &&
+				!platformAllows(input.manifests, target, input.options)
+			) {
+				input.addWarning({
+					from: next.key,
+					message: `Skipped optional ${target} — it does not build for the selected platforms.`,
+				});
+				continue;
+			}
+
+			queue.push({ key: target, reason: { _tag: "Edge", from: next.key, kind: edge.kind } });
+		}
+	}
+
+	return [...closure];
+};
+
+/** Adds a package, or merges one more justification into an existing entry. */
+const record = (
+	resolved: Map<PackageKey, ResolvedPackage>,
+	key: PackageKey,
+	manifest: PackageManifest,
+	locked: { readonly name: string; readonly version: string },
+	reason: Reason,
+): void => {
+	const existing = resolved.get(key);
+	if (existing === undefined) {
+		resolved.set(key, {
+			name: locked.name,
+			version: locked.version,
+			manifest,
+			reasons: [reason],
+		});
+		return;
+	}
+	if (hasReason(existing.reasons, reason)) return;
+	resolved.set(key, { ...existing, reasons: [...existing.reasons, reason] });
+};
+
+const hasReason = (reasons: ReadonlyArray<Reason>, candidate: Reason): boolean =>
+	reasons.some((reason) => {
+		if (reason._tag === "Root" && candidate._tag === "Root")
+			return reason.spec === candidate.spec;
+		if (reason._tag === "Edge" && candidate._tag === "Edge") {
+			return reason.from === candidate.from && reason.kind === candidate.kind;
+		}
+		return false;
+	});
+
+const platformAllows = (
+	manifests: ReadonlyMap<PackageKey, PackageManifest>,
+	key: PackageKey,
+	options: LockedResolveOptions,
+): boolean => {
+	const manifest = manifests.get(key);
+	if (manifest === undefined) return false;
+	return isIncluded(manifest, options.scope.platforms);
+};
+
+const compareResolved = (a: ResolvedPackage, b: ResolvedPackage): number => {
+	if (a.name !== b.name) return a.name < b.name ? -1 : 1;
+	return semver.valid(a.version) && semver.valid(b.version)
+		? semver.compare(a.version, b.version)
+		: a.version.localeCompare(b.version);
+};