@packall/core 0.0.1 → 0.2.0

package/src/lockfile/tree-builder.ts

@@ -0,0 +1,166 @@
+/**
+ * The bookkeeping every lockfile parser shares.
+ *
+ * Packages, edges and skip warnings accumulate here while a parser walks a
+ * file, which is what lets each parser stay focused on its own encoding.
+ */
+import semver from "semver";
+
+import { parseDependencyTarget } from "../dependency-range.js";
+import { LockedRootKind } from "../enums/edge-kind.js";
+import type { LockfileFormat } from "../enums/lockfile-format.js";
+import { LockfileError } from "../errors.js";
+import { packageKey } from "../resolve.js";
+import type { LockedEdge, LockedPackage, LockedRoot, LockedTree } from "./types.js";
+
+/**
+ * Which manifest block each root kind is declared in.
+ *
+ * The block names are the same across npm, pnpm and bun; only which of them a
+ * given format writes differs, which is why each parser supplies its own order
+ * rather than iterating this.
+ */
+export const fieldByRootKind = {
+	[LockedRootKind.Prod]: "dependencies",
+	[LockedRootKind.Optional]: "optionalDependencies",
+	[LockedRootKind.Dev]: "devDependencies",
+	[LockedRootKind.Peer]: "peerDependencies",
+} as const satisfies Record<LockedRootKind, string>;
+
+export class TreeBuilder {
+	readonly packages = new Map<string, LockedPackage>();
+	readonly roots: Array<LockedRoot> = [];
+	readonly warnings: Array<string> = [];
+	readonly incomplete: Array<string> = [];
+	/** Names dropped on purpose, so an edge into one is explained, not alarming. */
+	private readonly skipped = new Set<string>();
+	private readonly seenWarnings = new Set<string>();
+
+	add(pkg: LockedPackage): void {
+		const key = packageKey(pkg.name, pkg.version);
+		const existing = this.packages.get(key);
+		if (existing === undefined) {
+			this.packages.set(key, pkg);
+			return;
+		}
+		// The same package at the same version can appear at several paths in a
+		// nested layout. They are the same bytes; merge the edge sets so a copy
+		// recorded with fewer dependencies does not lose any.
+		this.packages.set(key, {
+			...existing,
+			dependencies: mergeEdges(existing.dependencies, pkg.dependencies),
+		});
+	}
+
+	addRoot(root: LockedRoot): void {
+		if (this.roots.some((existing) => existing.name === root.name)) return;
+		this.roots.push(root);
+	}
+
+	warn(message: string): void {
+		if (this.seenWarnings.has(message)) return;
+		this.seenWarnings.add(message);
+		this.warnings.push(message);
+	}
+
+	/** Notes an entry that is real but cannot come from a registry. */
+	skip(name: string, spec: string, reason: string): void {
+		this.skipped.add(name);
+		this.warn(`skipped ${name}@${spec} (${reason}) — not fetchable from a registry`);
+	}
+
+	/**
+	 * Records a required edge that resolved to nothing.
+	 *
+	 * An edge into something skipped on purpose is expected — you cannot bundle a
+	 * `workspace:` sibling from a registry — so it stays a warning. Anything else
+	 * means the file does not pin what it claims to.
+	 */
+	dangling(from: string, name: string): void {
+		if (this.skipped.has(name)) {
+			this.warn(`${from} requires ${name}, which is not fetchable from a registry`);
+			return;
+		}
+		this.incomplete.push(`${name} (required by ${from})`);
+	}
+
+	finish(input: {
+		readonly format: LockfileFormat;
+		readonly lockfileVersion: string;
+		readonly path: string;
+		readonly importer?: string | undefined;
+	}): LockedTree {
+		if (this.packages.size === 0) {
+			throw new LockfileError(input.path, "lockfile pins no packages that can be bundled");
+		}
+		return {
+			format: input.format,
+			lockfileVersion: input.lockfileVersion,
+			path: input.path,
+			importer: input.importer,
+			packages: this.packages,
+			roots: this.roots,
+			warnings: this.warnings,
+			incomplete: this.incomplete,
+		};
+	}
+}
+
+const mergeEdges = (
+	a: ReadonlyArray<LockedEdge>,
+	b: ReadonlyArray<LockedEdge>,
+): ReadonlyArray<LockedEdge> => {
+	const merged = new Map<string, LockedEdge>();
+	for (const edge of [...a, ...b]) {
+		merged.set(`${edge.name}@${edge.version}|${edge.kind}`, edge);
+	}
+	return [...merged.values()];
+};
+
+/**
+ * Reads the right-hand side of a locked entry as an exact version.
+ *
+ * Returns the version when it is one, or the reason it is not — `file:../x`,
+ * `workspace:*`, a git URL. `parseDependencyTarget` already classifies every
+ * one of those for package.json parsing, so the vocabulary of skip reasons
+ * stays identical between the two paths.
+ */
+export const asExactVersion = (
+	name: string,
+	raw: string,
+):
+	| { readonly _tag: "Version"; readonly version: string }
+	| { readonly _tag: "Unsupported"; readonly reason: string } => {
+	const exact = semver.valid(raw, { loose: true });
+	if (exact !== null) return { _tag: "Version", version: exact };
+
+	const target = parseDependencyTarget(name, raw);
+	if (target._tag === "Unsupported") return { _tag: "Unsupported", reason: target.reason };
+	return { _tag: "Unsupported", reason: "not an exact version" };
+};
+
+/**
+ * Raised when a lockfile found by walking up turns out not to cover the
+ * package.json that went looking for it.
+ *
+ * Worth distinguishing from "no lockfile at all": the file exists and is
+ * readable, it simply belongs to a different project, and bundling it would
+ * produce someone else's dependency set.
+ */
+export const notCovered = (path: string, importer: string, known: ReadonlyArray<string>): never => {
+	const listed = known
+		.slice(0, 8)
+		.map((entry) => `    ${entry}`)
+		.join("\n");
+	throw new LockfileError(
+		path,
+		`it does not cover ${importer}.\n` +
+			(known.length === 0
+				? "  It records no workspace members."
+				: `  Workspace members it does record:\n${listed}` +
+					(known.length > 8 ? `\n    … and ${known.length - 8} more` : "")),
+	);
+};
+
+/** `""` is how npm and bun spell the project at the lockfile's own level. */
+export const importerKey = (importer: string): string => (importer === "." ? "" : importer);

package/src/lockfile/types.ts

@@ -0,0 +1,74 @@
+/**
+ * The one output shape every lockfile parser produces.
+ *
+ * npm, pnpm and bun encode "package X at path P depends on Y" in three
+ * different ways; once flattened into this graph, nothing downstream has to
+ * know which manager wrote the file.
+ */
+import type { EdgeKind, LockedRootKind } from "../enums/edge-kind.js";
+import type { LockfileFormat } from "../enums/lockfile-format.js";
+
+/** One edge out of a locked package, already pinned to an exact version. */
+export type LockedEdge = {
+	readonly name: string;
+	readonly version: string;
+	readonly kind: EdgeKind;
+};
+
+/** One package pinned by a lockfile. */
+export type LockedPackage = {
+	readonly name: string;
+	readonly version: string;
+	readonly dependencies: ReadonlyArray<LockedEdge>;
+};
+
+/** A direct dependency of the locked project. */
+export type LockedRoot = {
+	readonly name: string;
+	readonly version: string;
+	readonly kind: LockedRootKind;
+	/** The range the manifest asked for, kept for reporting. */
+	readonly specifier?: string | undefined;
+};
+
+/** Options for reading a lockfile. */
+export type LockfileParseOptions = {
+	/**
+	 * Which workspace member's dependencies become the roots, as a path relative
+	 * to the lockfile in POSIX form — `"packages/core"`, or `"."` for the project
+	 * at the lockfile's own level.
+	 *
+	 * Left undefined, every workspace member in the file contributes, which is
+	 * what pointing `--file` straight at a lockfile should mean. It matters when
+	 * a package.json found a lockfile by walking *up*: the file then pins the
+	 * whole repository, and only one member of it was asked for.
+	 */
+	readonly importer?: string | undefined;
+};
+
+/** A whole lockfile, flattened. */
+export type LockedTree = {
+	readonly format: LockfileFormat;
+	/** As the file declares it: `3`, `'9.0'`, `1`. */
+	readonly lockfileVersion: string;
+	/** Where it was read from, for error messages. */
+	readonly path: string;
+	/** The workspace member the roots came from, when one was selected. */
+	readonly importer?: string | undefined;
+	/** Every pinned package, keyed by `name@version`. */
+	readonly packages: ReadonlyMap<string, LockedPackage>;
+	/** The project's direct dependencies. */
+	readonly roots: ReadonlyArray<LockedRoot>;
+	/** Entries that were recognised but cannot be fetched from a registry. */
+	readonly warnings: ReadonlyArray<string>;
+	/**
+	 * Required edges pointing at nothing the file pins.
+	 *
+	 * Distinct from `warnings`: a `workspace:` dependency that cannot be bundled
+	 * is expected and gets a warning, while an ordinary registry dependency with
+	 * no entry means the lockfile is genuinely partial and the bundle it produces
+	 * would be missing a package. The caller decides what to do about it, but the
+	 * two must not be confused.
+	 */
+	readonly incomplete: ReadonlyArray<string>;
+};

package/src/manifest.ts

@@ -0,0 +1,129 @@
+/**
+ * The manifest written into every bundle — the audit trail, answerable without
+ * unpacking a single tarball. See `notes/requirements.md` § FR-10.
+ */
+import { packagePath } from "./layout.js";
+import type { BundleOptions } from "./options.js";
+import { formatPlatformFilter } from "./platform.js";
+import type { Resolution, ResolvedPackage } from "./resolve.js";
+import { formatSpec } from "./spec.js";
+
+/** Schema version, bumped when the shape changes incompatibly. */
+export const MANIFEST_VERSION = 1;
+
+export type ManifestEntry = {
+	readonly name: string;
+	readonly version: string;
+	/** Path of this tarball inside the bundle. */
+	readonly path: string;
+	/** The URL it was downloaded from. */
+	readonly tarball: string;
+	/** SRI integrity string as advertised by the source registry, when present. */
+	readonly integrity?: string | undefined;
+	/** Legacy sha1, present on older published versions. */
+	readonly shasum?: string | undefined;
+	/** Actual size on disk, in bytes. */
+	readonly bytes?: number | undefined;
+	/** Human-readable justifications: `root:react@18.2.0`, `prod:scheduler@0.23.0`. */
+	readonly reasons: ReadonlyArray<string>;
+};
+
+export type BundleManifest = {
+	readonly manifestVersion: number;
+	readonly tool: { readonly name: string; readonly version: string };
+	readonly createdAt: string;
+	readonly registry: { readonly kind: string; readonly url: string };
+	/** What the user asked for, verbatim. */
+	readonly requested: ReadonlyArray<string>;
+	/** What each request expanded to. */
+	readonly roots: ReadonlyArray<{
+		readonly spec: string;
+		readonly versions: ReadonlyArray<string>;
+		readonly packageCount: number;
+	}>;
+	readonly options: {
+		readonly layout: string;
+		readonly optionalDependencies: boolean;
+		readonly peerDependencies: boolean;
+		readonly platforms: string;
+		readonly allVersions: boolean;
+		readonly includePrerelease: boolean;
+		readonly integrityVerified: boolean;
+	};
+	readonly packages: ReadonlyArray<ManifestEntry>;
+	readonly warnings: ReadonlyArray<string>;
+	readonly totals: {
+		readonly packages: number;
+		readonly bytes: number;
+	};
+};
+
+/** Renders a package's reasons into short, greppable strings. */
+export const formatReasons = (pkg: ResolvedPackage): ReadonlyArray<string> =>
+	pkg.reasons.map((reason) =>
+		reason._tag === "Root" ? `root:${reason.spec}` : `${reason.kind}:${reason.from}`,
+	);
+
+/** Builds the manifest for a completed (or dry) run. */
+export const buildManifest = (input: {
+	readonly resolution: Resolution;
+	/** Only the packages actually included in *this* archive. */
+	readonly included: ReadonlyArray<ResolvedPackage>;
+	readonly options: BundleOptions;
+	readonly registry: { readonly kind: string; readonly url: string };
+	readonly toolVersion: string;
+	readonly sizes: ReadonlyMap<string, number>;
+	readonly createdAt?: Date | undefined;
+}): BundleManifest => {
+	const includedNames = new Set(input.included.map((pkg) => `${pkg.name}@${pkg.version}`));
+
+	const packages: ReadonlyArray<ManifestEntry> = input.included.map((pkg) => {
+		const key = `${pkg.name}@${pkg.version}`;
+		return {
+			name: pkg.name,
+			version: pkg.version,
+			path: packagePath(pkg.name, pkg.version),
+			tarball: pkg.manifest.dist.tarball,
+			integrity: pkg.manifest.dist.integrity,
+			shasum: pkg.manifest.dist.shasum,
+			bytes: input.sizes.get(key),
+			reasons: formatReasons(pkg),
+		};
+	});
+
+	const totalBytes = packages.reduce((sum, entry) => sum + (entry.bytes ?? 0), 0);
+
+	return {
+		manifestVersion: MANIFEST_VERSION,
+		tool: { name: "packall", version: input.toolVersion },
+		createdAt: (input.createdAt ?? new Date()).toISOString(),
+		registry: input.registry,
+		requested: input.resolution.roots.map((root) => formatSpec(root.spec)),
+		roots: input.resolution.roots.map((root) => ({
+			spec: formatSpec(root.spec),
+			versions: root.versions,
+			packageCount: root.closure.filter((key) => includedNames.has(key)).length,
+		})),
+		options: {
+			layout: input.options.layout,
+			optionalDependencies: input.options.scope.optional,
+			peerDependencies: input.options.scope.peer,
+			platforms: formatPlatformFilter(input.options.scope.platforms),
+			allVersions: input.options.allVersions,
+			includePrerelease: input.options.includePrerelease,
+			integrityVerified: input.options.verifyIntegrity,
+		},
+		packages,
+		warnings: input.resolution.warnings.map((warning) =>
+			warning.from === undefined ? warning.message : `${warning.from}: ${warning.message}`,
+		),
+		totals: {
+			packages: packages.length,
+			bytes: totalBytes,
+		},
+	};
+};
+
+/** Serialises a manifest with stable key order and a trailing newline. */
+export const serializeManifest = (manifest: BundleManifest): string =>
+	`${JSON.stringify(manifest, null, 2)}\n`;

package/src/node/archiver.ts

@@ -0,0 +1,85 @@
+/**
+ * The Node `Archiver`, backed by `node-tar`.
+ *
+ * Behind `@packall/core/node` rather than the main barrel: importing
+ * `@packall/core` must not drag `node-tar` — and through it `node:fs` — into a
+ * browser bundle. Anything running on Node reaches for this explicitly.
+ *
+ * Uses `node-tar`, the same library npm itself packs with. Writing a USTAR
+ * encoder by hand would be a fun afternoon and a bad idea: scoped package
+ * paths routinely exceed the 100-byte name field, so correctness depends on
+ * prefix-splitting and PAX headers that `tar` already gets right.
+ */
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import type { PlatformError } from "effect/PlatformError";
+import { create } from "tar";
+
+import type { ArchiveResult, CreateArchiveOptions } from "../archive.js";
+import { Archiver } from "../archive.js";
+import { ArchiveError } from "../errors.js";
+
+const createArchive = (
+	options: CreateArchiveOptions,
+): Effect.Effect<ArchiveResult, ArchiveError | PlatformError, FileSystem.FileSystem> =>
+	Effect.gen(function* () {
+		const fs = yield* FileSystem.FileSystem;
+
+		yield* Effect.tryPromise({
+			try: () =>
+				create(
+					{
+						gzip: options.gzipLevel === undefined ? true : { level: options.gzipLevel },
+						file: options.outPath,
+						cwd: options.cwd,
+						// Normalises uid/gid/mtime so the same inputs produce byte-identical
+						// output — which matters when a security team wants to diff two
+						// bundles or re-derive one from a manifest.
+						portable: true,
+						// Undo the `./` added below, so recorded entry names stay exactly
+						// the registry paths the manifest lists.
+						onWriteEntry: (entry) => {
+							entry.path = entry.path.replace(/^\.\//, "");
+						},
+					},
+					// Sorted so archive order is stable across runs and platforms.
+					options.entries.toSorted().map(dotSlash),
+				),
+			catch: (cause) => new ArchiveError(options.outPath, describe(cause), { cause }),
+		});
+
+		const info = yield* fs.stat(options.outPath);
+
+		return { path: options.outPath, bytes: Number(info.size) };
+	});
+
+/** The Node packing backend. */
+export const layerArchiver: Layer.Layer<Archiver> = Layer.succeed(Archiver)({
+	create: createArchive,
+});
+
+/**
+ * Guards every entry against tar's `@` convention.
+ *
+ * In a tar file list, a leading `@` means "splice in the entries of this other
+ * archive" — a GNU convention `node-tar` implements by stripping the `@` and
+ * looking for what is left. Every scoped package is a top-level entry starting
+ * with `@`, so this hit the bundler in both possible ways:
+ *
+ * - `@oxc-project` became `oxc-project`, which does not exist, and the run died
+ *   with an ENOENT naming a path that appears nowhere in the staging tree.
+ * - `@esbuild` became `esbuild`, which *does* exist — the unscoped package of
+ *   the same name sitting right next to it. No error, and every scoped tarball
+ *   silently missing from a bundle that reported success. That is the dangerous
+ *   one: you would not find out until the install failed behind the firewall.
+ *
+ * `./@scope` is not subject to the convention and resolves identically.
+ *
+ * Any other backend has to answer this question too — the browser writer must
+ * be held to the same test.
+ */
+const dotSlash = (entry: string): string => (entry.startsWith("./") ? entry : `./${entry}`);
+
+const describe = (cause: unknown): string =>
+	cause instanceof Error ? cause.message : String(cause);

package/src/node/index.ts

@@ -0,0 +1,13 @@
+/**
+ * `@packall/core/node` — the engine's Node-specific layers.
+ *
+ * Everything here is separated from the main barrel for one reason: importing
+ * `@packall/core` must stay safe in a browser. The engine itself reaches the
+ * outside world only through services — `Registry`, `Progress`, `Archiver`,
+ * `FileSystem`, `Path`, `Crypto` — so the platform lives at the edges, and this
+ * is the Node edge. A browser supplies its own.
+ *
+ * This is the entry point's whole public API.
+ */
+
+export { layerArchiver } from "./archiver.js";

package/src/options.ts

@@ -0,0 +1,103 @@
+/**
+ * Option types shared by the resolver and the bundler, plus their defaults.
+ *
+ * The rule the defaults below are derived from is stated once, beside the flags
+ * that expose them — see `descriptors.ts` in `@packall/cli`, and
+ * `notes/requirements.md` § FR-13.
+ */
+import { Layout } from "./enums/layout.js";
+import type { PlatformFilter } from "./platform.js";
+import { allPlatforms } from "./platform.js";
+
+/** Which dependency edges the resolver follows. */
+export type DependencyScope = {
+	/**
+	 * Follow `optionalDependencies`. On by default — this is where per-platform
+	 * native binaries live, and omitting them is the most common reason an
+	 * offline install fails on a machine other than the one that built it.
+	 */
+	readonly optional: boolean;
+	/**
+	 * Follow non-optional `peerDependencies`. On by default, because npm 7+
+	 * auto-installs peers and will reach for the network if they are absent.
+	 */
+	readonly peer: boolean;
+	/** Which platforms optional dependencies are kept for. */
+	readonly platforms: PlatformFilter;
+};
+
+export const defaultScope: DependencyScope = {
+	optional: true,
+	peer: true,
+	platforms: allPlatforms,
+};
+
+/** Options governing version selection and the dependency walk. */
+export type ResolveOptions = {
+	readonly scope: DependencyScope;
+	/**
+	 * When a root spec is a range, take *every* published version that satisfies
+	 * it rather than only the best match.
+	 *
+	 * `npmb react@^18` bundles one version; `npmb --all-versions react@^18`
+	 * bundles all of them.
+	 */
+	readonly allVersions: boolean;
+	/** With `allVersions`, keep at most this many (newest first). */
+	readonly maxVersions?: number | undefined;
+	/** Let prereleases satisfy ranges that would not normally admit them. */
+	readonly includePrerelease: boolean;
+	/** How many registry requests to have in flight at once. */
+	readonly concurrency: number;
+};
+
+export const defaultResolveOptions: ResolveOptions = {
+	scope: defaultScope,
+	allVersions: false,
+	includePrerelease: false,
+	concurrency: 10,
+};
+
+/** Options for a full bundle run. */
+export type BundleOptions = ResolveOptions & {
+	readonly layout: Layout;
+	/** Directory the finished artifacts are written to. */
+	readonly outDir: string;
+	/** Base name for the archive in `single` layout. Defaults to `bundle`. */
+	readonly archiveName?: string | undefined;
+	/** Resolve and report, but download nothing and write nothing. */
+	readonly dryRun: boolean;
+	/**
+	 * Verify every tarball against the checksum the registry advertised.
+	 *
+	 * On by default and there is essentially no good reason to turn it off; the
+	 * escape hatch exists only for registries that serve broken metadata.
+	 */
+	readonly verifyIntegrity: boolean;
+	/** Overwrite existing files in `outDir` instead of refusing. */
+	readonly force: boolean;
+	/**
+	 * Planned output files to leave untouched, by file name.
+	 *
+	 * The engine never asks anything; deciding is the caller's job. The CLI's
+	 * `--force prompt` resolves its per-file answers into this set, so "keep the
+	 * existing one" means that artifact is simply not produced, and the rest of
+	 * the run continues.
+	 */
+	readonly skipExisting?: ReadonlySet<string> | undefined;
+	/**
+	 * Planned output files that may be replaced, by file name.
+	 *
+	 * A narrower `force`: `--force tsdown` says "replace tsdown's bundle, and
+	 * still ask about anything else".
+	 */
+	readonly overwrite?: ReadonlySet<string> | undefined;
+};
+
+export const defaultBundleOptions: Omit<BundleOptions, "outDir"> = {
+	...defaultResolveOptions,
+	layout: Layout.PerSpec,
+	dryRun: false,
+	verifyIntegrity: true,
+	force: false,
+};