@packall/core 0.0.1 → 0.2.0

package/src/resolve.ts

@@ -0,0 +1,560 @@
+/**
+ * Version selection and the dependency closure walk.
+ *
+ * This is the part that decides *what* ends up in a bundle. Its only side
+ * effect is asking the `Registry` service for metadata, which makes the whole
+ * of it testable against a fake registry with no network at all.
+ *
+ * The walk is breadth-first in waves: resolve the current frontier
+ * concurrently, collect the newly discovered edges, repeat. Wave-based BFS
+ * gives real parallelism without a dynamic worker pool, and because each wave
+ * merges its results serially there is no shared mutable state being written
+ * from more than one fiber.
+ *
+ * One subtlety worth calling out: metadata is cached across walks, but each
+ * walk still traverses the full graph. That is what lets `per-spec` layout ask
+ * "what does *this* root version depend on" many times over while the network
+ * is touched exactly once per package.
+ */
+import * as Effect from "effect/Effect";
+import semver from "semver";
+
+import { parseDependencyTarget } from "./dependency-range.js";
+import { EdgeKind } from "./enums/edge-kind.js";
+import { Phase } from "./enums/phase.js";
+import type { BundlerError } from "./errors.js";
+import { NoMatchingVersionsError, VersionNotFoundError } from "./errors.js";
+import type { ResolveOptions } from "./options.js";
+import { isIncluded } from "./platform.js";
+import * as Progress from "./progress.js";
+import type { PackageManifest, Packument } from "./registry.js";
+import { Registry } from "./registry.js";
+import type { PackageSpec, Selector } from "./spec.js";
+import { formatSelector, formatSpec } from "./spec.js";
+
+/** A single justification for including a package. */
+export type Reason =
+	| { readonly _tag: "Root"; readonly spec: string }
+	| { readonly _tag: "Edge"; readonly from: string; readonly kind: EdgeKind };
+
+/** `name@version` — the identity of a resolved package throughout the engine. */
+export type PackageKey = string;
+
+/** One package, pinned to one version, with every reason it was pulled in. */
+export type ResolvedPackage = {
+	readonly name: string;
+	readonly version: string;
+	readonly manifest: PackageManifest;
+	readonly reasons: ReadonlyArray<Reason>;
+};
+
+/** Something we could not include, reported rather than thrown. */
+export type ResolutionWarning = {
+	readonly message: string;
+	/** `name@version` of the package whose manifest contained the problem. */
+	readonly from?: string | undefined;
+};
+
+/** What one requested spec expanded to. */
+export type RootResolution = {
+	readonly spec: PackageSpec;
+	/** The concrete versions the spec selected — more than one under `--all-versions`. */
+	readonly versions: ReadonlyArray<string>;
+	/** Closure per selected version, keyed by version. Drives `per-spec` layout. */
+	readonly closures: ReadonlyMap<string, ReadonlyArray<PackageKey>>;
+	/** Union of every per-version closure. */
+	readonly closure: ReadonlyArray<PackageKey>;
+};
+
+/** The complete result of resolving every requested spec. */
+export type Resolution = {
+	readonly roots: ReadonlyArray<RootResolution>;
+	/** The deduplicated union of every closure, sorted by name then version. */
+	readonly packages: ReadonlyArray<ResolvedPackage>;
+	readonly warnings: ReadonlyArray<ResolutionWarning>;
+};
+
+/** One pending item on the BFS frontier. */
+export type Task = {
+	readonly name: string;
+	readonly selector: Selector;
+	readonly reason: Reason;
+	/** Optional edges tolerate resolution failure; required ones do not. */
+	readonly tolerateFailure: boolean;
+};
+
+export const packageKey = (name: string, version: string): PackageKey => `${name}@${version}`;
+
+/** Index of a resolution by `name@version`, for lookups during bundling. */
+export const indexPackages = (resolution: Resolution): ReadonlyMap<PackageKey, ResolvedPackage> =>
+	new Map(resolution.packages.map((pkg) => [packageKey(pkg.name, pkg.version), pkg]));
+
+/* -------------------------------------------------------------------------- */
+/* Version selection                                                          */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Turns a selector into the concrete version(s) it names.
+ *
+ * `all` controls whether a range collapses to its best match — npm's behaviour,
+ * and what you always want for a transitive dependency — or expands to every
+ * satisfying published version, which is what `--all-versions` is for.
+ */
+export const selectVersions = (
+	packument: Packument,
+	selector: Selector,
+	options: {
+		readonly all: boolean;
+		readonly maxVersions?: number | undefined;
+		readonly includePrerelease: boolean;
+	},
+): ReadonlyArray<string> => {
+	const available = Object.keys(packument.versions);
+
+	switch (selector._tag) {
+		case "Exact": {
+			if (!Object.hasOwn(packument.versions, selector.version)) {
+				throw new VersionNotFoundError(
+					packument.name,
+					selector.version,
+					sortVersions(available),
+				);
+			}
+			return [selector.version];
+		}
+
+		case "Tag": {
+			const version = packument.distTags[selector.tag];
+			if (version === undefined || !Object.hasOwn(packument.versions, version)) {
+				throw new NoMatchingVersionsError(
+					packument.name,
+					selector.tag,
+					sortVersions(available),
+				);
+			}
+			return [version];
+		}
+
+		case "Range": {
+			const satisfying = available
+				.filter((version) =>
+					semver.satisfies(version, selector.range, {
+						loose: true,
+						includePrerelease: options.includePrerelease,
+					}),
+				)
+				.toSorted(semver.rcompare);
+
+			const [best] = satisfying;
+			if (best === undefined) {
+				throw new NoMatchingVersionsError(
+					packument.name,
+					selector.range,
+					sortVersions(available),
+				);
+			}
+
+			if (!options.all) return [best];
+
+			return options.maxVersions !== undefined && options.maxVersions > 0
+				? satisfying.slice(0, options.maxVersions)
+				: satisfying;
+		}
+	}
+};
+
+const sortVersions = (versions: ReadonlyArray<string>): ReadonlyArray<string> =>
+	versions.toSorted((a, b) =>
+		semver.valid(a) && semver.valid(b) ? semver.compare(a, b) : a.localeCompare(b),
+	);
+
+/* -------------------------------------------------------------------------- */
+/* Walk state                                                                 */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * State shared across every walk in a run.
+ *
+ * Caching packuments and expanded edges here is what makes repeated walks
+ * cheap: the second walk over `react@18.2.0` costs no network at all.
+ */
+type WalkState = {
+	readonly resolved: Map<PackageKey, ResolvedPackage>;
+	readonly edges: Map<PackageKey, ReadonlyArray<Task>>;
+	readonly packuments: Map<string, Packument>;
+	readonly warnings: Array<ResolutionWarning>;
+	/** Warnings are deduplicated — the same skipped edge appears in many walks. */
+	readonly warningKeys: Set<string>;
+};
+
+const makeWalkState = (): WalkState => ({
+	resolved: new Map(),
+	edges: new Map(),
+	packuments: new Map(),
+	warnings: [],
+	warningKeys: new Set(),
+});
+
+const addWarning = (state: WalkState, warning: ResolutionWarning): void => {
+	const key = `${warning.from ?? ""}|${warning.message}`;
+	if (state.warningKeys.has(key)) return;
+	state.warningKeys.add(key);
+	state.warnings.push(warning);
+};
+
+/**
+ * Fetches a packument, reusing anything already seen in this run.
+ *
+ * Two fibers in the same wave can race here and both fetch. That is one
+ * duplicated GET at worst, and de-racing it would cost more complexity than it
+ * saves.
+ */
+const getPackument = (
+	state: WalkState,
+	name: string,
+): Effect.Effect<Packument, BundlerError, Registry> =>
+	Effect.gen(function* () {
+		const cached = state.packuments.get(name);
+		if (cached !== undefined) return cached;
+		const registry = yield* Registry;
+		const packument = yield* registry.packument(name);
+		state.packuments.set(name, packument);
+		return packument;
+	});
+
+/**
+ * Re-fails a thrown selection error into the Effect error channel.
+ *
+ * `selectVersions` throws `BundlerError` subclasses and nothing else, so this
+ * narrows rather than casting; anything unexpected is rethrown as a defect,
+ * which is what an unexpected exception actually is.
+ */
+const asBundlerError = (error: unknown): BundlerError => {
+	if (error instanceof VersionNotFoundError || error instanceof NoMatchingVersionsError) {
+		return error;
+	}
+	throw error;
+};
+
+/* -------------------------------------------------------------------------- */
+/* Edge expansion                                                             */
+/* -------------------------------------------------------------------------- */
+
+/** Expands one resolved manifest into the edges that follow from it. */
+export const edgesOf = (
+	manifest: PackageManifest,
+	options: ResolveOptions,
+): {
+	readonly tasks: ReadonlyArray<Task>;
+	readonly warnings: ReadonlyArray<ResolutionWarning>;
+} => {
+	const from = packageKey(manifest.name, manifest.version);
+	const tasks: Array<Task> = [];
+	const warnings: Array<ResolutionWarning> = [];
+
+	const add = (
+		entries: Readonly<Record<string, string>> | undefined,
+		kind: EdgeKind,
+		tolerateFailure: boolean,
+		skip?: (name: string) => boolean,
+	): void => {
+		if (entries === undefined) return;
+		for (const [name, raw] of Object.entries(entries)) {
+			if (skip?.(name)) continue;
+			const target = parseDependencyTarget(name, raw);
+			if (target._tag === "Unsupported") {
+				warnings.push({
+					from,
+					message:
+						`Skipped ${name}@${target.raw} (${target.reason}) — it cannot be fetched from a ` +
+						`registry. Vendor it separately if the offline install needs it.`,
+				});
+				continue;
+			}
+			tasks.push({
+				name: target.name,
+				selector: target.selector,
+				reason: { _tag: "Edge", from, kind },
+				tolerateFailure,
+			});
+		}
+	};
+
+	// Runtime dependencies are non-negotiable.
+	add(manifest.dependencies, EdgeKind.Prod, false);
+
+	// Optional dependencies are allowed to be missing — npm tolerates a 404 here
+	// and so must we, or one unpublished platform package fails the whole run.
+	if (options.scope.optional) {
+		add(manifest.optionalDependencies, EdgeKind.Optional, true);
+	}
+
+	if (options.scope.peer) {
+		const meta = manifest.peerDependenciesMeta ?? {};
+		add(
+			manifest.peerDependencies,
+			EdgeKind.Peer,
+			false,
+			(name) => meta[name]?.optional === true,
+		);
+	}
+
+	return { tasks, warnings };
+};
+
+/** Edges for a package, computed once and cached. */
+const edgesFor = (
+	state: WalkState,
+	manifest: PackageManifest,
+	options: ResolveOptions,
+): ReadonlyArray<Task> => {
+	const key = packageKey(manifest.name, manifest.version);
+	const cached = state.edges.get(key);
+	if (cached !== undefined) return cached;
+
+	const { tasks, warnings } = edgesOf(manifest, options);
+	for (const warning of warnings) addWarning(state, warning);
+	state.edges.set(key, tasks);
+	return tasks;
+};
+
+/* -------------------------------------------------------------------------- */
+/* The walk                                                                   */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Resolves everything reachable from `seeds`, returning the closure.
+ *
+ * `state` is mutated with newly resolved packages and warnings; the returned
+ * array is scoped to *this* walk, which is what makes per-root-version
+ * closures possible.
+ */
+const walk = (
+	state: WalkState,
+	seeds: ReadonlyArray<Task>,
+	options: ResolveOptions,
+): Effect.Effect<ReadonlyArray<PackageKey>, BundlerError, Registry | Progress.Progress> =>
+	Effect.gen(function* () {
+		const discovered = new Set<PackageKey>();
+		// Guards against re-resolving the same (name, selector) pair. Large trees
+		// ask for `tslib@^2` hundreds of times, and this is also what terminates
+		// dependency cycles.
+		const attempted = new Set<string>();
+		let frontier: ReadonlyArray<Task> = seeds;
+
+		while (frontier.length > 0) {
+			const wave: Array<Task> = [];
+			for (const task of frontier) {
+				const id = `${task.name} ${formatSelector(task.selector)}`;
+				if (attempted.has(id)) continue;
+				attempted.add(id);
+				wave.push(task);
+			}
+			if (wave.length === 0) break;
+
+			const results = yield* Effect.forEach(
+				wave,
+				(task) => resolveTask(state, task, options),
+				{ concurrency: options.concurrency },
+			);
+
+			const next: Array<Task> = [];
+
+			// Merged serially: every mutation of `state` happens on one fiber.
+			for (const result of results) {
+				if (result === null) continue;
+
+				for (const { manifest, reason } of result) {
+					const key = packageKey(manifest.name, manifest.version);
+					const existing = state.resolved.get(key);
+
+					if (existing === undefined) {
+						state.resolved.set(key, {
+							name: manifest.name,
+							version: manifest.version,
+							manifest,
+							reasons: [reason],
+						});
+
+						yield* Progress.emit({
+							_tag: "PackageResolved",
+							name: manifest.name,
+							version: manifest.version,
+							resolvedCount: state.resolved.size,
+							pendingCount: next.length,
+						});
+
+						if (manifest.deprecated !== undefined && manifest.deprecated !== "") {
+							addWarning(state, {
+								from: key,
+								message: `${key} is deprecated: ${manifest.deprecated}`,
+							});
+						}
+					} else if (!hasReason(existing.reasons, reason)) {
+						state.resolved.set(key, {
+							...existing,
+							reasons: [...existing.reasons, reason],
+						});
+					}
+
+					// Expand regardless of whether the package is new. A package resolved
+					// during an earlier walk still has to contribute its subtree to
+					// *this* walk's closure; `attempted` stops that from looping.
+					if (!discovered.has(key)) {
+						discovered.add(key);
+						next.push(...edgesFor(state, manifest, options));
+					}
+				}
+			}
+
+			frontier = next;
+		}
+
+		return [...discovered];
+	});
+
+const hasReason = (reasons: ReadonlyArray<Reason>, candidate: Reason): boolean =>
+	reasons.some((reason) => {
+		if (reason._tag === "Root" && candidate._tag === "Root")
+			return reason.spec === candidate.spec;
+		if (reason._tag === "Edge" && candidate._tag === "Edge") {
+			return reason.from === candidate.from && reason.kind === candidate.kind;
+		}
+		return false;
+	});
+
+/**
+ * Resolves one task to its manifest(s).
+ *
+ * Returns `null` when an optional edge could not be resolved — the caller
+ * treats that as "skip quietly", which is what npm does for
+ * `optionalDependencies`.
+ */
+const resolveTask = (
+	state: WalkState,
+	task: Task,
+	options: ResolveOptions,
+): Effect.Effect<
+	ReadonlyArray<{ readonly manifest: PackageManifest; readonly reason: Reason }> | null,
+	BundlerError,
+	Registry
+> =>
+	Effect.gen(function* () {
+		const packument = yield* getPackument(state, task.name);
+
+		// Transitive edges always collapse to a single best match; only root specs
+		// ever fan out, and those arrive here already pinned to an exact version.
+		const versions = yield* Effect.try({
+			try: () =>
+				selectVersions(packument, task.selector, {
+					all: false,
+					includePrerelease: options.includePrerelease,
+				}),
+			catch: asBundlerError,
+		});
+
+		return versions.flatMap((version) => {
+			const manifest = packument.versions[version];
+			if (manifest === undefined) return [];
+
+			// Platform narrowing only ever applies to optional edges. Filtering a
+			// required dependency by platform would produce a bundle that cannot
+			// install anywhere.
+			if (
+				task.reason._tag === "Edge" &&
+				task.reason.kind === EdgeKind.Optional &&
+				!isIncluded(manifest, options.scope.platforms)
+			) {
+				return [];
+			}
+
+			return { manifest, reason: task.reason };
+		});
+	}).pipe(
+		Effect.catch((error) => {
+			// A required edge that cannot be resolved is fatal. An optional one is
+			// recorded and skipped, matching how npm treats `optionalDependencies`.
+			if (!task.tolerateFailure) return Effect.fail(error);
+			return Effect.sync(() => {
+				addWarning(state, {
+					message:
+						`Optional dependency ${task.name}@${formatSelector(task.selector)} could not be ` +
+						`resolved and was skipped: ${error.message}`,
+				});
+				return null;
+			});
+		}),
+	);
+
+/* -------------------------------------------------------------------------- */
+/* Entry point                                                                */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Resolves every requested spec.
+ *
+ * Each selected root *version* is walked separately so `per-spec` layout has an
+ * exact closure per tarball, while all walks share one cache, so a package
+ * reached from twenty roots is fetched once.
+ */
+export const resolve = (
+	specs: ReadonlyArray<PackageSpec>,
+	options: ResolveOptions,
+): Effect.Effect<Resolution, BundlerError, Registry | Progress.Progress> =>
+	Effect.gen(function* () {
+		yield* Progress.emit({ _tag: "PhaseStarted", phase: Phase.Resolve, total: specs.length });
+
+		const state = makeWalkState();
+		const roots: Array<RootResolution> = [];
+
+		for (const spec of specs) {
+			const packument = yield* getPackument(state, spec.name);
+
+			const versions = yield* Effect.try({
+				try: () =>
+					selectVersions(packument, spec.selector, {
+						all: options.allVersions,
+						maxVersions: options.maxVersions,
+						includePrerelease: options.includePrerelease,
+					}),
+				catch: asBundlerError,
+			});
+
+			const closures = new Map<string, ReadonlyArray<PackageKey>>();
+			const union = new Set<PackageKey>();
+
+			for (const version of versions) {
+				const closure = yield* walk(
+					state,
+					[
+						{
+							name: spec.name,
+							selector: { _tag: "Exact", version },
+							reason: { _tag: "Root", spec: formatSpec(spec) },
+							tolerateFailure: false,
+						},
+					],
+					options,
+				);
+				closures.set(version, closure);
+				for (const key of closure) union.add(key);
+			}
+
+			roots.push({ spec, versions, closures, closure: [...union] });
+		}
+
+		yield* Progress.emit({ _tag: "PhaseCompleted", phase: Phase.Resolve });
+
+		return {
+			roots,
+			packages: [...state.resolved.values()].toSorted(compareResolved),
+			warnings: state.warnings,
+		};
+	});
+
+const compareResolved = (a: ResolvedPackage, b: ResolvedPackage): number => {
+	if (a.name !== b.name) return a.name < b.name ? -1 : 1;
+	return semver.valid(a.version) && semver.valid(b.version)
+		? semver.compare(a.version, b.version)
+		: a.version.localeCompare(b.version);
+};

package/src/schemas/lenient.ts

@@ -0,0 +1,130 @@
+/**
+ * Schemas for JSON nobody controls.
+ *
+ * Registry responses, lockfiles and hand-edited package.json files all arrive
+ * as untyped JSON, and the rule at every one of those boundaries is the same:
+ * drop what cannot be understood, keep the rest. A bundler that rejects a
+ * packument because one published version carries a field it has never seen is
+ * useless against exactly the private registries it exists to serve.
+ *
+ * Effect Schema does the narrowing, which is what keeps those boundaries free
+ * of casts and hand-rolled `typeof` ladders. What this module adds on top of
+ * the stock combinators is that tolerance: `Schema.Record(String, String)`
+ * fails the whole record over one numeric value, where the wire format wants
+ * that entry skipped and its neighbours kept.
+ */
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import * as Schema from "effect/Schema";
+import * as Getter from "effect/SchemaGetter";
+
+import { isRecord } from "../utils/is-record.js";
+
+/**
+ * An optional field that goes *absent* rather than failing when it is
+ * malformed.
+ *
+ * This is the difference between "we do not understand this field" and "we do
+ * not understand this document". An `os: "linux"` where an array was expected
+ * costs the platform filter one signal; it should not cost the run the package.
+ */
+export const tolerant = <S extends Schema.Top>(schema: S) =>
+	Schema.optional(
+		Schema.UndefinedOr(schema).pipe(
+			Schema.catchDecoding(() => Effect.succeed(Option.some(undefined))),
+		),
+	);
+
+/**
+ * Keeps only the string-valued members of an untyped object.
+ *
+ * `{ "lodash": "^4.0.0", "broken": 3 }` decodes to `{ "lodash": "^4.0.0" }`.
+ * Per-entry rather than whole-record tolerance, because one unreadable edge in
+ * a dependency block is no reason to forget the others.
+ */
+export const StringRecordSchema = Schema.Record(Schema.String, Schema.Unknown).pipe(
+	Schema.decodeTo(Schema.Record(Schema.String, Schema.String), {
+		decode: Getter.transform((raw: Record<string, unknown>) => {
+			const out: Record<string, string> = {};
+			for (const [key, value] of Object.entries(raw)) {
+				if (typeof value === "string") out[key] = value;
+			}
+			return out;
+		}),
+		encode: Getter.passthroughSubtype(),
+	}),
+);
+
+/** The same per-entry tolerance for a list: `["linux", 7]` decodes to `["linux"]`. */
+export const StringArraySchema = Schema.Array(Schema.Unknown).pipe(
+	Schema.decodeTo(Schema.Array(Schema.String), {
+		decode: Getter.transform((raw: ReadonlyArray<unknown>) =>
+			raw.flatMap((entry) => (typeof entry === "string" ? entry : [])),
+		),
+		encode: Getter.passthroughSubtype(),
+	}),
+);
+
+/**
+ * `peerDependenciesMeta`, normalised.
+ *
+ * Only `optional` is read, and only its `true` is meaningful — npm writes the
+ * flag as a boolean, a hand-edited file may carry anything, and every other
+ * value means "not optional".
+ */
+export const OptionalFlagRecordSchema = Schema.Record(Schema.String, Schema.Unknown).pipe(
+	Schema.decodeTo(Schema.Record(Schema.String, Schema.Struct({ optional: Schema.Boolean })), {
+		decode: Getter.transform((raw: Record<string, unknown>) => {
+			const out: Record<string, { optional: boolean }> = {};
+			for (const [key, value] of Object.entries(raw)) {
+				if (isRecord(value)) out[key] = { optional: value["optional"] === true };
+			}
+			return out;
+		}),
+		encode: Getter.passthroughSubtype(),
+	}),
+);
+
+/*
+ * The `Optional…` variants below decode an empty result to `undefined`.
+ *
+ * Absent has to stay absent. `dependencies: {}` and no `dependencies` at all
+ * read the same to the resolver, but the first one travels into a bundle
+ * manifest claiming the registry published an empty block — a difference
+ * somebody diffing two manifests has to explain.
+ */
+
+const absentWhenEmpty = <T extends Record<string, unknown>>(value: T): T | undefined =>
+	Object.keys(value).length > 0 ? value : undefined;
+
+export const OptionalStringRecordSchema = StringRecordSchema.pipe(
+	Schema.decodeTo(Schema.UndefinedOr(Schema.Record(Schema.String, Schema.String)), {
+		decode: Getter.transform(absentWhenEmpty<Record<string, string>>),
+		encode: Getter.transform((value: Record<string, string> | undefined) => value ?? {}),
+	}),
+);
+
+export const OptionalStringArraySchema = StringArraySchema.pipe(
+	Schema.decodeTo(Schema.UndefinedOr(Schema.Array(Schema.String)), {
+		decode: Getter.transform((value: ReadonlyArray<string>) =>
+			value.length > 0 ? value : undefined,
+		),
+		encode: Getter.transform((value: ReadonlyArray<string> | undefined) => value ?? []),
+	}),
+);
+
+export const OptionalFlagRecordOrAbsentSchema = OptionalFlagRecordSchema.pipe(
+	Schema.decodeTo(
+		Schema.UndefinedOr(
+			Schema.Record(Schema.String, Schema.Struct({ optional: Schema.Boolean })),
+		),
+		{
+			decode: Getter.transform(
+				absentWhenEmpty<Record<string, { readonly optional: boolean }>>,
+			),
+			encode: Getter.transform(
+				(value: Record<string, { readonly optional: boolean }> | undefined) => value ?? {},
+			),
+		},
+	),
+);