@packall/core 0.0.1

package/src/Registry.ts

@@ -0,0 +1,121 @@
+/**
+ * The registry seam.
+ *
+ * This is the **only** place in the engine that knows a registry exists.
+ * Everything downstream — range resolution, the dependency walk, layout,
+ * archiving — is expressed against these three operations, so swapping npm for
+ * something else means implementing this interface and nothing more.
+ *
+ * `@packall/registry-npm` provides the implementation that speaks the plain
+ * npm registry protocol, which is what Artifactory, Nexus, Verdaccio, GitHub
+ * Packages and registry.npmjs.org all serve.
+ */
+import * as Context from "effect/Context"
+import type * as Effect from "effect/Effect"
+import type { BundlerError } from "./Errors.js"
+
+/**
+ * A single published version, as the registry describes it.
+ *
+ * This is a narrowed view of a package.json — only the fields that affect what
+ * has to be downloaded. Unknown fields are ignored rather than rejected,
+ * because registries add metadata over time and a bundler that breaks on new
+ * metadata is worse than useless.
+ */
+export interface PackageManifest {
+  readonly name: string
+  readonly version: string
+
+  readonly dependencies?: Readonly<Record<string, string>> | undefined
+  readonly optionalDependencies?: Readonly<Record<string, string>> | undefined
+  readonly peerDependencies?: Readonly<Record<string, string>> | undefined
+  readonly peerDependenciesMeta?:
+    | Readonly<Record<string, { readonly optional?: boolean | undefined }>>
+    | undefined
+  readonly devDependencies?: Readonly<Record<string, string>> | undefined
+  readonly bundleDependencies?: ReadonlyArray<string> | undefined
+
+  /** Platform constraints, used when `--platform` narrows optional deps. */
+  readonly os?: ReadonlyArray<string> | undefined
+  readonly cpu?: ReadonlyArray<string> | undefined
+  readonly libc?: ReadonlyArray<string> | undefined
+
+  readonly deprecated?: string | undefined
+
+  readonly dist: PackageDist
+}
+
+/** Where the tarball lives and how to prove it arrived intact. */
+export interface PackageDist {
+  readonly tarball: string
+  /** SRI string, e.g. `sha512-...`. Preferred over `shasum`. */
+  readonly integrity?: string | undefined
+  /** Legacy hex sha1. Still the only checksum on very old versions. */
+  readonly shasum?: string | undefined
+  readonly unpackedSize?: number | undefined
+  readonly fileCount?: number | undefined
+}
+
+/** Everything the registry knows about one package. */
+export interface Packument {
+  readonly name: string
+  /** `{ latest: "1.2.3", next: "2.0.0-beta.1" }` */
+  readonly distTags: Readonly<Record<string, string>>
+  /** Every published version, keyed by version string. */
+  readonly versions: Readonly<Record<string, PackageManifest>>
+}
+
+/** The set of operations a registry backend has to provide. */
+export interface RegistryService {
+  /** Short identifier for the manifest and for error messages, e.g. `"npm"`. */
+  readonly kind: string
+
+  /**
+   * The registry URL that will actually be used for a given package, after
+   * per-scope configuration is applied. Used for reporting only.
+   */
+  readonly registryFor: (packageName: string) => string
+
+  /**
+   * Cheap liveness check, run once before any real work.
+   *
+   * This is what turns "hangs forever behind a dead VPN" into a five-second
+   * error naming the host we could not reach.
+   */
+  readonly preflight: Effect.Effect<void, BundlerError>
+
+  /**
+   * Full metadata for a package. Implementations are expected to memoise:
+   * the dependency walk asks for the same popular packages repeatedly.
+   */
+  readonly packument: (packageName: string) => Effect.Effect<Packument, BundlerError>
+
+  /** Fetches the tarball bytes for one published version. */
+  readonly download: (manifest: PackageManifest) => Effect.Effect<Uint8Array, BundlerError>
+}
+
+/**
+ * Service tag for the registry backend.
+ *
+ * @example
+ * ```ts
+ * import { Effect } from "effect"
+ * import { Registry } from "@packall/core"
+ *
+ * const program = Effect.gen(function*() {
+ *   const registry = yield* Registry
+ *   return yield* registry.packument("lodash")
+ * })
+ * ```
+ */
+export class Registry extends Context.Service<Registry, RegistryService>()(
+  "@packall/core/Registry"
+) {}
+
+/** Extracts the dist-tag mapping, defaulting sensibly when a registry omits it. */
+export const resolveDistTag = (packument: Packument, tag: string): string | undefined =>
+  packument.distTags[tag]
+
+/** All published version strings, in registry order. */
+export const publishedVersions = (packument: Packument): ReadonlyArray<string> =>
+  Object.keys(packument.versions)

package/src/Resolve.ts

@@ -0,0 +1,528 @@
+/**
+ * Version selection and the dependency closure walk.
+ *
+ * This is the part that decides *what* ends up in a bundle. Its only side
+ * effect is asking the `Registry` service for metadata, which makes the whole
+ * of it testable against a fake registry with no network at all.
+ *
+ * The walk is breadth-first in waves: resolve the current frontier
+ * concurrently, collect the newly discovered edges, repeat. Wave-based BFS
+ * gives real parallelism without a dynamic worker pool, and because each wave
+ * merges its results serially there is no shared mutable state being written
+ * from more than one fiber.
+ *
+ * One subtlety worth calling out: metadata is cached across walks, but each
+ * walk still traverses the full graph. That is what lets `per-spec` layout ask
+ * "what does *this* root version depend on" many times over while the network
+ * is touched exactly once per package.
+ */
+import * as Effect from "effect/Effect"
+import semver from "semver"
+import { parseDependencyTarget } from "./DependencyRange.js"
+import type { BundlerError } from "./Errors.js"
+import { NoMatchingVersionsError, VersionNotFoundError } from "./Errors.js"
+import type { ResolveOptions } from "./Options.js"
+import { isIncluded } from "./Platform.js"
+import type { Packument, PackageManifest } from "./Registry.js"
+import { Registry } from "./Registry.js"
+import * as Progress from "./Progress.js"
+import type { PackageSpec, Selector } from "./Spec.js"
+import { formatSelector, formatSpec } from "./Spec.js"
+
+/** Why a package is in the bundle. */
+export type EdgeKind = "prod" | "optional" | "peer"
+
+/** A single justification for including a package. */
+export type Reason =
+  | { readonly _tag: "Root"; readonly spec: string }
+  | { readonly _tag: "Edge"; readonly from: string; readonly kind: EdgeKind }
+
+/** `name@version` — the identity of a resolved package throughout the engine. */
+export type PackageKey = string
+
+export const packageKey = (name: string, version: string): PackageKey => `${name}@${version}`
+
+/** One package, pinned to one version, with every reason it was pulled in. */
+export interface ResolvedPackage {
+  readonly name: string
+  readonly version: string
+  readonly manifest: PackageManifest
+  readonly reasons: ReadonlyArray<Reason>
+}
+
+/** Something we could not include, reported rather than thrown. */
+export interface ResolutionWarning {
+  readonly message: string
+  /** `name@version` of the package whose manifest contained the problem. */
+  readonly from?: string | undefined
+}
+
+/** What one requested spec expanded to. */
+export interface RootResolution {
+  readonly spec: PackageSpec
+  /** The concrete versions the spec selected — more than one under `--all-versions`. */
+  readonly versions: ReadonlyArray<string>
+  /** Closure per selected version, keyed by version. Drives `per-spec` layout. */
+  readonly closures: ReadonlyMap<string, ReadonlyArray<PackageKey>>
+  /** Union of every per-version closure. */
+  readonly closure: ReadonlyArray<PackageKey>
+}
+
+/** The complete result of resolving every requested spec. */
+export interface Resolution {
+  readonly roots: ReadonlyArray<RootResolution>
+  /** The deduplicated union of every closure, sorted by name then version. */
+  readonly packages: ReadonlyArray<ResolvedPackage>
+  readonly warnings: ReadonlyArray<ResolutionWarning>
+}
+
+/** Index of a resolution by `name@version`, for lookups during bundling. */
+export const indexPackages = (
+  resolution: Resolution
+): ReadonlyMap<PackageKey, ResolvedPackage> =>
+  new Map(resolution.packages.map((p) => [packageKey(p.name, p.version), p]))
+
+/* -------------------------------------------------------------------------- */
+/* Version selection                                                          */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Turns a selector into the concrete version(s) it names.
+ *
+ * `all` controls whether a range collapses to its best match — npm's behaviour,
+ * and what you always want for a transitive dependency — or expands to every
+ * satisfying published version, which is what `--all-versions` is for.
+ */
+export const selectVersions = (
+  packument: Packument,
+  selector: Selector,
+  options: {
+    readonly all: boolean
+    readonly maxVersions?: number | undefined
+    readonly includePrerelease: boolean
+  }
+): ReadonlyArray<string> => {
+  const available = Object.keys(packument.versions)
+
+  switch (selector._tag) {
+    case "Exact": {
+      if (!Object.hasOwn(packument.versions, selector.version)) {
+        throw new VersionNotFoundError(packument.name, selector.version, sortVersions(available))
+      }
+      return [selector.version]
+    }
+
+    case "Tag": {
+      const version = packument.distTags[selector.tag]
+      if (version === undefined || !Object.hasOwn(packument.versions, version)) {
+        throw new NoMatchingVersionsError(packument.name, selector.tag, sortVersions(available))
+      }
+      return [version]
+    }
+
+    case "Range": {
+      const satisfying = available
+        .filter((v) =>
+          semver.satisfies(v, selector.range, {
+            loose: true,
+            includePrerelease: options.includePrerelease
+          })
+        )
+        .sort(semver.rcompare)
+
+      if (satisfying.length === 0) {
+        throw new NoMatchingVersionsError(packument.name, selector.range, sortVersions(available))
+      }
+
+      if (!options.all) return [satisfying[0]!]
+
+      return options.maxVersions !== undefined && options.maxVersions > 0
+        ? satisfying.slice(0, options.maxVersions)
+        : satisfying
+    }
+  }
+}
+
+const sortVersions = (versions: ReadonlyArray<string>): ReadonlyArray<string> =>
+  [...versions].sort((a, b) =>
+    semver.valid(a) && semver.valid(b) ? semver.compare(a, b) : a.localeCompare(b)
+  )
+
+/* -------------------------------------------------------------------------- */
+/* Walk state                                                                 */
+/* -------------------------------------------------------------------------- */
+
+/** One pending item on the BFS frontier. */
+export interface Task {
+  readonly name: string
+  readonly selector: Selector
+  readonly reason: Reason
+  /** Optional edges tolerate resolution failure; required ones do not. */
+  readonly tolerateFailure: boolean
+}
+
+/**
+ * State shared across every walk in a run.
+ *
+ * Caching packuments and expanded edges here is what makes repeated walks
+ * cheap: the second walk over `react@18.2.0` costs no network at all.
+ */
+interface WalkState {
+  readonly resolved: Map<PackageKey, ResolvedPackage>
+  readonly edges: Map<PackageKey, ReadonlyArray<Task>>
+  readonly packuments: Map<string, Packument>
+  readonly warnings: Array<ResolutionWarning>
+  /** Warnings are deduplicated — the same skipped edge appears in many walks. */
+  readonly warningKeys: Set<string>
+}
+
+const makeWalkState = (): WalkState => ({
+  resolved: new Map(),
+  edges: new Map(),
+  packuments: new Map(),
+  warnings: [],
+  warningKeys: new Set()
+})
+
+const addWarning = (state: WalkState, warning: ResolutionWarning): void => {
+  const key = `${warning.from ?? ""}|${warning.message}`
+  if (state.warningKeys.has(key)) return
+  state.warningKeys.add(key)
+  state.warnings.push(warning)
+}
+
+/**
+ * Fetches a packument, reusing anything already seen in this run.
+ *
+ * Two fibers in the same wave can race here and both fetch. That is one
+ * duplicated GET at worst, and de-racing it would cost more complexity than it
+ * saves.
+ */
+const getPackument = (
+  state: WalkState,
+  name: string
+): Effect.Effect<Packument, BundlerError, Registry> =>
+  Effect.gen(function*() {
+    const cached = state.packuments.get(name)
+    if (cached !== undefined) return cached
+    const registry = yield* Registry
+    const packument = yield* registry.packument(name)
+    state.packuments.set(name, packument)
+    return packument
+  })
+
+/* -------------------------------------------------------------------------- */
+/* Edge expansion                                                             */
+/* -------------------------------------------------------------------------- */
+
+/** Expands one resolved manifest into the edges that follow from it. */
+export const edgesOf = (
+  manifest: PackageManifest,
+  options: ResolveOptions
+): {
+  readonly tasks: ReadonlyArray<Task>
+  readonly warnings: ReadonlyArray<ResolutionWarning>
+} => {
+  const from = packageKey(manifest.name, manifest.version)
+  const tasks: Array<Task> = []
+  const warnings: Array<ResolutionWarning> = []
+
+  const add = (
+    entries: Readonly<Record<string, string>> | undefined,
+    kind: EdgeKind,
+    tolerateFailure: boolean,
+    skip?: (name: string) => boolean
+  ): void => {
+    if (entries === undefined) return
+    for (const [name, raw] of Object.entries(entries)) {
+      if (skip?.(name)) continue
+      const target = parseDependencyTarget(name, raw)
+      if (target._tag === "Unsupported") {
+        warnings.push({
+          from,
+          message:
+            `Skipped ${name}@${target.raw} (${target.reason}) — it cannot be fetched from a ` +
+            `registry. Vendor it separately if the offline install needs it.`
+        })
+        continue
+      }
+      tasks.push({
+        name: target.name,
+        selector: target.selector,
+        reason: { _tag: "Edge", from, kind },
+        tolerateFailure
+      })
+    }
+  }
+
+  // Runtime dependencies are non-negotiable.
+  add(manifest.dependencies, "prod", false)
+
+  // Optional dependencies are allowed to be missing — npm tolerates a 404 here
+  // and so must we, or one unpublished platform package fails the whole run.
+  if (options.scope.optional) {
+    add(manifest.optionalDependencies, "optional", true)
+  }
+
+  if (options.scope.peer) {
+    const meta = manifest.peerDependenciesMeta ?? {}
+    add(manifest.peerDependencies, "peer", false, (name) => meta[name]?.optional === true)
+  }
+
+  return { tasks, warnings }
+}
+
+/** Edges for a package, computed once and cached. */
+const edgesFor = (
+  state: WalkState,
+  manifest: PackageManifest,
+  options: ResolveOptions
+): ReadonlyArray<Task> => {
+  const key = packageKey(manifest.name, manifest.version)
+  const cached = state.edges.get(key)
+  if (cached !== undefined) return cached
+
+  const { tasks, warnings } = edgesOf(manifest, options)
+  for (const warning of warnings) addWarning(state, warning)
+  state.edges.set(key, tasks)
+  return tasks
+}
+
+/* -------------------------------------------------------------------------- */
+/* The walk                                                                   */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Resolves everything reachable from `seeds`, returning the closure.
+ *
+ * `state` is mutated with newly resolved packages and warnings; the returned
+ * array is scoped to *this* walk, which is what makes per-root-version
+ * closures possible.
+ */
+const walk = (
+  state: WalkState,
+  seeds: ReadonlyArray<Task>,
+  options: ResolveOptions
+): Effect.Effect<ReadonlyArray<PackageKey>, BundlerError, Registry | Progress.Progress> =>
+  Effect.gen(function*() {
+    const discovered = new Set<PackageKey>()
+    // Guards against re-resolving the same (name, selector) pair. Large trees
+    // ask for `tslib@^2` hundreds of times, and this is also what terminates
+    // dependency cycles.
+    const attempted = new Set<string>()
+    let frontier: ReadonlyArray<Task> = seeds
+
+    while (frontier.length > 0) {
+      const wave: Array<Task> = []
+      for (const task of frontier) {
+        const id = `${task.name} ${formatSelector(task.selector)}`
+        if (attempted.has(id)) continue
+        attempted.add(id)
+        wave.push(task)
+      }
+      if (wave.length === 0) break
+
+      const results = yield* Effect.forEach(
+        wave,
+        (task) => resolveTask(state, task, options),
+        { concurrency: options.concurrency }
+      )
+
+      const next: Array<Task> = []
+
+      // Merged serially: every mutation of `state` happens on one fiber.
+      for (const result of results) {
+        if (result === null) continue
+
+        for (const { manifest, reason } of result) {
+          const key = packageKey(manifest.name, manifest.version)
+          const isNew = !state.resolved.has(key)
+
+          if (isNew) {
+            state.resolved.set(key, {
+              name: manifest.name,
+              version: manifest.version,
+              manifest,
+              reasons: [reason]
+            })
+
+            yield* Progress.emit({
+              _tag: "PackageResolved",
+              name: manifest.name,
+              version: manifest.version,
+              resolvedCount: state.resolved.size,
+              pendingCount: next.length
+            })
+
+            if (manifest.deprecated !== undefined && manifest.deprecated !== "") {
+              addWarning(state, { from: key, message: `${key} is deprecated: ${manifest.deprecated}` })
+            }
+          } else {
+            const existing = state.resolved.get(key)!
+            if (!hasReason(existing.reasons, reason)) {
+              state.resolved.set(key, { ...existing, reasons: [...existing.reasons, reason] })
+            }
+          }
+
+          // Expand regardless of whether the package is new. A package resolved
+          // during an earlier walk still has to contribute its subtree to
+          // *this* walk's closure; `attempted` stops that from looping.
+          if (!discovered.has(key)) {
+            discovered.add(key)
+            next.push(...edgesFor(state, manifest, options))
+          }
+        }
+      }
+
+      frontier = next
+    }
+
+    return [...discovered]
+  })
+
+const hasReason = (reasons: ReadonlyArray<Reason>, candidate: Reason): boolean =>
+  reasons.some((reason) => {
+    if (reason._tag === "Root" && candidate._tag === "Root") return reason.spec === candidate.spec
+    if (reason._tag === "Edge" && candidate._tag === "Edge") {
+      return reason.from === candidate.from && reason.kind === candidate.kind
+    }
+    return false
+  })
+
+/**
+ * Resolves one task to its manifest(s).
+ *
+ * Returns `null` when an optional edge could not be resolved — the caller
+ * treats that as "skip quietly", which is what npm does for
+ * `optionalDependencies`.
+ */
+const resolveTask = (
+  state: WalkState,
+  task: Task,
+  options: ResolveOptions
+): Effect.Effect<
+  ReadonlyArray<{ manifest: PackageManifest; reason: Reason }> | null,
+  BundlerError,
+  Registry
+> =>
+  Effect.gen(function*() {
+    const packument = yield* getPackument(state, task.name)
+
+    // Transitive edges always collapse to a single best match; only root specs
+    // ever fan out, and those arrive here already pinned to an exact version.
+    const versions = yield* Effect.try({
+      try: () =>
+        selectVersions(packument, task.selector, {
+          all: false,
+          includePrerelease: options.includePrerelease
+        }),
+      catch: (error) => error as BundlerError
+    })
+
+    const out: Array<{ manifest: PackageManifest; reason: Reason }> = []
+    for (const version of versions) {
+      const manifest = packument.versions[version]
+      if (manifest === undefined) continue
+
+      // Platform narrowing only ever applies to optional edges. Filtering a
+      // required dependency by platform would produce a bundle that cannot
+      // install anywhere.
+      if (
+        task.reason._tag === "Edge" &&
+        task.reason.kind === "optional" &&
+        !isIncluded(manifest, options.scope.platforms)
+      ) {
+        continue
+      }
+
+      out.push({ manifest, reason: task.reason })
+    }
+    return out
+  }).pipe(
+    Effect.catch((error) => {
+      // A required edge that cannot be resolved is fatal. An optional one is
+      // recorded and skipped, matching how npm treats `optionalDependencies`.
+      if (!task.tolerateFailure) return Effect.fail(error)
+      return Effect.sync(() => {
+        addWarning(state, {
+          message:
+            `Optional dependency ${task.name}@${formatSelector(task.selector)} could not be ` +
+            `resolved and was skipped: ${error.message}`
+        })
+        return null
+      })
+    })
+  )
+
+/* -------------------------------------------------------------------------- */
+/* Entry point                                                                */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Resolves every requested spec.
+ *
+ * Each selected root *version* is walked separately so `per-spec` layout has an
+ * exact closure per tarball, while all walks share one cache, so a package
+ * reached from twenty roots is fetched once.
+ */
+export const resolve = (
+  specs: ReadonlyArray<PackageSpec>,
+  options: ResolveOptions
+): Effect.Effect<Resolution, BundlerError, Registry | Progress.Progress> =>
+  Effect.gen(function*() {
+    yield* Progress.emit({ _tag: "PhaseStarted", phase: "resolve", total: specs.length })
+
+    const state = makeWalkState()
+    const roots: Array<RootResolution> = []
+
+    for (const spec of specs) {
+      const packument = yield* getPackument(state, spec.name)
+
+      const versions = yield* Effect.try({
+        try: () =>
+          selectVersions(packument, spec.selector, {
+            all: options.allVersions,
+            maxVersions: options.maxVersions,
+            includePrerelease: options.includePrerelease
+          }),
+        catch: (error) => error as BundlerError
+      })
+
+      const closures = new Map<string, ReadonlyArray<PackageKey>>()
+      const union = new Set<PackageKey>()
+
+      for (const version of versions) {
+        const closure = yield* walk(
+          state,
+          [
+            {
+              name: spec.name,
+              selector: { _tag: "Exact", version },
+              reason: { _tag: "Root", spec: formatSpec(spec) },
+              tolerateFailure: false
+            }
+          ],
+          options
+        )
+        closures.set(version, closure)
+        for (const key of closure) union.add(key)
+      }
+
+      roots.push({ spec, versions, closures, closure: [...union] })
+    }
+
+    yield* Progress.emit({ _tag: "PhaseCompleted", phase: "resolve" })
+
+    return {
+      roots,
+      packages: [...state.resolved.values()].sort(compareResolved),
+      warnings: state.warnings
+    }
+  })
+
+const compareResolved = (a: ResolvedPackage, b: ResolvedPackage): number => {
+  if (a.name !== b.name) return a.name < b.name ? -1 : 1
+  return semver.valid(a.version) && semver.valid(b.version)
+    ? semver.compare(a.version, b.version)
+    : a.version.localeCompare(b.version)
+}