@packall/core 0.0.1

package/src/Bundle.ts

@@ -0,0 +1,570 @@
+/**
+ * End-to-end orchestration: specs in, tarballs out.
+ *
+ * Two properties this module is responsible for, both of them fixes for how
+ * the original tool behaved:
+ *
+ * 1. **Nothing is written outside the output directory.** Every intermediate
+ *    file lives in a scoped temp directory obtained from
+ *    `makeTempDirectoryScoped`, so the scope closing — on success, on failure,
+ *    or on Ctrl-C — takes the whole staging tree with it. The working
+ *    directory is never touched.
+ *
+ * 2. **Each package is downloaded exactly once**, no matter how many bundles
+ *    it ends up in. Per-spec archives are assembled by hard-linking out of one
+ *    shared download tree, so `per-spec` costs extra disk in the *output*, not
+ *    extra network.
+ */
+import * as Effect from "effect/Effect"
+import * as FileSystem from "effect/FileSystem"
+import * as Path from "effect/Path"
+import type { PlatformError } from "effect/PlatformError"
+import type { Scope } from "effect/Scope"
+import { createArchive } from "./Archive.js"
+import { downloadAll } from "./Download.js"
+import type { BundlerError } from "./Errors.js"
+import { OutputError } from "./Errors.js"
+import {
+  importGuide,
+  MANIFEST_FILE,
+  packagePath,
+  perSpecArchiveName,
+  README_FILE,
+  singleArchiveName
+} from "./Layout.js"
+import { buildManifest, serializeManifest } from "./Manifest.js"
+import type { BundleOptions, Layout } from "./Options.js"
+import * as Progress from "./Progress.js"
+import { Registry } from "./Registry.js"
+import type { PackageKey, Resolution, ResolvedPackage } from "./Resolve.js"
+import { indexPackages, resolve } from "./Resolve.js"
+import type { PackageSpec } from "./Spec.js"
+import { formatSpec } from "./Spec.js"
+
+/** One thing written to the output directory. */
+export interface BundleArtifact {
+  readonly kind: "archive" | "directory"
+  readonly path: string
+  readonly bytes: number
+  readonly packageCount: number
+  /** Which spec this artifact covers. Absent for `single` and `dir`. */
+  readonly spec?: string | undefined
+  readonly version?: string | undefined
+}
+
+/** The outcome of a bundle run. */
+export interface BundleResult {
+  readonly resolution: Resolution
+  readonly artifacts: ReadonlyArray<BundleArtifact>
+  /** Total bytes downloaded from the registry. */
+  readonly downloadedBytes: number
+  /** Packages the registry advertised no usable checksum for. */
+  readonly unverified: ReadonlyArray<string>
+  readonly dryRun: boolean
+}
+
+/** Everything `bundle` needs beyond the user-facing options. */
+export interface BundleContext extends BundleOptions {
+  /** Recorded in the manifest so a bundle can be traced to the tool that built it. */
+  readonly toolVersion: string
+}
+
+/**
+ * A summary of what a resolution would produce, for deciding whether the
+ * chosen layout is still the right one.
+ */
+export interface PlanSummary {
+  /** Distinct packages in the deduplicated union. */
+  readonly uniquePackages: number
+  /** How many archives `per-spec` layout would emit. */
+  readonly perSpecArchives: number
+  /**
+   * Total package entries across all per-spec archives, counting shared
+   * dependencies once per archive they appear in. The gap between this and
+   * `uniquePackages` is exactly what `single` layout would save.
+   */
+  readonly perSpecEntries: number
+}
+
+/** Computes the plan summary for a resolution. */
+export const summarize = (resolution: Resolution): PlanSummary => {
+  let perSpecArchives = 0
+  let perSpecEntries = 0
+  for (const root of resolution.roots) {
+    for (const version of root.versions) {
+      perSpecArchives += 1
+      perSpecEntries += (root.closures.get(version) ?? []).length
+    }
+  }
+  return {
+    uniquePackages: resolution.packages.length,
+    perSpecArchives,
+    perSpecEntries
+  }
+}
+
+/** Replaces the layout on a set of options. */
+export const withLayout = <T extends BundleOptions>(options: T, layout: Layout): T => ({
+  ...options,
+  layout
+})
+
+type BundleEnv = Registry | Progress.Progress | FileSystem.FileSystem | Path.Path
+
+/**
+ * Checks the registry is reachable, then resolves every spec.
+ *
+ * Split out from `bundle` so a caller can inspect the plan — how many packages,
+ * how many archives — and act on it before any bytes move. The CLI uses this to
+ * offer a different layout when a run turns out to be much larger than
+ * expected.
+ */
+export const plan = (
+  specs: ReadonlyArray<PackageSpec>,
+  options: BundleOptions
+): Effect.Effect<Resolution, BundlerError, Registry | Progress.Progress> =>
+  Effect.gen(function*() {
+    const registry = yield* Registry
+
+    yield* Progress.emit({ _tag: "PhaseStarted", phase: "preflight" })
+    yield* registry.preflight
+    yield* Progress.emit({ _tag: "PhaseCompleted", phase: "preflight" })
+
+    return yield* resolve(specs, options)
+  })
+
+/**
+ * Resolves, downloads and packages a set of specs.
+ *
+ * Under `dryRun` this stops after resolution and reports the plan — useful for
+ * checking what a command *would* pull before committing to a multi-gigabyte
+ * download over a slow VPN.
+ */
+export const bundle = (
+  specs: ReadonlyArray<PackageSpec>,
+  options: BundleContext
+): Effect.Effect<BundleResult, BundlerError | PlatformError, BundleEnv> =>
+  Effect.gen(function*() {
+    const resolution = yield* plan(specs, options)
+    return yield* bundleResolved(resolution, options)
+  })
+
+/**
+ * Downloads and packages an already-computed resolution.
+ *
+ * Everything from here on touches the disk, and all of it happens inside
+ * `Effect.scoped` — so the staging tree's lifetime is exactly this call,
+ * including when it fails partway through or the user hits Ctrl-C.
+ */
+export const bundleResolved = (
+  resolution: Resolution,
+  options: BundleContext
+): Effect.Effect<BundleResult, BundlerError | PlatformError, BundleEnv> =>
+  Effect.gen(function*() {
+    if (options.dryRun) {
+      return {
+        resolution,
+        artifacts: [],
+        downloadedBytes: 0,
+        unverified: [],
+        dryRun: true
+      }
+    }
+    return yield* Effect.scoped(runBundle(resolution, options))
+  })
+
+const runBundle = (
+  resolution: Resolution,
+  options: BundleContext
+): Effect.Effect<
+  BundleResult,
+  BundlerError | PlatformError,
+  BundleEnv | Scope
+> =>
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const registry = yield* Registry
+
+    yield* prepareOutputDir(resolution, options)
+
+    const staging = yield* fs.makeTempDirectoryScoped({ prefix: "packall-" })
+    const packagesDir = path.join(staging, "packages")
+    yield* fs.makeDirectory(packagesDir, { recursive: true })
+
+    const report = yield* downloadAll(resolution.packages, packagesDir, {
+      concurrency: options.concurrency,
+      verifyIntegrity: options.verifyIntegrity
+    })
+
+    yield* Progress.emit({ _tag: "PhaseStarted", phase: "archive" })
+
+    const registryInfo = {
+      kind: registry.kind,
+      url: registry.registryFor(resolution.roots[0]?.spec.name ?? "")
+    }
+
+    const index = indexPackages(resolution)
+
+    const artifacts =
+      options.layout === "single"
+        ? yield* emitSingle({ resolution, options, staging, packagesDir, report, registryInfo })
+        : options.layout === "dir"
+          ? yield* emitDirectory({ resolution, options, packagesDir, report, registryInfo })
+          : yield* emitPerSpec({
+              resolution,
+              options,
+              staging,
+              packagesDir,
+              report,
+              registryInfo,
+              index
+            })
+
+    yield* Progress.emit({ _tag: "PhaseCompleted", phase: "archive" })
+    yield* Progress.emit({ _tag: "PhaseStarted", phase: "done" })
+
+    return {
+      resolution,
+      artifacts,
+      downloadedBytes: report.totalBytes,
+      unverified: report.unverified,
+      dryRun: false
+    }
+  })
+
+/* -------------------------------------------------------------------------- */
+/* Emitters                                                                   */
+/* -------------------------------------------------------------------------- */
+
+interface EmitInput {
+  readonly resolution: Resolution
+  readonly options: BundleContext
+  readonly packagesDir: string
+  readonly report: { readonly sizes: ReadonlyMap<string, number> }
+  readonly registryInfo: { readonly kind: string; readonly url: string }
+}
+
+/** One tarball containing the deduplicated union of every closure. */
+const emitSingle = (
+  input: EmitInput & { readonly staging: string }
+): Effect.Effect<
+  ReadonlyArray<BundleArtifact>,
+  BundlerError | PlatformError,
+  FileSystem.FileSystem | Path.Path | Progress.Progress
+> =>
+  Effect.gen(function*() {
+    const path = yield* Path.Path
+
+    const archiveName = singleArchiveName(input.options.archiveName ?? "bundle")
+    // Declined at the prompt, and it is the run's only output.
+    if (input.options.skipExisting?.has(archiveName) === true) return []
+
+    yield* writeBundleMetadata({
+      dir: input.packagesDir,
+      resolution: input.resolution,
+      included: input.resolution.packages,
+      options: input.options,
+      registryInfo: input.registryInfo,
+      sizes: input.report.sizes
+    })
+
+    const outPath = path.join(input.options.outDir, archiveName)
+
+    const result = yield* createArchive({
+      cwd: input.packagesDir,
+      entries: yield* topLevelEntries(input.packagesDir),
+      outPath
+    })
+
+    return [
+      {
+        kind: "archive" as const,
+        path: result.path,
+        bytes: result.bytes,
+        packageCount: input.resolution.packages.length
+      }
+    ]
+  })
+
+/** The raw npm-layout tree, no archive. */
+const emitDirectory = (
+  input: EmitInput
+): Effect.Effect<
+  ReadonlyArray<BundleArtifact>,
+  BundlerError | PlatformError,
+  FileSystem.FileSystem | Path.Path
+> =>
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+
+    yield* writeBundleMetadata({
+      dir: input.packagesDir,
+      resolution: input.resolution,
+      included: input.resolution.packages,
+      options: input.options,
+      registryInfo: input.registryInfo,
+      sizes: input.report.sizes
+    })
+
+    // The staging tree is about to be deleted with its scope, so this is a
+    // real copy rather than a move — `copy` also works across devices, which
+    // matters because the temp dir often lives on a different filesystem.
+    yield* fs.copy(input.packagesDir, input.options.outDir, { overwrite: true }).pipe(
+      Effect.mapError(
+        (cause) => new OutputError(input.options.outDir, "could not write output tree", { cause })
+      )
+    )
+
+    const bytes = [...input.report.sizes.values()].reduce((a, b) => a + b, 0)
+
+    return [
+      {
+        kind: "directory" as const,
+        path: input.options.outDir,
+        bytes,
+        packageCount: input.resolution.packages.length
+      }
+    ]
+  })
+
+/**
+ * One tarball per resolved root *version*.
+ *
+ * Under `--all-versions react@^18` this produces `react-18.0.0.tgz`,
+ * `react-18.1.0.tgz` and so on, each independently importable — which is the
+ * whole reason to prefer this layout.
+ */
+const emitPerSpec = (
+  input: EmitInput & {
+    readonly staging: string
+    readonly index: ReadonlyMap<PackageKey, ResolvedPackage>
+  }
+): Effect.Effect<
+  ReadonlyArray<BundleArtifact>,
+  BundlerError | PlatformError,
+  FileSystem.FileSystem | Path.Path | Progress.Progress
+> =>
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const artifacts: Array<BundleArtifact> = []
+
+    let counter = 0
+    for (const root of input.resolution.roots) {
+      for (const version of root.versions) {
+        // Declined at the prompt: the existing file stays, and this archive is
+        // simply never built. Skipping the staging work too, not just the write.
+        if (input.options.skipExisting?.has(perSpecArchiveName(root.spec.name, version)) === true) {
+          continue
+        }
+
+        const closure = root.closures.get(version) ?? []
+        const included = closure
+          .map((key) => input.index.get(key))
+          .filter((pkg): pkg is ResolvedPackage => pkg !== undefined)
+
+        const rootDir = path.join(input.staging, "roots", String(counter++))
+        yield* fs.makeDirectory(rootDir, { recursive: true })
+
+        for (const pkg of included) {
+          const relative = packagePath(pkg.name, pkg.version)
+          const parts = relative.split("/")
+          const source = path.join(input.packagesDir, ...parts)
+          const target = path.join(rootDir, ...parts)
+          yield* fs.makeDirectory(path.dirname(target), { recursive: true })
+          yield* linkOrCopy(source, target)
+        }
+
+        yield* writeBundleMetadata({
+          dir: rootDir,
+          resolution: {
+            ...input.resolution,
+            roots: [{ ...root, versions: [version] }]
+          },
+          included,
+          options: input.options,
+          registryInfo: input.registryInfo,
+          sizes: input.report.sizes
+        })
+
+        const outPath = path.join(
+          input.options.outDir,
+          perSpecArchiveName(root.spec.name, version)
+        )
+
+        const result = yield* createArchive({
+          cwd: rootDir,
+          entries: yield* topLevelEntries(rootDir),
+          outPath
+        })
+
+        artifacts.push({
+          kind: "archive",
+          path: result.path,
+          bytes: result.bytes,
+          packageCount: included.length,
+          spec: formatSpec(root.spec),
+          version
+        })
+      }
+    }
+
+    return artifacts
+  })
+
+/* -------------------------------------------------------------------------- */
+/* Helpers                                                                    */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Hard-links a file, falling back to a copy.
+ *
+ * Hard links make `per-spec` layout nearly free in the staging tree. They fail
+ * across devices and on some Windows configurations, so the copy fallback is
+ * not optional.
+ */
+const linkOrCopy = (
+  source: string,
+  target: string
+): Effect.Effect<void, PlatformError, FileSystem.FileSystem> =>
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+    yield* fs.link(source, target).pipe(
+      Effect.catch(() => fs.copyFile(source, target))
+    )
+  })
+
+/** Writes the manifest and import guide into a staging directory. */
+const writeBundleMetadata = (input: {
+  readonly dir: string
+  readonly resolution: Resolution
+  readonly included: ReadonlyArray<ResolvedPackage>
+  readonly options: BundleContext
+  readonly registryInfo: { readonly kind: string; readonly url: string }
+  readonly sizes: ReadonlyMap<string, number>
+}): Effect.Effect<void, PlatformError, FileSystem.FileSystem | Path.Path> =>
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+
+    const createdAt = new Date()
+    const manifest = buildManifest({
+      resolution: input.resolution,
+      included: input.included,
+      options: input.options,
+      registry: input.registryInfo,
+      toolVersion: input.options.toolVersion,
+      sizes: input.sizes,
+      createdAt
+    })
+
+    yield* fs.writeFileString(path.join(input.dir, MANIFEST_FILE), serializeManifest(manifest))
+    yield* fs.writeFileString(
+      path.join(input.dir, README_FILE),
+      importGuide({
+        packageCount: input.included.length,
+        createdAt: createdAt.toISOString(),
+        toolVersion: input.options.toolVersion
+      })
+    )
+  })
+
+/** Sorted top-level entries of a directory, used as the tar entry list. */
+const topLevelEntries = (
+  dir: string
+): Effect.Effect<ReadonlyArray<string>, PlatformError, FileSystem.FileSystem> =>
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+    const entries = yield* fs.readDirectory(dir)
+    return [...entries].sort()
+  })
+
+/**
+ * Creates the output directory, refusing to clobber an existing non-empty one
+ * unless `--force` was passed.
+ *
+ * Silently overwriting somebody's previous bundle is the kind of thing a tool
+ * only gets forgiven for once.
+ */
+/**
+ * The files a run will write, known before anything is downloaded.
+ *
+ * `dir` layout is the odd one: it merges a package tree into `outDir`, and
+ * package paths are `name/-/name-version.tgz` — the same package at the same
+ * version is the same bytes, so an overlap there is idempotent rather than
+ * destructive. Only the two summary files are genuinely replaced, and those are
+ * the ones worth guarding.
+ */
+export interface PlannedOutput {
+  readonly file: string
+  /** The root this file belongs to, when one file corresponds to one spec. */
+  readonly name?: string | undefined
+  readonly version?: string | undefined
+}
+
+export const plannedOutputs = (
+  resolution: Resolution,
+  options: BundleContext
+): ReadonlyArray<PlannedOutput> => {
+  if (options.layout === "dir") return [{ file: MANIFEST_FILE }, { file: README_FILE }]
+  if (options.layout === "single") {
+    return [{ file: singleArchiveName(options.archiveName ?? "bundle") }]
+  }
+  return resolution.roots.flatMap((root) =>
+    root.versions.map((version) => ({
+      file: perSpecArchiveName(root.spec.name, version),
+      name: root.spec.name,
+      version
+    }))
+  )
+}
+
+/**
+ * Fails if the run would overwrite something, unless `--force`.
+ *
+ * The guard is on the *files this run writes*, not on whether the directory has
+ * anything in it. An output directory accumulating bundles is the normal way to
+ * use this — bundling `esbuild` into an `out/` that already holds `tsdown` is
+ * not a conflict, and refusing it made `--out` a single-use directory and the
+ * default `.` unusable.
+ *
+ * Checked before anything is downloaded. Discovering the collision *after* a
+ * twenty-minute download would be a uniquely irritating way to fail.
+ */
+const prepareOutputDir = (
+  resolution: Resolution,
+  options: BundleContext
+): Effect.Effect<void, OutputError | PlatformError, FileSystem.FileSystem | Path.Path> =>
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const outDir = options.outDir
+
+    if (!options.force) {
+      const clashes: Array<string> = []
+      for (const planned of plannedOutputs(resolution, options)) {
+        if (options.skipExisting?.has(planned.file) === true) continue
+        if (options.overwrite?.has(planned.file) === true) continue
+        const exists = yield* fs
+          .exists(path.join(outDir, planned.file))
+          .pipe(Effect.orElseSucceed(() => false))
+        if (exists) clashes.push(planned.file)
+      }
+
+      if (clashes.length > 0) {
+        return yield* Effect.fail(
+          new OutputError(
+            outDir,
+            `would overwrite ${clashes.length} existing file${clashes.length === 1 ? "" : "s"} ` +
+              `(${clashes.slice(0, 3).join(", ")}${clashes.length > 3 ? ", …" : ""}). ` +
+              `Pass --force true to replace ${clashes.length === 1 ? "it" : "them"}, or choose a different --out.`
+          )
+        )
+      }
+    }
+
+    yield* fs.makeDirectory(outDir, { recursive: true }).pipe(
+      Effect.mapError((cause) => new OutputError(outDir, "could not create output directory", { cause }))
+    )
+  })

package/src/DependencyRange.ts

@@ -0,0 +1,115 @@
+/**
+ * Classification of the right-hand side of a dependency entry.
+ *
+ * `"dependencies": { "x": "^1.0.0" }` is the easy case. Real package.json files
+ * also contain aliases, git URLs, tarball URLs, `file:` links and
+ * `workspace:` protocols — none of which can be fetched from a registry.
+ *
+ * Rather than crashing or silently producing a bundle that will not install,
+ * we classify each edge and let the caller report precisely what was skipped
+ * and why.
+ */
+import semver from "semver"
+import type { Selector } from "./Spec.js"
+
+/** What a dependency edge points at. */
+export type DependencyTarget =
+  /** A normal registry dependency. `name` may differ from the key under an alias. */
+  | {
+      readonly _tag: "Registry"
+      readonly name: string
+      readonly selector: Selector
+      /** Set when the edge is an `npm:` alias, e.g. `"lodash4": "npm:lodash@^4"`. */
+      readonly aliasOf?: string | undefined
+    }
+  /** Something we cannot fetch from a registry. */
+  | { readonly _tag: "Unsupported"; readonly name: string; readonly raw: string; readonly reason: string }
+
+const UNSUPPORTED_PROTOCOLS: ReadonlyArray<readonly [string, string]> = [
+  ["file:", "local file path"],
+  ["link:", "local link"],
+  ["workspace:", "workspace protocol"],
+  ["portal:", "portal protocol"],
+  ["patch:", "patch protocol"],
+  ["git:", "git dependency"],
+  ["git+", "git dependency"],
+  ["github:", "GitHub shorthand"],
+  ["gitlab:", "GitLab shorthand"],
+  ["bitbucket:", "Bitbucket shorthand"],
+  ["http:", "remote tarball URL"],
+  ["https:", "remote tarball URL"]
+]
+
+/**
+ * Classifies one `name -> range` entry.
+ *
+ * An empty range, `*`, and `latest` all mean "any published version"; npm
+ * treats them interchangeably and so do we.
+ */
+export const parseDependencyTarget = (name: string, raw: string): DependencyTarget => {
+  const text = raw.trim()
+
+  if (text.length === 0 || text === "*" || text === "x" || text === "latest") {
+    return { _tag: "Registry", name, selector: { _tag: "Range", range: "*" } }
+  }
+
+  if (text.startsWith("npm:")) {
+    return parseAlias(name, text)
+  }
+
+  for (const [prefix, reason] of UNSUPPORTED_PROTOCOLS) {
+    if (text.startsWith(prefix)) {
+      return { _tag: "Unsupported", name, raw: text, reason }
+    }
+  }
+
+  // Bare `owner/repo` is GitHub shorthand, but only when it is not a valid
+  // range — `>=1.0.0/2` is not a thing, so this ordering is safe.
+  if (semver.validRange(text, { loose: true }) === null && /^[\w.-]+\/[\w.-]+/.test(text)) {
+    return { _tag: "Unsupported", name, raw: text, reason: "git shorthand" }
+  }
+
+  const exact = semver.valid(text, { loose: true })
+  if (exact !== null) {
+    return { _tag: "Registry", name, selector: { _tag: "Exact", version: exact } }
+  }
+
+  if (semver.validRange(text, { loose: true }) !== null) {
+    return { _tag: "Registry", name, selector: { _tag: "Range", range: text } }
+  }
+
+  // Whatever is left is most plausibly a dist-tag published by a private
+  // registry, e.g. `"@acme/sdk": "stable"`.
+  if (/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(text)) {
+    return { _tag: "Registry", name, selector: { _tag: "Tag", tag: text } }
+  }
+
+  return { _tag: "Unsupported", name, raw: text, reason: "unrecognised version specifier" }
+}
+
+/**
+ * Parses `npm:<name>[@<range>]`.
+ *
+ * The nested name may itself be scoped, so the `@` split has the same
+ * "not at index 0" caveat as top-level spec parsing.
+ */
+const parseAlias = (key: string, text: string): DependencyTarget => {
+  const body = text.slice("npm:".length)
+  if (body.length === 0) {
+    return { _tag: "Unsupported", name: key, raw: text, reason: "empty npm: alias" }
+  }
+
+  const at = body.lastIndexOf("@")
+  const target = at > 0 ? body.slice(0, at) : body
+  const rangeText = at > 0 ? body.slice(at + 1) : ""
+
+  if (target.length === 0) {
+    return { _tag: "Unsupported", name: key, raw: text, reason: "empty npm: alias target" }
+  }
+
+  const inner = parseDependencyTarget(target, rangeText)
+  if (inner._tag === "Unsupported") {
+    return { ...inner, name: key, raw: text }
+  }
+  return { _tag: "Registry", name: target, selector: inner.selector, aliasOf: key }
+}