@packall/core 0.0.1

package/src/Layout.ts

@@ -0,0 +1,103 @@
+/**
+ * The on-disk shape of a bundle.
+ *
+ * Everything is laid out exactly the way a registry serves it:
+ *
+ *     lodash/-/lodash-4.17.21.tgz
+ *     @babel/core/-/core-7.24.0.tgz
+ *
+ * That is not an Artifactory convention — it is the path structure in every
+ * `dist.tarball` URL npm publishes, which is why the same tree imports into
+ * Artifactory, Nexus and Verdaccio without translation. Note that a scoped
+ * package's file name drops the scope: `@babel/core` becomes `core-7.24.0.tgz`
+ * under an `@babel/core/-/` directory.
+ */
+
+/** Splits `@scope/name` into its parts. `scope` is undefined when unscoped. */
+export const splitName = (
+  name: string
+): { readonly scope: string | undefined; readonly bare: string } => {
+  if (!name.startsWith("@")) return { scope: undefined, bare: name }
+  const slash = name.indexOf("/")
+  if (slash === -1) return { scope: undefined, bare: name }
+  return { scope: name.slice(0, slash), bare: name.slice(slash + 1) }
+}
+
+/** `@babel/core` + `7.24.0` -> `core-7.24.0.tgz` */
+export const tarballFileName = (name: string, version: string): string => {
+  const { bare } = splitName(name)
+  return `${bare}-${version}.tgz`
+}
+
+/**
+ * The path of one package tarball within a bundle, using `/` separators
+ * regardless of host platform — these become tar entry names, and tar entries
+ * are always POSIX.
+ */
+export const packagePath = (name: string, version: string): string =>
+  `${name}/-/${tarballFileName(name, version)}`
+
+/** File name of the manifest that ships inside every bundle. */
+export const MANIFEST_FILE = "bundle-manifest.json"
+
+/** File name of the short import guide that ships inside every bundle. */
+export const README_FILE = "IMPORT.md"
+
+/**
+ * Name of the archive produced for one root spec in `per-spec` layout.
+ *
+ * Scoped names are flattened (`@babel/core` -> `babel-core`) because a `/` in a
+ * file name is not a thing, and `@` confuses enough shells and web UIs to be
+ * worth avoiding.
+ */
+export const perSpecArchiveName = (name: string, version: string): string => {
+  const { scope, bare } = splitName(name)
+  const prefix = scope === undefined ? bare : `${scope.slice(1)}-${bare}`
+  return `${prefix}-${version}.tgz`
+}
+
+/** Name of the archive produced in `single` layout. */
+export const singleArchiveName = (base = "bundle"): string => `${base}.tgz`
+
+/**
+ * The import guide written into each bundle.
+ *
+ * Kept short and copy-pasteable on purpose: whoever opens this is mid-task on a
+ * restricted network and does not want prose.
+ */
+export const importGuide = (options: {
+  readonly packageCount: number
+  readonly createdAt: string
+  readonly toolVersion: string
+}): string =>
+  `# Importing this bundle
+
+${options.packageCount} package tarball(s), laid out exactly as an npm registry serves them:
+
+    <package-name>/-/<file>.tgz
+    @<scope>/<name>/-/<file>.tgz
+
+Created ${options.createdAt} by packall ${options.toolVersion}.
+See ${MANIFEST_FILE} for the full list with checksums.
+
+## JFrog Artifactory
+
+Unpack, then upload the tree into a **local npm** repository. The layout already
+matches what Artifactory expects, so no path rewriting is needed:
+
+    tar -xzf <this-bundle>.tgz -C ./bundle
+    jf rt upload "bundle/(**)" "<npm-local-repo>/{1}" --flat=false
+
+## Verdaccio / Nexus / any npm registry
+
+Publish each tarball individually:
+
+    find ./bundle -name '*.tgz' -exec npm publish --registry <url> {} \\;
+
+## Verifying before import
+
+    node -e "const m=require('./bundle/${MANIFEST_FILE}');console.log(m.packages.length+' packages')"
+
+Every entry in ${MANIFEST_FILE} carries the integrity string the source registry
+advertised, and each tarball was checked against it at download time.
+`

package/src/Manifest.ts

@@ -0,0 +1,133 @@
+/**
+ * The manifest written into every bundle.
+ *
+ * This is the audit trail. Before anything is imported into a corporate
+ * registry somebody is usually required to answer "what exactly is in here and
+ * where did it come from" — this file answers that without unpacking a single
+ * tarball.
+ */
+import type { Resolution, ResolvedPackage } from "./Resolve.js"
+import { packagePath } from "./Layout.js"
+import type { BundleOptions } from "./Options.js"
+import { formatPlatformFilter } from "./Platform.js"
+import { formatSpec } from "./Spec.js"
+
+/** Schema version, bumped when the shape changes incompatibly. */
+export const MANIFEST_VERSION = 1
+
+export interface ManifestEntry {
+  readonly name: string
+  readonly version: string
+  /** Path of this tarball inside the bundle. */
+  readonly path: string
+  /** The URL it was downloaded from. */
+  readonly tarball: string
+  /** SRI integrity string as advertised by the source registry, when present. */
+  readonly integrity?: string | undefined
+  /** Legacy sha1, present on older published versions. */
+  readonly shasum?: string | undefined
+  /** Actual size on disk, in bytes. */
+  readonly bytes?: number | undefined
+  /** Human-readable justifications: `root:react@18.2.0`, `prod:scheduler@0.23.0`. */
+  readonly reasons: ReadonlyArray<string>
+}
+
+export interface BundleManifest {
+  readonly manifestVersion: number
+  readonly tool: { readonly name: string; readonly version: string }
+  readonly createdAt: string
+  readonly registry: { readonly kind: string; readonly url: string }
+  /** What the user asked for, verbatim. */
+  readonly requested: ReadonlyArray<string>
+  /** What each request expanded to. */
+  readonly roots: ReadonlyArray<{
+    readonly spec: string
+    readonly versions: ReadonlyArray<string>
+    readonly packageCount: number
+  }>
+  readonly options: {
+    readonly layout: string
+    readonly optionalDependencies: boolean
+    readonly peerDependencies: boolean
+    readonly platforms: string
+    readonly allVersions: boolean
+    readonly includePrerelease: boolean
+    readonly integrityVerified: boolean
+  }
+  readonly packages: ReadonlyArray<ManifestEntry>
+  readonly warnings: ReadonlyArray<string>
+  readonly totals: {
+    readonly packages: number
+    readonly bytes: number
+  }
+}
+
+/** Renders a package's reasons into short, greppable strings. */
+export const formatReasons = (pkg: ResolvedPackage): ReadonlyArray<string> =>
+  pkg.reasons.map((reason) =>
+    reason._tag === "Root" ? `root:${reason.spec}` : `${reason.kind}:${reason.from}`
+  )
+
+/** Builds the manifest for a completed (or dry) run. */
+export const buildManifest = (input: {
+  readonly resolution: Resolution
+  /** Only the packages actually included in *this* archive. */
+  readonly included: ReadonlyArray<ResolvedPackage>
+  readonly options: BundleOptions
+  readonly registry: { readonly kind: string; readonly url: string }
+  readonly toolVersion: string
+  readonly sizes: ReadonlyMap<string, number>
+  readonly createdAt?: Date | undefined
+}): BundleManifest => {
+  const includedNames = new Set(input.included.map((p) => `${p.name}@${p.version}`))
+
+  const packages: ReadonlyArray<ManifestEntry> = input.included.map((pkg) => {
+    const key = `${pkg.name}@${pkg.version}`
+    return {
+      name: pkg.name,
+      version: pkg.version,
+      path: packagePath(pkg.name, pkg.version),
+      tarball: pkg.manifest.dist.tarball,
+      integrity: pkg.manifest.dist.integrity,
+      shasum: pkg.manifest.dist.shasum,
+      bytes: input.sizes.get(key),
+      reasons: formatReasons(pkg)
+    }
+  })
+
+  const totalBytes = packages.reduce((sum, entry) => sum + (entry.bytes ?? 0), 0)
+
+  return {
+    manifestVersion: MANIFEST_VERSION,
+    tool: { name: "packall", version: input.toolVersion },
+    createdAt: (input.createdAt ?? new Date()).toISOString(),
+    registry: input.registry,
+    requested: input.resolution.roots.map((root) => formatSpec(root.spec)),
+    roots: input.resolution.roots.map((root) => ({
+      spec: formatSpec(root.spec),
+      versions: root.versions,
+      packageCount: root.closure.filter((key) => includedNames.has(key)).length
+    })),
+    options: {
+      layout: input.options.layout,
+      optionalDependencies: input.options.scope.optional,
+      peerDependencies: input.options.scope.peer,
+      platforms: formatPlatformFilter(input.options.scope.platforms),
+      allVersions: input.options.allVersions,
+      includePrerelease: input.options.includePrerelease,
+      integrityVerified: input.options.verifyIntegrity
+    },
+    packages,
+    warnings: input.resolution.warnings.map((warning) =>
+      warning.from === undefined ? warning.message : `${warning.from}: ${warning.message}`
+    ),
+    totals: {
+      packages: packages.length,
+      bytes: totalBytes
+    }
+  }
+}
+
+/** Serialises a manifest with stable key order and a trailing newline. */
+export const serializeManifest = (manifest: BundleManifest): string =>
+  `${JSON.stringify(manifest, null, 2)}\n`

package/src/Options.ts

@@ -0,0 +1,123 @@
+/**
+ * Option types shared by the resolver and the bundler, plus their defaults.
+ *
+ * Defaults are chosen so that the *unflagged* command produces a bundle that
+ * actually installs offline on a machine that is not the one you built it on.
+ * Every switch that makes the bundle smaller is opt-in.
+ */
+import type { PlatformFilter } from "./Platform.js"
+import { allPlatforms } from "./Platform.js"
+
+/** How the resulting tarball(s) are shaped. */
+export type Layout =
+  /**
+   * One tarball per requested spec, each carrying its own complete dependency
+   * closure. Shared dependencies repeat across tarballs, so the total is
+   * larger — but each one imports, rolls back and hands off independently.
+   */
+  | "per-spec"
+  /**
+   * One tarball for everything, with the union of all closures deduplicated.
+   * Smallest output and a single import step; all-or-nothing to roll back.
+   */
+  | "single"
+  /**
+   * No archive at all — just the npm-layout directory tree, ready for
+   * `jf rt upload` or rsync.
+   */
+  | "dir"
+
+export const layouts: ReadonlyArray<Layout> = ["per-spec", "single", "dir"]
+
+/** Which dependency edges the resolver follows. */
+export interface DependencyScope {
+  /**
+   * Follow `optionalDependencies`. On by default — this is where per-platform
+   * native binaries live, and omitting them is the most common reason an
+   * offline install fails on a machine other than the one that built it.
+   */
+  readonly optional: boolean
+  /**
+   * Follow non-optional `peerDependencies`. On by default, because npm 7+
+   * auto-installs peers and will reach for the network if they are absent.
+   */
+  readonly peer: boolean
+  /** Which platforms optional dependencies are kept for. */
+  readonly platforms: PlatformFilter
+}
+
+export const defaultScope: DependencyScope = {
+  optional: true,
+  peer: true,
+  platforms: allPlatforms
+}
+
+/** Options governing version selection and the dependency walk. */
+export interface ResolveOptions {
+  readonly scope: DependencyScope
+  /**
+   * When a root spec is a range, take *every* published version that satisfies
+   * it rather than only the best match.
+   *
+   * `npmb react@^18` bundles one version; `npmb --all-versions react@^18`
+   * bundles all of them.
+   */
+  readonly allVersions: boolean
+  /** With `allVersions`, keep at most this many (newest first). */
+  readonly maxVersions?: number | undefined
+  /** Let prereleases satisfy ranges that would not normally admit them. */
+  readonly includePrerelease: boolean
+  /** How many registry requests to have in flight at once. */
+  readonly concurrency: number
+}
+
+export const defaultResolveOptions: ResolveOptions = {
+  scope: defaultScope,
+  allVersions: false,
+  includePrerelease: false,
+  concurrency: 10
+}
+
+/** Options for a full bundle run. */
+export interface BundleOptions extends ResolveOptions {
+  readonly layout: Layout
+  /** Directory the finished artifacts are written to. */
+  readonly outDir: string
+  /** Base name for the archive in `single` layout. Defaults to `bundle`. */
+  readonly archiveName?: string | undefined
+  /** Resolve and report, but download nothing and write nothing. */
+  readonly dryRun: boolean
+  /**
+   * Verify every tarball against the checksum the registry advertised.
+   *
+   * On by default and there is essentially no good reason to turn it off; the
+   * escape hatch exists only for registries that serve broken metadata.
+   */
+  readonly verifyIntegrity: boolean
+  /** Overwrite existing files in `outDir` instead of refusing. */
+  readonly force: boolean
+  /**
+   * Planned output files to leave untouched, by file name.
+   *
+   * The engine never asks anything; deciding is the caller's job. The CLI's
+   * `--force prompt` resolves its per-file answers into this set, so "keep the
+   * existing one" means that artifact is simply not produced, and the rest of
+   * the run continues.
+   */
+  readonly skipExisting?: ReadonlySet<string> | undefined
+  /**
+   * Planned output files that may be replaced, by file name.
+   *
+   * A narrower `force`: `--force tsdown` says "replace tsdown's bundle, and
+   * still ask about anything else".
+   */
+  readonly overwrite?: ReadonlySet<string> | undefined
+}
+
+export const defaultBundleOptions: Omit<BundleOptions, "outDir"> = {
+  ...defaultResolveOptions,
+  layout: "per-spec",
+  dryRun: false,
+  verifyIntegrity: true,
+  force: false
+}

package/src/Platform.ts

@@ -0,0 +1,182 @@
+/**
+ * Platform matching for optional dependencies.
+ *
+ * This module exists because of one specific, very annoying failure mode.
+ *
+ * Packages with native code — esbuild, rollup, sharp, swc, lightningcss,
+ * `@parcel/watcher` — publish their binary as a *set* of sibling packages, one
+ * per platform, and list all of them in `optionalDependencies`. npm installs
+ * only the one matching the current machine and silently skips the rest.
+ *
+ * So if you bundle on a Mac and skip the non-matching optional deps, the
+ * resulting bundle installs fine on your Mac and fails on the Linux CI box
+ * inside the firewall — which is the only place it actually matters. Hence the
+ * default is "take every platform" and narrowing is opt-in.
+ */
+
+/** A target to keep, e.g. `linux-x64`, `linux-x64-musl`, or just `linux`. */
+export interface PlatformTarget {
+  readonly os: string
+  /** Undefined means "every architecture for this OS". */
+  readonly cpu?: string | undefined
+  /** `glibc` / `musl`. Undefined means "don't care". */
+  readonly libc?: string | undefined
+}
+
+/**
+ * Which platforms optional dependencies should be resolved for.
+ *
+ * - `All` — keep everything, regardless of `os`/`cpu`/`libc` (the default).
+ * - `Targets` — keep only packages that could install on one of these.
+ */
+export type PlatformFilter =
+  | { readonly _tag: "All" }
+  | { readonly _tag: "Targets"; readonly targets: ReadonlyArray<PlatformTarget> }
+
+export const allPlatforms: PlatformFilter = { _tag: "All" }
+
+export const platformTargets = (targets: ReadonlyArray<PlatformTarget>): PlatformFilter => ({
+  _tag: "Targets",
+  targets
+})
+
+/** The machine we are currently running on. */
+export const currentPlatform = (): PlatformTarget => ({
+  os: process.platform,
+  cpu: process.arch
+})
+
+/**
+ * npm's documented `os` values, plus the names people actually type.
+ *
+ * An OS on its own is a valid target, so a typo has to be caught here — the
+ * whole failure mode this tool exists to prevent is a bundle that looks fine
+ * and installs nothing. `--platform windows` silently matching no binding at
+ * all would be precisely that, so it is corrected rather than accepted.
+ */
+const OS_ALIASES: Readonly<Record<string, string>> = {
+  win: "win32",
+  windows: "win32",
+  mac: "darwin",
+  macos: "darwin",
+  osx: "darwin"
+}
+
+const KNOWN_OS: ReadonlySet<string> = new Set([
+  "aix",
+  "android",
+  "cygwin",
+  "darwin",
+  "freebsd",
+  "haiku",
+  "linux",
+  "netbsd",
+  "openbsd",
+  "sunos",
+  "win32"
+])
+
+const LIBC = new Set(["musl", "glibc"])
+
+/**
+ * Parses `linux`, `linux-x64`, `darwin-arm64`, `win32-x64`, `linux-x64-musl`,
+ * `linux-musl`.
+ *
+ * An OS on its own means every architecture for that OS, which is almost always
+ * what you want: "the Windows and Linux bindings" is a far more natural way to
+ * describe a target set than enumerating four `os-arch` pairs, and it keeps
+ * working when a package adds an arm64 build.
+ *
+ * Returns `null` rather than throwing so the CLI can report every bad value at
+ * once alongside the list of accepted forms.
+ */
+export const parsePlatformTarget = (input: string): PlatformTarget | null => {
+  const parts = input.trim().toLowerCase().split("-").filter((p) => p.length > 0)
+  if (parts.length === 0 || parts.length > 3) return null
+
+  const rawOs = parts[0]!
+  const os = OS_ALIASES[rawOs] ?? rawOs
+  if (!KNOWN_OS.has(os)) return null
+
+  if (parts.length === 1) return { os }
+
+  // `linux-musl` is an OS plus a libc, not an OS plus an architecture.
+  if (parts.length === 2 && LIBC.has(parts[1]!)) return { os, libc: parts[1]! }
+
+  const cpu = parts[1]!
+  if (parts.length === 2) return { os, cpu }
+
+  const libc = parts[2]!
+  if (!LIBC.has(libc)) return null
+  return { os, cpu, libc }
+}
+
+/**
+ * npm's `os`/`cpu`/`libc` fields use a list of allowed values, where a leading
+ * `!` negates. An empty or absent list means "no constraint".
+ *
+ * The semantics npm implements: if any negated entry matches, reject. Otherwise
+ * if there are any positive entries, at least one must match.
+ */
+const listAllows = (list: ReadonlyArray<string> | undefined, value: string): boolean => {
+  if (list === undefined || list.length === 0) return true
+
+  let hasPositive = false
+  let positiveMatched = false
+
+  for (const entry of list) {
+    const normalized = entry.trim().toLowerCase()
+    if (normalized.length === 0) continue
+    if (normalized === "any" || normalized === "*") return true
+
+    if (normalized.startsWith("!")) {
+      if (normalized.slice(1) === value) return false
+    } else {
+      hasPositive = true
+      if (normalized === value) positiveMatched = true
+    }
+  }
+
+  return hasPositive ? positiveMatched : true
+}
+
+/** Constraints declared by a package, as they appear in its manifest. */
+export interface PlatformConstraints {
+  readonly os?: ReadonlyArray<string> | undefined
+  readonly cpu?: ReadonlyArray<string> | undefined
+  readonly libc?: ReadonlyArray<string> | undefined
+}
+
+/** Would this package install on this specific target? */
+export const matchesTarget = (
+  constraints: PlatformConstraints,
+  target: PlatformTarget
+): boolean => {
+  if (!listAllows(constraints.os, target.os)) return false
+  // An undefined cpu means the target is the whole OS, so any architecture the
+  // package declares qualifies.
+  if (target.cpu !== undefined && !listAllows(constraints.cpu, target.cpu)) return false
+  if (target.libc !== undefined && !listAllows(constraints.libc, target.libc)) return false
+  return true
+}
+
+/**
+ * Should this package be included, given the active filter?
+ *
+ * Under `All` this is unconditionally `true` — that is the whole point.
+ */
+export const isIncluded = (
+  constraints: PlatformConstraints,
+  filter: PlatformFilter
+): boolean => {
+  if (filter._tag === "All") return true
+  return filter.targets.some((target) => matchesTarget(constraints, target))
+}
+
+/** Renders a filter for the manifest and for `--dry-run` output. */
+export const formatPlatformFilter = (filter: PlatformFilter): string => {
+  if (filter._tag === "All") return "all"
+  return filter.targets
+    .map((t) => [t.os, t.cpu, t.libc].filter((part) => part !== undefined).join("-"))
+    .join(",")
+}

package/src/Progress.ts

@@ -0,0 +1,106 @@
+/**
+ * Progress reporting.
+ *
+ * The engine emits structured events; it never writes to a terminal. That
+ * separation is what lets the CLI draw a live TTY view, CI print plain lines,
+ * and the test suite assert on an array of events without any of them
+ * interfering with each other.
+ */
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as Ref from "effect/Ref"
+
+/** The phases a bundle run moves through, in order. */
+export type Phase = "preflight" | "resolve" | "download" | "archive" | "done"
+
+/** A structured progress event. */
+export type ProgressEvent =
+  /** A phase began. `total` is set only when it is known up front. */
+  | { readonly _tag: "PhaseStarted"; readonly phase: Phase; readonly total?: number | undefined }
+  /** A phase finished. */
+  | { readonly _tag: "PhaseCompleted"; readonly phase: Phase }
+  /** The resolver discovered a package that has to be included. */
+  | {
+      readonly _tag: "PackageResolved"
+      readonly name: string
+      readonly version: string
+      /** Running count of resolved packages, for a live counter. */
+      readonly resolvedCount: number
+      /** Packages still queued for resolution. */
+      readonly pendingCount: number
+    }
+  /** A tarball download started. */
+  | { readonly _tag: "DownloadStarted"; readonly name: string; readonly version: string }
+  /** A tarball download finished. */
+  | {
+      readonly _tag: "DownloadCompleted"
+      readonly name: string
+      readonly version: string
+      readonly bytes: number
+      readonly completedCount: number
+      readonly totalCount: number
+    }
+  /** A download failed and will be retried. */
+  | {
+      readonly _tag: "DownloadRetrying"
+      readonly name: string
+      readonly version: string
+      readonly attempt: number
+      readonly reason: string
+    }
+  /** An archive is being written. */
+  | { readonly _tag: "ArchiveStarted"; readonly path: string; readonly entryCount: number }
+  /** An archive finished. */
+  | { readonly _tag: "ArchiveCompleted"; readonly path: string; readonly bytes: number }
+  /** Something noteworthy but non-fatal, e.g. a deprecated package. */
+  | { readonly _tag: "Warning"; readonly message: string }
+
+/** The reporting sink. */
+export interface ProgressService {
+  readonly emit: (event: ProgressEvent) => Effect.Effect<void>
+}
+
+/** Service tag for progress reporting. */
+export class Progress extends Context.Service<Progress, ProgressService>()(
+  "@packall/core/Progress"
+) {}
+
+/** Emits an event to whichever reporter is installed. */
+export const emit = (event: ProgressEvent): Effect.Effect<void, never, Progress> =>
+  Effect.gen(function*() {
+    const progress = yield* Progress
+    yield* progress.emit(event)
+  })
+
+/**
+ * Discards every event.
+ *
+ * The default for library consumers and for tests that do not care about
+ * progress — silence should never require ceremony.
+ */
+export const layerSilent: Layer.Layer<Progress> = Layer.succeed(Progress)({
+  emit: () => Effect.void
+})
+
+/** Sends every event to a callback. Used by the CLI renderer. */
+export const layerCallback = (
+  onEvent: (event: ProgressEvent) => void
+): Layer.Layer<Progress> =>
+  Layer.succeed(Progress)({
+    emit: (event) => Effect.sync(() => onEvent(event))
+  })
+
+/**
+ * Accumulates every event into a `Ref`, for assertions.
+ *
+ * Returned as `[layer, ref]` so a test can provide the layer and then read the
+ * transcript afterwards.
+ */
+export const makeCollector = Effect.gen(function*() {
+  const ref = yield* Ref.make<ReadonlyArray<ProgressEvent>>([])
+  const layer = Layer.succeed(Progress)({
+    emit: (event: ProgressEvent) => Ref.update(ref, (events) => [...events, event])
+  })
+  return { layer, events: Ref.get(ref) } as const
+})