@sous-io/sous 0.2.25 → 0.2.27

package/src/lib/repos/release/index-builder.ts

@@ -6,8 +6,10 @@
  * and this module exists to keep them:
  *
  * 1. A published version is IMMUTABLE. An entry that is already in the index
- *    keeps the hash it was published with; a recomputed hash that disagrees is
- *    reported as an error, never quietly written over.
+ *    and carries a tag keeps the hash and the dependencies it was published
+ *    with; a recomputed hash that disagrees, or dependencies that no longer
+ *    agree with what the version's manifest declares, are reported as errors,
+ *    never quietly written over.
  * 2. A version is published when a tag carries it, and every version the index
  *    records has to have one. The single exception is the version a recipe's
  *    manifest declares right now: a release regenerates the index, commits it
@@ -16,12 +18,14 @@
  *    OLDER version without a tag is an error, because that is a published
  *    version nothing can fetch.
  * 3. The tags are the backstop. A version that has a tag but is missing from the
- *    index is rebuilt from the tagged tree, so deleting the index file and
- *    regenerating it restores the same catalog.
+ *    index is rebuilt from the tagged tree, dependencies included, so deleting
+ *    the index file and regenerating it restores the same catalog.
  *
  * Each version entry also records what it was released against, so a consumer
  * installs the versions a recipe was published with rather than re-resolving
- * its ranges long afterwards.
+ * its ranges long afterwards. Those dependencies are resolved exactly once: when
+ * the version is first recorded, against the repository as it stands then (or,
+ * for a version rebuilt from its tag, as it stood at that tag), and never again.
  */
 
 import fs from "node:fs";
@@ -51,6 +55,7 @@ import {
 } from "./tags.js";
 import {
   describeError,
+  validateRepo,
   type RepoValidation,
   type ValidatedRecipe,
   type ValidationProblem,
@@ -160,20 +165,7 @@ export async function buildIndex(
   // and every version each of them publishes. Both are needed to turn a
   // dependency's declared range into the exact version a consumer installs.
   const publishing = options.publishing ?? {};
-  const siblingVersions = new Map<string, string>();
-  const publishedVersions = new Map<string, string[]>();
-  for (const recipe of validation.recipes) {
-    const settled = publishing[recipe.key] ?? recipe.manifest.version;
-    siblingVersions.set(recipe.key, settled);
-    const known = new Set(
-      (tagsByKey.get(recipe.key) ?? []).map((tag) => tag.version)
-    );
-    for (const version of Object.keys(existing?.recipes[recipe.key]?.versions ?? {})) {
-      known.add(version);
-    }
-    known.add(settled);
-    publishedVersions.set(recipe.key, [...known]);
-  }
+  const current = siblingState(validation, tagsByKey, existing, publishing);
 
   for (const recipe of validation.recipes) {
     if (!Object.hasOwn(namespaces, recipe.manifest.namespace)) continue;
@@ -185,14 +177,18 @@ export async function buildIndex(
     const manifestName = path.basename(recipe.manifestPath);
 
     // Rule 3: every tagged version belongs in the index, including ones the
-    // committed index has lost.
+    // committed index has lost. Its dependencies are resolved against the
+    // repository as it stood at the tag, which is what it was released against.
     for (const tag of recipeTags) {
       if (Object.hasOwn(versions, tag.version)) continue;
+      const rebuilt = await rebuildTaggedVersion(rootDir, tag, recipe, tagsByKey, existing, run);
+      problems.push(...rebuilt.problems);
       versions[tag.version] = {
-        hash: await hashTaggedRecipe(rootDir, tag.tag, recipe.path, run),
+        hash: rebuilt.hash,
         tag: tag.tag,
         prerelease: semver.prerelease(tag.version) !== null,
         ...(await releasedAtOf(rootDir, tag.tag, run)),
+        ...(rebuilt.dependencies === undefined ? {} : { dependencies: rebuilt.dependencies }),
       };
     }
 
@@ -201,12 +197,7 @@ export async function buildIndex(
     const hasTag = recipeTags.some((entry) => entry.tag === tagName);
     const workingHash = await hashDirectory(recipe.dir);
     const where = relativeTo(rootDir, recipe.manifestPath);
-    const dependencies = resolveIndexDependencies(
-      recipe,
-      validation,
-      siblingVersions,
-      publishedVersions
-    );
+    const declared = declaredDependencies(recipe, validation);
 
     // A version the index publishes must have a tag, with one exception: the
     // version the manifest declares right now, which is the one a release is in
@@ -242,21 +233,24 @@ export async function buildIndex(
             `now carries. A published version never changes, so either the tag was moved ` +
             `or the index was edited; restore whichever is wrong before releasing again.`,
         });
-      } else {
-        // A release records a version before its tag exists, dating it from the
-        // run; the tag that follows carries a date of its own. Keeping the date
-        // the index already published is what stops the very next regeneration
-        // from calling a perfectly current index stale.
-        versions[version] = {
-          hash: taggedHash,
-          tag: tagName,
-          prerelease: semver.prerelease(version) !== null,
-          ...(published?.releasedAt === undefined
-            ? await releasedAtOf(rootDir, tagName, run)
-            : { releasedAt: published.releasedAt }),
-          ...(dependencies === undefined ? {} : { dependencies }),
-        };
+      } else if (published !== undefined) {
+        // Already published: the entry is carried forward as it stands, its
+        // dependencies and any field a later sous wrote included. Only a missing
+        // release date is filled in, from the tag.
+        if (published.releasedAt === undefined) {
+          versions[version] = { ...published, ...(await releasedAtOf(rootDir, tagName, run)) };
+        }
+        // The check needs the manifest that was published, which the working one
+        // is only while the folder still matches the tag. An entry with no
+        // dependencies at all was published before sous recorded them; a
+        // consumer resolves that version's ranges, and it stays that way.
+        if (taggedHash === workingHash && published.dependencies !== undefined) {
+          problems.push(
+            ...checkRecordedDependencies(key, version, published.dependencies, declared)
+          );
+        }
       }
+      // A tagged version the index did not record was rebuilt from its tag above.
 
       if (taggedHash !== workingHash) {
         // A recipe this run is not publishing is allowed to have unpublished
@@ -284,6 +278,7 @@ export async function buildIndex(
       // follows on the very commit that carries this index.
       const published = existingVersions[version];
       if (options.publishing?.[key] === version || published !== undefined) {
+        const dependencies = resolveIndexDependencies(declared, current);
         versions[version] = {
           hash: workingHash,
           tag: tagName,
@@ -420,50 +415,76 @@ export function describeIndexDrift(
 // --- Internals ----------------------------------------------------------------------------------
 
 /**
- * Turns one recipe's declared dependencies into the exact versions its index
- * entry publishes.
- *
- * This is the whole point of recording dependencies in the index: a consumer
- * installing version 1.4.0 of a recipe installs what 1.4.0 was released
- * against, not whatever its ranges happen to reach months later.
- *
- *   - A SIBLING with no range means "the version released alongside me", which
- *     is the version this run settles that sibling on.
- *   - A SIBLING with a range resolves against the versions this repository
- *     publishes, including the ones this run is adding.
- *   - A CROSS-REPOSITORY dependency carries the identity of the repository it
- *     lives in, which is what a consumer needs in order to add that repository
- *     and find the recipe. Its exact version belongs to that repository's own
- *     index, so the range the manifest declared is recorded instead.
+ * One dependency a recipe's manifest declares, keyed `namespace/recipe` in
+ * `DeclaredDependencies`. A whole-namespace declaration is expanded into one
+ * entry per recipe the namespace holds.
+ */
+type DeclaredDependency =
+  | {
+      /** A recipe in another repository, named by its location. */
+      kind: "remote";
+      /** The canonical identity of the repository publishing it. */
+      repo: string;
+      /** The range the manifest declared, `*` when it declared none. */
+      range: string;
+    }
+  | {
+      /** A recipe in this same repository. */
+      kind: "sibling";
+      /** The range the manifest declared, when it declared one. */
+      range?: string;
+      /**
+       * True when the manifest names this recipe itself, rather than only the
+       * namespace holding it. A recipe a namespace gained after a version was
+       * published is rightly missing from that version's entry; one the
+       * manifest names is not.
+       */
+      named: boolean;
+    };
+
+/** Everything one recipe's manifest declares, under `depends` and `subscribes`. */
+type DeclaredDependencies = {
+  /** Every dependency, keyed `namespace/recipe`; a later declaration wins over an earlier one. */
+  byKey: Map<string, DeclaredDependency>;
+  /** The namespaces of this repository the manifest declares whole. */
+  namespaces: Set<string>;
+};
+
+/** What each recipe of a repository is published at, as one resolution sees it. */
+type SiblingState = {
+  /** The version each recipe is released at, which is what "released alongside me" means. */
+  settled: Map<string, string>;
+  /** Every version each recipe publishes, for resolving a declared range. */
+  published: Map<string, string[]>;
+};
+
+/**
+ * Reads the dependencies a recipe's manifest declares, expanding a
+ * whole-namespace declaration into the recipes that namespace holds.
  *
- * @param recipe - The recipe whose dependencies are being resolved.
+ * @param recipe - The recipe whose dependencies are being read.
  * @param validation - The validated repository, for expanding namespace refs.
- * @param siblingVersions - What each recipe here is published at after this run.
- * @param publishedVersions - Every version each recipe here publishes.
  */
-function resolveIndexDependencies(
+function declaredDependencies(
   recipe: ValidatedRecipe,
-  validation: RepoValidation,
-  siblingVersions: Map<string, string>,
-  publishedVersions: Map<string, string[]>
-): Record<string, IndexDependency> | undefined {
+  validation: RepoValidation
+): DeclaredDependencies {
+  const byKey = new Map<string, DeclaredDependency>();
+  const namespaces = new Set<string>();
   const declared = [
     ...(recipe.manifest.depends ?? []),
     ...(recipe.manifest.subscribes ?? []),
   ];
-  if (declared.length === 0) return undefined;
 
-  const resolved: Record<string, IndexDependency> = {};
-
-  /** Records one sibling, at the version this run settles it on. */
-  const recordSibling = (key: string, range: string | undefined): void => {
-    if (range === undefined) {
-      const settled = siblingVersions.get(key);
-      resolved[key] = settled === undefined ? { range: "*" } : { version: settled };
-      return;
-    }
-    const best = semver.maxSatisfying(publishedVersions.get(key) ?? [], range);
-    resolved[key] = best === null ? { range } : { version: best };
+  /** Records one sibling, remembering whether any declaration named it. */
+  const addSibling = (key: string, range: string | undefined, named: boolean): void => {
+    const before = byKey.get(key);
+    const namedBefore = before?.kind === "sibling" && before.named;
+    byKey.set(key, {
+      kind: "sibling",
+      ...(range === undefined ? {} : { range }),
+      named: named || namedBefore,
+    });
   };
 
   for (const written of declared) {
@@ -476,29 +497,299 @@ function resolveIndexDependencies(
     }
 
     if (parsed.kind === "remote") {
-      resolved[`${parsed.namespace}/${parsed.recipe}`] = {
+      byKey.set(`${parsed.namespace}/${parsed.recipe}`, {
+        kind: "remote",
         repo: parsed.canonicalRepo!,
         range: parsed.range ?? "*",
-      };
+      });
       continue;
     }
 
     if (parsed.recipe !== undefined) {
-      recordSibling(`${parsed.namespace}/${parsed.recipe}`, parsed.range);
+      addSibling(`${parsed.namespace}/${parsed.recipe}`, parsed.range, true);
       continue;
     }
 
     // A whole-namespace dependency means every recipe in that namespace.
+    namespaces.add(parsed.namespace);
     for (const entry of validation.recipes) {
       if (entry.manifest.namespace !== parsed.namespace) continue;
       if (entry.key === recipe.key) continue;
-      recordSibling(entry.key, undefined);
+      addSibling(entry.key, undefined, false);
+    }
+  }
+
+  return { byKey, namespaces };
+}
+
+/**
+ * Works out what each recipe of a repository is published at, for resolving
+ * the dependencies of a version being recorded.
+ *
+ * Two moments call this. A run recording a NEW version resolves against the
+ * repository as it stands, counting the versions the run itself is publishing.
+ * A run rebuilding a version from its TAG resolves against the repository as
+ * it stood at that tag (no `publishing` given): a sibling is settled on the
+ * version its manifest declared there, and only versions up to that one count
+ * as published.
+ *
+ * @param validation - The repository, as it stands or as it stood at a tag.
+ * @param tagsByKey - Every release tag, grouped by recipe.
+ * @param existing - The committed index, when there is one.
+ * @param publishing - The versions this run publishes; omitted for a tag.
+ */
+function siblingState(
+  validation: RepoValidation,
+  tagsByKey: Map<string, RecipeTag[]>,
+  existing: IndexFile | undefined,
+  publishing?: Record<string, string>
+): SiblingState {
+  const settled = new Map<string, string>();
+  const published = new Map<string, string[]>();
+
+  for (const recipe of validation.recipes) {
+    const version = publishing?.[recipe.key] ?? recipe.manifest.version;
+    settled.set(recipe.key, version);
+
+    const known = new Set((tagsByKey.get(recipe.key) ?? []).map((tag) => tag.version));
+    for (const other of Object.keys(existing?.recipes[recipe.key]?.versions ?? {})) {
+      known.add(other);
     }
+    known.add(version);
+
+    const all = [...known];
+    published.set(
+      recipe.key,
+      publishing === undefined ? all.filter((other) => semver.lte(other, version)) : all
+    );
+  }
+
+  return { settled, published };
+}
+
+/**
+ * Turns one recipe's declared dependencies into the exact versions its index
+ * entry publishes.
+ *
+ * This is the whole point of recording dependencies in the index: a consumer
+ * installing version 1.4.0 of a recipe installs what 1.4.0 was released
+ * against, not whatever its ranges happen to reach months later. That is also
+ * why this runs only when a version is first recorded; an entry the index
+ * already publishes is carried forward and never resolved again.
+ *
+ *   - A SIBLING with no range means "the version released alongside me", which
+ *     is the version that sibling is settled on.
+ *   - A SIBLING with a range resolves against the versions this repository
+ *     publishes.
+ *   - A CROSS-REPOSITORY dependency carries the identity of the repository it
+ *     lives in, which is what a consumer needs in order to add that repository
+ *     and find the recipe. Its exact version belongs to that repository's own
+ *     index, so the range the manifest declared is recorded instead.
+ *
+ * @param declared - What the recipe's manifest declares.
+ * @param siblings - What each recipe here is published at, for this resolution.
+ */
+function resolveIndexDependencies(
+  declared: DeclaredDependencies,
+  siblings: SiblingState
+): Record<string, IndexDependency> | undefined {
+  const resolved: Record<string, IndexDependency> = {};
+
+  for (const [key, dependency] of declared.byKey) {
+    if (dependency.kind === "remote") {
+      resolved[key] = { repo: dependency.repo, range: dependency.range };
+      continue;
+    }
+    if (dependency.range === undefined) {
+      const settled = siblings.settled.get(key);
+      resolved[key] = settled === undefined ? { range: "*" } : { version: settled };
+      continue;
+    }
+    const best = semver.maxSatisfying(siblings.published.get(key) ?? [], dependency.range);
+    resolved[key] = best === null ? { range: dependency.range } : { version: best };
   }
 
   return Object.keys(resolved).length === 0 ? undefined : resolved;
 }
 
+/**
+ * Checks the dependencies the index records for an already published version
+ * against what that version's manifest declares.
+ *
+ * The recorded list is frozen, so it is never compared with a fresh
+ * resolution: a sibling released since, or a recipe a declared namespace has
+ * gained since, is exactly what freezing keeps out of it. What must still hold
+ * is that the list honours the manifest: every recipe the manifest names is
+ * there, nothing is there that the manifest does not declare, a sibling sits
+ * inside its declared range, and a dependency in another repository carries the
+ * repository and range the manifest wrote. A disagreement means the index was
+ * edited, or this sous reads the manifest differently from the one that
+ * published the version.
+ *
+ * @param key - The recipe's key, `namespace/recipe`.
+ * @param version - The published version.
+ * @param recorded - The dependencies the index records for it.
+ * @param declared - What the version's manifest declares.
+ */
+function checkRecordedDependencies(
+  key: string,
+  version: string,
+  recorded: Record<string, IndexDependency> | undefined,
+  declared: DeclaredDependencies
+): ValidationProblem[] {
+  const differences: string[] = [];
+  const entries = recorded ?? {};
+
+  for (const [name, dependency] of declared.byKey) {
+    const entry = entries[name];
+    if (entry === undefined) {
+      if (dependency.kind === "remote" || dependency.named) {
+        differences.push(
+          `'${name}': the manifest declares ${describeDeclared(dependency)}, and the index ` +
+            `records nothing for it`
+        );
+      }
+      continue;
+    }
+    if (!recordedSatisfies(entry, dependency)) {
+      differences.push(
+        `'${name}': the index records ${describeRecorded(entry)}, and the manifest declares ` +
+          `${describeDeclared(dependency)}`
+      );
+    }
+  }
+
+  for (const [name, entry] of Object.entries(entries)) {
+    if (declared.byKey.has(name)) continue;
+    // A recipe the declared namespace held when the version was published, and
+    // no longer does, is history rather than a disagreement.
+    const namespace = name.slice(0, name.indexOf("/"));
+    if (entry.repo === undefined && declared.namespaces.has(namespace)) continue;
+    differences.push(
+      `'${name}': the index records ${describeRecorded(entry)}, and the manifest does not ` +
+        `declare it`
+    );
+  }
+
+  if (differences.length === 0) return [];
+  return [
+    {
+      level: "error",
+      where: `${INDEX_FILENAME} recipes['${key}'].versions['${version}'].dependencies`,
+      message:
+        `version ${version} of '${key}' is already published, and the dependencies the index ` +
+        `records for it disagree with what its manifest declares:\n` +
+        differences.map((line) => `    - ${line}\n`).join("") +
+        `  A published version never changes, so a release carries its dependencies forward ` +
+        `exactly as they were published. If the index was edited, restore the entry. To ` +
+        `publish different dependencies, bump the version in this manifest, or with ` +
+        `'sous repo release --bump patch'.`,
+    },
+  ];
+}
+
+/** True when a recorded dependency honours what the manifest declares for it. */
+function recordedSatisfies(entry: IndexDependency, dependency: DeclaredDependency): boolean {
+  if (dependency.kind === "remote") {
+    return (
+      entry.repo === dependency.repo &&
+      entry.range === dependency.range &&
+      entry.version === undefined
+    );
+  }
+  if (entry.repo !== undefined) return false;
+  if (dependency.range === undefined) return true;
+  if (entry.version !== undefined) return semver.satisfies(entry.version, dependency.range);
+  return entry.range === dependency.range;
+}
+
+/** Describes a recorded dependency the way an author would say it. */
+function describeRecorded(entry: IndexDependency): string {
+  const what =
+    entry.version !== undefined ? `version ${entry.version}` : `the range '${entry.range}'`;
+  return entry.repo === undefined ? what : `${what} from ${entry.repo}`;
+}
+
+/** Describes a declared dependency the way an author would say it. */
+function describeDeclared(dependency: DeclaredDependency): string {
+  if (dependency.kind === "remote") {
+    return `the range '${dependency.range}' from ${dependency.repo}`;
+  }
+  return dependency.range === undefined ? "it with no range" : `the range '${dependency.range}'`;
+}
+
+/** What rebuilding one tagged version from its tag produced. */
+type RebuiltVersion = {
+  /** The content hash of the recipe folder at the tag. */
+  hash: string;
+  /** The dependencies it was released against, when they could be worked out. */
+  dependencies?: Record<string, IndexDependency>;
+  /** A warning when they could not. */
+  problems: ValidationProblem[];
+};
+
+/**
+ * Rebuilds a tagged version the index does not record, from the tree its tag
+ * carries: the content hash of the recipe folder, and the dependencies its
+ * manifest declared there, resolved against the repository as it stood at the
+ * tag. That is as close as sous can get to what the release that cut the tag
+ * recorded; it is resolved once, then frozen like any other entry.
+ *
+ * @param rootDir - The repository's root directory.
+ * @param tag - The tag to rebuild from.
+ * @param recipe - The recipe as it stands now, for its folder and its key.
+ * @param tagsByKey - Every release tag, grouped by recipe.
+ * @param existing - The committed index, when there is one.
+ * @param run - The command runner git calls go through.
+ */
+async function rebuildTaggedVersion(
+  rootDir: string,
+  tag: RecipeTag,
+  recipe: ValidatedRecipe,
+  tagsByKey: Map<string, RecipeTag[]>,
+  existing: IndexFile | undefined,
+  run: RunOptions["run"]
+): Promise<RebuiltVersion> {
+  return withTaggedTree(
+    rootDir,
+    tag.tag,
+    ".",
+    async (treeDir) => {
+      const hash = await hashDirectory(path.join(treeDir, ...recipe.path.split("/")));
+
+      let reason: string;
+      try {
+        const snapshot = validateRepo(treeDir);
+        const tagged = snapshot.recipes.find((entry) => entry.key === recipe.key);
+        if (tagged !== undefined) {
+          const dependencies = resolveIndexDependencies(
+            declaredDependencies(tagged, snapshot),
+            siblingState(snapshot, tagsByKey, existing)
+          );
+          return { hash, ...(dependencies === undefined ? {} : { dependencies }), problems: [] };
+        }
+        reason = `The repository manifest at that tag does not list '${recipe.key}'.`;
+      } catch (error) {
+        reason = describeError(error);
+      }
+
+      return {
+        hash,
+        problems: [
+          {
+            level: "warning",
+            where: `${INDEX_FILENAME} recipes['${recipe.key}'].versions['${tag.version}']`,
+            message:
+              `version ${tag.version} was rebuilt from the tag '${tag.tag}' without its ` +
+              `dependencies, because sous could not read the recipe at that tag.\n  ${reason}`,
+          },
+        ],
+      };
+    },
+    { run }
+  );
+}
+
 /** Groups release tags by the recipe key they belong to. */
 function groupTagsByKey(tags: ReadonlyArray<RecipeTag>): Map<string, RecipeTag[]> {
   const grouped = new Map<string, RecipeTag[]>();