@sous-io/sous 0.2.25 → 0.2.27

package/src/lib/repos/locked-namespace-resolver.ts

@@ -12,25 +12,37 @@
  *
  *   - a file inside a recipe may address only that recipe's own declared
  *     dependencies (`depends` plus `subscribes`), at their pinned versions;
- *   - a file in the project's own templates may address the project's
- *     subscriptions.
+ *   - a file in the project's own templates may address every recipe the
+ *     lockfile pins, whatever holds it: the project, a set it subscribes to,
+ *     or another recipe's `depends`.
+ *
+ * When a project template names a recipe the lockfile does not pin, the error
+ * says which recipe used to bring it in, when the cached indexes or the config
+ * can tell.
  *
  * A project with no lockfile entries gets no resolver at all, so a project that
  * uses no repositories behaves exactly as it did before: `~` in an include line
  * means an alias and nothing else.
  */
 
+import semver from "semver";
 import type { Settings } from "../settings.js";
+import { resolveStoreRoot } from "../sous-home.js";
 import {
   StaticNamespaceResolver,
+  normalizeRef,
+  type DroppedRecipe,
   type NamespaceResolver,
 } from "./namespace-resolver.js";
 import {
   listLockedRecipes,
-  projectSubscriptionRefs,
+  readProjectLockfile,
   readRecipeManifestIn,
   type LockedRecipeLocation,
 } from "./locked-recipes.js";
+import { createIndexCache } from "./providers/index.js";
+import type { IndexFile } from "./formats/index-file.js";
+import type { Lockfile } from "./formats/lockfile.js";
 
 /** How the project's namespace resolver is built. */
 export type ProjectNamespaceResolverOptions = {
@@ -82,9 +94,125 @@ export function createProjectNamespaceResolver(
     if (declared.length > 0) dependencies[recipe.key] = declared;
   }
 
+  // No `projectScope`: a project template may address every recipe the
+  // lockfile pins, and those are exactly the recipes this resolver knows.
   return new StaticNamespaceResolver({
     recipes,
     dependencies,
-    projectScope: projectSubscriptionRefs(options.settings, locked),
+    explainMissing: (recipe) => explainUnpinnedRecipe(recipe, options),
   });
 }
+
+/**
+ * Why the lockfile does not pin a recipe a project template asked for, when
+ * that can be known without the network. Two things are checked, in order:
+ *
+ *   - a subscription to it (or to its namespace) that is switched off;
+ *   - another published version of a pinned recipe that brought it in, read
+ *     from the cached indexes of the lockfile's repositories. That version may
+ *     declare it directly, or declare a recipe that no longer is pinned either
+ *     and that declares it in turn; the shortest such chain is reported, and
+ *     among the versions of one pinned recipe the newest wins. This is the
+ *     usual story of a set that dropped one of its members, and the members'
+ *     own libraries with it.
+ *
+ * Only ever runs while an error is being reported, so it reads the lockfile
+ * and the cached indexes then rather than on every build.
+ *
+ * @param recipe - The `namespace/recipe` the template asked for.
+ * @param options - The project's `.sous/` directory, its config and the environment.
+ */
+export function explainUnpinnedRecipe(
+  recipe: string,
+  options: Pick<ProjectNamespaceResolverOptions, "sousDir" | "settings" | "env">
+): DroppedRecipe | undefined {
+  const namespace = recipe.slice(0, recipe.indexOf("/"));
+  for (const [key, entry] of Object.entries(options.settings.subscriptions ?? {})) {
+    if (entry?.enabled !== false) continue;
+    const ref = normalizeRef(key);
+    if (ref === recipe || ref === namespace) {
+      return { by: "disabled-subscription", subscription: key };
+    }
+  }
+
+  let lock: Lockfile;
+  try {
+    lock = readProjectLockfile(options.sousDir);
+  } catch {
+    // The build reports a broken lockfile on its own; an error message is no
+    // place to report it a second time.
+    return undefined;
+  }
+
+  const published = publishedVersions(lock, options.env ?? process.env);
+
+  /** What one published version of a recipe declares, as recipe keys. */
+  const declaredBy = (key: string, version: string): string[] =>
+    Object.keys(published.get(key)?.[version]?.dependencies ?? {});
+
+  // Breadth first from every other version of every pinned recipe, through
+  // recipes the lockfile does not pin, so the shortest chain is found first.
+  type Step = { key: string; root: { recipe: string; version: string }; through: string[] };
+  const queue: Step[] = [];
+  for (const holder of Object.keys(lock.recipes).sort()) {
+    const pinned = lock.recipes[holder]!.version;
+    const versions = Object.keys(published.get(holder) ?? {})
+      .filter((version) => version !== pinned && semver.valid(version) !== null)
+      .sort(semver.rcompare);
+    for (const version of versions) {
+      for (const key of declaredBy(holder, version)) {
+        queue.push({ key, root: { recipe: holder, version }, through: [] });
+      }
+    }
+  }
+
+  const seen = new Set<string>();
+  while (queue.length > 0) {
+    const step = queue.shift()!;
+    if (step.key === recipe) {
+      const root = step.root;
+      return {
+        by: "recipe",
+        recipe: root.recipe,
+        declaredAt: root.version,
+        pinned: lock.recipes[root.recipe]!.version,
+        ...(step.through.length > 0 ? { through: step.through } : {}),
+      };
+    }
+    // A pinned recipe is explained by its own entry, and each recipe is
+    // walked once.
+    if (Object.hasOwn(lock.recipes, step.key) || seen.has(step.key)) continue;
+    seen.add(step.key);
+    for (const version of Object.keys(published.get(step.key) ?? {})) {
+      for (const key of declaredBy(step.key, version)) {
+        queue.push({ key, root: step.root, through: [...step.through, step.key] });
+      }
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Every published version of every recipe in the cached indexes of the
+ * repositories the lockfile names, keyed by recipe. A repository with no
+ * cached index contributes nothing.
+ *
+ * @param lock - The project's lockfile.
+ * @param env - The environment, which decides where the store is.
+ */
+function publishedVersions(
+  lock: Lockfile,
+  env: NodeJS.ProcessEnv
+): Map<string, IndexFile["recipes"][string]["versions"]> {
+  const cache = createIndexCache({ storeRoot: resolveStoreRoot(env) });
+  const found = new Map<string, IndexFile["recipes"][string]["versions"]>();
+  const identities = new Set(Object.values(lock.repos).map((repo) => repo.identity));
+  for (const identity of [...identities].sort()) {
+    const index = cache.readCached(identity);
+    for (const [key, entry] of Object.entries(index?.recipes ?? {})) {
+      if (!found.has(key)) found.set(key, entry.versions);
+    }
+  }
+  return found;
+}

package/src/lib/repos/locked-recipes.ts

@@ -20,7 +20,6 @@
 
 import fs from "node:fs";
 import path from "node:path";
-import type { Settings } from "../settings.js";
 import { resolveStoreRoot } from "../sous-home.js";
 import { parseLockfile, type LockKind, type Lockfile } from "./formats/lockfile.js";
 import { parseRecipeManifest, type RecipeManifest } from "./formats/recipe-manifest.js";
@@ -34,7 +33,6 @@ import {
 } from "./load-manifest.js";
 import { readEffectiveLinks } from "./links.js";
 import { identitySegments } from "./identity.js";
-import { enabledSubscriptions } from "./defaults.js";
 import { PROJECT_HOLDER } from "./formats/lockfile.js";
 
 /** One locked recipe, together with the directory its files are read from. */
@@ -233,26 +231,6 @@ export function keysHeldBySubscription(lock: Lockfile, key: string): string[] {
     .sort();
 }
 
-/**
- * The refs a project's own templates may address: everything it subscribed to
- * directly. Both sources are read, because a subscription may be written by
- * `sous subscribe` into the managed layer, hand-written in the primary config,
- * or (for an older project) recorded only in the lockfile.
- *
- * @param settings - The merged project config.
- * @param locked - The locked recipes, as returned by listLockedRecipes.
- */
-export function projectSubscriptionRefs(
-  settings: Settings,
-  locked: LockedRecipeLocation[]
-): string[] {
-  const refs = new Set<string>(Object.keys(enabledSubscriptions(settings)));
-  for (const recipe of locked) {
-    if (recipe.requestedBy.includes(PROJECT_HOLDER)) refs.add(recipe.key);
-  }
-  return [...refs].sort();
-}
-
 /**
  * Every locked recipe reachable from a set of keys through the lockfile's
  * holder lists: the keys themselves, whatever they hold, and so on.

package/src/lib/repos/namespace-resolver.ts

@@ -26,10 +26,31 @@ import path from "node:path";
  * the including file:
  *   - a file that lives inside a recipe may address only that recipe's declared
  *     dependencies (`depends` plus `subscribes`) at their pinned versions;
- *   - a file in the project's own templates may address the project's
- *     subscriptions.
+ *   - a file in the project's own templates may address every recipe the
+ *     project's lockfile pins, whatever holds it.
  */
 
+/**
+ * Why a recipe a project template asked for is not pinned any more, when that
+ * can be known. Only ever attached to an answer for a project template.
+ */
+export type DroppedRecipe =
+  /**
+   * Another version of a pinned recipe declares it (in `depends` or
+   * `subscribes`), and the version the lockfile pins does not: the usual story
+   * of a set that dropped one of its members.
+   */
+  | {
+      by: "recipe";
+      recipe: string;
+      declaredAt: string;
+      pinned: string;
+      /** The recipes, no longer pinned either, that the chain ran through, in order. */
+      through?: string[];
+    }
+  /** The project subscribes to it, but the subscription is switched off. */
+  | { by: "disabled-subscription"; subscription: string };
+
 /** A single `~namespace/rest` resolution request. */
 export type NamespaceRequest = {
   /** The namespace name, with the leading `~` already stripped (e.g. `workflow`). */
@@ -59,20 +80,24 @@ export type NamespaceRequest = {
 export type NamespaceResolution =
   /** Ordered absolute paths to try, most preferred first. An empty list means the lookup produced nothing. */
   | { kind: "candidates"; candidates: string[] }
-  /** No such namespace is known at all. `known` lists the namespaces that are. */
-  | { kind: "unknown-namespace"; known: string[] }
+  /**
+   * No such namespace is known at all. `known` lists the namespaces that are.
+   * `dropped` says why a project template's recipe is no longer pinned, when
+   * that can be known.
+   */
+  | { kind: "unknown-namespace"; known: string[]; dropped?: DroppedRecipe }
   /**
    * The namespace exists but holds no such recipe. `recipe` is the fully
    * qualified ref that was asked for; `known` lists the recipe refs the
-   * namespace does hold.
+   * namespace does hold; `dropped` is as above.
    */
-  | { kind: "unknown-recipe"; recipe: string; known: string[] }
+  | { kind: "unknown-recipe"; recipe: string; known: string[]; dropped?: DroppedRecipe }
   /**
    * The recipe exists but the including file is not allowed to address it.
    * `recipe` is the fully qualified ref that was asked for. `includingRecipe`
    * is the ref of the recipe the including file belongs to, or `null` when the
-   * including file is one of the project's own templates (in which case the
-   * project simply does not subscribe to the recipe).
+   * including file is one of the project's own templates (which only happens
+   * when the resolver was given a narrower `projectScope`).
    */
   | { kind: "not-a-dependency"; recipe: string; includingRecipe: string | null }
   /**
@@ -126,6 +151,7 @@ export function formatNamespaceProblem(opts: {
         ? `Available namespaces: ${resolution.known.join(", ")}.`
         : "This project has no recipe namespaces available yet."
     );
+    if (resolution.dropped) lines.push(...describeDropped(resolution.dropped, opts));
   } else if (resolution.kind === "unknown-recipe") {
     lines.push(`recipe: ${resolution.recipe}`);
     lines.push(`The namespace "${opts.namespace}" holds no recipe named "${resolution.recipe}".`);
@@ -134,6 +160,7 @@ export function formatNamespaceProblem(opts: {
         ? `Recipes in this namespace: ${resolution.known.join(", ")}.`
         : `The namespace "${opts.namespace}" currently holds no recipes.`
     );
+    if (resolution.dropped) lines.push(...describeDropped(resolution.dropped, opts));
   } else if (resolution.kind === "not-a-dependency") {
     lines.push(`recipe: ${resolution.recipe}`);
     if (resolution.includingRecipe) {
@@ -144,10 +171,7 @@ export function formatNamespaceProblem(opts: {
         `Add "${resolution.recipe}" to the "depends" list in that recipe's manifest before addressing it as "~${opts.namespace}".`
       );
     } else {
-      lines.push(`This project does not subscribe to "${resolution.recipe}".`);
-      lines.push(
-        `Subscribe to it before addressing it as "~${opts.namespace}" from a project template.`
-      );
+      lines.push(`The project's own templates may not address "${resolution.recipe}".`);
     }
   } else if (resolution.kind === "escapes-recipe") {
     lines.push(`recipe: ${resolution.recipe}`);
@@ -167,6 +191,37 @@ export function formatNamespaceProblem(opts: {
   return lines.map((line) => `  ${line}`).join("\n");
 }
 
+/**
+ * The lines that say why a recipe a project template asked for is no longer
+ * pinned, and what brings it back.
+ *
+ * @param dropped - What is known about how the recipe used to be pinned.
+ * @param opts - The namespace and the rest of the reference, for the recipe's name.
+ */
+function describeDropped(
+  dropped: DroppedRecipe,
+  opts: { namespace: string; rest: string }
+): string[] {
+  const recipe = `${opts.namespace}/${opts.rest.split("/")[0] ?? ""}`;
+  if (dropped.by === "disabled-subscription") {
+    return [
+      `This project subscribes to "${dropped.subscription}", but that subscription is ` +
+        `switched off ("enabled: false"), so the lockfile does not pin "${recipe}".`,
+      `Switch the subscription back on, or remove the include.`,
+    ];
+  }
+  const through =
+    dropped.through === undefined || dropped.through.length === 0
+      ? ""
+      : ` (through ${dropped.through.map((key) => `"${key}"`).join(", ")})`;
+  return [
+    `Version ${dropped.declaredAt} of "${dropped.recipe}" brought "${recipe}" in${through}, but ` +
+      `the version this project pins, ${dropped.pinned}, does not, so the lockfile no longer ` +
+      `pins it.`,
+    `Subscribe to "${recipe}" directly to keep including it, or remove the include.`,
+  ];
+}
+
 /** Options for {@link StaticNamespaceResolver}. */
 export type StaticNamespaceResolverOptions = {
   /**
@@ -186,9 +241,16 @@ export type StaticNamespaceResolverOptions = {
   /**
    * What the project's own templates may address, in the same ref forms as
    * `dependencies`. Omit it to make every known recipe addressable from
-   * project templates (the convenient default for tests).
+   * project templates, which is the rule for a real project: its templates may
+   * address everything its lockfile pins.
    */
   projectScope?: string[];
+  /**
+   * Says why a recipe a project template asked for is not among `recipes`,
+   * when that can be known. Consulted only for a project template's reference
+   * to a recipe the resolver does not know.
+   */
+  explainMissing?: (recipe: string) => DroppedRecipe | undefined;
 };
 
 /**
@@ -196,7 +258,7 @@ export type StaticNamespaceResolverOptions = {
  * recipe refs to directories.
  *
  * It implements the full scoping rule (recipe files see their declared
- * dependencies; project files see the project's subscriptions) without knowing
+ * dependencies; project files see every known recipe, or `projectScope`) without knowing
  * anything about repositories, versions or the store, which makes it the
  * resolver used by tests and a usable core for the real implementation to wrap.
  */
@@ -204,6 +266,7 @@ export class StaticNamespaceResolver implements NamespaceResolver {
   private readonly recipes: Record<string, string>;
   private readonly dependencies: Record<string, string[]>;
   private readonly projectScope?: string[];
+  private readonly explainMissing?: (recipe: string) => DroppedRecipe | undefined;
 
   constructor(options: StaticNamespaceResolverOptions) {
     this.recipes = {};
@@ -212,6 +275,18 @@ export class StaticNamespaceResolver implements NamespaceResolver {
     }
     this.dependencies = options.dependencies ?? {};
     this.projectScope = options.projectScope;
+    this.explainMissing = options.explainMissing;
+  }
+
+  /**
+   * Why a project template's reference names a recipe this resolver does not
+   * know, or undefined when the including file is a recipe's own or nothing is
+   * known.
+   */
+  private droppedFor(recipe: string, fromFile: string): { dropped?: DroppedRecipe } {
+    if (this.explainMissing === undefined || this.includingRecipe(fromFile) !== null) return {};
+    const dropped = this.explainMissing(recipe);
+    return dropped === undefined ? {} : { dropped };
   }
 
   /** Every namespace this resolver knows about, sorted. */
@@ -254,17 +329,27 @@ export class StaticNamespaceResolver implements NamespaceResolver {
   resolve(request: NamespaceRequest): NamespaceResolution {
     const { namespace, rest, fromFile } = request;
 
-    if (!this.knownNamespaces().includes(namespace)) {
-      return { kind: "unknown-namespace", known: this.knownNamespaces() };
-    }
-
     const segments = rest.split("/").filter((segment) => segment.length > 0);
     const recipeName = segments[0] ?? "";
     const ref = `${namespace}/${recipeName}`;
+
+    if (!this.knownNamespaces().includes(namespace)) {
+      return {
+        kind: "unknown-namespace",
+        known: this.knownNamespaces(),
+        ...this.droppedFor(ref, fromFile),
+      };
+    }
+
     const recipeDir = this.recipes[ref];
 
     if (!recipeDir) {
-      return { kind: "unknown-recipe", recipe: ref, known: this.recipesIn(namespace) };
+      return {
+        kind: "unknown-recipe",
+        recipe: ref,
+        known: this.recipesIn(namespace),
+        ...this.droppedFor(ref, fromFile),
+      };
     }
 
     const includingRecipe = this.includingRecipe(fromFile);