@sous-io/sous 0.2.37 → 1.0.2

package/src/lib/repos/locked-namespace-resolver.ts

@@ -1,232 +0,0 @@
-/**
- * The real namespace resolver: the one the compiler uses.
- *
- * Phase 4 defined the `NamespaceResolver` contract and shipped an in-memory
- * implementation for tests. This is the implementation backed by the project's
- * actual state: the lockfile says which recipes are in play and at what version,
- * the links map and the store say where each one's files are, and the
- * lockfile's holder lists say what each recipe is allowed to address.
- *
- * The scoping rule is the whole point, and it is enforced by handing
- * `StaticNamespaceResolver` the right two maps:
- *
- *   - a file inside a recipe may address only that recipe's own declared
- *     dependencies (`depends` plus `subscribes`), at their pinned versions. The
- *     lockfile records which keys each recipe holds, however its manifest wrote
- *     them (a sibling ref, a locator, a URL copied from the browser), so that
- *     record is what decides it;
- *   - a file in the project's own templates may address every recipe the
- *     lockfile pins, whatever holds it: the project, a set it subscribes to,
- *     or another recipe's `depends`.
- *
- * When a project template names a recipe the lockfile does not pin, the error
- * says which recipe used to bring it in, when the cached indexes or the config
- * can tell.
- *
- * A project with no lockfile entries gets no resolver at all, so a project that
- * uses no repositories behaves exactly as it did before: a `~` reference in an
- * include line is just a relative path.
- */
-
-import semver from "semver";
-import type { Settings } from "../settings.js";
-import { resolveStoreRoot } from "../sous-home.js";
-import {
-  RefSource,
-  namespaceOfKey,
-  parseNamedRef,
-  shortKey,
-} from "../../services/ref-resolver/index.js";
-import {
-  StaticNamespaceResolver,
-  type DroppedRecipe,
-  type NamespaceResolver,
-} from "./namespace-resolver.js";
-import {
-  listLockedRecipes,
-  readProjectLockfile,
-  type LockedRecipeLocation,
-} from "./locked-recipes.js";
-import { createIndexCache } from "./providers/index.js";
-import type { IndexFile } from "./formats/index-file.js";
-import type { Lockfile } from "./formats/lockfile.js";
-
-/** How the project's namespace resolver is built. */
-export type ProjectNamespaceResolverOptions = {
-  /** The project's `.sous/` directory, which holds the lockfile and the links map. */
-  sousDir: string;
-  /** The merged project config, read for its subscriptions. */
-  settings: Settings;
-  /** The environment to read; decides where the store and machine-wide links map are. */
-  env?: NodeJS.ProcessEnv;
-  /** The locked recipes, when the caller has already located them. */
-  locked?: LockedRecipeLocation[];
-};
-
-/**
- * Builds the namespace resolver for a project, or undefined when the project
- * locks no recipes at all.
- *
- * createProjectNamespaceResolver({ sousDir, settings })
- * // -> resolves "@~workflow/task-files/_partials/resume.md" (or a glob such as
- * //    "@~*" + "/*" + "/memories/*.md") to the pinned
- * //    recipe directory, when the project (or the including recipe) may address it
- *
- * @param options - The project's `.sous/` directory, its config and the environment.
- */
-export function createProjectNamespaceResolver(
-  options: ProjectNamespaceResolverOptions
-): NamespaceResolver | undefined {
-  const locked =
-    options.locked ??
-    listLockedRecipes({
-      sousDir: options.sousDir,
-      ...(options.env === undefined ? {} : { env: options.env }),
-    });
-
-  if (locked.length === 0) return undefined;
-
-  const recipes: Record<string, string> = {};
-  const repos: Record<string, string> = {};
-  const dependencies: Record<string, string[]> = {};
-
-  for (const recipe of locked) {
-    recipes[recipe.key] = recipe.dir;
-    repos[recipe.key] = recipe.repo;
-
-    // Every recipe holding this one declared it, in `depends` or `subscribes`,
-    // and the lockfile records the key that declaration resolved to.
-    for (const holder of recipe.requestedBy) {
-      (dependencies[holder] ??= []).push(recipe.key);
-    }
-  }
-
-  // No `projectScope`: a project template may address every recipe the
-  // lockfile pins, and those are exactly the recipes this resolver knows.
-  return new StaticNamespaceResolver({
-    recipes,
-    repos,
-    dependencies,
-    explainMissing: (recipe) => explainUnpinnedRecipe(recipe, options),
-  });
-}
-
-/**
- * Why the lockfile does not pin a recipe a project template asked for, when
- * that can be known without the network. Two things are checked, in order:
- *
- *   - a subscription to it (or to its namespace) that is switched off;
- *   - another published version of a pinned recipe that brought it in, read
- *     from the cached indexes of the lockfile's repositories. That version may
- *     declare it directly, or declare a recipe that no longer is pinned either
- *     and that declares it in turn; the shortest such chain is reported, and
- *     among the versions of one pinned recipe the newest wins. This is the
- *     usual story of a set that dropped one of its members, and the members'
- *     own libraries with it.
- *
- * Only ever runs while an error is being reported, so it reads the lockfile
- * and the cached indexes then rather than on every build.
- *
- * @param recipe - The `namespace/recipe` the template asked for.
- * @param options - The project's `.sous/` directory, its config and the environment.
- */
-export function explainUnpinnedRecipe(
-  recipe: string,
-  options: Pick<ProjectNamespaceResolverOptions, "sousDir" | "settings" | "env">
-): DroppedRecipe | undefined {
-  const namespace = namespaceOfKey(recipe);
-  for (const [key, entry] of Object.entries(options.settings.subscriptions ?? {})) {
-    if (entry?.enabled !== false) continue;
-    let ref: string;
-    try {
-      ref = shortKey(parseNamedRef(key, RefSource.Config));
-    } catch {
-      // The config schema already refuses a key that is not a ref.
-      continue;
-    }
-    if (ref === recipe || ref === namespace) {
-      return { by: "disabled-subscription", subscription: key };
-    }
-  }
-
-  let lock: Lockfile;
-  try {
-    lock = readProjectLockfile(options.sousDir);
-  } catch {
-    // The build reports a broken lockfile on its own; an error message is no
-    // place to report it a second time.
-    return undefined;
-  }
-
-  const published = publishedVersions(lock, options.env ?? process.env);
-
-  /** What one published version of a recipe declares, as recipe keys. */
-  const declaredBy = (key: string, version: string): string[] =>
-    Object.keys(published.get(key)?.[version]?.dependencies ?? {});
-
-  // Breadth first from every other version of every pinned recipe, through
-  // recipes the lockfile does not pin, so the shortest chain is found first.
-  type Step = { key: string; root: { recipe: string; version: string }; through: string[] };
-  const queue: Step[] = [];
-  for (const holder of Object.keys(lock.recipes).sort()) {
-    const pinned = lock.recipes[holder]!.version;
-    const versions = Object.keys(published.get(holder) ?? {})
-      .filter((version) => version !== pinned && semver.valid(version) !== null)
-      .sort(semver.rcompare);
-    for (const version of versions) {
-      for (const key of declaredBy(holder, version)) {
-        queue.push({ key, root: { recipe: holder, version }, through: [] });
-      }
-    }
-  }
-
-  const seen = new Set<string>();
-  while (queue.length > 0) {
-    const step = queue.shift()!;
-    if (step.key === recipe) {
-      const root = step.root;
-      return {
-        by: "recipe",
-        recipe: root.recipe,
-        declaredAt: root.version,
-        pinned: lock.recipes[root.recipe]!.version,
-        ...(step.through.length > 0 ? { through: step.through } : {}),
-      };
-    }
-    // A pinned recipe is explained by its own entry, and each recipe is
-    // walked once.
-    if (Object.hasOwn(lock.recipes, step.key) || seen.has(step.key)) continue;
-    seen.add(step.key);
-    for (const version of Object.keys(published.get(step.key) ?? {})) {
-      for (const key of declaredBy(step.key, version)) {
-        queue.push({ key, root: step.root, through: [...step.through, step.key] });
-      }
-    }
-  }
-
-  return undefined;
-}
-
-/**
- * Every published version of every recipe in the cached indexes of the
- * repositories the lockfile names, keyed by recipe. A repository with no
- * cached index contributes nothing.
- *
- * @param lock - The project's lockfile.
- * @param env - The environment, which decides where the store is.
- */
-function publishedVersions(
-  lock: Lockfile,
-  env: NodeJS.ProcessEnv
-): Map<string, IndexFile["recipes"][string]["versions"]> {
-  const cache = createIndexCache({ storeRoot: resolveStoreRoot(env) });
-  const found = new Map<string, IndexFile["recipes"][string]["versions"]>();
-  const identities = new Set(Object.values(lock.repos).map((repo) => repo.identity));
-  for (const identity of [...identities].sort()) {
-    const index = cache.readCached(identity);
-    for (const [key, entry] of Object.entries(index?.recipes ?? {})) {
-      if (!found.has(key)) found.set(key, entry.versions);
-    }
-  }
-  return found;
-}

package/src/lib/repos/locked-recipe-lookup.ts

@@ -1,123 +0,0 @@
-/**
- * What an include line may name: the recipes the lockfile pins.
- *
- * A `RefLookup` over the pinned recipes, answering a `recipeFile` reading (the
- * namespace and recipe names, and the path inside the recipe) with every pinned
- * recipe whose names match it. Names are compared as the ref service compares
- * every name: exact spelling first, ignoring case as the fallback, and a name
- * may be a glob (`*`, `**`, `?`, `[..]`, `{a,b}`). The path inside the recipe
- * is carried through as written; expanding it into files is the include
- * resolver's job, because only it knows which directory each recipe lives in.
- */
-
-import {
-  compareBytewise,
-  matchName,
-  repoOf,
-  type NameMatch,
-  type RefMatch,
-  type SousRef,
-  type SyncRefLookup,
-} from "../../services/ref-resolver/index.js";
-
-/** One pinned recipe, as far as a lookup needs to know it. */
-export type RecipeFileEntry = {
-  /** The recipe's namespace. */
-  namespace: string;
-  /** The recipe's name. */
-  name: string;
-  /** The short name of the repository it came from, when known. */
-  repo?: string;
-};
-
-/** The weakest of two matches: a name that matched only ignoring case spoils an exact one. */
-function combine(left: NameMatch, right: NameMatch): NameMatch {
-  if (left === undefined || right === undefined) return undefined;
-  return left === "exact" && right === "exact" ? "exact" : "folded";
-}
-
-/** Answers which pinned recipes an include reading names. */
-export class LockedRecipeFileLookup implements SyncRefLookup {
-  /**
-   * @param entries - Every recipe the lockfile pins.
-   */
-  constructor(private readonly entries: readonly RecipeFileEntry[]) {}
-
-  /** The distinct namespaces of the pinned recipes, sorted. */
-  namespaces(): string[] {
-    return [...new Set(this.entries.map((entry) => entry.namespace))].sort(compareBytewise);
-  }
-
-  /**
-   * The pinned recipes of the namespaces a written namespace matches, sorted
-   * by key: those whose spelling matches exactly, or, when none does, those
-   * that differ only in case.
-   *
-   * @param written - The namespace as written, possibly a glob.
-   */
-  recipesInNamespace(written: string): string[] {
-    const matched = this.entries
-      .map((entry) => ({ entry, match: matchName(written, entry.namespace) }))
-      .filter((hit) => hit.match !== undefined);
-    const exact = matched.filter((hit) => hit.match === "exact");
-    return (exact.length > 0 ? exact : matched)
-      .map((hit) => `${hit.entry.namespace}/${hit.entry.name}`)
-      .sort(compareBytewise);
-  }
-
-  /**
-   * True when a written namespace matches a pinned one in some spelling.
-   *
-   * @param written - The namespace as written, possibly a glob.
-   */
-  knowsNamespace(written: string): boolean {
-    return this.entries.some((entry) => matchName(written, entry.namespace) !== undefined);
-  }
-
-  findSync(candidate: SousRef): RefMatch[] {
-    if (candidate.kind !== "recipeFile") return [];
-    const written = candidate.recipe;
-    const namespaceName = written.namespace?.name;
-    if (namespaceName === undefined) return [];
-    const repoName = repoOf(candidate)?.name;
-
-    const found: RefMatch[] = [];
-    for (const entry of this.entries) {
-      let match = combine(
-        matchName(namespaceName, entry.namespace),
-        matchName(written.name, entry.name)
-      );
-      if (repoName !== undefined) {
-        match = combine(
-          match,
-          entry.repo === undefined ? undefined : matchName(repoName, entry.repo)
-        );
-      }
-      if (match === undefined) continue;
-      found.push({
-        ref: {
-          kind: "recipeFile",
-          path: candidate.path,
-          recipe: {
-            kind: "recipe",
-            name: entry.name,
-            namespace: { kind: "namespace", name: entry.namespace },
-          },
-          ...(candidate.glob === undefined ? {} : { glob: true as const }),
-          ...(candidate.vars === undefined ? {} : { vars: candidate.vars }),
-        },
-        exactSpelling: match === "exact",
-      });
-    }
-    return found.sort((left, right) => compareBytewise(keyOf(left.ref), keyOf(right.ref)));
-  }
-
-  async find(candidate: SousRef): Promise<RefMatch[]> {
-    return this.findSync(candidate);
-  }
-}
-
-/** The `namespace/recipe` key of a recipe file ref, for ordering. */
-function keyOf(ref: SousRef): string {
-  return ref.kind === "recipeFile" ? `${ref.recipe.namespace?.name}/${ref.recipe.name}` : "";
-}

package/src/lib/repos/locked-recipes.ts

@@ -1,254 +0,0 @@
-/**
- * Where a locked recipe's files actually are.
- *
- * The lockfile pins a recipe to a repository, a version and a content hash; it
- * does not say where the bytes live. Three consumers need that answer and must
- * all get the same one: the namespace resolver (which recipe directory a
- * `~namespace` include lands in), the variable definition source (which
- * manifests `sous vars` reads), and the recipe compile targets (which files a
- * build copies or renders).
- *
- * There are two possible homes, and the order matters. A LINKED repository is
- * read from its working copy, because a link is a deliberate instruction to
- * bypass versions and the lockfile; everything else is read from the immutable
- * store entry at the locked version. Nothing here fetches anything: a recipe the
- * store does not hold yet is reported as absent so the caller can restore it.
- *
- * Reading is done once per call and cached inside the returned list, because a
- * build asks for the same answer several times in a row.
- */
-
-import fs from "node:fs";
-import path from "node:path";
-import { resolveStoreRoot } from "../sous-home.js";
-import { parseLockfile, type LockKind, type Lockfile } from "./formats/lockfile.js";
-import { parseRecipeManifest, type RecipeManifest } from "./formats/recipe-manifest.js";
-import { parseRepoManifest, type RepoManifest } from "./formats/repo-manifest.js";
-import { LOCKFILE_FILENAME } from "./formats/common.js";
-import {
-  findRecipeManifest,
-  findRepoManifest,
-  loadJsonFile,
-  loadManifestFile,
-} from "./load-manifest.js";
-import { readEffectiveLinks } from "./links.js";
-import { identitySegments } from "./identity.js";
-import { splitRecipeKey } from "../../services/ref-resolver/index.js";
-import { PROJECT_HOLDER } from "./formats/lockfile.js";
-
-/** One locked recipe, together with the directory its files are read from. */
-export type LockedRecipeLocation = {
-  /** The recipe key, `namespace/recipe`. */
-  key: string;
-  /** The short name of the repository it came from. */
-  repo: string;
-  /** The recipe's namespace. */
-  namespace: string;
-  /** The recipe's name. */
-  name: string;
-  /** The exact version the lockfile pins. */
-  version: string;
-  /** The content hash the lockfile pins. */
-  hash: string;
-  /** Whether anything holds it as a co-subscription, or only as a build dependency. */
-  kind: LockKind;
-  /** Everyone holding it: "project", and the key of every recipe that asked. */
-  requestedBy: string[];
-  /** Absolute directory holding the recipe's files. */
-  dir: string;
-  /** True when `dir` is a linked working copy rather than a store entry. */
-  linked: boolean;
-  /** True when `dir` exists on disk right now. */
-  present: boolean;
-};
-
-/** How locked recipes are located. */
-export type LockedRecipeOptions = {
-  /** The project's `.sous/` directory, which holds the lockfile and the links map. */
-  sousDir: string;
-  /** The environment to read; decides where the store and the machine-wide links map are. */
-  env?: NodeJS.ProcessEnv;
-  /** The store root to use instead of the one the environment implies. */
-  storeRoot?: string;
-};
-
-/**
- * Reads and validates a project's lockfile, returning an empty one when the
- * project has locked nothing yet.
- *
- * @param sousDir - The project's `.sous/` directory.
- */
-export function readProjectLockfile(sousDir: string): Lockfile {
-  const filePath = path.join(sousDir, LOCKFILE_FILENAME);
-  if (!fs.existsSync(filePath)) return { formatVersion: 1, repos: {}, recipes: {} };
-  return parseLockfile(loadJsonFile(filePath, "lockfile"), filePath);
-}
-
-/**
- * Reads and validates the recipe manifest in a directory, returning undefined
- * when the directory has none or does not exist. A recipe that is not on disk
- * yet is an ordinary state (a fresh clone before restore), never an error here.
- *
- * @param recipeDir - The directory holding the recipe's files.
- */
-export function readRecipeManifestIn(recipeDir: string): RecipeManifest | undefined {
-  let manifestPath: string | undefined;
-  try {
-    manifestPath = findRecipeManifest(recipeDir);
-  } catch {
-    return undefined;
-  }
-  if (manifestPath === undefined) return undefined;
-  return parseRecipeManifest(loadManifestFile(manifestPath), manifestPath);
-}
-
-/**
- * Reads and validates the repo manifest at the root of a checkout, returning
- * undefined when there is none.
- *
- * @param repoRoot - The checkout's root directory.
- */
-export function readRepoManifestIn(repoRoot: string): RepoManifest | undefined {
-  let manifestPath: string | undefined;
-  try {
-    manifestPath = findRepoManifest(repoRoot);
-  } catch {
-    return undefined;
-  }
-  if (manifestPath === undefined) return undefined;
-  return parseRepoManifest(loadManifestFile(manifestPath), manifestPath);
-}
-
-/**
- * Maps every recipe a linked checkout publishes to its directory, by reading the
- * checkout's repo manifest and then each recipe folder's own manifest. A folder
- * the repo manifest lists but that holds no readable recipe manifest is skipped
- * rather than failing the build: a working copy is edited by hand and is allowed
- * to be mid-change.
- *
- * @param checkoutDir - The linked working copy's root directory.
- */
-export function mapLinkedRecipes(checkoutDir: string): Record<string, string> {
-  const manifest = readRepoManifestIn(checkoutDir);
-  if (manifest === undefined) return {};
-
-  const found: Record<string, string> = {};
-  for (const relative of manifest.recipes) {
-    const recipeDir = path.join(checkoutDir, relative);
-    const recipe = readRecipeManifestIn(recipeDir);
-    if (recipe === undefined) continue;
-    found[`${recipe.namespace}/${recipe.name}`] = recipeDir;
-  }
-  return found;
-}
-
-/**
- * Every recipe the lockfile pins, with the directory each one's files are read
- * from. Linked repositories win over the store, and a recipe whose directory is
- * not there yet comes back with `present: false`.
- *
- * @param options - The project's `.sous/` directory and the environment.
- */
-export function listLockedRecipes(
-  options: LockedRecipeOptions
-): LockedRecipeLocation[] {
-  const env = options.env ?? process.env;
-  const lock = readProjectLockfile(options.sousDir);
-  const keys = Object.keys(lock.recipes);
-  if (keys.length === 0) return [];
-
-  const storeRoot = options.storeRoot ?? resolveStoreRoot(env);
-  const links = readEffectiveLinks(options.sousDir, env);
-  const linkedRecipes = new Map<string, Record<string, string>>();
-
-  const located: LockedRecipeLocation[] = [];
-
-  for (const key of keys.sort()) {
-    const entry = lock.recipes[key]!;
-    const { namespace, name } = splitRecipeKey(key);
-
-    let dir: string | undefined;
-    let linked = false;
-
-    const checkout = links[entry.repo]?.path;
-    if (checkout !== undefined) {
-      if (!linkedRecipes.has(entry.repo)) {
-        linkedRecipes.set(entry.repo, mapLinkedRecipes(checkout));
-      }
-      dir = linkedRecipes.get(entry.repo)![key];
-      linked = dir !== undefined;
-    }
-
-    if (dir === undefined) {
-      // The store is machine-wide, so it files an entry under the repository's
-      // canonical identity rather than under this project's short name for it.
-      const identity = lock.repos[entry.repo]?.identity;
-      dir =
-        identity === undefined
-          ? undefined
-          : path.join(storeRoot, ...identitySegments(identity), namespace, name, entry.version);
-    }
-
-    located.push({
-      key,
-      repo: entry.repo,
-      namespace,
-      name,
-      version: entry.version,
-      hash: entry.hash,
-      kind: entry.kind,
-      requestedBy: [...entry.requestedBy],
-      dir: dir ?? "",
-      linked,
-      present: dir !== undefined && fs.existsSync(dir),
-    });
-  }
-
-  return located;
-}
-
-/**
- * The lockfile keys one subscription holds DIRECTLY. A recipe ref holds its own
- * key; a namespace ref holds every recipe in that namespace.
- *
- * Only entries the project itself holds count. A recipe that arrived purely as
- * another recipe's `depends` sits under the same namespace but was never the
- * project's to hold, and counting it made `sous unsubscribe <namespace>` report
- * it as having "stayed" when the project had never held it in the first place.
- *
- * @param lock - The lockfile as it stands.
- * @param key - The subscription's ref key: a namespace, or `namespace/recipe`.
- */
-export function keysHeldBySubscription(lock: Lockfile, key: string): string[] {
-  const heldByProject = (entry: string): boolean =>
-    lock.recipes[entry]?.requestedBy.includes(PROJECT_HOLDER) === true;
-
-  if (key.includes("/")) {
-    return heldByProject(key) ? [key] : [];
-  }
-  return Object.keys(lock.recipes)
-    .filter((entry) => entry.startsWith(`${key}/`) && heldByProject(entry))
-    .sort();
-}
-
-/**
- * Every locked recipe reachable from a set of keys through the lockfile's
- * holder lists: the keys themselves, whatever they hold, and so on.
- *
- * @param lock - The lockfile.
- * @param roots - The keys to start from.
- */
-export function lockedClosure(lock: Lockfile, roots: string[]): string[] {
-  const reached = new Set(roots.filter((key) => Object.hasOwn(lock.recipes, key)));
-  const queue = [...reached];
-  while (queue.length > 0) {
-    const holder = queue.shift()!;
-    for (const [key, entry] of Object.entries(lock.recipes)) {
-      if (reached.has(key) || !entry.requestedBy.includes(holder)) continue;
-      reached.add(key);
-      queue.push(key);
-    }
-  }
-  return [...reached].sort();
-}
-