@sous-io/sous 0.2.37 → 1.0.1

package/src/lib/repos/recipe-config-layers.ts

@@ -1,316 +0,0 @@
-/**
- * Config layers that come from subscribed recipes.
- *
- * A recipe may contribute `config` content: files that are merged into the
- * subscribing project's configuration rather than written anywhere in it. They
- * are loaded AFTER the primary config and BEFORE the `conf.d/` drop-ins, so a
- * recipe can supply defaults and the project always wins over them.
- *
- * This has to work before variable resolution, and before the settings even
- * exist, because these layers are part of what the settings are built from. It
- * therefore reads nothing but the lockfile, the links map and the store, all of
- * which are locatable from the `.sous/` directory and the environment alone.
- *
- * A recipe layer is JSON (`.json`, or `.jsonc` for JSON with comments) or YAML
- * only. The config kernel would happily import a
- * `.js` layer, and a repository's whole trust story rests on sous being able to
- * read what it publishes without running any of it, so an executable layer from
- * a recipe is refused rather than loaded.
- *
- * A recipe layer is also read HERE rather than by the config kernel, and only
- * the keys on the allowlist below survive the reading. A recipe is content you
- * subscribed to; it does not get to decide what sous trusts or what sous runs.
- * See RECIPE_CONFIG_ALLOWED_KEYS.
- */
-
-import fs from "node:fs";
-import path from "node:path";
-import { globSync } from "glob";
-import { parse as parseYaml } from "yaml";
-import { toGlobPattern } from "../glob-pattern.js";
-import { parseJsoncText } from "./load-manifest.js";
-import { listLockedRecipes, readRecipeManifestIn } from "./locked-recipes.js";
-
-/** The layer extensions a recipe may contribute; the executable ones are refused. */
-export const RECIPE_LAYER_EXTENSIONS = [".json", ".jsonc", ".yaml", ".yml"] as const;
-
-/** Extensions sous can load as a config layer but deliberately will not take from a recipe. */
-export const RECIPE_LAYER_EXECUTABLE_EXTENSIONS = [".js", ".mjs", ".cjs", ".ts"] as const;
-
-/**
- * The only top-level config keys a recipe's config layer may set.
- *
- * A recipe layer is merged into the project's configuration, so without this
- * allowlist a recipe could add a repository to `repos:` (granting itself, and
- * anything published beside it, trust the person never gave), add a
- * subscription, or point a `tools.<name>.command` at a program that `sous
- * launch` then spawns. Every key here is inert with respect to trust and to
- * what sous executes: a recipe can supply variables, aliases, compilation
- * targets, runtime context, where recipe output lands, store knobs and variable
- * mappings, and nothing else.
- *
- * Anything not on this list is dropped with a warning naming the recipe and the
- * key, including keys sous does not recognise at all.
- */
-export const RECIPE_CONFIG_ALLOWED_KEYS = [
-  "_aliases",
-  "_vars",
-  "compilation",
-  "recipes",
-  "runtimeContext",
-  "store",
-  "varMappings",
-] as const;
-
-/**
- * The keys under `recipes` a recipe's config layer may set. `recipes.memories`
- * decides which memories the project includes and in what order, which is the
- * project's own choice, so a recipe may not set it.
- */
-export const RECIPE_CONFIG_ALLOWED_RECIPES_KEYS = ["skills"] as const;
-
-/**
- * Plain-language reasons for the refused keys a recipe is most likely to try,
- * so the warning says why rather than only that.
- */
-const REFUSAL_REASONS: Record<string, string> = {
-  repos:
-    "only you decide which repositories this project trusts, so a recipe may not add one",
-  subscriptions:
-    "only you decide what this project subscribes to, so a recipe may not subscribe on your behalf",
-  tools:
-    "a tool entry names a program sous launches, so a recipe may not add one or change one",
-  _env: "an _env mapping decides which environment variables reach the build",
-  version: "the config version is the project's own to declare",
-  name: "the project's display name is the project's own to declare",
-  $schema: "the schema binding is the project's own editor setting",
-};
-
-/** One recipe config layer, already read and already filtered. */
-export type RecipeConfigLayer = {
-  /** Absolute path of the file it was read from, used for reporting and tracing. */
-  path: string;
-  /** The `namespace/recipe` key of the recipe that contributed it. */
-  recipeKey: string;
-  /** The layer's content, with every key outside the allowlist removed. */
-  config: Record<string, unknown>;
-};
-
-/** What a recipe's config contents came to. */
-export type RecipeConfigLayers = {
-  /** The layers, in the order they should be merged. */
-  layers: RecipeConfigLayer[];
-  /** Absolute paths of those layers, in the same order. */
-  paths: string[];
-  /** Complete, plain-language sentences about anything that was skipped. */
-  warnings: string[];
-};
-
-/**
- * Removes every top-level key a recipe is not allowed to set.
- *
- * Pure, so the rule can be tested on its own: give it whatever a layer file
- * parsed to and it returns what may be merged, plus a sentence for everything
- * it took out.
- *
- * @param recipeKey - The `namespace/recipe` key, named in every warning.
- * @param layerPath - The layer file, named in every warning.
- * @param raw - Whatever the layer file parsed to.
- */
-export function filterRecipeConfigLayer(
-  recipeKey: string,
-  layerPath: string,
-  raw: unknown
-): { config: Record<string, unknown>; warnings: string[] } {
-  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
-    return {
-      config: {},
-      warnings: [
-        `The recipe ${recipeKey} contributes a config layer that is not a set of ` +
-          `configuration keys, so none of it was merged:\n  ${layerPath}\n` +
-          `A config layer has to be an object with configuration keys at its top level.`,
-      ],
-    };
-  }
-
-  const warnings: string[] = [];
-  const allowed = new Set<string>(RECIPE_CONFIG_ALLOWED_KEYS);
-  const config: Record<string, unknown> = {};
-
-  for (const key of Object.keys(raw)) {
-    if (key === "recipes") {
-      const value = (raw as Record<string, unknown>)[key];
-      const kept: Record<string, unknown> = {};
-      if (value !== null && typeof value === "object" && !Array.isArray(value)) {
-        for (const sub of Object.keys(value)) {
-          if ((RECIPE_CONFIG_ALLOWED_RECIPES_KEYS as readonly string[]).includes(sub)) {
-            kept[sub] = (value as Record<string, unknown>)[sub];
-            continue;
-          }
-          warnings.push(
-            `The recipe ${recipeKey} tried to set 'recipes.${sub}' in a config layer, and sous ` +
-              `removed it before merging anything:\n  ${layerPath}\n` +
-              `That key is not one a recipe may set (it configures the project, not the recipe). ` +
-              `Under 'recipes' a recipe's config layer may set only: ` +
-              `${RECIPE_CONFIG_ALLOWED_RECIPES_KEYS.join(", ")}.`
-          );
-        }
-      }
-      config[key] = kept;
-      continue;
-    }
-    if (allowed.has(key)) {
-      config[key] = (raw as Record<string, unknown>)[key];
-      continue;
-    }
-    const reason =
-      REFUSAL_REASONS[key] ?? "sous does not recognise it as a key a recipe may set";
-    warnings.push(
-      `The recipe ${recipeKey} tried to set '${key}' in a config layer, and sous ` +
-        `removed it before merging anything:\n  ${layerPath}\n` +
-        `That key is not one a recipe may set (${reason}). A recipe's config layer may ` +
-        `set only: ${RECIPE_CONFIG_ALLOWED_KEYS.join(", ")}.`
-    );
-  }
-
-  return { config, warnings };
-}
-
-/**
- * Every config layer the recipes this project subscribes to contribute, read
- * and filtered, ordered by recipe key and then by path so the merge order is
- * the same on every machine.
- *
- * Only recipes held through `subscribes` contribute; a build dependency is
- * addressable from the recipe that declared it and changes nothing about the
- * project, its configuration included.
- *
- * @param sousDir - The project's `.sous/` directory.
- * @param env - The environment to read; decides where the store is.
- */
-export function listRecipeConfigLayers(
-  sousDir: string,
-  env: NodeJS.ProcessEnv = process.env
-): RecipeConfigLayers {
-  const result: RecipeConfigLayers = { layers: [], paths: [], warnings: [] };
-
-  let locked;
-  try {
-    locked = listLockedRecipes({ sousDir, env });
-  } catch {
-    // Config discovery runs before anything can report a problem nicely. A
-    // lockfile that does not parse is reported by every other reader of it, so
-    // contributing no layers here is the quiet, correct answer.
-    return result;
-  }
-
-  const executable: string[] = [];
-  const unsupported: string[] = [];
-
-  for (const recipe of locked) {
-    if (recipe.kind !== "subscribes") continue;
-    if (!recipe.present) continue;
-
-    let manifest;
-    try {
-      manifest = readRecipeManifestIn(recipe.dir);
-    } catch {
-      continue;
-    }
-    if (manifest === undefined) continue;
-
-    const found: string[] = [];
-
-    for (const content of manifest.contents) {
-      if (content.kind !== "config") continue;
-
-      const ignore = (content.exclude ?? []).map((pattern) =>
-        toGlobPattern(path.join(recipe.dir, pattern))
-      );
-
-      for (const include of content.include) {
-        for (const filePath of globSync(toGlobPattern(path.join(recipe.dir, include)), {
-          absolute: true,
-          ignore,
-        })) {
-          if (!isFile(filePath)) continue;
-          const extension = path.extname(filePath).toLowerCase();
-          if (
-            (RECIPE_LAYER_EXECUTABLE_EXTENSIONS as readonly string[]).includes(extension)
-          ) {
-            executable.push(filePath);
-            continue;
-          }
-          if (!(RECIPE_LAYER_EXTENSIONS as readonly string[]).includes(extension)) {
-            unsupported.push(filePath);
-            continue;
-          }
-          found.push(path.normalize(filePath));
-        }
-      }
-    }
-
-    for (const layerPath of [...new Set(found)].sort()) {
-      let raw: unknown;
-      try {
-        raw = readLayerFile(layerPath);
-      } catch (error) {
-        result.warnings.push(
-          `The recipe ${recipe.key} contributes a config layer sous could not read, so ` +
-            `none of it was merged:\n  ${layerPath}\n` +
-            `${error instanceof Error ? error.message : String(error)}`
-        );
-        continue;
-      }
-
-      const filtered = filterRecipeConfigLayer(recipe.key, layerPath, raw);
-      result.warnings.push(...filtered.warnings);
-      result.layers.push({
-        path: layerPath,
-        recipeKey: recipe.key,
-        config: filtered.config,
-      });
-      result.paths.push(layerPath);
-    }
-  }
-
-  if (executable.length > 0) {
-    result.warnings.push(
-      `Some subscribed recipes contribute config layers that are program code, and sous ` +
-        `refused to load them:\n` +
-        executable.map((entry) => `  ${entry}`).join("\n") +
-        `\nSous must be able to read everything a repository publishes without running ` +
-        `any of it, so an executable layer from a recipe is never loaded. Publish the ` +
-        `same settings as ${RECIPE_LAYER_EXTENSIONS.join(", ")} instead.`
-    );
-  }
-
-  if (unsupported.length > 0) {
-    result.warnings.push(
-      `Some subscribed recipes contribute config layers written in a format sous does ` +
-        `not read, and they were skipped:\n` +
-        unsupported.map((entry) => `  ${entry}`).join("\n") +
-        `\nA recipe's config layer is written as ${RECIPE_LAYER_EXTENSIONS.join(", ")}.`
-    );
-  }
-
-  return result;
-}
-
-/** Reads and parses one recipe config layer according to its extension. */
-function readLayerFile(layerPath: string): unknown {
-  const text = fs.readFileSync(layerPath, "utf8");
-  const extension = path.extname(layerPath).toLowerCase();
-  if (extension === ".jsonc") return parseJsoncText(text, layerPath);
-  if (extension === ".json") return JSON.parse(text);
-  return parseYaml(text);
-}
-
-/** True when the path is a regular file. */
-function isFile(candidate: string): boolean {
-  try {
-    return fs.statSync(candidate).isFile();
-  } catch {
-    return false;
-  }
-}

package/src/lib/repos/recipe-key-matcher.ts

@@ -1,62 +0,0 @@
-/**
- * Matching recipe keys (`namespace/recipe`) against the lists a config holds, such
- * as `recipes.memories.first` and `recipes.memories.exclude`.
- *
- * Each entry of such a list is a glob (`communication/*`), or, when it is
- * written between two slashes (`/^tool-usage\//`), a regular expression. A
- * regular expression is written as a string because the config kernel passes
- * every layer through JSON, which has no way to carry a `RegExp`.
- */
-
-import { minimatch } from "minimatch";
-import { ConfigError } from "../errors.js";
-
-/** True when the string is written `/.../`, the spelling of a regular expression. */
-export function isRegexSpelling(entry: string): boolean {
-  return entry.length >= 3 && entry.startsWith("/") && entry.endsWith("/");
-}
-
-/**
- * Compiles one entry of a recipe key list into a matcher.
- *
- * @param entry - A glob, or a regular expression written `/.../`.
- * @param where - The config key the entry came from, named in the error, such as
- *   `recipes.memories.exclude`.
- * @throws ConfigError when the entry is a regular expression that does not compile.
- */
-function compileEntry(entry: string, where: string): (key: string) => boolean {
-  if (isRegexSpelling(entry)) {
-    let expression: RegExp;
-    try {
-      expression = new RegExp(entry.slice(1, -1));
-    } catch (error) {
-      const reason = error instanceof Error ? error.message : String(error);
-      throw new ConfigError(
-        `The value ${JSON.stringify(entry)} in '${where}' is written as a regular ` +
-          `expression, but it is not a valid one: ${reason}`
-      );
-    }
-    return (key) => expression.test(key);
-  }
-  return (key) => minimatch(key, entry);
-}
-
-/**
- * Compiles a list of globs and `/.../` regular expressions into one matcher over
- * recipe keys. The matcher is true when ANY entry matches; an empty or missing
- * list matches nothing.
- *
- * compileRecipeKeyMatcher(["communication/*", "/^tool-usage\\//"], "recipes.memories.first")("communication/tone");
- * // -> true
- *
- * @param entries - The configured list.
- * @param where - The config key the list came from, named in any error.
- * @throws ConfigError when an entry is an invalid regular expression.
- */
-export function compileRecipeKeyMatcher(
-  entries: readonly string[] | undefined,
-  where: string
-): (recipeKey: string) => boolean {
-  const matchers = (entries ?? []).map((entry) => compileEntry(entry, where));
-  return (recipeKey) => matchers.some((matches) => matches(recipeKey));
-}

package/src/lib/repos/recipe-memories.ts

@@ -1,178 +0,0 @@
-/**
- * The memories the project's active recipes publish, listed for the `#memories`
- * view and for the build's check that every one of them reached an output.
- *
- * A recipe is ACTIVE when the project holds it directly or a co-subscription
- * does (its lockfile kind is `subscribes`); a recipe held only through
- * `depends` is a library and contributes nothing. A memory is a file matched by
- * an include pattern of a manifest `contents` entry of `kind: memories`, and its
- * virtual path is `<namespace>/<recipe>/<path>`, the path being relative to the
- * static base of that pattern.
- *
- * Order: a recipe comes after every pinned recipe it depends on or subscribes
- * to, ties go to the recipe key (bytewise), and the recipes matching
- * `recipes.memories.first` move to the front in that same relative order. Within
- * a recipe, files are in bytewise path order. Recipes matching
- * `recipes.memories.exclude` are not listed.
- */
-
-import fs from "node:fs";
-import path from "node:path";
-import { globSync } from "glob";
-import { compareBytewise } from "../../services/ref-resolver/index.js";
-import { realPathInside } from "../include-resolver.js";
-import { inferGlobBase } from "../markdown-compiler.js";
-import { toGlobPattern } from "../glob-pattern.js";
-import { compileRecipeKeyMatcher } from "./recipe-key-matcher.js";
-import {
-  listLockedRecipes,
-  readRecipeManifestIn,
-  type LockedRecipeLocation,
-} from "./locked-recipes.js";
-
-/** One memory file an active recipe publishes. */
-export type MemoryFile = {
-  /** The recipe key, `namespace/recipe`. */
-  recipe: string;
-  /** The path inside the `#memories` view, without the `#memories/` prefix. */
-  path: string;
-  /** The real file's absolute path. */
-  file: string;
-  /** The file's path relative to its recipe's directory, for messages. */
-  relative: string;
-};
-
-/** What listing the memories needs. */
-export type MemoryListingOptions = {
-  /** The project's `.sous/` directory. */
-  sousDir: string;
-  /** The environment to read; decides where the store is. */
-  env?: NodeJS.ProcessEnv;
-  /** `recipes.memories.first`: recipes whose memories lead. */
-  first?: readonly string[] | undefined;
-  /** `recipes.memories.exclude`: recipes whose memories are left out. */
-  exclude?: readonly string[] | undefined;
-  /** The locked recipes, when the caller has already located them. */
-  locked?: LockedRecipeLocation[];
-  /**
-   * Receives a sentence for each memory skipped because its real path is
-   * outside its recipe's directory. Defaults to one line on stderr per file.
-   */
-  onWarning?: (message: string) => void;
-};
-
-/** Files already warned about by the default sink, so a repeated listing warns once. */
-const warnedFiles = new Set<string>();
-
-/** The default warning sink: one line on stderr, once per file. */
-function warnOnce(message: string): void {
-  if (warnedFiles.has(message)) return;
-  warnedFiles.add(message);
-  process.stderr.write(`Warning: ${message}\n`);
-}
-
-/**
- * Orders recipe keys so each comes after every pinned recipe it depends on or
- * subscribes to, ties by key. A lockfile entry's `requestedBy` names the recipes
- * that asked for it, which is the dependency edge read backwards. A cycle (which
- * a resolved lockfile should not hold) leaves its members in key order at the end.
- *
- * @param locked - Every pinned recipe.
- */
-export function dependencyOrder(locked: readonly LockedRecipeLocation[]): string[] {
-  // A recipe waits for every pinned recipe that lists it among its holders.
-  const waitingOn = new Map<string, Set<string>>();
-  for (const recipe of locked) waitingOn.set(recipe.key, new Set());
-  for (const dependency of locked) {
-    for (const holder of dependency.requestedBy) {
-      if (holder !== dependency.key) waitingOn.get(holder)?.add(dependency.key);
-    }
-  }
-
-  const ordered: string[] = [];
-  const done = new Set<string>();
-  const remaining = [...waitingOn.keys()].sort(compareBytewise);
-  while (remaining.length > 0) {
-    const index = remaining.findIndex((key) => [...waitingOn.get(key)!].every((dep) => done.has(dep)));
-    if (index === -1) {
-      ordered.push(...remaining);
-      break;
-    }
-    const [key] = remaining.splice(index, 1);
-    ordered.push(key!);
-    done.add(key!);
-  }
-  return ordered;
-}
-
-/**
- * Lists the memories of every active recipe, in view order, with the recipes
- * that `exclude` names left out.
- *
- * @param options - The project, and the `first` and `exclude` lists.
- */
-export function listMemories(options: MemoryListingOptions): MemoryFile[] {
-  const locked =
-    options.locked ??
-    listLockedRecipes({
-      sousDir: options.sousDir,
-      ...(options.env === undefined ? {} : { env: options.env }),
-    });
-  if (locked.length === 0) return [];
-
-  const isFirst = compileRecipeKeyMatcher(options.first, "recipes.memories.first");
-  const isExcluded = compileRecipeKeyMatcher(options.exclude, "recipes.memories.exclude");
-  const byKey = new Map(locked.map((recipe) => [recipe.key, recipe]));
-
-  const order = dependencyOrder(locked);
-  const leading = order.filter((key) => isFirst(key));
-  const rest = order.filter((key) => !isFirst(key));
-
-  const files: MemoryFile[] = [];
-  for (const key of [...leading, ...rest]) {
-    const recipe = byKey.get(key)!;
-    if (recipe.kind !== "subscribes" || !recipe.present || isExcluded(key)) continue;
-    const manifest = readRecipeManifestIn(recipe.dir);
-    if (manifest === undefined) continue;
-
-    const mine = new Map<string, MemoryFile>();
-    for (const content of manifest.contents) {
-      if (content.kind !== "memories") continue;
-      const ignore = (content.exclude ?? []).map((pattern) =>
-        toGlobPattern(path.join(recipe.dir, pattern))
-      );
-      for (const include of content.include) {
-        const pattern = toGlobPattern(path.join(recipe.dir, include));
-        const base = inferGlobBase(pattern);
-        for (const file of globSync(pattern, { absolute: true, ignore, dot: true })) {
-          if (!isFile(file)) continue;
-          if (!realPathInside(file, recipe.dir)) {
-            (options.onWarning ?? warnOnce)(
-              `The memory ${file} of the recipe ${key} was left out, because it is a link ` +
-                `that leads outside the recipe's directory. Replace the link with the file itself.`
-            );
-            continue;
-          }
-          const inside = path.relative(base, file).split(path.sep).join("/");
-          mine.set(file, {
-            recipe: key,
-            path: `${key}/${inside}`,
-            file,
-            relative: path.relative(recipe.dir, file).split(path.sep).join("/"),
-          });
-        }
-      }
-    }
-    files.push(...[...mine.values()].sort((a, b) => compareBytewise(a.path, b.path)));
-  }
-  return files;
-}
-
-/** True when the path is a regular file. */
-function isFile(candidate: string): boolean {
-  try {
-    return fs.statSync(candidate).isFile();
-  } catch {
-    return false;
-  }
-}