@sous-io/sous 0.2.32 → 0.2.34

package/src/lib/settings.ts

@@ -14,6 +14,7 @@ import {
   type DiscoveredConfig,
 } from "./config-discovery.js";
 import { ConfigError } from "./errors.js";
+import { sharedHashNames } from "../services/ref-resolver/index.js";
 import { resolveSousHome } from "./sous-home.js";
 import { validateSettings } from "./config-schema.js";
 import { applyRepoDefaults } from "./repos/defaults.js";
@@ -148,23 +149,25 @@ type StoreConfig = {
 };
 
 /**
- * Where the files a subscribed recipe contributes are written, one list of
- * destination directories per content kind. Each destination is `${var}`
- * substituted like any other config path, and a kind may name several so the
- * same recipe feeds more than one agent directory.
+ * What a project does with each content kind its subscribed recipes publish.
  *
- * Only `skills` has a default (`<project root>/.claude/skills`, the project root
- * being the parent of `.sous/`). A kind with no destination is skipped, with one
- * warning naming this config key, because sous cannot guess where a project
- * wants its memories or its prompts.
+ * `skills.outputs` lists the destination directories for recipe skills, each
+ * `${var}` substituted; the default is `<project root>/.claude/skills`, the
+ * project root being the parent of `.sous/`. `memories.first` and
+ * `memories.exclude` are lists of globs, or `/.../` regular expressions, over
+ * `namespace/recipe` keys.
  */
-type RecipeOutputs = {
-  /** Where recipe skill bundles are written. */
-  skills?: string[];
-  /** Where recipe memory files are written. */
-  memories?: string[];
-  /** Where recipe prompt files are written. */
-  prompts?: string[];
+type RecipesConfig = {
+  skills?: {
+    /** Where recipe skill bundles are written. */
+    outputs?: string[];
+  };
+  memories?: {
+    /** Recipes whose memories come first. */
+    first?: string[];
+    /** Recipes whose memories are left out. */
+    exclude?: string[];
+  };
 };
 
 /** Configuration for a launchable tool (e.g. claude, codex). */
@@ -204,9 +207,9 @@ export type Settings = {
   /** Knobs for the machine-wide recipe store. */
   store?: StoreConfig;
   /**
-   * Where the files subscribed recipes contribute are written, per content kind.
+   * What the project does with each content kind its subscribed recipes publish.
    */
-  recipeOutputs?: RecipeOutputs;
+  recipes?: RecipesConfig;
   /**
    * Variable mapping records, keyed by environment variable name, each bound to
    * one recipe variable written as `namespace/recipe/variableName` with an
@@ -820,30 +823,29 @@ function resolveAnswerLayer(
 }
 
 /**
- * Built-in `@include` aliases, always available and reserved (their names begin
- * with `~` so user `_aliases` can never shadow them). Add new entries here as
- * needed; keep names kebab-case.
+ * Built-in `@include` aliases: every `#` name the registry holds
+ * (`sharedHashNames`), each standing for its directories in this project. They
+ * are reserved (user `_aliases` may not start with `#`), so a user alias can
+ * never shadow one.
  *
- * - `~project` → the consuming project's root (`projectRoot`).
+ * - `#project` is the consuming project's root (`projectRoot`).
  *
- * There is exactly one, on purpose. Files that used to be reached through a
- * built-in alias pointing inside the sous package are published as recipes now,
- * and a recipe's files are addressed by its namespace (`@~workflow/task-files/
- * _partials/resume-task.md`), resolved against what the project has pinned. A
- * `~namespace` reference is NOT an alias: it is resolved separately, after the
- * alias map has been tried; see `locked-namespace-resolver.ts`.
+ * Files that used to be reached through a built-in alias pointing inside the
+ * sous package are published as recipes now, and a recipe's files are addressed
+ * by its namespace (`@~workflow/task-files/_partials/resume-task.md`),
+ * resolved against what the project has pinned. A `~` reference is NOT an
+ * alias: it is resolved separately, after the alias map has been tried; see
+ * `locked-namespace-resolver.ts`.
  */
 export function buildBuiltInAliases(scope: VarScope): AliasMap {
-  const builtIns: AliasMap = {};
-  if (scope.projectRoot) builtIns["~project"] = [scope.projectRoot];
-  return builtIns;
+  return sharedHashNames().aliasMap(scope as Record<string, string>);
 }
 
 /**
  * Resolve the full `@include` alias map: built-ins, then the config's
  * `_aliases` block (user entries prepend, so they are tried first and fall
  * through to built-in bases of the same name). User alias names starting
- * with `~` are rejected (reserved).
+ * with `~` or `#` are a ConfigError (reserved).
  *
  * @param settings - The loaded settings (for the `_aliases` block).
  * @param scope - The resolved settings scope (for ${var} substitution + projectRoot).
@@ -853,7 +855,9 @@ export function resolveAliases(settings: Settings, scope: VarScope): AliasMap {
     builtIns: buildBuiltInAliases(scope),
     userAliases: [settings._aliases],
     scope,
-    onError: warning,
+    onError: (message) => {
+      throw new ConfigError(message);
+    },
   });
 }
 
@@ -1002,7 +1006,7 @@ export function resolveCompilation(
 
       /* c8 ignore start */
       // Glob target: expand pattern into one CompilationTarget per matched file, skipping dirs.
-      // A leading alias (`~project/skills/**`) expands to one candidate pattern per alias
+      // A leading alias (`#project/skills/**`) expands to one candidate pattern per alias
       // base; the first base that matches any files wins, mirroring the first-existing-wins
       // rule of @include resolution.
       const rawPattern = substituteVarsStrict(target.entryGlob!, targetScope, `${where}.entryGlob`);

package/src/lib/vars/answers.ts

@@ -23,6 +23,11 @@
  *     is something to tell the user about, not a reason to refuse to build the
  *     rest of the project.
  *
+ * Alongside them comes `secrets`: every name some definition declares
+ * `secret: true`. The tags that dump a whole scope (`{% showVars %}`,
+ * `{% exportScalarVarsJs %}`) read it so a secret answer never lands in
+ * compiled output unless a template names it explicitly.
+ *
  * A definition's `default` counts as an answer of last resort: the
  * description a publisher writes promises what the default does, and a
  * template rendering an empty string instead would break that promise.
@@ -49,6 +54,8 @@ export interface RecipeAnswers {
   byRecipe: Map<string, VarScope>;
   /** Every required definition that nothing answered and that has no default. */
   unanswered: DefinedVariable[];
+  /** Every variable name some definition declares `secret: true`, answered or not. */
+  secrets: Set<string>;
 }
 
 /** How the answers are resolved. */
@@ -76,7 +83,7 @@ export interface RecipeAnswerOptions {
 
 /** The empty result, for a project that locks nothing. */
 export function noRecipeAnswers(): RecipeAnswers {
-  return { merged: {}, byRecipe: new Map(), unanswered: [] };
+  return { merged: {}, byRecipe: new Map(), unanswered: [], secrets: new Set() };
 }
 
 /**
@@ -103,6 +110,7 @@ export function resolveRecipeAnswers(options: RecipeAnswerOptions): RecipeAnswer
   for (const defined of definitions) {
     const value = answerFor(defined, context);
     const name = defined.definition.name;
+    if (defined.definition.secret) result.secrets.add(name);
 
     if (value === undefined) {
       if (defined.definition.required) result.unanswered.push(defined);

package/src/lib/vars/ask.ts

@@ -42,7 +42,7 @@ import { choicePrompt } from "../../utils/choice-prompt.js";
 import { confirmPrompt } from "../../utils/confirm-prompt.js";
 import { valuePrompt } from "../../utils/value-prompt.js";
 import { ENV_VAR_NAME_PATTERN } from "../repos/formats/patterns.js";
-import { variableReferenceKey } from "../refs/find.js";
+import { refKey, variableRefOf } from "../../services/ref-resolver/index.js";
 import type { VariableDefinition } from "../repos/formats/recipe-manifest.js";
 import {
   definedVariableKey,
@@ -152,7 +152,7 @@ export interface AskOptions {
    * Limit the run to these variables. Each entry is a variable's bare name, its
    * `namespace/recipe.name` key, or its fully qualified
    * `repository:namespace/recipe.name` reference. A command that resolved a
-   * reference through `src/lib/refs/` passes the fully qualified form, which is
+   * reference through `src/services/ref-resolver/` passes the fully qualified form, which is
    * the only spelling that cannot mean two variables at once.
    */
   only?: string[];
@@ -182,7 +182,7 @@ export function answerFileFor(definition: VariableDefinition): AnswerFile {
 function isNamed(defined: DefinedVariable, only: string[] | undefined): boolean {
   if (only === undefined) return false;
   const key = definedVariableKey(defined);
-  const qualified = variableReferenceKey(defined);
+  const qualified = refKey(variableRefOf(defined));
   return only.some(
     (name) => name === defined.definition.name || name === key || name === qualified
   );

package/src/lib/vars/display.ts

@@ -19,9 +19,7 @@ import {
   type DefiningRecipe,
 } from "./definition-source.js";
 import { constraintBullets } from "./validate.js";
-
-/** What the value column shows for a secret whose answer is known. */
-export const HIDDEN_VALUE = "(hidden)";
+import { HIDDEN_VALUE } from "./secrets.js";
 
 /** What the value column shows for a variable nothing has answered. */
 export const UNANSWERED_VALUE = "(unanswered)";

package/src/lib/vars/index.ts

@@ -17,3 +17,4 @@ export * from "./ask.js";
 export * from "./report.js";
 export * from "./preanswers.js";
 export * from "./question-plan.js";
+export * from "./secrets.js";

package/src/lib/vars/mappings.ts

@@ -27,8 +27,14 @@ import {
   updateManagedLayer,
 } from "../repos/managed-layer.js";
 import { VARIABLE_NAME_PATTERN } from "../repos/formats/patterns.js";
-import { RefSource } from "../refs/scopes.js";
-import { formatRef, parseShortRef, type ParsedRef } from "../refs/parse.js";
+import {
+  RefSource,
+  formatRef,
+  namespaceOf,
+  parseNamedRef,
+  repoOf,
+  type NamedRef,
+} from "../../services/ref-resolver/index.js";
 import type { DefinedVariable } from "./definition-source.js";
 
 /** File name of the machine-written mapping record layer. */
@@ -85,18 +91,20 @@ export function parseMappingTarget(input: string): MappingTarget {
   // ref part is read by the one ref parser and held to the stored form.
   const slash = trimmed.lastIndexOf("/");
   const variable = slash === -1 ? "" : trimmed.slice(slash + 1);
-  let ref: ParsedRef = { namespace: "" };
+  let ref: NamedRef | undefined;
   try {
-    ref = parseShortRef(trimmed.slice(0, Math.max(slash, 0)), RefSource.CommandLine);
+    ref = parseNamedRef(trimmed.slice(0, Math.max(slash, 0)), RefSource.CommandLine);
   } catch (error) {
     fail((error as Error).message.split("\n")[0]!.replace(/^Invalid ref '[^']*': /, ""));
   }
-  const { repo, namespace, recipe } = ref;
-  if (recipe === undefined || ref.range !== undefined) {
+  if (ref?.kind !== "recipe" || ref.range !== undefined) {
     fail("a target names a namespace, a recipe and a variable, joined by slashes.");
   }
-  if (formatRef(ref) !== formatRef(ref).toLowerCase()) {
-    fail(`the repository, namespace and recipe in '${formatRef(ref)}' must be lowercase.`);
+  const repo = repoOf(ref!)?.name;
+  const namespace = namespaceOf(ref!)!.name;
+  const recipe = ref!.kind === "recipe" ? ref!.name : "";
+  if (formatRef(ref!) !== formatRef(ref!).toLowerCase()) {
+    fail(`the repository, namespace and recipe in '${formatRef(ref!)}' must be lowercase.`);
   }
   if (!VARIABLE_NAME_PATTERN.test(variable)) {
     fail(
@@ -105,7 +113,7 @@ export function parseMappingTarget(input: string): MappingTarget {
     );
   }
 
-  const parsed: MappingTarget = { namespace, recipe: recipe!, variable };
+  const parsed: MappingTarget = { namespace, recipe, variable };
   if (repo !== undefined) parsed.repo = repo;
   return parsed;
 }

package/src/lib/vars/secrets.ts

@@ -0,0 +1,131 @@
+/**
+ * Which variables hold a secret.
+ *
+ * A recipe marks a variable `secret: true`, and that declaration is the
+ * authority. Everything else in a template's scope (an `_env` mapping, a
+ * project `_vars` entry, an answer to a definition that forgot the flag) is
+ * checked by a deliberately thin heuristic: a name whose last words say it is
+ * a secret (`githubToken`, `OPENAI_API_KEY`, `dbPassword`), or a value in one
+ * of the widely used token formats (a GitHub or GitLab token, an AWS access
+ * key, a private key block, a URL carrying a password). The heuristic only
+ * catches what is very probably a secret; it is a safety net, not a scanner.
+ *
+ * This module imports nothing, so the template tags can use it without
+ * pulling the variables layer into the template engine.
+ */
+
+/** What a hidden value shows in place of the secret. */
+export const HIDDEN_VALUE = "(hidden)";
+
+/**
+ * Name endings that mark a secret, as lowercase words. A name matches when its
+ * last words are one of these, so `apiToken` and `GITHUB_TOKEN` match while
+ * `tokenLimit` does not.
+ */
+const SECRET_NAME_ENDINGS: readonly (readonly string[])[] = [
+  ["token"],
+  ["secret"],
+  ["secrets"],
+  ["password"],
+  ["passwd"],
+  ["passphrase"],
+  ["credential"],
+  ["credentials"],
+  ["apikey"],
+  ["api", "key"],
+  ["private", "key"],
+  ["access", "key"],
+  ["secret", "key"],
+  ["signing", "key"],
+  ["encryption", "key"],
+];
+
+/** Widely used secret formats, each specific enough that a match is very probably a secret. */
+const SECRET_VALUE_PATTERNS: readonly RegExp[] = [
+  /-----BEGIN [A-Z ]*PRIVATE KEY-----/, // a PEM private key
+  /\bgh[pousr]_[A-Za-z0-9]{36,}/, // GitHub token
+  /\bgithub_pat_[A-Za-z0-9_]{22,}/, // GitHub fine-grained token
+  /\bglpat-[A-Za-z0-9_-]{20,}/, // GitLab personal access token
+  /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/, // AWS access key id
+  /\bxox[abposr]-[A-Za-z0-9-]{10,}/, // Slack token
+  /\b[rs]k_live_[A-Za-z0-9]{16,}/, // Stripe live key
+  /\bsk-ant-[A-Za-z0-9_-]{20,}/, // Anthropic API key
+  /\bsk-(?:proj-)?[A-Za-z0-9_-]{32,}/, // OpenAI API key
+  /\bAIza[0-9A-Za-z_-]{35}/, // Google API key
+  /\bnpm_[A-Za-z0-9]{36}\b/, // npm token
+  /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/, // JSON Web Token
+  /^[a-z][a-z0-9+.-]*:\/\/[^/\s:@]+:[^/\s@]+@/i, // a URL carrying a password
+];
+
+/** Values a secret-sounding name may hold without being a secret: nothing, a number, a switch. */
+const PLAIN_VALUE = /^(?:|-?\d+(?:\.\d+)?|true|false|yes|no|on|off)$/i;
+
+/**
+ * The words in a variable name, lowercased: split at `_`, `-`, `.`, spaces and
+ * camelCase boundaries, so `OPENAI_API_KEY`, `openaiApiKey` and `openai-api-key`
+ * all read as `openai api key`.
+ *
+ * @param name - The variable name.
+ */
+function nameWords(name: string): string[] {
+  return name
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+    .split(/[^A-Za-z0-9]+/)
+    .filter((word) => word.length > 0)
+    .map((word) => word.toLowerCase());
+}
+
+/**
+ * Whether a variable name says it holds a secret.
+ *
+ * @param name - The variable name.
+ */
+export function hasSecretName(name: string): boolean {
+  const words = nameWords(name);
+  return SECRET_NAME_ENDINGS.some(
+    (ending) =>
+      words.length >= ending.length &&
+      ending.every((word, i) => words[words.length - ending.length + i] === word)
+  );
+}
+
+/**
+ * Whether a value is in one of the widely used secret formats.
+ *
+ * @param value - The value.
+ */
+export function hasSecretValue(value: string): boolean {
+  return SECRET_VALUE_PATTERNS.some((pattern) => pattern.test(value));
+}
+
+/**
+ * Whether a variable very probably holds a secret: its name says so (and its
+ * value is more than a number or a switch), or its value is in a known secret
+ * format.
+ *
+ * @param name - The variable name.
+ * @param value - Its value.
+ */
+export function looksLikeSecret(name: string, value: string): boolean {
+  if (hasSecretValue(value)) return true;
+  return hasSecretName(name) && !PLAIN_VALUE.test(value.trim());
+}
+
+/**
+ * Whether a variable is to be treated as a secret: a recipe declared it one,
+ * or the heuristic says it very probably is.
+ *
+ * @param name - The variable name.
+ * @param value - Its value.
+ * @param declared - The names recipes declared `secret: true`.
+ */
+export function isSecretVariable(
+  name: string,
+  value: unknown,
+  declared: ReadonlySet<string> = new Set()
+): boolean {
+  if (typeof value !== "string") return false;
+  if (declared.has(name)) return value.length > 0;
+  return looksLikeSecret(name, value);
+}

package/src/services/ref-resolver/container.ts

@@ -0,0 +1,107 @@
+/**
+ * The Inversify container of the ref services.
+ *
+ * Every part is bound once and shared: the parser, every splitter, every
+ * pruner, the resolver and the picker. Splitters, pruners and providers are
+ * bound many times under one token each (`REF_TOKENS`), and the parser and the
+ * resolver receive them as lists, so a plugin adds a splitter, a pruner or a
+ * provider by binding one more under the same token on a container it builds
+ * with `createRefContainer()`:
+ *
+ *     const container = createRefContainer();
+ *     container.bind(REF_TOKENS.Splitter).to(MyNewFormSplitter);
+ *     const resolver = container.get<RefResolverService>(REF_TOKENS.Resolver);
+ *
+ * The `#` names are bound the same way: one more `HashName` under
+ * `REF_TOKENS.HashName` registers a view or a built-in, and a name bound twice
+ * is an error when the registry is first read.
+ *
+ * Callers that have no container of their own use `sharedRefResolver`,
+ * `sharedHashNames` and `sharedRefPicker`, which come from one container built the first time they
+ * are read.
+ */
+
+import { Container } from "inversify";
+import { builtInProviders } from "../../lib/repos/providers/index.js";
+import { EnvVarSplitter } from "./parser/env-var-splitter.js";
+import { LocationSplitter } from "./parser/location-splitter.js";
+import { NamePathSplitter } from "./parser/name-path-splitter.js";
+import { QuerySplitter } from "./parser/query-splitter.js";
+import { RangeSplitter } from "./parser/range-splitter.js";
+import { RefParser } from "./parser/ref-parser.js";
+import { RepoQualifierSplitter } from "./parser/repo-qualifier-splitter.js";
+import { CommandLinePruner } from "./pruners/command-line-pruner.js";
+import { IncludePruner } from "./pruners/include-pruner.js";
+import { ManifestPruner } from "./pruners/manifest-pruner.js";
+import { ConfigPruner, LockfilePruner } from "./pruners/stored-key-pruner.js";
+import { MemoriesHashName } from "../../lib/repos/memories-hash-name.js";
+import { HashNameRegistry, ProjectHashName } from "./hash-names.js";
+import { RefPickerService } from "./ref-picker-service.js";
+import { RefResolverService } from "./ref-resolver-service.js";
+import { REF_TOKENS } from "./tokens.js";
+
+/**
+ * Builds a container holding every ref service. Each call returns a fresh
+ * container, so a plugin or a test may bind more splitters or pruners without
+ * touching anyone else's.
+ */
+export function createRefContainer(): Container {
+  const container = new Container({ defaultScope: "Singleton" });
+
+  for (const provider of builtInProviders()) {
+    container.bind(REF_TOKENS.Provider).toConstantValue(provider);
+  }
+
+  for (const splitter of [
+    QuerySplitter,
+    RangeSplitter,
+    LocationSplitter,
+    RepoQualifierSplitter,
+    NamePathSplitter,
+    EnvVarSplitter,
+  ]) {
+    container.bind(REF_TOKENS.Splitter).to(splitter);
+  }
+
+  for (const pruner of [
+    CommandLinePruner,
+    ConfigPruner,
+    ManifestPruner,
+    LockfilePruner,
+    IncludePruner,
+  ]) {
+    container.bind(REF_TOKENS.Pruner).to(pruner);
+  }
+
+  container.bind(REF_TOKENS.HashName).to(ProjectHashName);
+  container.bind(REF_TOKENS.HashName).to(MemoriesHashName);
+  container.bind(REF_TOKENS.HashNames).to(HashNameRegistry);
+  container.bind(REF_TOKENS.Parser).to(RefParser);
+  container.bind(REF_TOKENS.Resolver).to(RefResolverService);
+  container.bind(REF_TOKENS.Picker).to(RefPickerService);
+  return container;
+}
+
+/** The container the shared instances come from, built on first use. */
+let sharedContainer: Container | undefined;
+
+/** The one container callers with no container of their own share. */
+export function getSharedRefContainer(): Container {
+  sharedContainer ??= createRefContainer();
+  return sharedContainer;
+}
+
+/** The shared resolver, for callers that have no container. */
+export function sharedRefResolver(): RefResolverService {
+  return getSharedRefContainer().get<RefResolverService>(REF_TOKENS.Resolver);
+}
+
+/** The shared registry of `#` names, for callers that have no container. */
+export function sharedHashNames(): HashNameRegistry {
+  return getSharedRefContainer().get<HashNameRegistry>(REF_TOKENS.HashNames);
+}
+
+/** The shared picker, for callers that have no container. */
+export function sharedRefPicker(): RefPickerService {
+  return getSharedRefContainer().get<RefPickerService>(REF_TOKENS.Picker);
+}

package/src/services/ref-resolver/dependency.ts

@@ -0,0 +1,60 @@
+/**
+ * Settling a manifest dependency that reads more than one way, from what is
+ * already at hand.
+ *
+ * A manifest entry usually says exactly what it means. A GitLab URL with nested
+ * groups does not say where the project path ends, and a browser URL names a
+ * folder, so the parser returns every reading. A release settles such an entry
+ * and records the repository it settled on beside each key it resolved
+ * (`RecordedDependencies`); a consumer reads that record and never probes.
+ */
+
+import { sharedRefResolver } from "./container.js";
+import {
+  isConfirmedByRecord,
+  type RecordedDependencies,
+} from "./lookups/recorded-dependency-lookup.js";
+import { RefSource } from "./source.js";
+import type { SousRef } from "./types.js";
+
+/** What `settleDependency` may consult, beyond the dependency itself. */
+export type SettleDependencyOptions = {
+  /** What the declaring version's index entry records about its dependencies. */
+  recorded?: RecordedDependencies;
+  /**
+   * Whether a reading is already known to be right from what is on this
+   * machine: its repository is trusted and its cached index publishes what it
+   * names. Used only when the index recorded nothing, and it never fetches.
+   */
+  known?: (reading: SousRef) => boolean;
+};
+
+/**
+ * The one reading a manifest dependency settles on, or undefined when it reads
+ * more than one way and nothing at hand says which: the declaring recipe's
+ * index recorded no answer, and what this machine already knows does not pick
+ * exactly one.
+ *
+ * settleDependency("workflow/alpha");
+ * // -> the recipe workflow/alpha
+ *
+ * @param written - The dependency as the manifest wrote it.
+ * @param options - The index's record, and what is known locally.
+ * @throws A ConfigError when a manifest may not write the entry that way.
+ */
+export function settleDependency(
+  written: string,
+  options: SettleDependencyOptions = {}
+): SousRef | undefined {
+  const { refs } = sharedRefResolver().parse(written, RefSource.Manifest);
+  if (refs.length === 1) return refs[0];
+
+  const confirmed = refs.filter((ref) => isConfirmedByRecord(ref, options.recorded));
+  if (confirmed.length === 1) return confirmed[0];
+
+  if (options.known !== undefined) {
+    const known = refs.filter(options.known);
+    if (known.length === 1) return known[0];
+  }
+  return undefined;
+}