@sous-io/sous 0.2.32 → 0.2.33

package/src/lib/markdown-compiler.ts

@@ -11,7 +11,13 @@ import {
   hashContent,
   recordDirCreation,
 } from "./state.js";
-import { resolveInclude, type AliasMap } from "./include-resolver.js";
+import {
+  resolveIncludeFiles,
+  splitIncludeQuery,
+  type AliasMap,
+  type ViewMap,
+} from "./include-resolver.js";
+import { RefSource, sharedRefResolver } from "../services/ref-resolver/index.js";
 import {
   formatNamespaceProblem,
   type NamespaceResolver,
@@ -58,6 +64,8 @@ export type CompilationConfig = {
   targets: CompilationTarget[];
   /** Resolved `@include` alias map (name → ordered base dirs). */
   aliases?: Record<string, string[]>;
+  /** The files each view lists (`#memories`), keyed by the name with its `#`. */
+  views?: ViewMap;
   /** Variable scope for `${var}` substitution in `@include` paths. */
   includeScope?: Record<string, string>;
 };
@@ -145,12 +153,55 @@ export function stableVarsFingerprint(vars: Record<string, string>): string {
   return JSON.stringify(Object.keys(vars).sort().map((key) => [key, vars[key]]));
 }
 
+/**
+ * An include line: `@`, a path that ends in `.md` and an optional
+ * `?name=value` query, and nothing else. The first character is a sigil, a
+ * variable, a dot or a letter, a digit or a glob character; the rest may hold
+ * the characters of a path, a `${var}`, a glob and a recipe reference's `repo:`
+ * qualifier.
+ */
+export const INCLUDE_LINE_PATTERN =
+  /^@([~#a-zA-Z0-9_.${}*?[][a-zA-Z0-9_\-/.:${}*?[\],!~#%+^()]*\.md(?:\?[^?\s]*=\S*)?)$/;
+
+/**
+ * Whether a line that is not a well-formed include still looks like one: it
+ * starts with `@` and is a single path-like word, meaning it holds a `/` or a
+ * `.md`, or starts with a sigil or a variable (`@~`, `@#`, `@.`, `@$`). A line
+ * that merely starts with `@` (a mention or an email address followed by
+ * words, or a lone `@name`) does not.
+ *
+ * looksLikeIncludeLine("@docs/notes.txt"); // -> true, the include has no .md
+ * looksLikeIncludeLine("@alice thanks for the review"); // -> false
+ * looksLikeIncludeLine("@alice"); // -> false
+ *
+ * @param line - One line of a file, with trailing whitespace already removed.
+ */
+export function looksLikeIncludeLine(line: string): boolean {
+  if (!/^@\S+$/.test(line)) return false;
+  const word = line.slice(1);
+  return /[/]/.test(word) || /\.md\b/.test(word) || /^[~#.$]/.test(word);
+}
+
+/** One piece of a read file: a line of its own text, or a file it includes. */
+type Part = string | { file: string };
+
+/** A file read for a target: its text, whether its name makes it a template, and its pieces. */
+type FileNode = {
+  path: string;
+  content: string;
+  /** True when the file's own name contains `.tpl.`: the only files that render as Liquid. */
+  isTpl: boolean;
+  parts: Part[];
+};
+
 export class CompilationService {
   private strict: boolean;
   private rebuild: boolean;
   private dryRun: boolean;
-  private visited: Set<string>;
   private includeStack: string[];
+  private nodes: Map<string, FileNode>;
+  private views: ViewMap;
+  private includedFilesSeen: Set<string>;
   private errors: string[];
   private includeSourceComments: boolean;
   private currentIncludeSourceComments: boolean;
@@ -170,8 +221,10 @@ export class CompilationService {
     this.strict = options.strict ?? false;
     this.rebuild = options.rebuild ?? false;
     this.dryRun = options.dryRun ?? false;
-    this.visited = new Set();
     this.includeStack = [];
+    this.nodes = new Map();
+    this.views = {};
+    this.includedFilesSeen = new Set();
     this.errors = [];
     this.includeSourceComments = false;
     this.currentIncludeSourceComments = false;
@@ -214,23 +267,36 @@ export class CompilationService {
   /**
    * Process @<path> includes in content.
    *
-   * Matches an `@`-prefixed `.md` path on its own line. The path may be:
+   * Matches an `@`-prefixed `.md` path on its own line, optionally followed by
+   * a `?name=value` query (read by the ref service and otherwise ignored). The
+   * path may be:
    *   - relative to the including file (`@sections/intro.md`),
    *   - a `${var}`-substituted path (`@${sousRootPath}/x.md`),
-   *   - an alias path (`@~project/memories/x.md`, `@docs/x.md`), where the
-   *     first segment (up to `/` or `:`) names a registered alias,
-   *   - or a recipe namespace path (`@~workflow/task-files/_partials/x.md`),
-   *     where the `~` sigil names a namespace and the rest names a recipe and a
-   *     file inside it. Namespaces are only consulted when a namespace resolver
-   *     was supplied, and always after aliases.
+   *   - a `#name` path (`@#project/memories/x.md`), or an alias path
+   *     (`@docs/x.md`), where the first segment (up to `/` or `:`) names a
+   *     registered alias,
+   *   - a home-relative path (`@~/notes/x.md`),
+   *   - or a recipe reference (`@~workflow/task-files/_partials/x.md`), where
+   *     the `~` sigil is followed by a namespace, a recipe and a file inside it.
+   *     Recipe references are only consulted when a namespace resolver was
+   *     supplied.
+   * Any of them may use glob syntax, and then includes every file it matches,
+   * in sorted path order; a glob that matches nothing is an error.
+   *
+   * A file may be included any number of times; only a file that includes
+   * itself (directly or through others) is an error.
    *
    * Lines inside fenced code blocks (``` or ~~~, per CommonMark) are left
    * verbatim, so include syntax can be documented without being executed.
    *
-   * Resolution produces an ordered candidate list (see include-resolver); the
-   * first candidate that exists on disk is used. If none exist, it errors,
-   * naming the including file and listing every path tried, plus what went
-   * wrong with the namespace lookup when one was attempted.
+   * A line that starts with `@` and looks like an include (see
+   * {@link looksLikeIncludeLine}) but is not a well-formed one is a compile
+   * error naming the file and the line, never text copied to the output.
+   *
+   * Resolution produces ordered groups of candidates (see include-resolver);
+   * the first group that names a file is used. If none does, it errors, naming
+   * the including file and listing every path tried, plus what went wrong with
+   * a `~` or `#` reference when one was attempted.
    *
    * @param content - The file's raw text.
    * @param baseDir - Directory the relative candidate resolves against.
@@ -242,18 +308,16 @@ export class CompilationService {
     baseDir: string,
     projectRoot: string,
     fromFile?: string
-  ): string {
-    // First segment allows ~ and . (so ./ and ../ work), then path chars;
-    // separators / and :; allows ${...}.
-    const includePattern = /^@([~a-zA-Z0-9_.${}][a-zA-Z0-9_\-/.:${}]*\.md)$/;
+  ): Part[] {
     const fenceOpenPattern = /^ {0,3}(`{3,}|~{3,})/;
     const fenceClosePattern = /^ {0,3}(`{3,}|~{3,})[ \t]*$/;
+    const where = fromFile ?? baseDir;
 
-    const out: string[] = [];
+    const out: Part[] = [];
     let fenceChar: string | null = null;
     let fenceLength = 0;
 
-    for (const line of content.split("\n")) {
+    for (const [lineIndex, line] of content.split("\n").entries()) {
       if (fenceChar !== null) {
         // Inside a fence: emit verbatim; only a matching closing fence ends it.
         const close = fenceClosePattern.exec(line);
@@ -272,28 +336,63 @@ export class CompilationService {
         continue;
       }
 
-      const match = includePattern.exec(line);
+      const trimmed = line.trimEnd();
+      const match = INCLUDE_LINE_PATTERN.exec(trimmed);
       if (!match) {
+        if (looksLikeIncludeLine(trimmed)) {
+          this.handleError(
+            `Malformed include line: ${trimmed}\n  in file: ${where}, line ${lineIndex + 1}\n` +
+              `  An include is "@" followed by a path that ends in .md, optionally followed by ` +
+              `"?name=value". The path may hold letters, digits and . _ - / : $ { } * ? [ ] , ~ #.\n` +
+              `  To write a line that starts with "@" as text, put a word after it.`
+          );
+        }
         out.push(line);
         continue;
       }
 
       const includePath = match[1].trim();
-      const { candidates, namespaceIssue } = resolveInclude(includePath, {
-        aliases: this.aliases,
-        scope: this.includeScope,
-        baseDir,
-        namespaceResolver: this.namespaceResolver,
-        fromFile: fromFile ?? baseDir,
-      });
-      const fullPath = candidates.find((c) => fs.existsSync(c));
-
-      if (!fullPath) {
+      const { files, glob, candidates, namespaceIssue, hashIssue, view } = resolveIncludeFiles(
+        includePath,
+        {
+          aliases: this.aliases,
+          views: this.views,
+          scope: this.includeScope,
+          baseDir,
+          namespaceResolver: this.namespaceResolver,
+          fromFile: where,
+        }
+      );
+
+      if (files.length === 0) {
+        // A view is a list that may be empty: a glob over it that selects
+        // nothing includes nothing, and the line leaves no trace.
+        if (view === true && glob && hashIssue === undefined) continue;
+        if (this.namespaceResolver === undefined && includePath.startsWith("~")) {
+          // Without a resolver the reference is only a relative path, so say
+          // what is wrong with a malformed one instead of "not found".
+          const reference = splitIncludeQuery(includePath).path.slice(1);
+          if (reference.includes("/") && !reference.includes("${")) {
+            try {
+              sharedRefResolver().parse(reference, RefSource.Include);
+            } catch (error) {
+              this.handleError(
+                `Malformed include line: ${trimmed}\n  in file: ${where}, line ${lineIndex + 1}\n  ${
+                  error instanceof Error ? error.message : String(error)
+                }`
+              );
+              out.push("");
+              continue;
+            }
+          }
+        }
         const explanation = namespaceIssue
           ? `\n${formatNamespaceProblem(namespaceIssue)}`
-          : `\n  in file: ${fromFile ?? baseDir}`;
+          : hashIssue
+            ? `\n  in file: ${where}\n  ${hashIssue.split("\n").join("\n  ")}`
+            : `\n  in file: ${where}`;
         this.handleError(
-          `Include not found: @${includePath}${explanation}\n  tried:\n${candidates
+          `${glob ? "Include matched no files" : "Include not found"}: @${includePath}${explanation}\n  tried:\n${candidates
             .map((c) => `    - ${c}`)
             .join("\n")}`
         );
@@ -301,57 +400,121 @@ export class CompilationService {
         continue;
       }
 
-      if (this.includeStack.includes(fullPath)) {
-        this.handleError(
-          `Circular dependency detected: ${this.includeStack.join(" -> ")} -> ${fullPath}`
-        );
-        out.push("");
-        continue;
-      }
+      for (const fullPath of files) {
+        if (this.includeStack.includes(fullPath)) {
+          this.handleError(
+            `Circular dependency detected: ${this.includeStack.join(" -> ")} -> ${fullPath}`
+          );
+          out.push("");
+          continue;
+        }
 
-      const includedContent = this.loadFile(fullPath, projectRoot);
-      if (includedContent !== null) {
-        const relativePath = path.relative(projectRoot, fullPath);
-        const sourceComment = this.currentIncludeSourceComments
-          ? `<!-- from: ${relativePath} -->\n`
-          : "";
-        out.push(sourceComment + includedContent);
-      } else {
-        out.push("");
+        out.push(this.loadFile(fullPath, projectRoot) === null ? "" : { file: fullPath });
       }
     }
 
-    return out.join("\n");
+    return out;
   }
 
-  /** Load a file and recursively process its includes. */
-  private loadFile(filePath: string, projectRoot: string): string | null {
+  /**
+   * Reads a file and, recursively, every file it includes, without rendering
+   * anything. A file already read for this target is not read again (its
+   * includes resolve the same way whoever includes it), so a file may be
+   * included any number of times; only a cycle is an error.
+   */
+  private loadFile(filePath: string, projectRoot: string): FileNode | null {
+    const known = this.nodes.get(filePath);
+    if (known !== undefined) return known;
+
     if (!fs.existsSync(filePath)) {
       this.handleError(`File not found: ${filePath}`);
       return null;
     }
 
-    if (this.visited.has(filePath)) {
-      return "";
-    }
-
     this.includeStack.push(filePath);
 
     try {
       const content = fs.readFileSync(filePath, "utf8");
       const baseDir = path.dirname(filePath);
-      const processedContent = this.processIncludes(content, baseDir, projectRoot, filePath);
-      this.visited.add(filePath);
-      return processedContent;
+      const node: FileNode = {
+        path: filePath,
+        content,
+        isTpl: path.basename(filePath).includes(".tpl."),
+        parts: [],
+      };
+      // Registered before its includes are read so the graph lists parents
+      // first; the cycle check uses the stack, not this map.
+      this.nodes.set(filePath, node);
+      node.parts = this.processIncludes(content, baseDir, projectRoot, filePath);
+      return node;
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       this.handleError(`Failed to read file ${filePath}: ${message}`);
+      this.nodes.delete(filePath);
       return null;
     } finally {
       this.includeStack.pop();
     }
   }
 
+  /**
+   * Turns a read file into its final text.
+   *
+   * A file whose own name contains `.tpl.` is rendered as Liquid with the
+   * given variables; no other file ever is. Each include is left as a unique
+   * marker while the file renders, so a condition around an include line is
+   * honored, and an included file's own output is never rendered again. Every
+   * marker that survives the render is then replaced by that file's final text,
+   * produced the same way.
+   *
+   * @param filePath - The file to expand.
+   * @param vars - The output's variable scope; undefined means templates stay unrendered.
+   * @param entryDir - The directory of the target's entry point, also searched by `{% render %}`.
+   * @param projectRoot - Root used to render source comments as relative paths.
+   */
+  private async expand(
+    filePath: string,
+    vars: Record<string, string> | undefined,
+    entryDir: string,
+    projectRoot: string
+  ): Promise<string> {
+    const node = this.nodes.get(filePath)!;
+    const nonce = `${process.pid}-${Math.random().toString(36).slice(2)}`;
+    const markers = new Map<string, string>();
+    const chunks = node.parts.map((part) => {
+      if (typeof part === "string") return part;
+      const marker = `\u0001SOUS-INCLUDE-${nonce}-${markers.size}\u0001`;
+      markers.set(marker, part.file);
+      return marker;
+    });
+
+    let text = chunks.join("\n");
+    if (node.isTpl && vars !== undefined) {
+      const dir = path.dirname(node.path);
+      text = await this.renderContent(
+        text,
+        { ...vars, sousTemplatePath: node.path, sousTemplateDir: dir },
+        [...new Set([dir, entryDir])],
+        node.path
+      );
+    }
+
+    for (const [marker, child] of markers) {
+      if (!text.includes(marker)) continue;
+      const body = await this.expand(child, vars, entryDir, projectRoot);
+      const comment = this.currentIncludeSourceComments
+        ? `<!-- from: ${path.relative(projectRoot, child)} -->\n`
+        : "";
+      text = text.split(marker).join(comment + body);
+    }
+    return text;
+  }
+
+  /** Every file read for the current target, parents before the files they include. */
+  private graphNodes(): FileNode[] {
+    return [...this.nodes.values()];
+  }
+
   /**
    * Render template content using LiquidJS with the given variable scope.
    *
@@ -368,6 +531,7 @@ export class CompilationService {
   ): Promise<string> {
     const engine = createLiquidEngine(roots, {
       aliases: this.aliases,
+      views: this.views,
       scope: { ...this.includeScope, ...vars },
       namespaceResolver: this.namespaceResolver,
       fromFile,
@@ -473,8 +637,8 @@ ${taskFileContents}
 
     this.initializeEncoder();
 
-    this.visited.clear();
     this.includeStack = [];
+    this.nodes = new Map();
     this.currentIncludeSourceComments =
       typeof target.includeSourceComments === "boolean"
         ? target.includeSourceComments
@@ -487,9 +651,9 @@ ${taskFileContents}
     }
 
     const promptsRoot = path.dirname(target.rootInputPath);
-    const content = this.loadFile(target.rootInputPath, promptsRoot);
+    const entry = this.loadFile(target.rootInputPath, promptsRoot);
 
-    if (content === null) {
+    if (entry === null) {
       displayError(`Failed to compile ${target.rootInputPath}`);
       return false;
     }
@@ -499,12 +663,24 @@ ${taskFileContents}
     // be written: the previous copy stays in place.
     const assembled = this.errors.length === errorsBefore;
 
-    // Compute source hash once per target from the assembled content
-    const contentHash = hashContent(content);
+    // Compute the source hash once per target from every file read: the entry
+    // point and everything it includes, whatever a condition decides later.
+    const graph = this.graphNodes();
+    for (const node of graph) this.includedFilesSeen.add(realPathOf(node.path));
+    const templates = graph.filter((node) => node.isTpl);
+    const contentHash = hashContent(
+      JSON.stringify([
+        this.currentIncludeSourceComments,
+        graph.map((node) => [path.relative(promptsRoot, node.path), node.content]),
+      ])
+    );
 
     let allSucceeded = true;
 
-    const isTpl = path.basename(target.rootInputPath).includes(".tpl.");
+    const isTpl = entry.isTpl;
+    // Whether any file of this target renders as Liquid, which makes the
+    // output depend on its variables.
+    const rendersAnything = templates.length > 0;
 
     for (const output of target.outputs) {
       // Resolve destination path: prefer destinationFile, fall back to destinationDir mirroring
@@ -525,8 +701,8 @@ ${taskFileContents}
       // changed answer or `_vars` value with the same template must re-render,
       // so the variable scope is part of a `.tpl.` output's source hash. A
       // verbatim copy hashes its content alone.
-      const srcHash = isTpl && output.vars
-        ? hashContent(`${content}\n${stableVarsFingerprint(output.vars)}`)
+      const srcHash = rendersAnything && output.vars
+        ? hashContent(`${contentHash}\n${stableVarsFingerprint(output.vars)}`)
         : contentHash;
 
       // Skip if content is unchanged and file already exists (unless --rebuild)
@@ -554,8 +730,10 @@ ${taskFileContents}
       // resolver always sets `vars` on every output (at minimum the inherited
       // scope). It is a real guard for callers that build a CompilationConfig
       // directly, and it stays as a tripwire in case the resolver ever changes.
-      if (isTpl && !output.vars) {
-        this.unrenderedTemplates.push(`${target.rootInputPath} → ${destFile}`);
+      if (rendersAnything && !output.vars) {
+        for (const template of templates) {
+          this.unrenderedTemplates.push(`${template.path} → ${destFile}`);
+        }
         // In strict mode this warning is an error, so the output is not written.
         if (this.strict) {
           this.reportNotWritten(destFile);
@@ -565,18 +743,12 @@ ${taskFileContents}
       }
 
       const errorsBeforeRender = this.errors.length;
-      const resolvedContent = (isTpl && output.vars)
-        ? await this.renderContent(
-            content,
-            {
-              ...output.vars,
-              sousTemplatePath: target.rootInputPath,
-              sousTemplateDir: promptsRoot,
-            },
-            [promptsRoot],
-            target.rootInputPath
-          )
-        : content;
+      const resolvedContent = await this.expand(
+        target.rootInputPath,
+        output.vars,
+        promptsRoot,
+        promptsRoot
+      );
 
       // A template that failed to render is not written either.
       if (this.errors.length > errorsBeforeRender) {
@@ -644,6 +816,15 @@ ${taskFileContents}
     return allSucceeded;
   }
 
+  /**
+   * Every file the last compile read, as real paths: the entry points and every
+   * file any include line of any target named, by any route (exact path, glob
+   * or view). Includes inside an unmet condition count, since they are read.
+   */
+  includedFiles(): ReadonlySet<string> {
+    return this.includedFilesSeen;
+  }
+
   /**
    * Compile all targets from the given config. Every target is compiled even
    * after an error, so every error is listed; an output whose target had an
@@ -675,6 +856,8 @@ ${taskFileContents}
 
       this.includeSourceComments = config.includeSourceComments === true;
       this.aliases = config.aliases ?? {};
+      this.views = config.views ?? {};
+      this.includedFilesSeen = new Set();
       this.includeScope = config.includeScope ?? {};
 
       this.initializeEncoder();
@@ -738,3 +921,12 @@ ${taskFileContents}
 // Backward-compat alias so existing imports of MarkdownCompiler keep working
 export { CompilationService as MarkdownCompiler };
 export type { CompilationServiceOptions as MarkdownCompilerOptions };
+
+/** The path with symbolic links resolved, or the path itself when it cannot be read. */
+function realPathOf(filePath: string): string {
+  try {
+    return fs.realpathSync(filePath);
+  } catch {
+    return filePath;
+  }
+}

package/src/lib/project-scaffold/templates.ts

@@ -33,7 +33,7 @@ export type ProjectConfigFormat = (typeof PROJECT_CONFIG_FORMATS)[number];
  * compiled output (`AGENTS.md` at the project root) are never mistaken for one
  * another.
  */
-export const STARTER_PROMPT_RELATIVE_PATH = "memories/AGENTS.md";
+export const STARTER_PROMPT_RELATIVE_PATH = "memories/AGENTS.tpl.md";
 
 /** The name of the file the starter prompt is compiled into, at the project root. */
 export const STARTER_OUTPUT_NAME = "AGENTS.md";
@@ -98,14 +98,19 @@ export const config = {
     ],
   },
 
-  // Where the recipes this project subscribes to write their files. Every
-  // project subscribes to the \`core\` namespace on its own, which is how the
-  // skills that teach an agent about sous reach \`.claude/skills\`. Memories and
-  // prompts have no default home; name one to receive them.
-  recipeOutputs: {
-    skills: ["\${projectRoot}/.claude/skills"],
-    // memories: ["\${projectRoot}/.claude/memories"],
-    // prompts: ["\${projectRoot}/.claude/prompts"],
+  // What to do with the content the recipes this project subscribes to publish.
+  // Every project subscribes to the \`core\` namespace on its own, which is how
+  // the skills that teach an agent about sous reach \`.claude/skills\`. Memories
+  // are not copied anywhere: the starter prompt pulls them in with one include
+  // line, and the lists below (uncomment to use them) choose which come first
+  // and which are left out. An entry is a glob over \`namespace/recipe\`, or a
+  // regular expression written between two slashes.
+  recipes: {
+    skills: { outputs: ["\${projectRoot}/.claude/skills"] },
+    // memories: {
+    //   first: ["communication/*"],
+    //   exclude: ["tool-usage/automated-browser-tasks", "/^experimental\\//"],
+    // },
   },
 };
 `;
@@ -132,8 +137,8 @@ export function buildConfigJson(context: ProjectScaffoldContext): string {
         },
       ],
     },
-    recipeOutputs: {
-      skills: ["${projectRoot}/.claude/skills"],
+    recipes: {
+      skills: { outputs: ["${projectRoot}/.claude/skills"] },
     },
   };
   return `${JSON.stringify(config, null, 2)}\n`;
@@ -158,6 +163,9 @@ source, run \`sous build\`, and the compiled copy follows; never edit the compil
   each one in with a line holding only \`@sections/name.md\`.
 - The skills under \`.claude/skills\` are written by sous from the recipes this
   project subscribes to. \`sous recipe list\` shows what is available.
+
+{% comment %}This line pulls in the memories of every recipe this project subscribes to.{% endcomment %}
+@#memories/**/*.md
 `;
 }
 

package/src/lib/repos/catalog-inputs.ts

@@ -6,7 +6,7 @@
  * deliberately the only place that knows: the subscription service for the
  * trusted repositories and their cached indexes, the lockfile service for what
  * is pinned, the links map and the store for a recipe's own files, and
- * `recipeOutputs` for where a content kind lands.
+ * `recipes` for where a content kind lands.
  *
  * By default nothing here downloads anything: a repository whose index has
  * never been fetched is left out of the catalog and named separately, so a