@sous-io/sous 0.2.32 → 0.2.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +7 -6
  2. package/docs/markdown/_sidebar.md +1 -0
  3. package/docs/markdown/commands.md +8 -4
  4. package/docs/markdown/configuration.md +28 -4
  5. package/docs/markdown/repositories-authoring.md +22 -2
  6. package/docs/markdown/repositories-consuming.md +40 -11
  7. package/docs/markdown/repositories-file-formats.md +28 -13
  8. package/docs/markdown/repositories-quickstart.md +1 -1
  9. package/docs/markdown/repositories-variables.md +16 -0
  10. package/docs/markdown/repositories.md +8 -4
  11. package/package.json +2 -1
  12. package/recipes/core/sous-skills/skills/about-liquid-templates/SKILL.tpl.md +33 -20
  13. package/recipes/core/sous-skills/skills/about-sous-configuration/SKILL.tpl.md +4 -0
  14. package/recipes/core/sous-skills/skills/about-sous-recipes/SKILL.tpl.md +21 -9
  15. package/recipes/core/sous-skills/sous.recipe.yaml +1 -1
  16. package/sous.config.schema.json +27 -15
  17. package/src/commands/lock/rebuild.ts +3 -6
  18. package/src/commands/namespace/show.ts +2 -2
  19. package/src/commands/recipe/show.ts +26 -9
  20. package/src/commands/repo/contribute.ts +32 -23
  21. package/src/commands/repo/gc.ts +1 -1
  22. package/src/commands/repo/remove.ts +1 -1
  23. package/src/commands/repo/search.ts +1 -1
  24. package/src/commands/vars/ask.ts +75 -54
  25. package/src/lib/build-service.ts +93 -15
  26. package/src/lib/config-schema.ts +59 -19
  27. package/src/lib/include-resolver.ts +307 -69
  28. package/src/lib/markdown-compiler.ts +289 -82
  29. package/src/lib/project-scaffold/templates.ts +19 -11
  30. package/src/lib/repos/catalog-inputs.ts +1 -1
  31. package/src/lib/repos/catalog.ts +98 -72
  32. package/src/lib/repos/declarations.ts +22 -12
  33. package/src/lib/repos/formats/index-file.ts +1 -1
  34. package/src/lib/repos/formats/recipe-manifest.ts +3 -4
  35. package/src/lib/repos/lock-service.ts +1 -1
  36. package/src/lib/repos/locked-namespace-resolver.ts +14 -6
  37. package/src/lib/repos/locked-recipe-lookup.ts +123 -0
  38. package/src/lib/repos/locked-recipes.ts +1 -1
  39. package/src/lib/repos/memories-hash-name.ts +33 -0
  40. package/src/lib/repos/namespace-resolver.ts +198 -119
  41. package/src/lib/repos/recipe-config-layers.ts +29 -1
  42. package/src/lib/repos/recipe-key-matcher.ts +62 -0
  43. package/src/lib/repos/recipe-memories.ts +175 -0
  44. package/src/lib/repos/recipe-targets.ts +32 -28
  45. package/src/lib/repos/ref-search.ts +1 -1
  46. package/src/lib/repos/release/index-builder.ts +23 -15
  47. package/src/lib/repos/release/plan.ts +14 -7
  48. package/src/lib/repos/release/settle.ts +38 -121
  49. package/src/lib/repos/release/submit-checkout.ts +42 -23
  50. package/src/lib/repos/release/tags.ts +1 -1
  51. package/src/lib/repos/resolver.ts +90 -72
  52. package/src/lib/repos/subscription-service.ts +192 -170
  53. package/src/lib/settings.ts +37 -33
  54. package/src/lib/vars/answers.ts +9 -1
  55. package/src/lib/vars/ask.ts +3 -3
  56. package/src/lib/vars/display.ts +1 -3
  57. package/src/lib/vars/index.ts +1 -0
  58. package/src/lib/vars/mappings.ts +17 -9
  59. package/src/lib/vars/secrets.ts +131 -0
  60. package/src/services/ref-resolver/container.ts +107 -0
  61. package/src/services/ref-resolver/dependency.ts +60 -0
  62. package/src/services/ref-resolver/format.ts +197 -0
  63. package/src/services/ref-resolver/glob.ts +115 -0
  64. package/src/services/ref-resolver/hash-names.ts +155 -0
  65. package/src/services/ref-resolver/index.ts +55 -0
  66. package/src/services/ref-resolver/injectable.ts +36 -0
  67. package/src/services/ref-resolver/location.ts +62 -0
  68. package/src/services/ref-resolver/lookups/cached-index-lookup.ts +53 -0
  69. package/src/services/ref-resolver/lookups/catalog-lookup.ts +23 -0
  70. package/src/services/ref-resolver/lookups/catalog-matcher.ts +261 -0
  71. package/src/services/ref-resolver/lookups/chained-lookup.ts +27 -0
  72. package/src/services/ref-resolver/lookups/env-var-lookup.ts +65 -0
  73. package/src/services/ref-resolver/lookups/fetched-index-lookup.ts +150 -0
  74. package/src/services/ref-resolver/lookups/recorded-dependency-lookup.ts +62 -0
  75. package/src/services/ref-resolver/lookups/ref-lookup.ts +54 -0
  76. package/src/services/ref-resolver/lookups/variable-lookup.ts +123 -0
  77. package/src/services/ref-resolver/named-refs.ts +144 -0
  78. package/src/services/ref-resolver/parser/env-var-splitter.ts +27 -0
  79. package/src/services/ref-resolver/parser/location-splitter.ts +239 -0
  80. package/src/services/ref-resolver/parser/name-path-splitter.ts +200 -0
  81. package/src/services/ref-resolver/parser/partial-ref.ts +84 -0
  82. package/src/services/ref-resolver/parser/query-splitter.ts +61 -0
  83. package/src/services/ref-resolver/parser/range-splitter.ts +52 -0
  84. package/src/services/ref-resolver/parser/ref-parser.ts +115 -0
  85. package/src/services/ref-resolver/parser/repo-qualifier-splitter.ts +51 -0
  86. package/src/services/ref-resolver/parts.ts +115 -0
  87. package/src/services/ref-resolver/pruners/command-line-pruner.ts +47 -0
  88. package/src/services/ref-resolver/pruners/include-pruner.ts +66 -0
  89. package/src/services/ref-resolver/pruners/manifest-pruner.ts +130 -0
  90. package/src/services/ref-resolver/pruners/ref-pruner.ts +135 -0
  91. package/src/services/ref-resolver/pruners/rule-helpers.ts +44 -0
  92. package/src/services/ref-resolver/pruners/stored-key-pruner.ts +136 -0
  93. package/src/services/ref-resolver/ref-helpers.ts +62 -0
  94. package/src/services/ref-resolver/ref-picker-service.ts +204 -0
  95. package/src/{lib/repos/reference-report.ts → services/ref-resolver/ref-report.ts} +2 -40
  96. package/src/services/ref-resolver/ref-resolve-arguments.ts +47 -0
  97. package/src/services/ref-resolver/ref-resolve-result.ts +36 -0
  98. package/src/services/ref-resolver/ref-resolver-service.ts +323 -0
  99. package/src/services/ref-resolver/source.ts +48 -0
  100. package/src/services/ref-resolver/tokens.ts +25 -0
  101. package/src/services/ref-resolver/types.ts +155 -0
  102. package/src/templating/init-liquid-engine.ts +33 -18
  103. package/src/templating/lib/secret-scope.ts +41 -0
  104. package/src/templating/tags/exportScalarVarsJs.ts +5 -2
  105. package/src/templating/tags/showVars.ts +8 -2
  106. package/src/lib/refs/find.ts +0 -770
  107. package/src/lib/refs/index.ts +0 -14
  108. package/src/lib/refs/parse.ts +0 -844
  109. package/src/lib/refs/pick.ts +0 -147
  110. package/src/lib/refs/scopes.ts +0 -95
  111. package/src/lib/refs/settle.ts +0 -219
@@ -11,7 +11,13 @@ import {
11
11
  hashContent,
12
12
  recordDirCreation,
13
13
  } from "./state.js";
14
- import { resolveInclude, type AliasMap } from "./include-resolver.js";
14
+ import {
15
+ resolveIncludeFiles,
16
+ splitIncludeQuery,
17
+ type AliasMap,
18
+ type ViewMap,
19
+ } from "./include-resolver.js";
20
+ import { RefSource, sharedRefResolver } from "../services/ref-resolver/index.js";
15
21
  import {
16
22
  formatNamespaceProblem,
17
23
  type NamespaceResolver,
@@ -58,8 +64,16 @@ export type CompilationConfig = {
58
64
  targets: CompilationTarget[];
59
65
  /** Resolved `@include` alias map (name → ordered base dirs). */
60
66
  aliases?: Record<string, string[]>;
67
+ /** The files each view lists (`#memories`), keyed by the name with its `#`. */
68
+ views?: ViewMap;
61
69
  /** Variable scope for `${var}` substitution in `@include` paths. */
62
70
  includeScope?: Record<string, string>;
71
+ /**
72
+ * Every variable name a recipe declares `secret: true`. The tags that dump a
73
+ * whole scope hide these (and anything that very probably is a secret); a
74
+ * template that names one explicitly still renders it.
75
+ */
76
+ secretVariables?: string[];
63
77
  };
64
78
 
65
79
  export type CompilationServiceOptions = {
@@ -145,12 +159,55 @@ export function stableVarsFingerprint(vars: Record<string, string>): string {
145
159
  return JSON.stringify(Object.keys(vars).sort().map((key) => [key, vars[key]]));
146
160
  }
147
161
 
162
+ /**
163
+ * An include line: `@`, a path that ends in `.md` and an optional
164
+ * `?name=value` query, and nothing else. The first character is a sigil, a
165
+ * variable, a dot or a letter, a digit or a glob character; the rest may hold
166
+ * the characters of a path, a `${var}`, a glob and a recipe reference's `repo:`
167
+ * qualifier.
168
+ */
169
+ export const INCLUDE_LINE_PATTERN =
170
+ /^@([~#a-zA-Z0-9_.${}*?[][a-zA-Z0-9_\-/.:${}*?[\],!~#%+^()]*\.md(?:\?[^?\s]*=\S*)?)$/;
171
+
172
+ /**
173
+ * Whether a line that is not a well-formed include still looks like one: it
174
+ * starts with `@` and is a single path-like word, meaning it holds a `/` or a
175
+ * `.md`, or starts with a sigil or a variable (`@~`, `@#`, `@.`, `@$`). A line
176
+ * that merely starts with `@` (a mention or an email address followed by
177
+ * words, or a lone `@name`) does not.
178
+ *
179
+ * looksLikeIncludeLine("@docs/notes.txt"); // -> true, the include has no .md
180
+ * looksLikeIncludeLine("@alice thanks for the review"); // -> false
181
+ * looksLikeIncludeLine("@alice"); // -> false
182
+ *
183
+ * @param line - One line of a file, with trailing whitespace already removed.
184
+ */
185
+ export function looksLikeIncludeLine(line: string): boolean {
186
+ if (!/^@\S+$/.test(line)) return false;
187
+ const word = line.slice(1);
188
+ return /[/]/.test(word) || /\.md\b/.test(word) || /^[~#.$]/.test(word);
189
+ }
190
+
191
+ /** One piece of a read file: a line of its own text, or a file it includes. */
192
+ type Part = string | { file: string };
193
+
194
+ /** A file read for a target: its text, whether its name makes it a template, and its pieces. */
195
+ type FileNode = {
196
+ path: string;
197
+ content: string;
198
+ /** True when the file's own name contains `.tpl.`: the only files that render as Liquid. */
199
+ isTpl: boolean;
200
+ parts: Part[];
201
+ };
202
+
148
203
  export class CompilationService {
149
204
  private strict: boolean;
150
205
  private rebuild: boolean;
151
206
  private dryRun: boolean;
152
- private visited: Set<string>;
153
207
  private includeStack: string[];
208
+ private nodes: Map<string, FileNode>;
209
+ private views: ViewMap;
210
+ private includedFilesSeen: Set<string>;
154
211
  private errors: string[];
155
212
  private includeSourceComments: boolean;
156
213
  private currentIncludeSourceComments: boolean;
@@ -158,6 +215,8 @@ export class CompilationService {
158
215
  private numberFormatter: Intl.NumberFormat;
159
216
  private aliases: AliasMap;
160
217
  private includeScope: Record<string, string>;
218
+ /** Every variable name a recipe declares secret, for the scope-dumping tags. */
219
+ private secretVariables: string[];
161
220
  private namespaceResolver?: NamespaceResolver;
162
221
  /**
163
222
  * `.tpl.` outputs that were written without a variable scope, so LiquidJS never
@@ -170,8 +229,10 @@ export class CompilationService {
170
229
  this.strict = options.strict ?? false;
171
230
  this.rebuild = options.rebuild ?? false;
172
231
  this.dryRun = options.dryRun ?? false;
173
- this.visited = new Set();
174
232
  this.includeStack = [];
233
+ this.nodes = new Map();
234
+ this.views = {};
235
+ this.includedFilesSeen = new Set();
175
236
  this.errors = [];
176
237
  this.includeSourceComments = false;
177
238
  this.currentIncludeSourceComments = false;
@@ -179,6 +240,7 @@ export class CompilationService {
179
240
  this.numberFormatter = new Intl.NumberFormat("en-US");
180
241
  this.aliases = {};
181
242
  this.includeScope = {};
243
+ this.secretVariables = [];
182
244
  this.namespaceResolver = options.namespaceResolver;
183
245
  this.unrenderedTemplates = [];
184
246
  }
@@ -214,23 +276,36 @@ export class CompilationService {
214
276
  /**
215
277
  * Process @<path> includes in content.
216
278
  *
217
- * Matches an `@`-prefixed `.md` path on its own line. The path may be:
279
+ * Matches an `@`-prefixed `.md` path on its own line, optionally followed by
280
+ * a `?name=value` query (read by the ref service and otherwise ignored). The
281
+ * path may be:
218
282
  * - relative to the including file (`@sections/intro.md`),
219
283
  * - a `${var}`-substituted path (`@${sousRootPath}/x.md`),
220
- * - an alias path (`@~project/memories/x.md`, `@docs/x.md`), where the
221
- * first segment (up to `/` or `:`) names a registered alias,
222
- * - or a recipe namespace path (`@~workflow/task-files/_partials/x.md`),
223
- * where the `~` sigil names a namespace and the rest names a recipe and a
224
- * file inside it. Namespaces are only consulted when a namespace resolver
225
- * was supplied, and always after aliases.
284
+ * - a `#name` path (`@#project/memories/x.md`), or an alias path
285
+ * (`@docs/x.md`), where the first segment (up to `/` or `:`) names a
286
+ * registered alias,
287
+ * - a home-relative path (`@~/notes/x.md`),
288
+ * - or a recipe reference (`@~workflow/task-files/_partials/x.md`), where
289
+ * the `~` sigil is followed by a namespace, a recipe and a file inside it.
290
+ * Recipe references are only consulted when a namespace resolver was
291
+ * supplied.
292
+ * Any of them may use glob syntax, and then includes every file it matches,
293
+ * in sorted path order; a glob that matches nothing is an error.
294
+ *
295
+ * A file may be included any number of times; only a file that includes
296
+ * itself (directly or through others) is an error.
226
297
  *
227
298
  * Lines inside fenced code blocks (``` or ~~~, per CommonMark) are left
228
299
  * verbatim, so include syntax can be documented without being executed.
229
300
  *
230
- * Resolution produces an ordered candidate list (see include-resolver); the
231
- * first candidate that exists on disk is used. If none exist, it errors,
232
- * naming the including file and listing every path tried, plus what went
233
- * wrong with the namespace lookup when one was attempted.
301
+ * A line that starts with `@` and looks like an include (see
302
+ * {@link looksLikeIncludeLine}) but is not a well-formed one is a compile
303
+ * error naming the file and the line, never text copied to the output.
304
+ *
305
+ * Resolution produces ordered groups of candidates (see include-resolver);
306
+ * the first group that names a file is used. If none does, it errors, naming
307
+ * the including file and listing every path tried, plus what went wrong with
308
+ * a `~` or `#` reference when one was attempted.
234
309
  *
235
310
  * @param content - The file's raw text.
236
311
  * @param baseDir - Directory the relative candidate resolves against.
@@ -242,18 +317,16 @@ export class CompilationService {
242
317
  baseDir: string,
243
318
  projectRoot: string,
244
319
  fromFile?: string
245
- ): string {
246
- // First segment allows ~ and . (so ./ and ../ work), then path chars;
247
- // separators / and :; allows ${...}.
248
- const includePattern = /^@([~a-zA-Z0-9_.${}][a-zA-Z0-9_\-/.:${}]*\.md)$/;
320
+ ): Part[] {
249
321
  const fenceOpenPattern = /^ {0,3}(`{3,}|~{3,})/;
250
322
  const fenceClosePattern = /^ {0,3}(`{3,}|~{3,})[ \t]*$/;
323
+ const where = fromFile ?? baseDir;
251
324
 
252
- const out: string[] = [];
325
+ const out: Part[] = [];
253
326
  let fenceChar: string | null = null;
254
327
  let fenceLength = 0;
255
328
 
256
- for (const line of content.split("\n")) {
329
+ for (const [lineIndex, line] of content.split("\n").entries()) {
257
330
  if (fenceChar !== null) {
258
331
  // Inside a fence: emit verbatim; only a matching closing fence ends it.
259
332
  const close = fenceClosePattern.exec(line);
@@ -272,28 +345,63 @@ export class CompilationService {
272
345
  continue;
273
346
  }
274
347
 
275
- const match = includePattern.exec(line);
348
+ const trimmed = line.trimEnd();
349
+ const match = INCLUDE_LINE_PATTERN.exec(trimmed);
276
350
  if (!match) {
351
+ if (looksLikeIncludeLine(trimmed)) {
352
+ this.handleError(
353
+ `Malformed include line: ${trimmed}\n in file: ${where}, line ${lineIndex + 1}\n` +
354
+ ` An include is "@" followed by a path that ends in .md, optionally followed by ` +
355
+ `"?name=value". The path may hold letters, digits and . _ - / : $ { } * ? [ ] , ~ #.\n` +
356
+ ` To write a line that starts with "@" as text, put a word after it.`
357
+ );
358
+ }
277
359
  out.push(line);
278
360
  continue;
279
361
  }
280
362
 
281
363
  const includePath = match[1].trim();
282
- const { candidates, namespaceIssue } = resolveInclude(includePath, {
283
- aliases: this.aliases,
284
- scope: this.includeScope,
285
- baseDir,
286
- namespaceResolver: this.namespaceResolver,
287
- fromFile: fromFile ?? baseDir,
288
- });
289
- const fullPath = candidates.find((c) => fs.existsSync(c));
290
-
291
- if (!fullPath) {
364
+ const { files, glob, candidates, namespaceIssue, hashIssue, view } = resolveIncludeFiles(
365
+ includePath,
366
+ {
367
+ aliases: this.aliases,
368
+ views: this.views,
369
+ scope: this.includeScope,
370
+ baseDir,
371
+ namespaceResolver: this.namespaceResolver,
372
+ fromFile: where,
373
+ }
374
+ );
375
+
376
+ if (files.length === 0) {
377
+ // A view is a list that may be empty: a glob over it that selects
378
+ // nothing includes nothing, and the line leaves no trace.
379
+ if (view === true && glob && hashIssue === undefined) continue;
380
+ if (this.namespaceResolver === undefined && includePath.startsWith("~")) {
381
+ // Without a resolver the reference is only a relative path, so say
382
+ // what is wrong with a malformed one instead of "not found".
383
+ const reference = splitIncludeQuery(includePath).path.slice(1);
384
+ if (reference.includes("/") && !reference.includes("${")) {
385
+ try {
386
+ sharedRefResolver().parse(reference, RefSource.Include);
387
+ } catch (error) {
388
+ this.handleError(
389
+ `Malformed include line: ${trimmed}\n in file: ${where}, line ${lineIndex + 1}\n ${
390
+ error instanceof Error ? error.message : String(error)
391
+ }`
392
+ );
393
+ out.push("");
394
+ continue;
395
+ }
396
+ }
397
+ }
292
398
  const explanation = namespaceIssue
293
399
  ? `\n${formatNamespaceProblem(namespaceIssue)}`
294
- : `\n in file: ${fromFile ?? baseDir}`;
400
+ : hashIssue
401
+ ? `\n in file: ${where}\n ${hashIssue.split("\n").join("\n ")}`
402
+ : `\n in file: ${where}`;
295
403
  this.handleError(
296
- `Include not found: @${includePath}${explanation}\n tried:\n${candidates
404
+ `${glob ? "Include matched no files" : "Include not found"}: @${includePath}${explanation}\n tried:\n${candidates
297
405
  .map((c) => ` - ${c}`)
298
406
  .join("\n")}`
299
407
  );
@@ -301,57 +409,121 @@ export class CompilationService {
301
409
  continue;
302
410
  }
303
411
 
304
- if (this.includeStack.includes(fullPath)) {
305
- this.handleError(
306
- `Circular dependency detected: ${this.includeStack.join(" -> ")} -> ${fullPath}`
307
- );
308
- out.push("");
309
- continue;
310
- }
412
+ for (const fullPath of files) {
413
+ if (this.includeStack.includes(fullPath)) {
414
+ this.handleError(
415
+ `Circular dependency detected: ${this.includeStack.join(" -> ")} -> ${fullPath}`
416
+ );
417
+ out.push("");
418
+ continue;
419
+ }
311
420
 
312
- const includedContent = this.loadFile(fullPath, projectRoot);
313
- if (includedContent !== null) {
314
- const relativePath = path.relative(projectRoot, fullPath);
315
- const sourceComment = this.currentIncludeSourceComments
316
- ? `<!-- from: ${relativePath} -->\n`
317
- : "";
318
- out.push(sourceComment + includedContent);
319
- } else {
320
- out.push("");
421
+ out.push(this.loadFile(fullPath, projectRoot) === null ? "" : { file: fullPath });
321
422
  }
322
423
  }
323
424
 
324
- return out.join("\n");
425
+ return out;
325
426
  }
326
427
 
327
- /** Load a file and recursively process its includes. */
328
- private loadFile(filePath: string, projectRoot: string): string | null {
428
+ /**
429
+ * Reads a file and, recursively, every file it includes, without rendering
430
+ * anything. A file already read for this target is not read again (its
431
+ * includes resolve the same way whoever includes it), so a file may be
432
+ * included any number of times; only a cycle is an error.
433
+ */
434
+ private loadFile(filePath: string, projectRoot: string): FileNode | null {
435
+ const known = this.nodes.get(filePath);
436
+ if (known !== undefined) return known;
437
+
329
438
  if (!fs.existsSync(filePath)) {
330
439
  this.handleError(`File not found: ${filePath}`);
331
440
  return null;
332
441
  }
333
442
 
334
- if (this.visited.has(filePath)) {
335
- return "";
336
- }
337
-
338
443
  this.includeStack.push(filePath);
339
444
 
340
445
  try {
341
446
  const content = fs.readFileSync(filePath, "utf8");
342
447
  const baseDir = path.dirname(filePath);
343
- const processedContent = this.processIncludes(content, baseDir, projectRoot, filePath);
344
- this.visited.add(filePath);
345
- return processedContent;
448
+ const node: FileNode = {
449
+ path: filePath,
450
+ content,
451
+ isTpl: path.basename(filePath).includes(".tpl."),
452
+ parts: [],
453
+ };
454
+ // Registered before its includes are read so the graph lists parents
455
+ // first; the cycle check uses the stack, not this map.
456
+ this.nodes.set(filePath, node);
457
+ node.parts = this.processIncludes(content, baseDir, projectRoot, filePath);
458
+ return node;
346
459
  } catch (error) {
347
460
  const message = error instanceof Error ? error.message : String(error);
348
461
  this.handleError(`Failed to read file ${filePath}: ${message}`);
462
+ this.nodes.delete(filePath);
349
463
  return null;
350
464
  } finally {
351
465
  this.includeStack.pop();
352
466
  }
353
467
  }
354
468
 
469
+ /**
470
+ * Turns a read file into its final text.
471
+ *
472
+ * A file whose own name contains `.tpl.` is rendered as Liquid with the
473
+ * given variables; no other file ever is. Each include is left as a unique
474
+ * marker while the file renders, so a condition around an include line is
475
+ * honored, and an included file's own output is never rendered again. Every
476
+ * marker that survives the render is then replaced by that file's final text,
477
+ * produced the same way.
478
+ *
479
+ * @param filePath - The file to expand.
480
+ * @param vars - The output's variable scope; undefined means templates stay unrendered.
481
+ * @param entryDir - The directory of the target's entry point, also searched by `{% render %}`.
482
+ * @param projectRoot - Root used to render source comments as relative paths.
483
+ */
484
+ private async expand(
485
+ filePath: string,
486
+ vars: Record<string, string> | undefined,
487
+ entryDir: string,
488
+ projectRoot: string
489
+ ): Promise<string> {
490
+ const node = this.nodes.get(filePath)!;
491
+ const nonce = `${process.pid}-${Math.random().toString(36).slice(2)}`;
492
+ const markers = new Map<string, string>();
493
+ const chunks = node.parts.map((part) => {
494
+ if (typeof part === "string") return part;
495
+ const marker = `\u0001SOUS-INCLUDE-${nonce}-${markers.size}\u0001`;
496
+ markers.set(marker, part.file);
497
+ return marker;
498
+ });
499
+
500
+ let text = chunks.join("\n");
501
+ if (node.isTpl && vars !== undefined) {
502
+ const dir = path.dirname(node.path);
503
+ text = await this.renderContent(
504
+ text,
505
+ { ...vars, sousTemplatePath: node.path, sousTemplateDir: dir },
506
+ [...new Set([dir, entryDir])],
507
+ node.path
508
+ );
509
+ }
510
+
511
+ for (const [marker, child] of markers) {
512
+ if (!text.includes(marker)) continue;
513
+ const body = await this.expand(child, vars, entryDir, projectRoot);
514
+ const comment = this.currentIncludeSourceComments
515
+ ? `<!-- from: ${path.relative(projectRoot, child)} -->\n`
516
+ : "";
517
+ text = text.split(marker).join(comment + body);
518
+ }
519
+ return text;
520
+ }
521
+
522
+ /** Every file read for the current target, parents before the files they include. */
523
+ private graphNodes(): FileNode[] {
524
+ return [...this.nodes.values()];
525
+ }
526
+
355
527
  /**
356
528
  * Render template content using LiquidJS with the given variable scope.
357
529
  *
@@ -368,9 +540,11 @@ export class CompilationService {
368
540
  ): Promise<string> {
369
541
  const engine = createLiquidEngine(roots, {
370
542
  aliases: this.aliases,
543
+ views: this.views,
371
544
  scope: { ...this.includeScope, ...vars },
372
545
  namespaceResolver: this.namespaceResolver,
373
546
  fromFile,
547
+ secretVariables: this.secretVariables,
374
548
  });
375
549
  try {
376
550
  return await engine.parseAndRender(content, vars);
@@ -473,8 +647,8 @@ ${taskFileContents}
473
647
 
474
648
  this.initializeEncoder();
475
649
 
476
- this.visited.clear();
477
650
  this.includeStack = [];
651
+ this.nodes = new Map();
478
652
  this.currentIncludeSourceComments =
479
653
  typeof target.includeSourceComments === "boolean"
480
654
  ? target.includeSourceComments
@@ -487,9 +661,9 @@ ${taskFileContents}
487
661
  }
488
662
 
489
663
  const promptsRoot = path.dirname(target.rootInputPath);
490
- const content = this.loadFile(target.rootInputPath, promptsRoot);
664
+ const entry = this.loadFile(target.rootInputPath, promptsRoot);
491
665
 
492
- if (content === null) {
666
+ if (entry === null) {
493
667
  displayError(`Failed to compile ${target.rootInputPath}`);
494
668
  return false;
495
669
  }
@@ -499,12 +673,24 @@ ${taskFileContents}
499
673
  // be written: the previous copy stays in place.
500
674
  const assembled = this.errors.length === errorsBefore;
501
675
 
502
- // Compute source hash once per target from the assembled content
503
- const contentHash = hashContent(content);
676
+ // Compute the source hash once per target from every file read: the entry
677
+ // point and everything it includes, whatever a condition decides later.
678
+ const graph = this.graphNodes();
679
+ for (const node of graph) this.includedFilesSeen.add(realPathOf(node.path));
680
+ const templates = graph.filter((node) => node.isTpl);
681
+ const contentHash = hashContent(
682
+ JSON.stringify([
683
+ this.currentIncludeSourceComments,
684
+ graph.map((node) => [path.relative(promptsRoot, node.path), node.content]),
685
+ ])
686
+ );
504
687
 
505
688
  let allSucceeded = true;
506
689
 
507
- const isTpl = path.basename(target.rootInputPath).includes(".tpl.");
690
+ const isTpl = entry.isTpl;
691
+ // Whether any file of this target renders as Liquid, which makes the
692
+ // output depend on its variables.
693
+ const rendersAnything = templates.length > 0;
508
694
 
509
695
  for (const output of target.outputs) {
510
696
  // Resolve destination path: prefer destinationFile, fall back to destinationDir mirroring
@@ -524,9 +710,13 @@ ${taskFileContents}
524
710
  // A rendered output depends on its variables as much as on its source: a
525
711
  // changed answer or `_vars` value with the same template must re-render,
526
712
  // so the variable scope is part of a `.tpl.` output's source hash. A
527
- // verbatim copy hashes its content alone.
528
- const srcHash = isTpl && output.vars
529
- ? hashContent(`${content}\n${stableVarsFingerprint(output.vars)}`)
713
+ // verbatim copy hashes its content alone. The secret names are part of it
714
+ // too, because they decide what a scope-dumping tag hides.
715
+ const srcHash = rendersAnything && output.vars
716
+ ? hashContent(
717
+ `${contentHash}\n${stableVarsFingerprint(output.vars)}\n` +
718
+ `secrets:${JSON.stringify([...this.secretVariables].sort())}`
719
+ )
530
720
  : contentHash;
531
721
 
532
722
  // Skip if content is unchanged and file already exists (unless --rebuild)
@@ -554,8 +744,10 @@ ${taskFileContents}
554
744
  // resolver always sets `vars` on every output (at minimum the inherited
555
745
  // scope). It is a real guard for callers that build a CompilationConfig
556
746
  // directly, and it stays as a tripwire in case the resolver ever changes.
557
- if (isTpl && !output.vars) {
558
- this.unrenderedTemplates.push(`${target.rootInputPath} → ${destFile}`);
747
+ if (rendersAnything && !output.vars) {
748
+ for (const template of templates) {
749
+ this.unrenderedTemplates.push(`${template.path} → ${destFile}`);
750
+ }
559
751
  // In strict mode this warning is an error, so the output is not written.
560
752
  if (this.strict) {
561
753
  this.reportNotWritten(destFile);
@@ -565,18 +757,12 @@ ${taskFileContents}
565
757
  }
566
758
 
567
759
  const errorsBeforeRender = this.errors.length;
568
- const resolvedContent = (isTpl && output.vars)
569
- ? await this.renderContent(
570
- content,
571
- {
572
- ...output.vars,
573
- sousTemplatePath: target.rootInputPath,
574
- sousTemplateDir: promptsRoot,
575
- },
576
- [promptsRoot],
577
- target.rootInputPath
578
- )
579
- : content;
760
+ const resolvedContent = await this.expand(
761
+ target.rootInputPath,
762
+ output.vars,
763
+ promptsRoot,
764
+ promptsRoot
765
+ );
580
766
 
581
767
  // A template that failed to render is not written either.
582
768
  if (this.errors.length > errorsBeforeRender) {
@@ -644,6 +830,15 @@ ${taskFileContents}
644
830
  return allSucceeded;
645
831
  }
646
832
 
833
+ /**
834
+ * Every file the last compile read, as real paths: the entry points and every
835
+ * file any include line of any target named, by any route (exact path, glob
836
+ * or view). Includes inside an unmet condition count, since they are read.
837
+ */
838
+ includedFiles(): ReadonlySet<string> {
839
+ return this.includedFilesSeen;
840
+ }
841
+
647
842
  /**
648
843
  * Compile all targets from the given config. Every target is compiled even
649
844
  * after an error, so every error is listed; an output whose target had an
@@ -675,7 +870,10 @@ ${taskFileContents}
675
870
 
676
871
  this.includeSourceComments = config.includeSourceComments === true;
677
872
  this.aliases = config.aliases ?? {};
873
+ this.views = config.views ?? {};
874
+ this.includedFilesSeen = new Set();
678
875
  this.includeScope = config.includeScope ?? {};
876
+ this.secretVariables = config.secretVariables ?? [];
679
877
 
680
878
  this.initializeEncoder();
681
879
 
@@ -738,3 +936,12 @@ ${taskFileContents}
738
936
  // Backward-compat alias so existing imports of MarkdownCompiler keep working
739
937
  export { CompilationService as MarkdownCompiler };
740
938
  export type { CompilationServiceOptions as MarkdownCompilerOptions };
939
+
940
+ /** The path with symbolic links resolved, or the path itself when it cannot be read. */
941
+ function realPathOf(filePath: string): string {
942
+ try {
943
+ return fs.realpathSync(filePath);
944
+ } catch {
945
+ return filePath;
946
+ }
947
+ }
@@ -33,7 +33,7 @@ export type ProjectConfigFormat = (typeof PROJECT_CONFIG_FORMATS)[number];
33
33
  * compiled output (`AGENTS.md` at the project root) are never mistaken for one
34
34
  * another.
35
35
  */
36
- export const STARTER_PROMPT_RELATIVE_PATH = "memories/AGENTS.md";
36
+ export const STARTER_PROMPT_RELATIVE_PATH = "memories/AGENTS.tpl.md";
37
37
 
38
38
  /** The name of the file the starter prompt is compiled into, at the project root. */
39
39
  export const STARTER_OUTPUT_NAME = "AGENTS.md";
@@ -98,14 +98,19 @@ export const config = {
98
98
  ],
99
99
  },
100
100
 
101
- // Where the recipes this project subscribes to write their files. Every
102
- // project subscribes to the \`core\` namespace on its own, which is how the
103
- // skills that teach an agent about sous reach \`.claude/skills\`. Memories and
104
- // prompts have no default home; name one to receive them.
105
- recipeOutputs: {
106
- skills: ["\${projectRoot}/.claude/skills"],
107
- // memories: ["\${projectRoot}/.claude/memories"],
108
- // prompts: ["\${projectRoot}/.claude/prompts"],
101
+ // What to do with the content the recipes this project subscribes to publish.
102
+ // Every project subscribes to the \`core\` namespace on its own, which is how
103
+ // the skills that teach an agent about sous reach \`.claude/skills\`. Memories
104
+ // are not copied anywhere: the starter prompt pulls them in with one include
105
+ // line, and the lists below (uncomment to use them) choose which come first
106
+ // and which are left out. An entry is a glob over \`namespace/recipe\`, or a
107
+ // regular expression written between two slashes.
108
+ recipes: {
109
+ skills: { outputs: ["\${projectRoot}/.claude/skills"] },
110
+ // memories: {
111
+ // first: ["communication/*"],
112
+ // exclude: ["tool-usage/automated-browser-tasks", "/^experimental\\//"],
113
+ // },
109
114
  },
110
115
  };
111
116
  `;
@@ -132,8 +137,8 @@ export function buildConfigJson(context: ProjectScaffoldContext): string {
132
137
  },
133
138
  ],
134
139
  },
135
- recipeOutputs: {
136
- skills: ["${projectRoot}/.claude/skills"],
140
+ recipes: {
141
+ skills: { outputs: ["${projectRoot}/.claude/skills"] },
137
142
  },
138
143
  };
139
144
  return `${JSON.stringify(config, null, 2)}\n`;
@@ -158,6 +163,9 @@ source, run \`sous build\`, and the compiled copy follows; never edit the compil
158
163
  each one in with a line holding only \`@sections/name.md\`.
159
164
  - The skills under \`.claude/skills\` are written by sous from the recipes this
160
165
  project subscribes to. \`sous recipe list\` shows what is available.
166
+
167
+ {% comment %}This line pulls in the memories of every recipe this project subscribes to.{% endcomment %}
168
+ @#memories/**/*.md
161
169
  `;
162
170
  }
163
171
 
@@ -6,7 +6,7 @@
6
6
  * deliberately the only place that knows: the subscription service for the
7
7
  * trusted repositories and their cached indexes, the lockfile service for what
8
8
  * is pinned, the links map and the store for a recipe's own files, and
9
- * `recipeOutputs` for where a content kind lands.
9
+ * `recipes` for where a content kind lands.
10
10
  *
11
11
  * By default nothing here downloads anything: a repository whose index has
12
12
  * never been fetched is left out of the catalog and named separately, so a