@sous-io/sous 0.2.32 → 0.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/docs/markdown/_sidebar.md +1 -0
- package/docs/markdown/commands.md +8 -4
- package/docs/markdown/configuration.md +28 -4
- package/docs/markdown/repositories-authoring.md +22 -2
- package/docs/markdown/repositories-consuming.md +40 -11
- package/docs/markdown/repositories-file-formats.md +28 -13
- package/docs/markdown/repositories-quickstart.md +1 -1
- package/docs/markdown/repositories-variables.md +16 -0
- package/docs/markdown/repositories.md +8 -4
- package/package.json +2 -1
- package/recipes/core/sous-skills/skills/about-liquid-templates/SKILL.tpl.md +33 -20
- package/recipes/core/sous-skills/skills/about-sous-configuration/SKILL.tpl.md +4 -0
- package/recipes/core/sous-skills/skills/about-sous-recipes/SKILL.tpl.md +21 -9
- package/recipes/core/sous-skills/sous.recipe.yaml +1 -1
- package/sous.config.schema.json +27 -15
- package/src/commands/lock/rebuild.ts +3 -6
- package/src/commands/namespace/show.ts +2 -2
- package/src/commands/recipe/show.ts +26 -9
- package/src/commands/repo/contribute.ts +32 -23
- package/src/commands/repo/gc.ts +1 -1
- package/src/commands/repo/remove.ts +1 -1
- package/src/commands/repo/search.ts +1 -1
- package/src/commands/vars/ask.ts +75 -54
- package/src/lib/build-service.ts +93 -15
- package/src/lib/config-schema.ts +59 -19
- package/src/lib/include-resolver.ts +307 -69
- package/src/lib/markdown-compiler.ts +289 -82
- package/src/lib/project-scaffold/templates.ts +19 -11
- package/src/lib/repos/catalog-inputs.ts +1 -1
- package/src/lib/repos/catalog.ts +98 -72
- package/src/lib/repos/declarations.ts +22 -12
- package/src/lib/repos/formats/index-file.ts +1 -1
- package/src/lib/repos/formats/recipe-manifest.ts +3 -4
- package/src/lib/repos/lock-service.ts +1 -1
- package/src/lib/repos/locked-namespace-resolver.ts +14 -6
- package/src/lib/repos/locked-recipe-lookup.ts +123 -0
- package/src/lib/repos/locked-recipes.ts +1 -1
- package/src/lib/repos/memories-hash-name.ts +33 -0
- package/src/lib/repos/namespace-resolver.ts +198 -119
- package/src/lib/repos/recipe-config-layers.ts +29 -1
- package/src/lib/repos/recipe-key-matcher.ts +62 -0
- package/src/lib/repos/recipe-memories.ts +175 -0
- package/src/lib/repos/recipe-targets.ts +32 -28
- package/src/lib/repos/ref-search.ts +1 -1
- package/src/lib/repos/release/index-builder.ts +23 -15
- package/src/lib/repos/release/plan.ts +14 -7
- package/src/lib/repos/release/settle.ts +38 -121
- package/src/lib/repos/release/submit-checkout.ts +42 -23
- package/src/lib/repos/release/tags.ts +1 -1
- package/src/lib/repos/resolver.ts +90 -72
- package/src/lib/repos/subscription-service.ts +192 -170
- package/src/lib/settings.ts +37 -33
- package/src/lib/vars/answers.ts +9 -1
- package/src/lib/vars/ask.ts +3 -3
- package/src/lib/vars/display.ts +1 -3
- package/src/lib/vars/index.ts +1 -0
- package/src/lib/vars/mappings.ts +17 -9
- package/src/lib/vars/secrets.ts +131 -0
- package/src/services/ref-resolver/container.ts +107 -0
- package/src/services/ref-resolver/dependency.ts +60 -0
- package/src/services/ref-resolver/format.ts +197 -0
- package/src/services/ref-resolver/glob.ts +115 -0
- package/src/services/ref-resolver/hash-names.ts +155 -0
- package/src/services/ref-resolver/index.ts +55 -0
- package/src/services/ref-resolver/injectable.ts +36 -0
- package/src/services/ref-resolver/location.ts +62 -0
- package/src/services/ref-resolver/lookups/cached-index-lookup.ts +53 -0
- package/src/services/ref-resolver/lookups/catalog-lookup.ts +23 -0
- package/src/services/ref-resolver/lookups/catalog-matcher.ts +261 -0
- package/src/services/ref-resolver/lookups/chained-lookup.ts +27 -0
- package/src/services/ref-resolver/lookups/env-var-lookup.ts +65 -0
- package/src/services/ref-resolver/lookups/fetched-index-lookup.ts +150 -0
- package/src/services/ref-resolver/lookups/recorded-dependency-lookup.ts +62 -0
- package/src/services/ref-resolver/lookups/ref-lookup.ts +54 -0
- package/src/services/ref-resolver/lookups/variable-lookup.ts +123 -0
- package/src/services/ref-resolver/named-refs.ts +144 -0
- package/src/services/ref-resolver/parser/env-var-splitter.ts +27 -0
- package/src/services/ref-resolver/parser/location-splitter.ts +239 -0
- package/src/services/ref-resolver/parser/name-path-splitter.ts +200 -0
- package/src/services/ref-resolver/parser/partial-ref.ts +84 -0
- package/src/services/ref-resolver/parser/query-splitter.ts +61 -0
- package/src/services/ref-resolver/parser/range-splitter.ts +52 -0
- package/src/services/ref-resolver/parser/ref-parser.ts +115 -0
- package/src/services/ref-resolver/parser/repo-qualifier-splitter.ts +51 -0
- package/src/services/ref-resolver/parts.ts +115 -0
- package/src/services/ref-resolver/pruners/command-line-pruner.ts +47 -0
- package/src/services/ref-resolver/pruners/include-pruner.ts +66 -0
- package/src/services/ref-resolver/pruners/manifest-pruner.ts +130 -0
- package/src/services/ref-resolver/pruners/ref-pruner.ts +135 -0
- package/src/services/ref-resolver/pruners/rule-helpers.ts +44 -0
- package/src/services/ref-resolver/pruners/stored-key-pruner.ts +136 -0
- package/src/services/ref-resolver/ref-helpers.ts +62 -0
- package/src/services/ref-resolver/ref-picker-service.ts +204 -0
- package/src/{lib/repos/reference-report.ts → services/ref-resolver/ref-report.ts} +2 -40
- package/src/services/ref-resolver/ref-resolve-arguments.ts +47 -0
- package/src/services/ref-resolver/ref-resolve-result.ts +36 -0
- package/src/services/ref-resolver/ref-resolver-service.ts +323 -0
- package/src/services/ref-resolver/source.ts +48 -0
- package/src/services/ref-resolver/tokens.ts +25 -0
- package/src/services/ref-resolver/types.ts +155 -0
- package/src/templating/init-liquid-engine.ts +33 -18
- package/src/templating/lib/secret-scope.ts +41 -0
- package/src/templating/tags/exportScalarVarsJs.ts +5 -2
- package/src/templating/tags/showVars.ts +8 -2
- package/src/lib/refs/find.ts +0 -770
- package/src/lib/refs/index.ts +0 -14
- package/src/lib/refs/parse.ts +0 -844
- package/src/lib/refs/pick.ts +0 -147
- package/src/lib/refs/scopes.ts +0 -95
- package/src/lib/refs/settle.ts +0 -219
|
@@ -11,7 +11,13 @@ import {
|
|
|
11
11
|
hashContent,
|
|
12
12
|
recordDirCreation,
|
|
13
13
|
} from "./state.js";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
resolveIncludeFiles,
|
|
16
|
+
splitIncludeQuery,
|
|
17
|
+
type AliasMap,
|
|
18
|
+
type ViewMap,
|
|
19
|
+
} from "./include-resolver.js";
|
|
20
|
+
import { RefSource, sharedRefResolver } from "../services/ref-resolver/index.js";
|
|
15
21
|
import {
|
|
16
22
|
formatNamespaceProblem,
|
|
17
23
|
type NamespaceResolver,
|
|
@@ -58,8 +64,16 @@ export type CompilationConfig = {
|
|
|
58
64
|
targets: CompilationTarget[];
|
|
59
65
|
/** Resolved `@include` alias map (name → ordered base dirs). */
|
|
60
66
|
aliases?: Record<string, string[]>;
|
|
67
|
+
/** The files each view lists (`#memories`), keyed by the name with its `#`. */
|
|
68
|
+
views?: ViewMap;
|
|
61
69
|
/** Variable scope for `${var}` substitution in `@include` paths. */
|
|
62
70
|
includeScope?: Record<string, string>;
|
|
71
|
+
/**
|
|
72
|
+
* Every variable name a recipe declares `secret: true`. The tags that dump a
|
|
73
|
+
* whole scope hide these (and anything that very probably is a secret); a
|
|
74
|
+
* template that names one explicitly still renders it.
|
|
75
|
+
*/
|
|
76
|
+
secretVariables?: string[];
|
|
63
77
|
};
|
|
64
78
|
|
|
65
79
|
export type CompilationServiceOptions = {
|
|
@@ -145,12 +159,55 @@ export function stableVarsFingerprint(vars: Record<string, string>): string {
|
|
|
145
159
|
return JSON.stringify(Object.keys(vars).sort().map((key) => [key, vars[key]]));
|
|
146
160
|
}
|
|
147
161
|
|
|
162
|
+
/**
|
|
163
|
+
* An include line: `@`, a path that ends in `.md` and an optional
|
|
164
|
+
* `?name=value` query, and nothing else. The first character is a sigil, a
|
|
165
|
+
* variable, a dot or a letter, a digit or a glob character; the rest may hold
|
|
166
|
+
* the characters of a path, a `${var}`, a glob and a recipe reference's `repo:`
|
|
167
|
+
* qualifier.
|
|
168
|
+
*/
|
|
169
|
+
export const INCLUDE_LINE_PATTERN =
|
|
170
|
+
/^@([~#a-zA-Z0-9_.${}*?[][a-zA-Z0-9_\-/.:${}*?[\],!~#%+^()]*\.md(?:\?[^?\s]*=\S*)?)$/;
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Whether a line that is not a well-formed include still looks like one: it
|
|
174
|
+
* starts with `@` and is a single path-like word, meaning it holds a `/` or a
|
|
175
|
+
* `.md`, or starts with a sigil or a variable (`@~`, `@#`, `@.`, `@$`). A line
|
|
176
|
+
* that merely starts with `@` (a mention or an email address followed by
|
|
177
|
+
* words, or a lone `@name`) does not.
|
|
178
|
+
*
|
|
179
|
+
* looksLikeIncludeLine("@docs/notes.txt"); // -> true, the include has no .md
|
|
180
|
+
* looksLikeIncludeLine("@alice thanks for the review"); // -> false
|
|
181
|
+
* looksLikeIncludeLine("@alice"); // -> false
|
|
182
|
+
*
|
|
183
|
+
* @param line - One line of a file, with trailing whitespace already removed.
|
|
184
|
+
*/
|
|
185
|
+
export function looksLikeIncludeLine(line: string): boolean {
|
|
186
|
+
if (!/^@\S+$/.test(line)) return false;
|
|
187
|
+
const word = line.slice(1);
|
|
188
|
+
return /[/]/.test(word) || /\.md\b/.test(word) || /^[~#.$]/.test(word);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** One piece of a read file: a line of its own text, or a file it includes. */
|
|
192
|
+
type Part = string | { file: string };
|
|
193
|
+
|
|
194
|
+
/** A file read for a target: its text, whether its name makes it a template, and its pieces. */
|
|
195
|
+
type FileNode = {
|
|
196
|
+
path: string;
|
|
197
|
+
content: string;
|
|
198
|
+
/** True when the file's own name contains `.tpl.`: the only files that render as Liquid. */
|
|
199
|
+
isTpl: boolean;
|
|
200
|
+
parts: Part[];
|
|
201
|
+
};
|
|
202
|
+
|
|
148
203
|
export class CompilationService {
|
|
149
204
|
private strict: boolean;
|
|
150
205
|
private rebuild: boolean;
|
|
151
206
|
private dryRun: boolean;
|
|
152
|
-
private visited: Set<string>;
|
|
153
207
|
private includeStack: string[];
|
|
208
|
+
private nodes: Map<string, FileNode>;
|
|
209
|
+
private views: ViewMap;
|
|
210
|
+
private includedFilesSeen: Set<string>;
|
|
154
211
|
private errors: string[];
|
|
155
212
|
private includeSourceComments: boolean;
|
|
156
213
|
private currentIncludeSourceComments: boolean;
|
|
@@ -158,6 +215,8 @@ export class CompilationService {
|
|
|
158
215
|
private numberFormatter: Intl.NumberFormat;
|
|
159
216
|
private aliases: AliasMap;
|
|
160
217
|
private includeScope: Record<string, string>;
|
|
218
|
+
/** Every variable name a recipe declares secret, for the scope-dumping tags. */
|
|
219
|
+
private secretVariables: string[];
|
|
161
220
|
private namespaceResolver?: NamespaceResolver;
|
|
162
221
|
/**
|
|
163
222
|
* `.tpl.` outputs that were written without a variable scope, so LiquidJS never
|
|
@@ -170,8 +229,10 @@ export class CompilationService {
|
|
|
170
229
|
this.strict = options.strict ?? false;
|
|
171
230
|
this.rebuild = options.rebuild ?? false;
|
|
172
231
|
this.dryRun = options.dryRun ?? false;
|
|
173
|
-
this.visited = new Set();
|
|
174
232
|
this.includeStack = [];
|
|
233
|
+
this.nodes = new Map();
|
|
234
|
+
this.views = {};
|
|
235
|
+
this.includedFilesSeen = new Set();
|
|
175
236
|
this.errors = [];
|
|
176
237
|
this.includeSourceComments = false;
|
|
177
238
|
this.currentIncludeSourceComments = false;
|
|
@@ -179,6 +240,7 @@ export class CompilationService {
|
|
|
179
240
|
this.numberFormatter = new Intl.NumberFormat("en-US");
|
|
180
241
|
this.aliases = {};
|
|
181
242
|
this.includeScope = {};
|
|
243
|
+
this.secretVariables = [];
|
|
182
244
|
this.namespaceResolver = options.namespaceResolver;
|
|
183
245
|
this.unrenderedTemplates = [];
|
|
184
246
|
}
|
|
@@ -214,23 +276,36 @@ export class CompilationService {
|
|
|
214
276
|
/**
|
|
215
277
|
* Process @<path> includes in content.
|
|
216
278
|
*
|
|
217
|
-
* Matches an `@`-prefixed `.md` path on its own line
|
|
279
|
+
* Matches an `@`-prefixed `.md` path on its own line, optionally followed by
|
|
280
|
+
* a `?name=value` query (read by the ref service and otherwise ignored). The
|
|
281
|
+
* path may be:
|
|
218
282
|
* - relative to the including file (`@sections/intro.md`),
|
|
219
283
|
* - a `${var}`-substituted path (`@${sousRootPath}/x.md`),
|
|
220
|
-
* -
|
|
221
|
-
* first segment (up to `/` or `:`) names a
|
|
222
|
-
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
284
|
+
* - a `#name` path (`@#project/memories/x.md`), or an alias path
|
|
285
|
+
* (`@docs/x.md`), where the first segment (up to `/` or `:`) names a
|
|
286
|
+
* registered alias,
|
|
287
|
+
* - a home-relative path (`@~/notes/x.md`),
|
|
288
|
+
* - or a recipe reference (`@~workflow/task-files/_partials/x.md`), where
|
|
289
|
+
* the `~` sigil is followed by a namespace, a recipe and a file inside it.
|
|
290
|
+
* Recipe references are only consulted when a namespace resolver was
|
|
291
|
+
* supplied.
|
|
292
|
+
* Any of them may use glob syntax, and then includes every file it matches,
|
|
293
|
+
* in sorted path order; a glob that matches nothing is an error.
|
|
294
|
+
*
|
|
295
|
+
* A file may be included any number of times; only a file that includes
|
|
296
|
+
* itself (directly or through others) is an error.
|
|
226
297
|
*
|
|
227
298
|
* Lines inside fenced code blocks (``` or ~~~, per CommonMark) are left
|
|
228
299
|
* verbatim, so include syntax can be documented without being executed.
|
|
229
300
|
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
* naming the
|
|
233
|
-
*
|
|
301
|
+
* A line that starts with `@` and looks like an include (see
|
|
302
|
+
* {@link looksLikeIncludeLine}) but is not a well-formed one is a compile
|
|
303
|
+
* error naming the file and the line, never text copied to the output.
|
|
304
|
+
*
|
|
305
|
+
* Resolution produces ordered groups of candidates (see include-resolver);
|
|
306
|
+
* the first group that names a file is used. If none does, it errors, naming
|
|
307
|
+
* the including file and listing every path tried, plus what went wrong with
|
|
308
|
+
* a `~` or `#` reference when one was attempted.
|
|
234
309
|
*
|
|
235
310
|
* @param content - The file's raw text.
|
|
236
311
|
* @param baseDir - Directory the relative candidate resolves against.
|
|
@@ -242,18 +317,16 @@ export class CompilationService {
|
|
|
242
317
|
baseDir: string,
|
|
243
318
|
projectRoot: string,
|
|
244
319
|
fromFile?: string
|
|
245
|
-
):
|
|
246
|
-
// First segment allows ~ and . (so ./ and ../ work), then path chars;
|
|
247
|
-
// separators / and :; allows ${...}.
|
|
248
|
-
const includePattern = /^@([~a-zA-Z0-9_.${}][a-zA-Z0-9_\-/.:${}]*\.md)$/;
|
|
320
|
+
): Part[] {
|
|
249
321
|
const fenceOpenPattern = /^ {0,3}(`{3,}|~{3,})/;
|
|
250
322
|
const fenceClosePattern = /^ {0,3}(`{3,}|~{3,})[ \t]*$/;
|
|
323
|
+
const where = fromFile ?? baseDir;
|
|
251
324
|
|
|
252
|
-
const out:
|
|
325
|
+
const out: Part[] = [];
|
|
253
326
|
let fenceChar: string | null = null;
|
|
254
327
|
let fenceLength = 0;
|
|
255
328
|
|
|
256
|
-
for (const line of content.split("\n")) {
|
|
329
|
+
for (const [lineIndex, line] of content.split("\n").entries()) {
|
|
257
330
|
if (fenceChar !== null) {
|
|
258
331
|
// Inside a fence: emit verbatim; only a matching closing fence ends it.
|
|
259
332
|
const close = fenceClosePattern.exec(line);
|
|
@@ -272,28 +345,63 @@ export class CompilationService {
|
|
|
272
345
|
continue;
|
|
273
346
|
}
|
|
274
347
|
|
|
275
|
-
const
|
|
348
|
+
const trimmed = line.trimEnd();
|
|
349
|
+
const match = INCLUDE_LINE_PATTERN.exec(trimmed);
|
|
276
350
|
if (!match) {
|
|
351
|
+
if (looksLikeIncludeLine(trimmed)) {
|
|
352
|
+
this.handleError(
|
|
353
|
+
`Malformed include line: ${trimmed}\n in file: ${where}, line ${lineIndex + 1}\n` +
|
|
354
|
+
` An include is "@" followed by a path that ends in .md, optionally followed by ` +
|
|
355
|
+
`"?name=value". The path may hold letters, digits and . _ - / : $ { } * ? [ ] , ~ #.\n` +
|
|
356
|
+
` To write a line that starts with "@" as text, put a word after it.`
|
|
357
|
+
);
|
|
358
|
+
}
|
|
277
359
|
out.push(line);
|
|
278
360
|
continue;
|
|
279
361
|
}
|
|
280
362
|
|
|
281
363
|
const includePath = match[1].trim();
|
|
282
|
-
const { candidates, namespaceIssue } =
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
364
|
+
const { files, glob, candidates, namespaceIssue, hashIssue, view } = resolveIncludeFiles(
|
|
365
|
+
includePath,
|
|
366
|
+
{
|
|
367
|
+
aliases: this.aliases,
|
|
368
|
+
views: this.views,
|
|
369
|
+
scope: this.includeScope,
|
|
370
|
+
baseDir,
|
|
371
|
+
namespaceResolver: this.namespaceResolver,
|
|
372
|
+
fromFile: where,
|
|
373
|
+
}
|
|
374
|
+
);
|
|
375
|
+
|
|
376
|
+
if (files.length === 0) {
|
|
377
|
+
// A view is a list that may be empty: a glob over it that selects
|
|
378
|
+
// nothing includes nothing, and the line leaves no trace.
|
|
379
|
+
if (view === true && glob && hashIssue === undefined) continue;
|
|
380
|
+
if (this.namespaceResolver === undefined && includePath.startsWith("~")) {
|
|
381
|
+
// Without a resolver the reference is only a relative path, so say
|
|
382
|
+
// what is wrong with a malformed one instead of "not found".
|
|
383
|
+
const reference = splitIncludeQuery(includePath).path.slice(1);
|
|
384
|
+
if (reference.includes("/") && !reference.includes("${")) {
|
|
385
|
+
try {
|
|
386
|
+
sharedRefResolver().parse(reference, RefSource.Include);
|
|
387
|
+
} catch (error) {
|
|
388
|
+
this.handleError(
|
|
389
|
+
`Malformed include line: ${trimmed}\n in file: ${where}, line ${lineIndex + 1}\n ${
|
|
390
|
+
error instanceof Error ? error.message : String(error)
|
|
391
|
+
}`
|
|
392
|
+
);
|
|
393
|
+
out.push("");
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
}
|
|
292
398
|
const explanation = namespaceIssue
|
|
293
399
|
? `\n${formatNamespaceProblem(namespaceIssue)}`
|
|
294
|
-
:
|
|
400
|
+
: hashIssue
|
|
401
|
+
? `\n in file: ${where}\n ${hashIssue.split("\n").join("\n ")}`
|
|
402
|
+
: `\n in file: ${where}`;
|
|
295
403
|
this.handleError(
|
|
296
|
-
|
|
404
|
+
`${glob ? "Include matched no files" : "Include not found"}: @${includePath}${explanation}\n tried:\n${candidates
|
|
297
405
|
.map((c) => ` - ${c}`)
|
|
298
406
|
.join("\n")}`
|
|
299
407
|
);
|
|
@@ -301,57 +409,121 @@ export class CompilationService {
|
|
|
301
409
|
continue;
|
|
302
410
|
}
|
|
303
411
|
|
|
304
|
-
|
|
305
|
-
this.
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
412
|
+
for (const fullPath of files) {
|
|
413
|
+
if (this.includeStack.includes(fullPath)) {
|
|
414
|
+
this.handleError(
|
|
415
|
+
`Circular dependency detected: ${this.includeStack.join(" -> ")} -> ${fullPath}`
|
|
416
|
+
);
|
|
417
|
+
out.push("");
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
311
420
|
|
|
312
|
-
|
|
313
|
-
if (includedContent !== null) {
|
|
314
|
-
const relativePath = path.relative(projectRoot, fullPath);
|
|
315
|
-
const sourceComment = this.currentIncludeSourceComments
|
|
316
|
-
? `<!-- from: ${relativePath} -->\n`
|
|
317
|
-
: "";
|
|
318
|
-
out.push(sourceComment + includedContent);
|
|
319
|
-
} else {
|
|
320
|
-
out.push("");
|
|
421
|
+
out.push(this.loadFile(fullPath, projectRoot) === null ? "" : { file: fullPath });
|
|
321
422
|
}
|
|
322
423
|
}
|
|
323
424
|
|
|
324
|
-
return out
|
|
425
|
+
return out;
|
|
325
426
|
}
|
|
326
427
|
|
|
327
|
-
/**
|
|
328
|
-
|
|
428
|
+
/**
|
|
429
|
+
* Reads a file and, recursively, every file it includes, without rendering
|
|
430
|
+
* anything. A file already read for this target is not read again (its
|
|
431
|
+
* includes resolve the same way whoever includes it), so a file may be
|
|
432
|
+
* included any number of times; only a cycle is an error.
|
|
433
|
+
*/
|
|
434
|
+
private loadFile(filePath: string, projectRoot: string): FileNode | null {
|
|
435
|
+
const known = this.nodes.get(filePath);
|
|
436
|
+
if (known !== undefined) return known;
|
|
437
|
+
|
|
329
438
|
if (!fs.existsSync(filePath)) {
|
|
330
439
|
this.handleError(`File not found: ${filePath}`);
|
|
331
440
|
return null;
|
|
332
441
|
}
|
|
333
442
|
|
|
334
|
-
if (this.visited.has(filePath)) {
|
|
335
|
-
return "";
|
|
336
|
-
}
|
|
337
|
-
|
|
338
443
|
this.includeStack.push(filePath);
|
|
339
444
|
|
|
340
445
|
try {
|
|
341
446
|
const content = fs.readFileSync(filePath, "utf8");
|
|
342
447
|
const baseDir = path.dirname(filePath);
|
|
343
|
-
const
|
|
344
|
-
|
|
345
|
-
|
|
448
|
+
const node: FileNode = {
|
|
449
|
+
path: filePath,
|
|
450
|
+
content,
|
|
451
|
+
isTpl: path.basename(filePath).includes(".tpl."),
|
|
452
|
+
parts: [],
|
|
453
|
+
};
|
|
454
|
+
// Registered before its includes are read so the graph lists parents
|
|
455
|
+
// first; the cycle check uses the stack, not this map.
|
|
456
|
+
this.nodes.set(filePath, node);
|
|
457
|
+
node.parts = this.processIncludes(content, baseDir, projectRoot, filePath);
|
|
458
|
+
return node;
|
|
346
459
|
} catch (error) {
|
|
347
460
|
const message = error instanceof Error ? error.message : String(error);
|
|
348
461
|
this.handleError(`Failed to read file ${filePath}: ${message}`);
|
|
462
|
+
this.nodes.delete(filePath);
|
|
349
463
|
return null;
|
|
350
464
|
} finally {
|
|
351
465
|
this.includeStack.pop();
|
|
352
466
|
}
|
|
353
467
|
}
|
|
354
468
|
|
|
469
|
+
/**
|
|
470
|
+
* Turns a read file into its final text.
|
|
471
|
+
*
|
|
472
|
+
* A file whose own name contains `.tpl.` is rendered as Liquid with the
|
|
473
|
+
* given variables; no other file ever is. Each include is left as a unique
|
|
474
|
+
* marker while the file renders, so a condition around an include line is
|
|
475
|
+
* honored, and an included file's own output is never rendered again. Every
|
|
476
|
+
* marker that survives the render is then replaced by that file's final text,
|
|
477
|
+
* produced the same way.
|
|
478
|
+
*
|
|
479
|
+
* @param filePath - The file to expand.
|
|
480
|
+
* @param vars - The output's variable scope; undefined means templates stay unrendered.
|
|
481
|
+
* @param entryDir - The directory of the target's entry point, also searched by `{% render %}`.
|
|
482
|
+
* @param projectRoot - Root used to render source comments as relative paths.
|
|
483
|
+
*/
|
|
484
|
+
private async expand(
|
|
485
|
+
filePath: string,
|
|
486
|
+
vars: Record<string, string> | undefined,
|
|
487
|
+
entryDir: string,
|
|
488
|
+
projectRoot: string
|
|
489
|
+
): Promise<string> {
|
|
490
|
+
const node = this.nodes.get(filePath)!;
|
|
491
|
+
const nonce = `${process.pid}-${Math.random().toString(36).slice(2)}`;
|
|
492
|
+
const markers = new Map<string, string>();
|
|
493
|
+
const chunks = node.parts.map((part) => {
|
|
494
|
+
if (typeof part === "string") return part;
|
|
495
|
+
const marker = `\u0001SOUS-INCLUDE-${nonce}-${markers.size}\u0001`;
|
|
496
|
+
markers.set(marker, part.file);
|
|
497
|
+
return marker;
|
|
498
|
+
});
|
|
499
|
+
|
|
500
|
+
let text = chunks.join("\n");
|
|
501
|
+
if (node.isTpl && vars !== undefined) {
|
|
502
|
+
const dir = path.dirname(node.path);
|
|
503
|
+
text = await this.renderContent(
|
|
504
|
+
text,
|
|
505
|
+
{ ...vars, sousTemplatePath: node.path, sousTemplateDir: dir },
|
|
506
|
+
[...new Set([dir, entryDir])],
|
|
507
|
+
node.path
|
|
508
|
+
);
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
for (const [marker, child] of markers) {
|
|
512
|
+
if (!text.includes(marker)) continue;
|
|
513
|
+
const body = await this.expand(child, vars, entryDir, projectRoot);
|
|
514
|
+
const comment = this.currentIncludeSourceComments
|
|
515
|
+
? `<!-- from: ${path.relative(projectRoot, child)} -->\n`
|
|
516
|
+
: "";
|
|
517
|
+
text = text.split(marker).join(comment + body);
|
|
518
|
+
}
|
|
519
|
+
return text;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/** Every file read for the current target, parents before the files they include. */
|
|
523
|
+
private graphNodes(): FileNode[] {
|
|
524
|
+
return [...this.nodes.values()];
|
|
525
|
+
}
|
|
526
|
+
|
|
355
527
|
/**
|
|
356
528
|
* Render template content using LiquidJS with the given variable scope.
|
|
357
529
|
*
|
|
@@ -368,9 +540,11 @@ export class CompilationService {
|
|
|
368
540
|
): Promise<string> {
|
|
369
541
|
const engine = createLiquidEngine(roots, {
|
|
370
542
|
aliases: this.aliases,
|
|
543
|
+
views: this.views,
|
|
371
544
|
scope: { ...this.includeScope, ...vars },
|
|
372
545
|
namespaceResolver: this.namespaceResolver,
|
|
373
546
|
fromFile,
|
|
547
|
+
secretVariables: this.secretVariables,
|
|
374
548
|
});
|
|
375
549
|
try {
|
|
376
550
|
return await engine.parseAndRender(content, vars);
|
|
@@ -473,8 +647,8 @@ ${taskFileContents}
|
|
|
473
647
|
|
|
474
648
|
this.initializeEncoder();
|
|
475
649
|
|
|
476
|
-
this.visited.clear();
|
|
477
650
|
this.includeStack = [];
|
|
651
|
+
this.nodes = new Map();
|
|
478
652
|
this.currentIncludeSourceComments =
|
|
479
653
|
typeof target.includeSourceComments === "boolean"
|
|
480
654
|
? target.includeSourceComments
|
|
@@ -487,9 +661,9 @@ ${taskFileContents}
|
|
|
487
661
|
}
|
|
488
662
|
|
|
489
663
|
const promptsRoot = path.dirname(target.rootInputPath);
|
|
490
|
-
const
|
|
664
|
+
const entry = this.loadFile(target.rootInputPath, promptsRoot);
|
|
491
665
|
|
|
492
|
-
if (
|
|
666
|
+
if (entry === null) {
|
|
493
667
|
displayError(`Failed to compile ${target.rootInputPath}`);
|
|
494
668
|
return false;
|
|
495
669
|
}
|
|
@@ -499,12 +673,24 @@ ${taskFileContents}
|
|
|
499
673
|
// be written: the previous copy stays in place.
|
|
500
674
|
const assembled = this.errors.length === errorsBefore;
|
|
501
675
|
|
|
502
|
-
// Compute source hash once per target from the
|
|
503
|
-
|
|
676
|
+
// Compute the source hash once per target from every file read: the entry
|
|
677
|
+
// point and everything it includes, whatever a condition decides later.
|
|
678
|
+
const graph = this.graphNodes();
|
|
679
|
+
for (const node of graph) this.includedFilesSeen.add(realPathOf(node.path));
|
|
680
|
+
const templates = graph.filter((node) => node.isTpl);
|
|
681
|
+
const contentHash = hashContent(
|
|
682
|
+
JSON.stringify([
|
|
683
|
+
this.currentIncludeSourceComments,
|
|
684
|
+
graph.map((node) => [path.relative(promptsRoot, node.path), node.content]),
|
|
685
|
+
])
|
|
686
|
+
);
|
|
504
687
|
|
|
505
688
|
let allSucceeded = true;
|
|
506
689
|
|
|
507
|
-
const isTpl =
|
|
690
|
+
const isTpl = entry.isTpl;
|
|
691
|
+
// Whether any file of this target renders as Liquid, which makes the
|
|
692
|
+
// output depend on its variables.
|
|
693
|
+
const rendersAnything = templates.length > 0;
|
|
508
694
|
|
|
509
695
|
for (const output of target.outputs) {
|
|
510
696
|
// Resolve destination path: prefer destinationFile, fall back to destinationDir mirroring
|
|
@@ -524,9 +710,13 @@ ${taskFileContents}
|
|
|
524
710
|
// A rendered output depends on its variables as much as on its source: a
|
|
525
711
|
// changed answer or `_vars` value with the same template must re-render,
|
|
526
712
|
// so the variable scope is part of a `.tpl.` output's source hash. A
|
|
527
|
-
// verbatim copy hashes its content alone.
|
|
528
|
-
|
|
529
|
-
|
|
713
|
+
// verbatim copy hashes its content alone. The secret names are part of it
|
|
714
|
+
// too, because they decide what a scope-dumping tag hides.
|
|
715
|
+
const srcHash = rendersAnything && output.vars
|
|
716
|
+
? hashContent(
|
|
717
|
+
`${contentHash}\n${stableVarsFingerprint(output.vars)}\n` +
|
|
718
|
+
`secrets:${JSON.stringify([...this.secretVariables].sort())}`
|
|
719
|
+
)
|
|
530
720
|
: contentHash;
|
|
531
721
|
|
|
532
722
|
// Skip if content is unchanged and file already exists (unless --rebuild)
|
|
@@ -554,8 +744,10 @@ ${taskFileContents}
|
|
|
554
744
|
// resolver always sets `vars` on every output (at minimum the inherited
|
|
555
745
|
// scope). It is a real guard for callers that build a CompilationConfig
|
|
556
746
|
// directly, and it stays as a tripwire in case the resolver ever changes.
|
|
557
|
-
if (
|
|
558
|
-
|
|
747
|
+
if (rendersAnything && !output.vars) {
|
|
748
|
+
for (const template of templates) {
|
|
749
|
+
this.unrenderedTemplates.push(`${template.path} → ${destFile}`);
|
|
750
|
+
}
|
|
559
751
|
// In strict mode this warning is an error, so the output is not written.
|
|
560
752
|
if (this.strict) {
|
|
561
753
|
this.reportNotWritten(destFile);
|
|
@@ -565,18 +757,12 @@ ${taskFileContents}
|
|
|
565
757
|
}
|
|
566
758
|
|
|
567
759
|
const errorsBeforeRender = this.errors.length;
|
|
568
|
-
const resolvedContent =
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
sousTemplateDir: promptsRoot,
|
|
575
|
-
},
|
|
576
|
-
[promptsRoot],
|
|
577
|
-
target.rootInputPath
|
|
578
|
-
)
|
|
579
|
-
: content;
|
|
760
|
+
const resolvedContent = await this.expand(
|
|
761
|
+
target.rootInputPath,
|
|
762
|
+
output.vars,
|
|
763
|
+
promptsRoot,
|
|
764
|
+
promptsRoot
|
|
765
|
+
);
|
|
580
766
|
|
|
581
767
|
// A template that failed to render is not written either.
|
|
582
768
|
if (this.errors.length > errorsBeforeRender) {
|
|
@@ -644,6 +830,15 @@ ${taskFileContents}
|
|
|
644
830
|
return allSucceeded;
|
|
645
831
|
}
|
|
646
832
|
|
|
833
|
+
/**
|
|
834
|
+
* Every file the last compile read, as real paths: the entry points and every
|
|
835
|
+
* file any include line of any target named, by any route (exact path, glob
|
|
836
|
+
* or view). Includes inside an unmet condition count, since they are read.
|
|
837
|
+
*/
|
|
838
|
+
includedFiles(): ReadonlySet<string> {
|
|
839
|
+
return this.includedFilesSeen;
|
|
840
|
+
}
|
|
841
|
+
|
|
647
842
|
/**
|
|
648
843
|
* Compile all targets from the given config. Every target is compiled even
|
|
649
844
|
* after an error, so every error is listed; an output whose target had an
|
|
@@ -675,7 +870,10 @@ ${taskFileContents}
|
|
|
675
870
|
|
|
676
871
|
this.includeSourceComments = config.includeSourceComments === true;
|
|
677
872
|
this.aliases = config.aliases ?? {};
|
|
873
|
+
this.views = config.views ?? {};
|
|
874
|
+
this.includedFilesSeen = new Set();
|
|
678
875
|
this.includeScope = config.includeScope ?? {};
|
|
876
|
+
this.secretVariables = config.secretVariables ?? [];
|
|
679
877
|
|
|
680
878
|
this.initializeEncoder();
|
|
681
879
|
|
|
@@ -738,3 +936,12 @@ ${taskFileContents}
|
|
|
738
936
|
// Backward-compat alias so existing imports of MarkdownCompiler keep working
|
|
739
937
|
export { CompilationService as MarkdownCompiler };
|
|
740
938
|
export type { CompilationServiceOptions as MarkdownCompilerOptions };
|
|
939
|
+
|
|
940
|
+
/** The path with symbolic links resolved, or the path itself when it cannot be read. */
|
|
941
|
+
function realPathOf(filePath: string): string {
|
|
942
|
+
try {
|
|
943
|
+
return fs.realpathSync(filePath);
|
|
944
|
+
} catch {
|
|
945
|
+
return filePath;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
@@ -33,7 +33,7 @@ export type ProjectConfigFormat = (typeof PROJECT_CONFIG_FORMATS)[number];
|
|
|
33
33
|
* compiled output (`AGENTS.md` at the project root) are never mistaken for one
|
|
34
34
|
* another.
|
|
35
35
|
*/
|
|
36
|
-
export const STARTER_PROMPT_RELATIVE_PATH = "memories/AGENTS.md";
|
|
36
|
+
export const STARTER_PROMPT_RELATIVE_PATH = "memories/AGENTS.tpl.md";
|
|
37
37
|
|
|
38
38
|
/** The name of the file the starter prompt is compiled into, at the project root. */
|
|
39
39
|
export const STARTER_OUTPUT_NAME = "AGENTS.md";
|
|
@@ -98,14 +98,19 @@ export const config = {
|
|
|
98
98
|
],
|
|
99
99
|
},
|
|
100
100
|
|
|
101
|
-
//
|
|
102
|
-
// project subscribes to the \`core\` namespace on its own, which is how
|
|
103
|
-
// skills that teach an agent about sous reach \`.claude/skills\`. Memories
|
|
104
|
-
//
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
101
|
+
// What to do with the content the recipes this project subscribes to publish.
|
|
102
|
+
// Every project subscribes to the \`core\` namespace on its own, which is how
|
|
103
|
+
// the skills that teach an agent about sous reach \`.claude/skills\`. Memories
|
|
104
|
+
// are not copied anywhere: the starter prompt pulls them in with one include
|
|
105
|
+
// line, and the lists below (uncomment to use them) choose which come first
|
|
106
|
+
// and which are left out. An entry is a glob over \`namespace/recipe\`, or a
|
|
107
|
+
// regular expression written between two slashes.
|
|
108
|
+
recipes: {
|
|
109
|
+
skills: { outputs: ["\${projectRoot}/.claude/skills"] },
|
|
110
|
+
// memories: {
|
|
111
|
+
// first: ["communication/*"],
|
|
112
|
+
// exclude: ["tool-usage/automated-browser-tasks", "/^experimental\\//"],
|
|
113
|
+
// },
|
|
109
114
|
},
|
|
110
115
|
};
|
|
111
116
|
`;
|
|
@@ -132,8 +137,8 @@ export function buildConfigJson(context: ProjectScaffoldContext): string {
|
|
|
132
137
|
},
|
|
133
138
|
],
|
|
134
139
|
},
|
|
135
|
-
|
|
136
|
-
skills: ["${projectRoot}/.claude/skills"],
|
|
140
|
+
recipes: {
|
|
141
|
+
skills: { outputs: ["${projectRoot}/.claude/skills"] },
|
|
137
142
|
},
|
|
138
143
|
};
|
|
139
144
|
return `${JSON.stringify(config, null, 2)}\n`;
|
|
@@ -158,6 +163,9 @@ source, run \`sous build\`, and the compiled copy follows; never edit the compil
|
|
|
158
163
|
each one in with a line holding only \`@sections/name.md\`.
|
|
159
164
|
- The skills under \`.claude/skills\` are written by sous from the recipes this
|
|
160
165
|
project subscribes to. \`sous recipe list\` shows what is available.
|
|
166
|
+
|
|
167
|
+
{% comment %}This line pulls in the memories of every recipe this project subscribes to.{% endcomment %}
|
|
168
|
+
@#memories/**/*.md
|
|
161
169
|
`;
|
|
162
170
|
}
|
|
163
171
|
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* deliberately the only place that knows: the subscription service for the
|
|
7
7
|
* trusted repositories and their cached indexes, the lockfile service for what
|
|
8
8
|
* is pinned, the links map and the store for a recipe's own files, and
|
|
9
|
-
* `
|
|
9
|
+
* `recipes` for where a content kind lands.
|
|
10
10
|
*
|
|
11
11
|
* By default nothing here downloads anything: a repository whose index has
|
|
12
12
|
* never been fetched is left out of the catalog and named separately, so a
|