@sous-io/sous 0.2.33 → 0.2.35

package/docs/markdown/repositories-variables.md

@@ -341,6 +341,22 @@ name in `_vars`, for instance `taskFileDir: "${sousDir}/../${taskFileRoot}"`.
 
 The `_env` block is still the way to reach any environment variable no recipe asks about.
 
+### Secrets stay out of scope dumps
+
+A secret renders only where a template names it, as in `{{ apiToken }}`. The two tags that print the whole
+scope leave secrets out: `{% showVars %}` prints `(hidden)` in place of the value, and
+`{% exportScalarVarsJs %}` drops the variable, because runtime code importing the module would read a mask
+as a real value. Both treat a variable as a secret when either holds:
+
+- A recipe declares it `secret: true`.
+- It very probably is one, by a thin check that needs no declaration: a name whose last words say so
+  (`githubToken`, `OPENAI_API_KEY`, `dbPassword`, `clientSecret`) holding more than a number or a switch, or
+  a value in a widely used token format (a GitHub or GitLab token, an AWS access key id, a private key block,
+  a JSON Web Token, a URL carrying a password).
+
+So `serviceApiKey: "abc123"` in `_vars` is hidden from a dump, while `maxTokens: "4096"` and
+`requireToken: "true"` still show.
+
 ## Where to go next
 
 - [Consuming recipes](repositories-consuming.md): subscribing, building, and what lands where

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sous-io/sous",
-  "version": "0.2.33",
+  "version": "0.2.35",
   "description": "Compiles AI coding agent configuration (CLAUDE.md, skills, memories) from LiquidJS templates",
   "type": "module",
   "license": "Apache-2.0",

package/recipes/core/sous-skills/skills/about-liquid-templates/SKILL.tpl.md

@@ -205,7 +205,10 @@ Sous plugin; a Sous plugin comes only from sous's built-ins or an npm-style modu
 
 **`showVars`**: dumps all variables currently in scope as a fenced JSON block.
 Useful during development to see exactly what variables are available at a given point
-in a template. Remove before finalizing.
+in a template. Remove before finalizing. A secret prints as `(hidden)`: a variable a
+recipe declares `secret: true`, or one that very probably is a secret by its name
+(`githubToken`, `OPENAI_API_KEY`) or its format (a GitHub token, a private key block). A
+template that names a secret explicitly, as in `{% raw %}{{ apiToken }}{% endraw %}`, still renders it.
 
 **`getFiles`**: globs files under a root directory and assigns the resulting array to a
 template variable. It renders nothing; present the results yourself with a `for` loop.
@@ -238,7 +241,8 @@ render paths relative to the root instead of bare file names:
 
 **`exportScalarVarsJs`**: emits every in-scope scalar variable (string, finite number,
 boolean) as an ES module default export, keys sorted. Objects, arrays, `null` and
-non-finite numbers are skipped. Use it to compile a settings module that runtime code
+non-finite numbers are skipped, and so is every secret (the same ones `showVars` hides),
+since runtime code would read a mask as a real value. Use it to compile a settings module that runtime code
 imports, rather than re-deriving project configuration:
 
 {% raw %}
@@ -294,7 +298,8 @@ project's variables change.
 
 ## Available Variables
 
-The following variables are in scope at compile time in this project:
+The following variables are in scope at compile time in this project (secrets show as
+`(hidden)`):
 
 {% showVars %}
 

package/recipes/core/sous-skills/sous.recipe.yaml

@@ -11,7 +11,7 @@ formatVersion: 1
 
 namespace: core
 name: sous-skills
-version: 0.2.33
+version: 0.2.35
 
 description: >-
   The skills that teach an agent what sous is and how it works: which files sous

package/src/lib/build-service.ts

@@ -118,13 +118,15 @@ export function resolveRecipeTargets(
 
   const missing = unansweredWarning(resolved, rootScope);
   if (missing !== undefined) recipes.warnings.push(missing);
-  return recipes;
+  return { ...recipes, secretVariables: [...resolved.secrets] };
 }
 
 /**
  * Adds the recipe targets to a project's own compilation config. A project with
  * no compilation block of its own still compiles its recipes, so the config is
- * created when there is none and there is something to compile.
+ * created when there is none and there is something to compile. The names the
+ * recipes declare secret go onto the config either way, because a project's own
+ * template may dump the scope too.
  *
  * @param config - The project's own compilation config, or null when it has none.
  * @param recipes - The targets the subscribed recipes contribute.
@@ -137,15 +139,17 @@ export function withRecipeTargets(
   settings: Settings,
   rootScope: Record<string, string>
 ): CompilationConfig | null {
-  if (recipes.targets.length === 0) return config;
+  const secrets = recipes.secretVariables?.length ? { secretVariables: recipes.secretVariables } : {};
+  if (recipes.targets.length === 0) return config === null ? null : { ...config, ...secrets };
   if (config === null) {
     return {
       targets: recipes.targets,
       aliases: resolveAliases(settings, rootScope),
       includeScope: rootScope,
+      ...secrets,
     };
   }
-  return { ...config, targets: [...config.targets, ...recipes.targets] };
+  return { ...config, targets: [...config.targets, ...recipes.targets], ...secrets };
 }
 
 /**

package/src/lib/markdown-compiler.ts

@@ -68,6 +68,12 @@ export type CompilationConfig = {
   views?: ViewMap;
   /** Variable scope for `${var}` substitution in `@include` paths. */
   includeScope?: Record<string, string>;
+  /**
+   * Every variable name a recipe declares `secret: true`. The tags that dump a
+   * whole scope hide these (and anything that very probably is a secret); a
+   * template that names one explicitly still renders it.
+   */
+  secretVariables?: string[];
 };
 
 export type CompilationServiceOptions = {
@@ -209,6 +215,8 @@ export class CompilationService {
   private numberFormatter: Intl.NumberFormat;
   private aliases: AliasMap;
   private includeScope: Record<string, string>;
+  /** Every variable name a recipe declares secret, for the scope-dumping tags. */
+  private secretVariables: string[];
   private namespaceResolver?: NamespaceResolver;
   /**
    * `.tpl.` outputs that were written without a variable scope, so LiquidJS never
@@ -232,6 +240,7 @@ export class CompilationService {
     this.numberFormatter = new Intl.NumberFormat("en-US");
     this.aliases = {};
     this.includeScope = {};
+    this.secretVariables = [];
     this.namespaceResolver = options.namespaceResolver;
     this.unrenderedTemplates = [];
   }
@@ -535,6 +544,7 @@ export class CompilationService {
       scope: { ...this.includeScope, ...vars },
       namespaceResolver: this.namespaceResolver,
       fromFile,
+      secretVariables: this.secretVariables,
     });
     try {
       return await engine.parseAndRender(content, vars);
@@ -700,9 +710,13 @@ ${taskFileContents}
       // A rendered output depends on its variables as much as on its source: a
       // changed answer or `_vars` value with the same template must re-render,
       // so the variable scope is part of a `.tpl.` output's source hash. A
-      // verbatim copy hashes its content alone.
+      // verbatim copy hashes its content alone. The secret names are part of it
+      // too, because they decide what a scope-dumping tag hides.
       const srcHash = rendersAnything && output.vars
-        ? hashContent(`${contentHash}\n${stableVarsFingerprint(output.vars)}`)
+        ? hashContent(
+            `${contentHash}\n${stableVarsFingerprint(output.vars)}\n` +
+              `secrets:${JSON.stringify([...this.secretVariables].sort())}`
+          )
         : contentHash;
 
       // Skip if content is unchanged and file already exists (unless --rebuild)
@@ -859,6 +873,7 @@ ${taskFileContents}
       this.views = config.views ?? {};
       this.includedFilesSeen = new Set();
       this.includeScope = config.includeScope ?? {};
+      this.secretVariables = config.secretVariables ?? [];
 
       this.initializeEncoder();
 

package/src/lib/repos/git-clone.ts

@@ -16,6 +16,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { spawnSync } from "node:child_process";
 import { ConfigError } from "../errors.js";
+import { LF_CHECKOUT_OPTIONS } from "./store/hash.js";
 
 /** What one git invocation produced. */
 export type GitResult = {
@@ -211,15 +212,20 @@ export function cloneRepo(
   const depth = options.depth ?? 1;
   const runner = options.runner ?? runGit;
 
+  // A linked checkout is hashed when its recipes are released or compiled, so
+  // it keeps the committed bytes whatever this machine's line-ending settings are.
   if (depth > 0) {
-    const shallow = runner(["clone", "--depth", String(depth), "--", url, destDir], {});
+    const shallow = runner(
+      ["clone", ...LF_CHECKOUT_OPTIONS, "--depth", String(depth), "--", url, destDir],
+      {}
+    );
     if (shallow.status === 0) return { depth, fellBackToFullClone: false };
     // git leaves the destination behind on some failures; clear it so the retry
     // is not refused by its own leftovers.
     fs.rmSync(destDir, { recursive: true, force: true });
   }
 
-  runGitOrThrow(["clone", "--", url, destDir], {
+  runGitOrThrow(["clone", ...LF_CHECKOUT_OPTIONS, "--", url, destDir], {
     runner: options.runner,
     what: `Cloning ${url}`,
   });

package/src/lib/repos/providers/git.ts

@@ -14,6 +14,7 @@ import { spawn } from "node:child_process";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { ConfigError } from "../../errors.js";
+import { LF_CHECKOUT_OPTIONS } from "../store/hash.js";
 
 /** What a finished subprocess reports back. */
 export type CommandResult = {
@@ -174,6 +175,9 @@ export async function fetchSubtree(options: {
         "1",
         "--filter=blob:none",
         "--sparse",
+        // The fetched subtree is hashed against its pin, so its bytes must be the
+        // committed ones whatever this machine's line-ending settings are.
+        ...LF_CHECKOUT_OPTIONS,
         "--branch",
         tag,
         // Everything after this is a path or a URL, never an option, whatever it

package/src/lib/repos/recipe-targets.ts

@@ -78,6 +78,11 @@ export type RecipeTargets = {
   watchDirs: string[];
   /** Complete, plain-language sentences a build should print before it compiles. */
   warnings: string[];
+  /**
+   * Every variable name a recipe declares `secret: true`, which the scope-dumping
+   * template tags hide. Set by the build from the resolved answers.
+   */
+  secretVariables?: string[];
 };
 
 /**

package/src/lib/repos/store/hash.ts

@@ -35,6 +35,23 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import { STORE_ENTRY_FILENAME } from "../formats/common.js";
 
+/**
+ * `git clone` options that check files out exactly as they were committed.
+ *
+ * A content hash is over raw bytes, and Git for Windows installs with
+ * `core.autocrlf=true` by default, which rewrites every text file to CRLF on
+ * checkout. A recipe published with LF then hashed differently on such a
+ * machine and failed its own pin. `--config` also writes these into the
+ * clone's own `.git/config`, so later checkouts in a linked working copy keep
+ * the same bytes.
+ */
+export const LF_CHECKOUT_OPTIONS = [
+  "--config",
+  "core.autocrlf=false",
+  "--config",
+  "core.eol=lf",
+] as const;
+
 /** Directory name never included in a content hash. */
 const GIT_DIR_NAME = ".git";
 

package/src/lib/vars/answers.ts

@@ -23,6 +23,11 @@
  *     is something to tell the user about, not a reason to refuse to build the
  *     rest of the project.
  *
+ * Alongside them comes `secrets`: every name some definition declares
+ * `secret: true`. The tags that dump a whole scope (`{% showVars %}`,
+ * `{% exportScalarVarsJs %}`) read it so a secret answer never lands in
+ * compiled output unless a template names it explicitly.
+ *
  * A definition's `default` counts as an answer of last resort: the
  * description a publisher writes promises what the default does, and a
  * template rendering an empty string instead would break that promise.
@@ -49,6 +54,8 @@ export interface RecipeAnswers {
   byRecipe: Map<string, VarScope>;
   /** Every required definition that nothing answered and that has no default. */
   unanswered: DefinedVariable[];
+  /** Every variable name some definition declares `secret: true`, answered or not. */
+  secrets: Set<string>;
 }
 
 /** How the answers are resolved. */
@@ -76,7 +83,7 @@ export interface RecipeAnswerOptions {
 
 /** The empty result, for a project that locks nothing. */
 export function noRecipeAnswers(): RecipeAnswers {
-  return { merged: {}, byRecipe: new Map(), unanswered: [] };
+  return { merged: {}, byRecipe: new Map(), unanswered: [], secrets: new Set() };
 }
 
 /**
@@ -103,6 +110,7 @@ export function resolveRecipeAnswers(options: RecipeAnswerOptions): RecipeAnswer
   for (const defined of definitions) {
     const value = answerFor(defined, context);
     const name = defined.definition.name;
+    if (defined.definition.secret) result.secrets.add(name);
 
     if (value === undefined) {
       if (defined.definition.required) result.unanswered.push(defined);

package/src/lib/vars/display.ts

@@ -19,9 +19,7 @@ import {
   type DefiningRecipe,
 } from "./definition-source.js";
 import { constraintBullets } from "./validate.js";
-
-/** What the value column shows for a secret whose answer is known. */
-export const HIDDEN_VALUE = "(hidden)";
+import { HIDDEN_VALUE } from "./secrets.js";
 
 /** What the value column shows for a variable nothing has answered. */
 export const UNANSWERED_VALUE = "(unanswered)";

package/src/lib/vars/index.ts

@@ -17,3 +17,4 @@ export * from "./ask.js";
 export * from "./report.js";
 export * from "./preanswers.js";
 export * from "./question-plan.js";
+export * from "./secrets.js";

package/src/lib/vars/secrets.ts

@@ -0,0 +1,131 @@
+/**
+ * Which variables hold a secret.
+ *
+ * A recipe marks a variable `secret: true`, and that declaration is the
+ * authority. Everything else in a template's scope (an `_env` mapping, a
+ * project `_vars` entry, an answer to a definition that forgot the flag) is
+ * checked by a deliberately thin heuristic: a name whose last words say it is
+ * a secret (`githubToken`, `OPENAI_API_KEY`, `dbPassword`), or a value in one
+ * of the widely used token formats (a GitHub or GitLab token, an AWS access
+ * key, a private key block, a URL carrying a password). The heuristic only
+ * catches what is very probably a secret; it is a safety net, not a scanner.
+ *
+ * This module imports nothing, so the template tags can use it without
+ * pulling the variables layer into the template engine.
+ */
+
+/** What a hidden value shows in place of the secret. */
+export const HIDDEN_VALUE = "(hidden)";
+
+/**
+ * Name endings that mark a secret, as lowercase words. A name matches when its
+ * last words are one of these, so `apiToken` and `GITHUB_TOKEN` match while
+ * `tokenLimit` does not.
+ */
+const SECRET_NAME_ENDINGS: readonly (readonly string[])[] = [
+  ["token"],
+  ["secret"],
+  ["secrets"],
+  ["password"],
+  ["passwd"],
+  ["passphrase"],
+  ["credential"],
+  ["credentials"],
+  ["apikey"],
+  ["api", "key"],
+  ["private", "key"],
+  ["access", "key"],
+  ["secret", "key"],
+  ["signing", "key"],
+  ["encryption", "key"],
+];
+
+/** Widely used secret formats, each specific enough that a match is very probably a secret. */
+const SECRET_VALUE_PATTERNS: readonly RegExp[] = [
+  /-----BEGIN [A-Z ]*PRIVATE KEY-----/, // a PEM private key
+  /\bgh[pousr]_[A-Za-z0-9]{36,}/, // GitHub token
+  /\bgithub_pat_[A-Za-z0-9_]{22,}/, // GitHub fine-grained token
+  /\bglpat-[A-Za-z0-9_-]{20,}/, // GitLab personal access token
+  /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/, // AWS access key id
+  /\bxox[abposr]-[A-Za-z0-9-]{10,}/, // Slack token
+  /\b[rs]k_live_[A-Za-z0-9]{16,}/, // Stripe live key
+  /\bsk-ant-[A-Za-z0-9_-]{20,}/, // Anthropic API key
+  /\bsk-(?:proj-)?[A-Za-z0-9_-]{32,}/, // OpenAI API key
+  /\bAIza[0-9A-Za-z_-]{35}/, // Google API key
+  /\bnpm_[A-Za-z0-9]{36}\b/, // npm token
+  /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/, // JSON Web Token
+  /^[a-z][a-z0-9+.-]*:\/\/[^/\s:@]+:[^/\s@]+@/i, // a URL carrying a password
+];
+
+/** Values a secret-sounding name may hold without being a secret: nothing, a number, a switch. */
+const PLAIN_VALUE = /^(?:|-?\d+(?:\.\d+)?|true|false|yes|no|on|off)$/i;
+
+/**
+ * The words in a variable name, lowercased: split at `_`, `-`, `.`, spaces and
+ * camelCase boundaries, so `OPENAI_API_KEY`, `openaiApiKey` and `openai-api-key`
+ * all read as `openai api key`.
+ *
+ * @param name - The variable name.
+ */
+function nameWords(name: string): string[] {
+  return name
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+    .split(/[^A-Za-z0-9]+/)
+    .filter((word) => word.length > 0)
+    .map((word) => word.toLowerCase());
+}
+
+/**
+ * Whether a variable name says it holds a secret.
+ *
+ * @param name - The variable name.
+ */
+export function hasSecretName(name: string): boolean {
+  const words = nameWords(name);
+  return SECRET_NAME_ENDINGS.some(
+    (ending) =>
+      words.length >= ending.length &&
+      ending.every((word, i) => words[words.length - ending.length + i] === word)
+  );
+}
+
+/**
+ * Whether a value is in one of the widely used secret formats.
+ *
+ * @param value - The value.
+ */
+export function hasSecretValue(value: string): boolean {
+  return SECRET_VALUE_PATTERNS.some((pattern) => pattern.test(value));
+}
+
+/**
+ * Whether a variable very probably holds a secret: its name says so (and its
+ * value is more than a number or a switch), or its value is in a known secret
+ * format.
+ *
+ * @param name - The variable name.
+ * @param value - Its value.
+ */
+export function looksLikeSecret(name: string, value: string): boolean {
+  if (hasSecretValue(value)) return true;
+  return hasSecretName(name) && !PLAIN_VALUE.test(value.trim());
+}
+
+/**
+ * Whether a variable is to be treated as a secret: a recipe declared it one,
+ * or the heuristic says it very probably is.
+ *
+ * @param name - The variable name.
+ * @param value - Its value.
+ * @param declared - The names recipes declared `secret: true`.
+ */
+export function isSecretVariable(
+  name: string,
+  value: unknown,
+  declared: ReadonlySet<string> = new Set()
+): boolean {
+  if (typeof value !== "string") return false;
+  if (declared.has(name)) return value.length > 0;
+  return looksLikeSecret(name, value);
+}

package/src/templating/init-liquid-engine.ts

@@ -3,6 +3,7 @@ import path from "node:path";
 import { Liquid, type FS } from "liquidjs";
 import filterRegistrars from "./filters/index.js";
 import tagRegistrars from "./tags/index.js";
+import { setSecretNames } from "./lib/secret-scope.js";
 import { resolveIncludeFiles, type AliasMap, type ViewMap } from "../lib/include-resolver.js";
 import {
   formatNamespaceProblem,
@@ -25,6 +26,12 @@ export type EngineAliasOptions = {
    * resolves them against, which is enough to locate the owning recipe.
    */
   fromFile?: string;
+  /**
+   * The variable names a recipe declared `secret: true`. `{% showVars %}` and
+   * `{% exportScalarVarsJs %}` hide them, along with anything that very probably
+   * is a secret.
+   */
+  secretVariables?: Iterable<string>;
 };
 
 /**
@@ -127,5 +134,6 @@ export function createLiquidEngine(roots: string[], aliasOpts: EngineAliasOption
     register(engine);
   }
 
+  setSecretNames(engine, aliasOpts.secretVariables ?? []);
   return engine;
 }

package/src/templating/lib/secret-scope.ts

@@ -0,0 +1,41 @@
+import type { Liquid } from "liquidjs";
+import { HIDDEN_VALUE, isSecretVariable } from "../../lib/vars/secrets.js";
+
+/**
+ * The variable names each engine was told are secret. Kept beside the engine
+ * rather than in its scope, so the list itself never shows up in a dump.
+ */
+const secretNamesByEngine = new WeakMap<Liquid, ReadonlySet<string>>();
+
+/**
+ * Records the variable names a recipe declared `secret: true` for one engine.
+ *
+ * @param engine - The engine the tags render with.
+ * @param names - The declared names.
+ */
+export function setSecretNames(engine: Liquid, names: Iterable<string>): void {
+  secretNamesByEngine.set(engine, new Set(names));
+}
+
+/**
+ * The declared secret names for an engine; empty when none were recorded.
+ *
+ * @param engine - The engine the tags render with.
+ */
+export function secretNamesFor(engine: Liquid): ReadonlySet<string> {
+  return secretNamesByEngine.get(engine) ?? new Set();
+}
+
+/**
+ * Whether a scope entry is a secret a whole-scope dump must not print: a
+ * declared secret, or one the heuristic in `lib/vars/secrets.ts` recognizes.
+ *
+ * @param engine - The engine the tag renders with.
+ * @param name - The variable name.
+ * @param value - Its value.
+ */
+export function isSecretEntry(engine: Liquid, name: string, value: unknown): boolean {
+  return isSecretVariable(name, value, secretNamesFor(engine));
+}
+
+export { HIDDEN_VALUE };

package/src/templating/tags/exportScalarVarsJs.ts

@@ -1,6 +1,7 @@
 import type { Liquid } from "liquidjs";
 import type { Context } from "liquidjs/dist/context/context.js";
 import { sortObjectKeys } from "../../utils/formatting.js";
+import { isSecretEntry } from "../lib/secret-scope.js";
 
 /**
  * A "scalar" for export purposes: the value types that can be safely embedded
@@ -20,7 +21,9 @@ function isScalar(value: unknown): value is string | number | boolean {
  * Iterates every variable currently in the LiquidJS scope, keeps only scalars
  * (strings, finite numbers, booleans), sorts them by key, and emits a valid
  * `export default { ... };` block. Objects, arrays, functions, null, undefined,
- * and non-finite numbers are skipped.
+ * and non-finite numbers are skipped, and so is every secret (one a recipe
+ * declared, or one that very probably is): runtime code would read a masked
+ * value as a real one, so a secret is left out rather than hidden.
  *
  * Intended for compiling a `settings.tpl.mjs` that downstream runtime code
  * (e.g. browser automation scripts) imports for project configuration.
@@ -32,7 +35,7 @@ export function registerExportScalarVarsJsTag(engine: Liquid): void {
       const scalars: Record<string, string | number | boolean> = {};
 
       for (const [key, value] of Object.entries(scope)) {
-        if (isScalar(value)) scalars[key] = value;
+        if (isScalar(value) && !isSecretEntry(engine, key, value)) scalars[key] = value;
       }
 
       const sorted = sortObjectKeys(scalars);

package/src/templating/tags/showVars.ts

@@ -1,18 +1,24 @@
 import type { Liquid } from "liquidjs";
 import type { Context } from "liquidjs/dist/context/context.js";
 import { sortObjectKeys } from "../../utils/formatting.js";
+import { HIDDEN_VALUE, isSecretEntry } from "../lib/secret-scope.js";
 
-/** Dumps all variables currently in scope as a fenced JSON block. */
+/**
+ * Dumps all variables currently in scope as a fenced JSON block. A secret (one a
+ * recipe declared, or one that very probably is) prints as `(hidden)`, so the
+ * dump never carries it into compiled output.
+ */
 export function registerShowVarsTag(engine: Liquid): void {
   engine.registerTag("showVars", {
     render(ctx: Context) {
       const seen = new WeakSet();
       const scope = sortObjectKeys(ctx.getAll() as Record<string, unknown>);
-      const json = JSON.stringify(scope, (_key, value) => {
+      const json = JSON.stringify(scope, (key, value) => {
         if (typeof value === "object" && value !== null) {
           if (seen.has(value)) return "[Circular]";
           seen.add(value);
         }
+        if (key !== "" && isSecretEntry(engine, key, value)) return HIDDEN_VALUE;
         return value;
       }, 2);