@sous-io/sous 0.2.32 → 0.2.34

package/src/services/ref-resolver/source.ts

@@ -0,0 +1,48 @@
+/**
+ * Where a ref was written. The parser reads every form wherever a ref comes
+ * from, and the place decides which of those forms it allows: the pruner for
+ * that place drops the readings it refuses, saying what to write there instead.
+ */
+export enum RefSource {
+  /**
+   * A ref typed on the command line. Every form is allowed: a bare name, a
+   * namespace and recipe, a `repo:` qualifier, a version range, a
+   * provider-scheme locator, an HTTPS, SSH or scheme-less URL, and a browser
+   * URL copied from a host's file view. Names may be written in any case.
+   */
+  CommandLine = "commandLine",
+  /**
+   * A subscription key in a config layer. Only the stored form is allowed:
+   * `namespace` or `namespace/recipe`, lowercase. The repository a
+   * subscription resolves into is recorded in the lockfile, and its range in
+   * the entry's own `range` field.
+   */
+  Config = "config",
+  /**
+   * An entry of a recipe manifest's `depends` or `subscribes` list. A bare ref
+   * names a recipe in the same repository; every locator and URL form names
+   * one in another repository. A `repo:` qualifier is refused, because it is
+   * one project's private name for a repository, and so is a local path.
+   */
+  Manifest = "manifest",
+  /**
+   * A key sous wrote itself: the lockfile, the index and the store. Only the
+   * canonical `namespace` or `namespace/recipe` is allowed.
+   */
+  Lockfile = "lockfile",
+  /**
+   * The path of a template include line, once its sigil is taken off. Only a
+   * file inside a recipe is allowed (`namespace/recipe/path`), and the path may
+   * be a glob.
+   */
+  Include = "include",
+}
+
+/** What each place is called in a sentence. */
+export const SOURCE_LABELS: Record<RefSource, string> = {
+  [RefSource.CommandLine]: "on the command line",
+  [RefSource.Config]: "as a subscription key in a config file",
+  [RefSource.Manifest]: "in a recipe manifest's 'depends' or 'subscribes' list",
+  [RefSource.Lockfile]: "as a key sous stores",
+  [RefSource.Include]: "in an include line",
+};

package/src/services/ref-resolver/tokens.ts

@@ -0,0 +1,25 @@
+/**
+ * The Inversify tokens of the ref services, in one place. A plugin adds a
+ * splitter, a pruner or a provider by binding one more value under the matching
+ * token; nothing else in the module needs to change.
+ */
+
+/** Every token the ref services are bound under. */
+export const REF_TOKENS = {
+  /** A `RefSplitter`; bound once per splitter, injected as a list. */
+  Splitter: Symbol.for("sous.ref.Splitter"),
+  /** A `RefPruner`; bound once per pruner, injected as a list. */
+  Pruner: Symbol.for("sous.ref.Pruner"),
+  /** A `RepoProvider`; bound once per provider, injected as a list. */
+  Provider: Symbol.for("sous.ref.Provider"),
+  /** A `HashName`; bound once per `#` name, injected as a list. */
+  HashName: Symbol.for("sous.ref.HashName"),
+  /** The `HashNameRegistry`. */
+  HashNames: Symbol.for("sous.ref.HashNames"),
+  /** The `RefParser`. */
+  Parser: Symbol.for("sous.ref.Parser"),
+  /** The `RefResolverService`. */
+  Resolver: Symbol.for("sous.ref.Resolver"),
+  /** The `RefPickerService`. */
+  Picker: Symbol.for("sous.ref.Picker"),
+} as const;

package/src/services/ref-resolver/types.ts

@@ -0,0 +1,155 @@
+/**
+ * What a ref is: plain data, never a class instance, so it survives a JSON
+ * round trip, can be compared structurally and can be extended by a plugin.
+ *
+ * A ref is a discriminated union on `kind`. Every kind with parents nests them:
+ * a recipe file carries its recipe, which carries its namespace, which carries
+ * the repository qualifier it was written with. A parent is optional wherever
+ * the written form may leave it out (a bare `task-files` is a recipe whose
+ * namespace is unknown), and a parent is a candidate of its own only when the
+ * written text could mean it.
+ *
+ * Globs are a flag on the ref (`glob`), not a kind of their own: a glob is a
+ * way of writing the name or path of a namespace, a recipe or a file, and the
+ * kind is what a lookup needs to know where to search. `glob` is true when any
+ * name or path part of the ref (a parent's included) is a pattern.
+ */
+
+import type { ProviderId } from "../../lib/repos/providers/index.js";
+
+/** Where a ref says a repository lives. */
+export type RefLocation = {
+  /** The provider that reads the location. */
+  provider: ProviderId;
+  /** The host, such as `github.com`. */
+  host: string;
+  /** The repository's path on the host, such as `sous-io/sous-recipes`. */
+  repoPath: string;
+  /** The repository's canonical identity, as the store and the lockfile key it. */
+  identity: string;
+  /** The repository's HTTPS URL, which is what adding it is handed. */
+  url: string;
+};
+
+/** What every ref may carry. */
+export type RefBase = {
+  /** True when any name or path part of the ref is a glob pattern. */
+  glob?: true;
+  /** The `?name=value` pairs the ref was written with, percent-decoded. */
+  vars?: Record<string, string>;
+  /**
+   * What the publisher says the thing is, in one line. Never read from written
+   * text; a lookup fills it in so a person choosing between matches can tell
+   * them apart.
+   */
+  description?: string;
+};
+
+/**
+ * A repository: named by the short name a project gave it, by where it lives,
+ * or both. As the parent of another ref it is the repository qualifier.
+ */
+export type RepoRef = RefBase & {
+  kind: "repo";
+  /** The project's short name for the repository (may be a glob). */
+  name?: string;
+  /** Where the repository lives, when the ref said so. */
+  location?: RefLocation;
+  /** A branch or tag, once a lookup has told it apart from the path after it. */
+  revision?: string;
+  /**
+   * A folder or file path copied from a host's file view, branch still in
+   * front of it (`main/recipes/workflow/alpha`). Which recipe or namespace it
+   * is, and where the branch ends, only the repository's index can say.
+   */
+  browsed?: string;
+  /** The version range a browsed path was written with; it applies to the recipe it settles to. */
+  range?: string;
+};
+
+/** A namespace published by a repository. */
+export type NamespaceRef = RefBase & {
+  kind: "namespace";
+  /** The namespace name (may be a glob). */
+  name: string;
+  /** The repository qualifier, when one was written. */
+  repo?: RepoRef;
+  /** True when the namespace was spelled out as `namespace/*`. */
+  wildcard?: true;
+};
+
+/** A recipe published in a namespace. */
+export type RecipeRef = RefBase & {
+  kind: "recipe";
+  /** The recipe name (may be a glob). */
+  name: string;
+  /** The namespace, when the ref named it. */
+  namespace?: NamespaceRef;
+  /** The repository qualifier, set here only when there is no namespace to hold it. */
+  repo?: RepoRef;
+  /** The semantic version range, when one was given. */
+  range?: string;
+};
+
+/** A file inside a recipe. */
+export type RecipeFileRef = RefBase & {
+  kind: "recipeFile";
+  /** The path inside the recipe, `/`-separated (may be a glob). */
+  path: string;
+  /** The recipe holding it. */
+  recipe: RecipeRef;
+  /** The semantic version range, when one was given. */
+  range?: string;
+};
+
+/** A variable a recipe declares, written `namespace/recipe.variableName`. */
+export type VariableRef = RefBase & {
+  kind: "variable";
+  /** The variable's own name, as its author wrote it (may be a glob). */
+  name: string;
+  /** The recipe declaring it, when the ref named it. */
+  recipe?: RecipeRef;
+  /** The repository qualifier, set here only when there is no recipe to hold it. */
+  repo?: RepoRef;
+};
+
+/** An environment variable name, held whole: it is never split into scopes. */
+export type EnvVarRef = RefBase & {
+  kind: "envVar";
+  /** The name, exactly as written. */
+  name: string;
+  /** The variables this name answers, once a lookup has said. */
+  variables?: VariableRef[];
+};
+
+/** Anything a written ref can name. */
+export type SousRef =
+  | RepoRef
+  | NamespaceRef
+  | RecipeRef
+  | RecipeFileRef
+  | VariableRef
+  | EnvVarRef;
+
+/** Every kind, in the order matches of equal specificity are listed in. */
+export const REF_KINDS = [
+  "repo",
+  "namespace",
+  "recipe",
+  "recipeFile",
+  "variable",
+  "envVar",
+] as const;
+
+/** One kind of ref. */
+export type RefKind = (typeof REF_KINDS)[number];
+
+/** What each kind is called in a sentence. */
+export const KIND_LABELS: Record<RefKind, string> = {
+  repo: "repository",
+  namespace: "namespace",
+  recipe: "recipe",
+  recipeFile: "recipe file",
+  variable: "variable",
+  envVar: "environment variable",
+};

package/src/templating/init-liquid-engine.ts

@@ -3,7 +3,8 @@ import path from "node:path";
 import { Liquid, type FS } from "liquidjs";
 import filterRegistrars from "./filters/index.js";
 import tagRegistrars from "./tags/index.js";
-import { resolveInclude, type AliasMap } from "../lib/include-resolver.js";
+import { setSecretNames } from "./lib/secret-scope.js";
+import { resolveIncludeFiles, type AliasMap, type ViewMap } from "../lib/include-resolver.js";
 import {
   formatNamespaceProblem,
   type NamespaceResolver,
@@ -13,9 +14,11 @@ import {
 export type EngineAliasOptions = {
   /** Resolved alias map (name → ordered base dirs). */
   aliases?: AliasMap;
+  /** The files each view lists, so a render path may name `#memories/...`. */
+  views?: ViewMap;
   /** Variable scope for `${var}` substitution in render paths. */
   scope?: Record<string, string>;
-  /** Resolver consulted for a `~namespace` first segment in a render path. */
+  /** Resolver consulted for a `~` recipe reference in a render path. */
   namespaceResolver?: NamespaceResolver;
   /**
    * Absolute path of the template being rendered, handed to the namespace
@@ -23,16 +26,23 @@ export type EngineAliasOptions = {
    * resolves them against, which is enough to locate the owning recipe.
    */
   fromFile?: string;
+  /**
+   * The variable names a recipe declared `secret: true`. `{% showVars %}` and
+   * `{% exportScalarVarsJs %}` hide them, along with anything that very probably
+   * is a secret.
+   */
+  secretVariables?: Iterable<string>;
 };
 
 /**
- * A node-backed LiquidJS FS that additionally understands alias and namespace
- * render paths — `{% render "@~project/x.md" %}`, `{% render "@docs/y.md" %}`,
- * `{% render "@${var}/z.md" %}` and `{% render "~workflow/task-files/x.md" %}` —
- * resolving them through the same alias/namespace/var/relative candidate logic
- * as `@include`. The leading `@` is optional for a `~`-sigil path, since `~`
- * already marks the path as symbolic rather than relative. Every other path
- * uses standard root-based resolution.
+ * A node-backed LiquidJS FS that additionally understands alias, `#` name and
+ * recipe render paths: `{% render "#project/x.md" %}`, `{% render "@docs/y.md" %}`,
+ * `{% render "@${var}/z.md" %}` and `{% render "~workflow/task-files/x.md" %}`.
+ * They resolve through the same alias/`#`/recipe/var/relative candidate logic
+ * as `@include`, and a glob renders the first file it matches. The leading `@`
+ * is optional for a `~` or `#` path, since the sigil already marks the path as
+ * symbolic rather than relative. Every other path uses standard root-based
+ * resolution.
  *
  * @param opts - Alias map, variable scope, namespace resolver and including file.
  * @returns A LiquidJS FS implementation.
@@ -42,27 +52,31 @@ function createAliasFS(opts: EngineAliasOptions): FS {
   const scope = opts.scope ?? {};
 
   /**
-   * Resolve an `@`-path or `~`-path to its first existing candidate, or the
-   * first candidate. Returns null for ordinary paths so the caller falls back
-   * to root-based resolution. Throws when a `~namespace` reference resolved to
-   * nothing and the resolver explained why, so the reason reaches the user
+   * Resolve an `@`-path, `~`-path or `#`-path to its first existing file, or
+   * the first candidate. Returns null for ordinary paths so the caller falls
+   * back to root-based resolution. Throws when a `~` or `#` reference resolved
+   * to nothing and the resolver explained why, so the reason reaches the user
    * instead of a bare "file not found".
    */
   const resolveSymbolic = (file: string, dir: string): string | null => {
     const isAt = file.startsWith("@");
-    if (!isAt && !file.startsWith("~")) return null;
+    if (!isAt && !file.startsWith("~") && !file.startsWith("#")) return null;
 
     const rawPath = isAt ? file.slice(1) : file;
-    const { candidates, namespaceIssue } = resolveInclude(rawPath, {
+    const { files, candidates, namespaceIssue, hashIssue } = resolveIncludeFiles(rawPath, {
       aliases,
+      views: opts.views,
       scope,
       baseDir: dir,
       namespaceResolver: opts.namespaceResolver,
       fromFile: opts.fromFile ?? dir,
     });
 
-    const existing = candidates.find((c) => fs.existsSync(c));
-    if (existing) return existing;
+    if (files[0] !== undefined) return files[0];
+
+    if (hashIssue) {
+      throw new Error(`Cannot render "${file}"\n  ${hashIssue.split("\n").join("\n  ")}`);
+    }
 
     if (namespaceIssue) {
       throw new Error(
@@ -100,7 +114,7 @@ function createAliasFS(opts: EngineAliasOptions): FS {
  * @param roots - Filesystem root paths searched (in order) when resolving
  *   `{% render %}` partials (relative paths resolve against these).
  * @param aliasOpts - Optional alias map, variable scope and namespace resolver,
- *   enabling `@alias/...`, `@${var}/...` and `~namespace/...` paths in
+ *   enabling `@alias/...`, `#name/...`, `@${var}/...` and `~namespace/...` paths in
  *   `{% render %}` (parity with `@include`).
  */
 export function createLiquidEngine(roots: string[], aliasOpts: EngineAliasOptions = {}): Liquid {
@@ -120,5 +134,6 @@ export function createLiquidEngine(roots: string[], aliasOpts: EngineAliasOption
     register(engine);
   }
 
+  setSecretNames(engine, aliasOpts.secretVariables ?? []);
   return engine;
 }

package/src/templating/lib/secret-scope.ts

@@ -0,0 +1,41 @@
+import type { Liquid } from "liquidjs";
+import { HIDDEN_VALUE, isSecretVariable } from "../../lib/vars/secrets.js";
+
+/**
+ * The variable names each engine was told are secret. Kept beside the engine
+ * rather than in its scope, so the list itself never shows up in a dump.
+ */
+const secretNamesByEngine = new WeakMap<Liquid, ReadonlySet<string>>();
+
+/**
+ * Records the variable names a recipe declared `secret: true` for one engine.
+ *
+ * @param engine - The engine the tags render with.
+ * @param names - The declared names.
+ */
+export function setSecretNames(engine: Liquid, names: Iterable<string>): void {
+  secretNamesByEngine.set(engine, new Set(names));
+}
+
+/**
+ * The declared secret names for an engine; empty when none were recorded.
+ *
+ * @param engine - The engine the tags render with.
+ */
+export function secretNamesFor(engine: Liquid): ReadonlySet<string> {
+  return secretNamesByEngine.get(engine) ?? new Set();
+}
+
+/**
+ * Whether a scope entry is a secret a whole-scope dump must not print: a
+ * declared secret, or one the heuristic in `lib/vars/secrets.ts` recognizes.
+ *
+ * @param engine - The engine the tag renders with.
+ * @param name - The variable name.
+ * @param value - Its value.
+ */
+export function isSecretEntry(engine: Liquid, name: string, value: unknown): boolean {
+  return isSecretVariable(name, value, secretNamesFor(engine));
+}
+
+export { HIDDEN_VALUE };

package/src/templating/tags/exportScalarVarsJs.ts

@@ -1,6 +1,7 @@
 import type { Liquid } from "liquidjs";
 import type { Context } from "liquidjs/dist/context/context.js";
 import { sortObjectKeys } from "../../utils/formatting.js";
+import { isSecretEntry } from "../lib/secret-scope.js";
 
 /**
  * A "scalar" for export purposes: the value types that can be safely embedded
@@ -20,7 +21,9 @@ function isScalar(value: unknown): value is string | number | boolean {
  * Iterates every variable currently in the LiquidJS scope, keeps only scalars
  * (strings, finite numbers, booleans), sorts them by key, and emits a valid
  * `export default { ... };` block. Objects, arrays, functions, null, undefined,
- * and non-finite numbers are skipped.
+ * and non-finite numbers are skipped, and so is every secret (one a recipe
+ * declared, or one that very probably is): runtime code would read a masked
+ * value as a real one, so a secret is left out rather than hidden.
  *
  * Intended for compiling a `settings.tpl.mjs` that downstream runtime code
  * (e.g. browser automation scripts) imports for project configuration.
@@ -32,7 +35,7 @@ export function registerExportScalarVarsJsTag(engine: Liquid): void {
       const scalars: Record<string, string | number | boolean> = {};
 
       for (const [key, value] of Object.entries(scope)) {
-        if (isScalar(value)) scalars[key] = value;
+        if (isScalar(value) && !isSecretEntry(engine, key, value)) scalars[key] = value;
       }
 
       const sorted = sortObjectKeys(scalars);

package/src/templating/tags/showVars.ts

@@ -1,18 +1,24 @@
 import type { Liquid } from "liquidjs";
 import type { Context } from "liquidjs/dist/context/context.js";
 import { sortObjectKeys } from "../../utils/formatting.js";
+import { HIDDEN_VALUE, isSecretEntry } from "../lib/secret-scope.js";
 
-/** Dumps all variables currently in scope as a fenced JSON block. */
+/**
+ * Dumps all variables currently in scope as a fenced JSON block. A secret (one a
+ * recipe declared, or one that very probably is) prints as `(hidden)`, so the
+ * dump never carries it into compiled output.
+ */
 export function registerShowVarsTag(engine: Liquid): void {
   engine.registerTag("showVars", {
     render(ctx: Context) {
       const seen = new WeakSet();
       const scope = sortObjectKeys(ctx.getAll() as Record<string, unknown>);
-      const json = JSON.stringify(scope, (_key, value) => {
+      const json = JSON.stringify(scope, (key, value) => {
         if (typeof value === "object" && value !== null) {
           if (seen.has(value)) return "[Circular]";
           seen.add(value);
         }
+        if (key !== "" && isSecretEntry(engine, key, value)) return HIDDEN_VALUE;
         return value;
       }, 2);