@sous-io/sous 0.2.25 → 0.2.27

package/src/lib/build-service.ts

@@ -10,14 +10,16 @@ import { CompilationService, resolveOutputPath } from "./markdown-compiler.js";
 import type { CompilationConfig, CompilationTarget } from "./markdown-compiler.js";
 import { StateService } from "./state.js";
 import { isProtectedPath } from "./state.js";
-import { protectedRepoPaths } from "./repos/links.js";
+import { describeLinkedRepos, protectedRepoPaths } from "./repos/links.js";
+import { subscriptionServiceFor } from "./repos/subscription-service.js";
+import { prepareRepositoriesForBuild } from "./build-preparation.js";
 import {
   noRecipeAnswers,
   resolveRecipeAnswers,
   unansweredWarning,
   type RecipeAnswers,
 } from "./vars/answers.js";
-import { log, warning } from "../utils/formatting.js";
+import { footer, heading, log, warning } from "../utils/formatting.js";
 
 export type BuildOptions = {
   strict?: boolean;
@@ -437,22 +439,64 @@ export class BuildService {
   }
 }
 
+/** What {@link runProjectBuild} is given. */
+export type ProjectBuildInput = {
+  /** The merged project config, reloaded after any change a command made. */
+  settings: Settings;
+  /** Where the active config was discovered. */
+  configContext: ConfigContext;
+  /** The shell environment as it was before the env files were injected. */
+  shellEnv?: NodeJS.ProcessEnv;
+  /** The build's own options; every one defaults to what a plain `sous build` uses. */
+  options?: Omit<BuildOptions, "configContext">;
+  /** The heading printed above the compile and prune step. Defaults to "Building". */
+  heading?: string;
+};
+
 /**
- * Compiles and prunes a project exactly the way `sous build` does with no flags.
+ * The one build every command runs: `build`, `init`, `launch`, `prune`,
+ * `repo remove`, `repo unlink`, `subscription add`, `subscription remove` and
+ * `subscription update` all come through here, so a project is built the same
+ * way whichever of them the user typed.
  *
- * Shared by the subscription commands, which rebuild the project the moment they
- * have changed what it subscribes to: a newly subscribed recipe's files appear,
- * and a removed one's files are pruned, without anyone having to remember a
- * second command. Every option is left at its default on purpose, because the
- * point is to run the ordinary build and nothing else.
+ * In order, it announces every linked repository, prepares the project's
+ * recipes (seeds the core recipe, locks new subscriptions, restores whatever
+ * the store is missing, checks upstream and reports newer versions), and then
+ * compiles and prunes. A dry run prepares nothing, because preparing writes the
+ * lockfile and the store. A partial rebuild in watch mode (one changed source
+ * file) prepares nothing either: the config did not change, so neither did the
+ * recipes it needs.
  *
- * @param settings - The merged project config, reloaded after the change.
- * @param configContext - Where the active config was discovered.
- * @returns True when compile and prune both succeeded.
+ * @param input - The project, the build options and the heading.
+ * @returns True when compile and prune both succeeded; false after any compile error.
  */
-export async function buildProjectOutputs(
-  settings: Settings,
-  configContext: ConfigContext
-): Promise<boolean> {
-  return new BuildService().build(settings, { configContext });
+export async function runProjectBuild(input: ProjectBuildInput): Promise<boolean> {
+  const options = input.options ?? {};
+  const partial = options.changedFile !== undefined;
+
+  // A linked repository is read from a working copy instead of a published
+  // version, so it is announced on every build; a build that silently produced
+  // something different would be far worse than a noisy one.
+  if (!partial) {
+    const linked = describeLinkedRepos(input.configContext.sousDir);
+    if (linked.length > 0) warning(linked.join("\n"));
+  }
+
+  if (!options.dryRun && !partial) {
+    await prepareRepositoriesForBuild(
+      subscriptionServiceFor({
+        configContext: input.configContext,
+        settings: input.settings,
+        ...(input.shellEnv === undefined ? {} : { shellEnv: input.shellEnv }),
+      })
+    );
+  }
+
+  heading(input.heading ?? "Building");
+  const succeeded = await new BuildService().build(input.settings, {
+    ...options,
+    configContext: input.configContext,
+  });
+  footer();
+  return succeeded;
 }

package/src/lib/markdown-compiler.ts

@@ -63,6 +63,11 @@ export type CompilationConfig = {
 };
 
 export type CompilationServiceOptions = {
+  /**
+   * Treat compile warnings as errors. An error always fails the compile; this
+   * only decides whether a warning (a `.tpl.` file copied without being
+   * rendered, a git branch that could not be read) does too.
+   */
   strict?: boolean;
   rebuild?: boolean;
   dryRun?: boolean;
@@ -184,14 +189,26 @@ export class CompilationService {
     this.encoder = get_encoding("o200k_base");
   }
 
-  /** Handle errors according to strict mode. */
+  /**
+   * Records a compile error and prints it. The compile carries on, so every
+   * target is compiled and every error is listed; the output the error belongs
+   * to is not written, and the compile as a whole reports failure.
+   */
   private handleError(message: string): void {
     this.errors.push(message);
     displayError(message);
+  }
 
+  /**
+   * Records a compile warning: printed as a warning, or, in strict mode,
+   * recorded as an error, with everything an error brings.
+   */
+  private handleWarning(message: string): void {
     if (this.strict) {
-      process.exit(1);
+      this.handleError(message);
+      return;
     }
+    warning(message);
   }
 
   /**
@@ -383,7 +400,10 @@ export class CompilationService {
       ).trim();
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
-      this.handleError(`Failed to resolve current git branch: ${message}`);
+      this.handleWarning(
+        `Sous could not read the current git branch, so the runtime context says ` +
+          `"unknown": ${message}`
+      );
     }
 
     const runtimeHeader = `## Runtime Session Context
@@ -427,6 +447,21 @@ ${taskFileContents}
     fs.writeFileSync(ctx.outputPath, `${runtimeHeader}${taskFileBlock}\n`, "utf8");
   }
 
+  /**
+   * Says that an output was not written because its target had an error, and
+   * what that leaves on disk.
+   *
+   * @param destFile - The output that was not written.
+   */
+  private reportNotWritten(destFile: string): void {
+    const verb = this.dryRun ? "would not be written" : "not written";
+    log(
+      fs.existsSync(destFile)
+        ? `  ✗ ${destFile} (${verb} because of an error; the previous copy stays)`
+        : `  ✗ ${destFile} (${verb} because of an error)`
+    );
+  }
+
   /** Compile a single target, writing to all of its outputs. */
   private async compileTarget(
     target: CompilationTarget,
@@ -445,6 +480,8 @@ ${taskFileContents}
         ? target.includeSourceComments
         : this.includeSourceComments;
 
+    const errorsBefore = this.errors.length;
+
     if (target.runtimeContext) {
       this.generateRuntimeSessionContext(target);
     }
@@ -457,6 +494,11 @@ ${taskFileContents}
       return false;
     }
 
+    // An include that failed left a hole in the assembled content (and in strict
+    // mode, a warning counts as an error), so none of this target's outputs may
+    // be written: the previous copy stays in place.
+    const assembled = this.errors.length === errorsBefore;
+
     // Compute source hash once per target from the assembled content
     const contentHash = hashContent(content);
 
@@ -473,6 +515,12 @@ ${taskFileContents}
       if (resolvedDest === undefined) continue;
       destFile = resolvedDest;
 
+      if (!assembled) {
+        this.reportNotWritten(destFile);
+        allSucceeded = false;
+        continue;
+      }
+
       // A rendered output depends on its variables as much as on its source: a
       // changed answer or `_vars` value with the same template must re-render,
       // so the variable scope is part of a `.tpl.` output's source hash. A
@@ -508,8 +556,15 @@ ${taskFileContents}
       // directly, and it stays as a tripwire in case the resolver ever changes.
       if (isTpl && !output.vars) {
         this.unrenderedTemplates.push(`${target.rootInputPath} → ${destFile}`);
+        // In strict mode this warning is an error, so the output is not written.
+        if (this.strict) {
+          this.reportNotWritten(destFile);
+          allSucceeded = false;
+          continue;
+        }
       }
 
+      const errorsBeforeRender = this.errors.length;
       const resolvedContent = (isTpl && output.vars)
         ? await this.renderContent(
             content,
@@ -522,6 +577,14 @@ ${taskFileContents}
             target.rootInputPath
           )
         : content;
+
+      // A template that failed to render is not written either.
+      if (this.errors.length > errorsBeforeRender) {
+        this.reportNotWritten(destFile);
+        allSucceeded = false;
+        continue;
+      }
+
       const fileContent = resolvedContent;
       const outputDir = path.dirname(destFile);
 
@@ -582,8 +645,11 @@ ${taskFileContents}
   }
 
   /**
-   * Compile all targets from the given config.
-   * Returns true if all targets compiled successfully.
+   * Compile all targets from the given config. Every target is compiled even
+   * after an error, so every error is listed; an output whose target had an
+   * error is not written and keeps its previous copy and its state entry.
+   *
+   * @returns True when nothing reported an error; false after any error.
    */
   async compile(config: CompilationConfig, stateFilePath?: string): Promise<boolean> {
     const stateService = new StateService();
@@ -632,15 +698,7 @@ ${taskFileContents}
           `\nFix: add a '_vars' block to the output (an empty '_vars: {}' is enough to\n` +
           `enable rendering), or rename the source so it does not contain '.tpl.'.`;
 
-        if (this.strict) {
-          // Recorded as a failure rather than exiting here, so state still gets
-          // written and the caller decides the exit code.
-          this.errors.push(message);
-          displayError(message);
-          allSucceeded = false;
-        } else {
-          warning(message);
-        }
+        this.handleWarning(message);
       }
 
       if (this.errors.length > 0) {
@@ -665,7 +723,9 @@ ${taskFileContents}
         await stateService.save(stateFilePath, state);
       }
 
-      return allSucceeded;
+      // Any error fails the compile, including one that belongs to no single
+      // target (a bad config option).
+      return allSucceeded && this.errors.length === 0;
     } finally {
       if (this.encoder) {
         this.encoder.free();

package/src/lib/repos/formats/common.ts

@@ -307,12 +307,29 @@ function stripExtensionKeys(value: unknown): unknown {
  * Builds a strict object schema for a HAND-WRITTEN format: unknown keys are
  * rejected so typos surface immediately, except keys in the reserved `x-`
  * extension namespace, which are accepted and ignored. Machine-written formats
- * use plain `z.strictObject` instead; nothing writes extension keys into them.
+ * use `z.strictObject` or `forwardCompatibleObject` instead; nothing writes
+ * extension keys into them.
  */
 export function extensibleObject<Shape extends z.ZodRawShape>(shape: Shape) {
   return z.preprocess(stripExtensionKeys, z.strictObject(shape));
 }
 
+/**
+ * Builds an object schema for a machine-written format that other versions of
+ * sous publish and this one reads: the repo index. Every known key is validated
+ * exactly as strictly as `z.strictObject` would, and a key this version does not
+ * define is kept, unread, rather than refused.
+ *
+ * A later sous adds fields to the index, and a repository published by it is
+ * read by every older sous a project happens to pin; refusing the whole index
+ * over a field it cannot use would break those projects for nothing. The key is
+ * kept rather than dropped, so a release carrying a published version forward
+ * writes it back byte for byte, whichever sous runs the release.
+ */
+export function forwardCompatibleObject<Shape extends z.ZodRawShape>(shape: Shape) {
+  return z.looseObject(shape);
+}
+
 /**
  * The `submissions` block a repo manifest and a recipe manifest may both carry:
  * whether the recipes it covers take proposed changes, and where to send a

package/src/lib/repos/formats/index-file.ts

@@ -9,12 +9,17 @@
  *
  * Adding a repo fetches only this file. Nothing else is downloaded until a
  * project subscribes to something inside it.
+ *
+ * Every object in it is a `forwardCompatibleObject`: a field a later sous adds is
+ * kept and ignored rather than refused, so an index published by a newer sous
+ * still reads here. The fields this version defines are validated in full.
  */
 
 import { z } from "zod";
 import {
   contentHashSchema,
   formatVersionSchema,
+  forwardCompatibleObject,
   isoTimestampSchema,
   namespaceNameSchema,
   parseFormat,
@@ -38,30 +43,25 @@ import {
  * belongs to that repository's own index, so this entry carries the range the
  * manifest declared instead.
  */
-export const indexDependencySchema = z
-  .strictObject({
-    /** The exact version this dependency resolved to, when the release could resolve one. */
-    version: semverVersionSchema.optional(),
-    /** The range the manifest declared, recorded when no exact version could be resolved. */
-    range: semverRangeSchema.optional(),
-    /**
-     * The canonical identity of the repository publishing it
-     * (`github.com/sous-io/sous-recipes`). Omitted for a sibling, which lives in
-     * this same repository.
-     */
-    repo: repoIdentitySchema.optional(),
-  })
-  .refine(
-    (entry) => entry.version !== undefined || entry.range !== undefined,
-    {
-      message:
-        "must record either the exact version this dependency resolved to or the range " +
-        "the recipe declared",
-    }
-  );
+export const indexDependencySchema = forwardCompatibleObject({
+  /** The exact version this dependency resolved to, when the release could resolve one. */
+  version: semverVersionSchema.optional(),
+  /** The range the manifest declared, recorded when no exact version could be resolved. */
+  range: semverRangeSchema.optional(),
+  /**
+   * The canonical identity of the repository publishing it
+   * (`github.com/sous-io/sous-recipes`). Omitted for a sibling, which lives in
+   * this same repository.
+   */
+  repo: repoIdentitySchema.optional(),
+}).refine((entry) => entry.version !== undefined || entry.range !== undefined, {
+  message:
+    "must record either the exact version this dependency resolved to or the range " +
+    "the recipe declared",
+});
 
 /** One published version of one recipe. */
-export const indexVersionSchema = z.strictObject({
+export const indexVersionSchema = forwardCompatibleObject({
   /** Content hash of the recipe folder at this version, verified after every fetch. */
   hash: contentHashSchema,
   /**
@@ -98,7 +98,7 @@ export const indexVersionSchema = z.strictObject({
 });
 
 /** One recipe, with every version the repo publishes of it. */
-export const indexRecipeSchema = z.strictObject({
+export const indexRecipeSchema = forwardCompatibleObject({
   /** The recipe folder, relative to the repo root. */
   path: relativePathSchema("a recipe path"),
   /** One-paragraph summary, copied from the recipe manifest at release time. */
@@ -112,71 +112,69 @@ export const indexRecipeSchema = z.strictObject({
 });
 
 /** One namespace declaration, copied from the repo manifest at release time. */
-export const indexNamespaceSchema = z.strictObject({
+export const indexNamespaceSchema = forwardCompatibleObject({
   description: z.string().optional(),
 });
 
 /** The repo index schema. */
-export const indexFileSchema = z
-  .strictObject({
-    /**
-     * A plain-language note about where this copy of the index came from. JSON
-     * has no comment syntax and an index is machine-written, so this is the one
-     * place a writer can say something to whoever opens the file. Sous ignores
-     * the value everywhere except one place: the seed index it writes for its
-     * own built-in repository carries `SEED_INDEX_COMMENT`, which is how a
-     * later run recognizes its own placeholder and is willing to replace it.
-     */
-    $comment: z.string().optional(),
-    formatVersion: formatVersionSchema,
-    /** The repo's suggested short name, copied from its manifest. */
-    name: repoNameSchema,
-    /** When this index was generated. */
-    generatedAt: isoTimestampSchema,
-    /** The version of sous that generated it. */
-    generator: semverVersionSchema,
-    /** Every namespace the repo publishes. */
-    namespaces: z.record(namespaceNameSchema, indexNamespaceSchema),
-    /** Every recipe the repo publishes, keyed `namespace/recipe`. */
-    recipes: z.record(recipeKeySchema, indexRecipeSchema),
-  })
-  .superRefine((index, ctx) => {
-    // A recipe whose namespace is not declared could never be resolved, so a
-    // release that produced one is broken; say which recipe and which namespace.
-    for (const key of Object.keys(index.recipes)) {
-      const namespace = key.slice(0, key.indexOf("/"));
-      if (!Object.hasOwn(index.namespaces, namespace)) {
-        ctx.addIssue({
-          code: "custom",
-          path: ["recipes", key],
-          message:
-            `belongs to the namespace '${namespace}', which this index does not ` +
-            `declare under 'namespaces'`,
-        });
-      }
+export const indexFileSchema = forwardCompatibleObject({
+  /**
+   * A plain-language note about where this copy of the index came from. JSON
+   * has no comment syntax and an index is machine-written, so this is the one
+   * place a writer can say something to whoever opens the file. Sous ignores
+   * the value everywhere except one place: the seed index it writes for its
+   * own built-in repository carries `SEED_INDEX_COMMENT`, which is how a
+   * later run recognizes its own placeholder and is willing to replace it.
+   */
+  $comment: z.string().optional(),
+  formatVersion: formatVersionSchema,
+  /** The repo's suggested short name, copied from its manifest. */
+  name: repoNameSchema,
+  /** When this index was generated. */
+  generatedAt: isoTimestampSchema,
+  /** The version of sous that generated it. */
+  generator: semverVersionSchema,
+  /** Every namespace the repo publishes. */
+  namespaces: z.record(namespaceNameSchema, indexNamespaceSchema),
+  /** Every recipe the repo publishes, keyed `namespace/recipe`. */
+  recipes: z.record(recipeKeySchema, indexRecipeSchema),
+}).superRefine((index, ctx) => {
+  // A recipe whose namespace is not declared could never be resolved, so a
+  // release that produced one is broken; say which recipe and which namespace.
+  for (const key of Object.keys(index.recipes)) {
+    const namespace = key.slice(0, key.indexOf("/"));
+    if (!Object.hasOwn(index.namespaces, namespace)) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["recipes", key],
+        message:
+          `belongs to the namespace '${namespace}', which this index does not ` +
+          `declare under 'namespaces'`,
+      });
     }
+  }
 
-    // A version's tag is what the provider fetches, so a tag that does not
-    // name this exact recipe and version is a version pointing somewhere else.
-    // Nothing on the consumer side could otherwise tell: an index publishing
-    // `1.0.0` with `tag: "main"` would hand `git clone --branch main` a moving
-    // target, whose content changes on every push and whose pinned hash then
-    // simply starts failing. Sous writes these tags itself, so requiring the
-    // shape it writes costs a correct index nothing.
-    for (const [key, recipe] of Object.entries(index.recipes)) {
-      for (const [version, published] of Object.entries(recipe.versions)) {
-        const expected = `${key}@${version}`;
-        if (published.tag === expected) continue;
-        ctx.addIssue({
-          code: "custom",
-          path: ["recipes", key, "versions", version, "tag"],
-          message:
-            `is '${published.tag}', but a published version's tag names the recipe and ` +
-            `the version it carries, so this one must be '${expected}'`,
-        });
-      }
+  // A version's tag is what the provider fetches, so a tag that does not
+  // name this exact recipe and version is a version pointing somewhere else.
+  // Nothing on the consumer side could otherwise tell: an index publishing
+  // `1.0.0` with `tag: "main"` would hand `git clone --branch main` a moving
+  // target, whose content changes on every push and whose pinned hash then
+  // simply starts failing. Sous writes these tags itself, so requiring the
+  // shape it writes costs a correct index nothing.
+  for (const [key, recipe] of Object.entries(index.recipes)) {
+    for (const [version, published] of Object.entries(recipe.versions)) {
+      const expected = `${key}@${version}`;
+      if (published.tag === expected) continue;
+      ctx.addIssue({
+        code: "custom",
+        path: ["recipes", key, "versions", version, "tag"],
+        message:
+          `is '${published.tag}', but a published version's tag names the recipe and ` +
+          `the version it carries, so this one must be '${expected}'`,
+      });
     }
-  });
+  }
+});
 
 /** A validated repo index. */
 export type IndexFile = z.infer<typeof indexFileSchema>;