@sous-io/sous 0.2.17 → 0.2.18

package/src/commands/subscription/update.ts

@@ -0,0 +1,215 @@
+/**
+ * `sous subscription update [ref]`.
+ *
+ * Moves the lockfile's pins to the newest versions their ranges allow. With no
+ * reference it covers every subscription; a reference narrows it to one
+ * repository, one namespace or one recipe:
+ *
+ *     sous subscription update
+ *     sous subscription update sous-recipes
+ *     sous subscription update workflow
+ *     sous subscription update workflow/task-files
+ *
+ * Every trusted repository's index is fetched fresh first. A pin moves only
+ * within the range its subscription (or the recipe depending on it) declares,
+ * and dependencies move with the closure. Only the lockfile changes; the
+ * subscriptions themselves are never edited. The whole change is printed as a
+ * plan and asked about once, and the project is rebuilt afterwards.
+ */
+
+import { Args, Flags } from "@oclif/core";
+import { BaseCommand } from "../../base-command.js";
+import { buildProjectOutputs } from "../../lib/build-service.js";
+import { ConfigError } from "../../lib/errors.js";
+import { subscriptionServiceFor } from "../../lib/repos/subscription-service.js";
+import { describeUpdateScope } from "../../lib/repos/update-plan.js";
+import { formatAskReport } from "../../lib/vars/ask.js";
+import { collectProvidedAnswers } from "../../lib/vars/index.js";
+import {
+  blankLine,
+  dryRunNotice,
+  footer,
+  heading,
+  indent,
+  log,
+  paragraph,
+  showCommandVars,
+  subheading,
+  warning,
+} from "../../utils/formatting.js";
+import { answerFlags, confirmationFlag } from "../../utils/flags.js";
+
+export default class SubscriptionUpdate extends BaseCommand {
+  static description =
+    "Move this project's pinned recipe versions to the newest ones their ranges allow";
+
+  /**
+   * The other spelling of the topic. It lives under a hidden topic, so it is
+   * typable everywhere without ever reaching the top-level listing.
+   */
+  static aliases = ["subscriptions:update"];
+
+  static examples = [
+    "<%= config.bin %> subscription update",
+    "<%= config.bin %> subscription update sous-recipes",
+    "<%= config.bin %> subscription update workflow/task-files",
+    "<%= config.bin %> subscription update --dry-run",
+    "<%= config.bin %> subscription update --yes",
+  ];
+
+  static args = {
+    ref: Args.string({
+      description:
+        "What to update: a repository, a namespace or a recipe. Leave it out to update everything",
+      required: false,
+    }),
+  };
+
+  static flags = {
+    ...BaseCommand.baseFlags,
+    // One flag answers both questions this command can ask: the plan, and the
+    // trust question for a repository a newer version needs.
+    yes: confirmationFlag({ extraAliases: ["trust"] }),
+    "accept-first": Flags.boolean({
+      description:
+        "When the reference matches several things, take the first one listed",
+      default: false,
+    }),
+    "dry-run": Flags.boolean({
+      description:
+        "Print what would change without writing anything or downloading any recipe",
+      default: false,
+    }),
+    "no-build": Flags.boolean({
+      description: "Change the lockfile without rebuilding the project",
+      default: false,
+    }),
+    ...answerFlags(),
+  };
+
+  async run(): Promise<void> {
+    const { args, flags } = await this.parse(SubscriptionUpdate);
+    const dryRun = flags["dry-run"];
+
+    showCommandVars({
+      Project: this.projectLabel,
+      Config: this.configContext.configPath,
+      Updating: args.ref ?? "every subscription",
+      "Dry Run": dryRun,
+    });
+
+    heading("Updating");
+
+    if (dryRun) {
+      dryRunNotice(
+        "The indexes are fetched so the plan is current; no recipe is downloaded and " +
+          "nothing in this project is written."
+      );
+    }
+
+    const service = subscriptionServiceFor({
+      configContext: this.configContext,
+      settings: this.settings,
+      shellEnv: this.shellEnv,
+    });
+
+    const provided = collectProvidedAnswers({
+      ...(flags.answer === undefined ? {} : { answer: flags.answer }),
+      ...(flags["answers-file"] === undefined ? {} : { answersFile: flags["answers-file"] }),
+    });
+
+    const outcome = await service.update({
+      ...(args.ref === undefined ? {} : { ref: args.ref }),
+      yes: flags.yes,
+      acceptFirst: flags["accept-first"],
+      answers: provided,
+      dryRun,
+    });
+
+    const scope = describeUpdateScope(outcome.scope);
+
+    if (outcome.nothingToUpdate) {
+      footer();
+      return;
+    }
+
+    if (dryRun) {
+      paragraph(
+        `Nothing was written. Run the same command without '--dry-run' to update ${scope}.`
+      );
+      footer();
+      return;
+    }
+
+    blankLine();
+    subheading("Lockfile");
+    blankLine();
+    if (outcome.diff.unchanged) {
+      paragraph("Nothing changed.");
+    } else {
+      for (const line of outcome.diff.lines) log(indent(line));
+    }
+
+    if (outcome.trusted.length > 0) {
+      blankLine();
+      paragraph(
+        `Repositories trusted along the way: ${outcome.trusted.join(", ")}. They are ` +
+          `now recorded in this project's config, and your colleagues inherit them.`
+      );
+    }
+
+    if (outcome.answers !== undefined) {
+      blankLine();
+      subheading("Variables");
+      for (const line of formatAskReport(outcome.answers)) {
+        log(line === "" ? "" : indent(line));
+      }
+    }
+
+    if (outcome.cycles.length > 0) {
+      warning(
+        `Some of these recipes co-subscribe to each other in a circle:\n` +
+          outcome.cycles.map((cycle) => `  ${cycle.join(" -> ")}`).join("\n") +
+          `\nThat is unusual but not broken, and the lockfile was updated.`
+      );
+    }
+
+    const rebuilding = !flags["no-build"] && !outcome.diff.unchanged;
+
+    blankLine();
+    paragraph(
+      `The lockfile now pins the newest versions the ranges allow for ${scope}. ` +
+        `This project's subscriptions are exactly as they were.` +
+        (rebuilding || outcome.diff.unchanged ? "" : " The project was not rebuilt.")
+    );
+
+    footer();
+
+    if (rebuilding) await this.rebuildProject();
+  }
+
+  /**
+   * Rebuilds the project now that the lockfile has moved, so the outputs match
+   * the new versions when this command returns. The config is reloaded first,
+   * because a repository trusted along the way was written into a managed layer.
+   * A build that fails leaves the update in place, because it is already
+   * written; the message says so.
+   */
+  private async rebuildProject(): Promise<void> {
+    await this.reloadDiscoveredConfig();
+
+    heading("Building the project");
+
+    const succeeded = await buildProjectOutputs(this.settings, this.configContext);
+
+    footer();
+
+    if (!succeeded) {
+      throw new ConfigError(
+        `The lockfile was updated, but the build that followed it failed, so this ` +
+          `project's outputs may not match the new versions yet. The update itself is ` +
+          `recorded; fix what the build reported above and run 'sous build' again.`
+      );
+    }
+  }
+}

package/src/lib/repos/formats/links-map.ts

@@ -4,7 +4,8 @@
  * A link redirects a repo's resolution away from the store and at a real
  * working copy on disk, which is how a maintainer edits recipes: edits happen
  * in a checkout, never in the store. `sous repo link` writes an entry;
- * `sous repo unlink` removes it and leaves the checkout in place.
+ * `sous repo unlink` removes it and leaves the checkout in place unless asked
+ * to delete one sous cloned.
  *
  * Two maps are read: the project's `.sous/sous.links.json` and the machine-wide
  * `$SOUS_HOME/sous.links.json`, with the project map winning on conflict. The
@@ -34,8 +35,9 @@ export const repoLinkSchema = z.strictObject({
   linkedAt: isoTimestampSchema,
   /**
    * Whether sous cloned the working copy itself ('clone') or was pointed at an
-   * existing checkout ('path'). Unlinking never deletes either, but the origin
-   * tells the user what sous put there.
+   * existing checkout ('path'). Unlinking leaves either in place unless
+   * `--remove` is passed, and even then only a 'clone' is ever deleted: a
+   * 'path' checkout belongs to whoever linked it.
    */
   origin: z.enum(LINK_ORIGINS),
 });

package/src/lib/repos/git-clone.ts

@@ -227,6 +227,77 @@ export function cloneRepo(
   return { depth: 0, fellBackToFullClone: depth > 0 };
 }
 
+/** Work in a checkout that exists nowhere else, and would be lost with it. */
+export type UnsavedWork = {
+  /** Every changed or untracked path, as `git status --porcelain` prints it. */
+  uncommitted: string[];
+  /** Every commit on a local branch that no remote has, one line each. */
+  unpushed: string[];
+  /** Every stash entry, one line each. */
+  stashes: string[];
+  /**
+   * Why git could not be asked, when it could not. Nothing is known about the
+   * checkout then, which a caller must treat as possibly holding work.
+   */
+  unknown?: string;
+};
+
+/**
+ * What in a checkout would be lost if it were deleted: uncommitted changes,
+ * commits no remote has, and stashes.
+ *
+ * unsavedWork("/path/to/checkout")
+ * // -> { uncommitted: ["M README.md"], unpushed: [], stashes: [] }
+ *
+ * @param directory - The checkout to inspect.
+ * @param options - The git runner to use.
+ */
+export function unsavedWork(directory: string, options: GitOptions = {}): UnsavedWork {
+  const runner = options.runner ?? runGit;
+  const lines = (text: string): string[] =>
+    text
+      .split("\n")
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0);
+
+  const status = runner(["status", "--porcelain"], { cwd: directory });
+  if (status.status !== 0) {
+    return {
+      uncommitted: [],
+      unpushed: [],
+      stashes: [],
+      unknown:
+        `git could not read the checkout at ${directory}` +
+        (status.stderr.length > 0 ? `: ${status.stderr}` : "."),
+    };
+  }
+
+  const unpushed = runner(["log", "--branches", "--not", "--remotes", "--oneline"], {
+    cwd: directory,
+  });
+  const stashes = runner(["stash", "list"], { cwd: directory });
+
+  return {
+    uncommitted: lines(status.stdout),
+    unpushed: unpushed.status === 0 ? lines(unpushed.stdout) : [],
+    stashes: stashes.status === 0 ? lines(stashes.stdout) : [],
+  };
+}
+
+/**
+ * True when an inspection found nothing that would be lost.
+ *
+ * @param work - What `unsavedWork` found.
+ */
+export function hasNoUnsavedWork(work: UnsavedWork): boolean {
+  return (
+    work.unknown === undefined &&
+    work.uncommitted.length === 0 &&
+    work.unpushed.length === 0 &&
+    work.stashes.length === 0
+  );
+}
+
 /**
  * True when two remote URLs name the same repository, ignoring the differences
  * that never change what is fetched: a `.git` suffix, a trailing slash, the

package/src/lib/repos/links.ts

@@ -4,7 +4,8 @@
  * A link redirects one repository's resolution away from the store and at a
  * real working copy on disk, which is how a maintainer edits recipes: edits
  * happen in a checkout, never in the store. `sous repo link` writes an entry
- * here; `sous repo unlink` removes it and leaves the checkout alone.
+ * here; `sous repo unlink` removes it and leaves the checkout alone unless
+ * `--remove` asks it to delete one sous cloned.
  *
  * Two maps exist. The project's `.sous/sous.links.json` covers one project; the
  * machine-wide `$SOUS_HOME/sous.links.json` covers every project on the machine,

package/src/lib/repos/locked-recipes.ts

@@ -252,3 +252,25 @@ export function projectSubscriptionRefs(
   }
   return [...refs].sort();
 }
+
+/**
+ * Every locked recipe reachable from a set of keys through the lockfile's
+ * holder lists: the keys themselves, whatever they hold, and so on.
+ *
+ * @param lock - The lockfile.
+ * @param roots - The keys to start from.
+ */
+export function lockedClosure(lock: Lockfile, roots: string[]): string[] {
+  const reached = new Set(roots.filter((key) => Object.hasOwn(lock.recipes, key)));
+  const queue = [...reached];
+  while (queue.length > 0) {
+    const holder = queue.shift()!;
+    for (const [key, entry] of Object.entries(lock.recipes)) {
+      if (reached.has(key) || !entry.requestedBy.includes(holder)) continue;
+      reached.add(key);
+      queue.push(key);
+    }
+  }
+  return [...reached].sort();
+}
+

package/src/lib/repos/resolver.ts

@@ -95,6 +95,14 @@ export type ResolveContext = {
   loadManifest: RecipeManifestLoader;
   /** Whether prereleases are allowed when a request does not say. Defaults to false. */
   prerelease?: boolean;
+  /**
+   * Versions to hold where they are, keyed by recipe key. A recipe named here
+   * resolves to that version whenever it still satisfies every range asked of
+   * it, rather than to the newest one that does; a range that no longer allows
+   * it wins, and the newest satisfying version is chosen as usual. This is how
+   * an update moves only the pins it was asked to move.
+   */
+  keep?: Record<string, string>;
 };
 
 /** One recipe version the resolver settled on. */
@@ -608,7 +616,7 @@ function resolveRecipeRef(
     (previous?.prerelease ?? false) ||
     (item.prerelease ?? context.prerelease ?? false);
 
-  const version = pickVersion(key, entry, ranges, prerelease);
+  const version = pickVersion(key, entry, ranges, prerelease, context.keep?.[key]);
   const versionEntry = entry.versions[version]!;
 
   const requestedBy = [...(previous?.requestedBy ?? [])];
@@ -645,12 +653,14 @@ function resolveRecipeRef(
  * @param entry - The recipe's index entry.
  * @param ranges - Every range that has to hold, with who asked for it.
  * @param prerelease - Whether prereleases may match.
+ * @param keep - A version to hold, chosen whenever it is among the candidates.
  */
 function pickVersion(
   key: string,
   entry: IndexFile["recipes"][string],
   ranges: Array<{ range: string; requestedBy: string }>,
-  prerelease: boolean
+  prerelease: boolean,
+  keep?: string
 ): string {
   const published = Object.keys(entry.versions);
   const eligible = prerelease
@@ -664,6 +674,19 @@ function pickVersion(
     );
   }
 
+  // A held version is kept exactly as long as every range still allows it. It
+  // is checked against the published list rather than the eligible one, so a
+  // prerelease a subscription once opted into is not moved just for being one.
+  if (
+    keep !== undefined &&
+    Object.hasOwn(entry.versions, keep) &&
+    ranges.every(({ range }) =>
+      semver.satisfies(keep, range, { includePrerelease: true })
+    )
+  ) {
+    return keep;
+  }
+
   const best = semver.maxSatisfying(candidates, "*", { includePrerelease: prerelease });
   if (best !== null) return best;
 

package/src/lib/repos/seed.ts

@@ -48,6 +48,7 @@ import {
   type IndexOverlay,
 } from "./providers/index-cache.js";
 import { warning } from "../../utils/formatting.js";
+import { hashDirectorySync } from "./store/hash.js";
 import { identitySegments } from "./identity.js";
 import { ensureIndexCacheDirectory } from "../../utils/sous-directory.js";
 import type { RecipeStoreLike, StoreKey } from "./store/contract.js";
@@ -204,14 +205,70 @@ export async function seedCoreRecipe(
 export type CoreIndexOverlayOptions = {
   /** The packaged version, which is by rule the running sous version. */
   version: string;
-  /** The content hash of the entry the seed put in the store. */
-  hash: string;
+  /**
+   * The content hash of the entry the seed put in the store, or a function
+   * that works it out the first time it is needed. The function form is what
+   * lets an overlay be installed before anything has been seeded.
+   */
+  hash: string | (() => string);
   /** The installed package's root directory. Defaults to the running CLI's own. */
   packageRoot?: string;
   /** Where the one warning this can produce goes. Defaults to the console banner. */
   warn?: (message: string) => void;
 };
 
+/**
+ * The overlay every index cache a command builds starts with: the packaged core
+ * version, hashed from the package itself the first time the official
+ * repository's index is read.
+ *
+ * Seeding installs a precise overlay of its own (carrying the hash of the entry
+ * it just wrote), but only the commands that seed get that one. This is what
+ * makes the packaged version resolvable everywhere else too: a lockfile rebuild,
+ * a browsing command, anything that reads the official index. The hash of the
+ * packaged folder is the hash a seeded store entry carries, because the store
+ * copies the folder byte for byte.
+ *
+ * A package whose core recipe cannot be read adds nothing, silently: seeding
+ * reports that failure in full, and a read-only command has nothing better to
+ * say about it.
+ *
+ * @param options - The packaged version, and where the package is.
+ */
+export function packagedCoreIndexOverlay(options: {
+  /** The packaged version, which is by rule the running sous version. */
+  version: string;
+  /** The installed package's root directory. Defaults to the running CLI's own. */
+  packageRoot?: string;
+  /** Where the one warning the overlay can produce goes. */
+  warn?: (message: string) => void;
+}): IndexOverlay {
+  let hash: string | null | undefined;
+  const packagedHash = (): string | null => {
+    if (hash === undefined) {
+      try {
+        hash = hashDirectorySync(packagedCoreRecipeDir(options.packageRoot));
+      } catch {
+        hash = null;
+      }
+    }
+    return hash;
+  };
+
+  const overlay = coreIndexOverlay({
+    version: options.version,
+    hash: () => packagedHash()!,
+    ...(options.packageRoot === undefined ? {} : { packageRoot: options.packageRoot }),
+    ...(options.warn === undefined ? {} : { warn: options.warn }),
+  });
+
+  return (identity, index) => {
+    if (identity !== OFFICIAL_REPO_IDENTITY) return index;
+    if (packagedHash() === null) return index;
+    return overlay(identity, index);
+  };
+}
+
 /**
  * Builds the overlay that makes the packaged core recipe resolvable whatever
  * the official repository has published so far.
@@ -236,6 +293,8 @@ export type CoreIndexOverlayOptions = {
  */
 export function coreIndexOverlay(options: CoreIndexOverlayOptions): IndexOverlay {
   let warned = false;
+  const packagedHash = (): string =>
+    typeof options.hash === "function" ? options.hash() : options.hash;
 
   return (identity: string, index: IndexFile): IndexFile => {
     if (identity !== OFFICIAL_REPO_IDENTITY) return index;
@@ -243,14 +302,14 @@ export function coreIndexOverlay(options: CoreIndexOverlayOptions): IndexOverlay
     const published = index.recipes[CORE_RECIPE_KEY]?.versions[options.version];
 
     if (published !== undefined) {
-      if (published.hash !== options.hash && !warned) {
+      if (published.hash !== packagedHash() && !warned) {
         warned = true;
         (options.warn ?? warning)(
           `The repository '${OFFICIAL_REPO_NAME}' publishes version ${options.version} of ` +
             `'${CORE_RECIPE_KEY}' with different contents from the copy inside this ` +
             `installation of sous, so sous is using the published one.\n` +
             `  Published: ${published.hash}\n` +
-            `  Packaged:  ${options.hash}\n` +
+            `  Packaged:  ${packagedHash()}\n` +
             `  Reinstalling sous will bring the two back into line.`
         );
       }
@@ -276,7 +335,7 @@ export function coreIndexOverlay(options: CoreIndexOverlayOptions): IndexOverlay
           versions: {
             ...recipe?.versions,
             [options.version]: {
-              hash: options.hash,
+              hash: packagedHash(),
               tag: `${CORE_RECIPE_KEY}@${options.version}`,
               prerelease: semver.prerelease(options.version) !== null,
               seeded: true,

package/src/lib/repos/store/hash.ts

@@ -29,7 +29,8 @@
  * recipe folder containing a link.
  */
 
-import { createHash } from "node:crypto";
+import { createHash, type Hash } from "node:crypto";
+import fsSync from "node:fs";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { STORE_ENTRY_FILENAME } from "../formats/common.js";
@@ -89,18 +90,77 @@ export async function hashDirectory(dir: string): Promise<string> {
   const hash = createHash("sha256");
 
   for (const relative of files) {
-    const bytes = await fs.readFile(path.join(root, ...relative.split("/")));
-    hash.update(Buffer.from(relative, "utf8"));
-    hash.update(FIELD_SEPARATOR);
-    hash.update(Buffer.from(String(bytes.byteLength), "utf8"));
-    hash.update(FIELD_SEPARATOR);
-    hash.update(bytes);
-    hash.update(FIELD_SEPARATOR);
+    updateWithFile(hash, relative, await fs.readFile(path.join(root, ...relative.split("/"))));
   }
 
   return `sha256-${hash.digest("hex")}`;
 }
 
+/**
+ * The same hash as `hashDirectory`, computed synchronously. It exists for the
+ * one place that needs a hash inside a synchronous read: folding the packaged
+ * core recipe into a cached index as the index is read.
+ *
+ * hashDirectorySync(dir) === (await hashDirectory(dir))
+ * // -> true, for any directory
+ *
+ * @param dir - Absolute path to the directory to hash.
+ */
+export function hashDirectorySync(dir: string): string {
+  const root = path.resolve(dir);
+  const files = collectFilesSync(root).sort(bytewiseCompare);
+  const hash = createHash("sha256");
+
+  for (const relative of files) {
+    updateWithFile(hash, relative, fsSync.readFileSync(path.join(root, ...relative.split("/"))));
+  }
+
+  return `sha256-${hash.digest("hex")}`;
+}
+
+/**
+ * Collects every hashable file under `dir`, synchronously. The rules are exactly
+ * those of `collectFiles`; the two differ only in how they wait.
+ *
+ * @param dir - The directory to walk.
+ * @param prefix - The relative path of `dir` within the tree being hashed.
+ */
+function collectFilesSync(dir: string, prefix = ""): string[] {
+  const entries = fsSync.readdirSync(dir, { withFileTypes: true });
+  const found: string[] = [];
+
+  for (const entry of entries) {
+    if (entry.name === GIT_DIR_NAME) continue;
+    if (entry.name === STORE_ENTRY_FILENAME) continue;
+    if (entry.isSymbolicLink()) continue;
+
+    const relative = prefix.length > 0 ? `${prefix}/${entry.name}` : entry.name;
+    const absolute = path.join(dir, entry.name);
+
+    if (entry.isDirectory()) found.push(...collectFilesSync(absolute, relative));
+    else if (entry.isFile()) found.push(relative);
+  }
+
+  return found;
+}
+
+/**
+ * Feeds one file's canonical record into a hash: its path, its byte length and
+ * its bytes, each followed by the separator.
+ *
+ * @param hash - The hash being built.
+ * @param relative - The file's path relative to the tree root, posix separators.
+ * @param bytes - The file's contents.
+ */
+function updateWithFile(hash: Hash, relative: string, bytes: Buffer): void {
+  hash.update(Buffer.from(relative, "utf8"));
+  hash.update(FIELD_SEPARATOR);
+  hash.update(Buffer.from(String(bytes.byteLength), "utf8"));
+  hash.update(FIELD_SEPARATOR);
+  hash.update(bytes);
+  hash.update(FIELD_SEPARATOR);
+}
+
 /**
  * Compares two content hashes. Both are canonical lowercase strings, so this is
  * an exact comparison; it exists so callers read as intent rather than as